diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index 89ace0803..6fb7c6a52 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -622,7 +622,7 @@ definitions: properties: data: items: - $ref: '#/definitions/store.RuntimeNode' + $ref: '#/definitions/deployment.Node' type: array type: object api.SessionDiagnosticFailure: @@ -902,176 +902,16 @@ definitions: service: $ref: '#/definitions/coremetrics.ServiceState' type: object - projects.APIKey: - properties: - created_at: - type: string - id: - type: string - name: - type: string - prefix: - type: string - project_id: - type: string - revoked_at: - type: string - type: object - projects.IssuedAPIKey: - properties: - created_at: - type: string - id: - type: string - key: - type: string - name: - type: string - prefix: - type: string - project_id: - type: string - revoked_at: - type: string - type: object - projects.KeyPage: - properties: - data: - items: - $ref: '#/definitions/projects.APIKey' - type: array - has_more: - type: boolean - type: object - projects.Page: - properties: - data: - items: - $ref: '#/definitions/projects.Project' - type: array - has_more: - type: boolean - type: object - projects.Project: - properties: - active_key_count: - type: integer - archived_at: - type: string - created_at: - type: string - id: - type: string - name: - type: string - type: object - sandbox.ConfigurationDiscoveryInput: - properties: - configuration: - type: object - credential: - type: object - query: - type: object - type: object - sandbox.DeploymentSpec: - properties: - resources: - $ref: '#/definitions/sandbox.Resources' - runtime: - $ref: '#/definitions/sandbox.RuntimeRelease' - type: object - sandbox.Resources: - properties: - cpus: - type: integer - environment_disk_mib: - type: integer - memory_mib: - type: integer - root_disk_mib: - type: integer - type: object - sandbox.RuntimeRelease: - properties: - firmware_sha256: - type: string - image_id: - type: string - image_manifest_digest: - type: string - microsandbox_ref: - type: string - runtime_sha256: - type: string - source_commit: - type: string - type: object - store.AddressBindings: - properties: - hosted_sandboxes: - type: integer - nodes: - type: integer - nodes_on_other_address: - type: integer - self_hosted_executors: - type: integer - type: object - store.AdminAssetCounts: - properties: - agents: - type: integer - credentials: - type: integer - environment_templates: - type: integer - files: - type: integer - skills: - type: integer - vaults: - type: integer - type: object - store.ExecutorCredential: - properties: - created_at: - type: string - key_id: - format: uuid - type: string - revoked_at: - type: string - x-nullable: true - type: object - store.IssuedExecutorCredential: - properties: - environment_id: - type: string - executor_token: - type: string - key_id: - type: string - type: object - store.ManagedSessionArchive: - properties: - environment_id: - type: string - session_id: - type: string - state: - type: string - type: object - store.NodeHostHistory: + deployment.HostHistory: properties: points: items: - $ref: '#/definitions/store.NodeHostHistoryPoint' + $ref: '#/definitions/deployment.HostHistoryPoint' type: array resolution_seconds: type: integer type: object - store.NodeHostHistoryPoint: + deployment.HostHistoryPoint: properties: available_disk_bytes_min: type: integer @@ -1085,46 +925,7 @@ definitions: start: type: string type: object - store.RuntimeDeploymentView: - properties: - configuration: - type: object - core_url: - description: 'Read-only: the installation public URL (OAC_PUBLIC_URL), which nodes and sandboxes use to reach Core. The deployment API does not accept it.' - type: string - credential_configured: - type: boolean - generation: - type: integer - installation_id: - type: string - metadata: - type: object - mode: - type: string - owner_epoch: - type: integer - provider: - type: string - reset: - allOf: - - $ref: '#/definitions/store.SandboxResetView' - x-nullable: true - resources: - $ref: '#/definitions/store.SandboxDeploymentResources' - rollout: - $ref: '#/definitions/store.SandboxRollout' - specification: - $ref: '#/definitions/sandbox.DeploymentSpec' - specification_digest: - type: string - suspension: - allOf: - - $ref: '#/definitions/store.SandboxSuspensionView' - description: Idle suspension policy; microsandbox only, otherwise null. - x-nullable: true - type: object - store.RuntimeNode: + deployment.Node: properties: active: type: integer @@ -1180,42 +981,13 @@ definitions: retained: type: integer rollout: - $ref: '#/definitions/store.SandboxNodeRollout' + $ref: '#/definitions/deployment.NodeRollout' running: type: integer snapshots: type: integer type: object - store.RuntimeNodeAllocation: - properties: - compute_phase: - type: string - compute_phase_changed_at: - description: The time the allocation entered its current compute_phase, or null when unknown; an allocation that existed before Core recorded it reports null until its next phase change. For a suspended microsandbox allocation, this time plus the deployment's snapshot retention tells roughly when Core reclaims it. - type: string - x-nullable: true - created_at: - type: string - deployment_generation: - type: integer - diagnostic: - type: string - environment_id: - type: string - id: - type: string - initialization: - type: string - node_id: - type: string - session_id: - type: string - state: - type: string - tenant_id: - type: string - type: object - store.RuntimeNodeDetail: + deployment.NodeDetail: properties: active: type: integer @@ -1251,9 +1023,9 @@ definitions: type: string x-nullable: true history: - $ref: '#/definitions/store.NodeHostHistory' + $ref: '#/definitions/deployment.HostHistory' host: - $ref: '#/definitions/store.RuntimeNodeHost' + $ref: '#/definitions/deployment.NodeHost' id: type: string last_seen_at: @@ -1275,13 +1047,13 @@ definitions: retained: type: integer rollout: - $ref: '#/definitions/store.SandboxNodeRollout' + $ref: '#/definitions/deployment.NodeRollout' running: type: integer snapshots: type: integer type: object - store.RuntimeNodeHost: + deployment.NodeHost: properties: available_disk_bytes: type: integer @@ -1302,24 +1074,7 @@ definitions: type: integer x-nullable: true type: object - store.RuntimeNodeUpdate: - properties: - max_active: - type: integer - max_retained: - description: Docker never suspends, so Core replaces this with max_active; microsandbox uses both limits. - type: integer - name: - type: string - type: object - store.SandboxDeploymentResources: - properties: - allocations: - type: integer - pending: - type: integer - type: object - store.SandboxNodeRollout: + deployment.NodeRollout: properties: diagnostic: enum: @@ -1346,49 +1101,17 @@ definitions: - unknown type: string type: object - store.SandboxResetOfflineNode: + deployment.NodeUpdate: properties: + max_active: + type: integer + max_retained: + description: Docker never suspends, so Core replaces this with max_active; microsandbox uses both limits. + type: integer name: type: string - node_id: - type: string - resources: - type: integer type: object - store.SandboxResetRemaining: - properties: - busy: - type: integer - cleanup: - type: integer - idle: - type: integer - offline_nodes: - items: - $ref: '#/definitions/store.SandboxResetOfflineNode' - type: array - on_offline_nodes: - type: integer - type: object - store.SandboxResetRequest: - properties: - clear: - enum: - - auto - - force - type: string - deadline_seconds: - maximum: 86400 - minimum: 300 - type: integer - expected_generation: - minimum: 0 - type: integer - required: - - clear - - expected_generation - type: object - store.SandboxResetView: + deployment.Reset: properties: clear: type: string @@ -1399,15 +1122,46 @@ definitions: type: string x-nullable: true remaining: - $ref: '#/definitions/store.SandboxResetRemaining' + $ref: '#/definitions/deployment.ResetRemaining' requested_at: type: string type: object - store.SandboxRollout: + deployment.ResetOfflineNode: + properties: + name: + type: string + node_id: + type: string + resources: + type: integer + type: object + deployment.ResetRemaining: + properties: + busy: + type: integer + cleanup: + type: integer + idle: + type: integer + offline_nodes: + items: + $ref: '#/definitions/deployment.ResetOfflineNode' + type: array + on_offline_nodes: + type: integer + type: object + deployment.Resources: + properties: + allocations: + type: integer + pending: + type: integer + type: object + deployment.Rollout: properties: nodes: allOf: - - $ref: '#/definitions/store.SandboxRolloutNodes' + - $ref: '#/definitions/deployment.RolloutNodes' x-nullable: true previous_generation_sandboxes: type: integer @@ -1417,7 +1171,7 @@ definitions: - preparing type: string type: object - store.SandboxRolloutNodes: + deployment.RolloutNodes: properties: failed: type: integer @@ -1430,13 +1184,259 @@ definitions: update_required: type: integer type: object - store.SandboxSuspensionView: + deployment.Suspension: properties: idle_seconds: type: integer retention_seconds: type: integer type: object + deployment.View: + properties: + configuration: + type: object + core_url: + description: 'Read-only: the installation public URL (OAC_PUBLIC_URL), which nodes and sandboxes use to reach Core. The deployment API does not accept it.' + type: string + credential_configured: + type: boolean + generation: + type: integer + installation_id: + type: string + metadata: + type: object + mode: + type: string + owner_epoch: + type: integer + provider: + type: string + reset: + allOf: + - $ref: '#/definitions/deployment.Reset' + x-nullable: true + resources: + $ref: '#/definitions/deployment.Resources' + rollout: + $ref: '#/definitions/deployment.Rollout' + specification: + $ref: '#/definitions/sandbox.DeploymentSpec' + specification_digest: + type: string + suspension: + allOf: + - $ref: '#/definitions/deployment.Suspension' + description: Idle suspension policy; microsandbox only, otherwise null. + x-nullable: true + type: object + projects.APIKey: + properties: + created_at: + type: string + id: + type: string + name: + type: string + prefix: + type: string + project_id: + type: string + revoked_at: + type: string + type: object + projects.IssuedAPIKey: + properties: + created_at: + type: string + id: + type: string + key: + type: string + name: + type: string + prefix: + type: string + project_id: + type: string + revoked_at: + type: string + type: object + projects.KeyPage: + properties: + data: + items: + $ref: '#/definitions/projects.APIKey' + type: array + has_more: + type: boolean + type: object + projects.Page: + properties: + data: + items: + $ref: '#/definitions/projects.Project' + type: array + has_more: + type: boolean + type: object + projects.Project: + properties: + active_key_count: + type: integer + archived_at: + type: string + created_at: + type: string + id: + type: string + name: + type: string + type: object + sandbox.ConfigurationDiscoveryInput: + properties: + configuration: + type: object + credential: + type: object + query: + type: object + type: object + sandbox.DeploymentSpec: + properties: + resources: + $ref: '#/definitions/sandbox.Resources' + runtime: + $ref: '#/definitions/sandbox.RuntimeRelease' + type: object + sandbox.Resources: + properties: + cpus: + type: integer + environment_disk_mib: + type: integer + memory_mib: + type: integer + root_disk_mib: + type: integer + type: object + sandbox.RuntimeRelease: + properties: + firmware_sha256: + type: string + image_id: + type: string + image_manifest_digest: + type: string + microsandbox_ref: + type: string + runtime_sha256: + type: string + source_commit: + type: string + type: object + store.AddressBindings: + properties: + hosted_sandboxes: + type: integer + nodes: + type: integer + nodes_on_other_address: + type: integer + self_hosted_executors: + type: integer + type: object + store.AdminAssetCounts: + properties: + agents: + type: integer + credentials: + type: integer + environment_templates: + type: integer + files: + type: integer + skills: + type: integer + vaults: + type: integer + type: object + store.ExecutorCredential: + properties: + created_at: + type: string + key_id: + format: uuid + type: string + revoked_at: + type: string + x-nullable: true + type: object + store.IssuedExecutorCredential: + properties: + environment_id: + type: string + executor_token: + type: string + key_id: + type: string + type: object + store.ManagedSessionArchive: + properties: + environment_id: + type: string + session_id: + type: string + state: + type: string + type: object + store.RuntimeNodeAllocation: + properties: + compute_phase: + type: string + compute_phase_changed_at: + description: The time the allocation entered its current compute_phase, or null when unknown; an allocation that existed before Core recorded it reports null until its next phase change. For a suspended microsandbox allocation, this time plus the deployment's snapshot retention tells roughly when Core reclaims it. + type: string + x-nullable: true + created_at: + type: string + deployment_generation: + type: integer + diagnostic: + type: string + environment_id: + type: string + id: + type: string + initialization: + type: string + node_id: + type: string + session_id: + type: string + state: + type: string + tenant_id: + type: string + type: object + store.SandboxResetRequest: + properties: + clear: + enum: + - auto + - force + type: string + deadline_seconds: + maximum: 86400 + minimum: 300 + type: integer + expected_generation: + minimum: 0 + type: integer + required: + - clear + - expected_generation + type: object v1.Agent: properties: id: @@ -6159,7 +6159,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/store.RuntimeDeploymentView' + $ref: '#/definitions/deployment.View' "400": description: Bad Request schema: @@ -6206,7 +6206,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/store.RuntimeDeploymentView' + $ref: '#/definitions/deployment.View' "400": description: Bad Request schema: @@ -6249,7 +6249,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/store.RuntimeDeploymentView' + $ref: '#/definitions/deployment.View' "400": description: Bad Request schema: @@ -6277,7 +6277,7 @@ paths: - Sandbox Manager /core/v1/sandbox/deployment/reset: delete: - description: Restores admission but never restores Sessions already archived. With no reset running this is an idempotent read, provided the generation still matches. + description: Restores admission but never restores Sessions already archived. With no reset running this is an idempotent read, provided the generation still matches. The response is the current deployment read after the cancellation commits. parameters: - description: Current deployment generation in: query @@ -6290,7 +6290,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/store.RuntimeDeploymentView' + $ref: '#/definitions/deployment.View' "400": description: Bad Request schema: @@ -6319,7 +6319,7 @@ paths: post: consumes: - application/json - description: Archives hosted Sessions and waits for confirmed provider cleanup, preserving history and Files/Artifacts. Auto waits for started or waiting Turns and file writes until the durable deadline; force cancels them. The same clear is idempotent; force escalates auto. Requires the current generation. Self-hosted Sessions are unchanged. + description: Archives hosted Sessions and waits for confirmed provider cleanup, preserving history and Files/Artifacts. Auto waits for started or waiting Turns and file writes until the durable deadline; force cancels them. The same clear is idempotent; force escalates auto. Requires the current generation. Self-hosted Sessions are unchanged. The response is the current deployment read after the reset commits; resource counts are live and may already differ. parameters: - description: Reset mode and current deployment generation in: body @@ -6333,7 +6333,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/store.RuntimeDeploymentView' + $ref: '#/definitions/deployment.View' "400": description: Bad Request schema: @@ -6513,7 +6513,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/store.RuntimeNodeDetail' + $ref: '#/definitions/deployment.NodeDetail' "400": description: Bad Request schema: @@ -6554,7 +6554,7 @@ paths: name: body required: true schema: - $ref: '#/definitions/store.RuntimeNodeUpdate' + $ref: '#/definitions/deployment.NodeUpdate' produces: - application/json responses: diff --git a/contracts/agents-api/runtime.openapi.yaml b/contracts/agents-api/runtime.openapi.yaml index 329d7fce1..fc109bc6f 100644 --- a/contracts/agents-api/runtime.openapi.yaml +++ b/contracts/agents-api/runtime.openapi.yaml @@ -34,40 +34,27 @@ definitions: executor_token: type: string type: object - sandbox.DeploymentSpec: - properties: - resources: - $ref: '#/definitions/sandbox.Resources' - runtime: - $ref: '#/definitions/sandbox.RuntimeRelease' - type: object - sandbox.Resources: - properties: - cpus: - type: integer - environment_disk_mib: - type: integer - memory_mib: - type: integer - root_disk_mib: - type: integer - type: object - sandbox.RuntimeRelease: + deployment.Enrollment: properties: - firmware_sha256: + backend_fingerprint: type: string - image_id: + core_url: + description: The Core origin this node stores and connects to, such as https://core.example. It must equal the installation public URL; otherwise enrollment gets 409 sandbox_node_address_mismatch and the token stays unused. type: string - image_manifest_digest: + credential: type: string - microsandbox_ref: + deployment_generation: + type: integer + name: type: string - runtime_sha256: + node_id: type: string - source_commit: + provider: + type: string + specification_digest: type: string type: object - store.RuntimeNodeConfiguration: + deployment.NodeConfiguration: properties: core_url: type: string @@ -86,27 +73,7 @@ definitions: specification_digest: type: string type: object - store.RuntimeNodeEnrollment: - properties: - backend_fingerprint: - type: string - core_url: - description: The Core origin this node stores and connects to, such as https://core.example. It must equal the installation public URL; otherwise enrollment gets 409 sandbox_node_address_mismatch and the token stays unused. - type: string - credential: - type: string - deployment_generation: - type: integer - name: - type: string - node_id: - type: string - provider: - type: string - specification_digest: - type: string - type: object - store.RuntimeNodeIdentity: + deployment.NodeIdentity: properties: deployment_generation: type: integer @@ -123,7 +90,7 @@ definitions: specification_digest: type: string type: object - store.RuntimeNodeStatus: + deployment.NodeStatus: properties: connected: type: boolean @@ -144,6 +111,39 @@ definitions: specification_digest: type: string type: object + sandbox.DeploymentSpec: + properties: + resources: + $ref: '#/definitions/sandbox.Resources' + runtime: + $ref: '#/definitions/sandbox.RuntimeRelease' + type: object + sandbox.Resources: + properties: + cpus: + type: integer + environment_disk_mib: + type: integer + memory_mib: + type: integer + root_disk_mib: + type: integer + type: object + sandbox.RuntimeRelease: + properties: + firmware_sha256: + type: string + image_id: + type: string + image_manifest_digest: + type: string + microsandbox_ref: + type: string + runtime_sha256: + type: string + source_commit: + type: string + type: object v1.APIError: properties: code: @@ -268,7 +268,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/store.RuntimeNodeConfiguration' + $ref: '#/definitions/deployment.NodeConfiguration' "400": description: Bad Request schema: @@ -305,14 +305,14 @@ paths: name: body required: true schema: - $ref: '#/definitions/store.RuntimeNodeEnrollment' + $ref: '#/definitions/deployment.Enrollment' produces: - application/json responses: "201": description: Created schema: - $ref: '#/definitions/store.RuntimeNodeIdentity' + $ref: '#/definitions/deployment.NodeIdentity' "400": description: Bad Request schema: @@ -357,7 +357,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/store.RuntimeNodeStatus' + $ref: '#/definitions/deployment.NodeStatus' "400": description: Bad Request schema: diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 360c27fd3..b92550900 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -216,7 +216,7 @@ Some fields keep one name across providers but differ in meaning, or do not appl | 409 | `sandbox_deployment_conflict` | Another state the change cannot apply to | | 409 | `runtime_node_in_use` | Node removal while it holds resources | | 503 | `execution_unavailable` | Provider preparation is unavailable | -| 503 | `sandbox_credential_unavailable` | The credential encryption key is unavailable | +| 503 | `credential_storage_unavailable` | Core has no credential encryption key | Storage and credential failures stay errors: an empty or failed read never proves cleanup. The [machine connection API](machine-api.md#node-route-errors) lists the errors of the node routes. diff --git a/deploy/install/config_model.py b/deploy/install/config_model.py index 6736ef8a3..ca48fd5e2 100644 --- a/deploy/install/config_model.py +++ b/deploy/install/config_model.py @@ -62,7 +62,7 @@ def lookup(config, key): # Checks named by x-oac.check. Core stays the authority for its own semantic rules. def _origin(value, https_only=False): - """Core's ValidateSandboxCoreURL rule, through the installer's one implementation of it.""" + """Core's deployment.ValidateCoreURL rule, through the installer's one implementation of it.""" from configuration import valid_core_origin # configuration imports this module at load time return valid_core_origin(value) and (not https_only or value.startswith("https://")) diff --git a/deploy/install/configuration.py b/deploy/install/configuration.py index 0b1d83aaf..aa746ea2f 100644 --- a/deploy/install/configuration.py +++ b/deploy/install/configuration.py @@ -29,8 +29,8 @@ def valid_core_origin(value): - """Accept exactly the origins Core's ValidateSandboxCoreURL accepts - (services/core/internal/store/sandbox_deployment_setup.go), so an + """Accept exactly the origins Core's deployment.ValidateCoreURL accepts + (services/core/internal/deployment/public_url.go), so an installer value never fails Core's OAC_PUBLIC_URL check at startup.""" if not isinstance(value, str) or any(char in value for char in "?#@\\% \t\r\n"): return False diff --git a/packages/agents-client/src/sandbox-client.test.ts b/packages/agents-client/src/sandbox-client.test.ts index ca9a8fcb8..ec61820b4 100644 --- a/packages/agents-client/src/sandbox-client.test.ts +++ b/packages/agents-client/src/sandbox-client.test.ts @@ -6,7 +6,7 @@ import nodeDiagnosticFixture from "../../../services/core/internal/sandbox/testd function response(value: unknown, status = 200) { return new Response(JSON.stringify(value), { status }); } -// Shapes as Core serializes them (store.RuntimeNode, RuntimeNodeDetail, RuntimeNodeAllocation, RuntimeDeploymentView). +// Shapes as Core serializes them (deployment.Node, deployment.NodeDetail, store.RuntimeNodeAllocation, deployment.View). const created = "2026-09-25T08:00:00.123456789Z"; /** A ready node: Core omits `diagnostic`. */ const node = { diff --git a/packages/agents-client/src/sandbox-client.ts b/packages/agents-client/src/sandbox-client.ts index 2a5c87f83..47269b1ad 100644 --- a/packages/agents-client/src/sandbox-client.ts +++ b/packages/agents-client/src/sandbox-client.ts @@ -180,7 +180,7 @@ function invalidSandboxResponse(): never { /** * The object has every required member, optional ones only where listed, and nothing else. The projections below - * follow Core's store.RuntimeDeploymentView, RuntimeNode, RuntimeNodeDetail and RuntimeNodeAllocation serialization. + * follow Core's deployment.View, deployment.Node, deployment.NodeDetail and store.RuntimeNodeAllocation serialization. */ function members(value: unknown, required: readonly string[], optional: readonly string[] = []): Record { if (!isRecord(value) || !required.every((field) => hasOwn(value, field)) || !onlyFields(value, new Set([...required, ...optional]))) return invalidSandboxResponse(); diff --git a/services/core/IMPLEMENTATION.md b/services/core/IMPLEMENTATION.md index c99c1b5f8..fcc0cf90a 100644 --- a/services/core/IMPLEMENTATION.md +++ b/services/core/IMPLEMENTATION.md @@ -31,6 +31,7 @@ Domain owners, each with its PostgreSQL adapter under `internal/persistence/post - `environmenttemplates` (`templatepg`): Environment Templates, their validation and default network, their sealed setup, initial files, Skills and Plugins, and the resolved Template that Session creation composes into its Environment. - `modelconfiguration` (`modelconfigurationpg`): each Harness's deployment default model configuration and its last-use observations. - `skills` (`skillpg`): Skills and their immutable versions: archive checks, the default and latest pointers, version selection and deletion, and each version's sealed archive. Session creation freezes selected versions inside its `store` transaction with the `skills` rules. +- `deployment` (`deploymentpg`): the sandbox deployment and its nodes: provider configuration and the sealed credential, the specification and retained generations, setup, update and switch, node enrollment, identity and authentication, generation configuration, capacity, presence, status and host history. It interprets Sandbox Provider declarations through the `providers.Registry` it is given, whose lookups return typed errors. `cmd/server` builds that registry and calls it only to build a direct Provider and to discover a Provider's configuration; it takes each setup's mode and declared operations from the `Setup` that `deployment` returns. The only other reader is `store`, whose own `providers.Builtin()` validates the stored specification at Session admission and reads the public-origin requirement at placement. Deployment changes run through `deployment.ExecutionOperations` on `deploymentpg.NewExecution(lease, …)`, which the Worker receives as `execution.Owner.Deployment`; the Worker reads the deployment and prepares a selection's setup through the pooled `deployment.Service` in `execution.Dispatcher.Deployment`, and node management and reads use the pooled `deploymentpg.Store`, which `cmd/server` also reads the owner epoch from and runs the host-history sampler on. Provider calls run outside transactions, and the final transaction rechecks the expected generation. Allocations, placement and reset stay in `store`. ## Request handling @@ -88,7 +89,7 @@ A dedicated self-hosted device is bound to exactly one Environment's Session and Connection observations use the execution lease and the Session lock. A separate `environment_connections` row holds the current generation and revision, and `environments.status` commits together with its Session Environment event. The producer serializes replacements and numbers socket observations within each generation; duplicate or older revisions and superseded generations are inert, and a replacement retires the previous connected observation before registering the new one. Registration alone creates no `connected` event. Event payloads carry only public Environment identity, type, status and nullable error, never configuration, credentials, registration IDs or revisions, and have no Turn association. `connected` and `disconnected` are distinct from native readiness: never cast resource `expired` into this vocabulary or emit `ready` for a self-hosted connection. On restart the Worker reconciles old observations before admitting new ones, and a failed or stale observation never establishes a connection. -The sandbox node Hub's global mutex protects only in-memory connection state. Authentication, ownership and store callbacks run synchronously outside it, respect cancellation and have five seconds; database writes are never detached. Closing the Hub cancels opening and live connections without waiting for database callbacks, and each node reservation lasts until its fenced disconnect cleanup finishes. Presence is registered in an explicit transaction, so a canceled statement cannot publish it later through autocommit. Disconnect cleanup first locks the node row, then applies the connection and epoch fence with a fresh READ COMMITTED statement so an in-flight commit is not missed. These transactions never take the deployment-wide manager lock, and online-state writes compare the handshake epoch atomically so a stale Core cannot publish readiness for a new owner. +The sandbox node Hub's global mutex protects only in-memory connection state. Authentication, ownership and `deployment` callbacks run synchronously outside it, respect cancellation and have five seconds; database writes are never detached. Closing the Hub cancels opening and live connections without waiting for database callbacks, and each node reservation lasts until its fenced disconnect cleanup finishes. `deploymentpg` registers presence in an explicit transaction, so a canceled statement cannot publish it later through autocommit. Disconnect cleanup first locks the node row, then applies the connection and epoch fence with a fresh READ COMMITTED statement so an in-flight commit is not missed. These transactions never take the deployment-wide manager lock, and online-state writes compare the handshake epoch atomically so a stale Core cannot publish readiness for a new owner. ## Sessions, Turns and input @@ -183,7 +184,7 @@ At startup the Worker fails previously claimed work, keeps queued input and neve The [managed lifecycle](../../docs/sandbox-provider.md#managed-lifecycle) describes publication, allocation, per-node workers, placement, suspension, reset and archive; the [sandbox node protocol](../../contracts/agents-api/node-generation-protocol.md) the node connection. In code: -- Each allocation's immutable specification and the current credential resolve in one snapshot, with no stale-credential cache, no fallback to the current specification and no unloading of a provider map that retained allocations need. +- Each allocation's immutable specification and the current credential resolve in one `deploymentpg` snapshot, with no stale-credential cache, no fallback to the current specification and no unloading of a provider map that retained allocations need. - Credential replacement fences provider calls and waits for helper subprocesses to exit, even after cancellation, before verifying again and committing; the audit entry never carries secret fields. - Observations of offline, missing or unconfirmed resources persist only bounded, sanitized codes, separately from the lifecycle, and a host restart never fabricates a running observation. - Runtime observation of a node allocation goes through its immutable placement as one bounded read-only Provider operation; an absent capability or transport returns unavailable, never a Core-local fallback. @@ -191,7 +192,7 @@ The [managed lifecycle](../../docs/sandbox-provider.md#managed-lifecycle) descri ## Core administration errors, metrics and write provenance - Core error details are scoped by the `/core/v1` router's writer mark, never by a request path test. Write Core errors with `writeCoreError` and typed `CoreErrorDetails` values (string, number, null and string-array constructors); invalid or empty details are omitted as a whole. The mark preserves error observation, flushing and `http.ResponseController` access. A shared handler or a Core-looking path alone never changes a public or machine error envelope. Core authentication runs before operation configuration checks, and unknown paths keep their status and admission rules. When adding a code with details, document its fixed keys in `contracts/agents-api/core-errors.md`, and pass only safe Core-owned facts: never submitted values, secrets, native text or provider bodies. -- Operation validators keep their original error text, sentinel identity and validation precedence. Package-owned typed errors carry fixed field metadata; only the marked Core error mapper translates it into operation codes and safe bound or catalog details. Sandbox validation metadata travels through its store wrapper without changing transaction or provider authority. Public Session provider validation stays byte-for-byte unchanged; cover it with handler-level golden responses. The Core clients ignore malformed optional details and never retry a write. +- Operation validators keep their original error text, sentinel identity and validation precedence. Package-owned typed errors carry fixed field metadata; only the marked Core error mapper translates it into operation codes and safe bound or catalog details. Sandbox validation metadata travels through `deployment.ConfigurationError` without changing transaction or provider authority. Public Session provider validation stays byte-for-byte unchanged; cover it with handler-level golden responses. The Core clients ignore malformed optional details and never retry a write. - Core metrics instrument the existing worker and job owners without changing scheduling, lease or retention behavior. Count `execution_unavailable` at the HTTP error writer, once per rejected response; never capture request or response bodies and never infer the count from other 503s or failed Turns. Process CPU, RSS and cgroup limits are sampled by the 30-second Core metrics loop into the same bounded in-memory ring; the first CPU interval and restart gaps stay null, and host usage never substitutes for process usage. Root Turn history is queried read-only from PostgreSQL with native timestamps. Builds inject the source commit with `-ldflags` into `main.buildRevision`. Keep the response shape aligned with `packages/agents-client/src/core-metrics.ts`. - Public resource writes carry the authenticated key's provenance separately from the execution principal. Record the operation and any creation ownership with `auditpg.RecordWriteAudit` in the business transaction, never in response middleware or an asynchronous queue; a failed record rolls back the write. Internal lifecycle and refresh work never acquires public provenance, and retries never replace ownership. Environment uploads persist the safe request origin before dispatch and record success with the confirmed Runtime receipt, not the native filesystem call. Never put payloads, paths or secrets in audit metadata. Do not confuse key identity with the Session creator identity used for retries. - Administrator writes reuse the public resource deletion and serialization code and record their administrator audit entry with `auditpg.RecordAdminMutation`, or `RecordDeploymentMutation` for a deployment-wide write, in the same transaction. Administrator provenance takes precedence over a key's. diff --git a/services/core/cmd/provider-artifacts/main.go b/services/core/cmd/provider-artifacts/main.go index 2b9e9de8f..13e97839d 100644 --- a/services/core/cmd/provider-artifacts/main.go +++ b/services/core/cmd/provider-artifacts/main.go @@ -14,7 +14,7 @@ import ( func main() { write := flag.Bool("write", false, "write generated declarations from the repository root") flag.Parse() - catalog, err := providers.ArtifactCatalog() + catalog, err := providers.Builtin().ArtifactCatalog() if err != nil { panic(err) } diff --git a/services/core/cmd/sandbox-node/generations.go b/services/core/cmd/sandbox-node/generations.go index 095c4fe3a..3cd17d93f 100644 --- a/services/core/cmd/sandbox-node/generations.go +++ b/services/core/cmd/sandbox-node/generations.go @@ -19,7 +19,7 @@ import ( providerconfig "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" ) -func runGenerations(ctx context.Context, configFile, stateDir string) error { +func runGenerations(ctx context.Context, registry *providerconfig.Registry, configFile, stateDir string) error { root := filepath.Dir(configFile) if stateDir != filepath.Join(root, "state", "node") { return errors.New("generation state must belong to the installed node root") @@ -105,7 +105,7 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { collection = append(collection, sandbox.GenerationReference{Generation: config.Generation, SpecificationDigest: config.Specification.Digest(config.Provider)}) continue } - value, err := buildGeneration(config, stateDir) + value, err := buildGeneration(registry, config, stateDir) if errors.Is(err, os.ErrNotExist) { recovery = append(recovery, sandbox.GenerationReference{Generation: config.Generation, SpecificationDigest: config.Specification.Digest(config.Provider)}) continue @@ -146,7 +146,7 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { if config.InstallationID != base.InstallationID || config.Provider != base.Provider || config.Generation != generation || config.Specification.Digest(config.Provider) != digest { return node.GenerationProvider{}, sandbox.ErrOwnership } - value, err := buildGeneration(config, stateDir) + value, err := buildGeneration(registry, config, stateDir) if err != nil { return value, err } @@ -166,8 +166,8 @@ func runGenerations(ctx context.Context, configFile, stateDir string) error { return node.Run(ctx, node.AgentConfig{CoreURL: stored.CoreURL, StateDirectory: stateDir, Identity: stored.Identity, Credential: stored.Credential, Generations: manager}) } -func buildGeneration(config providerconfig.Config, stateDir string) (node.GenerationProvider, error) { - built, closeProvider, err := providerconfig.Build(config, providerconfig.LocalOptions{GenerationStateDirectory: stateDir}) +func buildGeneration(registry *providerconfig.Registry, config providerconfig.Config, stateDir string) (node.GenerationProvider, error) { + built, closeProvider, err := registry.Build(config, providerconfig.LocalOptions{GenerationStateDirectory: stateDir}) if err != nil { return node.GenerationProvider{}, err } diff --git a/services/core/cmd/sandbox-node/main.go b/services/core/cmd/sandbox-node/main.go index d947f8d92..68749e61f 100644 --- a/services/core/cmd/sandbox-node/main.go +++ b/services/core/cmd/sandbox-node/main.go @@ -65,13 +65,14 @@ func run(ctx context.Context, args []string) error { if !filepath.IsAbs(*configFile) || !filepath.IsAbs(*stateDir) { return errors.New("config and state-dir must be absolute paths") } + registry := providerconfig.Builtin() if args[0] == "run" { helper := filepath.Join(filepath.Dir(*configFile), "generation-preparer.pyz") if info, err := os.Lstat(helper); err == nil { if !info.Mode().IsRegular() || info.Mode().Perm() != 0600 { return errors.New("generation preparer must be a private regular file") } - return runGenerations(ctx, *configFile, *stateDir) + return runGenerations(ctx, registry, *configFile, *stateDir) } else if !os.IsNotExist(err) { return err } @@ -80,7 +81,7 @@ func run(ctx context.Context, args []string) error { if err != nil { return err } - built, closeProvider, err := providerconfig.Build(config, providerconfig.LocalOptions{Standalone: true}) + built, closeProvider, err := registry.Build(config, providerconfig.LocalOptions{Standalone: true}) if err != nil { return err } diff --git a/services/core/cmd/server/http_routes_test.go b/services/core/cmd/server/http_routes_test.go index 4d5061121..aecd31cfd 100644 --- a/services/core/cmd/server/http_routes_test.go +++ b/services/core/cmd/server/http_routes_test.go @@ -101,8 +101,8 @@ func daemonComposition(t testing.TB) http.Handler { RuntimeObservations: struct{ api.RuntimeObservations }{}, RuntimeHistory: struct{ api.RuntimeHistory }{}, Execution: &api.Execution{ExecutorURL: "wss://core.example/api/v1/agent-daemon/ws", Admission: struct{ api.Admission }{}, SessionArchive: struct{ api.SessionArchive }{}, Workspaces: struct{ api.EnvironmentWorkspaces }{}}, - Sandboxes: &api.Sandboxes{Deployment: struct{ api.Deployment }{}, DeploymentChanges: struct{ api.DeploymentChanges }{}, - ConfigurationDiscovery: struct{ api.ConfigurationDiscovery }{}}, + Sandboxes: &api.Sandboxes{Deployment: struct{ api.Deployment }{}, NodeAllocations: struct{ api.NodeAllocations }{}, DeploymentChanges: struct{ api.DeploymentChanges }{}, + DeploymentReset: struct{ api.DeploymentReset }{}, ConfigurationDiscovery: struct{ api.ConfigurationDiscovery }{}}, }) if err != nil { t.Fatal(err) diff --git a/services/core/cmd/server/installation.go b/services/core/cmd/server/installation.go index 68a0e9fbe..240e7e8d3 100644 --- a/services/core/cmd/server/installation.go +++ b/services/core/cmd/server/installation.go @@ -7,7 +7,7 @@ import ( "regexp" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" ) var sourceCommit = regexp.MustCompile(`^[0-9a-f]{40}$`) @@ -22,7 +22,7 @@ func installationFacts(publicURL string) (api.Installation, error) { } if publicURL != "" { base := publicURL + "/v1" - facts.PublicURL, facts.APIBaseURL, facts.LocalOnly = &publicURL, &base, store.LoopbackOrigin(publicURL) + facts.PublicURL, facts.APIBaseURL, facts.LocalOnly = &publicURL, &base, deployment.LoopbackOrigin(publicURL) } if sourceCommit.MatchString(buildRevision) { revision := buildRevision diff --git a/services/core/cmd/server/main.go b/services/core/cmd/server/main.go index b55976a4f..7e74fbfc4 100644 --- a/services/core/cmd/server/main.go +++ b/services/core/cmd/server/main.go @@ -35,6 +35,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/databaseurl" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmenttemplates" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/files" @@ -42,6 +43,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/nativeinstaller" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/agentpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/filepg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/modelconfigurationpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" @@ -57,6 +59,7 @@ import ( historystoreresolver "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimehistory/storeresolver" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" observationstoreresolver "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs/storeresolver" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/skills" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/vaults" @@ -153,6 +156,12 @@ func run() error { if err != nil { return err } + sandboxProviders := providers.Builtin() + deploymentStore := deploymentpg.New(units, credentialKey) + deploymentService, err := deployment.NewService(deploymentStore, deploymentStore, sandboxProviders, public) + if err != nil { + return err + } installation, err := installationFacts(public) if err != nil { return err @@ -172,7 +181,7 @@ func run() error { defer func() { cancelAuditCleanup(); <-auditCleanupDone }() var workerDone chan error var worker *execution.Worker - managedNodes, err := configureManagedNodes(executionStore, public, func(ctx context.Context) error { + managedNodes, err := configureManagedNodes(executionStore, deploymentService, deploymentStore, sandboxProviders, public, func(ctx context.Context) error { if worker == nil { return errors.New("sandbox execution owner is unavailable") } @@ -273,14 +282,22 @@ func run() error { } if registry != nil { dispatcher := &execution.Dispatcher{Store: executionStore, Registry: registry, - Credentials: vaultService, Observer: modelConfigurationStore, + Credentials: vaultService, Observer: modelConfigurationStore, Deployment: deploymentService, ManagedRuntimes: managed, MaxConcurrentExecutions: concurrency} lease, err := pgunit.AcquireLease(ctx, pool) if err != nil { return err } + deploymentExecution, err := deployment.NewExecutionOperations(deploymentService, deploymentpg.NewExecution(lease, credentialKey)) + if err != nil { + return errors.Join(err, lease.Close(ctx)) + } // From this call on the Worker closes the lease, even when it fails to start. - worker, err = execution.StartWorker(ctx, dispatcher, execution.Owner{Lease: lease, Store: store.NewExecution(executionStore, lease)}) + worker, err = execution.StartWorker(ctx, dispatcher, execution.Owner{ + Lease: lease, + Store: store.NewExecution(executionStore, lease), + Deployment: deploymentExecution, + }) if err != nil { return err } @@ -306,7 +323,7 @@ func run() error { sampleCtx, cancel := context.WithTimeout(ctx, 2*time.Second) sampleErr := worker.CheckOwnership(sampleCtx) if sampleErr == nil { - _, sampleErr = executionStore.SampleNodeHostHistory(sampleCtx) + _, sampleErr = deploymentStore.SampleHostHistory(sampleCtx) } cancel() if !result.Complete { @@ -362,7 +379,13 @@ func run() error { deps.Execution = &api.Execution{ExecutorURL: executorURL, Admission: worker, SessionArchive: worker, Workspaces: worker, NativeInstaller: nativeInstaller} } if managedNodes != nil { - deps.Sandboxes = &api.Sandboxes{Deployment: executionStore, DeploymentChanges: worker, ConfigurationDiscovery: managedNodes.setup} + deps.Sandboxes = &api.Sandboxes{ + Deployment: deploymentService, + NodeAllocations: executionStore, + DeploymentChanges: worker, + DeploymentReset: worker, + ConfigurationDiscovery: managedNodes.setup, + } } handler, err := api.NewHandler(deps) if err != nil { diff --git a/services/core/cmd/server/managed_generations.go b/services/core/cmd/server/managed_generations.go index e77a5520d..3d744c15a 100644 --- a/services/core/cmd/server/managed_generations.go +++ b/services/core/cmd/server/managed_generations.go @@ -6,28 +6,19 @@ import ( "maps" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) -type generationStore interface { - GetSandboxAllocationSetup(context.Context, sandbox.Reference) (store.SandboxSetup, error) - GetSandboxSetup(context.Context) (store.SandboxSetup, error) - SandboxGenerationPage(context.Context, int64) ([]store.SandboxSetup, error) - SandboxCredentialAllocationPage(context.Context, string) ([]store.RuntimeAllocation, error) -} - // Every facade, including already running lifecycles, resolves the allocation's // immutable specification and current credential. There is no mutable provider // map to unload and no current-generation fallback for a missing historical row. type generationRouter struct { setup *managedSetup - store generationStore operations providercontract.Operations providerType string } @@ -37,7 +28,7 @@ func (p *generationRouter) route(ctx context.Context, r sandbox.Reference) (sand if err != nil { return nil, nil, err } - setup, err := p.store.GetSandboxAllocationSetup(ctx, r) + setup, err := p.setup.deployment.AllocationSetup(ctx, r) if err != nil { release() return nil, nil, err @@ -120,23 +111,13 @@ func (p *observedGenerationRouter) Observe(ctx context.Context, t runtimeobs.Tar return source.Observe(ctx, t) } -func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeployment, setup store.SandboxSetup) (execution.PreparedRuntimeDeployment, error) { - usesCredential, err := providers.UsesCredential(setup.Provider) - if err != nil { - return execution.PreparedRuntimeDeployment{}, err - } - adapter, err := providers.Lookup(setup.Provider) - if err != nil { - return execution.PreparedRuntimeDeployment{}, err - } - if adapter.Mode == "nodes" { +// routeGenerations routes a direct provider's allocations through their own +// generations. Node providers route through the node transport instead. +func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeployment, setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { + if setup.Mode == "nodes" { return candidate, nil } - db, ok := s.store.(generationStore) - if !ok { - return execution.PreparedRuntimeDeployment{}, errors.New("sandbox generation store is unavailable") - } - router := &generationRouter{setup: s, store: db, providerType: candidate.Config.Provider.(runtimeobs.Source).ObservationProviderType(), operations: candidate.Config.Provider.ProviderOperations()} + router := &generationRouter{setup: s, providerType: candidate.Config.Provider.(runtimeobs.Source).ObservationProviderType(), operations: candidate.Config.Provider.ProviderOperations()} router.operations["ObserveBatch"] = providercontract.Support{State: providercontract.Unsupported, Reason: "allocations_require_individual_generation_routing"} router.operations["DiscoverSelection"] = providercontract.Support{State: providercontract.Unsupported, Reason: "generation_router_does_not_discover_configuration"} router.operations["VerifyCredential"] = providercontract.Support{State: providercontract.Unsupported, Reason: "generation_router_does_not_verify_configuration"} @@ -144,7 +125,7 @@ func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeplo if err := sandbox.ValidateProvider(candidate.Config.Provider); err != nil { return execution.PreparedRuntimeDeployment{}, err } - if !usesCredential { + if !setup.UsesCredential { return candidate, nil } candidate.FenceCredential = func(ctx context.Context) (func(), error) { @@ -159,7 +140,7 @@ func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeplo defer cancel() // Preserve the committed credential until its ownership anchor is verified. // Public template readability cannot establish which team owns a deployment. - verify := func(value store.SandboxSetup, refs []sandbox.Reference) error { + verify := func(value deployment.Setup, refs []sandbox.Reference) error { value.InstallationID = setup.InstallationID provider, err := s.provider(value) if err != nil { @@ -174,27 +155,26 @@ func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeplo } return verifier.VerifyCredential(ctx, refs) } - current, err := db.GetSandboxSetup(ctx) + current, err := s.deployment.Setup(ctx) if err != nil { return err } if current.Provider != setup.Provider { - return &store.SandboxResetRequiredError{CurrentProvider: current.Provider, RequestedProvider: setup.Provider} + return &deployment.ResetRequiredError{CurrentProvider: current.Provider, RequestedProvider: setup.Provider} } if err := verify(current, nil); err != nil { if errors.Is(err, sandbox.ErrCredentialRejected) || errors.Is(err, sandbox.ErrCredentialOwnership) { // A revoked legacy key or a public template outside its team cannot // anchor ownership. This says nothing about the candidate key's validity. - return &store.SandboxResetRequiredError{CurrentProvider: setup.Provider, RequestedProvider: setup.Provider} + return &deployment.ResetRequiredError{CurrentProvider: setup.Provider, RequestedProvider: setup.Provider} } return err } - withCandidateKey := func(value store.SandboxSetup, refs []sandbox.Reference) error { - selection, err := providers.WithCredential(sandbox.Selection{Provider: value.Provider, DeploymentSpec: value.Specification, Configuration: value.Configuration}, sandbox.Selection{Provider: setup.Provider, DeploymentSpec: setup.Specification, Configuration: setup.Configuration}) + withCandidateKey := func(value deployment.Setup, refs []sandbox.Reference) error { + value, err := s.deployment.WithCredential(value, setup) if err != nil { return err } - value.Configuration = selection.Configuration return verify(value, refs) } if err := withCandidateKey(current, nil); err != nil { @@ -203,9 +183,9 @@ func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeplo if err := verify(setup, nil); err != nil { return err } - generations := map[uint64]store.SandboxSetup{current.Generation: current} + generations := map[uint64]deployment.Setup{current.Generation: current} for after := int64(-1); ; { - page, err := db.SandboxGenerationPage(ctx, after) + page, err := s.deployment.GenerationPage(ctx, after) if err != nil { return err } @@ -221,7 +201,7 @@ func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeplo } } for after := ""; ; { - page, err := db.SandboxCredentialAllocationPage(ctx, after) + page, err := s.allocations.SandboxCredentialAllocationPage(ctx, after) if err != nil { return err } diff --git a/services/core/cmd/server/managed_generations_test.go b/services/core/cmd/server/managed_generations_test.go index aca3167eb..adc1a9519 100644 --- a/services/core/cmd/server/managed_generations_test.go +++ b/services/core/cmd/server/managed_generations_test.go @@ -10,28 +10,14 @@ import ( "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" ) -type routingSetupStore struct { - setupStore - old store.SandboxSetup - oldID string -} - -func (s *routingSetupStore) GetSandboxAllocationSetup(_ context.Context, ref sandbox.Reference) (store.SandboxSetup, error) { - value := s.value - if ref.AllocationID == s.oldID { - value = s.old - key := *value.Configuration.(*e2b.DeploymentConfiguration) - key.APIKey = s.value.Configuration.(*e2b.DeploymentConfiguration).APIKey - value.Configuration = &key - } - return value, nil -} func TestE2BRouterKeepsOldSpecificationWithCommittedCredential(t *testing.T) { state := filepath.Join(t.TempDir(), "e2b") if err := os.MkdirAll(state, 0700); err != nil { @@ -51,16 +37,27 @@ print(json.dumps({'Version':1,'Info':info})) } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - old := store.SandboxSetup{InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}, Configuration: &e2b.DeploymentConfiguration{APIKey: "old-key", Template: "old:" + uuid.NewString()}} + old := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, UsesCredential: true, Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}, Configuration: &e2b.DeploymentConfiguration{APIKey: "old-key", Template: "old:" + uuid.NewString()}} current := old current.Generation = 2 current.Specification.Resources.CPUs = 4 current.Configuration = &e2b.DeploymentConfiguration{APIKey: "new-key", Template: "new:" + uuid.NewString()} ref := sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()} - db := &routingSetupStore{setupStore: setupStore{value: current}, old: old, oldID: ref.AllocationID} - setup := &managedSetup{processPaths: paths, store: db, installationID: id} + // The deployment returns an allocation's own generation with the current + // credential. + allocation := func(_ context.Context, r sandbox.Reference) (deployment.Setup, error) { + if r.AllocationID != ref.AllocationID { + return current, nil + } + value := old + key := *old.Configuration.(*e2b.DeploymentConfiguration) + key.APIKey = current.Configuration.(*e2b.DeploymentConfiguration).APIKey + value.Configuration = &key + return value, nil + } + setup := &managedSetup{processPaths: paths, registry: providers.Builtin(), deployment: &fakeDeploymentSetups{t: t, allocationSetup: allocation}, installationID: id} // A facade retained by a generation-one lifecycle still reads current credentials. - router := &generationRouter{setup: setup, store: db} + router := &generationRouter{setup: setup} ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() if _, err := router.GetInfo(ctx, ref); err != nil { @@ -148,11 +145,14 @@ print(json.dumps(result)) } paths := testProviderPaths(t, helper, state) id, build := uuid.NewString(), ":"+uuid.NewString() - current := store.SandboxSetup{InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, + current := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, UsesCredential: true, Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}, Configuration: &e2b.DeploymentConfiguration{APIKey: tc.committedKey, Template: tc.committedTemplate + build}} - db := &setupStore{value: current} - s := &managedSetup{processPaths: paths, installationID: id, store: db} + committed := current + setups := &fakeDeploymentSetups{t: t, setup: committedSetup(&committed), withCredential: credentialService(t), + generationPage: func(context.Context, int64) ([]deployment.Setup, error) { return nil, nil }} + allocations := &fakeGenerationAllocations{t: t, sandboxCredentialAllocationPage: func(context.Context, string) ([]store.RuntimeAllocation, error) { return nil, nil }} + s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: setups, allocations: allocations} loaded, err := s.load(t.Context()) if err != nil { t.Fatal(err) @@ -164,7 +164,7 @@ print(json.dumps(result)) t.Fatal(err) } err = candidate.VerifyCredential(t.Context()) - var reset *store.SandboxResetRequiredError + var reset *deployment.ResetRequiredError if tc.reset { if !errors.As(err, &reset) || errors.Is(err, sandbox.ErrCredentialRejected) || errors.Is(err, sandbox.ErrCredentialOwnership) { t.Fatalf("unanchored ownership misattributed: %v", err) @@ -172,7 +172,7 @@ print(json.dumps(result)) } else if !errors.Is(err, tc.want) { t.Fatalf("got %v; want %v", err, tc.want) } - if db.value.Generation != 1 || db.value.Configuration.(*e2b.DeploymentConfiguration).APIKey != tc.committedKey || s.selected.Load().Config != loaded { + if committed.Generation != 1 || committed.Configuration.(*e2b.DeploymentConfiguration).APIKey != tc.committedKey || s.selected.Load().Config != loaded { t.Fatal("verification mutated committed selection") } raw, err := os.ReadFile(filepath.Join(state, "requests")) diff --git a/services/core/cmd/server/managed_nodes.go b/services/core/cmd/server/managed_nodes.go index 1e3e4fa2b..4ee963ac1 100644 --- a/services/core/cmd/server/managed_nodes.go +++ b/services/core/cmd/server/managed_nodes.go @@ -7,9 +7,12 @@ import ( "os" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/node" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" ) @@ -22,7 +25,11 @@ type managedNodes struct { closeProvider func() } -func configureManagedNodes(s *store.Store, publicURL string, owner func(context.Context) error) (*managedNodes, error) { +// configureManagedNodes serves the nodes of the Web-managed deployment. Node +// presence and health go through the deployment service, the owner epoch that +// fences connections is read from the deployment reader, and the store +// resolves the generation of each allocation. +func configureManagedNodes(s *store.Store, nodes *deployment.Service, reader deployment.Reader, registry *providers.Registry, publicURL string, owner func(context.Context) error) (*managedNodes, error) { setupID := os.Getenv("OAC_INSTALLATION_ID") if setupID == "" { return nil, nil @@ -54,17 +61,20 @@ func configureManagedNodes(s *store.Store, publicURL string, owner func(context. if err := owner(ctx); err != nil { return err } - return s.HeartbeatRuntimeNodeGenerations(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health), health.Generations) + return nodes.HeartbeatGenerations(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health), health.Generations) }, Retention: func(ctx context.Context, n node.Identity, connection string, epoch uint64, refs []sandbox.GenerationReference) (sandbox.NodeDeployment, []sandbox.GenerationRetention, error) { if err := owner(ctx); err != nil { return sandbox.NodeDeployment{}, nil, err } - return s.RuntimeNodeRetention(ctx, n.NodeID, connection, epoch, refs) + // The node waits for this answer; bound it like every leased operation. + ctx, cancel := context.WithTimeout(ctx, pgunit.ExecutionTimeout) + defer cancel() + return nodes.NodeRetention(ctx, n.NodeID, connection, epoch, refs) }, Authenticate: func(ctx context.Context, id, credential string) (node.Identity, error) { - n, err := s.AuthenticateRuntimeNode(ctx, id, credential) - if errors.Is(err, store.ErrRuntimeNodeCredential) { + n, err := nodes.AuthenticateNode(ctx, id, credential) + if errors.Is(err, deployment.ErrNodeCredential) { err = node.ErrAuthentication } return node.Identity{SpecificationDigest: n.SpecificationDigest, DeploymentGeneration: n.DeploymentGeneration, NodeID: n.NodeID, InstallationID: n.InstallationID, Provider: n.Provider, BackendFingerprint: n.BackendFingerprint, MaxActive: n.MaxActive, MaxRetained: n.MaxRetained}, err @@ -73,25 +83,25 @@ func configureManagedNodes(s *store.Store, publicURL string, owner func(context. if err := owner(ctx); err != nil { return 0, err } - return s.RuntimeOwnerEpoch(ctx) + return reader.OwnerEpoch(ctx) }, Connected: func(ctx context.Context, n node.Identity, connection string, epoch uint64) error { if err := owner(ctx); err != nil { return err } - return s.ConnectRuntimeNode(ctx, n.NodeID, connection, epoch) + return nodes.ConnectNode(ctx, n.NodeID, connection, epoch) }, Disconnected: func(ctx context.Context, n node.Identity, connection string, epoch uint64) { - _ = s.DisconnectRuntimeNode(ctx, n.NodeID, connection, epoch) + _ = nodes.DisconnectNode(ctx, n.NodeID, connection, epoch) }, Heartbeat: func(ctx context.Context, n node.Identity, connection string, epoch uint64, health node.Health) error { if err := owner(ctx); err != nil { return err } - return s.HeartbeatRuntimeNode(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health)) + return nodes.Heartbeat(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health)) }, }) - result.setup = &managedSetup{processPaths: providerProcessPaths(), store: s, hub: result.hub, installationID: setupID, publicURL: publicURL} + result.setup = &managedSetup{processPaths: providerProcessPaths(), registry: registry, deployment: nodes, allocations: s, hub: result.hub, installationID: setupID, publicURL: publicURL} result.runtime = execution.NewDeferredRuntimeProvider(setupID, result.setup.load, result.setup.prepare) result.runtime.PublishUnconfigured = result.setup.publishUnconfigured success = true @@ -128,6 +138,6 @@ func serverAddress() string { return "127.0.0.1:8091" } -func nodeHealthRecord(health node.Health) store.RuntimeNodeHealth { - return store.RuntimeNodeHealth{Host: &store.RuntimeNodeHost{EffectiveCPUCores: health.EffectiveCPUCores, CPUUtilization: health.CPUUtilization, TotalMemoryBytes: health.TotalMemoryBytes, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes, ObservedAt: &health.ObservedAt}, ProviderReady: health.ProviderReady, Diagnostic: health.Diagnostic, CPUCount: health.CPUCount, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes} +func nodeHealthRecord(health node.Health) deployment.NodeHealth { + return deployment.NodeHealth{Host: &deployment.NodeHost{EffectiveCPUCores: health.EffectiveCPUCores, CPUUtilization: health.CPUUtilization, TotalMemoryBytes: health.TotalMemoryBytes, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes, ObservedAt: &health.ObservedAt}, ProviderReady: health.ProviderReady, Diagnostic: health.Diagnostic, CPUCount: health.CPUCount, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes} } diff --git a/services/core/cmd/server/managed_setup.go b/services/core/cmd/server/managed_setup.go index 5cffa21b0..295075307 100644 --- a/services/core/cmd/server/managed_setup.go +++ b/services/core/cmd/server/managed_setup.go @@ -9,6 +9,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" @@ -22,10 +23,11 @@ import ( // observation. The database owns the selection; this cache is never a writer. type managedSetup struct { processPaths sandbox.ProcessPaths - store interface { - GetSandboxSetup(context.Context) (store.SandboxSetup, error) - ResolveRuntimeGeneration(context.Context, sandbox.Reference) (string, uint64, error) - } + // registry builds the selected direct provider and discovers configuration; + // the deployment setup reports what the registration declares. + registry *providers.Registry + deployment deploymentSetups + allocations generationAllocations hub *node.Hub installationID string // publicURL is OAC_PUBLIC_URL; every sandbox reaches Core through it. @@ -34,10 +36,26 @@ type managedSetup struct { providerCalls sandbox.CallFence } +// deploymentSetups reads the committed deployment setup and its retained +// generations. *deployment.Service implements it. +type deploymentSetups interface { + Setup(context.Context) (deployment.Setup, error) + AllocationSetup(context.Context, sandbox.Reference) (deployment.Setup, error) + GenerationPage(context.Context, int64) ([]deployment.Setup, error) + WithCredential(owner, candidate deployment.Setup) (deployment.Setup, error) +} + +// generationAllocations resolves the generation that owns each allocation. +// *store.Store implements it. +type generationAllocations interface { + ResolveRuntimeGeneration(context.Context, sandbox.Reference) (string, uint64, error) + SandboxCredentialAllocationPage(context.Context, string) ([]store.RuntimeAllocation, error) +} + // DiscoverConfiguration asks a Provider which configuration values its // credential can use, with this installation's process paths. func (s *managedSetup) DiscoverConfiguration(ctx context.Context, provider string, input sandbox.ConfigurationDiscoveryInput) (json.RawMessage, error) { - return providers.DiscoverConfiguration(ctx, provider, input, s.processPaths) + return s.registry.DiscoverConfiguration(ctx, provider, input, s.processPaths) } // Empty selections retain their generation so a delayed provider load cannot @@ -65,7 +83,7 @@ func (s *managedSetup) publish(config *execution.RuntimeProvider) { func (s *managedSetup) publishUnconfigured(generation uint64) { s.publishSelection(generation, nil) } func (s *managedSetup) load(ctx context.Context) (*execution.RuntimeProvider, error) { - setup, err := s.store.GetSandboxSetup(ctx) + setup, err := s.deployment.Setup(ctx) if err != nil { return nil, err } @@ -89,15 +107,9 @@ func (s *managedSetup) load(ctx context.Context) (*execution.RuntimeProvider, er return s.publishSelection(setup.Generation, candidate.Config), nil } -func (s *managedSetup) prepare(ctx context.Context, setup store.SandboxSetup) (execution.PreparedRuntimeDeployment, error) { - // Adapters declare whether their guests require a public Core origin. - requiresPublicOrigin, err := providers.RequiresPublicOrigin(setup.Provider) - if err != nil { - return execution.PreparedRuntimeDeployment{}, err - } - if requiresPublicOrigin && store.LoopbackOrigin(s.publicURL) { - return execution.PreparedRuntimeDeployment{}, store.ErrSandboxPublicURLUnreachable - } +// prepare validates a setup the deployment prepared for a selection, which has +// already rejected a provider whose guests cannot reach the public URL. +func (s *managedSetup) prepare(ctx context.Context, setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { candidate, err := s.configuration(setup) if err != nil { return execution.PreparedRuntimeDeployment{}, err @@ -123,7 +135,7 @@ func (s *managedSetup) prepare(ctx context.Context, setup store.SandboxSetup) (e // Loading an already committed selection must retain provider access to its // owned resources, even when a new-template validation would now fail. -func (s *managedSetup) configuration(setup store.SandboxSetup) (execution.PreparedRuntimeDeployment, error) { +func (s *managedSetup) configuration(setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { if setup.InstallationID != s.installationID { return execution.PreparedRuntimeDeployment{}, errors.New("sandbox installation does not match setup") } @@ -133,9 +145,9 @@ func (s *managedSetup) configuration(setup store.SandboxSetup) (execution.Prepar } selected := &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: s.publicURL + "/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} - if sandbox.SupportsCheckpoint(provider) { - selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.IdleSeconds) * time.Second, - Retention: time.Duration(setup.RetentionSeconds) * time.Second, MaxActive: 4, MaxRetained: 16} + if setup.Suspension != nil { + selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.Suspension.IdleSeconds) * time.Second, + Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second, MaxActive: 4, MaxRetained: 16} } return execution.PreparedRuntimeDeployment{Config: selected, Publish: s.publish}, nil } @@ -159,16 +171,14 @@ func (s *managedSetup) ResolveObservationSource(ctx context.Context) (runtimeobs return source, nil } -func (s *managedSetup) provider(setup store.SandboxSetup) (sandbox.SandboxProvider, error) { - adapter, err := providers.Lookup(setup.Provider) - if err != nil { - return nil, err - } - if adapter.Mode == "nodes" { +// provider builds the setup's provider. The setup carries the mode and +// declared operations that deployment read from the provider's registration. +func (s *managedSetup) provider(setup deployment.Setup) (sandbox.SandboxProvider, error) { + if setup.Mode == "nodes" { if s.hub == nil { return nil, errors.New("sandbox node transport is unavailable") } - return s.hub.GenerationProvider(setup.Provider, s.store.ResolveRuntimeGeneration), nil + return s.hub.GenerationProvider(setup.Provider, setup.Operations, s.allocations.ResolveRuntimeGeneration), nil } - return providers.BuildDirect(providers.DirectConfig{ProcessPaths: s.processPaths, InstallationID: setup.InstallationID, Selection: sandbox.Selection{Provider: setup.Provider, DeploymentSpec: setup.Specification, Configuration: setup.Configuration}, Fence: &s.providerCalls}) + return s.registry.BuildDirect(providers.DirectConfig{ProcessPaths: s.processPaths, InstallationID: setup.InstallationID, Selection: sandbox.Selection{Provider: setup.Provider, DeploymentSpec: setup.Specification, Configuration: setup.Configuration}, Fence: &s.providerCalls}) } diff --git a/services/core/cmd/server/managed_setup_preflight_test.go b/services/core/cmd/server/managed_setup_preflight_test.go index ceee02d7f..dc977e41a 100644 --- a/services/core/cmd/server/managed_setup_preflight_test.go +++ b/services/core/cmd/server/managed_setup_preflight_test.go @@ -9,10 +9,11 @@ import ( "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "github.com/google/uuid" ) @@ -27,13 +28,13 @@ func TestE2BRejectedSpecificationHasSafeActionableDiagnostic(t *testing.T) { } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - s := &managedSetup{processPaths: paths, installationID: id} - selection := store.SandboxSetup{InstallationID: id, Provider: "e2b", Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 3, MemoryMiB: 3072}}, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} + s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id} + selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 3, MemoryMiB: 3072}}, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} _, err := s.prepare(t.Context(), selection) if !errors.Is(err, sandbox.ErrConfigurationSelection) || strings.Contains(err.Error(), "synthetic-private-key") || s.selected.Load() != nil { t.Fatal("rejected candidate lost its safe diagnostic or was published", err) } - s.store = &setupStore{value: selection} + s.deployment = &fakeDeploymentSetups{t: t, setup: committedSetup(&selection)} if restored, err := s.load(t.Context()); err != nil || restored == nil || restored.Provider == nil { t.Fatal("template rejection prevented loading committed resource ownership", err) } @@ -64,10 +65,12 @@ else: } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - selection := store.SandboxSetup{InstallationID: id, Provider: "e2b", Generation: 1, + selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, UsesCredential: true, Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} - s := &managedSetup{processPaths: paths, installationID: id, store: &setupStore{value: selection}} + allocation := func(context.Context, sandbox.Reference) (deployment.Setup, error) { return selection, nil } + s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, + deployment: &fakeDeploymentSetups{t: t, setup: committedSetup(&selection), allocationSetup: allocation}} if _, err := s.prepare(t.Context(), selection); err == nil || s.selected.Load() != nil { t.Fatal("invalid new template selection was published", err) } @@ -104,8 +107,8 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - s := &managedSetup{processPaths: paths, installationID: id, store: &setupStore{}} - selection := store.SandboxSetup{InstallationID: id, Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} + s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} + selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} candidate, err := s.prepare(t.Context(), selection) disk := int32(24063) if err != nil || candidate.Selection.Configuration.(*e2b.DeploymentConfiguration).TemplateBuild == nil || candidate.Selection.Configuration.(*e2b.DeploymentConfiguration).TemplateBuild.CPUs != 4 || candidate.Selection.Configuration.(*e2b.DeploymentConfiguration).TemplateBuild.MemoryMiB != 4096 || @@ -139,43 +142,19 @@ func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - s := &managedSetup{processPaths: paths, installationID: id, store: &setupStore{}} - selection := store.SandboxSetup{InstallationID: id, Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-team-a", Template: "public-team-b:" + uuid.NewString()}} + s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} + selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-team-a", Template: "public-team-b:" + uuid.NewString()}} if _, err := s.prepare(t.Context(), selection); !errors.Is(err, sandbox.ErrCredentialOwnership) || s.selected.Load() != nil { t.Fatal("public readability accepted as team ownership", err) } } -func TestManagedSetupRejectsUnknownRegistrationBeforePreparationOrRouting(t *testing.T) { - // No store or node hub is available: rejection must precede any use of them. - s := &managedSetup{installationID: "installation", publicURL: "http://127.0.0.1"} - setup := store.SandboxSetup{InstallationID: "installation", Provider: "missing-registration"} - for _, call := range []struct { - name string - run func() (execution.PreparedRuntimeDeployment, error) - }{ - {"prepare", func() (execution.PreparedRuntimeDeployment, error) { - return s.prepare(t.Context(), setup) - }}, - {"route", func() (execution.PreparedRuntimeDeployment, error) { - return s.routeGenerations(execution.PreparedRuntimeDeployment{}, setup) - }}, - } { - t.Run(call.name, func(t *testing.T) { - candidate, err := call.run() - if !errors.Is(err, sandbox.ErrInvalid) || candidate.Config != nil || s.selected.Load() != nil { - t.Fatalf("invalid registration reached preparation or publication: %v", err) - } - }) - } -} - func TestManagedSetupRoutesProviderWithoutCredentialRequirement(t *testing.T) { s := &managedSetup{} config := &execution.RuntimeProvider{ProviderKind: "docker"} candidate, err := s.routeGenerations( execution.PreparedRuntimeDeployment{Config: config}, - store.SandboxSetup{Provider: "docker"}, + deployment.Setup{Provider: "docker", Mode: "nodes"}, ) if err != nil || candidate.Config != config || candidate.FenceCredential != nil || candidate.VerifyCredential != nil { t.Fatalf("explicit no-credential provider required credential routing: %v", err) diff --git a/services/core/cmd/server/managed_setup_test.go b/services/core/cmd/server/managed_setup_test.go index d43376144..1b779186b 100644 --- a/services/core/cmd/server/managed_setup_test.go +++ b/services/core/cmd/server/managed_setup_test.go @@ -10,11 +10,14 @@ import ( "strings" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -31,10 +34,10 @@ func TestWebSetupCreatesManagerWithoutLocalProvider(t *testing.T) { t.Fatal(err) } t.Setenv("OAC_CORE_KEY_DIGESTS_FILE", path) - if _, err := configureManagedNodes(nil, "", nil); err == nil || !strings.Contains(err.Error(), "OAC_PUBLIC_URL") { + if _, err := configureManagedNodes(nil, nil, nil, providers.Builtin(), "", nil); err == nil || !strings.Contains(err.Error(), "OAC_PUBLIC_URL") { t.Fatal("sandbox manager started without a public URL", err) } - m, err := configureManagedNodes(nil, "https://core.example", func(context.Context) error { return nil }) + m, err := configureManagedNodes(nil, nil, nil, providers.Builtin(), "https://core.example", func(context.Context) error { return nil }) if err != nil { t.Fatal(err) } @@ -43,43 +46,111 @@ func TestWebSetupCreatesManagerWithoutLocalProvider(t *testing.T) { t.Fatal("zero-node setup unexpectedly instantiated local compute or omitted management") } t.Setenv("OAC_CORE_KEY_DIGESTS_FILE", "") - if _, err := configureManagedNodes(nil, "https://core.example", nil); err == nil { + if _, err := configureManagedNodes(nil, nil, nil, providers.Builtin(), "https://core.example", nil); err == nil { t.Fatal("setup accepted without admin authentication") } } -type setupStore struct { - value store.SandboxSetup - err error +// fakeDeploymentSetups is a strict deploymentSetups: a call without a set +// function fails the test. +type fakeDeploymentSetups struct { + t testing.TB + setup func(context.Context) (deployment.Setup, error) + allocationSetup func(context.Context, sandbox.Reference) (deployment.Setup, error) + generationPage func(context.Context, int64) ([]deployment.Setup, error) + withCredential func(owner, candidate deployment.Setup) (deployment.Setup, error) } -func (s *setupStore) GetSandboxSetup(context.Context) (store.SandboxSetup, error) { - return s.value, s.err +func (f *fakeDeploymentSetups) Setup(ctx context.Context) (deployment.Setup, error) { + if f.setup == nil { + return deployment.Setup{}, unexpectedCall(f.t, "Setup") + } + return f.setup(ctx) +} +func (f *fakeDeploymentSetups) AllocationSetup(ctx context.Context, ref sandbox.Reference) (deployment.Setup, error) { + if f.allocationSetup == nil { + return deployment.Setup{}, unexpectedCall(f.t, "AllocationSetup") + } + return f.allocationSetup(ctx, ref) } -func (*setupStore) ResolveRuntimeNode(context.Context, string, string) (string, error) { - return "", errors.New("unexpected node lookup") +func (f *fakeDeploymentSetups) GenerationPage(ctx context.Context, after int64) ([]deployment.Setup, error) { + if f.generationPage == nil { + return nil, unexpectedCall(f.t, "GenerationPage") + } + return f.generationPage(ctx, after) +} +func (f *fakeDeploymentSetups) WithCredential(owner, candidate deployment.Setup) (deployment.Setup, error) { + if f.withCredential == nil { + return deployment.Setup{}, unexpectedCall(f.t, "WithCredential") + } + return f.withCredential(owner, candidate) +} + +// fakeGenerationAllocations is a strict generationAllocations. +type fakeGenerationAllocations struct { + t testing.TB + resolveRuntimeGeneration func(context.Context, sandbox.Reference) (string, uint64, error) + sandboxCredentialAllocationPage func(context.Context, string) ([]store.RuntimeAllocation, error) +} + +func (f *fakeGenerationAllocations) ResolveRuntimeGeneration(ctx context.Context, ref sandbox.Reference) (string, uint64, error) { + if f.resolveRuntimeGeneration == nil { + return "", 0, unexpectedCall(f.t, "ResolveRuntimeGeneration") + } + return f.resolveRuntimeGeneration(ctx, ref) +} +func (f *fakeGenerationAllocations) SandboxCredentialAllocationPage(ctx context.Context, after string) ([]store.RuntimeAllocation, error) { + if f.sandboxCredentialAllocationPage == nil { + return nil, unexpectedCall(f.t, "SandboxCredentialAllocationPage") + } + return f.sandboxCredentialAllocationPage(ctx, after) +} + +// unexpectedCall fails the test from any goroutine and returns the error the +// caller propagates. +func unexpectedCall(t testing.TB, method string) error { + t.Errorf("unexpected call to %s", method) + return errors.New("unexpected call to " + method) +} + +// committedSetup reads *value as the deployment's committed setup. +func committedSetup(value *deployment.Setup) func(context.Context) (deployment.Setup, error) { + return func(context.Context) (deployment.Setup, error) { return *value, nil } +} + +// credentialService gives an owner setup a candidate's credential as the +// deployment service does. That needs no storage. +func credentialService(t *testing.T) func(owner, candidate deployment.Setup) (deployment.Setup, error) { + t.Helper() + service, err := deployment.NewService(deploymentpg.New(nil, nil), deploymentpg.New(nil, nil), providers.Builtin(), "") + if err != nil { + t.Fatal(err) + } + return service.WithCredential } func TestManagedSetupNeverReusesAnotherGenerationOrUnverifiedState(t *testing.T) { - db := &setupStore{value: store.SandboxSetup{InstallationID: "installation", Provider: "docker", Generation: 1}} - s := &managedSetup{store: db, installationID: "installation"} + value := deployment.Setup{InstallationID: "installation", Provider: "docker", Mode: "nodes", Generation: 1} + var loadErr error + setups := &fakeDeploymentSetups{t: t, setup: func(context.Context) (deployment.Setup, error) { return value, loadErr }} + s := &managedSetup{registry: providers.Builtin(), deployment: setups, allocations: &fakeGenerationAllocations{t: t}, installationID: "installation"} cached := &execution.RuntimeProvider{InstallationID: "installation", ProviderKind: "docker", Generation: 1} s.publish(cached) if got, err := s.load(t.Context()); err != nil || got != cached { t.Fatal("matching immutable selection was not reused") } - db.err = errors.New("database unavailable") + loadErr = errors.New("database unavailable") if _, err := s.load(t.Context()); err == nil { t.Fatal("stale cached selection hid storage failure") } - db.err = nil - db.value.Generation = 2 + loadErr = nil + value.Generation = 2 // No Hub is installed; a changed generation must construct again and fail. if _, err := s.load(t.Context()); !errors.Is(err, execution.ErrExecutionUnavailable) { t.Fatal("changed provider availability must block execution without losing recovery", err) } - db.value.InstallationID = "other-installation" - db.value.Generation = 1 + value.InstallationID = "other-installation" + value.Generation = 1 if _, err := s.load(t.Context()); err == nil { t.Fatal("cache ignored installation identity") } @@ -87,10 +158,10 @@ func TestManagedSetupNeverReusesAnotherGenerationOrUnverifiedState(t *testing.T) func TestMissingE2BHelperReportsProviderUnavailable(t *testing.T) { id := uuid.NewString() - s := &managedSetup{processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, installationID: id, store: &setupStore{value: store.SandboxSetup{ - InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, - Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-key", Template: "runtime:" + uuid.NewString()}, - }}} + committed := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, UsesCredential: true, + Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-key", Template: "runtime:" + uuid.NewString()}} + s := &managedSetup{processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id, + deployment: &fakeDeploymentSetups{t: t, setup: committedSetup(&committed)}} if _, err := s.load(t.Context()); !errors.Is(err, execution.ErrExecutionUnavailable) { t.Fatal("missing local helper must leave administrative recovery available", err) } @@ -103,10 +174,10 @@ func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { id := uuid.NewString() hub := node.NewHub(node.HubOptions{}) defer hub.Close() - s := &managedSetup{installationID: id, hub: hub, store: &setupStore{}, publicURL: "https://core.example"} + s := &managedSetup{registry: providers.Builtin(), installationID: id, hub: hub, deployment: &fakeDeploymentSetups{t: t}, allocations: &fakeGenerationAllocations{t: t}, publicURL: "https://core.example"} previous := &execution.RuntimeProvider{InstallationID: id, Generation: 1, ProviderKind: "docker"} s.publish(previous) - candidate, err := s.prepare(t.Context(), store.SandboxSetup{InstallationID: id, Provider: "microsandbox", Mode: "nodes", IdleSeconds: 300, RetentionSeconds: 86400}) + candidate, err := s.prepare(t.Context(), deployment.Setup{InstallationID: id, Provider: "microsandbox", Mode: "nodes", Operations: microsandbox.Operations(), Suspension: &deployment.Suspension{IdleSeconds: 300, RetentionSeconds: 86400}}) if err != nil { t.Fatal(err) } @@ -123,45 +194,30 @@ func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { func TestManagedSetupRejectedCandidateRetainsSelection(t *testing.T) { id := uuid.NewString() - s := &managedSetup{processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, installationID: id} + s := &managedSetup{processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id} previous := &execution.RuntimeProvider{InstallationID: id, Generation: 1, ProviderKind: "docker"} s.publish(previous) - _, err := s.prepare(t.Context(), store.SandboxSetup{InstallationID: id, Provider: "e2b", Mode: "direct", + _, err := s.prepare(t.Context(), deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-key", Template: "runtime:" + uuid.NewString()}}) if !errors.Is(err, execution.ErrExecutionUnavailable) || s.selected.Load().Config != previous { t.Fatal("rejected candidate lost the previous selection", err) } } -func TestE2BRequiresAPublicURLOutsideTheHost(t *testing.T) { - id := uuid.NewString() - s := &managedSetup{installationID: id, publicURL: "http://127.0.0.1:8091"} - _, err := s.prepare(t.Context(), store.SandboxSetup{InstallationID: id, Provider: "e2b", Mode: "direct", - Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-key", Template: "runtime:" + uuid.NewString()}}) - if !errors.Is(err, store.ErrSandboxPublicURLUnreachable) { - t.Fatal("E2B accepted a loopback public URL", err) - } -} - -type delayedSetupStore struct { - setupStore - entered, release chan struct{} -} - -func (s *delayedSetupStore) GetSandboxSetup(ctx context.Context) (store.SandboxSetup, error) { - value := s.value - close(s.entered) - select { - case <-s.release: - return value, nil - case <-ctx.Done(): - return store.SandboxSetup{}, ctx.Err() - } -} func TestManagedSetupResetTombstoneRejectsDelayedProviderLoad(t *testing.T) { id := uuid.NewString() - db := &delayedSetupStore{setupStore: setupStore{value: store.SandboxSetup{InstallationID: id, Provider: "docker", Generation: 1}}, entered: make(chan struct{}), release: make(chan struct{})} - s := &managedSetup{installationID: id, store: db} + value := deployment.Setup{InstallationID: id, Provider: "docker", Mode: "nodes", Generation: 1} + entered, release := make(chan struct{}), make(chan struct{}) + delayed := func(ctx context.Context) (deployment.Setup, error) { + close(entered) + select { + case <-release: + return value, nil + case <-ctx.Done(): + return deployment.Setup{}, ctx.Err() + } + } + s := &managedSetup{registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: delayed}} done := make(chan error, 1) go func() { provider, err := s.load(t.Context()) @@ -170,9 +226,9 @@ func TestManagedSetupResetTombstoneRejectsDelayedProviderLoad(t *testing.T) { } done <- err }() - <-db.entered + <-entered s.publishUnconfigured(2) - close(db.release) + close(release) if err := <-done; err != nil { t.Fatal(err) } @@ -190,20 +246,6 @@ func TestManagedSetupResetTombstoneRejectsDelayedProviderLoad(t *testing.T) { } } -func (s *setupStore) GetSandboxAllocationSetup(_ context.Context, _ sandbox.Reference) (store.SandboxSetup, error) { - return s.value, nil -} -func (s *setupStore) SandboxGenerationPage(context.Context, int64) ([]store.SandboxSetup, error) { - return nil, nil -} -func (s *setupStore) SandboxCredentialAllocationPage(context.Context, string) ([]store.RuntimeAllocation, error) { - return nil, nil -} - -func (s *setupStore) ResolveRuntimeGeneration(context.Context, sandbox.Reference) (string, uint64, error) { - return "", 0, errors.New("unexpected node generation lookup") -} - func testProviderPaths(t *testing.T, helper, state string) sandbox.ProcessPaths { t.Helper() paths := sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: filepath.Dir(state)} @@ -225,20 +267,20 @@ func testProviderPaths(t *testing.T, helper, state string) sandbox.ProcessPaths } func TestManagedObservationSourceKeepsSelectionAcrossReconfiguration(t *testing.T) { - db := &setupStore{value: store.SandboxSetup{InstallationID: "installation", Generation: 1}} - setup := &managedSetup{store: db, installationID: "installation"} + value := deployment.Setup{InstallationID: "installation", Generation: 1} + setup := &managedSetup{registry: providers.Builtin(), deployment: &fakeDeploymentSetups{t: t, setup: committedSetup(&value)}, installationID: "installation"} if source, err := setup.ResolveObservationSource(t.Context()); source != nil || !errors.Is(err, runtimeobs.ErrUnavailable) { t.Fatal("unconfigured setup did not return typed unavailability", source, err) } first := &docker.Provider{} - db.value.Provider, db.value.Generation = "docker", 2 + value.Provider, value.Mode, value.Generation = "docker", "nodes", 2 setup.publish(&execution.RuntimeProvider{Generation: 2, Provider: first}) source, err := setup.ResolveObservationSource(t.Context()) if err != nil || source != first { t.Fatal(source, err) } next := µsandbox.Provider{} - db.value.Provider, db.value.Generation = "microsandbox", 3 + value.Provider, value.Mode, value.Generation = "microsandbox", "nodes", 3 setup.publish(&execution.RuntimeProvider{Generation: 3, Provider: next}) if source.ObservationProviderType() != "docker" { t.Fatal("in-flight identity changed") @@ -252,9 +294,11 @@ func TestManagedObservationSourceKeepsSelectionAcrossReconfiguration(t *testing. func TestObservationGenerationIdentityMustMatchRoutedAllocation(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() - db := &setupStore{value: store.SandboxSetup{Provider: "docker"}} - setup := &managedSetup{hub: hub, store: db} - source := &observedGenerationRouter{&generationRouter{setup: setup, store: db, providerType: "e2b"}} + routed := func(context.Context, sandbox.Reference) (deployment.Setup, error) { + return deployment.Setup{Provider: "docker", Mode: "nodes"}, nil + } + setup := &managedSetup{registry: providers.Builtin(), hub: hub, deployment: &fakeDeploymentSetups{t: t, allocationSetup: routed}, allocations: &fakeGenerationAllocations{t: t}} + source := &observedGenerationRouter{&generationRouter{setup: setup, providerType: "e2b"}} _, err := source.Observe(t.Context(), runtimeobs.Target{TenantID: "tenant", EnvironmentID: "environment", Instance: runtimeobs.Instance{AllocationID: "allocation"}}) if !errors.Is(err, providercontract.ErrContract) { t.Fatal("routed allocation was attributed to another provider", err) diff --git a/services/core/cmd/server/process_configuration.go b/services/core/cmd/server/process_configuration.go index e7f1d6924..4b41c3be5 100644 --- a/services/core/cmd/server/process_configuration.go +++ b/services/core/cmd/server/process_configuration.go @@ -7,9 +7,9 @@ import ( "strconv" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) func executionConcurrency() (int, error) { @@ -39,7 +39,7 @@ func publicURL() (string, error) { if value == "" { return "", nil } - if store.ValidateSandboxCoreURL(value) != nil { + if deployment.ValidateCoreURL(value) != nil { return "", errors.New("OAC_PUBLIC_URL must be a canonical HTTPS origin without path, credentials, query or fragment, such as https://core.example; plain HTTP is accepted only for a loopback host") } return value, nil diff --git a/services/core/cmd/specification-contract/main.go b/services/core/cmd/specification-contract/main.go index 3d02f3c97..bb2d671c0 100644 --- a/services/core/cmd/specification-contract/main.go +++ b/services/core/cmd/specification-contract/main.go @@ -12,7 +12,7 @@ import ( func main() { write := flag.Bool("write", false, "update deploy/install/node_spec.py from the repository root") flag.Parse() - projection, err := providers.PythonDeploymentContract() + projection, err := providers.Builtin().PythonDeploymentContract() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) diff --git a/services/core/internal/api/admin_session_archive_test.go b/services/core/internal/api/admin_session_archive_test.go index 68ce043fc..eb7f2c05b 100644 --- a/services/core/internal/api/admin_session_archive_test.go +++ b/services/core/internal/api/admin_session_archive_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) @@ -72,7 +73,7 @@ func TestAdminSessionArchiveAuthorityAndValidation(t *testing.T) { for _, failure := range []struct { err error status int - }{{store.ErrSandboxDeploymentConflict, 409}, {store.ErrNotFound, 404}, {store.ErrInvalidInput, 400}} { + }{{deployment.ErrConflict, 409}, {store.ErrNotFound, 404}, {store.ErrInvalidInput, 400}} { fixture.err = failure.err if w := projectKeyHTTP(h, http.MethodPost, path, "admin", `{"expected_generation":2}`); w.Code != failure.status { t.Fatalf("archive error: %d %s", w.Code, w.Body) diff --git a/services/core/internal/api/core_store_validation_test.go b/services/core/internal/api/core_store_validation_test.go index a815a237a..0cca0fb45 100644 --- a/services/core/internal/api/core_store_validation_test.go +++ b/services/core/internal/api/core_store_validation_test.go @@ -10,17 +10,24 @@ import ( "strings" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/projects" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" ) func TestCoreStoreValidationFieldsAndPublicFallback(t *testing.T) { - s := &store.Store{} - upperCapacityErr := s.UpdateRuntimeNode(context.Background(), managementProjectID, store.RuntimeNodeUpdate{Name: "node", MaxActive: 1000001, MaxRetained: 8}) - capacityErr := s.UpdateRuntimeNode(context.Background(), managementProjectID, store.RuntimeNodeUpdate{Name: "node", MaxActive: 0}) - _, resourceErr := store.SandboxSetupForSelection(managementProjectID, store.SandboxDeploymentSetupRequest{Provider: "docker", DeploymentSpec: sandbox.DeploymentSpec{}}) - _, runtimeErr := store.SandboxSetupForSelection(managementProjectID, store.SandboxDeploymentSetupRequest{Provider: "docker", DeploymentSpec: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 1, MemoryMiB: 512}}}) + // The deployment validates these inputs before it reaches storage, so + // storage without a database is enough. + nodes, err := deployment.NewService(deploymentpg.New(nil, nil), deploymentpg.New(nil, nil), providers.Builtin(), "") + if err != nil { + t.Fatal(err) + } + upperCapacityErr := nodes.UpdateNode(context.Background(), managementProjectID, deployment.NodeUpdate{Name: "node", MaxActive: 1000001, MaxRetained: 8}) + capacityErr := nodes.UpdateNode(context.Background(), managementProjectID, deployment.NodeUpdate{Name: "node", MaxActive: 0}) + _, resourceErr := nodes.SetupForSelection(managementProjectID, sandbox.Selection{Provider: "docker", DeploymentSpec: sandbox.DeploymentSpec{}}) + _, runtimeErr := nodes.SetupForSelection(managementProjectID, sandbox.Selection{Provider: "docker", DeploymentSpec: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 1, MemoryMiB: 512}}}) for _, tc := range []struct { err error code, param string @@ -30,10 +37,10 @@ func TestCoreStoreValidationFieldsAndPublicFallback(t *testing.T) { }{ {&sandbox.ValidationError{Param: "resources", Message: "E2B template build resources are outside the supported sandbox limits; select another build"}, "invalid_sandbox_configuration", "resources", nil, "E2B template build resources are outside the supported sandbox limits; select another build", "invalid_sandbox_configuration", nil}, {&projects.NameError{MaxLength: projects.ProjectNameMaxLength}, "invalid_name", "name", map[string]any{"max_length": float64(128)}, "Invalid resource identifier or request limits.", "invalid_request", writeProjectsError}, - {upperCapacityErr, "invalid_node_capacity", "max_active", map[string]any{"min": float64(1), "max": float64(1000000)}, "Invalid resource identifier or request limits.", "invalid_request", nil}, - {capacityErr, "invalid_node_capacity", "max_active", map[string]any{"min": float64(1), "max": float64(1000000)}, "Invalid resource identifier or request limits.", "invalid_request", nil}, - {resourceErr, "invalid_sandbox_configuration", "resources.cpus", map[string]any{"min": float64(1), "max": float64(255)}, "invalid sandbox configuration: cpus must be 1..255 and memory_mib must be 512..1048576", "invalid_sandbox_configuration", nil}, - {runtimeErr, "invalid_sandbox_configuration", "runtime", nil, "invalid sandbox configuration: managed nodes require a pinned Runtime release", "invalid_sandbox_configuration", nil}, + {upperCapacityErr, "invalid_node_capacity", "max_active", map[string]any{"min": float64(1), "max": float64(1000000)}, "Invalid resource identifier or request limits.", "invalid_request", writeDeploymentError}, + {capacityErr, "invalid_node_capacity", "max_active", map[string]any{"min": float64(1), "max": float64(1000000)}, "Invalid resource identifier or request limits.", "invalid_request", writeDeploymentError}, + {resourceErr, "invalid_sandbox_configuration", "resources.cpus", map[string]any{"min": float64(1), "max": float64(255)}, "invalid sandbox configuration: cpus must be 1..255 and memory_mib must be 512..1048576", "invalid_sandbox_configuration", writeDeploymentError}, + {runtimeErr, "invalid_sandbox_configuration", "runtime", nil, "invalid sandbox configuration: managed nodes require a pinned Runtime release", "invalid_sandbox_configuration", writeDeploymentError}, } { if tc.err == nil { t.Fatal("missing validator error") @@ -77,11 +84,11 @@ func TestCoreStoreValidationFieldsAndPublicFallback(t *testing.T) { func TestCoreActiveCapacityUpperBoundNamesSubmittedField(t *testing.T) { deps, fakes := sandboxFakes(t) - fakes.deployment.createRuntimeEnrollment = func(_ context.Context, capacity store.RuntimeNodeCapacity) (store.RuntimeNodeEnrollmentToken, error) { - return store.RuntimeNodeEnrollmentToken{}, storeCapacity(capacity.MaxActive) + fakes.deployment.createEnrollment = func(_ context.Context, capacity deployment.Capacity) (deployment.EnrollmentToken, error) { + return deployment.EnrollmentToken{}, nodeCapacity(capacity.MaxActive) } - fakes.deployment.updateRuntimeNode = func(_ context.Context, _ string, input store.RuntimeNodeUpdate) error { - return storeCapacity(input.MaxActive) + fakes.deployment.updateNode = func(_ context.Context, _ string, input deployment.NodeUpdate) error { + return nodeCapacity(input.MaxActive) } h := newTestHandler(t, deps) for _, tc := range []struct{ method, path, body string }{ diff --git a/services/core/internal/api/core_validation_errors.go b/services/core/internal/api/core_validation_errors.go index 03012481c..44be8bacc 100644 --- a/services/core/internal/api/core_validation_errors.go +++ b/services/core/internal/api/core_validation_errors.go @@ -6,6 +6,7 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/internal/harnessconfig/builtin" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) @@ -18,16 +19,20 @@ func writeCoreValidationError(w http.ResponseWriter, err error) bool { return false } var field *store.AdminValidationError - var configuration *store.SandboxConfigurationError + var node *deployment.NodeValidationError + var configuration *deployment.ConfigurationError switch { - case errors.As(err, &field): - details := CoreErrorDetails{"max_length": CoreErrorNumber(float64(field.MaxLength))} + case errors.As(err, &node): + details := CoreErrorDetails{"max_length": CoreErrorNumber(float64(node.MaxLength))} message := invalidNameMessage - if field.Code == "invalid_node_capacity" { + if node.Code == "invalid_node_capacity" { details = CoreErrorDetails{"min": CoreErrorNumber(1), "max": CoreErrorNumber(1000000)} message = "Node capacity must be positive, at most 1000000, and max_retained must be at least max_active." } - writeCoreError(w, http.StatusBadRequest, field.Code, message, details, field.Param) + writeCoreError(w, http.StatusBadRequest, node.Code, message, details, node.Param) + return true + case errors.As(err, &field): + writeCoreError(w, http.StatusBadRequest, field.Code, invalidNameMessage, CoreErrorDetails{"max_length": CoreErrorNumber(float64(field.MaxLength))}, field.Param) return true case errors.As(err, &configuration) && configuration.Validation != nil: field := configuration.Validation diff --git a/services/core/internal/api/dependencies.go b/services/core/internal/api/dependencies.go index 53d46e343..705be6b12 100644 --- a/services/core/internal/api/dependencies.go +++ b/services/core/internal/api/dependencies.go @@ -87,7 +87,9 @@ type Execution struct { // required. type Sandboxes struct { Deployment Deployment + NodeAllocations NodeAllocations DeploymentChanges DeploymentChanges + DeploymentReset DeploymentReset ConfigurationDiscovery ConfigurationDiscovery } @@ -148,7 +150,13 @@ func (d Dependencies) validate() error { if d.Execution == nil { return errors.New("api: Sandboxes requires Execution") } - return required(field{"Sandboxes.Deployment", s.Deployment}, field{"Sandboxes.DeploymentChanges", s.DeploymentChanges}, field{"Sandboxes.ConfigurationDiscovery", s.ConfigurationDiscovery}) + return required( + field{"Sandboxes.Deployment", s.Deployment}, + field{"Sandboxes.NodeAllocations", s.NodeAllocations}, + field{"Sandboxes.DeploymentChanges", s.DeploymentChanges}, + field{"Sandboxes.DeploymentReset", s.DeploymentReset}, + field{"Sandboxes.ConfigurationDiscovery", s.ConfigurationDiscovery}, + ) } return nil } diff --git a/services/core/internal/api/dependencies_test.go b/services/core/internal/api/dependencies_test.go index c7d2e935e..ea515c06f 100644 --- a/services/core/internal/api/dependencies_test.go +++ b/services/core/internal/api/dependencies_test.go @@ -47,7 +47,9 @@ type testFakes struct { sessionArchive *fakeSessionArchive workspaces *fakeEnvironmentWorkspaces deployment *fakeDeployment + nodeAllocations *fakeNodeAllocations deploymentChanges *fakeDeploymentChanges + deploymentReset *fakeDeploymentReset configurationDiscovery *fakeConfigurationDiscovery environmentTemplatesReader *fakeEnvironmentTemplatesReader @@ -74,7 +76,9 @@ func testDependencies(t testing.TB) (Dependencies, *testFakes) { admin: &fakeAdmin{t: t}, adminAudit: &fakeAdminAudit{t: t}, writeAudit: &fakeWriteAudit{t: t}, metrics: &fakeMetrics{t: t}, runtimeObservations: &fakeRuntimeObservations{t: t}, runtimeHistory: &fakeRuntimeHistory{t: t}, installationBindings: &fakeInstallationBindings{t: t}, admission: &fakeAdmission{t: t}, sessionArchive: &fakeSessionArchive{t: t}, workspaces: &fakeEnvironmentWorkspaces{t: t}, - deployment: &fakeDeployment{t: t}, deploymentChanges: &fakeDeploymentChanges{t: t}, configurationDiscovery: &fakeConfigurationDiscovery{t: t}, + deployment: &fakeDeployment{t: t}, nodeAllocations: &fakeNodeAllocations{t: t}, + deploymentChanges: &fakeDeploymentChanges{t: t}, deploymentReset: &fakeDeploymentReset{t: t}, + configurationDiscovery: &fakeConfigurationDiscovery{t: t}, } return Dependencies{ Engine: "codex", CoreKeys: coreKeys(t, "admin"), InstallationBindings: f.installationBindings, @@ -101,7 +105,13 @@ func (f *testFakes) execution() *Execution { // sandboxes is a Sandboxes group backed by f's strict fakes. It requires // Execution. func (f *testFakes) sandboxes() *Sandboxes { - return &Sandboxes{Deployment: f.deployment, DeploymentChanges: f.deploymentChanges, ConfigurationDiscovery: f.configurationDiscovery} + return &Sandboxes{ + Deployment: f.deployment, + NodeAllocations: f.nodeAllocations, + DeploymentChanges: f.deploymentChanges, + DeploymentReset: f.deploymentReset, + ConfigurationDiscovery: f.configurationDiscovery, + } } // coreKeys accepts each token as a Core key. diff --git a/services/core/internal/api/errors.go b/services/core/internal/api/errors.go index 83b7d2cfc..9e6170a16 100644 --- a/services/core/internal/api/errors.go +++ b/services/core/internal/api/errors.go @@ -12,8 +12,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/projects" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/textvalue" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" @@ -105,64 +103,22 @@ func writeFieldError(w http.ResponseWriter, err error) bool { } func writeStoreError(w http.ResponseWriter, r *http.Request, err error, notFoundParam ...string) { - // Adapter discovery and persisted configuration share the same public error. - var validation *sandbox.ValidationError - if errors.As(err, &validation) { - err = &store.SandboxConfigurationError{Message: validation.Message, Validation: validation} - } - if writeCoreValidationError(w, err) { - return - } - var configuration *sandbox.ConfigurationError - var unsupported *providercontract.UnsupportedError - var cursor *store.InvalidCursorError - var sandboxConfiguration *store.SandboxConfigurationError - var stale *store.SandboxGenerationStaleError - var resetRequired *store.SandboxResetRequiredError + // A reset's resource check unwraps to deployment.ErrConflict and admission + // paused by a reset is Session admission's, so both precede the deployment errors. var inUse *store.SandboxInUseError switch { - case errors.As(err, &configuration): - status := http.StatusInternalServerError - switch configuration.Class { - case sandbox.ConfigurationInvalid: - status = http.StatusBadRequest - case sandbox.ConfigurationConflict: - status = http.StatusConflict - case sandbox.ConfigurationUnconfirmed: - status = http.StatusServiceUnavailable - default: - writeError(w, status, "internal_error", "The operation could not be completed.") - return - } - if configuration.Param == "" { - writeError(w, status, configuration.Code, configuration.Message) - } else { - writeError(w, status, configuration.Code, configuration.Message, configuration.Param) - } - case errors.As(err, &unsupported): - writeError(w, http.StatusBadRequest, "sandbox_operation_unsupported", "The selected sandbox provider does not support this operation.") - case errors.Is(err, sandbox.ErrInvalid): - writeError(w, http.StatusBadRequest, "invalid_sandbox_configuration", "Invalid sandbox provider configuration.", "configuration") - case errors.As(err, &stale): - writeCoreError(w, http.StatusConflict, "sandbox_generation_stale", "The sandbox deployment generation changed. Refresh before submitting again.", CoreErrorDetails{"current_generation": CoreErrorNumber(float64(stale.CurrentGeneration))}) - case errors.As(err, &resetRequired): - writeCoreError(w, http.StatusConflict, "sandbox_reset_required", "Reset the sandbox deployment before changing this configuration.", CoreErrorDetails{"current_provider": CoreErrorString(resetRequired.CurrentProvider), "requested_provider": CoreErrorString(resetRequired.RequestedProvider)}) case errors.As(err, &inUse): writeCoreError(w, http.StatusConflict, "sandbox_in_use", "Hosted sandbox resources still belong to this deployment.", CoreErrorDetails{"allocations": CoreErrorNumber(float64(inUse.Resources.Allocations)), "pending": CoreErrorNumber(float64(inUse.Resources.Pending))}) + return case errors.Is(err, store.ErrSandboxResetAdmission): writeError(w, http.StatusServiceUnavailable, "sandbox_reset_in_progress", "A sandbox reset is in progress.") - case errors.Is(err, store.ErrSandboxResetInProgress): - writeError(w, http.StatusConflict, "sandbox_reset_in_progress", "A sandbox reset is in progress.") - case errors.Is(err, store.ErrSandboxNotConfigured): - writeError(w, http.StatusConflict, "sandbox_not_configured", "The sandbox deployment is not configured.") - case errors.As(err, &sandboxConfiguration): - writeError(w, http.StatusBadRequest, "invalid_sandbox_configuration", sandboxConfiguration.Message) - case errors.Is(err, store.ErrSandboxPublicURLUnreachable): - writeError(w, http.StatusConflict, "sandbox_configuration_error", err.Error()) - case errors.Is(err, store.ErrRuntimeNodeAddressMismatch): - writeError(w, http.StatusConflict, "sandbox_node_address_mismatch", "This node uses a different Core address than the installation public URL. Generate a new command on the Nodes page and run it on the host.") - case errors.Is(err, store.ErrRuntimeSpecificationMismatch): - writeError(w, http.StatusConflict, "sandbox_specification_mismatch", "The node resource limits or Runtime release do not match the active deployment. Restore its installed configuration or remove and enroll the node again after a drained deployment change.") + return + } + if writeSandboxError(w, err) { + return + } + var cursor *store.InvalidCursorError + switch { case errors.Is(err, projects.ErrArchived): // Executor credential management checks the Project in its own // transaction. @@ -171,21 +127,6 @@ func writeStoreError(w http.ResponseWriter, r *http.Request, err error, notFound writeError(w, http.StatusUnauthorized, "installation_authorization_invalid", store.ErrInstallationAuthorization.Error()) case errors.Is(err, store.ErrExecutorCredentialExists): writeError(w, http.StatusConflict, "executor_credential_exists", "This executor key ID already exists. Explicitly rotate it to replace the secret.") - case errors.Is(err, store.ErrSandboxCredentialUnavailable): - writeError(w, http.StatusServiceUnavailable, "sandbox_credential_unavailable", "Sandbox credentials are unavailable. Check the service credential encryption configuration.") - case errors.Is(err, store.ErrSandboxDeploymentConflict): - writeError(w, http.StatusConflict, "sandbox_deployment_conflict", "The sandbox deployment cannot change in its current state. Refresh the configuration and inspect its reset and resource state.") - case errors.Is(err, store.ErrRuntimeNodeCredential): - writeError(w, http.StatusUnauthorized, "invalid_node_credential", "A valid sandbox node enrollment or node credential is required.") - case errors.Is(err, store.ErrRuntimeNodeInUse): - writeError(w, http.StatusConflict, "runtime_node_in_use", "The sandbox node retains allocations, snapshots, reservations or pending cleanup.") - case errors.Is(err, store.ErrRuntimeLocalNodeConfigured): - writeError(w, http.StatusConflict, "runtime_local_node_configured", "The local sandbox node is enabled in deployment configuration. Drain it with the previous release and remove its file-managed configuration before replacing it.") - case errors.Is(err, store.ErrSandboxNodesPreparing): - writeError(w, http.StatusServiceUnavailable, "sandbox_nodes_preparing", "Sandbox nodes are preparing the requested Runtime.") - case errors.Is(err, store.ErrRuntimeNodeUnavailable): - writeError(w, http.StatusServiceUnavailable, "runtime_node_unavailable", "The selected sandbox node is unavailable or has no capacity.") - case errors.Is(err, execution.ErrModelProviderRequired): writeError(w, http.StatusBadRequest, "model_provider_required", "This Session was created without a model provider and cannot run. Create a new Session with x_agents_core.model_provider or an Agent that has one saved.") case errors.Is(err, store.ErrHostedEnvironmentFailed): diff --git a/services/core/internal/api/errors_deployment.go b/services/core/internal/api/errors_deployment.go new file mode 100644 index 000000000..62b78dff7 --- /dev/null +++ b/services/core/internal/api/errors_deployment.go @@ -0,0 +1,108 @@ +package api + +import ( + "errors" + "net/http" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// writeDeploymentError reports an error of the sandbox deployment, its nodes +// or the execution owner that changes them. +func writeDeploymentError(w http.ResponseWriter, r *http.Request, err error) { + if writeSandboxError(w, err) { + return + } + switch { + case errors.Is(err, execution.ErrExecutionUnavailable): + writeError(w, http.StatusServiceUnavailable, "execution_unavailable", "Execution is not available on this service.") + case errors.Is(err, deployment.ErrNotFound): + writeError(w, http.StatusNotFound, "not_found_error", "Resource not found.") + case errors.Is(err, deployment.ErrInvalidInput): + writeError(w, http.StatusBadRequest, "invalid_request", invalidInputMessage) + default: + if writeAuditSourceError(w, r, err) || writeTextValueError(w, r, err) || writeCredentialUnavailableError(w, r, err) { + return + } + writeInternalError(w, r) + } +} + +// writeSandboxError reports a sandbox deployment, node or Sandbox Provider +// error and returns false for any other error. Session and Environment +// admission report the same errors through writeStoreError. +func writeSandboxError(w http.ResponseWriter, err error) bool { + // Adapter discovery and persisted configuration share the same public error. + var validation *sandbox.ValidationError + if errors.As(err, &validation) { + err = &deployment.ConfigurationError{Message: validation.Message, Validation: validation} + } + if writeCoreValidationError(w, err) { + return true + } + var configuration *sandbox.ConfigurationError + var unsupported *providercontract.UnsupportedError + var deploymentConfiguration *deployment.ConfigurationError + var stale *deployment.GenerationStaleError + var resetRequired *deployment.ResetRequiredError + switch { + case errors.As(err, &configuration): + status := http.StatusInternalServerError + switch configuration.Class { + case sandbox.ConfigurationInvalid: + status = http.StatusBadRequest + case sandbox.ConfigurationConflict: + status = http.StatusConflict + case sandbox.ConfigurationUnconfirmed: + status = http.StatusServiceUnavailable + default: + writeError(w, status, "internal_error", "The operation could not be completed.") + return true + } + if configuration.Param == "" { + writeError(w, status, configuration.Code, configuration.Message) + } else { + writeError(w, status, configuration.Code, configuration.Message, configuration.Param) + } + case errors.As(err, &unsupported): + writeError(w, http.StatusBadRequest, "sandbox_operation_unsupported", "The selected sandbox provider does not support this operation.") + case errors.Is(err, sandbox.ErrInvalid): + writeError(w, http.StatusBadRequest, "invalid_sandbox_configuration", "Invalid sandbox provider configuration.", "configuration") + case errors.As(err, &stale): + writeCoreError(w, http.StatusConflict, "sandbox_generation_stale", "The sandbox deployment generation changed. Refresh before submitting again.", CoreErrorDetails{"current_generation": CoreErrorNumber(float64(stale.CurrentGeneration))}) + case errors.As(err, &resetRequired): + writeCoreError(w, http.StatusConflict, "sandbox_reset_required", "Reset the sandbox deployment before changing this configuration.", CoreErrorDetails{"current_provider": CoreErrorString(resetRequired.CurrentProvider), "requested_provider": CoreErrorString(resetRequired.RequestedProvider)}) + case errors.Is(err, deployment.ErrResetInProgress): + writeError(w, http.StatusConflict, "sandbox_reset_in_progress", "A sandbox reset is in progress.") + case errors.Is(err, deployment.ErrNotConfigured): + writeError(w, http.StatusConflict, "sandbox_not_configured", "The sandbox deployment is not configured.") + case errors.As(err, &deploymentConfiguration): + writeError(w, http.StatusBadRequest, "invalid_sandbox_configuration", deploymentConfiguration.Message) + case errors.Is(err, deployment.ErrPublicURLUnreachable): + writeError(w, http.StatusConflict, "sandbox_configuration_error", err.Error()) + case errors.Is(err, deployment.ErrNodeAddressMismatch): + writeError(w, http.StatusConflict, "sandbox_node_address_mismatch", "This node uses a different Core address than the installation public URL. Generate a new command on the Nodes page and run it on the host.") + case errors.Is(err, deployment.ErrSpecificationMismatch): + writeError(w, http.StatusConflict, "sandbox_specification_mismatch", "The node resource limits or Runtime release do not match the active deployment. Restore its installed configuration or remove and enroll the node again after a drained deployment change.") + case errors.Is(err, deployment.ErrNodeExists): + writeError(w, http.StatusConflict, "idempotency_conflict", "This idempotency key was used with different input.") + case errors.Is(err, deployment.ErrConflict): + writeError(w, http.StatusConflict, "sandbox_deployment_conflict", "The sandbox deployment cannot change in its current state. Refresh the configuration and inspect its reset and resource state.") + case errors.Is(err, deployment.ErrNodeCredential): + writeError(w, http.StatusUnauthorized, "invalid_node_credential", "A valid sandbox node enrollment or node credential is required.") + case errors.Is(err, deployment.ErrNodeInUse): + writeError(w, http.StatusConflict, "runtime_node_in_use", "The sandbox node retains allocations, snapshots, reservations or pending cleanup.") + case errors.Is(err, deployment.ErrLocalNodeConfigured): + writeError(w, http.StatusConflict, "runtime_local_node_configured", "The local sandbox node is enabled in deployment configuration. Drain it with the previous release and remove its file-managed configuration before replacing it.") + case errors.Is(err, deployment.ErrNodesPreparing): + writeError(w, http.StatusServiceUnavailable, "sandbox_nodes_preparing", "Sandbox nodes are preparing the requested Runtime.") + case errors.Is(err, deployment.ErrNodeUnavailable): + writeError(w, http.StatusServiceUnavailable, "runtime_node_unavailable", "The selected sandbox node is unavailable or has no capacity.") + default: + return false + } + return true +} diff --git a/services/core/internal/api/errors_test.go b/services/core/internal/api/errors_test.go index ef6fefee7..08a68ba2a 100644 --- a/services/core/internal/api/errors_test.go +++ b/services/core/internal/api/errors_test.go @@ -12,6 +12,7 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/files" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" @@ -132,10 +133,10 @@ func TestSessionDeletionConflictError(t *testing.T) { // turn_conflict because the official behavior there is unobserved. func TestConflictErrorsUseConflictType(t *testing.T) { for err, code := range map[error]string{ - store.ErrSandboxDeploymentConflict: "sandbox_deployment_conflict", - store.ErrRuntimeNodeInUse: "runtime_node_in_use", - store.ErrRuntimeLocalNodeConfigured: "runtime_local_node_configured", - store.ErrRuntimeNodeAddressMismatch: "sandbox_node_address_mismatch", + deployment.ErrConflict: "sandbox_deployment_conflict", + deployment.ErrNodeInUse: "runtime_node_in_use", + deployment.ErrLocalNodeConfigured: "runtime_local_node_configured", + deployment.ErrNodeAddressMismatch: "sandbox_node_address_mismatch", store.ErrEnvironmentUnavailable: "environment_unavailable", execution.ErrEnvironmentInputExpired: "environment_input_expired", execution.ErrEnvironmentInputCancelled: "environment_input_cancelled", diff --git a/services/core/internal/api/fakes_test.go b/services/core/internal/api/fakes_test.go index 52c83303a..d10ffce34 100644 --- a/services/core/internal/api/fakes_test.go +++ b/services/core/internal/api/fakes_test.go @@ -12,6 +12,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/agents" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmenttemplates" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/files" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" @@ -205,119 +206,135 @@ func (f *fakeConfigurationDiscovery) DiscoverConfiguration(a0 context.Context, a } type fakeDeployment struct { - t testing.TB - getRuntimeDeployment func(context.Context) (store.RuntimeDeploymentView, error) - listRuntimeNodes func(context.Context) ([]store.RuntimeNode, error) - getRuntimeNodeDetail func(context.Context, string, string) (store.RuntimeNodeDetail, error) - updateRuntimeNode func(context.Context, string, store.RuntimeNodeUpdate) error - removeRuntimeNode func(context.Context, string) error - listNodeRuntimeAllocations func(context.Context, string) ([]store.RuntimeNodeAllocation, error) - createRuntimeEnrollment func(context.Context, store.RuntimeNodeCapacity) (store.RuntimeNodeEnrollmentToken, error) - enrollRuntimeNode func(context.Context, string, store.RuntimeNodeEnrollment) (store.RuntimeNodeIdentity, error) - runtimeNodeGenerationConfiguration func(context.Context, string, string, uint64) (store.RuntimeNodeConfiguration, error) - runtimeNodeStatus func(context.Context, string, string) (store.RuntimeNodeStatus, error) + t testing.TB + view func(context.Context) (deployment.View, error) + listNodes func(context.Context) ([]deployment.Node, error) + nodeDetail func(context.Context, string, string) (deployment.NodeDetail, error) + updateNode func(context.Context, string, deployment.NodeUpdate) error + removeNode func(context.Context, string) error + createEnrollment func(context.Context, deployment.Capacity) (deployment.EnrollmentToken, error) + enroll func(context.Context, string, deployment.Enrollment) (deployment.NodeIdentity, error) + nodeConfiguration func(context.Context, string, string, uint64) (deployment.NodeConfiguration, error) + nodeStatus func(context.Context, string, string) (deployment.NodeStatus, error) + decodeConfiguration func(string, json.RawMessage, json.RawMessage) (sandbox.Configuration, error) } -func (f *fakeDeployment) GetRuntimeDeployment(a0 context.Context) (store.RuntimeDeploymentView, error) { - if f.getRuntimeDeployment == nil { - unexpectedCall(f.t, "GetRuntimeDeployment") +func (f *fakeDeployment) View(a0 context.Context) (deployment.View, error) { + if f.view == nil { + unexpectedCall(f.t, "View") } - return f.getRuntimeDeployment(a0) + return f.view(a0) } -func (f *fakeDeployment) ListRuntimeNodes(a0 context.Context) ([]store.RuntimeNode, error) { - if f.listRuntimeNodes == nil { - unexpectedCall(f.t, "ListRuntimeNodes") +func (f *fakeDeployment) ListNodes(a0 context.Context) ([]deployment.Node, error) { + if f.listNodes == nil { + unexpectedCall(f.t, "ListNodes") } - return f.listRuntimeNodes(a0) + return f.listNodes(a0) } -func (f *fakeDeployment) GetRuntimeNodeDetail(a0 context.Context, a1 string, a2 string) (store.RuntimeNodeDetail, error) { - if f.getRuntimeNodeDetail == nil { - unexpectedCall(f.t, "GetRuntimeNodeDetail") +func (f *fakeDeployment) NodeDetail(a0 context.Context, a1 string, a2 string) (deployment.NodeDetail, error) { + if f.nodeDetail == nil { + unexpectedCall(f.t, "NodeDetail") } - return f.getRuntimeNodeDetail(a0, a1, a2) + return f.nodeDetail(a0, a1, a2) } -func (f *fakeDeployment) UpdateRuntimeNode(a0 context.Context, a1 string, a2 store.RuntimeNodeUpdate) error { - if f.updateRuntimeNode == nil { - unexpectedCall(f.t, "UpdateRuntimeNode") +func (f *fakeDeployment) UpdateNode(a0 context.Context, a1 string, a2 deployment.NodeUpdate) error { + if f.updateNode == nil { + unexpectedCall(f.t, "UpdateNode") } - return f.updateRuntimeNode(a0, a1, a2) + return f.updateNode(a0, a1, a2) } -func (f *fakeDeployment) RemoveRuntimeNode(a0 context.Context, a1 string) error { - if f.removeRuntimeNode == nil { - unexpectedCall(f.t, "RemoveRuntimeNode") +func (f *fakeDeployment) RemoveNode(a0 context.Context, a1 string) error { + if f.removeNode == nil { + unexpectedCall(f.t, "RemoveNode") } - return f.removeRuntimeNode(a0, a1) + return f.removeNode(a0, a1) } -func (f *fakeDeployment) ListNodeRuntimeAllocations(a0 context.Context, a1 string) ([]store.RuntimeNodeAllocation, error) { - if f.listNodeRuntimeAllocations == nil { - unexpectedCall(f.t, "ListNodeRuntimeAllocations") +func (f *fakeDeployment) CreateEnrollment(a0 context.Context, a1 deployment.Capacity) (deployment.EnrollmentToken, error) { + if f.createEnrollment == nil { + unexpectedCall(f.t, "CreateEnrollment") } - return f.listNodeRuntimeAllocations(a0, a1) + return f.createEnrollment(a0, a1) } -func (f *fakeDeployment) CreateRuntimeEnrollment(a0 context.Context, a1 store.RuntimeNodeCapacity) (store.RuntimeNodeEnrollmentToken, error) { - if f.createRuntimeEnrollment == nil { - unexpectedCall(f.t, "CreateRuntimeEnrollment") +func (f *fakeDeployment) Enroll(a0 context.Context, a1 string, a2 deployment.Enrollment) (deployment.NodeIdentity, error) { + if f.enroll == nil { + unexpectedCall(f.t, "Enroll") } - return f.createRuntimeEnrollment(a0, a1) + return f.enroll(a0, a1, a2) } -func (f *fakeDeployment) EnrollRuntimeNode(a0 context.Context, a1 string, a2 store.RuntimeNodeEnrollment) (store.RuntimeNodeIdentity, error) { - if f.enrollRuntimeNode == nil { - unexpectedCall(f.t, "EnrollRuntimeNode") +func (f *fakeDeployment) NodeConfiguration(a0 context.Context, a1 string, a2 string, a3 uint64) (deployment.NodeConfiguration, error) { + if f.nodeConfiguration == nil { + unexpectedCall(f.t, "NodeConfiguration") } - return f.enrollRuntimeNode(a0, a1, a2) + return f.nodeConfiguration(a0, a1, a2, a3) } -func (f *fakeDeployment) RuntimeNodeGenerationConfiguration(a0 context.Context, a1 string, a2 string, a3 uint64) (store.RuntimeNodeConfiguration, error) { - if f.runtimeNodeGenerationConfiguration == nil { - unexpectedCall(f.t, "RuntimeNodeGenerationConfiguration") +func (f *fakeDeployment) NodeStatus(a0 context.Context, a1 string, a2 string) (deployment.NodeStatus, error) { + if f.nodeStatus == nil { + unexpectedCall(f.t, "NodeStatus") } - return f.runtimeNodeGenerationConfiguration(a0, a1, a2, a3) + return f.nodeStatus(a0, a1, a2) } -func (f *fakeDeployment) RuntimeNodeStatus(a0 context.Context, a1 string, a2 string) (store.RuntimeNodeStatus, error) { - if f.runtimeNodeStatus == nil { - unexpectedCall(f.t, "RuntimeNodeStatus") +func (f *fakeDeployment) DecodeConfiguration(a0 string, a1 json.RawMessage, a2 json.RawMessage) (sandbox.Configuration, error) { + if f.decodeConfiguration == nil { + unexpectedCall(f.t, "DecodeConfiguration") } - return f.runtimeNodeStatus(a0, a1, a2) + return f.decodeConfiguration(a0, a1, a2) +} + +type fakeNodeAllocations struct { + t testing.TB + listNodeRuntimeAllocations func(context.Context, string) ([]store.RuntimeNodeAllocation, error) +} + +func (f *fakeNodeAllocations) ListNodeRuntimeAllocations(a0 context.Context, a1 string) ([]store.RuntimeNodeAllocation, error) { + if f.listNodeRuntimeAllocations == nil { + unexpectedCall(f.t, "ListNodeRuntimeAllocations") + } + return f.listNodeRuntimeAllocations(a0, a1) } type fakeDeploymentChanges struct { t testing.TB - initializeSandboxDeployment func(context.Context, store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error) - updateSandboxDeployment func(context.Context, store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) - startSandboxReset func(context.Context, store.SandboxResetRequest) (store.RuntimeDeploymentView, error) - cancelSandboxReset func(context.Context, uint64) (store.RuntimeDeploymentView, error) + initializeSandboxDeployment func(context.Context, sandbox.Selection) (deployment.View, error) + updateSandboxDeployment func(context.Context, sandbox.Selection) (deployment.View, error) } -func (f *fakeDeploymentChanges) InitializeSandboxDeployment(a0 context.Context, a1 store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error) { +func (f *fakeDeploymentChanges) InitializeSandboxDeployment(a0 context.Context, a1 sandbox.Selection) (deployment.View, error) { if f.initializeSandboxDeployment == nil { unexpectedCall(f.t, "InitializeSandboxDeployment") } return f.initializeSandboxDeployment(a0, a1) } -func (f *fakeDeploymentChanges) UpdateSandboxDeployment(a0 context.Context, a1 store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) { +func (f *fakeDeploymentChanges) UpdateSandboxDeployment(a0 context.Context, a1 sandbox.Selection) (deployment.View, error) { if f.updateSandboxDeployment == nil { unexpectedCall(f.t, "UpdateSandboxDeployment") } return f.updateSandboxDeployment(a0, a1) } -func (f *fakeDeploymentChanges) StartSandboxReset(a0 context.Context, a1 store.SandboxResetRequest) (store.RuntimeDeploymentView, error) { +type fakeDeploymentReset struct { + t testing.TB + startSandboxReset func(context.Context, store.SandboxResetRequest) (deployment.View, error) + cancelSandboxReset func(context.Context, uint64) (deployment.View, error) +} + +func (f *fakeDeploymentReset) StartSandboxReset(a0 context.Context, a1 store.SandboxResetRequest) (deployment.View, error) { if f.startSandboxReset == nil { unexpectedCall(f.t, "StartSandboxReset") } return f.startSandboxReset(a0, a1) } -func (f *fakeDeploymentChanges) CancelSandboxReset(a0 context.Context, a1 uint64) (store.RuntimeDeploymentView, error) { +func (f *fakeDeploymentReset) CancelSandboxReset(a0 context.Context, a1 uint64) (deployment.View, error) { if f.cancelSandboxReset == nil { unexpectedCall(f.t, "CancelSandboxReset") } diff --git a/services/core/internal/api/installation_test.go b/services/core/internal/api/installation_test.go index e9b18bb12..37bc7f638 100644 --- a/services/core/internal/api/installation_test.go +++ b/services/core/internal/api/installation_test.go @@ -8,6 +8,9 @@ import ( "strings" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) @@ -63,15 +66,16 @@ func TestInstallationSnapshotCannotCarryASensitiveValue(t *testing.T) { func TestDeploymentAddressIsNotInput(t *testing.T) { deps, fakes := sandboxFakes(t) initializations := 0 - initialize := func(_ context.Context, input store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error) { + initialize := func(_ context.Context, input sandbox.Selection) (deployment.View, error) { initializations++ if input.Provider != "e2b" || input.ExpectedGeneration != 0 { t.Fatal("invalid selection reached initialization", input.Provider, input.ExpectedGeneration) } - return store.RuntimeDeploymentView{}, store.ErrSandboxPublicURLUnreachable + return deployment.View{}, deployment.ErrPublicURLUnreachable } // The strict fake fails the test if core_url reaches the update. fakes.deploymentChanges.initializeSandboxDeployment = initialize + fakes.deployment.decodeConfiguration = providers.Builtin().DecodeInput h := newTestHandler(t, deps) for _, test := range []struct { method, body, code string diff --git a/services/core/internal/api/sandbox_configuration_discovery.go b/services/core/internal/api/sandbox_configuration_discovery.go index ce69c15c6..95a4f3e59 100644 --- a/services/core/internal/api/sandbox_configuration_discovery.go +++ b/services/core/internal/api/sandbox_configuration_discovery.go @@ -6,8 +6,8 @@ import ( "net/http" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/go-chi/chi/v5" ) @@ -35,14 +35,14 @@ func (h *Handler) discoverSandboxConfiguration(w http.ResponseWriter, r *http.Re } var input sandbox.ConfigurationDiscoveryInput if decodeInputObject(raw, &input, "configuration", "credential", "query") != nil { - writeStoreError(w, r, store.ErrInvalidInput) + writeDeploymentError(w, r, deployment.ErrInvalidInput) return } ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) defer cancel() result, err := h.Sandboxes.ConfigurationDiscovery.DiscoverConfiguration(ctx, chi.URLParam(r, "provider"), input) if err != nil { - writeStoreError(w, r, err) + writeDeploymentError(w, r, err) return } writeJSON(w, http.StatusOK, result) diff --git a/services/core/internal/api/sandbox_deployment_changes_test.go b/services/core/internal/api/sandbox_deployment_changes_test.go index b1a2fe445..aee58af40 100644 --- a/services/core/internal/api/sandbox_deployment_changes_test.go +++ b/services/core/internal/api/sandbox_deployment_changes_test.go @@ -7,31 +7,34 @@ import ( "strings" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" - + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) func TestSandboxDeploymentChangesAuthenticateAndDecode(t *testing.T) { deps, fakes := sandboxFakes(t) updates, resets := 0, 0 - update := func(_ context.Context, in store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) { + update := func(_ context.Context, in sandbox.Selection) (deployment.View, error) { updates++ if in.Provider != "e2b" || in.ExpectedGeneration != 2 || in.Configuration == nil || in.Configuration.(*e2b.DeploymentConfiguration).APIKey != "synthetic-private-key" { t.Fatal("write-only fields were lost") } - return store.RuntimeDeploymentView{Provider: in.Provider}, nil + return deployment.View{Provider: in.Provider}, nil } - maintain := func(_ context.Context, in store.SandboxResetRequest) (store.RuntimeDeploymentView, error) { + maintain := func(_ context.Context, in store.SandboxResetRequest) (deployment.View, error) { resets++ if in.ExpectedGeneration != 2 { t.Fatal("generation was lost") } - return store.RuntimeDeploymentView{Reset: &store.SandboxResetView{Clear: in.Clear}}, nil + return deployment.View{Reset: &deployment.Reset{Clear: in.Clear}}, nil } - fakes.deploymentChanges.updateSandboxDeployment, fakes.deploymentChanges.startSandboxReset = update, maintain - fakes.deploymentChanges.cancelSandboxReset = func(context.Context, uint64) (store.RuntimeDeploymentView, error) { - return store.RuntimeDeploymentView{}, nil + fakes.deploymentChanges.updateSandboxDeployment, fakes.deploymentReset.startSandboxReset = update, maintain + fakes.deployment.decodeConfiguration = providers.Builtin().DecodeInput + fakes.deploymentReset.cancelSandboxReset = func(context.Context, uint64) (deployment.View, error) { + return deployment.View{}, nil } h := newTestHandler(t, deps) const selection = `{"provider":"e2b","expected_generation":2,"credential":{"api_key":"synthetic-private-key"},"configuration":{"template":"qualified:build"}}` @@ -77,11 +80,11 @@ func TestSandboxMutationErrorsExposeOnlyTypedCoreFacts(t *testing.T) { status int details string }{ - {&store.SandboxGenerationStaleError{CurrentGeneration: 8}, "sandbox_generation_stale", 409, `"current_generation":8`}, - {&store.SandboxResetRequiredError{CurrentProvider: "docker", RequestedProvider: "e2b"}, "sandbox_reset_required", 409, `"requested_provider":"e2b"`}, - {&store.SandboxResetRequiredError{CurrentProvider: "e2b", RequestedProvider: "e2b"}, "sandbox_reset_required", 409, `"current_provider":"e2b"`}, - {&store.SandboxInUseError{Resources: store.SandboxDeploymentResources{Allocations: 2, Pending: 1}}, "sandbox_in_use", 409, `"allocations":2`}, - {store.ErrSandboxResetInProgress, "sandbox_reset_in_progress", 409, ""}, + {&deployment.GenerationStaleError{CurrentGeneration: 8}, "sandbox_generation_stale", 409, `"current_generation":8`}, + {&deployment.ResetRequiredError{CurrentProvider: "docker", RequestedProvider: "e2b"}, "sandbox_reset_required", 409, `"requested_provider":"e2b"`}, + {&deployment.ResetRequiredError{CurrentProvider: "e2b", RequestedProvider: "e2b"}, "sandbox_reset_required", 409, `"current_provider":"e2b"`}, + {&store.SandboxInUseError{Resources: deployment.Resources{Allocations: 2, Pending: 1}}, "sandbox_in_use", 409, `"allocations":2`}, + {deployment.ErrResetInProgress, "sandbox_reset_in_progress", 409, ""}, {store.ErrSandboxResetAdmission, "sandbox_reset_in_progress", 503, ""}, } { for _, core := range []bool{false, true} { diff --git a/services/core/internal/api/sandbox_deployment_setup.go b/services/core/internal/api/sandbox_deployment_setup.go index 15ed035d5..c75146109 100644 --- a/services/core/internal/api/sandbox_deployment_setup.go +++ b/services/core/internal/api/sandbox_deployment_setup.go @@ -7,8 +7,7 @@ import ( "net/url" "strconv" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" - + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) @@ -28,21 +27,27 @@ type SandboxDeploymentChangeInput struct { SandboxDeploymentInput } -func (v SandboxDeploymentInput) request() (store.SandboxDeploymentSetupRequest, error) { - c, err := providers.DecodeInput(v.Provider, v.Configuration, v.Credential) +// selection decodes the submitted configuration with the provider's declared codec. +func (h *Handler) selection(v SandboxDeploymentInput) (sandbox.Selection, error) { + c, err := h.Sandboxes.Deployment.DecodeConfiguration(v.Provider, v.Configuration, v.Credential) if err != nil { - return store.SandboxDeploymentSetupRequest{}, err + return sandbox.Selection{}, err } - return store.SandboxDeploymentSetupRequest{ExpectedGeneration: *v.ExpectedGeneration, Provider: v.Provider, DeploymentSpec: sandbox.DeploymentSpec{Resources: v.Resources, Runtime: v.Runtime}, Configuration: c}, nil + return sandbox.Selection{ExpectedGeneration: *v.ExpectedGeneration, Provider: v.Provider, DeploymentSpec: sandbox.DeploymentSpec{Resources: v.Resources, Runtime: v.Runtime}, Configuration: c}, nil } -// DeploymentChanges sets up, updates and resets the sandbox deployment through -// the execution owner. +// DeploymentChanges sets up and updates the sandbox deployment through the +// execution owner. type DeploymentChanges interface { - InitializeSandboxDeployment(context.Context, store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error) - UpdateSandboxDeployment(context.Context, store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) - StartSandboxReset(context.Context, store.SandboxResetRequest) (store.RuntimeDeploymentView, error) - CancelSandboxReset(context.Context, uint64) (store.RuntimeDeploymentView, error) + InitializeSandboxDeployment(ctx context.Context, input sandbox.Selection) (deployment.View, error) + UpdateSandboxDeployment(ctx context.Context, input sandbox.Selection) (deployment.View, error) +} + +// DeploymentReset starts and cancels a sandbox deployment reset through the +// execution owner. +type DeploymentReset interface { + StartSandboxReset(ctx context.Context, input store.SandboxResetRequest) (deployment.View, error) + CancelSandboxReset(ctx context.Context, generation uint64) (deployment.View, error) } // @Summary Initialize the deployment sandbox provider @@ -52,7 +57,7 @@ type DeploymentChanges interface { // @Security DeploymentAdminAuth // @Accept json // @Param body body api.SandboxDeploymentInput true "Deployment selection" -// @Success 200 {object} store.RuntimeDeploymentView +// @Success 200 {object} deployment.View // @Failure 400,401,409,500,503 {object} CoreErrorResponse // @Router /core/v1/sandbox/deployment [post] func (h *Handler) initializeSandboxDeployment(w http.ResponseWriter, r *http.Request) { @@ -62,17 +67,17 @@ func (h *Handler) initializeSandboxDeployment(w http.ResponseWriter, r *http.Req } var input SandboxDeploymentInput if decodeInputObject(raw, &input, "provider", "configuration", "credential", "resources", "runtime", "expected_generation") != nil || input.ExpectedGeneration == nil { - writeStoreError(w, r, store.ErrInvalidInput) + writeDeploymentError(w, r, deployment.ErrInvalidInput) return } - selection, err := input.request() + selection, err := h.selection(input) if err != nil { - writeStoreError(w, r, err) + writeDeploymentError(w, r, err) return } result, err := h.Sandboxes.DeploymentChanges.InitializeSandboxDeployment(r.Context(), selection) if err != nil { - writeStoreError(w, r, err) + writeDeploymentError(w, r, err) return } writeJSON(w, http.StatusOK, result) @@ -85,7 +90,7 @@ func (h *Handler) initializeSandboxDeployment(w http.ResponseWriter, r *http.Req // @Security DeploymentAdminAuth // @Accept json // @Param body body api.SandboxDeploymentChangeInput true "Replacement deployment selection" -// @Success 200 {object} store.RuntimeDeploymentView +// @Success 200 {object} deployment.View // @Failure 400,401,409,500,503 {object} CoreErrorResponse // @Router /core/v1/sandbox/deployment [put] func (h *Handler) updateSandboxDeployment(w http.ResponseWriter, r *http.Request) { @@ -95,30 +100,30 @@ func (h *Handler) updateSandboxDeployment(w http.ResponseWriter, r *http.Request } var input SandboxDeploymentChangeInput if decodeInputObject(raw, &input, "provider", "configuration", "credential", "resources", "runtime", "expected_generation") != nil || input.ExpectedGeneration == nil { - writeStoreError(w, r, store.ErrInvalidInput) + writeDeploymentError(w, r, deployment.ErrInvalidInput) return } - selection, err := input.request() + selection, err := h.selection(input.SandboxDeploymentInput) if err != nil { - writeStoreError(w, r, err) + writeDeploymentError(w, r, err) return } - result, err := h.Sandboxes.DeploymentChanges.UpdateSandboxDeployment(r.Context(), store.SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: selection, ExpectedGeneration: *input.ExpectedGeneration}) + result, err := h.Sandboxes.DeploymentChanges.UpdateSandboxDeployment(r.Context(), selection) if err != nil { - writeStoreError(w, r, err) + writeDeploymentError(w, r, err) return } writeJSON(w, http.StatusOK, result) } // @Summary Start or escalate a durable sandbox deployment reset -// @Description Archives hosted Sessions and waits for confirmed provider cleanup, preserving history and Files/Artifacts. Auto waits for started or waiting Turns and file writes until the durable deadline; force cancels them. The same clear is idempotent; force escalates auto. Requires the current generation. Self-hosted Sessions are unchanged. +// @Description Archives hosted Sessions and waits for confirmed provider cleanup, preserving history and Files/Artifacts. Auto waits for started or waiting Turns and file writes until the durable deadline; force cancels them. The same clear is idempotent; force escalates auto. Requires the current generation. Self-hosted Sessions are unchanged. The response is the current deployment read after the reset commits; resource counts are live and may already differ. // @Tags Sandbox Manager // @Produce json // @Security DeploymentAdminAuth // @Accept json // @Param body body store.SandboxResetRequest true "Reset mode and current deployment generation" -// @Success 200 {object} store.RuntimeDeploymentView +// @Success 200 {object} deployment.View // @Failure 400,401,409,500,503 {object} CoreErrorResponse // @Router /core/v1/sandbox/deployment/reset [post] func (h *Handler) startSandboxReset(w http.ResponseWriter, r *http.Request) { @@ -150,7 +155,7 @@ func (h *Handler) startSandboxReset(w http.ResponseWriter, r *http.Request) { return } setAdminAuditSource(r, "") - result, err := h.Sandboxes.DeploymentChanges.StartSandboxReset(r.Context(), store.SandboxResetRequest{ExpectedGeneration: *input.ExpectedGeneration, Clear: input.Clear, DeadlineSeconds: input.DeadlineSeconds}) + result, err := h.Sandboxes.DeploymentReset.StartSandboxReset(r.Context(), store.SandboxResetRequest{ExpectedGeneration: *input.ExpectedGeneration, Clear: input.Clear, DeadlineSeconds: input.DeadlineSeconds}) if err != nil { writeStoreError(w, r, err) return @@ -159,12 +164,12 @@ func (h *Handler) startSandboxReset(w http.ResponseWriter, r *http.Request) { } // @Summary Cancel a sandbox deployment reset -// @Description Restores admission but never restores Sessions already archived. With no reset running this is an idempotent read, provided the generation still matches. +// @Description Restores admission but never restores Sessions already archived. With no reset running this is an idempotent read, provided the generation still matches. The response is the current deployment read after the cancellation commits. // @Tags Sandbox Manager // @Produce json // @Security DeploymentAdminAuth // @Param expected_generation query integer true "Current deployment generation" -// @Success 200 {object} store.RuntimeDeploymentView +// @Success 200 {object} deployment.View // @Failure 400,401,409,500,503 {object} CoreErrorResponse // @Router /core/v1/sandbox/deployment/reset [delete] func (h *Handler) cancelSandboxReset(w http.ResponseWriter, r *http.Request) { @@ -174,7 +179,7 @@ func (h *Handler) cancelSandboxReset(w http.ResponseWriter, r *http.Request) { return } setAdminAuditSource(r, "") - result, err := h.Sandboxes.DeploymentChanges.CancelSandboxReset(r.Context(), query) + result, err := h.Sandboxes.DeploymentReset.CancelSandboxReset(r.Context(), query) if err != nil { writeStoreError(w, r, err) return diff --git a/services/core/internal/api/sandbox_deployment_setup_test.go b/services/core/internal/api/sandbox_deployment_setup_test.go index 60cf5261f..c3106ad62 100644 --- a/services/core/internal/api/sandbox_deployment_setup_test.go +++ b/services/core/internal/api/sandbox_deployment_setup_test.go @@ -7,20 +7,23 @@ import ( "strings" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" ) func TestSandboxDeploymentSetupRequiresAdministratorAndStrictBody(t *testing.T) { deps, fakes := sandboxFakes(t) calls := 0 - initialize := func(_ context.Context, input store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error) { + initialize := func(_ context.Context, input sandbox.Selection) (deployment.View, error) { calls++ if input.Provider == "microsandbox" { - return store.RuntimeDeploymentView{}, store.ErrSandboxDeploymentConflict + return deployment.View{}, deployment.ErrConflict } - return store.RuntimeDeploymentView{Provider: input.Provider}, nil + return deployment.View{Provider: input.Provider}, nil } fakes.deploymentChanges.initializeSandboxDeployment = initialize + fakes.deployment.decodeConfiguration = providers.Builtin().DecodeInput h := newTestHandler(t, deps) for _, test := range []struct { token, body string diff --git a/services/core/internal/api/sandbox_manager.go b/services/core/internal/api/sandbox_manager.go index 259fe9bfa..4d008d197 100644 --- a/services/core/internal/api/sandbox_manager.go +++ b/services/core/internal/api/sandbox_manager.go @@ -2,15 +2,18 @@ package api import ( "context" + "encoding/json" "net/http" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/go-chi/chi/v5" ) type SandboxNodeList struct { - Data []store.RuntimeNode `json:"data"` + Data []deployment.Node `json:"data"` } type SandboxAllocationList struct { Data []store.RuntimeNodeAllocation `json:"data"` @@ -35,16 +38,23 @@ type SandboxEnrollmentTokenRequest struct { // Deployment reads the sandbox deployment and manages its nodes: enrollment, // identity, generation configuration, capacity and removal. type Deployment interface { - GetRuntimeDeployment(context.Context) (store.RuntimeDeploymentView, error) - ListRuntimeNodes(context.Context) ([]store.RuntimeNode, error) - GetRuntimeNodeDetail(context.Context, string, string) (store.RuntimeNodeDetail, error) - UpdateRuntimeNode(context.Context, string, store.RuntimeNodeUpdate) error - RemoveRuntimeNode(context.Context, string) error - ListNodeRuntimeAllocations(context.Context, string) ([]store.RuntimeNodeAllocation, error) - CreateRuntimeEnrollment(context.Context, store.RuntimeNodeCapacity) (store.RuntimeNodeEnrollmentToken, error) - EnrollRuntimeNode(context.Context, string, store.RuntimeNodeEnrollment) (store.RuntimeNodeIdentity, error) - RuntimeNodeGenerationConfiguration(context.Context, string, string, uint64) (store.RuntimeNodeConfiguration, error) - RuntimeNodeStatus(context.Context, string, string) (store.RuntimeNodeStatus, error) + View(ctx context.Context) (deployment.View, error) + ListNodes(ctx context.Context) ([]deployment.Node, error) + NodeDetail(ctx context.Context, id, window string) (deployment.NodeDetail, error) + UpdateNode(ctx context.Context, id string, update deployment.NodeUpdate) error + RemoveNode(ctx context.Context, id string) error + CreateEnrollment(ctx context.Context, capacity deployment.Capacity) (deployment.EnrollmentToken, error) + Enroll(ctx context.Context, token string, input deployment.Enrollment) (deployment.NodeIdentity, error) + NodeConfiguration(ctx context.Context, nodeID, token string, generation uint64) (deployment.NodeConfiguration, error) + NodeStatus(ctx context.Context, nodeID, credential string) (deployment.NodeStatus, error) + // DecodeConfiguration decodes a submitted provider configuration and + // credential with the provider's declared codec. + DecodeConfiguration(provider string, public, credential json.RawMessage) (sandbox.Configuration, error) +} + +// NodeAllocations lists the allocations a sandbox node holds. +type NodeAllocations interface { + ListNodeRuntimeAllocations(ctx context.Context, nodeID string) ([]store.RuntimeNodeAllocation, error) } // registerSandboxNodeRoutes serves node machine connections. They authenticate @@ -83,13 +93,13 @@ func (h *Handler) registerSandboxManagerRoutes(r chi.Router) { // @Tags Sandbox Manager // @Produce json // @Security DeploymentAdminAuth -// @Success 200 {object} store.RuntimeDeploymentView +// @Success 200 {object} deployment.View // @Failure 400,401,404,409,500,503 {object} CoreErrorResponse // @Router /core/v1/sandbox/deployment [get] func (h *Handler) sandboxDeployment(w http.ResponseWriter, r *http.Request) { - value, err := h.Sandboxes.Deployment.GetRuntimeDeployment(r.Context()) + value, err := h.Sandboxes.Deployment.View(r.Context()) if err != nil { - writeStoreError(w, r, err) + writeDeploymentError(w, r, err) return } writeJSON(w, http.StatusOK, value) @@ -104,9 +114,9 @@ func (h *Handler) sandboxDeployment(w http.ResponseWriter, r *http.Request) { // @Failure 400,401,404,409,500,503 {object} CoreErrorResponse // @Router /core/v1/sandbox/nodes [get] func (h *Handler) sandboxNodes(w http.ResponseWriter, r *http.Request) { - value, err := h.Sandboxes.Deployment.ListRuntimeNodes(r.Context()) + value, err := h.Sandboxes.Deployment.ListNodes(r.Context()) if err != nil { - writeStoreError(w, r, err) + writeDeploymentError(w, r, err) return } writeJSON(w, http.StatusOK, SandboxNodeList{Data: value}) @@ -119,7 +129,7 @@ func (h *Handler) sandboxNodes(w http.ResponseWriter, r *http.Request) { // @Security DeploymentAdminAuth // @Param node_id path string true "Sandbox node UUID" // @Accept json -// @Param body body store.RuntimeNodeUpdate true "Request" +// @Param body body deployment.NodeUpdate true "Request" // @Success 200 {object} api.SandboxMutationResponse // @Failure 400,401,404,409,500,503 {object} CoreErrorResponse // @Router /core/v1/sandbox/nodes/{node_id} [patch] @@ -128,14 +138,14 @@ func (h *Handler) updateSandboxNode(w http.ResponseWriter, r *http.Request) { if !ok { return } - var input store.RuntimeNodeUpdate + var input deployment.NodeUpdate if decodeInputObject(raw, &input, "name", "max_active", "max_retained") != nil { - writeStoreError(w, r, store.ErrInvalidInput) + writeDeploymentError(w, r, deployment.ErrInvalidInput) return } id := chi.URLParam(r, "node_id") - if err := h.Sandboxes.Deployment.UpdateRuntimeNode(r.Context(), id, input); err != nil { - writeStoreError(w, r, err) + if err := h.Sandboxes.Deployment.UpdateNode(r.Context(), id, input); err != nil { + writeDeploymentError(w, r, err) return } writeJSON(w, http.StatusOK, SandboxMutationResponse{ID: id, Updated: true}) @@ -152,8 +162,8 @@ func (h *Handler) updateSandboxNode(w http.ResponseWriter, r *http.Request) { // @Router /core/v1/sandbox/nodes/{node_id} [delete] func (h *Handler) removeSandboxNode(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "node_id") - if err := h.Sandboxes.Deployment.RemoveRuntimeNode(r.Context(), id); err != nil { - writeStoreError(w, r, err) + if err := h.Sandboxes.Deployment.RemoveNode(r.Context(), id); err != nil { + writeDeploymentError(w, r, err) return } writeJSON(w, http.StatusOK, SandboxMutationResponse{ID: id, Deleted: true}) @@ -169,7 +179,7 @@ func (h *Handler) removeSandboxNode(w http.ResponseWriter, r *http.Request) { // @Failure 400,401,404,409,500,503 {object} CoreErrorResponse // @Router /core/v1/sandbox/nodes/{node_id}/allocations [get] func (h *Handler) sandboxAllocations(w http.ResponseWriter, r *http.Request) { - value, err := h.Sandboxes.Deployment.ListNodeRuntimeAllocations(r.Context(), chi.URLParam(r, "node_id")) + value, err := h.Sandboxes.NodeAllocations.ListNodeRuntimeAllocations(r.Context(), chi.URLParam(r, "node_id")) if err != nil { writeStoreError(w, r, err) return @@ -194,19 +204,19 @@ func (h *Handler) createSandboxEnrollment(w http.ResponseWriter, r *http.Request } var input SandboxEnrollmentTokenRequest if decodeInputObject(raw, &input, "max_active", "max_retained") != nil { - writeStoreError(w, r, store.ErrInvalidInput) + writeDeploymentError(w, r, deployment.ErrInvalidInput) return } - capacity := store.RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 8} + capacity := deployment.Capacity{MaxActive: 2, MaxRetained: 8} if input.MaxActive != nil { capacity.MaxActive = *input.MaxActive } if input.MaxRetained != nil { capacity.MaxRetained = *input.MaxRetained } - enrollment, err := h.Sandboxes.Deployment.CreateRuntimeEnrollment(r.Context(), capacity) + enrollment, err := h.Sandboxes.Deployment.CreateEnrollment(r.Context(), capacity) if err != nil { - writeStoreError(w, r, err) + writeDeploymentError(w, r, err) return } writeJSON(w, http.StatusCreated, SandboxEnrollmentToken{Token: enrollment.Token, ExpiresAt: enrollment.ExpiresAt, EnrollmentID: enrollment.ID}) @@ -218,32 +228,32 @@ func (h *Handler) createSandboxEnrollment(w http.ResponseWriter, r *http.Request // @Produce json // @Security NodeEnrollmentAuth // @Accept json -// @Param body body store.RuntimeNodeEnrollment true "Request" -// @Success 201 {object} store.RuntimeNodeIdentity +// @Param body body deployment.Enrollment true "Request" +// @Success 201 {object} deployment.NodeIdentity // @Failure 400,401,404,409,500,503 {object} v1.ErrorResponse // @Router /api/v1/sandbox-node/enroll [post] func (h *Handler) enrollSandboxNode(w http.ResponseWriter, r *http.Request) { token, ok := sandboxBearer(r) if !ok { - writeStoreError(w, r, store.ErrRuntimeNodeCredential) + writeDeploymentError(w, r, deployment.ErrNodeCredential) return } raw, ok := readJSONBodyLimit(w, r, 16384, "Sandbox node enrollment is too large.") if !ok { return } - var input store.RuntimeNodeEnrollment + var input deployment.Enrollment if decodeInputObject(raw, &input, "node_id", "credential", "name", "provider", "backend_fingerprint", "deployment_generation", "specification_digest", "core_url") != nil { - writeStoreError(w, r, store.ErrInvalidInput) + writeDeploymentError(w, r, deployment.ErrInvalidInput) return } if input.CoreURL == "" { writeError(w, http.StatusBadRequest, "invalid_request_error", "Enrollment requires core_url, the Core origin this node uses. Install the node with this Core's node installer.", "core_url") return } - value, err := h.Sandboxes.Deployment.EnrollRuntimeNode(r.Context(), token, input) + value, err := h.Sandboxes.Deployment.Enroll(r.Context(), token, input) if err != nil { - writeStoreError(w, r, err) + writeDeploymentError(w, r, err) return } writeJSON(w, http.StatusCreated, value) @@ -255,23 +265,23 @@ func (h *Handler) enrollSandboxNode(w http.ResponseWriter, r *http.Request) { // @Produce json // @Security NodeAuth // @Param node_id query string true "Sandbox node UUID" -// @Success 200 {object} store.RuntimeNodeStatus +// @Success 200 {object} deployment.NodeStatus // @Failure 400,401,404,409,500,503 {object} v1.ErrorResponse // @Router /api/v1/sandbox-node/identity [get] func (h *Handler) sandboxNodeIdentity(w http.ResponseWriter, r *http.Request) { token, ok := sandboxBearer(r) if !ok { - writeStoreError(w, r, store.ErrRuntimeNodeCredential) + writeDeploymentError(w, r, deployment.ErrNodeCredential) return } ids := r.URL.Query()["node_id"] if len(ids) != 1 { - writeStoreError(w, r, store.ErrInvalidInput) + writeDeploymentError(w, r, deployment.ErrInvalidInput) return } - value, err := h.Sandboxes.Deployment.RuntimeNodeStatus(r.Context(), ids[0], token) + value, err := h.Sandboxes.Deployment.NodeStatus(r.Context(), ids[0], token) if err != nil { - writeStoreError(w, r, err) + writeDeploymentError(w, r, err) return } writeJSON(w, http.StatusOK, value) diff --git a/services/core/internal/api/sandbox_manager_test.go b/services/core/internal/api/sandbox_manager_test.go index b4322a3e2..61f816a93 100644 --- a/services/core/internal/api/sandbox_manager_test.go +++ b/services/core/internal/api/sandbox_manager_test.go @@ -7,8 +7,8 @@ import ( "strings" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) // sandboxFakes authenticates callerBinding() as a Project key and @@ -72,24 +72,25 @@ func TestSandboxEnrollmentDoesNotAcceptProjectAsAdmin(t *testing.T) { func TestSandboxLocalNodeRemovalExplainsDeploymentBinding(t *testing.T) { request := httptest.NewRequest(http.MethodDelete, "/core/v1/sandbox/nodes/local", nil) response := httptest.NewRecorder() - writeStoreError(response, request, store.ErrRuntimeLocalNodeConfigured) + writeStoreError(response, request, deployment.ErrLocalNodeConfigured) if response.Code != http.StatusConflict || !strings.Contains(response.Body.String(), "runtime_local_node_configured") || !strings.Contains(response.Body.String(), "previous release") { t.Fatal(response.Code, response.Body.String()) } } -// storeCapacity rejects max_active outside the store's node capacity bounds. -func storeCapacity(active int) error { +// nodeCapacity rejects max_active outside the deployment's node capacity +// bounds. +func nodeCapacity(active int) error { if active < 1 || active > 1000000 { - return &store.AdminValidationError{Code: "invalid_node_capacity", Param: "max_active"} + return &deployment.NodeValidationError{Code: "invalid_node_capacity", Param: "max_active"} } return nil } func TestSandboxEnrollmentCapacityIsAdministratorOnly(t *testing.T) { deps, fakes := sandboxFakes(t) - fakes.deployment.createRuntimeEnrollment = func(_ context.Context, capacity store.RuntimeNodeCapacity) (store.RuntimeNodeEnrollmentToken, error) { - return store.RuntimeNodeEnrollmentToken{}, storeCapacity(capacity.MaxActive) + fakes.deployment.createEnrollment = func(_ context.Context, capacity deployment.Capacity) (deployment.EnrollmentToken, error) { + return deployment.EnrollmentToken{}, nodeCapacity(capacity.MaxActive) } h := newTestHandler(t, deps) for _, test := range []struct{ path, token, body string }{ diff --git a/services/core/internal/api/sandbox_node_configuration.go b/services/core/internal/api/sandbox_node_configuration.go index b13a7d4d8..afab10889 100644 --- a/services/core/internal/api/sandbox_node_configuration.go +++ b/services/core/internal/api/sandbox_node_configuration.go @@ -1,10 +1,11 @@ package api import ( - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "math" "net/http" "strconv" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" ) // @Summary Read the active configuration for node installation @@ -14,31 +15,31 @@ import ( // @Security NodeEnrollmentAuth // @Param X-OAC-Node-ID header string false "Retained node UUID" // @Param generation query integer false "Exact kept generation (registered nodes only); omitted reads the current target" -// @Success 200 {object} store.RuntimeNodeConfiguration +// @Success 200 {object} deployment.NodeConfiguration // @Failure 400,401,409,500,503 {object} v1.ErrorResponse // @Router /api/v1/sandbox-node/configuration [get] func (h *Handler) sandboxNodeConfiguration(w http.ResponseWriter, r *http.Request) { token, ok := sandboxBearer(r) if !ok || len(r.Header.Values("X-OAC-Node-ID")) > 1 { - writeStoreError(w, r, store.ErrRuntimeNodeCredential) + writeDeploymentError(w, r, deployment.ErrNodeCredential) return } var generation uint64 if values, present := r.URL.Query()["generation"]; present { var err error if len(values) != 1 { - writeStoreError(w, r, store.ErrInvalidInput) + writeDeploymentError(w, r, deployment.ErrInvalidInput) return } generation, err = strconv.ParseUint(values[0], 10, 63) if err != nil || generation == 0 || generation > math.MaxInt64 || strconv.FormatUint(generation, 10) != values[0] { - writeStoreError(w, r, store.ErrInvalidInput) + writeDeploymentError(w, r, deployment.ErrInvalidInput) return } } - value, err := h.Sandboxes.Deployment.RuntimeNodeGenerationConfiguration(r.Context(), r.Header.Get("X-OAC-Node-ID"), token, generation) + value, err := h.Sandboxes.Deployment.NodeConfiguration(r.Context(), r.Header.Get("X-OAC-Node-ID"), token, generation) if err != nil { - writeStoreError(w, r, err) + writeDeploymentError(w, r, err) return } writeJSON(w, http.StatusOK, value) diff --git a/services/core/internal/api/sandbox_node_detail.go b/services/core/internal/api/sandbox_node_detail.go index 257d20951..28c3ddb01 100644 --- a/services/core/internal/api/sandbox_node_detail.go +++ b/services/core/internal/api/sandbox_node_detail.go @@ -6,7 +6,7 @@ import ( "net/url" "time" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/go-chi/chi/v5" ) @@ -17,13 +17,13 @@ import ( // @Security DeploymentAdminAuth // @Param node_id path string true "Sandbox node UUID" // @Param range query string false "Time range (default 1h)" Enums(1h,6h,24h) -// @Success 200 {object} store.RuntimeNodeDetail +// @Success 200 {object} deployment.NodeDetail // @Failure 400,401,404,500,503 {object} CoreErrorResponse // @Router /core/v1/sandbox/nodes/{node_id} [get] func (h *Handler) sandboxNodeDetail(w http.ResponseWriter, r *http.Request) { values, err := url.ParseQuery(r.URL.RawQuery) if err != nil || len(values) > 1 || len(values) == 1 && len(values["range"]) != 1 { - writeStoreError(w, r, store.ErrInvalidInput) + writeDeploymentError(w, r, deployment.ErrInvalidInput) return } name := "1h" @@ -32,9 +32,9 @@ func (h *Handler) sandboxNodeDetail(w http.ResponseWriter, r *http.Request) { } ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second) defer cancel() - value, err := h.Sandboxes.Deployment.GetRuntimeNodeDetail(ctx, chi.URLParam(r, "node_id"), name) + value, err := h.Sandboxes.Deployment.NodeDetail(ctx, chi.URLParam(r, "node_id"), name) if err != nil { - writeStoreError(w, r, err) + writeDeploymentError(w, r, err) return } writeJSON(w, http.StatusOK, value) diff --git a/services/core/internal/api/sandbox_node_detail_test.go b/services/core/internal/api/sandbox_node_detail_test.go index dead3a9a2..e43f73eab 100644 --- a/services/core/internal/api/sandbox_node_detail_test.go +++ b/services/core/internal/api/sandbox_node_detail_test.go @@ -4,14 +4,14 @@ import ( "context" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" ) func TestSandboxNodeDetailValidationAndAuthentication(t *testing.T) { deps, fakes := sandboxFakes(t) - // The store rejects an unknown range or a malformed node ID. - fakes.deployment.getRuntimeNodeDetail = func(context.Context, string, string) (store.RuntimeNodeDetail, error) { - return store.RuntimeNodeDetail{}, store.ErrInvalidInput + // The deployment rejects an unknown range or a malformed node ID. + fakes.deployment.nodeDetail = func(context.Context, string, string) (deployment.NodeDetail, error) { + return deployment.NodeDetail{}, deployment.ErrInvalidInput } h := newTestHandler(t, deps) path := "/core/v1/sandbox/nodes/11111111-1111-4111-8111-111111111111" diff --git a/services/core/internal/deployment/doc.go b/services/core/internal/deployment/doc.go new file mode 100644 index 000000000..4f8f60b4f --- /dev/null +++ b/services/core/internal/deployment/doc.go @@ -0,0 +1,6 @@ +// Package deployment owns the managed sandbox deployment: the provider, +// specification and configuration an administrator selects, its generations, +// and the sandbox nodes that enroll with it, authenticate, report their +// readiness and serve its generations. It is the one package that interprets +// the registered provider declarations for those rules. +package deployment diff --git a/services/core/internal/deployment/errors.go b/services/core/internal/deployment/errors.go new file mode 100644 index 000000000..907c64226 --- /dev/null +++ b/services/core/internal/deployment/errors.go @@ -0,0 +1,78 @@ +package deployment + +import ( + "errors" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +var ( + // ErrInvalidInput reports a malformed identifier, limit or request value. + ErrInvalidInput = errors.New("invalid sandbox deployment input") + // ErrNotFound reports a missing sandbox node, allocation or generation. + ErrNotFound = errors.New("sandbox deployment resource not found") + // ErrConflict reports a deployment that cannot change in its current state. + ErrConflict = errors.New("sandbox deployment is already configured differently") + // ErrSpecificationMismatch reports a node or stored generation whose + // specification differs from the deployment's. + ErrSpecificationMismatch = errors.New("node does not match the deployment specification") + ErrResetInProgress = errors.New("a sandbox reset is in progress") + ErrNotConfigured = errors.New("the sandbox deployment is not configured") + // ErrPublicURLUnreachable rejects selection and admission when the provider + // requires a reachable public origin and the installation is loopback. + ErrPublicURLUnreachable = errors.New("This sandbox provider needs a reachable HTTPS public URL before they can connect to Core.") + ErrNodesPreparing = errors.New("sandbox nodes are preparing the target generation") + ErrNodeUnavailable = errors.New("sandbox node unavailable") + ErrNodeInUse = errors.New("sandbox node retains resources") + ErrNodeCredential = errors.New("invalid sandbox node credential") + ErrLocalNodeConfigured = errors.New("local sandbox node is enabled in deployment configuration") + // ErrNodeAddressMismatch rejects an enrollment whose Core address is not + // the installation public URL. The token stays unconsumed. + ErrNodeAddressMismatch = errors.New("sandbox node Core address differs from the public URL") + // ErrNodeExists rejects an enrollment whose node ID is already in use. + ErrNodeExists = errors.New("sandbox node ID is already enrolled") +) + +// GenerationStaleError rejects a change whose expected generation is not the +// deployment's current one. +type GenerationStaleError struct{ CurrentGeneration uint64 } + +func (e *GenerationStaleError) Error() string { return "the sandbox deployment generation changed" } +func (e *GenerationStaleError) Unwrap() error { return ErrConflict } + +// ResetRequiredError rejects a change that needs a reset first, such as a +// change of backend. +type ResetRequiredError struct{ CurrentProvider, RequestedProvider string } + +func (e *ResetRequiredError) Error() string { + return "reset the sandbox deployment before changing its backend" +} +func (e *ResetRequiredError) Unwrap() error { return ErrConflict } + +// ConfigurationError contains only validated, non-secret configuration +// diagnostics. +type ConfigurationError struct { + Message string + Validation *sandbox.ValidationError +} + +func (e *ConfigurationError) Error() string { return e.Message } +func (e *ConfigurationError) Unwrap() error { return ErrInvalidInput } + +// configurationError wraps a provider's rejection of a selection. +func configurationError(err error) *ConfigurationError { + var validation *sandbox.ValidationError + errors.As(err, &validation) + return &ConfigurationError{Message: err.Error(), Validation: validation} +} + +// NodeValidationError rejects a node name or capacity. Code is +// invalid_name or invalid_node_capacity, and Param names the field. It adds +// field identity without changing the ErrInvalidInput text or classification. +type NodeValidationError struct { + Code, Param string + MaxLength int +} + +func (e *NodeValidationError) Error() string { return ErrInvalidInput.Error() } +func (e *NodeValidationError) Unwrap() error { return ErrInvalidInput } diff --git a/services/core/internal/deployment/execution.go b/services/core/internal/deployment/execution.go new file mode 100644 index 000000000..1b69cee68 --- /dev/null +++ b/services/core/internal/deployment/execution.go @@ -0,0 +1,423 @@ +package deployment + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// ExecutionOperations are the deployment changes only the execution owner +// makes. Every change runs in a transaction on the connection that holds the +// execution lease and repeats its checks there; provider work runs between +// transactions, never inside one. +type ExecutionOperations struct { + service *Service + storage ExecutionStorage +} + +// NewExecutionOperations binds the deployment changes to the lease-bound +// storage of one execution owner. +func NewExecutionOperations(service *Service, storage ExecutionStorage) (*ExecutionOperations, error) { + if service == nil || storage == nil { + return nil, errors.New("deployment execution operations require the deployment service and execution storage") + } + return &ExecutionOperations{service: service, storage: storage}, nil +} + +// Claim reserves the installation for Web setup, once per execution owner +// startup, and fences the previous owner epoch's node presence. +func (e *ExecutionOperations) Claim(ctx context.Context, installationID string) error { + id, err := parseID(installationID) + if err != nil { + return err + } + return e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { + d, err := tx.LoadDeployment() + if err != nil { + return err + } + if d.InstallationID != "" { + if !d.WebManaged || d.InstallationID != id { + return ErrConflict + } + } else { + resources, err := tx.CountResources() + if err != nil { + return err + } + if resources.Allocations != 0 || resources.Pending != 0 { + return ErrConflict + } + } + if d.Provider != "" { + if _, err := e.service.specification(d); err != nil { + return err + } + } + return tx.ClaimInstallation(id) + }) +} + +// ConfigureProcess records the deployment process configuration selects, +// before the Worker starts. AdmissionPaused must be committed for the old +// installation before any switch. A nil selection never forgets the previous +// identity or unresolved resources. +func (e *ExecutionOperations) ConfigureProcess(ctx context.Context, selected *ProcessDeployment) error { + var installation string + if selected != nil { + copy := *selected + selected = © + id, err := parseID(selected.InstallationID) + if err != nil { + return err + } + installation = id + if !validDigest(selected.BackendFingerprint) { + return fmt.Errorf("%w: invalid backend identity fingerprint", ErrInvalidInput) + } + } + return e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { + previous, err := tx.LoadDeployment() + if err != nil { + return err + } + if previous.WebManaged { + return ErrConflict + } + if selected != nil && previous.InstallationID == installation && previous.BackendFingerprint == selected.BackendFingerprint && (previous.Provider == "" || selected.ProviderKind == previous.Provider) { + if err := tx.SetProcessDeployment(installation, selected.BackendFingerprint, selected.AdmissionPaused); err != nil { + return err + } + return e.configureManager(tx, previous, selected, installation) + } + resources, err := tx.CountResources() + if err != nil { + return err + } + if selected == nil { + if previous.InstallationID != "" && (resources.Allocations != 0 || resources.Pending != 0) { + return fmt.Errorf("cannot disable managed sandbox provider: %d unreleased allocations (instances, retained snapshots, uncertain operations or pending cleanup) and %d pending hosted environments remain", resources.Allocations, resources.Pending) + } + return nil + } + if previous.InstallationID == "" { + if resources.Allocations != 0 { + return fmt.Errorf("cannot adopt sandbox installation: %d existing unreleased allocations (including retained snapshots and pending cleanup) have no verified backend identity", resources.Allocations) + } + } else { + if !previous.AdmissionPaused || !selected.AdmissionPaused { + return fmt.Errorf("cannot switch sandbox installation: persist maintenance on the previous installation and keep the new installation in maintenance") + } + if resources.Allocations != 0 || resources.Pending != 0 { + return fmt.Errorf("cannot switch sandbox installation: %d unreleased allocations (instances, retained snapshots, uncertain operations or pending cleanup) and %d pending hosted environments remain", resources.Allocations, resources.Pending) + } + } + if err := tx.SetProcessDeployment(installation, selected.BackendFingerprint, selected.AdmissionPaused); err != nil { + return err + } + return e.configureManager(tx, previous, selected, installation) + }) +} + +// configureManager records the node provider and the local node process +// configuration selects. +func (e *ExecutionOperations) configureManager(tx DeploymentTx, previous Record, selected *ProcessDeployment, installation string) error { + if selected.ProviderKind == "" { + return nil + } + isNode, err := e.service.registry.IsNode(selected.ProviderKind) + if err != nil { + return err + } + if !isNode { + return ErrInvalidInput + } + if previous.Provider == "" { + resources, err := tx.CountResources() + if err != nil { + return err + } + if resources.Allocations != 0 || resources.Pending != 0 { + return fmt.Errorf("cannot adopt historical sandbox resources: keep the original Core responsible for retained resources and install this release separately") + } + } + var localNode string + if selected.LocalNodeID != "" { + id, err := parseID(selected.LocalNodeID) + if err != nil { + return err + } + localNode = id + if previous.LocalNodeID != "" && previous.LocalNodeID != id { + resources, err := tx.CountResources() + if err != nil { + return err + } + if resources.Allocations != 0 || resources.Pending != 0 || !previous.AdmissionPaused || !selected.AdmissionPaused { + return fmt.Errorf("local sandbox node identity changed: restore its original state directory; replacement requires maintenance and no retained resources") + } + } + if !validDigest(selected.LocalCredentialSHA256) { + return ErrInvalidInput + } + if err := validateNode("Local", selected.LocalMaxActive, selected.LocalMaxRetained); err != nil { + return err + } + n, err := tx.LoadNode(id) + if errors.Is(err, ErrNotFound) { + _, err = tx.InsertNode(NewNode{ID: id, InstallationID: installation, Name: "Local", BackendFingerprint: selected.BackendFingerprint, CredentialDigest: selected.LocalCredentialSHA256, MaxActive: selected.LocalMaxActive, MaxRetained: selected.LocalMaxRetained}) + } else if err == nil { + if n.InstallationID != installation || n.BackendFingerprint != selected.BackendFingerprint || n.CredentialDigest != selected.LocalCredentialSHA256 { + return fmt.Errorf("local sandbox node identity does not match the retained backend") + } + err = tx.UpdateNode(id, NodeLimits{Name: n.Name, MaxActive: selected.LocalMaxActive, MaxRetained: selected.LocalMaxRetained}) + } + if err != nil { + return err + } + } + return tx.SetManagerDeployment(selected.ProviderKind, localNode) +} + +// CheckSetup rejects a stale or reset deployment before provider preparation. +// Initialize repeats the check in its committing transaction. +func (e *ExecutionOperations) CheckSetup(ctx context.Context, installation string, input sandbox.Selection) error { + return e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { + d, err := tx.LoadDeployment() + if err != nil { + return err + } + if err := checkGeneration(d, installation, input.ExpectedGeneration); err != nil { + return err + } + if d.Reset != nil { + return ErrResetInProgress + } + if err := e.service.validateSelection(input); err != nil { + return err + } + if d.Provider != "" { + if _, err := e.service.specification(d); err != nil { + return err + } + if d.Provider != input.Provider { + return &ResetRequiredError{CurrentProvider: d.Provider, RequestedProvider: input.Provider} + } + } + return nil + }) +} + +// Initialize stores the first selection of the installation. The same +// selection at the current generation is a no-op; a different one conflicts. +func (e *ExecutionOperations) Initialize(ctx context.Context, installation string, input sandbox.Selection) (View, error) { + if err := e.service.validateSelection(input); err != nil { + return View{}, err + } + id, err := parseID(installation) + if err != nil { + return View{}, err + } + var result View + err = e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { + d, err := tx.LoadDeployment() + if err != nil { + return err + } + if err := checkGeneration(d, installation, input.ExpectedGeneration); err != nil { + return err + } + if d.Reset != nil { + return ErrResetInProgress + } + if d.InstallationID != id { + return ErrConflict + } + if d.Provider != "" { + equal, err := e.service.selectionEqual(d, input) + if err != nil { + return err + } + if !equal { + return ErrConflict + } + if err := e.recordMetadata(tx, input); err != nil { + return err + } + } else if err := e.saveSelection(tx, d, input); err != nil { + return err + } + result, err = e.committedView(tx) + return err + }) + if err != nil { + return View{}, err + } + return result, nil +} + +// ClassifyChange resolves an omitted credential before validation and reports +// whether the change leaves the deployment unchanged. An explicitly submitted +// credential is a verified change even when its bytes are unchanged. +func (e *ExecutionOperations) ClassifyChange(ctx context.Context, installation string, input sandbox.Selection) (sandbox.Selection, bool, error) { + var unchanged bool + err := e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { + d, err := tx.LoadDeployment() + if err != nil { + return err + } + if err := e.checkSwitch(d, installation, input); err != nil { + return err + } + previous, err := e.service.setup(d) + if err != nil { + return err + } + resolved, err := e.service.registry.ResolveChange(input, previous.selection()) + if err != nil { + return configurationError(err) + } + resolved.ExpectedGeneration = input.ExpectedGeneration + input = resolved + if err := e.service.validateSelection(input); err != nil { + return err + } + equal, err := e.service.selectionEqual(d, input) + unchanged = equal && !input.ReplacesCredential() + return err + }) + return input, unchanged, err +} + +// checkSwitch rejects a change of a deployment that is stale, resetting, +// unconfigured or on another backend. +func (e *ExecutionOperations) checkSwitch(d Record, installation string, input sandbox.Selection) error { + if err := checkGeneration(d, installation, input.ExpectedGeneration); err != nil { + return err + } + if d.Reset != nil { + return ErrResetInProgress + } + if d.Provider == "" { + return ErrNotConfigured + } + if _, err := e.service.specification(d); err != nil { + return err + } + if d.Provider != input.Provider { + return &ResetRequiredError{CurrentProvider: d.Provider, RequestedProvider: input.Provider} + } + return nil +} + +// Update changes the selection on the same backend. A different selection or a +// submitted credential stores the next generation and retains the previous one +// for the allocations that still use it. +func (e *ExecutionOperations) Update(ctx context.Context, installation string, input sandbox.Selection) (View, error) { + if err := e.service.validateSelection(input); err != nil { + return View{}, err + } + var result View + err := e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { + d, err := tx.LoadDeployment() + if err != nil { + return err + } + if err := e.checkSwitch(d, installation, input); err != nil { + return err + } + equal, err := e.service.selectionEqual(d, input) + if err != nil { + return err + } + if !equal || input.ReplacesCredential() { + if err := tx.RetainGeneration(); err != nil { + return err + } + if err := e.saveSelection(tx, d, input); err != nil { + return err + } + if err := tx.CollectGenerations(); err != nil { + return err + } + action := "change" + if input.ReplacesCredential() { + action = "replace_credential" + } + if err := tx.RecordAudit(action, installation); err != nil { + return err + } + } else if err := e.recordMetadata(tx, input); err != nil { + return err + } + result, err = e.committedView(tx) + return err + }) + if err != nil { + return View{}, err + } + return result, nil +} + +// CollectGenerations deletes retained generations nothing uses, serialized +// with admission and deployment changes. +func (e *ExecutionOperations) CollectGenerations(ctx context.Context) error { + return e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { + return tx.CollectGenerations() + }) +} + +func (e *ExecutionOperations) committedView(tx DeploymentTx) (View, error) { + snapshot, err := tx.LoadSnapshot() + if err != nil { + return View{}, err + } + return e.service.view(snapshot) +} + +// saveSelection stores input as the deployment's next generation. +func (e *ExecutionOperations) saveSelection(tx DeploymentTx, d Record, input sandbox.Selection) error { + registry := e.service.registry + if err := registry.ValidateSpecification(input.Provider, input.DeploymentSpec); err != nil { + return configurationError(err) + } + if d.Generation >= math.MaxInt64 { + return ErrConflict + } + description, err := registry.Describe(input.Provider, d.InstallationID) + if err != nil { + return configurationError(err) + } + input, err = registry.Normalize(input) + if err != nil { + return configurationError(err) + } + record, err := registry.Encode(input.Provider, input.Configuration) + if err != nil { + return configurationError(err) + } + specification, err := json.Marshal(input.DeploymentSpec) + if err != nil { + return err + } + return tx.SaveSelection(SelectionRecord{InstallationID: d.InstallationID, Provider: input.Provider, BackendFingerprint: description.BackendFingerprint, Mode: description.Mode, + Generation: d.Generation + 1, IdleSeconds: description.IdleSeconds, RetentionSeconds: description.RetentionSeconds, Specification: specification, + Configuration: sandbox.ConfigurationRecord{Public: configurationJSON(record.Public), Metadata: configurationJSON(record.Metadata), Secret: record.Secret}}) +} + +// recordMetadata keeps the provider's latest non-secret metadata for an +// unchanged selection. +func (e *ExecutionOperations) recordMetadata(tx DeploymentTx, input sandbox.Selection) error { + record, err := e.service.registry.Encode(input.Provider, input.Configuration) + if err != nil { + return configurationError(err) + } + if len(record.Metadata) == 0 { + return nil + } + return tx.RecordConfigurationMetadata(record.Metadata) +} diff --git a/services/core/internal/deployment/fakes_test.go b/services/core/internal/deployment/fakes_test.go new file mode 100644 index 000000000..97086a8d1 --- /dev/null +++ b/services/core/internal/deployment/fakes_test.go @@ -0,0 +1,427 @@ +package deployment + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// The fakes are strict: each method runs its func field, and a nil field fails +// the test, so a test sets exactly the calls it expects. + +func unexpected(t testing.TB, method string) { + t.Helper() + t.Fatalf("unexpected call to %s", method) +} + +type fakeStorage struct { + t testing.TB + withNodes func(context.Context, func(NodeTx) error) error + connectNode func(context.Context, string, string, uint64) (bool, error) + disconnectNode func(context.Context, string, string, uint64) error + sampleHostHistory func(context.Context) (int64, error) +} + +func (f *fakeStorage) WithNodes(ctx context.Context, apply func(NodeTx) error) error { + if f.withNodes == nil { + unexpected(f.t, "WithNodes") + } + return f.withNodes(ctx, apply) +} + +func (f *fakeStorage) ConnectNode(ctx context.Context, nodeID, connectionID string, epoch uint64) (bool, error) { + if f.connectNode == nil { + unexpected(f.t, "ConnectNode") + } + return f.connectNode(ctx, nodeID, connectionID, epoch) +} + +func (f *fakeStorage) DisconnectNode(ctx context.Context, nodeID, connectionID string, epoch uint64) error { + if f.disconnectNode == nil { + unexpected(f.t, "DisconnectNode") + } + return f.disconnectNode(ctx, nodeID, connectionID, epoch) +} + +func (f *fakeStorage) SampleHostHistory(ctx context.Context) (int64, error) { + if f.sampleHostHistory == nil { + unexpected(f.t, "SampleHostHistory") + } + return f.sampleHostHistory(ctx) +} + +type fakeExecutionStorage struct { + t testing.TB + withDeployment func(context.Context, func(DeploymentTx) error) error +} + +func (f *fakeExecutionStorage) WithDeployment(ctx context.Context, apply func(DeploymentTx) error) error { + if f.withDeployment == nil { + unexpected(f.t, "WithDeployment") + } + return f.withDeployment(ctx, apply) +} + +type fakeReader struct { + t testing.TB + deployment func(context.Context) (Record, error) + snapshot func(context.Context) (Snapshot, error) + ownerEpoch func(context.Context) (uint64, error) + allocation func(context.Context, sandbox.Reference) (AllocationRecord, error) + generations func(context.Context, int64) ([]GenerationRecord, error) + nodes func(context.Context) ([]NodeRecord, error) + nodeHistory func(context.Context, string, coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) + readNodes func(context.Context, func(NodeReads) error) error +} + +func (f *fakeReader) Deployment(ctx context.Context) (Record, error) { + if f.deployment == nil { + unexpected(f.t, "Deployment") + } + return f.deployment(ctx) +} + +func (f *fakeReader) Snapshot(ctx context.Context) (Snapshot, error) { + if f.snapshot == nil { + unexpected(f.t, "Snapshot") + } + return f.snapshot(ctx) +} + +func (f *fakeReader) OwnerEpoch(ctx context.Context) (uint64, error) { + if f.ownerEpoch == nil { + unexpected(f.t, "OwnerEpoch") + } + return f.ownerEpoch(ctx) +} + +func (f *fakeReader) Allocation(ctx context.Context, ref sandbox.Reference) (AllocationRecord, error) { + if f.allocation == nil { + unexpected(f.t, "Allocation") + } + return f.allocation(ctx, ref) +} + +func (f *fakeReader) Generations(ctx context.Context, after int64) ([]GenerationRecord, error) { + if f.generations == nil { + unexpected(f.t, "Generations") + } + return f.generations(ctx, after) +} + +func (f *fakeReader) Nodes(ctx context.Context) ([]NodeRecord, error) { + if f.nodes == nil { + unexpected(f.t, "Nodes") + } + return f.nodes(ctx) +} + +func (f *fakeReader) NodeHistory(ctx context.Context, nodeID string, window coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) { + if f.nodeHistory == nil { + unexpected(f.t, "NodeHistory") + } + return f.nodeHistory(ctx, nodeID, window) +} + +func (f *fakeReader) ReadNodes(ctx context.Context, apply func(NodeReads) error) error { + if f.readNodes == nil { + unexpected(f.t, "ReadNodes") + } + return f.readNodes(ctx, apply) +} + +type fakeNodeReads struct { + t testing.TB + loadDeployment func() (Record, error) + loadNode func(string) (StoredNode, error) + loadEnrollment func(string) (EnrollmentRecord, error) + loadGenerationSpecification func(uint64) (GenerationSpecification, error) + generationKept func(string, uint64) (bool, error) +} + +func (f *fakeNodeReads) LoadDeployment() (Record, error) { + if f.loadDeployment == nil { + unexpected(f.t, "LoadDeployment") + } + return f.loadDeployment() +} + +func (f *fakeNodeReads) LoadNode(id string) (StoredNode, error) { + if f.loadNode == nil { + unexpected(f.t, "LoadNode") + } + return f.loadNode(id) +} + +func (f *fakeNodeReads) LoadEnrollment(digest string) (EnrollmentRecord, error) { + if f.loadEnrollment == nil { + unexpected(f.t, "LoadEnrollment") + } + return f.loadEnrollment(digest) +} + +func (f *fakeNodeReads) LoadGenerationSpecification(generation uint64) (GenerationSpecification, error) { + if f.loadGenerationSpecification == nil { + unexpected(f.t, "LoadGenerationSpecification") + } + return f.loadGenerationSpecification(generation) +} + +func (f *fakeNodeReads) GenerationKept(nodeID string, generation uint64) (bool, error) { + if f.generationKept == nil { + unexpected(f.t, "GenerationKept") + } + return f.generationKept(nodeID, generation) +} + +type fakeNodeTx struct { + t testing.TB + loadDeployment func() (Record, error) + loadNode func(string) (StoredNode, error) + loadEnrollment func(string) (EnrollmentRecord, error) + loadGenerationSpecification func(uint64) (GenerationSpecification, error) + generationKept func(string, uint64) (bool, error) + listNodes func() ([]NodeRecord, error) + insertNode func(NewNode) (StoredNode, error) + updateNode func(string, NodeLimits) error + removeNode func(string) error + createEnrollment func(NewEnrollment) (time.Time, error) + consumeEnrollment func(string, string) (bool, error) + heartbeatNode func(Heartbeat) (bool, error) + deleteGenerationStatus func(string, uint64) error + upsertGenerationStatus func(GenerationStatusRecord) error + promoteServingGeneration func(string, uint64) error + refreshServingReadiness func(string, int) error +} + +func (f *fakeNodeTx) LoadDeployment() (Record, error) { + if f.loadDeployment == nil { + unexpected(f.t, "LoadDeployment") + } + return f.loadDeployment() +} + +func (f *fakeNodeTx) LoadNode(id string) (StoredNode, error) { + if f.loadNode == nil { + unexpected(f.t, "LoadNode") + } + return f.loadNode(id) +} + +func (f *fakeNodeTx) LoadEnrollment(digest string) (EnrollmentRecord, error) { + if f.loadEnrollment == nil { + unexpected(f.t, "LoadEnrollment") + } + return f.loadEnrollment(digest) +} + +func (f *fakeNodeTx) LoadGenerationSpecification(generation uint64) (GenerationSpecification, error) { + if f.loadGenerationSpecification == nil { + unexpected(f.t, "LoadGenerationSpecification") + } + return f.loadGenerationSpecification(generation) +} + +func (f *fakeNodeTx) GenerationKept(nodeID string, generation uint64) (bool, error) { + if f.generationKept == nil { + unexpected(f.t, "GenerationKept") + } + return f.generationKept(nodeID, generation) +} + +func (f *fakeNodeTx) ListNodes() ([]NodeRecord, error) { + if f.listNodes == nil { + unexpected(f.t, "ListNodes") + } + return f.listNodes() +} + +func (f *fakeNodeTx) InsertNode(node NewNode) (StoredNode, error) { + if f.insertNode == nil { + unexpected(f.t, "InsertNode") + } + return f.insertNode(node) +} + +func (f *fakeNodeTx) UpdateNode(id string, limits NodeLimits) error { + if f.updateNode == nil { + unexpected(f.t, "UpdateNode") + } + return f.updateNode(id, limits) +} + +func (f *fakeNodeTx) RemoveNode(id string) error { + if f.removeNode == nil { + unexpected(f.t, "RemoveNode") + } + return f.removeNode(id) +} + +func (f *fakeNodeTx) CreateEnrollment(enrollment NewEnrollment) (time.Time, error) { + if f.createEnrollment == nil { + unexpected(f.t, "CreateEnrollment") + } + return f.createEnrollment(enrollment) +} + +func (f *fakeNodeTx) ConsumeEnrollment(digest, nodeID string) (bool, error) { + if f.consumeEnrollment == nil { + unexpected(f.t, "ConsumeEnrollment") + } + return f.consumeEnrollment(digest, nodeID) +} + +func (f *fakeNodeTx) HeartbeatNode(heartbeat Heartbeat) (bool, error) { + if f.heartbeatNode == nil { + unexpected(f.t, "HeartbeatNode") + } + return f.heartbeatNode(heartbeat) +} + +func (f *fakeNodeTx) DeleteGenerationStatus(nodeID string, generation uint64) error { + if f.deleteGenerationStatus == nil { + unexpected(f.t, "DeleteGenerationStatus") + } + return f.deleteGenerationStatus(nodeID, generation) +} + +func (f *fakeNodeTx) UpsertGenerationStatus(status GenerationStatusRecord) error { + if f.upsertGenerationStatus == nil { + unexpected(f.t, "UpsertGenerationStatus") + } + return f.upsertGenerationStatus(status) +} + +func (f *fakeNodeTx) PromoteServingGeneration(nodeID string, generation uint64) error { + if f.promoteServingGeneration == nil { + unexpected(f.t, "PromoteServingGeneration") + } + return f.promoteServingGeneration(nodeID, generation) +} + +func (f *fakeNodeTx) RefreshServingReadiness(nodeID string, protocol int) error { + if f.refreshServingReadiness == nil { + unexpected(f.t, "RefreshServingReadiness") + } + return f.refreshServingReadiness(nodeID, protocol) +} + +type fakeDeploymentTx struct { + t testing.TB + loadDeployment func() (Record, error) + loadSnapshot func() (Snapshot, error) + countResources func() (Resources, error) + claimInstallation func(string) error + setProcessDeployment func(string, string, bool) error + setManagerDeployment func(string, string) error + loadNode func(string) (StoredNode, error) + insertNode func(NewNode) (StoredNode, error) + updateNode func(string, NodeLimits) error + saveSelection func(SelectionRecord) error + recordConfigurationMetadata func(json.RawMessage) error + retainGeneration func() error + collectGenerations func() error + recordAudit func(string, string) error +} + +func (f *fakeDeploymentTx) LoadDeployment() (Record, error) { + if f.loadDeployment == nil { + unexpected(f.t, "LoadDeployment") + } + return f.loadDeployment() +} + +func (f *fakeDeploymentTx) LoadSnapshot() (Snapshot, error) { + if f.loadSnapshot == nil { + unexpected(f.t, "LoadSnapshot") + } + return f.loadSnapshot() +} + +func (f *fakeDeploymentTx) CountResources() (Resources, error) { + if f.countResources == nil { + unexpected(f.t, "CountResources") + } + return f.countResources() +} + +func (f *fakeDeploymentTx) ClaimInstallation(installationID string) error { + if f.claimInstallation == nil { + unexpected(f.t, "ClaimInstallation") + } + return f.claimInstallation(installationID) +} + +func (f *fakeDeploymentTx) SetProcessDeployment(installationID, backendFingerprint string, admissionPaused bool) error { + if f.setProcessDeployment == nil { + unexpected(f.t, "SetProcessDeployment") + } + return f.setProcessDeployment(installationID, backendFingerprint, admissionPaused) +} + +func (f *fakeDeploymentTx) SetManagerDeployment(provider, localNodeID string) error { + if f.setManagerDeployment == nil { + unexpected(f.t, "SetManagerDeployment") + } + return f.setManagerDeployment(provider, localNodeID) +} + +func (f *fakeDeploymentTx) LoadNode(id string) (StoredNode, error) { + if f.loadNode == nil { + unexpected(f.t, "LoadNode") + } + return f.loadNode(id) +} + +func (f *fakeDeploymentTx) InsertNode(node NewNode) (StoredNode, error) { + if f.insertNode == nil { + unexpected(f.t, "InsertNode") + } + return f.insertNode(node) +} + +func (f *fakeDeploymentTx) UpdateNode(id string, limits NodeLimits) error { + if f.updateNode == nil { + unexpected(f.t, "UpdateNode") + } + return f.updateNode(id, limits) +} + +func (f *fakeDeploymentTx) SaveSelection(selection SelectionRecord) error { + if f.saveSelection == nil { + unexpected(f.t, "SaveSelection") + } + return f.saveSelection(selection) +} + +func (f *fakeDeploymentTx) RecordConfigurationMetadata(metadata json.RawMessage) error { + if f.recordConfigurationMetadata == nil { + unexpected(f.t, "RecordConfigurationMetadata") + } + return f.recordConfigurationMetadata(metadata) +} + +func (f *fakeDeploymentTx) RetainGeneration() error { + if f.retainGeneration == nil { + unexpected(f.t, "RetainGeneration") + } + return f.retainGeneration() +} + +func (f *fakeDeploymentTx) CollectGenerations() error { + if f.collectGenerations == nil { + unexpected(f.t, "CollectGenerations") + } + return f.collectGenerations() +} + +func (f *fakeDeploymentTx) RecordAudit(action, installationID string) error { + if f.recordAudit == nil { + unexpected(f.t, "RecordAudit") + } + return f.recordAudit(action, installationID) +} diff --git a/services/core/internal/deployment/health.go b/services/core/internal/deployment/health.go new file mode 100644 index 000000000..e8e37ffca --- /dev/null +++ b/services/core/internal/deployment/health.go @@ -0,0 +1,94 @@ +package deployment + +import ( + "math" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// normalizeHealth validates a node's reported health. A diagnostic explains +// unreadiness only: unknown values, including arbitrary text, become +// provider_unavailable, a ready provider has none and empty stays empty. +func normalizeHealth(health NodeHealth) (NodeHealth, error) { + health.Diagnostic = sandbox.NormalizeNodeDiagnostic(health.Diagnostic) + if health.ProviderReady { + health.Diagnostic = "" + } + for _, value := range []*int64{health.CPUCount, health.AvailableMemoryBytes, health.AvailableDiskBytes} { + if value != nil && *value < 0 { + return health, ErrInvalidInput + } + } + if err := validateHost(health.Host); err != nil { + return health, err + } + return health, nil +} + +func validateHost(host *NodeHost) error { + if host == nil { + return nil + } + if host.ObservedAt == nil || host.ObservedAt.IsZero() || host.ObservedAt.Year() < 1970 || host.ObservedAt.Year() > 9999 { + return ErrInvalidInput + } + for _, value := range []*float64{host.CPUUtilization, host.EffectiveCPUCores} { + if value != nil && (math.IsNaN(*value) || math.IsInf(*value, 0) || *value < 0) { + return ErrInvalidInput + } + } + if host.CPUUtilization != nil && *host.CPUUtilization > 1 || host.EffectiveCPUCores != nil && *host.EffectiveCPUCores == 0 { + return ErrInvalidInput + } + for _, value := range []*int64{host.TotalMemoryBytes, host.AvailableMemoryBytes, host.AvailableDiskBytes} { + if value != nil && (*value < 0 || *value > 1<<53-1) { + return ErrInvalidInput + } + } + if host.TotalMemoryBytes != nil && (*host.TotalMemoryBytes == 0 || host.AvailableMemoryBytes != nil && *host.AvailableMemoryBytes > *host.TotalMemoryBytes) { + return ErrInvalidInput + } + return nil +} + +// historyPoints fills the window with one point per bucket; a bucket without +// samples has only its start. Neither reads nor offline nodes fill gaps. +func historyPoints(window coremetrics.Range, samples []HostHistoryPoint) []HostHistoryPoint { + byStart := make(map[time.Time]HostHistoryPoint, len(samples)) + for _, sample := range samples { + sample.Start = sample.Start.UTC() + byStart[sample.Start] = sample + } + points := make([]HostHistoryPoint, 0) + step := time.Duration(window.ResolutionSeconds) * time.Second + for start := window.Start; start.Before(window.End); start = start.Add(step) { + point, ok := byStart[start] + if !ok { + point.Start = start + } + points = append(points, point) + } + return points +} + +// nodeRollout reports a node's preparation of the target generation. +func nodeRollout(n NodeRecord) NodeRollout { + out := NodeRollout{State: "unknown", ReadyGeneration: n.ReadyGeneration} + if !n.Online { + return out + } + if n.ProtocolVersion == 1 && n.DeploymentGeneration != n.TargetGeneration { + out.State = "update_required" + return out + } + switch n.TargetState { + case "ready", "preparing", "failed": + out.State = n.TargetState + } + if out.State == "failed" && n.TargetDiagnostic != "" { + out.Diagnostic = sandbox.NormalizeNodeDiagnostic(n.TargetDiagnostic) + } + return out +} diff --git a/services/core/internal/deployment/node.go b/services/core/internal/deployment/node.go new file mode 100644 index 000000000..2509363da --- /dev/null +++ b/services/core/internal/deployment/node.go @@ -0,0 +1,131 @@ +package deployment + +import ( + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +type NodeIdentity struct { + SpecificationDigest string `json:"specification_digest"` + DeploymentGeneration uint64 `json:"deployment_generation"` + NodeID string `json:"node_id"` + InstallationID string `json:"installation_id"` + Provider string `json:"provider"` + BackendFingerprint string `json:"-"` + MaxActive int `json:"max_active"` + MaxRetained int `json:"max_retained"` +} + +// EnrollmentToken is an issued one-use node enrollment token. ID is a public, +// non-secret handle that never authenticates; the node the token registers +// reports it as enrollment_id. +type EnrollmentToken struct { + Token string + ExpiresAt time.Time + ID string +} + +type Capacity struct { + MaxActive int `json:"max_active"` + MaxRetained int `json:"max_retained"` +} + +type Enrollment struct { + SpecificationDigest string `json:"specification_digest"` + DeploymentGeneration uint64 `json:"deployment_generation"` + NodeID string `json:"node_id"` + Credential string `json:"credential"` + Name string `json:"name"` + Provider string `json:"provider"` + BackendFingerprint string `json:"backend_fingerprint"` + // The Core origin this node stores and connects to, such as https://core.example. It must equal the installation public URL; otherwise enrollment gets 409 sandbox_node_address_mismatch and the token stays unused. + CoreURL string `json:"core_url"` +} + +type NodeHealth struct { + Host *NodeHost `json:"-"` + // Fixed reason for the last reported unreadiness; absent while the provider is ready. Clients treat an unknown value as provider_unavailable. + Diagnostic string `json:"diagnostic,omitempty" enums:"provider_unavailable,docker_unavailable,docker_limits_unsupported,runtime_download_failed,runtime_image_unavailable,kvm_unavailable,microsandbox_artifacts_unavailable,capacity_insufficient"` + ProviderReady bool `json:"provider_ready"` + CPUCount *int64 `json:"cpu_count"` + AvailableMemoryBytes *int64 `json:"available_memory_bytes"` + AvailableDiskBytes *int64 `json:"available_disk_bytes"` +} + +type Node struct { + Rollout NodeRollout `json:"rollout"` + NodeHealth + Running int64 `json:"running"` + Snapshots int64 `json:"snapshots"` + ID string `json:"id"` + Name string `json:"name"` + Provider string `json:"provider"` + Online bool `json:"online"` + LastSeenAt *time.Time `json:"last_seen_at"` + MaxActive int `json:"max_active"` + MaxRetained int `json:"max_retained"` + Active int64 `json:"active"` + Reserved int64 `json:"reserved"` + Retained int64 `json:"retained"` + CleanupPending int64 `json:"cleanup_pending"` + CreatedAt time.Time `json:"created_at"` + // The Core address this node enrolled with. A node whose address differs + // from the installation public URL receives no new sandboxes; re-add it. + CoreURL string `json:"core_url"` + // The enrollment_id of the command that registered this node (POST /core/v1/sandbox/enrollment-tokens); null for nodes enrolled before Core recorded it. + EnrollmentID *string `json:"enrollment_id" extensions:"x-nullable"` +} + +type NodeUpdate struct { + Name string `json:"name"` + MaxActive int `json:"max_active"` + // Docker never suspends, so Core replaces this with max_active; microsandbox uses both limits. + MaxRetained int `json:"max_retained"` +} + +// NodeStatus reports only the authenticated node's Core-owned presence. +type NodeStatus struct { + NodeIdentity + Connected bool `json:"connected"` + ProviderReady bool `json:"provider_ready"` +} + +type NodeConfiguration struct { + MaxActive int `json:"max_active"` + MaxRetained int `json:"max_retained"` + InstallationID string `json:"installation_id"` + Provider string `json:"provider"` + CoreURL string `json:"core_url"` + Generation uint64 `json:"generation"` + Specification sandbox.DeploymentSpec `json:"specification"` + SpecificationDigest string `json:"specification_digest"` +} + +// NodeHost is deployment telemetry, not sandbox capacity authority. +type NodeHost struct { + EffectiveCPUCores *float64 `json:"effective_cpu_cores" extensions:"x-nullable"` + CPUUtilization *float64 `json:"cpu_utilization" extensions:"x-nullable"` + TotalMemoryBytes *int64 `json:"total_memory_bytes" extensions:"x-nullable"` + AvailableMemoryBytes *int64 `json:"available_memory_bytes" extensions:"x-nullable"` + AvailableDiskBytes *int64 `json:"available_disk_bytes" extensions:"x-nullable"` + ObservedAt *time.Time `json:"observed_at" extensions:"x-nullable"` +} + +type HostHistoryPoint struct { + Start time.Time `json:"start"` + CPUUtilizationMax *float64 `json:"cpu_utilization_max" extensions:"x-nullable"` + MemoryUsedBytesMax *int64 `json:"memory_used_bytes_max" extensions:"x-nullable"` + AvailableDiskBytesMin *int64 `json:"available_disk_bytes_min" extensions:"x-nullable"` +} + +type HostHistory struct { + ResolutionSeconds int64 `json:"resolution_seconds"` + Points []HostHistoryPoint `json:"points"` +} + +type NodeDetail struct { + Node + Host NodeHost `json:"host"` + History HostHistory `json:"history"` +} diff --git a/services/core/internal/deployment/nodes.go b/services/core/internal/deployment/nodes.go new file mode 100644 index 000000000..4feef8603 --- /dev/null +++ b/services/core/internal/deployment/nodes.go @@ -0,0 +1,643 @@ +package deployment + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "math" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/google/uuid" +) + +// ListNodes returns the installation's nodes. +func (s *Service) ListNodes(ctx context.Context) ([]Node, error) { + records, err := s.reader.Nodes(ctx) + if err != nil { + return nil, err + } + out := make([]Node, 0, len(records)) + for _, n := range records { + value, err := s.node(n) + if err != nil { + return nil, err + } + out = append(out, value) + } + return out, nil +} + +func (s *Service) node(n NodeRecord) (Node, error) { + retained, err := s.registry.RetainedLimit(n.Provider, n.MaxActive, n.MaxRetained) + if err != nil { + return Node{}, err + } + health := n.Health + health.Host = nil + health.ProviderReady = n.Online && n.ServingReady + return Node{Rollout: nodeRollout(n), NodeHealth: health, Running: n.Running, Snapshots: n.Snapshots, ID: n.ID, Name: n.Name, CoreURL: n.CoreURL, EnrollmentID: n.EnrollmentID, Provider: n.Provider, Online: n.Online, LastSeenAt: n.LastSeenAt, MaxActive: n.MaxActive, MaxRetained: retained, Active: n.Active, Reserved: n.Reserved, Retained: n.Retained, CleanupPending: n.CleanupPending, CreatedAt: n.CreatedAt}, nil +} + +// NodeDetail returns one node with its last host observation and its host +// history over the named window: 1h, 6h or 24h. +func (s *Service) NodeDetail(ctx context.Context, id, window string) (NodeDetail, error) { + nodeID, err := parseID(id) + if err != nil { + return NodeDetail{}, err + } + if window != "1h" && window != "6h" && window != "24h" { + return NodeDetail{}, ErrInvalidInput + } + span, err := coremetrics.Window(time.Now(), window) + if err != nil { + return NodeDetail{}, ErrInvalidInput + } + record, samples, err := s.reader.NodeHistory(ctx, nodeID, span) + if err != nil { + return NodeDetail{}, err + } + node, err := s.node(record) + if err != nil { + return NodeDetail{}, err + } + result := NodeDetail{Node: node, History: HostHistory{ResolutionSeconds: span.ResolutionSeconds, Points: historyPoints(span, samples)}} + if record.Health.Host != nil { + result.Host = *record.Health.Host + } + return result, nil +} + +// withManager runs apply over an initialized deployment. +func (s *Service) withManager(ctx context.Context, apply func(NodeTx, Record) error) error { + return s.storage.WithNodes(ctx, func(tx NodeTx) error { + d, err := tx.LoadDeployment() + if err != nil { + return err + } + if !initialized(d) { + return ErrNodeUnavailable + } + return apply(tx, d) + }) +} + +// UpdateNode renames a node and changes its capacity. +func (s *Service) UpdateNode(ctx context.Context, id string, update NodeUpdate) error { + // The retained limit depends on the provider, so the transaction checks it. + if err := validateNode(update.Name, update.MaxActive, update.MaxActive); err != nil { + return err + } + nodeID, err := parseID(id) + if err != nil { + return err + } + return s.withManager(ctx, func(tx NodeTx, d Record) error { + retained, err := s.registry.RetainedLimit(d.Provider, update.MaxActive, update.MaxRetained) + if err != nil { + return err + } + if err := validateNode(update.Name, update.MaxActive, retained); err != nil { + return err + } + n, err := tx.LoadNode(nodeID) + if err != nil { + return err + } + if n.InstallationID != d.InstallationID { + return ErrNotFound + } + return tx.UpdateNode(nodeID, NodeLimits{Name: update.Name, MaxActive: update.MaxActive, MaxRetained: retained}) + }) +} + +// RemoveNode removes a node that retains no resources and is not the local node. +func (s *Service) RemoveNode(ctx context.Context, id string) error { + nodeID, err := parseID(id) + if err != nil { + return err + } + return s.withManager(ctx, func(tx NodeTx, d Record) error { + nodes, err := tx.ListNodes() + if err != nil { + return err + } + for _, n := range nodes { + if n.ID != nodeID { + continue + } + if n.Retained != 0 || n.CleanupPending != 0 { + return ErrNodeInUse + } + if d.LocalNodeID == nodeID { + return ErrLocalNodeConfigured + } + return tx.RemoveNode(nodeID) + } + return ErrNotFound + }) +} + +// CreateEnrollment issues a one-use enrollment token. Its ID is a public, +// non-secret handle: the node the token registers reports it as enrollment_id. +func (s *Service) CreateEnrollment(ctx context.Context, capacity Capacity) (EnrollmentToken, error) { + // The retained limit depends on the provider, so the transaction checks it. + if err := validateNode("enrollment", capacity.MaxActive, capacity.MaxActive); err != nil { + return EnrollmentToken{}, err + } + var bytes [32]byte + if _, err := rand.Read(bytes[:]); err != nil { + return EnrollmentToken{}, err + } + result := EnrollmentToken{Token: hex.EncodeToString(bytes[:]), ID: uuid.NewString()} + err := s.withManager(ctx, func(tx NodeTx, d Record) error { + retained, err := s.registry.RetainedLimit(d.Provider, capacity.MaxActive, capacity.MaxRetained) + if err != nil { + return err + } + if err := validateNode("enrollment", capacity.MaxActive, retained); err != nil { + return err + } + if d.Reset != nil { + return ErrResetInProgress + } + if d.Mode != "nodes" || d.AdmissionPaused { + return ErrConflict + } + if _, err := s.specification(d); err != nil { + return ErrConflict + } + result.ExpiresAt, err = tx.CreateEnrollment(NewEnrollment{ID: result.ID, TokenDigest: tokenDigest(result.Token), InstallationID: d.InstallationID, MaxActive: capacity.MaxActive, MaxRetained: retained}) + return err + }) + if err != nil { + return EnrollmentToken{}, err + } + return result, nil +} + +// Enroll registers a node with a one-use enrollment token. +func (s *Service) Enroll(ctx context.Context, token string, input Enrollment) (NodeIdentity, error) { + nodeID, err := parseID(input.NodeID) + if err != nil || !validNodeCredential(input.Credential) || !validDigest(input.BackendFingerprint) { + return NodeIdentity{}, ErrInvalidInput + } + if err := validateNode(input.Name, 1, 1); err != nil { + return NodeIdentity{}, err + } + var result NodeIdentity + err = s.storage.WithNodes(ctx, func(tx NodeTx) error { + d, err := tx.LoadDeployment() + if err != nil { + return err + } + receipt, err := tx.LoadEnrollment(tokenDigest(token)) + if errors.Is(err, ErrNotFound) { + return ErrNodeCredential + } + if err != nil { + return ErrNodeUnavailable + } + if receipt.Consumed || !receipt.ExpiresAt.After(time.Now()) { + return ErrNodeCredential + } + if d.InstallationID != "" && receipt.InstallationID != d.InstallationID { + return ErrNodeCredential + } + if !initialized(d) { + return ErrNodeUnavailable + } + if d.Reset != nil { + return ErrResetInProgress + } + if d.Mode != "nodes" || d.AdmissionPaused || input.Provider != d.Provider { + return ErrInvalidInput + } + spec, err := s.specification(d) + if err != nil || input.DeploymentGeneration != d.Generation || input.SpecificationDigest != spec.Digest(d.Provider) { + return ErrSpecificationMismatch + } + // The node must use the address Core advertises now. It read that address + // from its configuration, but the public URL may have changed since, or an + // operator may have registered by hand with another origin. + if input.CoreURL != s.publicURL { + return ErrNodeAddressMismatch + } + retained, err := s.registry.RetainedLimit(d.Provider, receipt.MaxActive, receipt.MaxRetained) + if err != nil { + return err + } + n, err := tx.InsertNode(NewNode{ID: nodeID, InstallationID: d.InstallationID, Name: input.Name, BackendFingerprint: input.BackendFingerprint, CredentialDigest: tokenDigest(input.Credential), MaxActive: receipt.MaxActive, MaxRetained: retained, SpecificationDigest: input.SpecificationDigest, DeploymentGeneration: input.DeploymentGeneration, CoreURL: input.CoreURL, EnrollmentID: receipt.ID}) + if err != nil { + return err + } + consumed, err := tx.ConsumeEnrollment(tokenDigest(token), nodeID) + if err != nil { + return err + } + if !consumed { + return ErrNodeCredential + } + result, err = s.identity(n, d.Provider) + return err + }) + if err != nil { + return NodeIdentity{}, err + } + return result, nil +} + +func (s *Service) identity(n StoredNode, provider string) (NodeIdentity, error) { + retained, err := s.registry.RetainedLimit(provider, n.MaxActive, n.MaxRetained) + if err != nil { + return NodeIdentity{}, err + } + return NodeIdentity{SpecificationDigest: n.SpecificationDigest, DeploymentGeneration: n.DeploymentGeneration, NodeID: n.ID, InstallationID: n.InstallationID, Provider: provider, BackendFingerprint: n.BackendFingerprint, MaxActive: n.MaxActive, MaxRetained: retained}, nil +} + +// AuthenticateNode checks a node credential and returns the node's identity. +func (s *Service) AuthenticateNode(ctx context.Context, nodeID, credential string) (NodeIdentity, error) { + id, err := parseID(nodeID) + if err != nil { + return NodeIdentity{}, ErrNodeCredential + } + var result NodeIdentity + err = s.reader.ReadNodes(ctx, func(tx NodeReads) error { + n, err := tx.LoadNode(id) + if errors.Is(err, ErrNotFound) { + return ErrNodeCredential + } + if err != nil { + return ErrNodeUnavailable + } + if n.CredentialDigest != tokenDigest(credential) { + return ErrNodeCredential + } + d, err := tx.LoadDeployment() + if err != nil { + return ErrNodeUnavailable + } + if n.InstallationID != d.InstallationID || d.Mode != "nodes" { + return ErrNodeCredential + } + if err := s.checkEnrollmentIdentity(tx, d, n); err != nil { + return err + } + // Reset retires nodes before another backend lineage can be selected. + // Enrollment generation and digest remain immutable identity history; the + // current target and per-generation readiness do not replace that history. + result, err = s.identity(n, d.Provider) + return err + }) + if err != nil { + return NodeIdentity{}, err + } + return result, nil +} + +// checkEnrollmentIdentity keeps the enrollment identity valid after collection +// of its old specification. Whenever that specification is still +// authoritative, its exact digest must agree. +func (s *Service) checkEnrollmentIdentity(tx NodeReads, d Record, n StoredNode) error { + if n.DeploymentGeneration == 0 || n.DeploymentGeneration > d.Generation || !validDigest(n.SpecificationDigest) { + return ErrSpecificationMismatch + } + spec, err := s.generationSpecification(tx, d, n.DeploymentGeneration) + if errors.Is(err, ErrNotFound) && n.DeploymentGeneration < d.Generation { + return nil + } + if err != nil || spec.Digest(d.Provider) != n.SpecificationDigest { + return ErrSpecificationMismatch + } + return nil +} + +// generationSpecification returns a generation nodes may prepare when its +// provider still accepts it. +func (s *Service) generationSpecification(tx NodeReads, d Record, generation uint64) (sandbox.DeploymentSpec, error) { + var spec sandbox.DeploymentSpec + if !validGeneration(generation) { + return spec, ErrInvalidInput + } + row, err := tx.LoadGenerationSpecification(generation) + if err != nil { + return spec, err + } + if row.Provider != d.Provider || json.Unmarshal(row.Specification, &spec) != nil || s.registry.ValidateSpecification(d.Provider, spec) != nil { + return spec, ErrSpecificationMismatch + } + return spec, nil +} + +// NodeStatus reports the authenticated node's Core-owned presence. +func (s *Service) NodeStatus(ctx context.Context, nodeID, credential string) (NodeStatus, error) { + identity, err := s.AuthenticateNode(ctx, nodeID, credential) + if err != nil { + return NodeStatus{}, err + } + nodes, err := s.reader.Nodes(ctx) + if err != nil { + return NodeStatus{}, err + } + for _, n := range nodes { + if n.ID == identity.NodeID { + return NodeStatus{NodeIdentity: identity, Connected: n.Online, ProviderReady: n.Online && n.ServingReady}, nil + } + } + return NodeStatus{}, ErrNodeCredential +} + +// NodeConfiguration is the read-only bootstrap of an enrollment token or node +// credential. It never consumes tokens or reveals cloud credentials, and it +// authenticates the credential before reporting any deployment state. The +// deployment lock keeps authentication and the returned generation consistent. +// +// A nonzero generation recovers that exact generation, restricted to this +// node's current target, serving pin and unreleased ownership. It is never a +// general history read. +func (s *Service) NodeConfiguration(ctx context.Context, nodeID, token string, generation uint64) (NodeConfiguration, error) { + var result NodeConfiguration + if generation > math.MaxInt64 { + return result, ErrInvalidInput + } + err := s.storage.WithNodes(ctx, func(tx NodeTx) error { + d, err := tx.LoadDeployment() + if err != nil { + return err + } + var node *StoredNode + var installation string + var active, retained int + if nodeID == "" { + r, err := tx.LoadEnrollment(tokenDigest(token)) + if err != nil || r.Consumed || !r.ExpiresAt.After(time.Now()) { + return ErrNodeCredential + } + installation, active, retained = r.InstallationID, r.MaxActive, r.MaxRetained + } else { + id, err := parseID(nodeID) + if err != nil { + return ErrNodeCredential + } + n, err := tx.LoadNode(id) + if err != nil || n.CredentialDigest != tokenDigest(token) { + return ErrNodeCredential + } + node, installation, active, retained = &n, n.InstallationID, n.MaxActive, n.MaxRetained + } + // A claimed installation rejects foreign credentials identically before + // and after initialization. + if d.InstallationID != "" && installation != d.InstallationID { + return ErrNodeCredential + } + if !initialized(d) { + return ErrNodeUnavailable + } + if node == nil && d.Reset != nil { + return ErrResetInProgress + } + if d.Mode != "nodes" { + return ErrConflict + } + if node != nil { + if err := s.checkEnrollmentIdentity(tx, d, *node); err != nil { + return err + } + } + selected := d.Generation + if generation != 0 { + if node == nil { + return ErrNodeCredential + } + selected = generation + kept, err := tx.GenerationKept(node.ID, generation) + if err != nil { + return err + } + if !kept { + return ErrSpecificationMismatch + } + } + spec, err := s.generationSpecification(tx, d, selected) + if err != nil { + return err + } + if node == nil && d.AdmissionPaused { + return ErrConflict + } + limit, err := s.registry.RetainedLimit(d.Provider, active, retained) + if err != nil { + return err + } + result = NodeConfiguration{MaxActive: active, MaxRetained: limit, InstallationID: d.InstallationID, Provider: d.Provider, CoreURL: s.publicURL, Generation: selected, Specification: spec, SpecificationDigest: spec.Digest(d.Provider)} + return nil + }) + if err != nil { + return NodeConfiguration{}, err + } + return result, nil +} + +// connection returns the node of a current connection of this owner epoch. +func (s *Service) connection(tx NodeReads, d Record, nodeID, connectionID string, epoch uint64) (StoredNode, error) { + id, err := parseID(nodeID) + if err != nil { + return StoredNode{}, ErrNodeCredential + } + n, err := tx.LoadNode(id) + if errors.Is(err, ErrNotFound) { + return StoredNode{}, ErrNodeCredential + } + if err != nil { + return StoredNode{}, err + } + if n.ConnectionID != connectionID || n.ConnectedEpoch != epoch || epoch == 0 || epoch > math.MaxInt64 || d.OwnerEpoch != epoch || n.InstallationID != d.InstallationID || d.Mode != "nodes" { + return StoredNode{}, ErrNodeCredential + } + return n, nil +} + +// NodeRetention answers which reported generations a node keeps, under the +// same serialization as pin promotion and placement, so a dropped generation +// cannot later gain fresh ownership. It deletes the status of the rest. +func (s *Service) NodeRetention(ctx context.Context, nodeID, connectionID string, epoch uint64, refs []sandbox.GenerationReference) (sandbox.NodeDeployment, []sandbox.GenerationRetention, error) { + var deployment sandbox.NodeDeployment + if len(refs) > maxReportedGenerations { + return deployment, nil, ErrInvalidInput + } + grants := make([]sandbox.GenerationRetention, 0, len(refs)) + err := s.storage.WithNodes(ctx, func(tx NodeTx) error { + d, err := tx.LoadDeployment() + if err != nil { + return err + } + n, err := s.connection(tx, d, nodeID, connectionID, epoch) + if err != nil { + return err + } + deployment.Generation = d.Generation + spec, err := s.generationSpecification(tx, d, d.Generation) + if err != nil { + return err + } + deployment.SpecificationDigest = spec.Digest(d.Provider) + if n.ReadyGeneration != nil { + pin := *n.ReadyGeneration + deployment.ServingGeneration = &pin + } + seen := map[uint64]bool{} + for _, ref := range refs { + if !validGeneration(ref.Generation) || !validDigest(ref.SpecificationDigest) || seen[ref.Generation] { + return ErrInvalidInput + } + seen[ref.Generation] = true + kept, err := tx.GenerationKept(n.ID, ref.Generation) + if err != nil { + return err + } + if kept { + spec, err := s.generationSpecification(tx, d, ref.Generation) + if err != nil { + return err + } + if spec.Digest(d.Provider) != ref.SpecificationDigest { + return ErrSpecificationMismatch + } + } else if err := tx.DeleteGenerationStatus(n.ID, ref.Generation); err != nil { + return err + } + grants = append(grants, sandbox.GenerationRetention{GenerationReference: ref, Keep: kept}) + } + return nil + }) + if err != nil { + return sandbox.NodeDeployment{}, nil, err + } + return deployment, grants, nil +} + +// Heartbeat records a protocol 1 node's health, which reports readiness of its +// enrollment generation only. +func (s *Service) Heartbeat(ctx context.Context, nodeID, connectionID string, epoch uint64, health NodeHealth) error { + return s.heartbeat(ctx, nodeID, connectionID, epoch, health, nil, 1) +} + +// HeartbeatGenerations records a node's health and the preparation state of +// the generations it reports. +func (s *Service) HeartbeatGenerations(ctx context.Context, nodeID, connectionID string, epoch uint64, health NodeHealth, statuses []sandbox.GenerationStatus) error { + if len(statuses) > maxReportedGenerations { + return ErrInvalidInput + } + return s.heartbeat(ctx, nodeID, connectionID, epoch, health, statuses, 2) +} + +func (s *Service) heartbeat(ctx context.Context, nodeID, connectionID string, epoch uint64, health NodeHealth, statuses []sandbox.GenerationStatus, protocol int) error { + id, err := parseID(nodeID) + if err != nil { + return err + } + connection, err := parseID(connectionID) + if err != nil { + return err + } + health, err = normalizeHealth(health) + if err != nil { + return err + } + return s.storage.WithNodes(ctx, func(tx NodeTx) error { + d, err := tx.LoadDeployment() + if err != nil { + return err + } + n, err := s.connection(tx, d, id, connection, epoch) + if err != nil { + return err + } + current, err := tx.HeartbeatNode(Heartbeat{NodeID: id, ConnectionID: connection, Epoch: epoch, Health: health}) + if err != nil { + return err + } + if !current { + return ErrNodeCredential + } + if protocol == 1 { + state := "failed" + if health.ProviderReady { + state = "ready" + } + statuses = []sandbox.GenerationStatus{{Generation: n.DeploymentGeneration, SpecificationDigest: n.SpecificationDigest, State: state, Diagnostic: health.Diagnostic}} + } + return s.recordGenerations(tx, d, n, statuses, protocol) + }) +} + +func (s *Service) recordGenerations(tx NodeTx, d Record, n StoredNode, statuses []sandbox.GenerationStatus, protocol int) error { + seen := map[uint64]bool{} + for _, status := range statuses { + if !validGeneration(status.Generation) || !validDigest(status.SpecificationDigest) || seen[status.Generation] || (status.State != "ready" && status.State != "preparing" && status.State != "failed") || status.State == "ready" && status.Diagnostic != "" { + return ErrInvalidInput + } + seen[status.Generation] = true + kept, err := tx.GenerationKept(n.ID, status.Generation) + if err != nil { + return err + } + // A sparse report may race collection of a skipped target. It creates no + // readiness or pin; only the later correlated retention reply can drop it. + if !kept { + continue + } + spec, err := s.generationSpecification(tx, d, status.Generation) + if err != nil { + return err + } + if spec.Digest(d.Provider) != status.SpecificationDigest { + return ErrSpecificationMismatch + } + if err := tx.UpsertGenerationStatus(GenerationStatusRecord{NodeID: n.ID, ConnectionID: n.ConnectionID, Generation: status.Generation, SpecificationDigest: status.SpecificationDigest, OwnerEpoch: d.OwnerEpoch, State: status.State, Diagnostic: sandbox.NormalizeNodeDiagnostic(status.Diagnostic)}); err != nil { + return err + } + if status.State == "ready" && status.Generation == d.Generation { + if err := tx.PromoteServingGeneration(n.ID, status.Generation); err != nil { + return err + } + } + } + return tx.RefreshServingReadiness(n.ID, protocol) +} + +// ConnectNode records a node connection for the owner epoch. +func (s *Service) ConnectNode(ctx context.Context, nodeID, connectionID string, epoch uint64) error { + id, err := parseID(nodeID) + if err != nil { + return err + } + connection, err := parseID(connectionID) + if err != nil { + return err + } + connected, err := s.storage.ConnectNode(ctx, id, connection, epoch) + if err != nil { + return err + } + if !connected { + return ErrNodeCredential + } + return nil +} + +// DisconnectNode clears the node's connection when it is still this one. +func (s *Service) DisconnectNode(ctx context.Context, nodeID, connectionID string, epoch uint64) error { + id, err := parseID(nodeID) + if err != nil { + return err + } + connection, err := parseID(connectionID) + if err != nil { + return err + } + return s.storage.DisconnectNode(ctx, id, connection, epoch) +} diff --git a/services/core/internal/deployment/public_url.go b/services/core/internal/deployment/public_url.go new file mode 100644 index 000000000..6d9dead16 --- /dev/null +++ b/services/core/internal/deployment/public_url.go @@ -0,0 +1,62 @@ +package deployment + +import ( + "net" + "net/url" + "strconv" + "strings" +) + +// ValidateCoreURL accepts a canonical public origin, never a path or +// credential. Plain HTTP is reserved for explicit loopback development hosts. +// OAC_PUBLIC_URL must pass it. +func ValidateCoreURL(value string) error { + u, err := url.Parse(value) + if err != nil || u.Hostname() == "" || u.User != nil || u.Path != "" || u.RawPath != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || u.RawFragment != "" || u.Opaque != "" || u.String() != value || u.Host != strings.ToLower(u.Host) { + return ErrInvalidInput + } + if strings.ContainsAny(u.Host, "\\% \t\r\n") || strings.HasSuffix(u.Host, ":") { + return ErrInvalidInput + } + if port := u.Port(); port != "" { + n, err := strconv.Atoi(port) + if err != nil || n < 1 || n > 65535 || strconv.Itoa(n) != port { + return ErrInvalidInput + } + } + loopback := u.Hostname() == "localhost" + if ip := net.ParseIP(u.Hostname()); ip != nil { + loopback = ip.IsLoopback() + } else { + if len(u.Hostname()) > 253 || strings.ContainsAny(u.Host, "[]") { + return ErrInvalidInput + } + for _, label := range strings.Split(u.Hostname(), ".") { + if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { + return ErrInvalidInput + } + for _, char := range label { + if (char < 'a' || char > 'z') && (char < '0' || char > '9') && char != '-' { + return ErrInvalidInput + } + } + } + } + if u.Scheme != "https" && !(u.Scheme == "http" && loopback) { + return ErrInvalidInput + } + return nil +} + +// LoopbackOrigin reports whether a validated origin names a loopback host, which +// nothing outside the Core host can reach. +func LoopbackOrigin(value string) bool { + u, err := url.Parse(value) + if err != nil { + return false + } + if ip := net.ParseIP(u.Hostname()); ip != nil { + return ip.IsLoopback() + } + return u.Hostname() == "localhost" +} diff --git a/services/core/internal/deployment/rules.go b/services/core/internal/deployment/rules.go new file mode 100644 index 000000000..91717deb0 --- /dev/null +++ b/services/core/internal/deployment/rules.go @@ -0,0 +1,82 @@ +package deployment + +import ( + "crypto/sha256" + "encoding/hex" + "math" + "strings" + + "github.com/google/uuid" +) + +// Node capacity and name limits. +const ( + maxNodeName = 128 + maxNodeCapacity = 1000000 + // maxReportedGenerations bounds the generations one node report names. + maxReportedGenerations = 8 +) + +// parseID returns the canonical form of a nonzero UUID, and ErrInvalidInput +// for anything else. +func parseID(value string) (string, error) { + id, err := uuid.Parse(value) + if err != nil || id == uuid.Nil { + return "", ErrInvalidInput + } + return id.String(), nil +} + +// tokenDigest is the stored form of a node credential or enrollment token. +func tokenDigest(token string) string { + digest := sha256.Sum256([]byte(token)) + return hex.EncodeToString(digest[:]) +} + +// validDigest accepts a lowercase hexadecimal SHA-256 digest. +func validDigest(value string) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == 32 && hex.EncodeToString(decoded) == value +} + +// validGeneration accepts a generation a database row can hold. +func validGeneration(generation uint64) bool { + return generation != 0 && generation <= math.MaxInt64 +} + +// validateNode checks a node name and capacity. max_retained is at least +// max_active. +func validateNode(name string, active, retained int) error { + if strings.TrimSpace(name) == "" || len(name) > maxNodeName || strings.ContainsAny(name, "\x00\r\n") { + return &NodeValidationError{Code: "invalid_name", Param: "name", MaxLength: maxNodeName} + } + if active < 1 || active > maxNodeCapacity { + return &NodeValidationError{Code: "invalid_node_capacity", Param: "max_active"} + } + if retained < active || retained > maxNodeCapacity { + return &NodeValidationError{Code: "invalid_node_capacity", Param: "max_retained"} + } + return nil +} + +// validNodeCredential accepts a node credential as enrollment stores it. +func validNodeCredential(credential string) bool { + return len(credential) >= 32 && len(credential) <= 256 && !strings.ContainsAny(credential, " \t\r\n") +} + +// checkGeneration rejects a Web setup change whose expected generation or +// installation is not the deployment's. +func checkGeneration(d Record, installation string, generation uint64) error { + if d.Generation != generation { + return &GenerationStaleError{CurrentGeneration: d.Generation} + } + if !d.WebManaged || d.InstallationID != installation { + return ErrConflict + } + return nil +} + +// initialized reports whether an installation and a provider are selected. +func initialized(d Record) bool { + return d.InstallationID != "" && d.Provider != "" +} diff --git a/services/core/internal/deployment/rules_test.go b/services/core/internal/deployment/rules_test.go new file mode 100644 index 000000000..1812f531d --- /dev/null +++ b/services/core/internal/deployment/rules_test.go @@ -0,0 +1,281 @@ +package deployment + +import ( + "errors" + "math" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// deploy/install/test_install.py checks valid_core_origin against the same +// cases. +func TestValidateCoreURL(t *testing.T) { + for _, value := range []string{"https://core.example", "https://core.example:8443", "http://localhost:8091", "http://127.0.0.2:8091", "http://[::1]:8091", "https://[2001:db8::1]"} { + if err := ValidateCoreURL(value); err != nil { + t.Errorf("valid Core URL %q rejected: %v", value, err) + } + } + for _, value := range []string{ + "", "http://core.example", "http://core:8091", "http://host.localhost", "https://core.example/", "https://user:secret@core.example", + "https://core.example/path", "https://core.example?", "https://core.example?q=x", "https://core.example#x", "https://core.example#", + "https://CORE.example", "https://core.example:", "https://core.example:0", "https://core.example:65536", "https://core.example:0080", + "https://core.example\\evil", "https://[not-an-ip]", "https://-core.example", "https://core..example", "https://core_example", + "https://core.example.", "https://bücher.example", "https://core.example:0443", + } { + if err := ValidateCoreURL(value); !errors.Is(err, ErrInvalidInput) { + t.Errorf("invalid Core URL %q accepted: %v", value, err) + } + } +} + +func TestLoopbackOrigin(t *testing.T) { + for value, want := range map[string]bool{ + "http://localhost:8091": true, "http://127.0.0.1:8091": true, "http://127.0.0.2": true, "http://[::1]:8091": true, + "https://core.example": false, "https://[2001:db8::1]": false, "https://10.0.0.1": false, "": false, "http://host.localhost": false, + } { + if got := LoopbackOrigin(value); got != want { + t.Errorf("LoopbackOrigin(%q) = %v, want %v", value, got, want) + } + } +} + +func TestParseID(t *testing.T) { + id := uuid.New() + if got, err := parseID(strings.ToUpper(id.String())); err != nil || got != id.String() { + t.Fatalf("parseID returned %q, %v; want the canonical %q", got, err, id) + } + for _, value := range []string{"", "not-a-uuid", uuid.Nil.String()} { + if _, err := parseID(value); !errors.Is(err, ErrInvalidInput) { + t.Errorf("parseID(%q) = %v, want ErrInvalidInput", value, err) + } + } +} + +func TestDigestAndGenerationRules(t *testing.T) { + digest := tokenDigest("token") + if !validDigest(digest) { + t.Fatal("tokenDigest is not a valid digest") + } + for _, value := range []string{"", strings.ToUpper(digest), digest[:63], digest + "0", strings.Repeat("g", 64)} { + if validDigest(value) { + t.Errorf("validDigest(%q) accepted", value) + } + } + for generation, want := range map[uint64]bool{0: false, 1: true, math.MaxInt64: true, math.MaxInt64 + 1: false} { + if validGeneration(generation) != want { + t.Errorf("validGeneration(%d) != %v", generation, want) + } + } + for credential, want := range map[string]bool{ + strings.Repeat("n", 31): false, strings.Repeat("n", 32): true, strings.Repeat("n", 256): true, strings.Repeat("n", 257): false, + strings.Repeat("n", 32) + " ": false, strings.Repeat("n", 32) + "\t": false, strings.Repeat("n", 32) + "\n": false, + } { + if validNodeCredential(credential) != want { + t.Errorf("validNodeCredential(%d characters %q) != %v", len(credential), credential[len(credential)-1:], want) + } + } +} + +func TestValidateNode(t *testing.T) { + for _, c := range []struct { + name string + active, retained int + code, param string + }{ + {"node", 1, 1, "", ""}, + {strings.Repeat("n", 128), 1000000, 1000000, "", ""}, + // Node names keep their byte bound and limited control set; they do not + // use the stricter Project and key name validator. + {"node\tname", 1, 1000000, "", ""}, + {string([]byte{0xff}), 1, 1000000, "", ""}, + {"", 1, 1, "invalid_name", "name"}, + {"", 0, 0, "invalid_name", "name"}, + {strings.Repeat("界", 43), 1, 1, "invalid_name", "name"}, + {" ", 1, 1, "invalid_name", "name"}, + {strings.Repeat("n", 129), 1, 1, "invalid_name", "name"}, + {"a\nb", 1, 1, "invalid_name", "name"}, + {"a\rb", 1, 1, "invalid_name", "name"}, + {"a\x00b", 1, 1, "invalid_name", "name"}, + {"node", 0, 1, "invalid_node_capacity", "max_active"}, + {"node", 0, 0, "invalid_node_capacity", "max_active"}, + {"node", 1000001, 1000001, "invalid_node_capacity", "max_active"}, + {"node", 1000001, 8, "invalid_node_capacity", "max_active"}, + {"node", 2, 1, "invalid_node_capacity", "max_retained"}, + {"node", 1, 1000001, "invalid_node_capacity", "max_retained"}, + } { + err := validateNode(c.name, c.active, c.retained) + if c.code == "" { + if err != nil { + t.Errorf("validateNode(%q, %d, %d) = %v", c.name, c.active, c.retained, err) + } + continue + } + var validation *NodeValidationError + if !errors.As(err, &validation) || !errors.Is(err, ErrInvalidInput) || validation.Code != c.code || validation.Param != c.param || err.Error() != ErrInvalidInput.Error() { + t.Errorf("validateNode(%q, %d, %d) = %#v, want %s on %s", c.name, c.active, c.retained, err, c.code, c.param) + } + if c.code == "invalid_name" && validation != nil && validation.MaxLength != 128 { + t.Errorf("name rejection reports max length %d", validation.MaxLength) + } + } +} + +func TestCheckGeneration(t *testing.T) { + installation := uuid.NewString() + current := Record{InstallationID: installation, WebManaged: true, Generation: 3} + if err := checkGeneration(current, installation, 3); err != nil { + t.Fatal(err) + } + var stale *GenerationStaleError + if err := checkGeneration(current, installation, 2); !errors.As(err, &stale) || stale.CurrentGeneration != 3 || !errors.Is(err, ErrConflict) { + t.Fatalf("stale generation: %v", err) + } + process := current + process.WebManaged = false + for _, c := range []struct { + d Record + installation string + }{{process, installation}, {current, uuid.NewString()}} { + if err := checkGeneration(c.d, c.installation, 3); !errors.Is(err, ErrConflict) || errors.As(err, &stale) { + t.Errorf("checkGeneration(%+v, %s) = %v, want a plain conflict", c.d, c.installation, err) + } + } + if initialized(Record{InstallationID: installation}) || initialized(Record{Provider: "docker"}) || !initialized(Record{InstallationID: installation, Provider: "docker"}) { + t.Fatal("initialized needs both an installation and a provider") + } +} + +func TestTypedErrors(t *testing.T) { + if err := error(&ResetRequiredError{CurrentProvider: "docker", RequestedProvider: "e2b"}); !errors.Is(err, ErrConflict) { + t.Fatal("ResetRequiredError is not a conflict") + } + validation := &sandbox.ValidationError{Param: "resources.cpus", Message: "too many"} + err := configurationError(validation) + if !errors.Is(err, ErrInvalidInput) || err.Validation != validation || err.Message != "too many" { + t.Fatalf("configurationError = %#v", err) + } + if plain := configurationError(errors.New("rejected")); plain.Validation != nil || plain.Error() != "rejected" { + t.Fatalf("configurationError of a plain error = %#v", plain) + } + if got := string(configurationJSON(nil)); got != "{}" { + t.Fatalf("configurationJSON(nil) = %s", got) + } + if got := string(configurationJSON([]byte(`{"a":1}`))); got != `{"a":1}` { + t.Fatalf("configurationJSON kept %s", got) + } +} + +func TestNormalizeHealth(t *testing.T) { + negative, zero, one := int64(-1), float64(0), float64(1) + now := time.Now() + for _, c := range []struct { + name string + in NodeHealth + diagnostic string + invalid bool + }{ + {"ready clears the diagnostic", NodeHealth{ProviderReady: true, Diagnostic: sandbox.NodeProviderUnavailable}, "", false}, + {"unknown text becomes provider_unavailable", NodeHealth{Diagnostic: "disk on fire"}, sandbox.NodeProviderUnavailable, false}, + {"empty stays empty", NodeHealth{}, "", false}, + {"negative CPU count", NodeHealth{CPUCount: &negative}, "", true}, + {"negative memory", NodeHealth{AvailableMemoryBytes: &negative}, "", true}, + {"negative disk", NodeHealth{AvailableDiskBytes: &negative}, "", true}, + {"host without observation", NodeHealth{Host: &NodeHost{CPUUtilization: &one}}, "", true}, + {"valid host", NodeHealth{Host: &NodeHost{ObservedAt: &now, CPUUtilization: &one, EffectiveCPUCores: &one}}, "", false}, + {"zero effective cores", NodeHealth{Host: &NodeHost{ObservedAt: &now, EffectiveCPUCores: &zero}}, "", true}, + } { + got, err := normalizeHealth(c.in) + if c.invalid != errors.Is(err, ErrInvalidInput) || (!c.invalid && (err != nil || got.Diagnostic != c.diagnostic)) { + t.Errorf("%s: normalizeHealth = %+v, %v", c.name, got, err) + } + } +} + +func TestValidateHost(t *testing.T) { + now := time.Now() + early, late := time.Date(1969, 12, 31, 0, 0, 0, 0, time.UTC), time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC) + half, over, negative, nan, inf := 0.5, 1.5, -0.5, math.NaN(), math.Inf(1) + total, more, tooLarge, negativeBytes, zeroBytes := int64(100), int64(101), int64(1<<53), int64(-1), int64(0) + for _, c := range []struct { + name string + host *NodeHost + ok bool + }{ + {"absent", nil, true}, + {"complete", &NodeHost{ObservedAt: &now, CPUUtilization: &half, EffectiveCPUCores: &half, TotalMemoryBytes: &total, AvailableMemoryBytes: &total, AvailableDiskBytes: &total}, true}, + {"no observation time", &NodeHost{}, false}, + {"zero observation time", &NodeHost{ObservedAt: &time.Time{}}, false}, + {"before 1970", &NodeHost{ObservedAt: &early}, false}, + {"after 9999", &NodeHost{ObservedAt: &late}, false}, + {"utilization above 1", &NodeHost{ObservedAt: &now, CPUUtilization: &over}, false}, + {"negative utilization", &NodeHost{ObservedAt: &now, CPUUtilization: &negative}, false}, + {"NaN utilization", &NodeHost{ObservedAt: &now, CPUUtilization: &nan}, false}, + {"infinite cores", &NodeHost{ObservedAt: &now, EffectiveCPUCores: &inf}, false}, + {"negative bytes", &NodeHost{ObservedAt: &now, AvailableDiskBytes: &negativeBytes}, false}, + {"bytes beyond 2^53-1", &NodeHost{ObservedAt: &now, AvailableDiskBytes: &tooLarge}, false}, + {"zero total memory", &NodeHost{ObservedAt: &now, TotalMemoryBytes: &zeroBytes}, false}, + {"available above total", &NodeHost{ObservedAt: &now, TotalMemoryBytes: &total, AvailableMemoryBytes: &more}, false}, + } { + if err := validateHost(c.host); c.ok != (err == nil) || (!c.ok && !errors.Is(err, ErrInvalidInput)) { + t.Errorf("%s: validateHost = %v", c.name, err) + } + } +} + +func TestHistoryPoints(t *testing.T) { + start := time.Date(2026, 9, 30, 10, 0, 0, 0, time.UTC) + window := coremetrics.Range{Start: start, End: start.Add(3 * time.Minute), ResolutionSeconds: 60} + used := int64(7) + // A sample in another zone still fills its UTC bucket. + sample := HostHistoryPoint{Start: start.Add(time.Minute).In(time.FixedZone("east", 8*3600)), MemoryUsedBytesMax: &used} + points := historyPoints(window, []HostHistoryPoint{sample}) + if len(points) != 3 { + t.Fatalf("got %d points, want one per bucket", len(points)) + } + for i, p := range points { + if !p.Start.Equal(start.Add(time.Duration(i) * time.Minute)) { + t.Errorf("point %d starts at %v", i, p.Start) + } + if filled := p.MemoryUsedBytesMax != nil; filled != (i == 1) || p.CPUUtilizationMax != nil || p.AvailableDiskBytesMin != nil { + t.Errorf("point %d = %+v", i, p) + } + } + if points[1].Start.Location() != time.UTC { + t.Fatal("sample bucket is not UTC") + } + if got := historyPoints(coremetrics.Range{Start: start, End: start, ResolutionSeconds: 60}, nil); got == nil || len(got) != 0 { + t.Fatalf("empty window = %#v, want an empty list", got) + } +} + +func TestNodeRollout(t *testing.T) { + ready := uint64(2) + for _, c := range []struct { + name string + n NodeRecord + state string + diagnostic string + }{ + {"offline", NodeRecord{TargetState: "ready", ReadyGeneration: &ready}, "unknown", ""}, + {"protocol 1 on another generation", NodeRecord{Online: true, ProtocolVersion: 1, DeploymentGeneration: 1, TargetGeneration: 2, TargetState: "ready", ReadyGeneration: &ready}, "update_required", ""}, + {"protocol 1 on the target", NodeRecord{Online: true, ProtocolVersion: 1, DeploymentGeneration: 2, TargetGeneration: 2, TargetState: "ready", ReadyGeneration: &ready}, "ready", ""}, + {"preparing", NodeRecord{Online: true, ProtocolVersion: 2, TargetState: "preparing", ReadyGeneration: &ready}, "preparing", ""}, + {"failed with an unknown diagnostic", NodeRecord{Online: true, ProtocolVersion: 2, TargetState: "failed", TargetDiagnostic: "boom", ReadyGeneration: &ready}, "failed", sandbox.NodeProviderUnavailable}, + {"failed without diagnostic", NodeRecord{Online: true, ProtocolVersion: 2, TargetState: "failed", ReadyGeneration: &ready}, "failed", ""}, + {"ready ignores a diagnostic", NodeRecord{Online: true, ProtocolVersion: 2, TargetState: "ready", TargetDiagnostic: "boom", ReadyGeneration: &ready}, "ready", ""}, + {"unknown target state", NodeRecord{Online: true, ProtocolVersion: 2, TargetState: "other", ReadyGeneration: &ready}, "unknown", ""}, + {"protocol 1 failed on the target", NodeRecord{Online: true, ProtocolVersion: 1, DeploymentGeneration: 2, TargetGeneration: 2, TargetState: "failed", TargetDiagnostic: "kvm_unavailable", ReadyGeneration: &ready}, "failed", "kvm_unavailable"}, + {"protocol 1 without a target state", NodeRecord{Online: true, ProtocolVersion: 1, DeploymentGeneration: 2, TargetGeneration: 2, ReadyGeneration: &ready}, "unknown", ""}, + } { + got := nodeRollout(c.n) + if got.State != c.state || got.Diagnostic != c.diagnostic || got.ReadyGeneration != &ready { + t.Errorf("%s: nodeRollout = %+v", c.name, got) + } + } +} diff --git a/services/core/internal/deployment/service.go b/services/core/internal/deployment/service.go new file mode 100644 index 000000000..6be42ba1a --- /dev/null +++ b/services/core/internal/deployment/service.go @@ -0,0 +1,287 @@ +package deployment + +import ( + "context" + "encoding/json" + "errors" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" +) + +// Service reads the deployment and manages its nodes. It grants no execution +// authority: deployment changes run through ExecutionOperations. +type Service struct { + storage Storage + reader Reader + registry *providers.Registry + // publicURL is OAC_PUBLIC_URL. Core reports it as the deployment and node + // configuration core_url and records it on each node it enrolls. + publicURL string +} + +// NewService returns the deployment service. publicURL is the validated +// installation public URL, empty when the installation has none. +func NewService(storage Storage, reader Reader, registry *providers.Registry, publicURL string) (*Service, error) { + if storage == nil || reader == nil || registry == nil { + return nil, errors.New("deployment service requires storage, a reader and a provider registry") + } + return &Service{storage: storage, reader: reader, registry: registry, publicURL: publicURL}, nil +} + +// View returns the deployment as administrators read it. +func (s *Service) View(ctx context.Context) (View, error) { + snapshot, err := s.reader.Snapshot(ctx) + if err != nil { + return View{}, err + } + return s.view(snapshot) +} + +// view reports the public URL as the deployment's read-only core_url. +func (s *Service) view(snapshot Snapshot) (View, error) { + d := snapshot.Record + result := View{InstallationID: d.InstallationID, Provider: d.Provider, CoreURL: s.publicURL, OwnerEpoch: d.OwnerEpoch, Generation: d.Generation, Mode: d.Mode, Rollout: snapshot.Rollout, Resources: snapshot.Resources} + if len(d.Specification) > 0 && string(d.Specification) != "{}" { + var spec sandbox.DeploymentSpec + if json.Unmarshal(d.Specification, &spec) == nil { + result.Specification = &spec + result.SpecificationDigest = spec.Digest(d.Provider) + } + } + if d.Provider != "" { + value, err := s.registry.Decode(d.Provider, sandbox.ConfigurationRecord{Public: d.Configuration.Public, Metadata: d.Configuration.Metadata}) + if err != nil { + return View{}, ErrConflict + } + record, err := s.registry.Encode(d.Provider, value) + if err != nil { + return View{}, ErrConflict + } + result.Configuration = configurationJSON(record.Public) + result.Metadata = configurationJSON(record.Metadata) + result.CredentialConfigured = d.CredentialStored + checkpoint, err := s.registry.SupportsCheckpoint(d.Provider) + if err != nil { + return View{}, err + } + if checkpoint { + result.Suspension = &Suspension{IdleSeconds: d.IdleSeconds, RetentionSeconds: d.RetentionSeconds} + } + } + if d.Reset != nil { + result.Reset = &Reset{Clear: d.Reset.Clear, RequestedAt: d.Reset.RequestedAt, DeadlineAt: d.Reset.DeadlineAt, ForcedAt: d.Reset.ForcedAt, Remaining: snapshot.Remaining} + } + return result, nil +} + +// Setup returns the Web-managed selection with its credential. +func (s *Service) Setup(ctx context.Context) (Setup, error) { + d, err := s.reader.Deployment(ctx) + if err != nil { + return Setup{}, err + } + return s.setup(d) +} + +func (s *Service) setup(d Record) (Setup, error) { + if !d.WebManaged { + return Setup{}, ErrConflict + } + result := Setup{InstallationID: d.InstallationID, Provider: d.Provider, BackendFingerprint: d.BackendFingerprint, Generation: d.Generation, Mode: d.Mode, AdmissionPaused: d.AdmissionPaused} + if err := json.Unmarshal(d.Specification, &result.Specification); err != nil { + return Setup{}, err + } + if d.Provider == "" { + return result, nil + } + if err := s.registry.ValidateSpecification(d.Provider, result.Specification); err != nil { + return Setup{}, err + } + if d.CredentialError != nil { + return Setup{}, d.CredentialError + } + var err error + // A stored configuration the provider no longer decodes is a conflict the + // administrator resolves by saving the setup again. + result.Configuration, err = s.registry.Decode(d.Provider, d.Configuration) + if err != nil { + return Setup{}, ErrConflict + } + return s.describe(result, d.IdleSeconds, d.RetentionSeconds) +} + +// describe adds the provider's declared operations, suspension policy and +// credential use. +func (s *Service) describe(setup Setup, idleSeconds, retentionSeconds int64) (Setup, error) { + adapter, err := s.registry.Lookup(setup.Provider) + if err != nil { + return Setup{}, err + } + setup.Operations = adapter.Operations() + checkpoint, err := s.registry.SupportsCheckpoint(setup.Provider) + if err != nil { + return Setup{}, err + } + if checkpoint { + setup.Suspension = &Suspension{IdleSeconds: idleSeconds, RetentionSeconds: retentionSeconds} + } + setup.UsesCredential, err = s.registry.UsesCredential(setup.Provider) + if err != nil { + return Setup{}, err + } + return setup, nil +} + +// specification returns the deployment's specification when its provider +// still accepts it. +func (s *Service) specification(d Record) (sandbox.DeploymentSpec, error) { + var spec sandbox.DeploymentSpec + if json.Unmarshal(d.Specification, &spec) != nil || s.registry.ValidateSpecification(d.Provider, spec) != nil { + return spec, ErrSpecificationMismatch + } + return spec, nil +} + +// AllocationSetup returns the immutable generation an allocation was created +// with, configured with the current credential. A released allocation stays +// historical and is never rebound. +func (s *Service) AllocationSetup(ctx context.Context, ref sandbox.Reference) (Setup, error) { + a, err := s.reader.Allocation(ctx, ref) + if err != nil { + return Setup{}, err + } + if a.ID != ref.AllocationID || a.Generation == 0 || a.Released { + return Setup{}, ErrInvalidInput + } + if a.InstallationID != a.Deployment.InstallationID { + return Setup{}, sandbox.ErrOwnership + } + current, err := s.setup(a.Deployment) + if err != nil { + return Setup{}, err + } + if a.Generation == current.Generation { + return current, nil + } + g := a.Retained + if g == nil || g.Provider != current.Provider { + return Setup{}, ErrConflict + } + result := current + result.Generation = g.Generation + result.Specification = sandbox.DeploymentSpec{} + if err := json.Unmarshal(g.Specification, &result.Specification); err != nil { + return Setup{}, err + } + result.Configuration, err = s.registry.Decode(g.Provider, g.Configuration) + if err != nil { + return Setup{}, ErrConflict + } + if current.UsesCredential { + composed, err := s.registry.WithCredential(result.selection(), current.selection()) + if err != nil { + return Setup{}, ErrConflict + } + result.Configuration = composed.Configuration + } + return result, nil +} + +// GenerationPage returns up to 32 retained generations after the given one, +// without their credential. Callers keep a deadline over the full scan. +func (s *Service) GenerationPage(ctx context.Context, after int64) ([]Setup, error) { + rows, err := s.reader.Generations(ctx, after) + if err != nil { + return nil, err + } + result := make([]Setup, 0, len(rows)) + for _, r := range rows { + v := Setup{Generation: r.Generation, Provider: r.Provider} + if err := json.Unmarshal(r.Specification, &v.Specification); err != nil { + return nil, err + } + v.Configuration, err = s.registry.Decode(r.Provider, r.Configuration) + if err != nil { + return nil, ErrConflict + } + adapter, err := s.registry.Lookup(r.Provider) + if err != nil { + return nil, err + } + v.Mode, v.Operations = adapter.Mode, adapter.Operations() + result = append(result, v) + } + return result, nil +} + +// WithCredential returns owner configured with the credential of candidate, +// which selects the same provider. +func (s *Service) WithCredential(owner, candidate Setup) (Setup, error) { + selection, err := s.registry.WithCredential(owner.selection(), candidate.selection()) + if err != nil { + return Setup{}, err + } + owner.Configuration = selection.Configuration + return owner, nil +} + +// DecodeConfiguration reads a submitted provider configuration and credential. +func (s *Service) DecodeConfiguration(provider string, public, credential json.RawMessage) (sandbox.Configuration, error) { + return s.registry.DecodeInput(provider, public, credential) +} + +// SetupForSelection prepares a selection's setup without writing or allocating +// resources. It rejects a provider that requires a reachable public origin +// while the installation public URL is loopback. +func (s *Service) SetupForSelection(installationID string, input sandbox.Selection) (Setup, error) { + if _, err := parseID(installationID); err != nil { + return Setup{}, err + } + normalized, err := s.registry.Normalize(input) + if err != nil { + return Setup{}, configurationError(err) + } + description, err := s.registry.Describe(input.Provider, installationID) + if err != nil { + return Setup{}, configurationError(err) + } + // Adapters declare whether their guests require a public Core origin. + publicOrigin, err := s.registry.RequiresPublicOrigin(input.Provider) + if err != nil { + return Setup{}, err + } + if publicOrigin && LoopbackOrigin(s.publicURL) { + return Setup{}, ErrPublicURLUnreachable + } + result := Setup{InstallationID: installationID, Provider: input.Provider, Mode: description.Mode, Specification: normalized.DeploymentSpec, Configuration: normalized.Configuration, BackendFingerprint: description.BackendFingerprint} + return s.describe(result, description.IdleSeconds, description.RetentionSeconds) +} + +// validateSelection rejects a selection its provider cannot normalize. +func (s *Service) validateSelection(input sandbox.Selection) error { + if _, err := s.registry.Normalize(input); err != nil { + return configurationError(err) + } + return nil +} + +// selectionEqual reports whether input selects the stored provider, +// specification and configuration. +func (s *Service) selectionEqual(d Record, input sandbox.Selection) (bool, error) { + if d.Provider != input.Provider { + return false, nil + } + previous, err := s.setup(d) + if err != nil { + return false, err + } + normalized, err := s.registry.Normalize(input) + if err != nil { + return false, configurationError(err) + } + if previous.Specification.Digest(d.Provider) != normalized.DeploymentSpec.Digest(input.Provider) { + return false, nil + } + return s.registry.Equal(input.Provider, previous.Configuration, normalized.Configuration) +} diff --git a/services/core/internal/deployment/service_test.go b/services/core/internal/deployment/service_test.go new file mode 100644 index 000000000..92560479b --- /dev/null +++ b/services/core/internal/deployment/service_test.go @@ -0,0 +1,372 @@ +package deployment + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" +) + +const testPublicURL = "https://core.example" + +func newService(t *testing.T, storage *fakeStorage, reader *fakeReader, publicURL string) *Service { + t.Helper() + service, err := NewService(storage, reader, providers.Builtin(), publicURL) + if err != nil { + t.Fatal(err) + } + return service +} + +// operations builds execution operations whose every transaction runs on tx. +func operations(t *testing.T, publicURL string, tx *fakeDeploymentTx) *ExecutionOperations { + t.Helper() + storage := &fakeExecutionStorage{t: t} + if tx != nil { + storage.withDeployment = func(_ context.Context, apply func(DeploymentTx) error) error { return apply(tx) } + } + result, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, publicURL), storage) + if err != nil { + t.Fatal(err) + } + return result +} + +// testSpecification is a valid deployment specification for provider. +func testSpecification(provider string) sandbox.DeploymentSpec { + s := sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}} + s.Runtime = &sandbox.RuntimeRelease{SourceCommit: strings.Repeat("a", 40), ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64), MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64)} + if provider == "microsandbox" { + s.Resources.RootDiskMiB = 8192 + s.Resources.EnvironmentDiskMiB = 8192 + } + return s +} + +// webDeployment is a Web-managed node deployment of provider at generation. +func webDeployment(t *testing.T, installation, provider string, generation uint64) Record { + t.Helper() + specification, err := json.Marshal(testSpecification(provider)) + if err != nil { + t.Fatal(err) + } + return Record{InstallationID: installation, WebManaged: true, Provider: provider, Generation: generation, Mode: "nodes", Specification: specification, + Configuration: sandbox.ConfigurationRecord{Public: json.RawMessage(`{}`), Metadata: json.RawMessage(`{}`)}} +} + +func TestNewServiceAndOperationsRejectNilDependencies(t *testing.T) { + storage, reader, registry := &fakeStorage{t: t}, &fakeReader{t: t}, providers.Builtin() + for name, build := range map[string]func() (*Service, error){ + "storage": func() (*Service, error) { return NewService(nil, reader, registry, testPublicURL) }, + "reader": func() (*Service, error) { return NewService(storage, nil, registry, testPublicURL) }, + "registry": func() (*Service, error) { return NewService(storage, reader, nil, testPublicURL) }, + } { + if service, err := build(); service != nil || err == nil { + t.Errorf("NewService without %s = %v, %v", name, service, err) + } + } + service := newService(t, storage, reader, testPublicURL) + if result, err := NewExecutionOperations(nil, &fakeExecutionStorage{t: t}); result != nil || err == nil { + t.Errorf("NewExecutionOperations without the service = %v, %v", result, err) + } + if result, err := NewExecutionOperations(service, nil); result != nil || err == nil { + t.Errorf("NewExecutionOperations without storage = %v, %v", result, err) + } +} + +// A provider whose guests connect to Core from outside the host cannot use a +// loopback public URL. The check writes nothing: the fakes allow no call. +func TestSetupForSelectionRequiresAReachablePublicURL(t *testing.T) { + id := uuid.NewString() + e2bSelection := sandbox.Selection{Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-key", Template: "runtime:" + uuid.NewString()}} + for _, publicURL := range []string{"http://127.0.0.1:8091", "http://localhost:8091", "http://[::1]:8091"} { + if _, err := newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, publicURL).SetupForSelection(id, e2bSelection); !errors.Is(err, ErrPublicURLUnreachable) { + t.Errorf("E2B accepted the loopback public URL %s: %v", publicURL, err) + } + } + setup, err := newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL).SetupForSelection(id, e2bSelection) + if err != nil || setup.Provider != "e2b" || setup.Mode != "direct" || setup.InstallationID != id || !setup.UsesCredential { + t.Fatalf("E2B with a public URL = %+v, %v", setup, err) + } + docker := sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")} + if setup, err := newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, "http://127.0.0.1:8091").SetupForSelection(id, docker); err != nil || setup.Mode != "nodes" || setup.UsesCredential { + t.Fatalf("Docker with a loopback public URL = %+v, %v", setup, err) + } +} + +// An unregistered provider is rejected before any storage call: the fakes +// allow none. +func TestUnknownProviderIsRejectedBeforeStorage(t *testing.T) { + id := uuid.NewString() + selection := sandbox.Selection{Provider: "missing-registration", ExpectedGeneration: 1} + o := operations(t, "http://127.0.0.1", nil) + service := newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, "http://127.0.0.1") + for name, call := range map[string]func() error{ + "SetupForSelection": func() error { _, err := service.SetupForSelection(id, selection); return err }, + "Initialize": func() error { _, err := o.Initialize(t.Context(), id, selection); return err }, + "Update": func() error { _, err := o.Update(t.Context(), id, selection); return err }, + } { + var configuration *ConfigurationError + if err := call(); !errors.As(err, &configuration) || !errors.Is(err, ErrInvalidInput) || configuration.Message != providers.ErrUnknownProvider.Error() { + t.Errorf("%s accepted an unregistered provider: %v", name, err) + } + } +} + +// A provider validation failure reaches callers as a ConfigurationError that +// unwraps only ErrInvalidInput and carries the provider's validation metadata +// separately. +func TestSandboxValidationWrapperPreservesClassification(t *testing.T) { + service := newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL) + installation := "00000000-0000-4000-8000-000000000001" + _, err := service.SetupForSelection(installation, sandbox.Selection{Provider: "docker"}) + var configuration *ConfigurationError + if !errors.As(err, &configuration) || !errors.Is(err, ErrInvalidInput) || configuration.Validation == nil || configuration.Validation.Param != "resources.cpus" { + t.Fatalf("missing validation metadata: %#v", err) + } + if errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("wrapper changed sentinel identity") + } + _, err = service.SetupForSelection(installation, sandbox.Selection{Provider: "e2b", DeploymentSpec: sandbox.DeploymentSpec{Runtime: &sandbox.RuntimeRelease{}}}) + if !errors.As(err, &configuration) || configuration.Validation == nil || configuration.Validation.Param != "runtime" || err.Error() != "invalid sandbox configuration: E2B Runtime is selected by its immutable template build" { + t.Fatal("pending E2B validation order changed", err) + } +} + +// A stored provider the registry no longer holds fails every read and change +// that consults it. +func TestRegistryLookupFailuresPropagate(t *testing.T) { + stored := webDeployment(t, uuid.NewString(), "retired", 1) + reader := &fakeReader{t: t, + snapshot: func(context.Context) (Snapshot, error) { return Snapshot{Record: stored}, nil }, + deployment: func(context.Context) (Record, error) { return stored, nil }, + nodes: func(context.Context) ([]NodeRecord, error) { + return []NodeRecord{{ID: uuid.NewString(), Provider: "retired", MaxActive: 1, MaxRetained: 1}}, nil + }, + } + service := newService(t, &fakeStorage{t: t}, reader, testPublicURL) + // View decodes the stored configuration first, and reports that failure as a conflict. + if _, err := service.View(t.Context()); !errors.Is(err, ErrConflict) { + t.Errorf("View = %v, want a conflict", err) + } + if _, err := service.Setup(t.Context()); !errors.Is(err, providers.ErrUnknownProvider) { + t.Errorf("Setup = %v", err) + } + if _, err := service.ListNodes(t.Context()); !errors.Is(err, providers.ErrUnknownProvider) { + t.Errorf("ListNodes = %v", err) + } + tx := &fakeDeploymentTx{t: t, + loadDeployment: func() (Record, error) { return Record{}, nil }, + countResources: func() (Resources, error) { return Resources{}, nil }, + setProcessDeployment: func(string, string, bool) error { return nil }, + } + process := &ProcessDeployment{ProviderKind: "retired", InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("b", 64)} + if err := operations(t, testPublicURL, tx).ConfigureProcess(t.Context(), process); !errors.Is(err, providers.ErrUnknownProvider) { + t.Errorf("ConfigureProcess = %v", err) + } +} + +// Setup carries the mode and operations the provider declares. A stored +// configuration the provider no longer decodes is a conflict the administrator +// resolves by saving again. +func TestSetupReportsDeclarationsAndStoredConfigurationConflicts(t *testing.T) { + stored := webDeployment(t, uuid.NewString(), "docker", 1) + reader := &fakeReader{t: t, deployment: func(context.Context) (Record, error) { return stored, nil }} + service := newService(t, &fakeStorage{t: t}, reader, testPublicURL) + setup, err := service.Setup(t.Context()) + if err != nil || setup.Mode != "nodes" || setup.Operations["Create"].State != providercontract.Supported { + t.Fatal(setup, err) + } + stored.Configuration.Public = json.RawMessage(`{"retired":true}`) + if _, err := service.Setup(t.Context()); !errors.Is(err, ErrConflict) { + t.Fatal("an undecodable stored configuration was not a conflict", err) + } +} + +// Docker never suspends, so its nodes retain at most their active capacity. +func TestListNodesAppliesTheRetainedLimit(t *testing.T) { + reader := &fakeReader{t: t, nodes: func(context.Context) ([]NodeRecord, error) { + return []NodeRecord{{Provider: "docker", MaxActive: 2, MaxRetained: 5}, {Provider: "microsandbox", MaxActive: 2, MaxRetained: 5}}, nil + }} + nodes, err := newService(t, &fakeStorage{t: t}, reader, testPublicURL).ListNodes(t.Context()) + if err != nil || len(nodes) != 2 || nodes[0].MaxRetained != 2 || nodes[1].MaxRetained != 5 { + t.Fatalf("ListNodes = %+v, %v", nodes, err) + } +} + +func TestSetupChangeDecisions(t *testing.T) { + installation := uuid.NewString() + stored := webDeployment(t, installation, "docker", 2) + load := func() *fakeDeploymentTx { + return &fakeDeploymentTx{t: t, loadDeployment: func() (Record, error) { return stored, nil }} + } + docker := sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker"), ExpectedGeneration: 2} + + stale := docker + stale.ExpectedGeneration = 1 + for name, call := range map[string]func(*ExecutionOperations) error{ + "CheckSetup": func(o *ExecutionOperations) error { return o.CheckSetup(t.Context(), installation, stale) }, + "Initialize": func(o *ExecutionOperations) error { + _, err := o.Initialize(t.Context(), installation, stale) + return err + }, + "Update": func(o *ExecutionOperations) error { _, err := o.Update(t.Context(), installation, stale); return err }, + } { + var generation *GenerationStaleError + if err := call(operations(t, testPublicURL, load())); !errors.As(err, &generation) || generation.CurrentGeneration != 2 || !errors.Is(err, ErrConflict) { + t.Errorf("%s with a stale generation = %v", name, err) + } + } + + switched := sandbox.Selection{Provider: "microsandbox", DeploymentSpec: testSpecification("microsandbox"), ExpectedGeneration: 2} + for name, call := range map[string]func(*ExecutionOperations) error{ + "CheckSetup": func(o *ExecutionOperations) error { return o.CheckSetup(t.Context(), installation, switched) }, + "ClassifyChange": func(o *ExecutionOperations) error { + _, _, err := o.ClassifyChange(t.Context(), installation, switched) + return err + }, + "Update": func(o *ExecutionOperations) error { + _, err := o.Update(t.Context(), installation, switched) + return err + }, + } { + var reset *ResetRequiredError + if err := call(operations(t, testPublicURL, load())); !errors.As(err, &reset) || reset.CurrentProvider != "docker" || reset.RequestedProvider != "microsandbox" || !errors.Is(err, ErrConflict) { + t.Errorf("%s switching provider without a reset = %v", name, err) + } + } + + resolved, unchanged, err := operations(t, testPublicURL, load()).ClassifyChange(t.Context(), installation, docker) + if err != nil || !unchanged || resolved.Provider != "docker" || resolved.ExpectedGeneration != 2 { + t.Fatalf("ClassifyChange of the stored selection = %+v, %v, %v", resolved, unchanged, err) + } + changed := docker + changed.Resources.CPUs = 4 + if _, unchanged, err := operations(t, testPublicURL, load()).ClassifyChange(t.Context(), installation, changed); err != nil || unchanged { + t.Fatalf("ClassifyChange of a new specification = %v, %v", unchanged, err) + } +} + +// Enrollment checks the token before the deployment, so a bad token is a +// credential error whatever the deployment's state. +func TestEnrollChecksTheTokenBeforeTheDeployment(t *testing.T) { + installation, token := uuid.NewString(), "enrollment-token" + current := webDeployment(t, installation, "docker", 1) + valid := Enrollment{NodeID: uuid.NewString(), Name: "node", Credential: strings.Repeat("n", 64), Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), + DeploymentGeneration: 1, SpecificationDigest: testSpecification("docker").Digest("docker"), CoreURL: testPublicURL} + receipt := EnrollmentRecord{ID: uuid.NewString(), InstallationID: installation, ExpiresAt: time.Now().Add(10 * time.Minute), MaxActive: 1, MaxRetained: 1} + resetting := current + resetting.Reset = &ResetState{Clear: "sandboxes", RequestedAt: time.Now()} + paused := current + paused.AdmissionPaused = true + unselected := current + unselected.Provider = "" + consumed, expired, foreign := receipt, receipt, receipt + consumed.Consumed = true + expired.ExpiresAt = time.Now().Add(-time.Second) + foreign.InstallationID = uuid.NewString() + elsewhere := valid + elsewhere.CoreURL = "https://other.example" + stale := valid + stale.DeploymentGeneration = 2 + + for _, c := range []struct { + name string + deployment Record + receipt EnrollmentRecord + loadErr error + input Enrollment + want error + }{ + {"unknown token while resetting", resetting, EnrollmentRecord{}, ErrNotFound, valid, ErrNodeCredential}, + {"consumed token before setup", Record{}, consumed, nil, valid, ErrNodeCredential}, + {"expired token while paused", paused, expired, nil, valid, ErrNodeCredential}, + {"token of another installation while resetting", resetting, foreign, nil, valid, ErrNodeCredential}, + {"token store failure", current, EnrollmentRecord{}, errors.New("database down"), valid, ErrNodeUnavailable}, + {"no provider selected", unselected, receipt, nil, valid, ErrNodeUnavailable}, + {"reset in progress", resetting, receipt, nil, valid, ErrResetInProgress}, + {"admission paused", paused, receipt, nil, valid, ErrInvalidInput}, + {"stale generation", current, receipt, nil, stale, ErrSpecificationMismatch}, + // The token stays unused: the fake allows no insert or consumption. + {"another Core address", current, receipt, nil, elsewhere, ErrNodeAddressMismatch}, + } { + tx := &fakeNodeTx{t: t, + loadDeployment: func() (Record, error) { return c.deployment, nil }, + loadEnrollment: func(digest string) (EnrollmentRecord, error) { + if digest != tokenDigest(token) { + t.Fatalf("%s: enrollment loaded by %q", c.name, digest) + } + return c.receipt, c.loadErr + }, + } + storage := &fakeStorage{t: t, withNodes: func(_ context.Context, apply func(NodeTx) error) error { return apply(tx) }} + if _, err := newService(t, storage, &fakeReader{t: t}, testPublicURL).Enroll(t.Context(), token, c.input); !errors.Is(err, c.want) { + t.Errorf("%s: Enroll = %v, want %v", c.name, err, c.want) + } + } + + // Malformed input is rejected before storage. + service := newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL) + for _, mutate := range []func(*Enrollment){ + func(e *Enrollment) { e.NodeID = "node" }, + func(e *Enrollment) { e.Credential = "short" }, + func(e *Enrollment) { e.BackendFingerprint = "fingerprint" }, + func(e *Enrollment) { e.Name = "" }, + } { + input := valid + mutate(&input) + if _, err := service.Enroll(t.Context(), token, input); !errors.Is(err, ErrInvalidInput) { + t.Errorf("Enroll(%+v) = %v", input, err) + } + } +} + +// Node authentication checks the credential before it reads the deployment. +func TestAuthenticateNodeChecksTheCredentialFirst(t *testing.T) { + nodeID, credential := uuid.NewString(), strings.Repeat("n", 64) + stored := StoredNode{ID: nodeID, InstallationID: uuid.NewString(), CredentialDigest: tokenDigest(credential)} + service := func(reads *fakeNodeReads) *Service { + reader := &fakeReader{t: t, readNodes: func(_ context.Context, apply func(NodeReads) error) error { return apply(reads) }} + return newService(t, &fakeStorage{t: t}, reader, testPublicURL) + } + if _, err := newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL).AuthenticateNode(t.Context(), "node", credential); !errors.Is(err, ErrNodeCredential) { + t.Errorf("malformed node ID: %v", err) + } + for _, c := range []struct { + name string + node StoredNode + loadErr error + credential string + want error + }{ + {"unknown node", StoredNode{}, ErrNotFound, credential, ErrNodeCredential}, + {"node store failure", StoredNode{}, errors.New("database down"), credential, ErrNodeUnavailable}, + {"wrong credential", stored, nil, strings.Repeat("x", 64), ErrNodeCredential}, + } { + reads := &fakeNodeReads{t: t, loadNode: func(id string) (StoredNode, error) { + if id != nodeID { + t.Fatalf("%s: loaded node %q", c.name, id) + } + return c.node, c.loadErr + }} + if _, err := service(reads).AuthenticateNode(t.Context(), nodeID, c.credential); !errors.Is(err, c.want) { + t.Errorf("%s: AuthenticateNode = %v, want %v", c.name, err, c.want) + } + } + reads := &fakeNodeReads{t: t, + loadNode: func(string) (StoredNode, error) { return stored, nil }, + loadDeployment: func() (Record, error) { return webDeployment(t, uuid.NewString(), "docker", 1), nil }, + } + if _, err := service(reads).AuthenticateNode(t.Context(), nodeID, credential); !errors.Is(err, ErrNodeCredential) { + t.Errorf("node of another installation: %v", err) + } +} diff --git a/services/core/internal/deployment/setup.go b/services/core/internal/deployment/setup.go new file mode 100644 index 000000000..e40852e6c --- /dev/null +++ b/services/core/internal/deployment/setup.go @@ -0,0 +1,57 @@ +package deployment + +import ( + "encoding/json" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// Setup is the immutable configuration selected by the deployment admin, or one +// generation of it. An empty Provider means that Web setup has not yet selected +// an adapter. +type Setup struct { + Specification sandbox.DeploymentSpec + InstallationID, Provider, BackendFingerprint string + Generation uint64 + // Mode is where the provider runs: "nodes" for enrolled sandbox nodes and + // "direct" for a provider Core calls itself. + Mode string + AdmissionPaused bool + // Operations is the provider's declared operation support, which the node + // transport proxies. + Operations providercontract.Operations + Configuration sandbox.Configuration `json:"-"` + // Suspension is the idle suspension policy of a provider that suspends + // sandboxes, and nil otherwise. + Suspension *Suspension + // UsesCredential reports whether the provider's configuration carries a + // credential. + UsesCredential bool +} + +// selection is the provider selection a Setup describes. +func (s Setup) selection() sandbox.Selection { + return sandbox.Selection{Provider: s.Provider, DeploymentSpec: s.Specification, Configuration: s.Configuration} +} + +// ProcessDeployment is a deployment selected by process configuration instead of +// Web setup. Its fingerprint describes the backend namespace, never credentials +// or image contents. +type ProcessDeployment struct { + ProviderKind string + LocalNodeID string + LocalCredentialSHA256 string + LocalMaxActive, LocalMaxRetained int + InstallationID string + BackendFingerprint string + AdmissionPaused bool +} + +// configurationJSON reports an absent configuration object as {}. +func configurationJSON(raw json.RawMessage) json.RawMessage { + if len(raw) == 0 { + return json.RawMessage(`{}`) + } + return raw +} diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go new file mode 100644 index 000000000..5f09f0992 --- /dev/null +++ b/services/core/internal/deployment/storage.go @@ -0,0 +1,298 @@ +package deployment + +import ( + "context" + "encoding/json" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// Storage is the persistence node management writes through, on pooled +// connections. It grants no execution authority. +type Storage interface { + // WithNodes runs apply in one transaction that begins by locking the + // deployment, so node changes serialize with deployment changes, pin + // promotion and placement. It commits only when apply returns nil. + WithNodes(ctx context.Context, apply func(NodeTx) error) error + // ConnectNode records a node connection for the owner epoch, when the node + // belongs to the installation and no newer epoch connected it. It reports + // whether it did. + ConnectNode(ctx context.Context, nodeID, connectionID string, epoch uint64) (bool, error) + // DisconnectNode clears the node's connection when it is still this one. It + // waits for the node row first, so an uncertain connect cannot outlive it. + DisconnectNode(ctx context.Context, nodeID, connectionID string, epoch uint64) error + // SampleHostHistory copies each fresh authenticated heartbeat into the + // host history, at the Runtime sampler cadence, and returns the number of + // samples. Neither reads nor offline nodes fill gaps. + SampleHostHistory(ctx context.Context) (int64, error) +} + +// ExecutionStorage is the persistence deployment changes write through. Only +// the execution owner holds it: every transaction runs on the connection that +// holds the execution lease. +type ExecutionStorage interface { + // WithDeployment runs apply in one leased transaction that begins by + // locking the deployment. It commits only when apply returns nil. + WithDeployment(ctx context.Context, apply func(DeploymentTx) error) error +} + +// Reader answers deployment and node queries. +type Reader interface { + // Deployment returns the stored deployment. + Deployment(ctx context.Context) (Record, error) + // Snapshot returns the deployment with the counts and projections its View + // reports, read in one statement. + Snapshot(ctx context.Context) (Snapshot, error) + // OwnerEpoch returns the current execution owner epoch, which fences node + // connections. + OwnerEpoch(ctx context.Context) (uint64, error) + // Allocation returns the allocation of the Environment the reference names + // with the deployment, read in one snapshot. It returns ErrInvalidInput for + // malformed identifiers and ErrNotFound for a missing allocation. + Allocation(ctx context.Context, ref sandbox.Reference) (AllocationRecord, error) + // Generations returns up to 32 retained generations after the given one, + // in order, without their credential. + Generations(ctx context.Context, after int64) ([]GenerationRecord, error) + // Nodes returns the installation's nodes. + Nodes(ctx context.Context) ([]NodeRecord, error) + // NodeHistory returns one node and its host history samples in the window, + // one per bucket that has samples. It returns ErrInvalidInput for a + // malformed ID and ErrNotFound for a missing node. + NodeHistory(ctx context.Context, nodeID string, window coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) + // ReadNodes runs apply in one read-only snapshot. + ReadNodes(ctx context.Context, apply func(NodeReads) error) error +} + +// NodeReads loads node authentication facts. +type NodeReads interface { + // LoadDeployment returns the deployment, locked when the transaction locks it. + LoadDeployment() (Record, error) + // LoadNode returns ErrInvalidInput for a malformed ID and ErrNotFound for + // a missing or removed node. + LoadNode(id string) (StoredNode, error) + // LoadEnrollment returns ErrNotFound for an unknown token digest. + LoadEnrollment(tokenDigest string) (EnrollmentRecord, error) + // LoadGenerationSpecification returns ErrNotFound for a collected + // generation. + LoadGenerationSpecification(generation uint64) (GenerationSpecification, error) + // GenerationKept reports whether the node may keep the generation: its + // target, its serving pin or one with unreleased ownership on it. + GenerationKept(nodeID string, generation uint64) (bool, error) +} + +// NodeTx is one node management transaction. +type NodeTx interface { + NodeReads + ListNodes() ([]NodeRecord, error) + // InsertNode returns ErrNodeExists when the ID is in use, including by a + // removed node, and ErrInvalidInput for a malformed ID. + InsertNode(node NewNode) (StoredNode, error) + UpdateNode(id string, limits NodeLimits) error + RemoveNode(id string) error + // CreateEnrollment stores a token that expires in ten minutes and returns + // its expiry. + CreateEnrollment(enrollment NewEnrollment) (time.Time, error) + // ConsumeEnrollment marks an unexpired, unconsumed token of the current + // installation used by the node and reports whether it did. + ConsumeEnrollment(tokenDigest, nodeID string) (bool, error) + // HeartbeatNode records the node's health for its current connection and + // reports whether the connection is still current. + HeartbeatNode(heartbeat Heartbeat) (bool, error) + DeleteGenerationStatus(nodeID string, generation uint64) error + UpsertGenerationStatus(status GenerationStatusRecord) error + PromoteServingGeneration(nodeID string, generation uint64) error + RefreshServingReadiness(nodeID string, protocol int) error +} + +// DeploymentTx is one leased deployment change. +type DeploymentTx interface { + LoadDeployment() (Record, error) + LoadSnapshot() (Snapshot, error) + CountResources() (Resources, error) + // ClaimInstallation reserves the installation for Web setup and fences the + // previous owner epoch's node presence. + ClaimInstallation(installationID string) error + SetProcessDeployment(installationID, backendFingerprint string, admissionPaused bool) error + // SetManagerDeployment records the node provider and the local node, which + // is empty when there is none. + SetManagerDeployment(provider, localNodeID string) error + LoadNode(id string) (StoredNode, error) + InsertNode(node NewNode) (StoredNode, error) + UpdateNode(id string, limits NodeLimits) error + // SaveSelection stores the next generation. It seals a secret bound to the + // installation and generation; without a key it returns + // credentialcrypto.ErrUnavailable. + SaveSelection(selection SelectionRecord) error + RecordConfigurationMetadata(metadata json.RawMessage) error + // RetainGeneration keeps the current generation for the allocations that + // still use it. + RetainGeneration() error + // CollectGenerations deletes retained generations nothing uses. + CollectGenerations() error + // RecordAudit records an administrator mutation of the deployment. + RecordAudit(action, installationID string) error +} + +// Record is the stored deployment. +type Record struct { + // InstallationID is empty until an installation is claimed or configured. + InstallationID string + WebManaged bool + Provider string + BackendFingerprint string + Generation uint64 + OwnerEpoch uint64 + Mode string + AdmissionPaused bool + LocalNodeID string + IdleSeconds int64 + RetentionSeconds int64 + // Specification is the stored specification document. + Specification json.RawMessage + // Configuration holds the public configuration and metadata and, when a + // credential is stored and could be opened, its secret. + Configuration sandbox.ConfigurationRecord + CredentialStored bool + // CredentialError is why the stored credential could not be opened: no key + // (credentialcrypto.ErrUnavailable) or a ciphertext the key cannot open or + // authenticate (an internal error). + CredentialError error + // Reset is nil unless a reset is in progress. + Reset *ResetState +} + +// ResetState is the durable state of a reset in progress. +type ResetState struct { + Clear string + RequestedAt time.Time + DeadlineAt, ForcedAt *time.Time +} + +// Snapshot is the deployment with the counts and projections of its View. +type Snapshot struct { + Record Record + Resources Resources + Rollout Rollout + // Remaining partitions the resources while a reset is in progress. + Remaining ResetRemaining +} + +// SelectionRecord is one generation of the selection to store. +type SelectionRecord struct { + InstallationID, Provider, BackendFingerprint, Mode string + Generation uint64 + IdleSeconds, RetentionSeconds int64 + Specification json.RawMessage + // Configuration carries the secret in plaintext; the adapter seals it. + Configuration sandbox.ConfigurationRecord +} + +// GenerationRecord is a retained generation without its credential. +type GenerationRecord struct { + Generation uint64 + Provider string + Specification json.RawMessage + Configuration sandbox.ConfigurationRecord +} + +// GenerationSpecification is the provider and specification of a generation +// nodes may prepare. +type GenerationSpecification struct { + Provider string + Specification json.RawMessage +} + +// AllocationRecord is an allocation with the deployment that owns it. +type AllocationRecord struct { + ID string + // Generation is zero when the allocation has none. + Generation uint64 + Released bool + InstallationID string + Deployment Record + // Retained is the allocation's generation when it is not the deployment's + // current one and is still retained. + Retained *GenerationRecord +} + +// StoredNode is one node as stored. +type StoredNode struct { + ID, InstallationID, Name, BackendFingerprint string + CredentialDigest string + MaxActive, MaxRetained int + ConnectionID string + ConnectedEpoch uint64 + SpecificationDigest string + DeploymentGeneration uint64 + ReadyGeneration *uint64 +} + +// NodeRecord is a node of the installation with its presence and usage. +type NodeRecord struct { + ID, Name, Provider, CoreURL string + EnrollmentID *string + CreatedAt time.Time + LastSeenAt *time.Time + Health NodeHealth + Online, ServingReady bool + ProtocolVersion int + DeploymentGeneration uint64 + TargetGeneration uint64 + ReadyGeneration *uint64 + TargetState string + TargetDiagnostic string + MaxActive, MaxRetained int + Active, Retained, Reserved int64 + CleanupPending int64 + Running, Snapshots int64 +} + +// NewNode is a node to store. +type NewNode struct { + ID, InstallationID, Name, BackendFingerprint string + CredentialDigest string + MaxActive, MaxRetained int + SpecificationDigest string + DeploymentGeneration uint64 + CoreURL string + // EnrollmentID is empty for a node that no enrollment registered. + EnrollmentID string +} + +// NodeLimits is a node's name and capacity. +type NodeLimits struct { + Name string + MaxActive, MaxRetained int +} + +// EnrollmentRecord is a stored enrollment token. +type EnrollmentRecord struct { + ID, InstallationID string + ExpiresAt time.Time + Consumed bool + MaxActive, MaxRetained int +} + +// NewEnrollment is an enrollment token to store, by digest. +type NewEnrollment struct { + ID, TokenDigest, InstallationID string + MaxActive, MaxRetained int +} + +// Heartbeat is a node's health for one connection. +type Heartbeat struct { + NodeID, ConnectionID string + Epoch uint64 + Health NodeHealth +} + +// GenerationStatusRecord is a node's preparation state of one generation. +type GenerationStatusRecord struct { + NodeID, ConnectionID string + Generation uint64 + SpecificationDigest string + OwnerEpoch uint64 + State, Diagnostic string +} diff --git a/services/core/internal/deployment/view.go b/services/core/internal/deployment/view.go new file mode 100644 index 000000000..806a138fd --- /dev/null +++ b/services/core/internal/deployment/view.go @@ -0,0 +1,87 @@ +package deployment + +import ( + "encoding/json" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// View is the sandbox deployment as administrators read it. +type View struct { + Rollout Rollout `json:"rollout"` + Specification *sandbox.DeploymentSpec `json:"specification,omitempty"` + SpecificationDigest string `json:"specification_digest,omitempty"` + Generation uint64 `json:"generation"` + Mode string `json:"mode"` + Resources Resources `json:"resources"` + Configuration json.RawMessage `json:"configuration,omitempty" swaggertype:"object"` + Metadata json.RawMessage `json:"metadata,omitempty" swaggertype:"object"` + CredentialConfigured bool `json:"credential_configured"` + // Idle suspension policy; microsandbox only, otherwise null. + Suspension *Suspension `json:"suspension" extensions:"x-nullable"` + InstallationID string `json:"installation_id"` + Provider string `json:"provider"` + Reset *Reset `json:"reset" extensions:"x-nullable"` + OwnerEpoch uint64 `json:"owner_epoch"` + // Read-only: the installation public URL (OAC_PUBLIC_URL), which nodes and sandboxes use to reach Core. The deployment API does not accept it. + CoreURL string `json:"core_url"` +} + +// Resources counts what still belongs to the deployment. +type Resources struct { + Allocations int64 `json:"allocations"` + Pending int64 `json:"pending"` +} + +// Suspension is the idle suspension policy. Only microsandbox suspends +// sandboxes; Docker and E2B deployments return null. +type Suspension struct { + IdleSeconds int64 `json:"idle_seconds"` + RetentionSeconds int64 `json:"retention_seconds"` +} + +type NodeRollout struct { + // Target preparation, independent of an old pin's serving readiness. + State string `json:"state" enums:"ready,preparing,failed,update_required,unknown"` + // Durable serving-generation pin; online and provider_ready still gate placement. + ReadyGeneration *uint64 `json:"ready_generation" extensions:"x-nullable"` + Diagnostic string `json:"diagnostic,omitempty" enums:"provider_unavailable,docker_unavailable,docker_limits_unsupported,runtime_download_failed,runtime_image_unavailable,kvm_unavailable,microsandbox_artifacts_unavailable,capacity_insufficient"` +} + +type RolloutNodes struct { + Ready int64 `json:"ready"` + Preparing int64 `json:"preparing"` + Failed int64 `json:"failed"` + UpdateRequired int64 `json:"update_required"` + Unknown int64 `json:"unknown"` +} + +type Rollout struct { + State string `json:"state" enums:"settled,preparing"` + PreviousGenerationSandboxes int64 `json:"previous_generation_sandboxes"` + Nodes *RolloutNodes `json:"nodes" extensions:"x-nullable"` +} + +// Reset contains only durable state and a single-snapshot resource partition. +type Reset struct { + Clear string `json:"clear"` + RequestedAt time.Time `json:"requested_at"` + DeadlineAt *time.Time `json:"deadline_at" extensions:"x-nullable"` + ForcedAt *time.Time `json:"forced_at" extensions:"x-nullable"` + Remaining ResetRemaining `json:"remaining"` +} + +type ResetRemaining struct { + Busy int64 `json:"busy"` + Idle int64 `json:"idle"` + Cleanup int64 `json:"cleanup"` + OnOfflineNodes int64 `json:"on_offline_nodes"` + OfflineNodes []ResetOfflineNode `json:"offline_nodes"` +} + +type ResetOfflineNode struct { + NodeID string `json:"node_id"` + Name string `json:"name"` + Resources int64 `json:"resources"` +} diff --git a/services/core/internal/execution/archive_cancellation_cleanup_test.go b/services/core/internal/execution/archive_cancellation_cleanup_test.go index 9b49367e1..370b8b7a0 100644 --- a/services/core/internal/execution/archive_cancellation_cleanup_test.go +++ b/services/core/internal/execution/archive_cancellation_cleanup_test.go @@ -13,7 +13,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/projectpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/projects" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" @@ -66,18 +65,9 @@ func TestArchiveWaitingCleanupReceiptBarrier(t *testing.T) { }{{"Kill_no_delivery", false, false}, {"KillCompute_no_delivery", true, false}, {"Kill_live_delivery", false, true}, {"KillCompute_live_delivery", true, true}} { t.Run(scenario.name, func(t *testing.T) { checkpoint := scenario.checkpoint - s, leased, pool := resetManagerStoreDB(t, nil) + s, leased, _, pool := resetManagerStoreDB(t, nil) writer := leased.Store - installation := uuid.NewString() - if err := writer.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { - t.Fatal(err) - } - selection := store.SandboxDeploymentSetupRequest{Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}} - selection.Resources.CPUs = 2 - selection.Resources.MemoryMiB = 2048 - if _, err := writer.InitializeSandboxDeployment(t.Context(), installation, selection); err != nil { - t.Fatal(err) - } + installation := initializeE2BDeployment(t, leased) projectID := uuid.NewString() audit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) management, err := projects.NewService(projectpg.New(pgunit.NewPool(pool))) diff --git a/services/core/internal/execution/deployment_fixture_test.go b/services/core/internal/execution/deployment_fixture_test.go new file mode 100644 index 000000000..3ec740d90 --- /dev/null +++ b/services/core/internal/execution/deployment_fixture_test.go @@ -0,0 +1,181 @@ +package execution + +import ( + "context" + "errors" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" +) + +// fixturePublicURL is the installation public URL of these tests. E2B requires +// a public origin, so it is never loopback. +const fixturePublicURL = "https://core.example" + +// testDeployment builds the pooled deployment service and the deployment +// execution operations on lease, as cmd/server does for the Worker. cipher is +// nil when the owner has no credential key. +func testDeployment(t *testing.T, pool *pgxpool.Pool, cipher *credentialcrypto.Cipher, lease *pgunit.Lease) (*deployment.Service, *deployment.ExecutionOperations) { + t.Helper() + adapter := deploymentpg.New(pgunit.NewPool(pool), cipher) + return deploymentOperations(t, adapter, adapter, deploymentpg.NewExecution(lease, cipher)) +} + +// unitDeploymentService builds a deployment service for tests without a +// database. Only SetupForSelection, which never reads or writes, works; any +// storage call fails the test. +func unitDeploymentService(t *testing.T) *deployment.Service { + t.Helper() + service, _ := deploymentOperations(t, &strictDeploymentStorage{t: t}, &strictDeploymentReader{t: t}, &strictExecutionStorage{t: t}) + return service +} + +// deploymentOperations builds the deployment service on storage and reader and +// the execution operations on execution. +func deploymentOperations(t *testing.T, storage deployment.Storage, reader deployment.Reader, execution deployment.ExecutionStorage) (*deployment.Service, *deployment.ExecutionOperations) { + t.Helper() + service, err := deployment.NewService(storage, reader, providers.Builtin(), fixturePublicURL) + if err != nil { + t.Fatal(err) + } + operations, err := deployment.NewExecutionOperations(service, execution) + if err != nil { + t.Fatal(err) + } + return service, operations +} + +// unexpectedDeploymentCall fails the test for a storage call it did not set. +func unexpectedDeploymentCall(t *testing.T, name string) error { + t.Helper() + t.Error("unexpected call to " + name) + return errors.New("unexpected call to " + name) +} + +// strictDeploymentStorage runs each set func; any other call fails the test. +type strictDeploymentStorage struct { + t *testing.T + withNodes func(context.Context, func(deployment.NodeTx) error) error + connectNode func(context.Context, string, string, uint64) (bool, error) + disconnectNode func(context.Context, string, string, uint64) error + sampleHostHistory func(context.Context) (int64, error) +} + +func (s *strictDeploymentStorage) WithNodes(ctx context.Context, apply func(deployment.NodeTx) error) error { + if s.withNodes == nil { + return unexpectedDeploymentCall(s.t, "WithNodes") + } + return s.withNodes(ctx, apply) +} + +func (s *strictDeploymentStorage) ConnectNode(ctx context.Context, nodeID, connectionID string, epoch uint64) (bool, error) { + if s.connectNode == nil { + return false, unexpectedDeploymentCall(s.t, "ConnectNode") + } + return s.connectNode(ctx, nodeID, connectionID, epoch) +} + +func (s *strictDeploymentStorage) DisconnectNode(ctx context.Context, nodeID, connectionID string, epoch uint64) error { + if s.disconnectNode == nil { + return unexpectedDeploymentCall(s.t, "DisconnectNode") + } + return s.disconnectNode(ctx, nodeID, connectionID, epoch) +} + +func (s *strictDeploymentStorage) SampleHostHistory(ctx context.Context) (int64, error) { + if s.sampleHostHistory == nil { + return 0, unexpectedDeploymentCall(s.t, "SampleHostHistory") + } + return s.sampleHostHistory(ctx) +} + +// strictExecutionStorage runs withDeployment when set; otherwise any call +// fails the test. +type strictExecutionStorage struct { + t *testing.T + withDeployment func(context.Context, func(deployment.DeploymentTx) error) error +} + +func (s *strictExecutionStorage) WithDeployment(ctx context.Context, apply func(deployment.DeploymentTx) error) error { + if s.withDeployment == nil { + return unexpectedDeploymentCall(s.t, "WithDeployment") + } + return s.withDeployment(ctx, apply) +} + +// strictDeploymentReader runs each set func; any other call fails the test. +type strictDeploymentReader struct { + t *testing.T + deployment func(context.Context) (deployment.Record, error) + snapshot func(context.Context) (deployment.Snapshot, error) + ownerEpoch func(context.Context) (uint64, error) + allocation func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) + generations func(context.Context, int64) ([]deployment.GenerationRecord, error) + nodes func(context.Context) ([]deployment.NodeRecord, error) + nodeHistory func(context.Context, string, coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) + readNodes func(context.Context, func(deployment.NodeReads) error) error +} + +func (r *strictDeploymentReader) Deployment(ctx context.Context) (deployment.Record, error) { + if r.deployment == nil { + return deployment.Record{}, unexpectedDeploymentCall(r.t, "Deployment") + } + return r.deployment(ctx) +} + +func (r *strictDeploymentReader) Snapshot(ctx context.Context) (deployment.Snapshot, error) { + if r.snapshot == nil { + return deployment.Snapshot{}, unexpectedDeploymentCall(r.t, "Snapshot") + } + return r.snapshot(ctx) +} + +func (r *strictDeploymentReader) OwnerEpoch(ctx context.Context) (uint64, error) { + if r.ownerEpoch == nil { + return 0, unexpectedDeploymentCall(r.t, "OwnerEpoch") + } + return r.ownerEpoch(ctx) +} + +func (r *strictDeploymentReader) Allocation(ctx context.Context, ref sandbox.Reference) (deployment.AllocationRecord, error) { + if r.allocation == nil { + return deployment.AllocationRecord{}, unexpectedDeploymentCall(r.t, "Allocation") + } + return r.allocation(ctx, ref) +} + +func (r *strictDeploymentReader) Generations(ctx context.Context, after int64) ([]deployment.GenerationRecord, error) { + if r.generations == nil { + return nil, unexpectedDeploymentCall(r.t, "Generations") + } + return r.generations(ctx, after) +} + +func (r *strictDeploymentReader) Nodes(ctx context.Context) ([]deployment.NodeRecord, error) { + if r.nodes == nil { + return nil, unexpectedDeploymentCall(r.t, "Nodes") + } + return r.nodes(ctx) +} + +func (r *strictDeploymentReader) NodeHistory(ctx context.Context, nodeID string, window coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) { + if r.nodeHistory == nil { + return deployment.NodeRecord{}, nil, unexpectedDeploymentCall(r.t, "NodeHistory") + } + return r.nodeHistory(ctx, nodeID, window) +} + +func (r *strictDeploymentReader) ReadNodes(ctx context.Context, apply func(deployment.NodeReads) error) error { + if r.readNodes == nil { + return unexpectedDeploymentCall(r.t, "ReadNodes") + } + return r.readNodes(ctx, apply) +} diff --git a/services/core/internal/execution/deployment_provider_observations_test.go b/services/core/internal/execution/deployment_provider_observations_test.go index ac245d704..c1d7ea6d6 100644 --- a/services/core/internal/execution/deployment_provider_observations_test.go +++ b/services/core/internal/execution/deployment_provider_observations_test.go @@ -36,7 +36,7 @@ type finishObservationFixture struct { func newFinishObservationFixture(t *testing.T, maxConnections int32) finishObservationFixture { t.Helper() var cfg *pgxpool.Config - s, owner := resetManagerStoreConfig(t, func(c *pgxpool.Config) { + s, owner, _ := resetManagerStoreConfig(t, func(c *pgxpool.Config) { if maxConnections > 0 { c.MaxConns = maxConnections } diff --git a/services/core/internal/execution/dispatcher.go b/services/core/internal/execution/dispatcher.go index ba5292ad5..43bf3680a 100644 --- a/services/core/internal/execution/dispatcher.go +++ b/services/core/internal/execution/dispatcher.go @@ -10,6 +10,7 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/modelconfiguration" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -36,6 +37,9 @@ type Dispatcher struct { // Observer records which deployment default model configurations committed // root Turns used. It is required. Observer modelconfiguration.Observer + // Deployment reads the sandbox deployment and prepares a selection's setup. + // It is required; deployment changes go through Owner.Deployment. + Deployment *deployment.Service // ManagedRuntimes is optional internal provisioning; it does not admit hosted API requests. ManagedRuntimes *RuntimeProvider // MaxConcurrentExecutions bounds work admitted by this Core execution owner. diff --git a/services/core/internal/execution/environment_capabilities_test.go b/services/core/internal/execution/environment_capabilities_test.go index abe8a087b..b5678c2a7 100644 --- a/services/core/internal/execution/environment_capabilities_test.go +++ b/services/core/internal/execution/environment_capabilities_test.go @@ -2,10 +2,11 @@ package execution import ( "encoding/json" - "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "slices" "testing" + + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) func TestSelfHostedCapabilitySourcesAreFrozenAndStrict(t *testing.T) { diff --git a/services/core/internal/execution/owner.go b/services/core/internal/execution/owner.go index ed34437f5..38f2bcd2c 100644 --- a/services/core/internal/execution/owner.go +++ b/services/core/internal/execution/owner.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) @@ -17,8 +18,9 @@ type Ownership interface { // Owner is everything bound to one execution lease. Later cutovers add one explicit // field per domain's execution operations and delete the matching store calls. type Owner struct { - Lease Ownership - Store *store.Store // the remaining store execution operations, built by store.NewExecution(s, lease) + Lease Ownership + Store *store.Store // the remaining store execution operations, built by store.NewExecution(s, lease) + Deployment *deployment.ExecutionOperations // sandbox deployment changes on the lease-bound deploymentpg storage } // leaseCloseTimeout bounds releasing the lease once the Worker owns it. diff --git a/services/core/internal/execution/owner_test.go b/services/core/internal/execution/owner_test.go index 1cea828fc..2d703c5df 100644 --- a/services/core/internal/execution/owner_test.go +++ b/services/core/internal/execution/owner_test.go @@ -85,16 +85,26 @@ func TestStartWorkerFailureClosesLeaseOnce(t *testing.T) { _, err := StartWorker(canceled, &Dispatcher{Credentials: &recordingCredentials{}}, Owner{Lease: lease}) return err }, - "missing Store": func(t *testing.T, lease *closeCountingLease) error { + "missing deployment service": func(t *testing.T, lease *closeCountingLease) error { _, err := StartWorker(canceled, &Dispatcher{Credentials: &recordingCredentials{}, Observer: unusedObserver{t}}, Owner{Lease: lease}) return err }, + "missing Store": func(t *testing.T, lease *closeCountingLease) error { + _, _, deployments := resetManagerStore(t) + _, err := StartWorker(canceled, &Dispatcher{Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments}, Owner{Lease: lease}) + return err + }, + "missing deployment": func(t *testing.T, lease *closeCountingLease) error { + _, owner, deployments := resetManagerStore(t) + _, err := StartWorker(canceled, &Dispatcher{Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments}, Owner{Lease: lease, Store: owner.Store}) + return err + }, "deployment claim": func(t *testing.T, lease *closeCountingLease) error { - s, owner := resetManagerStore(t) + s, owner, deployments := resetManagerStore(t) lease.inner = owner.Lease id := uuid.NewString() - dispatcher := &Dispatcher{Store: s, Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })} - _, err := StartWorker(canceled, dispatcher, Owner{Lease: lease, Store: owner.Store}) + dispatcher := &Dispatcher{Store: s, Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })} + _, err := StartWorker(canceled, dispatcher, Owner{Lease: lease, Store: owner.Store, Deployment: owner.Deployment}) if ping := owner.Lease.CheckOwnership(t.Context()); !errors.Is(ping, pgunit.ErrLeaseClosed) { t.Error("failed start kept the database lease", ping) } @@ -114,12 +124,41 @@ func TestStartWorkerFailureClosesLeaseOnce(t *testing.T) { } } +func TestStartWorkerChecksDeploymentAfterItsDependencies(t *testing.T) { + _, owner, deployments := resetManagerStore(t) + credentials, observer := &recordingCredentials{}, unusedObserver{t} + for _, test := range []struct { + name string + dispatcher Dispatcher + store bool + want string + }{ + {"missing Credentials", Dispatcher{Observer: observer}, true, "execution worker requires MCP Credentials"}, + {"missing observer", Dispatcher{Credentials: credentials}, true, "execution requires a model configuration observer"}, + {"missing deployment service", Dispatcher{Credentials: credentials, Observer: observer}, true, "execution worker requires the deployment service"}, + {"missing Store", Dispatcher{Credentials: credentials, Observer: observer, Deployment: deployments}, false, "execution worker requires the execution Store"}, + {"missing deployment", Dispatcher{Credentials: credentials, Observer: observer, Deployment: deployments}, true, "execution worker requires the deployment execution operations"}, + } { + t.Run(test.name, func(t *testing.T) { + lease := &closeCountingLease{t: t} + lease.closable.Store(true) + started := Owner{Lease: lease} + if test.store { + started.Store = owner.Store + } + if _, err := StartWorker(t.Context(), &test.dispatcher, started); err == nil || err.Error() != test.want { + t.Fatalf("StartWorker without deployment operations = %v, want %q", err, test.want) + } + }) + } +} + func TestWorkerRunClosesLeaseAfterDrain(t *testing.T) { - s, owner := resetManagerStore(t) + s, owner, deployments := resetManagerStore(t) lease := &closeCountingLease{t: t, inner: owner.Lease} id := uuid.NewString() - dispatcher := &Dispatcher{Store: s, Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })} - worker, err := StartWorker(t.Context(), dispatcher, Owner{Lease: lease, Store: owner.Store}) + dispatcher := &Dispatcher{Store: s, Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })} + worker, err := StartWorker(t.Context(), dispatcher, Owner{Lease: lease, Store: owner.Store, Deployment: owner.Deployment}) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/execution/provider_operations_fixture_test.go b/services/core/internal/execution/provider_operations_fixture_test.go index 4b568704b..c8fc32805 100644 --- a/services/core/internal/execution/provider_operations_fixture_test.go +++ b/services/core/internal/execution/provider_operations_fixture_test.go @@ -2,6 +2,7 @@ package execution import ( "context" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" diff --git a/services/core/internal/execution/runtime_capabilities_test.go b/services/core/internal/execution/runtime_capabilities_test.go index d7231090f..c2c29bca1 100644 --- a/services/core/internal/execution/runtime_capabilities_test.go +++ b/services/core/internal/execution/runtime_capabilities_test.go @@ -4,12 +4,13 @@ import ( "bytes" "context" "errors" + "testing" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentcapabilities" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/agentplugin" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/google/uuid" - "testing" ) type capabilityFixture struct { diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 95643dcd7..0a9fee385 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -12,6 +12,7 @@ import ( "github.com/google/uuid" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -56,7 +57,7 @@ type runtimeLifecycle struct { wakeHints chan struct{} } -func newRuntimeManager(owner Owner, registry *runtimegateway.Registry, config *RuntimeProvider) (*runtimeManager, error) { +func newRuntimeManager(owner Owner, deployments *deployment.Service, registry *runtimegateway.Registry, config *RuntimeProvider) (*runtimeManager, error) { if config == nil { return nil, nil } @@ -74,7 +75,7 @@ func newRuntimeManager(owner Owner, registry *runtimegateway.Registry, config *R } } ctx, stop := context.WithCancel(context.Background()) - return &runtimeManager{store: owner.Store, lease: owner.Lease, registry: registry, config: copied, setupInstallationID: config.InstallationID, loadDeployment: config.loadDeployment, prepareDeployment: config.prepareDeployment, publishUnconfigured: config.PublishUnconfigured, setupGate: make(chan struct{}, 1), mutationGate: make(chan struct{}, 1), ctx: ctx, cancel: stop, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)}, nil + return &runtimeManager{store: owner.Store, deployment: owner.Deployment, deploymentService: deployments, lease: owner.Lease, registry: registry, config: copied, setupInstallationID: config.InstallationID, loadDeployment: config.loadDeployment, prepareDeployment: config.prepareDeployment, publishUnconfigured: config.PublishUnconfigured, setupGate: make(chan struct{}, 1), mutationGate: make(chan struct{}, 1), ctx: ctx, cancel: stop, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)}, nil } func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway.Registry) (RuntimeProvider, error) { diff --git a/services/core/internal/execution/runtime_manager.go b/services/core/internal/execution/runtime_manager.go index 18f0d874a..4ace4764f 100644 --- a/services/core/internal/execution/runtime_manager.go +++ b/services/core/internal/execution/runtime_manager.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) @@ -17,6 +18,8 @@ var errRuntimeTransition = fmt.Errorf("%w: sandbox configuration is changing", E type runtimeManager struct { store *store.Store + deployment *deployment.ExecutionOperations + deploymentService *deployment.Service lease Ownership registry *runtimegateway.Registry config RuntimeProvider @@ -235,7 +238,7 @@ func (m *runtimeManager) run(ctx context.Context) error { m.running = true m.mu.Unlock() if m.loadDeployment != nil { - if err := m.store.CollectSandboxGenerations(ctx); err != nil { + if err := m.deployment.CollectGenerations(ctx); err != nil { return err } } @@ -257,7 +260,7 @@ func (m *runtimeManager) run(ctx context.Context) error { return err case <-ticker.C: if m.loadDeployment != nil { - if err := m.store.CollectSandboxGenerations(ctx); err != nil { + if err := m.deployment.CollectGenerations(ctx); err != nil { return err } } diff --git a/services/core/internal/execution/runtime_retirement_failure_test.go b/services/core/internal/execution/runtime_retirement_failure_test.go index eda85686c..c0065b04e 100644 --- a/services/core/internal/execution/runtime_retirement_failure_test.go +++ b/services/core/internal/execution/runtime_retirement_failure_test.go @@ -29,7 +29,7 @@ func TestFailedInventoryRetirementClosesAdmissionAndRetainsGate(t *testing.T) { var readOnce sync.Once unblockRead := func() { readOnce.Do(func() { close(releaseRead) }) } defer unblockRead() - owner, pool := delayedReadWriter(t, &armed, reading, releaseRead) + owner, _, pool := delayedReadWriter(t, &armed, reading, releaseRead) m := testRuntimeManager(t) m.store, m.lease = owner.Store, owner.Lease m.loadDeployment = func(context.Context) (*RuntimeProvider, error) { return nil, nil } diff --git a/services/core/internal/execution/sandbox_deployment_drain_test.go b/services/core/internal/execution/sandbox_deployment_drain_test.go index 436c61ecf..ccfc8b937 100644 --- a/services/core/internal/execution/sandbox_deployment_drain_test.go +++ b/services/core/internal/execution/sandbox_deployment_drain_test.go @@ -10,9 +10,11 @@ import ( "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/node" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" @@ -46,7 +48,7 @@ func (c *delayedLeaseRead) Read(p []byte) (int, error) { // delayedReadWriter owns an isolated database, so each case has its own // advisory-lock namespace. The delayed driver read lets a cancellation fence hit // its own deadline without shortening production timeouts. -func delayedReadWriter(t *testing.T, armed *atomic.Bool, reading chan struct{}, release <-chan struct{}) (Owner, *pgxpool.Pool) { +func delayedReadWriter(t *testing.T, armed *atomic.Bool, reading chan struct{}, release <-chan struct{}) (Owner, *deployment.Service, *pgxpool.Pool) { t.Helper() pool := pgtest.OpenIsolated(t, func(cfg *pgxpool.Config) { dial := cfg.ConnConfig.DialFunc @@ -69,7 +71,8 @@ func delayedReadWriter(t *testing.T, armed *atomic.Bool, reading chan struct{}, t.Error(err) } }) - return Owner{Lease: lease, Store: store.NewExecution(store.New(pool), lease)}, pool + deployments, operations := testDeployment(t, pool, nil, lease) + return Owner{Lease: lease, Store: store.NewExecution(store.New(pool), lease), Deployment: operations}, deployments, pool } func TestSandboxDeploymentDrainPreservesLeaseInFlightRead(t *testing.T) { @@ -86,12 +89,12 @@ func testLifecycleCancellationPreservesLease(t *testing.T, mode string) { var releaseOnce sync.Once unblock := func() { releaseOnce.Do(func() { close(release) }) } defer unblock() - owner, _ := delayedReadWriter(t, &armed, reading, release) + owner, deployments, _ := delayedReadWriter(t, &armed, reading, release) hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", 1)} - m, err := newRuntimeManager(owner, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil })) + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + m, err := newRuntimeManager(owner, deployments, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil })) if err != nil { t.Fatal(err) } @@ -224,12 +227,12 @@ func (c *delayedCancellationContext) unblock() { } func TestSandboxDeploymentDrainFailureCannotReactivate(t *testing.T) { - _, owner := resetManagerStore(t) + _, owner, deployments := resetManagerStore(t) hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", 1)} - m, err := newRuntimeManager(owner, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil })) + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + m, err := newRuntimeManager(owner, deployments, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil })) if err != nil { t.Fatal(err) } @@ -254,7 +257,7 @@ func TestSandboxDeploymentDrainFailureCannotReactivate(t *testing.T) { if err := m.pauseDeployment(t.Context()); err == nil { t.Fatal("repeated drain forgot its failure") } - if err := m.activateDeployment(t.Context(), store.RuntimeDeploymentView{InstallationID: id, Generation: 1, Mode: "nodes", Provider: "docker"}); err == nil { + if err := m.activateDeployment(t.Context(), deployment.View{InstallationID: id, Generation: 1, Mode: "nodes", Provider: "docker"}); err == nil { t.Fatal("failed drain reopened the provider") } if _, _, err := m.enter(t.Context()); err == nil { diff --git a/services/core/internal/execution/sandbox_deployment_setup.go b/services/core/internal/execution/sandbox_deployment_setup.go index 10c39fadc..6bfa6d142 100644 --- a/services/core/internal/execution/sandbox_deployment_setup.go +++ b/services/core/internal/execution/sandbox_deployment_setup.go @@ -4,8 +4,8 @@ import ( "context" "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) // PreparedRuntimeDeployment has completed provider validation without publishing @@ -19,7 +19,7 @@ type PreparedRuntimeDeployment struct { FenceCredential func(context.Context) (func(), error) } -type RuntimeDeploymentPreparer func(context.Context, store.SandboxSetup) (PreparedRuntimeDeployment, error) +type RuntimeDeploymentPreparer func(context.Context, deployment.Setup) (PreparedRuntimeDeployment, error) // NewDeferredRuntimeProvider enables Web setup for one fixed installation. The // loader returns nil until selection, then the committed immutable generation. @@ -32,19 +32,19 @@ func NewDeferredRuntimeProvider(installationID string, load func(context.Context return config } -func (w *Worker) InitializeSandboxDeployment(ctx context.Context, input store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error) { +func (w *Worker) InitializeSandboxDeployment(ctx context.Context, input sandbox.Selection) (deployment.View, error) { unlock, err := w.runtimes.lockMutation(ctx) if err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } defer unlock() m := w.runtimes - if err := m.store.CheckSandboxDeploymentSetup(ctx, m.setupInstallationID, input); err != nil { - return store.RuntimeDeploymentView{}, err + if err := m.deployment.CheckSetup(ctx, m.setupInstallationID, input); err != nil { + return deployment.View{}, err } candidate, err := m.prepareCandidate(ctx, input) if err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } // An idempotent setup retry can arrive after an interrupted replacement. // It must not reopen admission while that replacement is still draining. @@ -53,12 +53,12 @@ func (w *Worker) InitializeSandboxDeployment(ctx context.Context, input store.Sa m.mu.Unlock() if switching { if err := m.pauseDeployment(ctx); err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } } - result, err := m.store.InitializeSandboxDeployment(ctx, m.setupInstallationID, *candidate.Selection) + result, err := m.deployment.Initialize(ctx, m.setupInstallationID, *candidate.Selection) if err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } m.publishDeployment(candidate, result) return result, nil @@ -118,11 +118,11 @@ func (m *runtimeManager) ensureDeployment(parent context.Context) (bool, error) // Preparation is outside the manager mutex and all database transactions. A // rejected candidate cannot retire the current generation or its node lanes. -func (m *runtimeManager) prepareCandidate(ctx context.Context, input store.SandboxDeploymentSetupRequest) (PreparedRuntimeDeployment, error) { +func (m *runtimeManager) prepareCandidate(ctx context.Context, input sandbox.Selection) (PreparedRuntimeDeployment, error) { if m.prepareDeployment == nil { return PreparedRuntimeDeployment{}, ErrExecutionUnavailable } - setup, err := store.SandboxSetupForSelection(m.setupInstallationID, input) + setup, err := m.deploymentService.SetupForSelection(m.setupInstallationID, input) if err != nil { return PreparedRuntimeDeployment{}, err } @@ -160,7 +160,7 @@ func (m *runtimeManager) prepareCandidate(ctx context.Context, input store.Sandb // The store commit is the point of no return. Publishing a validated candidate // is infallible, including when shutdown or request cancellation follows commit. -func (m *runtimeManager) publishDeployment(candidate PreparedRuntimeDeployment, committed store.RuntimeDeploymentView) { +func (m *runtimeManager) publishDeployment(candidate PreparedRuntimeDeployment, committed deployment.View) { m.mu.Lock() defer m.mu.Unlock() config := *candidate.Config diff --git a/services/core/internal/execution/sandbox_deployment_setup_test.go b/services/core/internal/execution/sandbox_deployment_setup_test.go index 50624769d..a5b5f59c6 100644 --- a/services/core/internal/execution/sandbox_deployment_setup_test.go +++ b/services/core/internal/execution/sandbox_deployment_setup_test.go @@ -9,12 +9,12 @@ import ( "testing" "time" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" - + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/node" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" ) @@ -24,8 +24,8 @@ func TestDeferredSandboxDeploymentLoadsOnceBeforeNodeCreation(t *testing.T) { id := uuid.NewString() var selected atomic.Bool var loads atomic.Int32 - configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", 1)} - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { loads.Add(1) if !selected.Load() { return nil, nil @@ -70,7 +70,7 @@ func TestDeferredSandboxDeploymentLoadsOnceBeforeNodeCreation(t *testing.T) { func TestDeferredSandboxDeploymentShutdownCancelsLoad(t *testing.T) { entered := make(chan struct{}) - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(uuid.NewString(), func(ctx context.Context) (*RuntimeProvider, error) { + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(uuid.NewString(), func(ctx context.Context) (*RuntimeProvider, error) { close(entered) <-ctx.Done() return nil, ctx.Err() @@ -94,8 +94,8 @@ func TestDeferredSandboxProviderFailureKeepsRecoveryAvailable(t *testing.T) { id := uuid.NewString() available := false loadErr := ErrExecutionUnavailable - configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", 1)} - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { if !available { return nil, loadErr } @@ -125,10 +125,10 @@ func TestRejectedSandboxCandidatePreservesActiveGeneration(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", 1)} + config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} rejected := errors.New("candidate provider unavailable") - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil }, - func(context.Context, store.SandboxSetup) (PreparedRuntimeDeployment, error) { + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, unitDeploymentService(t), runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil }, + func(context.Context, deployment.Setup) (PreparedRuntimeDeployment, error) { return PreparedRuntimeDeployment{}, rejected })) if err != nil { @@ -142,7 +142,7 @@ func TestRejectedSandboxCandidatePreservesActiveGeneration(t *testing.T) { if err != nil { t.Fatal(err) } - input := store.SandboxDeploymentSetupRequest{Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}, DeploymentSpec: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 1024}}} + input := sandbox.Selection{Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}, DeploymentSpec: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 1024}}} if _, err := m.prepareCandidate(t.Context(), input); !errors.Is(err, rejected) { t.Fatal("candidate rejection was lost", err) } @@ -159,8 +159,8 @@ func TestRejectedSandboxCandidatePreservesActiveGeneration(t *testing.T) { func TestSandboxCandidateValidationDoesNotHoldManagerLock(t *testing.T) { id := uuid.NewString() entered, release := make(chan struct{}), make(chan struct{}) - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, - func(context.Context, store.SandboxSetup) (PreparedRuntimeDeployment, error) { + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, unitDeploymentService(t), runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, + func(context.Context, deployment.Setup) (PreparedRuntimeDeployment, error) { close(entered) <-release return PreparedRuntimeDeployment{}, errors.New("rejected") @@ -172,7 +172,7 @@ func TestSandboxCandidateValidationDoesNotHoldManagerLock(t *testing.T) { done := make(chan struct{}) go func() { defer close(done) - _, _ = m.prepareCandidate(t.Context(), store.SandboxDeploymentSetupRequest{Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}, DeploymentSpec: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 1024}}}) + _, _ = m.prepareCandidate(t.Context(), sandbox.Selection{Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}, DeploymentSpec: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 1024}}}) }() <-entered stopped := make(chan struct{}) @@ -192,18 +192,18 @@ func TestCommittedSandboxCandidatePublishesAfterShutdown(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })) + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })) if err != nil { t.Fatal(err) } if err := m.pauseDeployment(t.Context()); err != nil { t.Fatal(err) } - config := &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), "docker", 1)} + config := &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} var published *RuntimeProvider candidate := PreparedRuntimeDeployment{Config: config, Publish: func(value *RuntimeProvider) { published = value }} m.stop() - m.publishDeployment(candidate, store.RuntimeDeploymentView{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b", Reset: &store.SandboxResetView{}}) + m.publishDeployment(candidate, deployment.View{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b", Reset: &deployment.Reset{}}) m.drain() if m.config.Generation != 2 || !m.config.AdmissionPaused || published == nil || published.Generation != 2 || !published.AdmissionPaused || m.switching { t.Fatal("committed candidate was lost during shutdown") diff --git a/services/core/internal/execution/sandbox_deployment_switch.go b/services/core/internal/execution/sandbox_deployment_switch.go index 05eb19111..6e18883cf 100644 --- a/services/core/internal/execution/sandbox_deployment_switch.go +++ b/services/core/internal/execution/sandbox_deployment_switch.go @@ -4,13 +4,13 @@ import ( "context" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) func (m *runtimeManager) lockMutation(ctx context.Context) (func(), error) { if m == nil || m.loadDeployment == nil { - return nil, store.ErrSandboxDeploymentConflict + return nil, deployment.ErrConflict } m.mu.Lock() closed := m.closed @@ -90,7 +90,7 @@ func (m *runtimeManager) drainDeployment(result *deploymentDrain) { // activateDeployment is serialized with every Web configuration mutation. The // loader reads the committed generation and publishes the same server snapshot. -func (m *runtimeManager) activateDeployment(ctx context.Context, expected store.RuntimeDeploymentView) error { +func (m *runtimeManager) activateDeployment(ctx context.Context, expected deployment.View) error { m.mu.Lock() if m.closed { m.mu.Unlock() @@ -112,7 +112,7 @@ func (m *runtimeManager) activateDeployment(ctx context.Context, expected store. return err } if config == nil && expected.Provider == "" { - m.publishEmptyDeployment(expected) + m.publishEmptyDeployment(expected.InstallationID, expected.Generation) return nil } if config == nil || config.InstallationID != expected.InstallationID || config.Generation != expected.Generation || config.Mode != expected.Mode || config.ProviderKind != expected.Provider || config.loadDeployment != nil || config.LocalNodeID != "" { @@ -137,49 +137,48 @@ func (m *runtimeManager) activateDeployment(ctx context.Context, expected store. return nil } -func (w *Worker) UpdateSandboxDeployment(ctx context.Context, input store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) { +func (w *Worker) UpdateSandboxDeployment(ctx context.Context, input sandbox.Selection) (deployment.View, error) { unlock, err := w.runtimes.lockMutation(ctx) if err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } defer unlock() m := w.runtimes - input, unchanged, err := m.store.ClassifySandboxDeploymentChange(ctx, m.setupInstallationID, input) + input, unchanged, err := m.deployment.ClassifyChange(ctx, m.setupInstallationID, input) if err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } if unchanged { - return m.store.GetRuntimeDeployment(ctx) + return m.deploymentService.View(ctx) } - candidate, err := m.prepareCandidate(ctx, input.SandboxDeploymentSetupRequest) + candidate, err := m.prepareCandidate(ctx, input) if err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } if input.HasCredential() || candidate.VerifyCredential != nil { if candidate.VerifyCredential == nil || candidate.FenceCredential == nil { - return store.RuntimeDeploymentView{}, ErrExecutionUnavailable + return deployment.View{}, ErrExecutionUnavailable } if err := candidate.VerifyCredential(ctx); err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } if input.ReplacesCredential() { fenceCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() release, err := candidate.FenceCredential(fenceCtx) if err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } defer release() // The final scan includes allocations admitted during preliminary verification. if err := candidate.VerifyCredential(fenceCtx); err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } } } - input.SandboxDeploymentSetupRequest = *candidate.Selection - result, err := m.store.UpdateSandboxDeployment(ctx, m.setupInstallationID, input) + result, err := m.deployment.Update(ctx, m.setupInstallationID, *candidate.Selection) if err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } m.publishDeployment(candidate, result) return result, nil @@ -190,7 +189,7 @@ func (w *Worker) UpdateSandboxDeployment(ctx context.Context, input store.Sandbo func (m *runtimeManager) restoreCommittedDeployment() error { ctx, cancel := context.WithTimeout(m.ctx, 5*time.Second) defer cancel() - committed, err := m.store.GetRuntimeDeployment(ctx) + committed, err := m.deploymentService.View(ctx) if err == nil { err = m.activateDeployment(ctx, committed) } @@ -204,14 +203,16 @@ func (m *runtimeManager) restoreCommittedDeployment() error { } // Empty-state publication is infallible after commit, even if the request was -// cancelled. Loader, preparer and installation ownership stay on the manager. -func (m *runtimeManager) publishEmptyDeployment(committed store.RuntimeDeploymentView) { +// cancelled. It needs only the installation and the committed generation, so +// no read follows the commit before the old configuration is retired. Loader, +// preparer and installation ownership stay on the manager. +func (m *runtimeManager) publishEmptyDeployment(installationID string, generation uint64) { m.mu.Lock() defer m.mu.Unlock() - m.config = RuntimeProvider{InstallationID: committed.InstallationID, Generation: committed.Generation} + m.config = RuntimeProvider{InstallationID: installationID, Generation: generation} m.nodes = make(map[string]*runtimeNode) if m.publishUnconfigured != nil { - m.publishUnconfigured(committed.Generation) + m.publishUnconfigured(generation) } m.switching = false m.switchDrained = nil diff --git a/services/core/internal/execution/sandbox_deployment_switch_test.go b/services/core/internal/execution/sandbox_deployment_switch_test.go index 6ed04d45b..6a26e6f94 100644 --- a/services/core/internal/execution/sandbox_deployment_switch_test.go +++ b/services/core/internal/execution/sandbox_deployment_switch_test.go @@ -7,9 +7,10 @@ import ( "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/node" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" ) @@ -17,8 +18,8 @@ func TestSandboxManagerSwitchDrainsBeforeDirectActivation(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", 1)} - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) + config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) if err != nil { t.Fatal(err) } @@ -61,8 +62,8 @@ func TestSandboxManagerSwitchDrainsBeforeDirectActivation(t *testing.T) { case <-time.After(time.Second): t.Fatal("switch drain blocked") } - config = &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", Generation: 2, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), "docker", 1)} - if err := m.activateDeployment(t.Context(), store.RuntimeDeploymentView{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b"}); err != nil { + config = &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", Generation: 2, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + if err := m.activateDeployment(t.Context(), deployment.View{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b"}); err != nil { t.Fatal(err) } if _, err := m.node(uuid.NewString()); !errors.Is(err, ErrExecutionUnavailable) { @@ -79,7 +80,7 @@ func TestSandboxManagerSwitchDrainsBeforeDirectActivation(t *testing.T) { func TestSandboxManagerFailedActivationStaysPaused(t *testing.T) { id := uuid.NewString() - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, errors.New("provider unavailable") })) + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, errors.New("provider unavailable") })) if err != nil { t.Fatal(err) } @@ -87,7 +88,7 @@ func TestSandboxManagerFailedActivationStaysPaused(t *testing.T) { if err := m.pauseDeployment(t.Context()); err != nil { t.Fatal(err) } - if err := m.activateDeployment(t.Context(), store.RuntimeDeploymentView{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b"}); err == nil { + if err := m.activateDeployment(t.Context(), deployment.View{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b"}); err == nil { t.Fatal("failed provider activated") } if _, _, err := m.enter(t.Context()); !errors.Is(err, errRuntimeTransition) { @@ -99,8 +100,8 @@ func TestSandboxManagerCancelledSwitchCannotResumeBeforeDrain(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", 1)} - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) + config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) if err != nil { t.Fatal(err) } @@ -128,8 +129,8 @@ func TestSandboxManagerCancelledSwitchCannotResumeBeforeDrain(t *testing.T) { m.mu.Lock() originalDrain := m.switchDrained m.mu.Unlock() - config = &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", Generation: 2, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), "docker", 1)} - expected := store.RuntimeDeploymentView{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b"} + config = &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", Generation: 2, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + expected := deployment.View{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b"} ctx, cancel = context.WithTimeout(t.Context(), 10*time.Millisecond) if err := m.activateDeployment(ctx, expected); !errors.Is(err, context.DeadlineExceeded) { cancel() @@ -154,10 +155,10 @@ func TestSandboxActivationCannotBypassOutstandingDrain(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, - func(_ context.Context, setup store.SandboxSetup) (PreparedRuntimeDeployment, error) { - return PreparedRuntimeDeployment{Config: &RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", 1)}}, nil + func(_ context.Context, setup deployment.Setup) (PreparedRuntimeDeployment, error) { + return PreparedRuntimeDeployment{Config: &RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)}}, nil })) if err != nil { t.Fatal(err) @@ -175,7 +176,7 @@ func TestSandboxActivationCannotBypassOutstandingDrain(t *testing.T) { cancel() ctx, cancel = context.WithTimeout(t.Context(), 10*time.Millisecond) defer cancel() - err = m.activateDeployment(ctx, store.RuntimeDeploymentView{InstallationID: id, Generation: 1, Provider: "e2b", Mode: "direct"}) + err = m.activateDeployment(ctx, deployment.View{InstallationID: id, Generation: 1, Provider: "e2b", Mode: "direct"}) if !errors.Is(err, context.DeadlineExceeded) { t.Fatal("activation bypassed the unfinished drain", err) } diff --git a/services/core/internal/execution/sandbox_generations_test.go b/services/core/internal/execution/sandbox_generations_test.go index feb60b4ca..01c99f724 100644 --- a/services/core/internal/execution/sandbox_generations_test.go +++ b/services/core/internal/execution/sandbox_generations_test.go @@ -6,40 +6,40 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/node" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" ) func TestE2BReplacementVerifiesTwiceAndNeverPublishesFailedCommit(t *testing.T) { - s, owner := resetManagerStore(t) - writer := owner.Store + _, owner, deployments := resetManagerStore(t) id := uuid.NewString() - if err := writer.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + if err := owner.Deployment.Claim(t.Context(), id); err != nil { t.Fatal(err) } - input := store.SandboxDeploymentSetupRequest{Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "old-key", Template: "runtime:" + uuid.NewString()}} + input := sandbox.Selection{Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "old-key", Template: "runtime:" + uuid.NewString()}} input.Resources.CPUs = 2 input.Resources.MemoryMiB = 2048 - if _, err := writer.InitializeSandboxDeployment(t.Context(), id, input); err != nil { + if _, err := owner.Deployment.Initialize(t.Context(), id, input); err != nil { t.Fatal(err) } hub := node.NewHub(node.HubOptions{}) defer hub.Close() - provider := hub.Proxy(uuid.NewString(), "docker", 1) + provider := hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1) verifyCalls, published, fenced, released := 0, 0, 0, 0 var rejectAt int var rejection error = sandbox.ErrCredentialOwnership config := NewDeferredRuntimeProvider(id, func(ctx context.Context) (*RuntimeProvider, error) { - setup, err := s.GetSandboxSetup(ctx) + setup, err := deployments.Setup(ctx) if err != nil { return nil, err } return &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", Generation: setup.Generation, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider}, nil - }, func(ctx context.Context, setup store.SandboxSetup) (PreparedRuntimeDeployment, error) { + }, func(ctx context.Context, setup deployment.Setup) (PreparedRuntimeDeployment, error) { return PreparedRuntimeDeployment{Config: &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider}, VerifyCredential: func(context.Context) error { verifyCalls++ @@ -50,7 +50,7 @@ func TestE2BReplacementVerifiesTwiceAndNeverPublishesFailedCommit(t *testing.T) }, FenceCredential: func(context.Context) (func(), error) { fenced++; return func() { released++ }, nil }, Publish: func(*RuntimeProvider) { published++ }}, nil }) - m, err := newRuntimeManager(owner, runtimegateway.NewRegistry(), config) + m, err := newRuntimeManager(owner, deployments, runtimegateway.NewRegistry(), config) if err != nil { t.Fatal(err) } @@ -64,7 +64,8 @@ func TestE2BReplacementVerifiesTwiceAndNeverPublishesFailedCommit(t *testing.T) } worker := &Worker{runtimes: m} input.Configuration.(*e2b.DeploymentConfiguration).APIKey = "candidate-key" - request := store.SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1} + request := input + request.ExpectedGeneration = 1 audit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "test", RequestID: "test", TraceID: "test"}) for _, failure := range []string{"preliminary", "final", "unanchored legacy or revoked committed key", "unknown ownership", "commit"} { verifyCalls = 0 @@ -77,7 +78,7 @@ func TestE2BReplacementVerifiesTwiceAndNeverPublishesFailedCommit(t *testing.T) rejectAt = 2 case "unanchored legacy or revoked committed key": rejectAt = 1 - rejection = &store.SandboxResetRequiredError{CurrentProvider: "e2b", RequestedProvider: "e2b"} + rejection = &deployment.ResetRequiredError{CurrentProvider: "e2b", RequestedProvider: "e2b"} case "unknown ownership": rejectAt = 1 rejection = sandbox.ErrConfigurationUnconfirmed @@ -87,7 +88,7 @@ func TestE2BReplacementVerifiesTwiceAndNeverPublishesFailedCommit(t *testing.T) if _, err := worker.UpdateSandboxDeployment(ctx, request); err == nil { t.Fatal("failure published", failure) } - committed, err := s.GetSandboxSetup(t.Context()) + committed, err := deployments.Setup(t.Context()) if err != nil || committed.Generation != 1 || committed.Configuration.(*e2b.DeploymentConfiguration).APIKey != "old-key" || published != 0 || fenced != released { t.Fatal("partial credential publication", failure, err) } @@ -110,7 +111,7 @@ func TestE2BReplacementVerifiesTwiceAndNeverPublishesFailedCommit(t *testing.T) if _, err := worker.UpdateSandboxDeployment(audit, request); err == nil { t.Fatal("stale replay accepted") } else { - var stale *store.SandboxGenerationStaleError + var stale *deployment.GenerationStaleError if !errors.As(err, &stale) { t.Fatal(err) } diff --git a/services/core/internal/execution/sandbox_provider_contract_test.go b/services/core/internal/execution/sandbox_provider_contract_test.go index b84b05c9a..dce51eba7 100644 --- a/services/core/internal/execution/sandbox_provider_contract_test.go +++ b/services/core/internal/execution/sandbox_provider_contract_test.go @@ -18,7 +18,7 @@ func TestSandboxProviderRegistrationDoesNotRequireAnExecutionVendorBranch(t *tes id := uuid.NewString() config := &RuntimeProvider{InstallationID: id, ProviderKind: "contract-fixture", Mode: mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: &lifecycleOnlySandbox{}} - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/execution/sandbox_reset.go b/services/core/internal/execution/sandbox_reset.go index d8c3cd16f..493f645fd 100644 --- a/services/core/internal/execution/sandbox_reset.go +++ b/services/core/internal/execution/sandbox_reset.go @@ -6,25 +6,58 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) -func (w *Worker) StartSandboxReset(ctx context.Context, input store.SandboxResetRequest) (store.RuntimeDeploymentView, error) { +func (w *Worker) StartSandboxReset(ctx context.Context, input store.SandboxResetRequest) (deployment.View, error) { unlock, err := w.runtimes.lockMutation(ctx) if err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } defer unlock() - return w.runtimes.store.StartSandboxReset(ctx, w.runtimes.setupInstallationID, input) + m := w.runtimes + if err := m.store.StartSandboxReset(ctx, m.setupInstallationID, input); err != nil { + return deployment.View{}, err + } + return m.committedView(ctx) } -func (w *Worker) CancelSandboxReset(ctx context.Context, generation uint64) (store.RuntimeDeploymentView, error) { +func (w *Worker) CancelSandboxReset(ctx context.Context, generation uint64) (deployment.View, error) { unlock, err := w.runtimes.lockMutation(ctx) if err != nil { - return store.RuntimeDeploymentView{}, err + return deployment.View{}, err } defer unlock() - return w.runtimes.store.CancelSandboxReset(ctx, w.runtimes.setupInstallationID, generation) + m := w.runtimes + if err := m.store.CancelSandboxReset(ctx, m.setupInstallationID, generation); err != nil { + return deployment.View{}, err + } + return m.committedView(ctx) +} + +// committedView reads the deployment after a committed reset change. The +// caller still holds the mutation gate, so no other Web change interleaves, +// but the gate does not freeze allocation counts or other live observations: +// the response is the current state read after commit. A failed read fails the +// response and never rolls back the change. The read runs on a pooled +// snapshot, so the lease check that follows proves no successor owner wrote +// before it; a lost lease stops this owner. +func (m *runtimeManager) committedView(ctx context.Context) (deployment.View, error) { + view, err := m.deploymentService.View(ctx) + if err != nil { + return deployment.View{}, err + } + check, cancel := context.WithTimeout(m.ctx, 5*time.Second) + defer cancel() + if err := m.lease.CheckOwnership(check); err != nil { + select { + case m.failed <- err: + default: + } + return deployment.View{}, ErrExecutionUnavailable + } + return view, nil } // resetStep runs only on the manager's uncounted coordinator loop. It must never @@ -51,7 +84,7 @@ func (m *runtimeManager) resetPage(parent, ctx context.Context) error { if err := m.store.AdvanceSandboxResetDeadline(ctx); err != nil { return err } - current, err := m.store.GetRuntimeDeployment(ctx) + current, err := m.deploymentService.View(ctx) if err != nil { return err } @@ -85,7 +118,7 @@ func (m *runtimeManager) resetPage(parent, ctx context.Context) error { if ctx.Err() != nil { return nil } - current, err = m.store.GetRuntimeDeployment(ctx) + current, err = m.deploymentService.View(ctx) if err != nil { return err } @@ -114,7 +147,9 @@ func (m *runtimeManager) resetPage(parent, ctx context.Context) error { log.Warn(parent, "Sandbox reset completion remains pending", "generation", current.Generation) return nil } - m.publishEmptyDeployment(committed) + // Publish from the committed generation alone: no read may stand between + // the commit and retiring the old runtime configuration. + m.publishEmptyDeployment(m.setupInstallationID, committed) m.resetCursor = "" return nil } diff --git a/services/core/internal/execution/sandbox_reset_test.go b/services/core/internal/execution/sandbox_reset_test.go index cc533668e..d4f3b8ff6 100644 --- a/services/core/internal/execution/sandbox_reset_test.go +++ b/services/core/internal/execution/sandbox_reset_test.go @@ -3,16 +3,21 @@ package execution import ( "bytes" "context" + "errors" + "reflect" "testing" "time" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/node" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" @@ -20,22 +25,23 @@ import ( ) // The execution lease is database-scoped, so these manager tests own a database. -// They receive the pooled Store and the Owner of its execution lease, which the -// test closes when it ends. -func resetManagerStore(t *testing.T) (*store.Store, Owner) { +// They receive the pooled Store, the Owner of its execution lease, which the +// test closes when it ends, and the pooled deployment service the Worker +// receives beside it. +func resetManagerStore(t *testing.T) (*store.Store, Owner, *deployment.Service) { t.Helper() return resetManagerStoreConfig(t, nil) } -func resetManagerStoreConfig(t *testing.T, configure func(*pgxpool.Config)) (*store.Store, Owner) { +func resetManagerStoreConfig(t *testing.T, configure func(*pgxpool.Config)) (*store.Store, Owner, *deployment.Service) { t.Helper() - s, owner, _ := resetManagerStoreDB(t, configure) - return s, owner + s, owner, deployments, _ := resetManagerStoreDB(t, configure) + return s, owner, deployments } // resetManagerStoreDB also returns the test database, for tests that build // adapters on it. -func resetManagerStoreDB(t *testing.T, configure func(*pgxpool.Config)) (*store.Store, Owner, *pgxpool.Pool) { +func resetManagerStoreDB(t *testing.T, configure func(*pgxpool.Config)) (*store.Store, Owner, *deployment.Service, *pgxpool.Pool) { t.Helper() pool := pgtest.OpenIsolated(t, configure) cipher, err := credentialcrypto.New(bytes.Repeat([]byte{8}, 32)) @@ -43,34 +49,28 @@ func resetManagerStoreDB(t *testing.T, configure func(*pgxpool.Config)) (*store. t.Fatal(err) } s := store.NewWithCredentialCipher(pool, cipher) - return s, testOwner(t, pool, s), pool + owner, deployments := testOwner(t, pool, cipher, s) + return s, owner, deployments, pool } // testOwner acquires the execution lease on pool and builds s's execution -// writer on it, as cmd/server does. The lease closes when the test ends. -func testOwner(t *testing.T, pool *pgxpool.Pool, s *store.Store) Owner { +// writer and the deployment execution operations on it, and the pooled +// deployment service, as cmd/server does. The lease closes when the test ends. +func testOwner(t *testing.T, pool *pgxpool.Pool, cipher *credentialcrypto.Cipher, s *store.Store) (Owner, *deployment.Service) { t.Helper() lease, err := pgunit.AcquireLease(t.Context(), pool) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = lease.Close(context.Background()) }) - return Owner{Lease: lease, Store: store.NewExecution(s, lease)} + deployments, operations := testDeployment(t, pool, cipher, lease) + return Owner{Lease: lease, Store: store.NewExecution(s, lease), Deployment: operations}, deployments } func TestSandboxResetPageTimeoutRecoversCommittedOwner(t *testing.T) { - s, owner := resetManagerStore(t) + _, owner, deployments := resetManagerStore(t) w := owner.Store - id := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { - t.Fatal(err) - } - selection := store.SandboxDeploymentSetupRequest{Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}} - selection.Resources.CPUs = 2 - selection.Resources.MemoryMiB = 2048 - if _, err := w.InitializeSandboxDeployment(t.Context(), id, selection); err != nil { - t.Fatal(err) - } + id := initializeE2BDeployment(t, owner) hub := node.NewHub(node.HubOptions{}) defer hub.Close() loads := 0 @@ -79,13 +79,13 @@ func TestSandboxResetPageTimeoutRecoversCommittedOwner(t *testing.T) { t.Error("recovery inherited cancelled page") } loads++ - setup, err := s.GetSandboxSetup(ctx) + setup, err := deployments.Setup(ctx) if err != nil || setup.Provider == "" { return nil, err } - return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Mode: setup.Mode, Generation: setup.Generation, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", 1)}, nil + return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Mode: setup.Mode, Generation: setup.Generation, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)}, nil }) - m, err := newRuntimeManager(owner, runtimegateway.NewRegistry(), config) + m, err := newRuntimeManager(owner, deployments, runtimegateway.NewRegistry(), config) if err != nil { t.Fatal(err) } @@ -108,10 +108,13 @@ func TestSandboxResetPageTimeoutRecoversCommittedOwner(t *testing.T) { } }() audit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "reset-test", ActorLabel: "operator", RequestID: "request", TraceID: "trace"}) - reset, err := w.StartSandboxReset(audit, id, store.SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}) - if err != nil { + if err := w.StartSandboxReset(audit, id, store.SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { t.Fatal(err) } + reset, err := deployments.View(t.Context()) + if err != nil || reset.Reset == nil { + t.Fatal("reset did not start", reset, err) + } page, cancel := context.WithCancel(t.Context()) defer cancel() done := make(chan error, 1) @@ -132,7 +135,7 @@ func TestSandboxResetPageTimeoutRecoversCommittedOwner(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("owner recovery blocked") } - current, err := s.GetRuntimeDeployment(t.Context()) + current, err := deployments.View(t.Context()) if err != nil || current.Reset == nil || current.Generation != 1 || !current.Reset.RequestedAt.Equal(reset.Reset.RequestedAt) { t.Fatal("page timeout lost durable reset", current, err) } @@ -147,8 +150,140 @@ func TestSandboxResetPageTimeoutRecoversCommittedOwner(t *testing.T) { if err := m.resetStep(t.Context()); err != nil { t.Fatal(err) } - current, err = s.GetRuntimeDeployment(t.Context()) + current, err = deployments.View(t.Context()) if err != nil || current.Reset != nil || current.Provider != "" || current.Generation != 2 { t.Fatal("next tick did not finish", current, err) } } + +// initializeE2BDeployment claims a new installation for Web setup and selects +// E2B at generation 1 through owner's deployment execution operations. +func initializeE2BDeployment(t *testing.T, owner Owner) string { + t.Helper() + id := uuid.NewString() + if err := owner.Deployment.Claim(t.Context(), id); err != nil { + t.Fatal(err) + } + selection := sandbox.Selection{Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}} + selection.Resources.CPUs = 2 + selection.Resources.MemoryMiB = 2048 + if _, err := owner.Deployment.Initialize(t.Context(), id, selection); err != nil { + t.Fatal(err) + } + return id +} + +func TestSandboxResetPublishesCommittedGenerationWithoutReading(t *testing.T) { + _, owner, pooled, pool := resetManagerStoreDB(t, nil) + id := initializeE2BDeployment(t, owner) + // The manager reads the deployment through a reader that fails every read + // of the committed reset, so publication cannot depend on one. + adapter := deploymentpg.New(pgunit.NewPool(pool), nil) + errCommittedRead := errors.New("committed deployment read failed") + committedReads := 0 + reader := &strictDeploymentReader{t: t, snapshot: func(ctx context.Context) (deployment.Snapshot, error) { + snapshot, err := adapter.Snapshot(ctx) + if err == nil && snapshot.Record.Generation == 2 { + committedReads++ + return deployment.Snapshot{}, errCommittedRead + } + return snapshot, err + }} + deployments, operations := deploymentOperations(t, &strictDeploymentStorage{t: t}, reader, &strictExecutionStorage{t: t}) + hub := node.NewHub(node.HubOptions{}) + defer hub.Close() + config := NewDeferredRuntimeProvider(id, func(ctx context.Context) (*RuntimeProvider, error) { + setup, err := pooled.Setup(ctx) + if err != nil || setup.Provider == "" { + return nil, err + } + return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Mode: setup.Mode, Generation: setup.Generation, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)}, nil + }) + var published []uint64 + config.PublishUnconfigured = func(generation uint64) { + if committedReads != 0 { + t.Error("a deployment read stood between the reset commit and its publication") + } + published = append(published, generation) + } + m, err := newRuntimeManager(Owner{Lease: owner.Lease, Store: owner.Store, Deployment: operations}, deployments, runtimegateway.NewRegistry(), config) + if err != nil { + t.Fatal(err) + } + defer func() { m.stop(); m.drain() }() + if _, err := m.ensureDeployment(t.Context()); err != nil { + t.Fatal(err) + } + audit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "reset-test", ActorLabel: "operator", RequestID: "request", TraceID: "trace"}) + if err := owner.Store.StartSandboxReset(audit, id, store.SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { + t.Fatal(err) + } + if err := m.resetStep(t.Context()); err != nil { + t.Fatal(err) + } + if len(published) != 1 || published[0] != 2 { + t.Fatal("reset did not publish its committed generation", published) + } + m.mu.Lock() + current := m.config + m.mu.Unlock() + if current.InstallationID != id || current.Generation != 2 || current.Provider != nil { + t.Fatal("reset did not retire the old configuration", current.InstallationID, current.Generation) + } + if _, err := m.deploymentService.View(t.Context()); !errors.Is(err, errCommittedRead) { + t.Fatal("the read after the reset commit did not fail", err) + } +} + +func TestCommittedResetViewStopsOwnerWithoutLease(t *testing.T) { + id := uuid.NewString() + reader := &strictDeploymentReader{t: t, snapshot: func(context.Context) (deployment.Snapshot, error) { + return deployment.Snapshot{Record: deployment.Record{InstallationID: id, WebManaged: true, Generation: 1}}, nil + }} + deployments, operations := deploymentOperations(t, &strictDeploymentStorage{t: t}, reader, &strictExecutionStorage{t: t}) + m, err := newRuntimeManager(Owner{Lease: lostLease{}, Deployment: operations}, deployments, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })) + if err != nil { + t.Fatal(err) + } + defer func() { m.stop(); m.drain() }() + view, err := m.committedView(t.Context()) + if !errors.Is(err, ErrExecutionUnavailable) || view.InstallationID != "" || view.Generation != 0 { + t.Fatal("committed view answered without the lease", view, err) + } + select { + case failure := <-m.failed: + if !errors.Is(failure, pgunit.ErrLeaseClosed) { + t.Fatal("owner stopped for another failure", failure) + } + default: + t.Fatal("lost lease did not stop the owner") + } +} + +func TestSandboxResetChangesReturnViewReadAfterCommit(t *testing.T) { + _, owner, deployments := resetManagerStore(t) + id := initializeE2BDeployment(t, owner) + m, err := newRuntimeManager(owner, deployments, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })) + if err != nil { + t.Fatal(err) + } + defer func() { m.stop(); m.drain() }() + worker := &Worker{runtimes: m} + audit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "reset-test", ActorLabel: "operator", RequestID: "request", TraceID: "trace"}) + started, err := worker.StartSandboxReset(audit, store.SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}) + if err != nil { + t.Fatal(err) + } + read, err := deployments.View(t.Context()) + if err != nil || started.Reset == nil || started.Reset.Clear != "force" || started.Generation != 1 || !reflect.DeepEqual(started, read) { + t.Fatal("start returned a view other than the committed reset", started, read, err) + } + cancelled, err := worker.CancelSandboxReset(audit, 1) + if err != nil { + t.Fatal(err) + } + read, err = deployments.View(t.Context()) + if err != nil || cancelled.Reset != nil || cancelled.Generation != 1 || cancelled.Provider != "e2b" || !reflect.DeepEqual(cancelled, read) { + t.Fatal("cancel returned a view other than the committed cancellation", cancelled, read, err) + } +} diff --git a/services/core/internal/execution/sandbox_snapshot_budget_test.go b/services/core/internal/execution/sandbox_snapshot_budget_test.go index 164f5f586..a17955865 100644 --- a/services/core/internal/execution/sandbox_snapshot_budget_test.go +++ b/services/core/internal/execution/sandbox_snapshot_budget_test.go @@ -8,10 +8,9 @@ import ( "testing" "time" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/node" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" @@ -43,8 +42,9 @@ func (d *snapshotBudget) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pg return } d.t.Logf("snapshot=%d query_elapsed=%s err=%v ctx=%v", q.ordinal, time.Since(q.started), data.Err, ctx.Err()) - // Model scheduling pressure on the first two reads without changing the real - // five-second page deadline. The final snapshot uses the lease connection. + // Model scheduling pressure on both page reads without changing the real + // five-second page deadline. Completion publishes the committed generation + // without a third read. if q.ordinal <= 2 && data.Err == nil { delay := 2200*time.Millisecond - time.Since(q.started) if delay > 0 { @@ -60,31 +60,22 @@ func (d *snapshotBudget) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pg func TestSandboxResetSnapshotFitsPageBudget(t *testing.T) { budget := &snapshotBudget{t: t} - s, owner := resetManagerStoreConfig(t, func(cfg *pgxpool.Config) { + _, owner, deployments := resetManagerStoreConfig(t, func(cfg *pgxpool.Config) { cfg.ConnConfig.RuntimeParams["jit"] = "on" cfg.ConnConfig.Tracer = budget }) w := owner.Store - id := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { - t.Fatal(err) - } - selection := store.SandboxDeploymentSetupRequest{Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-key", Template: "runtime:" + uuid.NewString()}} - selection.Resources.CPUs = 2 - selection.Resources.MemoryMiB = 2048 - if _, err := w.InitializeSandboxDeployment(t.Context(), id, selection); err != nil { - t.Fatal(err) - } + id := initializeE2BDeployment(t, owner) hub := node.NewHub(node.HubOptions{}) defer hub.Close() configuration := NewDeferredRuntimeProvider(id, func(ctx context.Context) (*RuntimeProvider, error) { - setup, err := s.GetSandboxSetup(ctx) + setup, err := deployments.Setup(ctx) if err != nil || setup.Provider == "" { return nil, err } - return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", 1)}, nil + return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)}, nil }) - m, err := newRuntimeManager(owner, runtimegateway.NewRegistry(), configuration) + m, err := newRuntimeManager(owner, deployments, runtimegateway.NewRegistry(), configuration) if err != nil { t.Fatal(err) } @@ -92,7 +83,7 @@ func TestSandboxResetSnapshotFitsPageBudget(t *testing.T) { if _, err = m.ensureDeployment(t.Context()); err != nil { t.Fatal(err) } - if _, err = w.StartSandboxReset(adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "reset-test", ActorLabel: "operator", RequestID: "request", TraceID: "trace"}), id, store.SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { + if err = w.StartSandboxReset(adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "reset-test", ActorLabel: "operator", RequestID: "request", TraceID: "trace"}), id, store.SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { t.Fatal(err) } budget.armed.Store(true) @@ -103,13 +94,13 @@ func TestSandboxResetSnapshotFitsPageBudget(t *testing.T) { if err != nil || ping != nil { t.Fatal("bounded snapshot lost execution ownership", err, ping) } - if budget.reads.Load() < 3 { - t.Fatal("did not exercise final leased snapshot") + if reads := budget.reads.Load(); reads != 2 { + t.Fatal("reset page read the deployment", reads, "times, want the two bounded reads before completion") } - if err := w.CollectSandboxGenerations(t.Context()); err != nil { + if err := owner.Deployment.CollectGenerations(t.Context()); err != nil { t.Fatal("next generation collection lost ownership", err) } - view, err := s.GetRuntimeDeployment(t.Context()) + view, err := deployments.View(t.Context()) if err != nil || view.Generation != 2 || view.Provider != "" || view.Reset != nil { t.Fatal("reset did not commit", view, err) } diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index e0e57680d..45eb241c3 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) @@ -55,14 +56,20 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher, owner Owner) (_ *W if dispatcher.Observer == nil { return nil, errors.New("execution requires a model configuration observer") } + if dispatcher.Deployment == nil { + return nil, errors.New("execution worker requires the deployment service") + } if owner.Store == nil { return nil, errors.New("execution worker requires the execution Store") } + if owner.Deployment == nil { + return nil, errors.New("execution worker requires the deployment execution operations") + } owned := *dispatcher owned.Store = owner.Store owned.notifications = &executionNotifications{} worker := &Worker{concurrency: dispatcher.MaxConcurrentExecutions, dispatcher: &owned, admission: dispatcher.Store, lease: owner.Lease, directoryReads: make(chan directoryReadRequest), fileWrites: make(chan fileWriteRequest), stopped: make(chan struct{}), scheduleWake: make(chan struct{}, 1), enrolledConnections: make(map[string]*runtimeConnection)} - worker.runtimes, err = newRuntimeManager(owner, owned.Registry, owned.ManagedRuntimes) + worker.runtimes, err = newRuntimeManager(owner, owned.Deployment, owned.Registry, owned.ManagedRuntimes) if err != nil { return nil, err } @@ -73,18 +80,18 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher, owner Owner) (_ *W } }() } - var deployment *store.RuntimeDeployment + var process *deployment.ProcessDeployment if worker.runtimes != nil && worker.runtimes.loadDeployment == nil { config := worker.runtimes.config - deployment = &store.RuntimeDeployment{ProviderKind: config.ProviderKind, LocalNodeID: config.LocalNodeID, LocalCredentialSHA256: config.LocalCredentialSHA256, LocalMaxActive: config.LocalMaxActive, LocalMaxRetained: config.LocalMaxRetained, InstallationID: config.InstallationID, BackendFingerprint: config.BackendFingerprint, AdmissionPaused: config.AdmissionPaused} + process = &deployment.ProcessDeployment{ProviderKind: config.ProviderKind, LocalNodeID: config.LocalNodeID, LocalCredentialSHA256: config.LocalCredentialSHA256, LocalMaxActive: config.LocalMaxActive, LocalMaxRetained: config.LocalMaxRetained, InstallationID: config.InstallationID, BackendFingerprint: config.BackendFingerprint, AdmissionPaused: config.AdmissionPaused} } if worker.runtimes != nil && worker.runtimes.loadDeployment != nil { - err = owned.Store.ClaimWebSandboxDeployment(ctx, worker.runtimes.setupInstallationID) + err = owner.Deployment.Claim(ctx, worker.runtimes.setupInstallationID) if err == nil { _, err = worker.runtimes.ensureDeployment(ctx) } } else { - err = owned.Store.ConfigureRuntimeDeployment(ctx, deployment) + err = owner.Deployment.ConfigureProcess(ctx, process) } if err != nil { return nil, err diff --git a/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go b/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go new file mode 100644 index 000000000..035a607be --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go @@ -0,0 +1,139 @@ +package deploymentpg_test + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" +) + +const fixturePublicURL = "https://core.example" + +// The deployment is a database singleton, so every test opens its own +// database. +type fixture struct { + pool *pgxpool.Pool + cipher *credentialcrypto.Cipher + adapter *deploymentpg.Store + service *deployment.Service +} + +func newFixture(t *testing.T) fixture { + t.Helper() + pool := pgtest.OpenIsolated(t, nil) + cipher, err := credentialcrypto.New(bytes.Repeat([]byte{13}, 32)) + if err != nil { + t.Fatal(err) + } + f := fixture{pool: pool, cipher: cipher} + f.adapter, f.service = f.withPublicURL(t, fixturePublicURL) + return f +} + +// withPublicURL builds the adapter and service as cmd/server does for an +// installation with this public URL. +func (f fixture) withPublicURL(t *testing.T, publicURL string) (*deploymentpg.Store, *deployment.Service) { + t.Helper() + adapter := deploymentpg.New(pgunit.NewPool(f.pool), f.cipher) + service, err := deployment.NewService(adapter, adapter, providers.Builtin(), publicURL) + if err != nil { + t.Fatal(err) + } + return adapter, service +} + +// execution acquires the execution lease and builds the deployment execution +// operations on it, as cmd/server does for the Worker. The lease closes when +// the test ends. +func (f fixture) execution(t *testing.T) (*deployment.ExecutionOperations, *pgunit.Lease) { + t.Helper() + lease, err := pgunit.AcquireLease(t.Context(), f.pool) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = lease.Close(context.Background()) }) + operations, err := deployment.NewExecutionOperations(f.service, deploymentpg.NewExecution(lease, f.cipher)) + if err != nil { + t.Fatal(err) + } + return operations, lease +} + +// admin carries the administrator provenance that deployment mutations audit. +func admin(t *testing.T) context.Context { + return adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", ActorLabel: "operator", RequestID: uuid.NewString(), TraceID: uuid.NewString()}) +} + +// testSpecification is a valid deployment specification for provider. +func testSpecification(provider string) sandbox.DeploymentSpec { + s := sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}} + if provider == "e2b" { + return s + } + s.Runtime = &sandbox.RuntimeRelease{SourceCommit: strings.Repeat("a", 40), ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64), MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64)} + if provider == "microsandbox" { + s.Resources.RootDiskMiB = 8192 + s.Resources.EnvironmentDiskMiB = 8192 + } + return s +} + +// initialize claims the deployment for a new installation and selects +// provider, returning the installation ID and the committed view. +func (f fixture) initialize(t *testing.T, changes *deployment.ExecutionOperations, input sandbox.Selection) (string, deployment.View) { + t.Helper() + installation := uuid.NewString() + if err := changes.Claim(t.Context(), installation); err != nil { + t.Fatal(err) + } + view, err := changes.Initialize(admin(t), installation, input) + if err != nil { + t.Fatal(err) + } + return installation, view +} + +// enroll issues an enrollment token and enrolls a node on the current +// generation with it. +func (f fixture) enroll(t *testing.T, view deployment.View, capacity deployment.Capacity) deployment.Enrollment { + t.Helper() + token, err := f.service.CreateEnrollment(admin(t), capacity) + if err != nil { + t.Fatal(err) + } + input := deployment.Enrollment{NodeID: uuid.NewString(), Name: "fixture node", Credential: strings.Repeat("n", 64), Provider: view.Provider, + BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest, CoreURL: fixturePublicURL} + if _, err := f.service.Enroll(t.Context(), token.Token, input); err != nil { + t.Fatal(err) + } + return input +} + +// connect connects the node for the current owner epoch and reports it ready. +func (f fixture) connect(t *testing.T, nodeID string) string { + t.Helper() + epoch, err := f.adapter.OwnerEpoch(t.Context()) + if err != nil { + t.Fatal(err) + } + connection := uuid.NewString() + if err := f.service.ConnectNode(t.Context(), nodeID, connection, epoch); err != nil { + t.Fatal(err) + } + if err := f.service.Heartbeat(t.Context(), nodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + t.Fatal(err) + } + return connection +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/generations_test.go b/services/core/internal/persistence/postgres/deploymentpg/generations_test.go new file mode 100644 index 000000000..b0463ac7d --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/generations_test.go @@ -0,0 +1,309 @@ +package deploymentpg_test + +import ( + "bytes" + "context" + "errors" + "slices" + "testing" + + "github.com/google/uuid" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" +) + +// generationsState is the stored deployment, its retained generations and the +// administrator audit, as one comparable value. +func generationsState(t *testing.T, f fixture) string { + t.Helper() + var state string + if err := f.pool.QueryRow(t.Context(), `SELECT jsonb_build_object( + 'deployment', (SELECT to_jsonb(d) FROM runtime_deployment d), + 'generations', (SELECT coalesce(jsonb_agg(to_jsonb(g) ORDER BY g.generation), '[]') FROM runtime_deployment_generations g), + 'audit', (SELECT coalesce(jsonb_agg(to_jsonb(a) ORDER BY a.created_at, a.id), '[]') FROM admin_audit_log a))::text`).Scan(&state); err != nil { + t.Fatal(err) + } + return state +} + +// generationsAudit lists the audited deployment actions in order. +func generationsAudit(t *testing.T, f fixture) []string { + t.Helper() + rows, err := f.pool.Query(t.Context(), "SELECT action FROM admin_audit_log WHERE resource_type='sandbox_deployment' ORDER BY created_at, id") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var actions []string + for rows.Next() { + var action string + if err := rows.Scan(&action); err != nil { + t.Fatal(err) + } + actions = append(actions, action) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return actions +} + +// generationsCredential returns the stored sealed credential after checking +// the deployment is at generation. +func generationsCredential(t *testing.T, f fixture, generation uint64) []byte { + t.Helper() + var stored int64 + var sealed []byte + if err := f.pool.QueryRow(t.Context(), "SELECT generation,provider_credential FROM runtime_deployment").Scan(&stored, &sealed); err != nil || uint64(stored) != generation || len(sealed) == 0 { + t.Fatal("stored credential", stored, len(sealed), err) + } + return sealed +} + +func TestE2BChangeClassifierOmittedKeyAndExplicitSameKey(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + input := setupE2BSelection() + id, _ := f.initialize(t, changes, input) + configuration := input.Configuration.(*e2b.DeploymentConfiguration) + key := configuration.APIKey + configuration.APIKey = "" + input.Resources = sandbox.Resources{} + input.ExpectedGeneration = 1 + resolved, unchanged, err := changes.ClassifyChange(t.Context(), id, input) + if err != nil || !unchanged || resolved.Configuration.(*e2b.DeploymentConfiguration).APIKey != key || resolved.Resources.CPUs == 0 { + t.Fatal(unchanged, err) + } + configuration.APIKey = key + configuration.CredentialSupplied = true + if _, unchanged, err = changes.ClassifyChange(t.Context(), id, input); err != nil || unchanged { + t.Fatal("explicit same key skipped verification", err) + } + invalid := input + invalid.Runtime = &sandbox.RuntimeRelease{} + if _, _, err := changes.ClassifyChange(t.Context(), id, invalid); err == nil { + t.Fatal("omitted resources erased forbidden Runtime input") + } + input.ExpectedGeneration = 0 + _, _, err = changes.ClassifyChange(t.Context(), id, input) + var stale *deployment.GenerationStaleError + if !errors.As(err, &stale) { + t.Fatal("stale did not precede no-op", err) + } +} + +func TestGenerationPinRetainsOfflineZeroResourceFallback(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + _, view := f.initialize(t, changes, sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 4, MaxRetained: 16}) + f.connect(t, node.NodeID) + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, node.NodeID); err != nil { + t.Fatal(err) + } + // Retain the generation and advance the deployment directly, isolating the + // persistence invariant from the node generation protocol. + tx, err := f.pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + q := sqlc.New(tx) + if _, err = q.LockRuntimeDeployment(t.Context()); err != nil { + t.Fatal(err) + } + if err = q.RetainSandboxGeneration(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(t.Context(), `UPDATE runtime_deployment SET generation=2`); err != nil { + t.Fatal(err) + } + if err = tx.Commit(t.Context()); err != nil { + t.Fatal(err) + } + if err = changes.CollectGenerations(t.Context()); err != nil { + t.Fatal(err) + } + rows, err := f.service.GenerationPage(t.Context(), -1) + if err != nil || len(rows) != 1 { + t.Fatal("offline pin was collected", rows, err) + } + nodes, err := f.service.ListNodes(t.Context()) + if err != nil || len(nodes) != 1 || nodes[0].Rollout.State != "unknown" || nodes[0].Rollout.ReadyGeneration == nil || *nodes[0].Rollout.ReadyGeneration != 1 { + t.Fatal(nodes, err) + } + if _, err = f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET removed_at=clock_timestamp(),ready_generation=NULL WHERE id=$1`, node.NodeID); err != nil { + t.Fatal(err) + } + if err = changes.CollectGenerations(t.Context()); err != nil { + t.Fatal(err) + } + rows, err = f.service.GenerationPage(t.Context(), -1) + if err != nil || len(rows) != 0 { + t.Fatal(rows, err) + } +} + +// Two changes submitted against the same generation: one commits, the other +// reports the generation it lost to. +func TestConcurrentUpdatesAtOneGenerationHaveOneWinner(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + input := sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")} + id, view := f.initialize(t, changes, input) + type outcome struct { + view deployment.View + cpus uint32 + err error + } + outcomes := make(chan outcome, 2) + start := make(chan struct{}) + for i := 1; i <= 2; i++ { + change := input + change.Resources.CPUs += uint32(i) + change.ExpectedGeneration = view.Generation + go func() { + <-start + committed, err := changes.Update(admin(t), id, change) + outcomes <- outcome{view: committed, cpus: change.Resources.CPUs, err: err} + }() + } + close(start) + var winners []outcome + for range 2 { + o := <-outcomes + var stale *deployment.GenerationStaleError + switch { + case o.err == nil: + winners = append(winners, o) + case errors.As(o.err, &stale): + if stale.CurrentGeneration != view.Generation+1 { + t.Fatal("stale change reported the wrong current generation", stale.CurrentGeneration) + } + default: + t.Fatal(o.err) + } + } + if len(winners) != 1 { + t.Fatal("changes at one generation did not have exactly one winner", len(winners)) + } + winner := winners[0] + if winner.view.Generation != view.Generation+1 || winner.view.Specification == nil || winner.view.Specification.Resources.CPUs != winner.cpus { + t.Fatal("winning change was not committed", winner.view) + } + current, err := f.service.View(t.Context()) + if err != nil || current.Generation != view.Generation+1 || current.Specification == nil || current.Specification.Resources.CPUs != winner.cpus { + t.Fatal("the losing change overwrote the winner", current, err) + } + if audit := generationsAudit(t, f); !slices.Equal(audit, []string{"change"}) { + t.Fatal("audit does not record exactly the winning change", audit) + } +} + +// A replaced E2B key is sealed for the new generation only; an omitted key +// keeps the current one, sealed again for the next generation. +func TestE2BCredentialReplacementSealsTheNewGeneration(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + input := setupE2BSelection() + configured := input.Configuration.(*e2b.DeploymentConfiguration) + id, view := f.initialize(t, changes, input) + open := func(sealed []byte, generation uint64) (string, error) { + secret, err := f.cipher.OpenSandboxDeployment(sealed, id, generation) + return string(secret), err + } + first := generationsCredential(t, f, view.Generation) + if key, err := open(first, view.Generation); err != nil || key != configured.APIKey { + t.Fatal("initial credential was not sealed for its generation", err) + } + + replacement := "replacement-private-api-key" + replace := sandbox.Selection{DeploymentSpec: input.DeploymentSpec, Provider: "e2b", ExpectedGeneration: view.Generation, + Configuration: &e2b.DeploymentConfiguration{APIKey: replacement, CredentialSupplied: true, Template: configured.Template}} + replaced, err := changes.Update(admin(t), id, replace) + if err != nil || replaced.Generation != view.Generation+1 || !replaced.CredentialConfigured { + t.Fatal("credential replacement was not committed", replaced, err) + } + second := generationsCredential(t, f, replaced.Generation) + if bytes.Equal(second, first) { + t.Fatal("replacement kept the previous ciphertext") + } + if key, err := open(second, replaced.Generation); err != nil || key != replacement { + t.Fatal("replacement was not sealed for the new generation", err) + } + if _, err := open(second, view.Generation); err == nil { + t.Fatal("replacement opened for the previous generation") + } + setup, err := f.service.Setup(t.Context()) + if err != nil || setup.Generation != replaced.Generation || setup.Configuration.(*e2b.DeploymentConfiguration).APIKey != replacement { + t.Fatal("setup does not use the replacement", err) + } + + omitted := sandbox.Selection{DeploymentSpec: input.DeploymentSpec, Provider: "e2b", ExpectedGeneration: replaced.Generation, + Configuration: &e2b.DeploymentConfiguration{Template: "next:" + uuid.NewString()}} + resolved, unchanged, err := changes.ClassifyChange(t.Context(), id, omitted) + if err != nil || unchanged || resolved.ReplacesCredential() || resolved.Configuration.(*e2b.DeploymentConfiguration).APIKey != replacement { + t.Fatal("omitted key did not resolve to the current key", unchanged, err) + } + changed, err := changes.Update(admin(t), id, resolved) + if err != nil || changed.Generation != replaced.Generation+1 { + t.Fatal(changed, err) + } + if key, err := open(generationsCredential(t, f, changed.Generation), changed.Generation); err != nil || key != replacement { + t.Fatal("omitted key did not keep the current key", err) + } + setup, err = f.service.Setup(t.Context()) + if err != nil || setup.Configuration.(*e2b.DeploymentConfiguration).APIKey != replacement || setup.Configuration.(*e2b.DeploymentConfiguration).Template != omitted.Configuration.(*e2b.DeploymentConfiguration).Template { + t.Fatal("setup lost the kept key or the new template", err) + } + if audit := generationsAudit(t, f); !slices.Equal(audit, []string{"replace_credential", "change"}) { + t.Fatal("audit does not distinguish the replacement", audit) + } +} + +// A rejected change leaves the deployment, its retained generations and the +// audit as they were, including writes made before the rejection. +func TestRejectedChangeLeavesDeploymentUnchanged(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + input := sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")} + id, view := f.initialize(t, changes, input) + // The ready node pins the current generation, so a committed change also + // retains it. + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + f.connect(t, node.NodeID) + before := generationsState(t, f) + change := input + change.Resources.CPUs++ + change.ExpectedGeneration = view.Generation + // The audit write comes last and fails without administrator provenance. + if _, err := changes.Update(t.Context(), id, change); !errors.Is(err, adminaudit.ErrInvalidSource) { + t.Fatal("change without administrator provenance", err) + } + if after := generationsState(t, f); after != before { + t.Fatal("failed audit write left the change behind", after) + } + other := sandbox.Selection{Provider: "microsandbox", DeploymentSpec: testSpecification("microsandbox"), ExpectedGeneration: view.Generation} + var reset *deployment.ResetRequiredError + if _, err := changes.Update(admin(t), id, other); !errors.As(err, &reset) || reset.CurrentProvider != "docker" || reset.RequestedProvider != "microsandbox" { + t.Fatal("backend change without a reset", err) + } + if after := generationsState(t, f); after != before { + t.Fatal("backend change without a reset changed the deployment", after) + } + committed, err := changes.Update(admin(t), id, change) + if err != nil || committed.Generation != view.Generation+1 { + t.Fatal(committed, err) + } + retained, err := f.service.GenerationPage(t.Context(), -1) + if err != nil || len(retained) != 1 || retained[0].Generation != view.Generation { + t.Fatal("committed change did not retain the pinned generation", retained, err) + } + if audit := generationsAudit(t, f); !slices.Equal(audit, []string{"change"}) { + t.Fatal("committed change was not audited", audit) + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/host_history_test.go b/services/core/internal/persistence/postgres/deploymentpg/host_history_test.go new file mode 100644 index 000000000..ee83493e4 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/host_history_test.go @@ -0,0 +1,171 @@ +package deploymentpg_test + +import ( + "encoding/json" + "errors" + "math" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +func hostHistoryPtr[T any](value T) *T { return &value } + +// hostHistoryNode initializes a Docker deployment and connects one enrolled +// node, returning it with its connection and the owner epoch. +func hostHistoryNode(t *testing.T) (fixture, string, string, uint64) { + t.Helper() + f := newFixture(t) + changes, _ := f.execution(t) + _, view := f.initialize(t, changes, sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 2, MaxRetained: 8}) + connection := f.connect(t, node.NodeID) + epoch, err := f.adapter.OwnerEpoch(t.Context()) + if err != nil { + t.Fatal(err) + } + return f, node.NodeID, connection, epoch +} + +func TestNodeHostHistorySamplingAndDetail(t *testing.T) { + f, node, connection, epoch := hostHistoryNode(t) + now := time.Now().UTC() + host := &deployment.NodeHost{EffectiveCPUCores: hostHistoryPtr(2.0), CPUUtilization: hostHistoryPtr(0.35), TotalMemoryBytes: hostHistoryPtr(int64(4096)), AvailableMemoryBytes: hostHistoryPtr(int64(1024)), AvailableDiskBytes: hostHistoryPtr(int64(8192)), ObservedAt: &now} + health := deployment.NodeHealth{ProviderReady: true, Host: host} + if err := f.service.Heartbeat(t.Context(), node, connection, epoch, health); err != nil { + t.Fatal(err) + } + for i, want := range []int64{1, 0} { + if n, err := f.adapter.SampleHostHistory(t.Context()); err != nil || n != want { + t.Fatal(i, n, err) + } + } + detail, err := f.service.NodeDetail(t.Context(), node, "1h") + if err != nil || detail.Host.TotalMemoryBytes == nil || *detail.Host.CPUUtilization != 0.35 || len(detail.History.Points) != 60 { + t.Fatal(detail, err) + } + // The current partial minute does not enter history yet. + for _, p := range detail.History.Points { + if p.CPUUtilizationMax != nil { + t.Fatal("partial bucket leaked", p) + } + } + list, err := f.service.ListNodes(t.Context()) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(list[0]) + var fields map[string]json.RawMessage + _ = json.Unmarshal(raw, &fields) + for _, key := range []string{"host", "history", "cpu_utilization", "total_memory_bytes", "effective_cpu_cores"} { + if _, exists := fields[key]; exists { + t.Fatal("list contract expanded", key, string(raw)) + } + } + // Sample maxima/minima differ from the latest reading. + end := time.Now().UTC().Truncate(time.Minute) + start := end.Add(-time.Minute) + for _, sample := range []struct { + at time.Time + cpu any + mem any + disk any + }{ + {start.Add(time.Second), 0.1, int64(100), int64(800)}, + {start.Add(20 * time.Second), 0.8, int64(70), int64(900)}, + {start.Add(40 * time.Second), nil, int64(500), int64(600)}, + {start.Add(-time.Minute), nil, nil, nil}, + } { + if _, err := f.pool.Exec(t.Context(), "INSERT INTO node_host_history_samples(node_id, observed_at,cpu_utilization,memory_used_bytes,available_disk_bytes) VALUES($1,$2,$3,$4,$5)", node, sample.at, sample.cpu, sample.mem, sample.disk); err != nil { + t.Fatal(err) + } + } + detail, err = f.service.NodeDetail(t.Context(), node, "1h") + if err != nil { + t.Fatal(err) + } + last := detail.History.Points[len(detail.History.Points)-1] + if !last.Start.Equal(start) || last.CPUUtilizationMax == nil || *last.CPUUtilizationMax != 0.8 || *last.MemoryUsedBytesMax != 500 || *last.AvailableDiskBytesMin != 600 { + t.Fatal(last) + } + previous := detail.History.Points[len(detail.History.Points)-2] + if previous.CPUUtilizationMax != nil || previous.MemoryUsedBytesMax != nil || previous.AvailableDiskBytesMin != nil { + t.Fatal(previous) + } + for name, want := range map[string]int{"6h": 72, "24h": 96} { + detail, err := f.service.NodeDetail(t.Context(), node, name) + if err != nil || len(detail.History.Points) != want { + t.Fatal(name, detail, err) + } + } + if _, err := f.service.NodeDetail(t.Context(), uuid.NewString(), "1h"); !errors.Is(err, deployment.ErrNotFound) { + t.Fatal(err) + } + for _, name := range []string{"", "7d", "other"} { + if _, err := f.service.NodeDetail(t.Context(), node, name); !errors.Is(err, deployment.ErrInvalidInput) { + t.Fatal(err) + } + } + // A disconnected node cannot create another history row, even with a fresh last observation. + next := now.Add(time.Millisecond) + host.ObservedAt = &next + if err := f.service.Heartbeat(t.Context(), node, connection, epoch, health); err != nil { + t.Fatal(err) + } + if err := f.service.DisconnectNode(t.Context(), node, connection, epoch); err != nil { + t.Fatal(err) + } + if n, err := f.adapter.SampleHostHistory(t.Context()); err != nil || n != 0 { + t.Fatal(n, err) + } + // A restarted service can query existing durable history without re-sampling. + _, fresh := f.withPublicURL(t, fixturePublicURL) + restored, err := fresh.NodeDetail(t.Context(), node, "1h") + if err != nil || restored.Online || restored.History.Points[59].CPUUtilizationMax == nil { + t.Fatal(restored, err) + } + if !restored.Host.ObservedAt.Equal(next) { + t.Fatal(restored.Host) + } +} + +func TestNodeHostHistoryFencingAndUnknown(t *testing.T) { + f, node, conn, epoch := hostHistoryNode(t) + for _, at := range []time.Time{time.Now().Add(-time.Minute), time.Now().Add(time.Hour)} { + if err := f.service.Heartbeat(t.Context(), node, conn, epoch, deployment.NodeHealth{Host: &deployment.NodeHost{ObservedAt: &at}}); err != nil { + t.Fatal(err) + } + if n, err := f.adapter.SampleHostHistory(t.Context()); err != nil || n != 0 { + t.Fatal(n, err) + } + } + now := time.Now().UTC() + health := deployment.NodeHealth{Host: &deployment.NodeHost{ObservedAt: &now}} + if err := f.service.Heartbeat(t.Context(), node, uuid.NewString(), epoch, health); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal(err) + } + if err := f.service.Heartbeat(t.Context(), node, conn, epoch, health); err != nil { + t.Fatal(err) + } + if n, err := f.adapter.SampleHostHistory(t.Context()); err != nil || n != 1 { + t.Fatal(n, err) + } + var cpu *float64 + var memory, disk *int64 + if err := f.pool.QueryRow(t.Context(), "SELECT cpu_utilization,memory_used_bytes,available_disk_bytes FROM node_host_history_samples WHERE node_id=$1", node).Scan(&cpu, &memory, &disk); err != nil || cpu != nil || memory != nil || disk != nil { + t.Fatal(cpu, memory, disk, err) + } + for _, host := range []*deployment.NodeHost{ + {ObservedAt: &now, CPUUtilization: hostHistoryPtr(1.1)}, {ObservedAt: &now, CPUUtilization: hostHistoryPtr(math.NaN())}, + {ObservedAt: &now, TotalMemoryBytes: hostHistoryPtr(int64(10)), AvailableMemoryBytes: hostHistoryPtr(int64(11))}, + {ObservedAt: &now, AvailableDiskBytes: hostHistoryPtr(int64(-1))}, {ObservedAt: &now, EffectiveCPUCores: hostHistoryPtr(0.0)}, {}, + } { + if err := f.service.Heartbeat(t.Context(), node, conn, epoch, deployment.NodeHealth{Host: host}); !errors.Is(err, deployment.ErrInvalidInput) { + t.Fatal(host, err) + } + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/nodes_test.go b/services/core/internal/persistence/postgres/deploymentpg/nodes_test.go new file mode 100644 index 000000000..a06cb97f0 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/nodes_test.go @@ -0,0 +1,421 @@ +package deploymentpg_test + +import ( + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "os" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// nodesExec changes rows the node operations read, standing in for state they +// cannot reach through their own API. +func nodesExec(t *testing.T, pool *pgxpool.Pool, query string, args ...any) { + t.Helper() + if _, err := pool.Exec(t.Context(), query, args...); err != nil { + t.Fatal(err) + } +} + +// nodesTokenDigest is the stored form of an enrollment token. +func nodesTokenDigest(token string) string { + digest := sha256.Sum256([]byte(token)) + return hex.EncodeToString(digest[:]) +} + +// nodesEnrollment is a valid enrollment of a new node on view. +func nodesEnrollment(view deployment.View, name, credential string) deployment.Enrollment { + return deployment.Enrollment{NodeID: uuid.NewString(), Name: name, Credential: strings.Repeat(credential, 64), Provider: view.Provider, BackendFingerprint: strings.Repeat("b", 64), + SpecificationDigest: view.SpecificationDigest, DeploymentGeneration: view.Generation, CoreURL: fixturePublicURL} +} + +func TestEnrollmentApprovedCapacity(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + _, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: testSpecification("microsandbox")}) + for _, capacity := range []deployment.Capacity{{MaxActive: 0, MaxRetained: 8}, {MaxActive: 3, MaxRetained: 2}, {MaxActive: 1, MaxRetained: 1000001}} { + if _, err := f.service.CreateEnrollment(t.Context(), capacity); !errors.Is(err, deployment.ErrInvalidInput) { + t.Fatal("invalid capacity accepted", err) + } + } + token, err := f.service.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 3}) + if err != nil { + t.Fatal(err) + } + config, err := f.service.NodeConfiguration(t.Context(), "", token.Token, 0) + if err != nil || config.MaxActive != 1 || config.MaxRetained != 3 { + t.Fatal("bootstrap lost approved capacity", config, err) + } + input := nodesEnrollment(view, "approved", "a") + identity, err := f.service.Enroll(t.Context(), token.Token, input) + if err != nil || identity.MaxActive != 1 || identity.MaxRetained != 3 { + t.Fatal("enrollment did not apply token capacity", identity, err) + } + if _, err := f.service.Enroll(t.Context(), token.Token, input); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("consumed token reused", err) + } + if err := f.service.UpdateNode(t.Context(), input.NodeID, deployment.NodeUpdate{Name: "approved", MaxActive: 2, MaxRetained: 5}); err != nil { + t.Fatal(err) + } + // Microsandbox uses both limits, so a retained limit below the active one is rejected and nothing is written. + if err := f.service.UpdateNode(t.Context(), input.NodeID, deployment.NodeUpdate{Name: "rejected", MaxActive: 4, MaxRetained: 3}); !errors.Is(err, deployment.ErrInvalidInput) { + t.Fatal("retained limit below active accepted", err) + } + var name string + if err := f.pool.QueryRow(t.Context(), "SELECT name FROM runtime_nodes WHERE id=$1", input.NodeID).Scan(&name); err != nil || name != "approved" { + t.Fatal("rejected update was written", name, err) + } + identity, err = f.service.AuthenticateNode(t.Context(), input.NodeID, input.Credential) + if err != nil || identity.MaxActive != 2 || identity.MaxRetained != 5 { + t.Fatal("credential read ignored admin update", identity, err) + } + config, err = f.service.NodeConfiguration(t.Context(), input.NodeID, input.Credential, 0) + if err != nil || config.MaxActive != 2 || config.MaxRetained != 5 { + t.Fatal("configuration read ignored admin update", config, err) + } +} + +// Docker never suspends a sandbox, so every read reports its retained limit as its +// active limit, including for a token or node stored before Core applied that rule. +func TestDockerRetainedLimitFollowsActive(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + _, view := f.initialize(t, changes, sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")}) + token, err := f.service.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 3, MaxRetained: 1}) + if err != nil { + t.Fatal(err) + } + current := nodesEnrollment(view, "Docker", "d") + if identity, err := f.service.Enroll(t.Context(), token.Token, current); err != nil || identity.MaxRetained != 3 { + t.Fatal("enrollment kept a separate Docker retained limit", identity, err) + } + legacyToken, err := f.service.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 2, MaxRetained: 8}) + if err != nil { + t.Fatal(err) + } + nodesExec(t, f.pool, "UPDATE runtime_node_enrollments SET max_retained=8 WHERE token_sha256=$1", nodesTokenDigest(legacyToken.Token)) + if config, err := f.service.NodeConfiguration(t.Context(), "", legacyToken.Token, 0); err != nil || config.MaxRetained != 2 { + t.Fatal("bootstrap reported a legacy Docker retained limit", config, err) + } + legacy := nodesEnrollment(view, "Legacy", "l") + if _, err := f.service.Enroll(t.Context(), legacyToken.Token, legacy); err != nil { + t.Fatal(err) + } + var stored int32 + if err := f.pool.QueryRow(t.Context(), "SELECT max_retained FROM runtime_nodes WHERE id=$1", legacy.NodeID).Scan(&stored); err != nil || stored != 2 { + t.Fatal("enrollment stored a legacy Docker retained limit", stored, err) + } + nodesExec(t, f.pool, "UPDATE runtime_nodes SET max_retained=8 WHERE id=$1", legacy.NodeID) + if identity, err := f.service.AuthenticateNode(t.Context(), legacy.NodeID, legacy.Credential); err != nil || identity.MaxRetained != 2 { + t.Fatal("node identity reported a legacy Docker retained limit", identity, err) + } + if config, err := f.service.NodeConfiguration(t.Context(), legacy.NodeID, legacy.Credential, 0); err != nil || config.MaxRetained != 2 { + t.Fatal("node configuration reported a legacy Docker retained limit", config, err) + } + if err := f.service.UpdateNode(t.Context(), current.NodeID, deployment.NodeUpdate{Name: "Docker", MaxActive: 5, MaxRetained: 12}); err != nil { + t.Fatal(err) + } + nodes, err := f.service.ListNodes(t.Context()) + if err != nil || len(nodes) != 2 { + t.Fatal(nodes, err) + } + for _, n := range nodes { + if n.MaxRetained != n.MaxActive || (n.ID == current.NodeID && n.MaxActive != 5) { + t.Fatal("node list kept a separate Docker retained limit", n) + } + } +} + +// An unknown or consumed token and a wrong node credential are rejected before +// any deployment state or enrollment input is judged, so they reveal nothing. +func TestNodeCredentialPrecedesDeploymentState(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + unknown := strings.Repeat("u", 64) + // Before initialization the deployment is unavailable. + early := deployment.Enrollment{NodeID: uuid.NewString(), Name: "early", Credential: strings.Repeat("e", 64), Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), + SpecificationDigest: strings.Repeat("d", 64), DeploymentGeneration: 1, CoreURL: fixturePublicURL} + if _, err := f.service.Enroll(t.Context(), unknown, early); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("uninitialized deployment judged before the token", err) + } + if _, err := f.service.NodeConfiguration(t.Context(), "", unknown, 0); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("uninitialized deployment judged before the token", err) + } + if _, err := f.service.AuthenticateNode(t.Context(), early.NodeID, early.Credential); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("unknown node authenticated", err) + } + _, view := f.initialize(t, changes, sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")}) + consumed, err := f.service.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + if err != nil { + t.Fatal(err) + } + node := nodesEnrollment(view, "ordered", "c") + if _, err := f.service.Enroll(t.Context(), consumed.Token, node); err != nil { + t.Fatal(err) + } + fresh, err := f.service.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + change func(*deployment.Enrollment) + want error + }{ + {"provider", func(e *deployment.Enrollment) { e.Provider = "microsandbox" }, deployment.ErrInvalidInput}, + {"generation", func(e *deployment.Enrollment) { e.DeploymentGeneration++ }, deployment.ErrSpecificationMismatch}, + {"specification", func(e *deployment.Enrollment) { e.SpecificationDigest = strings.Repeat("0", 64) }, deployment.ErrSpecificationMismatch}, + {"address", func(e *deployment.Enrollment) { e.CoreURL = "https://other.example" }, deployment.ErrNodeAddressMismatch}, + } { + input := nodesEnrollment(view, "refused", "r") + tc.change(&input) + for _, token := range []string{unknown, consumed.Token} { + if _, err := f.service.Enroll(t.Context(), token, input); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal(tc.name, "judged before the token", err) + } + } + if _, err := f.service.Enroll(t.Context(), fresh.Token, input); !errors.Is(err, tc.want) { + t.Fatal(tc.name, err) + } + } + if _, err := f.service.NodeConfiguration(t.Context(), "", unknown, 0); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("unknown token configured", err) + } + if _, err := f.service.NodeConfiguration(t.Context(), "", consumed.Token, 0); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("consumed token configured", err) + } + // None of the refusals consumed the valid token. + if _, err := f.service.NodeConfiguration(t.Context(), "", fresh.Token, 0); err != nil { + t.Fatal("refused enrollment consumed its token", err) + } + // A node whose enrollment identity no longer matches is refused only with + // its own credential. + nodesExec(t, f.pool, "UPDATE runtime_nodes SET specification_digest=$2 WHERE id=$1", node.NodeID, strings.Repeat("0", 64)) + wrong := strings.Repeat("w", 64) + if _, err := f.service.AuthenticateNode(t.Context(), node.NodeID, wrong); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("identity judged before the credential", err) + } + if _, err := f.service.NodeStatus(t.Context(), node.NodeID, wrong); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("identity judged before the credential", err) + } + if _, err := f.service.NodeConfiguration(t.Context(), node.NodeID, wrong, 0); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("identity judged before the credential", err) + } + if _, err := f.service.AuthenticateNode(t.Context(), node.NodeID, node.Credential); !errors.Is(err, deployment.ErrSpecificationMismatch) { + t.Fatal("mismatched identity authenticated", err) + } + if _, err := f.service.NodeConfiguration(t.Context(), node.NodeID, node.Credential, 0); !errors.Is(err, deployment.ErrSpecificationMismatch) { + t.Fatal("mismatched identity configured", err) + } +} + +// A node must connect to the address Core advertises now. A refused enrollment +// registers nothing and leaves its token usable. +func TestEnrollmentAddressMismatchKeepsToken(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + _, view := f.initialize(t, changes, sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")}) + token, err := f.service.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + if err != nil { + t.Fatal(err) + } + config, err := f.service.NodeConfiguration(t.Context(), "", token.Token, 0) + if err != nil || config.CoreURL != fixturePublicURL { + t.Fatal("bootstrap did not advertise the public URL", config, err) + } + input := nodesEnrollment(view, "addressed", "a") + input.CoreURL = "https://other.example" + if _, err := f.service.Enroll(t.Context(), token.Token, input); !errors.Is(err, deployment.ErrNodeAddressMismatch) { + t.Fatal("another Core address enrolled", err) + } + // The public URL changed after the node read its configuration. + _, moved := f.withPublicURL(t, "https://moved.example") + input.CoreURL = fixturePublicURL + if _, err := moved.Enroll(t.Context(), token.Token, input); !errors.Is(err, deployment.ErrNodeAddressMismatch) { + t.Fatal("a previous public URL enrolled", err) + } + var consumed bool + if err := f.pool.QueryRow(t.Context(), "SELECT consumed_at IS NOT NULL FROM runtime_node_enrollments WHERE token_sha256=$1", nodesTokenDigest(token.Token)).Scan(&consumed); err != nil || consumed { + t.Fatal("address mismatch consumed the token", consumed, err) + } + if nodes, err := f.service.ListNodes(t.Context()); err != nil || len(nodes) != 0 { + t.Fatal("address mismatch registered a node", nodes, err) + } + if _, err := f.service.Enroll(t.Context(), token.Token, input); err != nil { + t.Fatal("token unusable after an address mismatch", err) + } + nodes, err := f.service.ListNodes(t.Context()) + if err != nil || len(nodes) != 1 || nodes[0].ID != input.NodeID || nodes[0].CoreURL != fixturePublicURL { + t.Fatal(nodes, err) + } +} + +// A replayed enrollment finds its token consumed and recovers by authenticating. +// A node ID stays reserved, even after removal, and a refused ID keeps the token. +func TestEnrollmentReplay(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + _, view := f.initialize(t, changes, sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")}) + token, err := f.service.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + if err != nil { + t.Fatal(err) + } + input := nodesEnrollment(view, "replayed", "r") + identity, err := f.service.Enroll(t.Context(), token.Token, input) + if err != nil { + t.Fatal(err) + } + if _, err := f.service.Enroll(t.Context(), token.Token, input); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("replay reused the token", err) + } + if recovered, err := f.service.AuthenticateNode(t.Context(), input.NodeID, input.Credential); err != nil || recovered != identity { + t.Fatal("lost response cannot recover", recovered, err) + } + second, err := f.service.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + if err != nil { + t.Fatal(err) + } + other := input + other.Credential = strings.Repeat("o", 64) + for _, candidate := range []deployment.Enrollment{input, other} { + if _, err := f.service.Enroll(t.Context(), second.Token, candidate); !errors.Is(err, deployment.ErrNodeExists) { + t.Fatal("enrolled node ID reused", err) + } + } + if err := f.service.RemoveNode(t.Context(), input.NodeID); err != nil { + t.Fatal(err) + } + if _, err := f.service.Enroll(t.Context(), second.Token, input); !errors.Is(err, deployment.ErrNodeExists) { + t.Fatal("removed node ID reused", err) + } + if _, err := f.service.AuthenticateNode(t.Context(), input.NodeID, input.Credential); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("removed node credential accepted", err) + } + if _, err := f.service.Enroll(t.Context(), second.Token, nodesEnrollment(view, "next", "n")); err != nil { + t.Fatal("refused node ID consumed the token", err) + } +} + +func TestNodeUpdateBounds(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + _, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: testSpecification("microsandbox")}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 2, MaxRetained: 4}) + for _, tc := range []struct { + update deployment.NodeUpdate + code, param string + }{ + {deployment.NodeUpdate{Name: "", MaxActive: 1, MaxRetained: 1}, "invalid_name", "name"}, + {deployment.NodeUpdate{Name: " ", MaxActive: 1, MaxRetained: 1}, "invalid_name", "name"}, + {deployment.NodeUpdate{Name: strings.Repeat("n", 129), MaxActive: 1, MaxRetained: 1}, "invalid_name", "name"}, + {deployment.NodeUpdate{Name: "line\nbreak", MaxActive: 1, MaxRetained: 1}, "invalid_name", "name"}, + {deployment.NodeUpdate{Name: "bounded", MaxActive: 0, MaxRetained: 1}, "invalid_node_capacity", "max_active"}, + {deployment.NodeUpdate{Name: "bounded", MaxActive: 1000001, MaxRetained: 1000001}, "invalid_node_capacity", "max_active"}, + {deployment.NodeUpdate{Name: "bounded", MaxActive: 4, MaxRetained: 3}, "invalid_node_capacity", "max_retained"}, + {deployment.NodeUpdate{Name: "bounded", MaxActive: 1, MaxRetained: 1000001}, "invalid_node_capacity", "max_retained"}, + } { + err := f.service.UpdateNode(t.Context(), node.NodeID, tc.update) + var validation *deployment.NodeValidationError + if !errors.Is(err, deployment.ErrInvalidInput) || !errors.As(err, &validation) || validation.Code != tc.code || validation.Param != tc.param { + t.Fatal("update outside the bounds", tc.update, err) + } + } + var name string + var active, retained int32 + if err := f.pool.QueryRow(t.Context(), "SELECT name,max_active,max_retained FROM runtime_nodes WHERE id=$1", node.NodeID).Scan(&name, &active, &retained); err != nil || name != node.Name || active != 2 || retained != 4 { + t.Fatal("rejected update was written", name, active, retained, err) + } + for _, update := range []deployment.NodeUpdate{{Name: strings.Repeat("n", 128), MaxActive: 1000000, MaxRetained: 1000000}, {Name: "single", MaxActive: 1, MaxRetained: 1}} { + if err := f.service.UpdateNode(t.Context(), node.NodeID, update); err != nil { + t.Fatal("update at the bounds rejected", update, err) + } + } + nodes, err := f.service.ListNodes(t.Context()) + if err != nil || len(nodes) != 1 || nodes[0].Name != "single" || nodes[0].MaxActive != 1 || nodes[0].MaxRetained != 1 { + t.Fatal(nodes, err) + } + valid := deployment.NodeUpdate{Name: "missing", MaxActive: 1, MaxRetained: 1} + if err := f.service.UpdateNode(t.Context(), uuid.NewString(), valid); !errors.Is(err, deployment.ErrNotFound) { + t.Fatal("unknown node updated", err) + } + if err := f.service.UpdateNode(t.Context(), "not-a-node", valid); !errors.Is(err, deployment.ErrInvalidInput) { + t.Fatal("malformed node ID updated", err) + } + if err := f.service.RemoveNode(t.Context(), uuid.NewString()); !errors.Is(err, deployment.ErrNotFound) { + t.Fatal("unknown node removed", err) + } +} + +func TestNodeGenerationDowngradePreservesServingProtocol(t *testing.T) { + for _, mode := range []string{"v2", "old_v1", "current_v1"} { + t.Run(mode, func(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + input := sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")} + installation, first := f.initialize(t, changes, input) + node := f.enroll(t, first, deployment.Capacity{MaxActive: 4, MaxRetained: 16}) + f.connect(t, node.NodeID) + switch mode { + case "v2": + connection := f.connect(t, node.NodeID) + if err := f.service.HeartbeatGenerations(t.Context(), node.NodeID, connection, first.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: first.Generation, SpecificationDigest: first.SpecificationDigest, State: "ready"}}); err != nil { + t.Fatal(err) + } + case "old_v1": + input.Resources.CPUs++ + input.ExpectedGeneration = first.Generation + next, err := changes.Update(admin(t), installation, input) + if err != nil { + t.Fatal(err) + } + if next.Generation != first.Generation+1 || next.OwnerEpoch != first.OwnerEpoch { + t.Fatal("target change replaced execution ownership", next) + } + } + db := sql.OpenDB(stdlib.GetConnector(*f.pool.Config().ConnConfig)) + defer db.Close() + migration, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../../../migrations"), goose.WithTableName("agents_api_schema_version")) + if err != nil { + t.Fatal(err) + } + _, err = migration.DownTo(t.Context(), 81) + if mode == "current_v1" { + if err != nil { + t.Fatal("safe v1 downgrade refused", err) + } + } else { + if err == nil { + t.Fatal("downgrade discarded required node protocol") + } + // DownTo may have removed later, reversible migrations before the + // node protocol migration refused the downgrade. Restore the current + // schema before using this version of the adapter to verify recovery. + if _, err = migration.Up(t.Context()); err != nil { + t.Fatal("refused downgrade could not restore current schema", err) + } + if _, err = f.service.NodeConfiguration(t.Context(), node.NodeID, node.Credential, 1); err != nil { + t.Fatal("refused downgrade damaged retained recovery", err) + } + if err = f.service.RemoveNode(t.Context(), node.NodeID); err != nil { + t.Fatal(err) + } + if _, err = migration.DownTo(t.Context(), 81); err != nil { + t.Fatal("removed node blocked downgrade", err) + } + } + if _, err = migration.Up(t.Context()); err != nil { + t.Fatal("node schema could not upgrade again", err) + } + }) + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/presence_test.go b/services/core/internal/persistence/postgres/deploymentpg/presence_test.go new file mode 100644 index 000000000..62eb23d68 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/presence_test.go @@ -0,0 +1,346 @@ +package deploymentpg_test + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" +) + +func presenceContext(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + t.Cleanup(cancel) + return ctx +} + +// presenceFixture initializes a Docker deployment and enrolls two nodes, neither +// connected, and returns them with the current owner epoch. +func presenceFixture(t *testing.T) (fixture, string, string, uint64) { + t.Helper() + f := newFixture(t) + changes, _ := f.execution(t) + _, view := f.initialize(t, changes, sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + other := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + epoch, err := f.adapter.OwnerEpoch(t.Context()) + if err != nil { + t.Fatal(err) + } + return f, node.NodeID, other.NodeID, epoch +} + +// presenceWaitBlocked returns once blocker holds up another backend. +func presenceWaitBlocked(t *testing.T, ctx context.Context, pool *pgxpool.Pool, blocker int32, done <-chan error) { + t.Helper() + for { + var blocked bool + if err := pool.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE $1=ANY(pg_blocking_pids(pid)))", blocker).Scan(&blocked); err != nil { + t.Fatal(err) + } + if blocked { + return + } + select { + case err := <-done: + t.Fatal("operation bypassed the node lock", err) + case <-ctx.Done(): + t.Fatal("node lock wait not observed") + case <-time.After(5 * time.Millisecond): + } + } +} + +func assertPresence(t *testing.T, pool *pgxpool.Pool, node, want string) { + t.Helper() + var actual pgtype.UUID + // Wait for any canceled transaction to roll back before reading its outcome. + if err := pool.QueryRow(presenceContext(t), "SELECT connection_id FROM runtime_nodes WHERE id=$1 FOR UPDATE", node).Scan(&actual); err != nil { + t.Fatal(err) + } + got := "" + if actual.Valid { + got = uuid.UUID(actual.Bytes).String() + } + if got != want { + t.Fatalf("presence = %q, want %q", got, want) + } +} + +func TestNodePresenceCanceledBlockedConnect(t *testing.T) { + f, node, other, epoch := presenceFixture(t) + ctx := presenceContext(t) + lock, err := f.pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer lock.Rollback(ctx) + var blocker int32 + if err := lock.QueryRow(ctx, "SELECT pg_backend_pid() FROM runtime_nodes WHERE id=$1 FOR UPDATE", node).Scan(&blocker); err != nil { + t.Fatal(err) + } + connecting, cancel := context.WithCancel(ctx) + defer cancel() + connection := uuid.NewString() + done := make(chan error, 1) + go func() { done <- f.service.ConnectNode(connecting, node, connection, epoch) }() + presenceWaitBlocked(t, ctx, f.pool, blocker, done) + var writer int32 + if err := f.pool.QueryRow(ctx, "SELECT pid FROM pg_stat_activity WHERE $1=ANY(pg_blocking_pids(pid)) LIMIT 1", blocker).Scan(&writer); err != nil { + t.Fatal(err) + } + // Another node does not share the blocked presence transaction's lock. + otherConnection := uuid.NewString() + if err := f.service.ConnectNode(ctx, other, otherConnection, epoch); err != nil { + t.Fatal(err) + } + if err := f.service.DisconnectNode(ctx, other, otherConnection, epoch); err != nil { + t.Fatal(err) + } + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatal("blocked connect cancellation", err) + } + case <-ctx.Done(): + t.Fatal("blocked connect did not cancel") + } + if err := lock.Rollback(ctx); err != nil { + t.Fatal(err) + } + // pgx cancellation can return before PostgreSQL stops the original statement. + // Observe backend exit so a late autocommit cannot escape the assertion. + for { + var gone bool + if err := f.pool.QueryRow(ctx, "SELECT NOT EXISTS (SELECT 1 FROM pg_stat_activity WHERE pid=$1)", writer).Scan(&gone); err != nil { + t.Fatal(err) + } + if gone { + break + } + select { + case <-ctx.Done(): + t.Fatal("canceled backend did not finish") + case <-time.After(time.Millisecond): + } + } + assertPresence(t, f.pool, node, "") +} + +type cancelPresenceAfterUpdate struct{ cancel context.CancelFunc } + +func (trace cancelPresenceAfterUpdate) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + return ctx +} +func (trace cancelPresenceAfterUpdate) TraceQueryEnd(_ context.Context, _ *pgx.Conn, result pgx.TraceQueryEndData) { + if result.Err == nil && result.CommandTag.Update() { + trace.cancel() + } +} + +func TestNodePresenceCanceledBeforeCommit(t *testing.T) { + f, node, _, epoch := presenceFixture(t) + ctx, cancel := context.WithCancel(presenceContext(t)) + defer cancel() + cfg := f.pool.Config() + // Cancel after PostgreSQL acknowledges UPDATE, before the adapter can commit it. + cfg.ConnConfig.Tracer = cancelPresenceAfterUpdate{cancel: cancel} + pool, err := pgxpool.NewWithConfig(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + adapter := deploymentpg.New(pgunit.NewPool(pool), f.cipher) + service, err := deployment.NewService(adapter, adapter, providers.Builtin(), fixturePublicURL) + if err != nil { + t.Fatal(err) + } + if err := service.ConnectNode(ctx, node, uuid.NewString(), epoch); !errors.Is(err, context.Canceled) { + t.Fatal("canceled UPDATE published presence", err) + } + assertPresence(t, f.pool, node, "") +} + +func TestNodePresenceDisconnectWaitsForCommit(t *testing.T) { + for _, guard := range []string{"matching", "newer_connection", "newer_epoch"} { + t.Run(guard, func(t *testing.T) { + f, node, other, epoch := presenceFixture(t) + ctx := presenceContext(t) + pending, err := f.pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer pending.Rollback(ctx) + current, cleanup := uuid.NewString(), "" + cleanup = current + cleanupEpoch := epoch + want := "" + if guard == "newer_connection" { + cleanup = uuid.NewString() + want = current + } + if guard == "newer_epoch" { + cleanupEpoch-- + want = current + } + if _, err := pending.Exec(ctx, "UPDATE runtime_nodes SET connection_id=$2,connected_epoch=$3 WHERE id=$1", node, current, epoch); err != nil { + t.Fatal(err) + } + var blocker int32 + if err := pending.QueryRow(ctx, "SELECT pg_backend_pid()").Scan(&blocker); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- f.service.DisconnectNode(ctx, node, cleanup, cleanupEpoch) }() + presenceWaitBlocked(t, ctx, f.pool, blocker, done) + otherConnection := uuid.NewString() + if err := f.service.ConnectNode(ctx, other, otherConnection, epoch); err != nil { + t.Fatal(err) + } + if err := f.service.DisconnectNode(ctx, other, otherConnection, epoch); err != nil { + t.Fatal(err) + } + if err := pending.Commit(ctx); err != nil { + t.Fatal(err) + } + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatal("cleanup did not settle after commit") + } + assertPresence(t, f.pool, node, want) + }) + } +} + +// A current node connection survives callbacks from the previous owner epoch. +func TestNodeStaleEpochCannotReplaceCurrentConnection(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 4}) + epoch, err := f.adapter.OwnerEpoch(t.Context()) + if err != nil { + t.Fatal(err) + } + // A new execution owner claims the installation and fences the old epoch. + if err := changes.Claim(t.Context(), installation); err != nil { + t.Fatal(err) + } + connection := f.connect(t, node.NodeID) + if err := f.service.ConnectNode(t.Context(), node.NodeID, uuid.NewString(), epoch); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("old Core replaced new connection", err) + } + if err := f.service.DisconnectNode(t.Context(), node.NodeID, connection, epoch); err != nil { + t.Fatal(err) + } + if err := f.service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: false}); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("old Core rewrote health", err) + } + nodes, err := f.service.ListNodes(t.Context()) + if err != nil || !nodes[0].Online || !nodes[0].ProviderReady { + t.Fatal("stale callback changed current epoch", nodes, err) + } +} + +func TestNodeDiagnosticReachesListAndDetail(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + _, view := f.initialize(t, changes, sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + connection := f.connect(t, node.NodeID) + epoch, err := f.adapter.OwnerEpoch(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + reported, want string + ready bool + }{ + {reported: "docker_unavailable", want: "docker_unavailable"}, + {reported: "capacity_insufficient", want: "capacity_insufficient"}, + // Older nodes send provider_unavailable or nothing; unknown text is never stored. + {reported: "provider_unavailable", want: "provider_unavailable"}, + {reported: "", want: ""}, + {reported: "dial unix /var/run/docker.sock: permission denied", want: "provider_unavailable"}, + {reported: "kvm_unavailable", want: "", ready: true}, + } { + if err := f.service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: tc.ready, Diagnostic: tc.reported}); err != nil { + t.Fatal(tc.reported, err) + } + list, err := f.service.ListNodes(t.Context()) + if err != nil || len(list) != 1 || list[0].Diagnostic != tc.want { + t.Fatal(tc.reported, list, err) + } + detail, err := f.service.NodeDetail(t.Context(), node.NodeID, "1h") + if err != nil || detail.Diagnostic != tc.want { + t.Fatal(tc.reported, detail.Diagnostic, err) + } + var stored string + if err := f.pool.QueryRow(t.Context(), "SELECT health::text FROM runtime_nodes WHERE id=$1", node.NodeID).Scan(&stored); err != nil || strings.Contains(stored, "/var/run") { + t.Fatal("raw diagnostic stored", stored, err) + } + } +} + +func TestNodeStatusUsesAuthenticatedFreshPresence(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + _, view := f.initialize(t, changes, sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 4}) + status := func(connected, ready bool) { + t.Helper() + got, err := f.service.NodeStatus(t.Context(), node.NodeID, node.Credential) + if err != nil || got.NodeID != node.NodeID || got.Connected != connected || got.ProviderReady != ready { + t.Fatal(got, err) + } + raw, err := json.Marshal(got) + if err != nil || strings.Contains(string(raw), "credential") || strings.Contains(string(raw), "backend_fingerprint") { + t.Fatal("private identity leaked", string(raw), err) + } + } + status(false, false) + connection := f.connect(t, node.NodeID) + status(true, true) + if _, err := f.service.NodeStatus(t.Context(), node.NodeID, "different-credential"); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("status admitted another credential", err) + } + epoch, err := f.adapter.OwnerEpoch(t.Context()) + if err != nil { + t.Fatal(err) + } + if err := f.service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: false}); err != nil { + t.Fatal(err) + } + status(true, false) + if err := f.service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + t.Fatal(err) + } + status(true, true) + if _, err := f.pool.Exec(t.Context(), "UPDATE runtime_nodes SET last_seen_at=clock_timestamp()-interval '46 seconds' WHERE id=$1", node.NodeID); err != nil { + t.Fatal(err) + } + status(false, false) + f.connect(t, node.NodeID) + if _, err := f.pool.Exec(t.Context(), "UPDATE runtime_deployment SET owner_epoch=owner_epoch+1 WHERE singleton=true"); err != nil { + t.Fatal(err) + } + status(false, false) +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/records.go b/services/core/internal/persistence/postgres/deploymentpg/records.go new file mode 100644 index 000000000..b310e6688 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/records.go @@ -0,0 +1,132 @@ +package deploymentpg + +import ( + "encoding/json" + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/textvalue" +) + +// healthRecord is a node's stored health, with its last host observation. +type healthRecord struct { + deployment.NodeHealth + Host *deployment.NodeHost `json:"host,omitempty"` +} + +// parseID returns the stored form of an identifier, and ErrInvalidInput for a +// value that is not a nonzero UUID. +func parseID(value string) (pgtype.UUID, error) { + id, err := pgunit.ParseID(value) + if errors.Is(err, pgunit.ErrInvalidID) { + return id, deployment.ErrInvalidInput + } + return id, err +} + +func uuidString(id pgtype.UUID) string { + if !id.Valid { + return "" + } + return uuid.UUID(id.Bytes).String() +} + +func timestamp(value pgtype.Timestamptz) *time.Time { + if !value.Valid { + return nil + } + return &value.Time +} + +// translate replaces PostgreSQL's rejection of unstorable client text. +func translate(err error) error { + if pgunit.IsUnstorableText(err) { + return textvalue.ErrUnstorable + } + return err +} + +// record converts the stored deployment. open opens a stored credential; a +// view never needs it. Without a key the credential error is +// credentialcrypto.ErrUnavailable; a credential the key cannot open or +// authenticate is an internal decryption error. +func record(d sqlc.RuntimeDeployment, cipher *credentialcrypto.Cipher, open bool) deployment.Record { + r := deployment.Record{InstallationID: uuidString(d.InstallationID), WebManaged: d.WebManaged, Provider: d.ProviderKind, BackendFingerprint: d.BackendFingerprint, + Generation: uint64(d.Generation), OwnerEpoch: uint64(d.OwnerEpoch), Mode: d.Mode, AdmissionPaused: d.AdmissionPaused, LocalNodeID: uuidString(d.LocalNodeID), + IdleSeconds: d.IdleSeconds, RetentionSeconds: d.RetentionSeconds, Specification: d.Specification, + Configuration: sandbox.ConfigurationRecord{Public: d.ProviderConfig, Metadata: d.ProviderMetadata}, CredentialStored: len(d.ProviderCredential) > 0} + if r.CredentialStored && open { + if cipher == nil { + r.CredentialError = credentialcrypto.ErrUnavailable + } else if secret, err := cipher.OpenSandboxDeployment(d.ProviderCredential, r.InstallationID, r.Generation); err != nil { + r.CredentialError = errors.New("sandbox deployment credential decryption failed") + } else { + r.Configuration.Secret = secret + } + } + if d.ResetClear.Valid { + r.Reset = &deployment.ResetState{Clear: d.ResetClear.String, RequestedAt: d.ResetRequestedAt.Time, DeadlineAt: timestamp(d.ResetDeadlineAt), ForcedAt: timestamp(d.ResetForcedAt)} + } + return r +} + +func snapshot(row sqlc.GetSandboxDeploymentSnapshotRow) (deployment.Snapshot, error) { + result := deployment.Snapshot{Record: record(row.RuntimeDeployment, nil, false), Resources: deployment.Resources{Allocations: row.Allocations, Pending: row.Pending}} + if err := json.Unmarshal(row.Rollout, &result.Rollout); err != nil { + return deployment.Snapshot{}, err + } + if result.Record.Reset != nil { + if err := json.Unmarshal(row.Remaining, &result.Remaining); err != nil { + return deployment.Snapshot{}, err + } + } + return result, nil +} + +func generation(g sqlc.RuntimeDeploymentGeneration) deployment.GenerationRecord { + return deployment.GenerationRecord{Generation: uint64(g.Generation), Provider: g.ProviderKind, Specification: g.Specification, Configuration: sandbox.ConfigurationRecord{Public: g.ProviderConfig, Metadata: g.ProviderMetadata}} +} + +func storedNode(n sqlc.RuntimeNode) deployment.StoredNode { + result := deployment.StoredNode{ID: uuidString(n.ID), InstallationID: uuidString(n.InstallationID), Name: n.Name, BackendFingerprint: n.BackendFingerprint, CredentialDigest: n.CredentialSha256, + MaxActive: int(n.MaxActive), MaxRetained: int(n.MaxRetained), ConnectionID: uuidString(n.ConnectionID), ConnectedEpoch: uint64(n.ConnectedEpoch), + SpecificationDigest: n.SpecificationDigest, DeploymentGeneration: uint64(n.DeploymentGeneration)} + if n.ReadyGeneration.Valid { + ready := uint64(n.ReadyGeneration.Int64) + result.ReadyGeneration = &ready + } + return result +} + +func nodeRecords(rows []sqlc.ListRuntimeNodesRow) ([]deployment.NodeRecord, error) { + out := make([]deployment.NodeRecord, 0, len(rows)) + for _, n := range rows { + var health healthRecord + if err := json.Unmarshal(n.Health, &health); err != nil { + return nil, err + } + health.NodeHealth.Host = health.Host + record := deployment.NodeRecord{ID: uuidString(n.ID), Name: n.Name, Provider: n.ProviderKind, CoreURL: n.CoreUrl, CreatedAt: n.CreatedAt.Time, LastSeenAt: timestamp(n.LastSeenAt), + Health: health.NodeHealth, Online: n.Online, ServingReady: n.ServingReady, ProtocolVersion: int(n.ProtocolVersion), DeploymentGeneration: uint64(n.DeploymentGeneration), + TargetGeneration: uint64(n.TargetGeneration), TargetState: n.TargetState, TargetDiagnostic: n.TargetDiagnostic, MaxActive: int(n.MaxActive), MaxRetained: int(n.MaxRetained), + Active: n.Active, Retained: n.Retained, Reserved: n.Reserved, CleanupPending: n.CleanupPending, Running: n.Running, Snapshots: n.Snapshots} + if n.EnrollmentID.Valid { + id := uuidString(n.EnrollmentID) + record.EnrollmentID = &id + } + if n.ReadyGeneration.Valid { + ready := uint64(n.ReadyGeneration.Int64) + record.ReadyGeneration = &ready + } + out = append(out, record) + } + return out, nil +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/seal_test.go b/services/core/internal/persistence/postgres/deploymentpg/seal_test.go new file mode 100644 index 000000000..da1186920 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/seal_test.go @@ -0,0 +1,52 @@ +package deploymentpg_test + +import ( + "errors" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" +) + +// A missing key is credentialcrypto.ErrUnavailable, and a credential sealed +// with another binding is a decryption failure, never a missing key. Node +// transactions and snapshots never open the credential, so they need no key. +func TestStoredCredentialNeedsTheKeyAndItsBinding(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + id, view := f.initialize(t, changes, setupE2BSelection()) + keyless := deploymentpg.New(pgunit.NewPool(f.pool), nil) + service, err := deployment.NewService(keyless, keyless, providers.Builtin(), fixturePublicURL) + if err != nil { + t.Fatal(err) + } + if _, err := service.Setup(t.Context()); !errors.Is(err, credentialcrypto.ErrUnavailable) { + t.Fatal("a keyless Store opened the credential", err) + } + withoutCredential := func(reads deployment.NodeReads) error { + d, err := reads.LoadDeployment() + if err == nil && (!d.CredentialStored || d.CredentialError != nil || d.Configuration.Secret != nil) { + t.Error("a node read opened the credential", d.CredentialError) + } + return err + } + if err := keyless.WithNodes(t.Context(), func(tx deployment.NodeTx) error { return withoutCredential(tx) }); err != nil { + t.Fatal(err) + } + if err := keyless.ReadNodes(t.Context(), withoutCredential); err != nil { + t.Fatal(err) + } + sealed, err := f.cipher.SealSandboxDeployment([]byte("fixture-private-api-key"), id, view.Generation+1) + if err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), "UPDATE runtime_deployment SET provider_credential=$1", sealed); err != nil { + t.Fatal(err) + } + if _, err := f.service.Setup(t.Context()); err == nil || err.Error() != "sandbox deployment credential decryption failed" || errors.Is(err, credentialcrypto.ErrUnavailable) { + t.Fatal("a wrong binding was not a decryption failure", err) + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/setup_test.go b/services/core/internal/persistence/postgres/deploymentpg/setup_test.go new file mode 100644 index 000000000..f9ac218b8 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/setup_test.go @@ -0,0 +1,236 @@ +package deploymentpg_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "sync" + "testing" + + "github.com/google/uuid" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" +) + +// setupE2BSelection is a valid E2B selection with a new template. +func setupE2BSelection() sandbox.Selection { + return sandbox.Selection{DeploymentSpec: testSpecification("e2b"), Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-private-api-key", Template: "runtime:" + uuid.NewString()}} +} + +// setupE2BPublic decodes the public E2B configuration of a view. +func setupE2BPublic(t *testing.T, v deployment.View) *e2b.DeploymentConfiguration { + t.Helper() + c, err := (e2b.ConfigurationAdapter{}).Decode(sandbox.ConfigurationRecord{Public: v.Configuration, Metadata: v.Metadata}) + if err != nil { + t.Fatal(err) + } + return c.(*e2b.DeploymentConfiguration) +} + +// setupDigest is the hex SHA-256 form in which credentials and enrollment +// tokens are stored. +func setupDigest(value string) string { + digest := sha256.Sum256([]byte(value)) + return hex.EncodeToString(digest[:]) +} + +// setupClosedExecution builds the deployment execution operations on an +// execution lease that is already closed. Take it before f.execution: only one +// lease can be held at a time, so it returns after the server has released +// the closed lease. +func setupClosedExecution(t *testing.T, f fixture) *deployment.ExecutionOperations { + t.Helper() + lease, err := pgunit.AcquireLease(t.Context(), f.pool) + if err != nil { + t.Fatal(err) + } + awaitRelease := pgtest.ObserveExecutionLeaseRelease(t, f.pool) + if err := lease.Close(context.Background()); err != nil { + t.Fatal(err) + } + awaitRelease() + operations, err := deployment.NewExecutionOperations(f.service, deploymentpg.NewExecution(lease, f.cipher)) + if err != nil { + t.Fatal(err) + } + return operations +} + +func TestSandboxDeploymentSetupConcurrentSelection(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + id := uuid.NewString() + if err := changes.Claim(t.Context(), id); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + results := make(chan deployment.View, 20) + errorsFound := make(chan error, 20) + for i := range 20 { + wg.Add(1) + go func() { + defer wg.Done() + provider := "docker" + if i%2 == 1 { + provider = "microsandbox" + } + value, err := changes.Initialize(t.Context(), id, sandbox.Selection{DeploymentSpec: testSpecification(provider), Provider: provider}) + results <- value + errorsFound <- err + }() + } + wg.Wait() + close(results) + close(errorsFound) + conflicts := 0 + for err := range errorsFound { + if errors.Is(err, deployment.ErrConflict) { + conflicts++ + } else if err != nil { + t.Fatal(err) + } + } + if conflicts != 19 { + t.Fatal("both provider selections won", conflicts) + } + winner, err := f.service.Setup(t.Context()) + if err != nil { + t.Fatal(err) + } + for result := range results { + if result.Provider != "" && result.Provider != winner.Provider { + t.Fatal("inconsistent selection", result) + } + } +} + +func TestSandboxDeploymentSetupRejectsFileManagedAndUnleasedWrites(t *testing.T) { + f := newFixture(t) + input := sandbox.Selection{DeploymentSpec: testSpecification("docker"), Provider: "docker"} + process := deployment.ProcessDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64), ProviderKind: "docker", + LocalNodeID: uuid.NewString(), LocalCredentialSHA256: setupDigest("local-node-credential"), LocalMaxActive: 4, LocalMaxRetained: 16} + // Deployment changes run only on the execution lease; a closed one writes nothing. + closed := setupClosedExecution(t, f) + if err := closed.ConfigureProcess(t.Context(), &process); !errors.Is(err, pgunit.ErrLeaseClosed) { + t.Fatal("unleased process configuration accepted", err) + } + if _, err := closed.Initialize(t.Context(), process.InstallationID, input); !errors.Is(err, pgunit.ErrLeaseClosed) { + t.Fatal("unleased setup accepted", err) + } + if view, err := f.service.View(t.Context()); err != nil || view.InstallationID != "" || view.Provider != "" { + t.Fatal("unleased writes changed the deployment", view, err) + } + changes, _ := f.execution(t) + if err := changes.ConfigureProcess(t.Context(), &process); err != nil { + t.Fatal(err) + } + if _, err := changes.Initialize(t.Context(), process.InstallationID, input); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("file-managed deployment changed", err) + } + if err := changes.Claim(t.Context(), process.InstallationID); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("file-managed deployment adopted", err) + } +} + +func TestSandboxSelectionRejectsWhitespaceInE2BCredential(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + id := uuid.NewString() + if err := changes.Claim(t.Context(), id); err != nil { + t.Fatal(err) + } + for _, separator := range []string{" ", "\t", "\r", "\n", "\x00", " ", " ", "　"} { + t.Run(fmt.Sprintf("U+%04X", []rune(separator)[0]), func(t *testing.T) { + selection := setupE2BSelection() + selection.Configuration.(*e2b.DeploymentConfiguration).APIKey = "prefix" + separator + "suffix" + if _, err := changes.Initialize(t.Context(), id, selection); !errors.Is(err, deployment.ErrInvalidInput) { + t.Fatalf("credential containing whitespace or NUL accepted: %v", err) + } + }) + } + if _, err := changes.Initialize(t.Context(), id, setupE2BSelection()); err != nil { + t.Fatal(err) + } +} + +func TestSandboxE2BEndpointPersistenceAndOnlineSwitch(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + input := setupE2BSelection() + configured := input.Configuration.(*e2b.DeploymentConfiguration) + configured.APIURL, configured.Domain = "https://sandbox-test.sandbase.ai", "sandbox-test.sandbase.ai" + id, view := f.initialize(t, changes, input) + if view.Configuration == nil || setupE2BPublic(t, view).APIURL != configured.APIURL || setupE2BPublic(t, view).Domain != configured.Domain { + t.Fatal("custom endpoint was not returned", view) + } + setup, err := f.service.Setup(t.Context()) + if err != nil || setup.Configuration == nil || setup.Configuration.(*e2b.DeploymentConfiguration).APIURL != configured.APIURL || setup.Configuration.(*e2b.DeploymentConfiguration).Domain != configured.Domain { + t.Fatal("custom endpoint was not persisted", setup, err) + } + change := setupE2BSelection() + change.Configuration.(*e2b.DeploymentConfiguration).Template = configured.Template + change.ExpectedGeneration = view.Generation + changed, err := changes.Update(admin(t), id, change) + if err != nil || changed.Generation != view.Generation+1 || changed.Configuration == nil || setupE2BPublic(t, changed).APIURL != "https://api.e2b.app" || setupE2BPublic(t, changed).Domain != "e2b.app" { + t.Fatal("online endpoint switch failed", changed, err) + } +} + +func TestRuntimeDeploymentRequiresMaintenanceBeforeIdentityChange(t *testing.T) { + f := newFixture(t) + old := deployment.ProcessDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64)} + if err := setupClosedExecution(t, f).ConfigureProcess(t.Context(), &old); !errors.Is(err, pgunit.ErrLeaseClosed) { + t.Fatal("unleased configuration accepted", err) + } + changes, _ := f.execution(t) + configure := func(selected *deployment.ProcessDeployment) { + t.Helper() + if err := changes.ConfigureProcess(t.Context(), selected); err != nil { + t.Fatal(err) + } + } + configure(&old) + for _, changeID := range []bool{false, true} { + next := old + if changeID { + next.InstallationID = uuid.NewString() + } else { + next.BackendFingerprint = strings.Repeat("b", 64) + } + next.AdmissionPaused = true + if err := changes.ConfigureProcess(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "maintenance") { + t.Fatal("identity changed before prior maintenance", err) + } + } + old.AdmissionPaused = true + configure(&old) + next := old + next.BackendFingerprint = strings.Repeat("b", 64) + next.AdmissionPaused = false + if err := changes.ConfigureProcess(t.Context(), &next); err == nil { + t.Fatal("switch reopened creation in same operation") + } + next.AdmissionPaused = true + configure(&next) + var id, fingerprint string + var maintenance bool + if err := f.pool.QueryRow(t.Context(), "SELECT installation_id::text,backend_fingerprint,admission_paused FROM runtime_deployment").Scan(&id, &fingerprint, &maintenance); err != nil || id != next.InstallationID || fingerprint != next.BackendFingerprint || !maintenance { + t.Fatal("switch identity not durable", id, fingerprint, maintenance, err) + } + configure(nil) + // Disabling the configured adapter must not forget the old maintenance state. + next.AdmissionPaused = false + configure(&next) + another := deployment.ProcessDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64), AdmissionPaused: true} + if err := changes.ConfigureProcess(t.Context(), &another); err == nil { + t.Fatal("nil selection erased the maintenance prerequisite") + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/specification_test.go b/services/core/internal/persistence/postgres/deploymentpg/specification_test.go new file mode 100644 index 000000000..f739faaec --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/specification_test.go @@ -0,0 +1,149 @@ +package deploymentpg_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/google/uuid" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" +) + +func TestSandboxSpecificationRoundTripAndFileConfigurationCannotOverride(t *testing.T) { + for _, provider := range []string{"docker", "microsandbox", "e2b"} { + t.Run(provider, func(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + input := sandbox.Selection{Provider: provider, DeploymentSpec: testSpecification(provider)} + if provider == "e2b" { + input = setupE2BSelection() + } + _, view := f.initialize(t, changes, input) + setup, err := f.service.Setup(t.Context()) + if err != nil || !reflect.DeepEqual(setup.Specification, input.DeploymentSpec) || view.Specification == nil || !reflect.DeepEqual(*view.Specification, input.DeploymentSpec) || view.SpecificationDigest != input.DeploymentSpec.Digest(provider) { + t.Fatal("saved deployment lost its resources or Runtime provenance", err) + } + preview, err := f.service.SetupForSelection(view.InstallationID, input) + if err != nil || preview.Mode != setup.Mode || preview.BackendFingerprint != setup.BackendFingerprint || !reflect.DeepEqual(preview.Suspension, setup.Suspension) || !reflect.DeepEqual(preview.Configuration, setup.Configuration) { + t.Fatal("preview and persisted normalized deployment disagree", err) + } + input.ExpectedGeneration = view.Generation + retry, err := changes.Initialize(t.Context(), view.InstallationID, input) + if err != nil || !reflect.DeepEqual(retry, view) { + t.Fatal("identical specification changed the generation", err) + } + changed := input + changed.Resources.CPUs++ + if _, err := changes.Initialize(t.Context(), view.InstallationID, changed); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("initial setup silently resized a configured deployment", err) + } + file := deployment.ProcessDeployment{InstallationID: view.InstallationID, BackendFingerprint: setup.BackendFingerprint, ProviderKind: provider, AdmissionPaused: true} + for _, candidate := range []*deployment.ProcessDeployment{nil, &file} { + if err := changes.ConfigureProcess(t.Context(), candidate); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("file configuration replaced database ownership", err) + } + } + after, err := f.service.View(t.Context()) + if err != nil || !reflect.DeepEqual(after, view) { + t.Fatal("rejected writes changed the committed specification", err) + } + }) + } +} + +func TestSandboxSpecificationInitialCredentialRemainsPrivate(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + input := setupE2BSelection() + key := input.Configuration.(*e2b.DeploymentConfiguration).APIKey + _, view := f.initialize(t, changes, input) + raw, err := json.Marshal(view) + if err != nil || bytes.Contains(raw, []byte(key)) || bytes.Contains(raw, []byte("api_key")) { + t.Fatal("public deployment serialized a private credential", err) + } + var stored []byte + if err := f.pool.QueryRow(t.Context(), "SELECT provider_credential FROM runtime_deployment").Scan(&stored); err != nil || len(stored) == 0 || bytes.Contains(stored, []byte(key)) { + t.Fatal("private credential was not encrypted", err) + } + // The credential is rejected before the cloud deployment mode is reported. + if _, err := f.service.NodeConfiguration(t.Context(), "", key, 0); !errors.Is(err, deployment.ErrNodeCredential) { + t.Fatal("cloud key authorized node bootstrap", err) + } +} + +func TestEnrollmentRecordsItsIDAndRefusesAnotherAddress(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + _, view := f.initialize(t, changes, sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")}) + issued, err := f.service.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 2, MaxRetained: 2}) + if err != nil || uuid.Validate(issued.ID) != nil || issued.Token == "" { + t.Fatal(issued.ID, err) + } + input := deployment.Enrollment{NodeID: uuid.NewString(), Name: "addressed", Credential: strings.Repeat("a", 64), Provider: "docker", + BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest, CoreURL: "https://other.example"} + if _, err := f.service.Enroll(t.Context(), issued.Token, input); !errors.Is(err, deployment.ErrNodeAddressMismatch) { + t.Fatal("enrolled a node that uses another Core address", err) + } + input.CoreURL = fixturePublicURL + if _, err := f.service.Enroll(t.Context(), issued.Token, input); err != nil { + t.Fatal("the refused enrollment consumed its token", err) + } + earlier := strings.Repeat("e", 64) + if _, err := f.pool.Exec(t.Context(), "INSERT INTO runtime_node_enrollments(token_sha256,installation_id,expires_at) VALUES($1,$2,clock_timestamp()+interval '10 minutes')", + setupDigest(earlier), view.InstallationID); err != nil { + t.Fatal(err) + } + older := input + older.NodeID, older.Credential = uuid.NewString(), strings.Repeat("o", 64) + if _, err := f.service.Enroll(t.Context(), earlier, older); err != nil { + t.Fatal(err) + } + nodes, err := f.service.ListNodes(t.Context()) + if err != nil || len(nodes) != 2 { + t.Fatal(nodes, err) + } + for _, node := range nodes { + switch node.ID { + case input.NodeID: + if node.EnrollmentID == nil || *node.EnrollmentID != issued.ID || node.CoreURL != fixturePublicURL { + t.Fatal("the node did not record its enrollment", node.EnrollmentID, node.CoreURL) + } + case older.NodeID: + if node.EnrollmentID != nil { + t.Fatal("a token without an ID reported one", *node.EnrollmentID) + } + } + } +} + +func TestDatabaseDoesNotEnumerateProviderRegistrations(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + input := sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")} + id, view := f.initialize(t, changes, input) + tx, err := f.pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + if _, err = tx.Exec(t.Context(), "UPDATE runtime_deployment SET provider_kind='new-adapter' WHERE singleton=true"); err != nil { + t.Fatal("database enumerated provider implementations", err) + } + // Roll back before the deployment operation, which still rejects an + // unregistered provider even though persistence can represent a new adapter. + if err = tx.Rollback(t.Context()); err != nil { + t.Fatal(err) + } + input.Provider = "new-adapter" + input.ExpectedGeneration = view.Generation + if _, err = changes.Initialize(t.Context(), id, input); !errors.Is(err, deployment.ErrInvalidInput) { + t.Fatal("unknown adapter reached persistence", err) + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/store.go b/services/core/internal/persistence/postgres/deploymentpg/store.go new file mode 100644 index 000000000..40117d569 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/store.go @@ -0,0 +1,279 @@ +// Package deploymentpg stores the sandbox deployment, its retained generations +// and its nodes in PostgreSQL. It seals the deployment's provider credential +// with the Core credential key, bound to the installation and generation. +package deploymentpg + +import ( + "context" + "errors" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// Store is the deployment storage and reader on pooled connections. It grants +// no execution authority. +type Store struct { + pool *pgunit.Pool + cipher *credentialcrypto.Cipher +} + +// New returns the deployment store. cipher is nil when Core has no credential +// key; a stored credential then reads as credentialcrypto.ErrUnavailable. +func New(pool *pgunit.Pool, cipher *credentialcrypto.Cipher) *Store { + return &Store{pool: pool, cipher: cipher} +} + +var ( + _ deployment.Storage = (*Store)(nil) + _ deployment.Reader = (*Store)(nil) +) + +// Execution is the deployment storage of the execution owner: every +// transaction runs on the connection that holds the execution lease. +type Execution struct { + lease *pgunit.Lease + cipher *credentialcrypto.Cipher +} + +// NewExecution binds deployment changes to the execution lease. +func NewExecution(lease *pgunit.Lease, cipher *credentialcrypto.Cipher) *Execution { + return &Execution{lease: lease, cipher: cipher} +} + +var _ deployment.ExecutionStorage = (*Execution)(nil) + +func (e *Execution) WithDeployment(ctx context.Context, apply func(deployment.DeploymentTx) error) error { + return translate(e.lease.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { + q := sqlc.New(tx) + if _, err := q.LockRuntimeDeployment(ctx); err != nil { + return err + } + return apply(&deploymentTx{unit: unit{ctx: ctx, q: q, locked: true}, cipher: e.cipher}) + })) +} + +func (s *Store) WithNodes(ctx context.Context, apply func(deployment.NodeTx) error) error { + return translate(s.pool.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { + q := sqlc.New(tx) + if _, err := q.LockRuntimeDeployment(ctx); err != nil { + return err + } + return apply(&nodeTx{unit{ctx: ctx, q: q, locked: true}}) + })) +} + +func (s *Store) ReadNodes(ctx context.Context, apply func(deployment.NodeReads) error) error { + return s.pool.Snapshot(ctx, func(ctx context.Context, tx pgx.Tx) error { + return apply(unit{ctx: ctx, q: sqlc.New(tx)}) + }) +} + +// Deployment bounds its read by the execution deadline: the execution owner +// reads the setup through it before provider work. +func (s *Store) Deployment(ctx context.Context) (deployment.Record, error) { + ctx, cancel := context.WithTimeout(ctx, pgunit.ExecutionTimeout) + defer cancel() + d, err := s.pool.Queries().GetRuntimeDeployment(ctx) + if err != nil { + return deployment.Record{}, err + } + return record(d, s.cipher, true), nil +} + +func (s *Store) Snapshot(ctx context.Context) (deployment.Snapshot, error) { + row, err := s.pool.Queries().GetSandboxDeploymentSnapshot(ctx) + if err != nil { + return deployment.Snapshot{}, err + } + return snapshot(row) +} + +func (s *Store) OwnerEpoch(ctx context.Context) (uint64, error) { + d, err := s.pool.Queries().GetRuntimeDeployment(ctx) + if err != nil { + return 0, err + } + return uint64(d.OwnerEpoch), nil +} + +func (s *Store) Allocation(ctx context.Context, ref sandbox.Reference) (deployment.AllocationRecord, error) { + tenant, err := parseID(ref.TenantID) + if err != nil { + return deployment.AllocationRecord{}, err + } + environment, err := parseID(ref.EnvironmentID) + if err != nil { + return deployment.AllocationRecord{}, err + } + var result deployment.AllocationRecord + err = s.pool.Snapshot(ctx, func(ctx context.Context, tx pgx.Tx) error { + q := sqlc.New(tx) + row, err := q.GetRuntimeAllocation(ctx, sqlc.GetRuntimeAllocationParams{TenantID: tenant, EnvironmentID: environment}) + if errors.Is(err, pgx.ErrNoRows) { + return deployment.ErrNotFound + } + if err != nil { + return err + } + a := row.RuntimeAllocation + d, err := q.GetRuntimeDeployment(ctx) + if err != nil { + return err + } + result = deployment.AllocationRecord{ID: uuidString(a.ID), Released: a.State == "released", InstallationID: uuidString(a.ProviderKey), Deployment: record(d, s.cipher, true)} + if !a.DeploymentGeneration.Valid { + return nil + } + result.Generation = uint64(a.DeploymentGeneration.Int64) + if a.DeploymentGeneration.Int64 == d.Generation { + return nil + } + g, err := q.GetSandboxGeneration(ctx, a.DeploymentGeneration.Int64) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return err + } + retained := generation(g) + result.Retained = &retained + return nil + }) + if err != nil { + return deployment.AllocationRecord{}, err + } + return result, nil +} + +func (s *Store) Generations(ctx context.Context, after int64) ([]deployment.GenerationRecord, error) { + rows, err := s.pool.Queries().ListSandboxGenerations(ctx, after) + if err != nil { + return nil, err + } + out := make([]deployment.GenerationRecord, 0, len(rows)) + for _, g := range rows { + out = append(out, generation(g)) + } + return out, nil +} + +func (s *Store) Nodes(ctx context.Context) ([]deployment.NodeRecord, error) { + rows, err := s.pool.Queries().ListRuntimeNodes(ctx, pgtype.UUID{}) + if err != nil { + return nil, err + } + return nodeRecords(rows) +} + +func (s *Store) NodeHistory(ctx context.Context, nodeID string, window coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) { + id, err := parseID(nodeID) + if err != nil { + return deployment.NodeRecord{}, nil, err + } + var node deployment.NodeRecord + var points []deployment.HostHistoryPoint + err = s.pool.Snapshot(ctx, func(ctx context.Context, tx pgx.Tx) error { + q := sqlc.New(tx) + rows, err := q.ListRuntimeNodes(ctx, id) + if err != nil { + return err + } + if len(rows) == 0 { + return deployment.ErrNotFound + } + nodes, err := nodeRecords(rows) + if err != nil { + return err + } + node = nodes[0] + samples, err := q.ListNodeHostHistory(ctx, sqlc.ListNodeHostHistoryParams{ + NodeID: id, StartAt: pgtype.Timestamptz{Time: window.Start, Valid: true}, EndAt: pgtype.Timestamptz{Time: window.End, Valid: true}, + BucketWidth: pgtype.Interval{Microseconds: window.ResolutionSeconds * 1_000_000, Valid: true}, + }) + if err != nil { + return err + } + points = make([]deployment.HostHistoryPoint, 0, len(samples)) + for _, sample := range samples { + point := deployment.HostHistoryPoint{Start: sample.Start.Time} + if sample.CpuSamples > 0 { + point.CPUUtilizationMax = &sample.CpuUtilizationMax + } + if sample.MemorySamples > 0 { + point.MemoryUsedBytesMax = &sample.MemoryUsedBytesMax + } + if sample.DiskSamples > 0 { + point.AvailableDiskBytesMin = &sample.AvailableDiskBytesMin + } + points = append(points, point) + } + return nil + }) + if err != nil { + return deployment.NodeRecord{}, nil, err + } + return node, points, nil +} + +func (s *Store) ConnectNode(ctx context.Context, nodeID, connectionID string, epoch uint64) (bool, error) { + id, err := parseID(nodeID) + if err != nil { + return false, err + } + connection, err := parseID(connectionID) + if err != nil { + return false, err + } + var connected bool + // A canceled autocommit UPDATE may still finish on PostgreSQL after pgx + // returns. An explicit transaction cannot publish that late write. + err = s.pool.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { + changed, err := sqlc.New(tx).ConnectRuntimeNode(ctx, sqlc.ConnectRuntimeNodeParams{ID: id, ConnectionID: connection, OwnerEpoch: int64(epoch)}) + if err != nil { + return err + } + connected = changed == 1 + return ctx.Err() + }) + if err != nil { + return false, err + } + return connected, nil +} + +func (s *Store) DisconnectNode(ctx context.Context, nodeID, connectionID string, epoch uint64) error { + id, err := parseID(nodeID) + if err != nil { + return err + } + connection, err := parseID(connectionID) + if err != nil { + return err + } + return s.pool.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { + q := sqlc.New(tx) + // A Connect COMMIT may be uncertain. Wait on the node regardless of + // the visible connection, then fence cleanup in a fresh statement snapshot. + if _, err := q.LockRuntimeNodePresence(ctx, id); errors.Is(err, pgx.ErrNoRows) { + return nil + } else if err != nil { + return err + } + if err := q.DisconnectRuntimeNode(ctx, sqlc.DisconnectRuntimeNodeParams{ID: id, ConnectionID: connection, OwnerEpoch: int64(epoch)}); err != nil { + return err + } + return ctx.Err() + }) +} + +func (s *Store) SampleHostHistory(ctx context.Context) (int64, error) { + return s.pool.Queries().SampleNodeHostHistory(ctx) +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/tx.go b/services/core/internal/persistence/postgres/deploymentpg/tx.go new file mode 100644 index 000000000..4c78ffa32 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/tx.go @@ -0,0 +1,329 @@ +package deploymentpg + +import ( + "context" + "encoding/json" + "errors" + "math" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" +) + +// unit is one transaction or snapshot. locked reports that the transaction +// began by locking the deployment, so reads of it keep the lock. +type unit struct { + ctx context.Context + q *sqlc.Queries + locked bool +} + +var _ deployment.NodeReads = unit{} + +// LoadDeployment reads the deployment without opening its credential: node +// transactions and snapshots never need it. +func (u unit) LoadDeployment() (deployment.Record, error) { + d, err := u.loadDeployment() + if err != nil { + return deployment.Record{}, err + } + return record(d, nil, false), nil +} + +func (u unit) loadDeployment() (sqlc.RuntimeDeployment, error) { + if u.locked { + return u.q.LockRuntimeDeployment(u.ctx) + } + return u.q.GetRuntimeDeployment(u.ctx) +} + +func (u unit) LoadNode(id string) (deployment.StoredNode, error) { + nodeID, err := parseID(id) + if err != nil { + return deployment.StoredNode{}, err + } + n, err := u.q.GetRuntimeNode(u.ctx, nodeID) + if errors.Is(err, pgx.ErrNoRows) { + return deployment.StoredNode{}, deployment.ErrNotFound + } + if err != nil { + return deployment.StoredNode{}, err + } + return storedNode(n), nil +} + +func (u unit) LoadEnrollment(tokenDigest string) (deployment.EnrollmentRecord, error) { + r, err := u.q.GetRuntimeEnrollment(u.ctx, tokenDigest) + if errors.Is(err, pgx.ErrNoRows) { + return deployment.EnrollmentRecord{}, deployment.ErrNotFound + } + if err != nil { + return deployment.EnrollmentRecord{}, err + } + return deployment.EnrollmentRecord{ID: uuidString(r.ID), InstallationID: uuidString(r.InstallationID), ExpiresAt: r.ExpiresAt.Time, Consumed: r.ConsumedAt.Valid, MaxActive: int(r.MaxActive), MaxRetained: int(r.MaxRetained)}, nil +} + +func (u unit) LoadGenerationSpecification(generation uint64) (deployment.GenerationSpecification, error) { + if generation > math.MaxInt64 { + return deployment.GenerationSpecification{}, deployment.ErrInvalidInput + } + row, err := u.q.GetNodeGenerationSpecification(u.ctx, int64(generation)) + if errors.Is(err, pgx.ErrNoRows) { + return deployment.GenerationSpecification{}, deployment.ErrNotFound + } + if err != nil { + return deployment.GenerationSpecification{}, err + } + return deployment.GenerationSpecification{Provider: row.ProviderKind, Specification: row.Specification}, nil +} + +func (u unit) GenerationKept(nodeID string, generation uint64) (bool, error) { + id, err := parseID(nodeID) + if err != nil { + return false, err + } + if generation > math.MaxInt64 { + return false, deployment.ErrInvalidInput + } + return u.q.NodeGenerationKept(u.ctx, sqlc.NodeGenerationKeptParams{NodeID: id, Generation: int64(generation)}) +} + +func (u unit) InsertNode(node deployment.NewNode) (deployment.StoredNode, error) { + id, err := parseID(node.ID) + if err != nil { + return deployment.StoredNode{}, err + } + installation, err := parseID(node.InstallationID) + if err != nil { + return deployment.StoredNode{}, err + } + var enrollment pgtype.UUID + if node.EnrollmentID != "" { + if enrollment, err = parseID(node.EnrollmentID); err != nil { + return deployment.StoredNode{}, err + } + } + if node.DeploymentGeneration > math.MaxInt64 { + return deployment.StoredNode{}, deployment.ErrInvalidInput + } + row, err := u.q.InsertRuntimeNode(u.ctx, sqlc.InsertRuntimeNodeParams{ID: id, InstallationID: installation, Name: node.Name, BackendFingerprint: node.BackendFingerprint, CredentialSha256: node.CredentialDigest, MaxActive: int32(node.MaxActive), MaxRetained: int32(node.MaxRetained), SpecificationDigest: node.SpecificationDigest, DeploymentGeneration: int64(node.DeploymentGeneration), CoreUrl: node.CoreURL, EnrollmentID: enrollment}) + var databaseError *pgconn.PgError + if errors.As(err, &databaseError) && databaseError.Code == "23505" && databaseError.ConstraintName == "runtime_nodes_pkey" { + return deployment.StoredNode{}, deployment.ErrNodeExists + } + if err != nil { + return deployment.StoredNode{}, translate(err) + } + return storedNode(row), nil +} + +func (u unit) UpdateNode(id string, limits deployment.NodeLimits) error { + nodeID, err := parseID(id) + if err != nil { + return err + } + _, err = u.q.UpdateRuntimeNode(u.ctx, sqlc.UpdateRuntimeNodeParams{ID: nodeID, Name: limits.Name, MaxActive: int32(limits.MaxActive), MaxRetained: int32(limits.MaxRetained)}) + if errors.Is(err, pgx.ErrNoRows) { + return deployment.ErrNotFound + } + return translate(err) +} + +// nodeTx is one node management transaction. +type nodeTx struct{ unit } + +var _ deployment.NodeTx = (*nodeTx)(nil) + +func (t *nodeTx) ListNodes() ([]deployment.NodeRecord, error) { + rows, err := t.q.ListRuntimeNodes(t.ctx, pgtype.UUID{}) + if err != nil { + return nil, err + } + return nodeRecords(rows) +} + +func (t *nodeTx) RemoveNode(id string) error { + nodeID, err := parseID(id) + if err != nil { + return err + } + return t.q.RemoveRuntimeNode(t.ctx, nodeID) +} + +func (t *nodeTx) CreateEnrollment(enrollment deployment.NewEnrollment) (time.Time, error) { + id, err := parseID(enrollment.ID) + if err != nil { + return time.Time{}, err + } + installation, err := parseID(enrollment.InstallationID) + if err != nil { + return time.Time{}, err + } + if err := t.q.CreateRuntimeEnrollment(t.ctx, sqlc.CreateRuntimeEnrollmentParams{ID: id, TokenSha256: enrollment.TokenDigest, InstallationID: installation, MaxActive: int32(enrollment.MaxActive), MaxRetained: int32(enrollment.MaxRetained)}); err != nil { + return time.Time{}, err + } + row, err := t.q.GetRuntimeEnrollment(t.ctx, enrollment.TokenDigest) + if err != nil { + return time.Time{}, err + } + return row.ExpiresAt.Time, nil +} + +func (t *nodeTx) ConsumeEnrollment(tokenDigest, nodeID string) (bool, error) { + id, err := parseID(nodeID) + if err != nil { + return false, err + } + changed, err := t.q.ConsumeRuntimeEnrollment(t.ctx, sqlc.ConsumeRuntimeEnrollmentParams{TokenSha256: tokenDigest, NodeID: id}) + return changed == 1, err +} + +func (t *nodeTx) HeartbeatNode(heartbeat deployment.Heartbeat) (bool, error) { + id, err := parseID(heartbeat.NodeID) + if err != nil { + return false, err + } + connection, err := parseID(heartbeat.ConnectionID) + if err != nil { + return false, err + } + raw, err := json.Marshal(healthRecord{NodeHealth: heartbeat.Health, Host: heartbeat.Health.Host}) + if err != nil { + return false, err + } + changed, err := t.q.HeartbeatRuntimeNode(t.ctx, sqlc.HeartbeatRuntimeNodeParams{ID: id, ConnectionID: connection, OwnerEpoch: int64(heartbeat.Epoch), ProviderReady: heartbeat.Health.ProviderReady, Health: raw}) + return changed == 1, translate(err) +} + +func (t *nodeTx) DeleteGenerationStatus(nodeID string, generation uint64) error { + id, err := parseID(nodeID) + if err != nil { + return err + } + return t.q.DeleteNodeGenerationStatus(t.ctx, sqlc.DeleteNodeGenerationStatusParams{NodeID: id, Generation: int64(generation)}) +} + +func (t *nodeTx) UpsertGenerationStatus(status deployment.GenerationStatusRecord) error { + id, err := parseID(status.NodeID) + if err != nil { + return err + } + connection, err := parseID(status.ConnectionID) + if err != nil { + return err + } + return t.q.UpsertNodeGenerationStatus(t.ctx, sqlc.UpsertNodeGenerationStatusParams{NodeID: id, Generation: int64(status.Generation), SpecificationDigest: status.SpecificationDigest, ConnectionID: connection, OwnerEpoch: int64(status.OwnerEpoch), State: status.State, Diagnostic: status.Diagnostic}) +} + +func (t *nodeTx) PromoteServingGeneration(nodeID string, generation uint64) error { + id, err := parseID(nodeID) + if err != nil { + return err + } + return t.q.PromoteNodeServingGeneration(t.ctx, sqlc.PromoteNodeServingGenerationParams{ID: id, ReadyGeneration: pgtype.Int8{Int64: int64(generation), Valid: true}}) +} + +func (t *nodeTx) RefreshServingReadiness(nodeID string, protocol int) error { + id, err := parseID(nodeID) + if err != nil { + return err + } + return t.q.RefreshNodeServingReadiness(t.ctx, sqlc.RefreshNodeServingReadinessParams{ID: id, ProtocolVersion: int32(protocol)}) +} + +// deploymentTx is one leased deployment change. +type deploymentTx struct { + unit + cipher *credentialcrypto.Cipher +} + +var _ deployment.DeploymentTx = (*deploymentTx)(nil) + +// LoadDeployment opens the stored credential, which setup and switch need. +func (t *deploymentTx) LoadDeployment() (deployment.Record, error) { + d, err := t.loadDeployment() + if err != nil { + return deployment.Record{}, err + } + return record(d, t.cipher, true), nil +} + +func (t *deploymentTx) LoadSnapshot() (deployment.Snapshot, error) { + row, err := t.q.GetSandboxDeploymentSnapshot(t.ctx) + if err != nil { + return deployment.Snapshot{}, err + } + return snapshot(row) +} + +func (t *deploymentTx) CountResources() (deployment.Resources, error) { + row, err := t.q.CountRuntimeDeploymentResources(t.ctx) + if err != nil { + return deployment.Resources{}, err + } + return deployment.Resources{Allocations: row.Allocations, Pending: row.Pending}, nil +} + +func (t *deploymentTx) ClaimInstallation(installationID string) error { + id, err := parseID(installationID) + if err != nil { + return err + } + return t.q.ClaimWebSandboxDeployment(t.ctx, id) +} + +func (t *deploymentTx) SetProcessDeployment(installationID, backendFingerprint string, admissionPaused bool) error { + id, err := parseID(installationID) + if err != nil { + return err + } + return t.q.SetRuntimeDeployment(t.ctx, sqlc.SetRuntimeDeploymentParams{InstallationID: id, BackendFingerprint: backendFingerprint, AdmissionPaused: admissionPaused}) +} + +func (t *deploymentTx) SetManagerDeployment(provider, localNodeID string) error { + var local pgtype.UUID + if localNodeID != "" { + var err error + if local, err = parseID(localNodeID); err != nil { + return err + } + } + return t.q.SetRuntimeManagerDeployment(t.ctx, sqlc.SetRuntimeManagerDeploymentParams{ProviderKind: provider, LocalNodeID: local}) +} + +func (t *deploymentTx) SaveSelection(selection deployment.SelectionRecord) error { + if selection.Generation > math.MaxInt64 { + return deployment.ErrInvalidInput + } + params := sqlc.InitializeSandboxDeploymentParams{ProviderKind: selection.Provider, BackendFingerprint: selection.BackendFingerprint, Generation: int64(selection.Generation), Mode: selection.Mode, IdleSeconds: selection.IdleSeconds, RetentionSeconds: selection.RetentionSeconds, ProviderConfig: selection.Configuration.Public, ProviderMetadata: selection.Configuration.Metadata, Specification: selection.Specification} + if len(selection.Configuration.Secret) > 0 { + if t.cipher == nil { + return credentialcrypto.ErrUnavailable + } + sealed, err := t.cipher.SealSandboxDeployment(selection.Configuration.Secret, selection.InstallationID, selection.Generation) + if err != nil { + return errors.New("sandbox deployment credential encryption failed") + } + params.ProviderCredential = sealed + } + return translate(t.q.InitializeSandboxDeployment(t.ctx, params)) +} + +func (t *deploymentTx) RecordConfigurationMetadata(metadata json.RawMessage) error { + return translate(t.q.RecordSandboxConfigurationMetadata(t.ctx, metadata)) +} + +func (t *deploymentTx) RetainGeneration() error { return t.q.RetainSandboxGeneration(t.ctx) } + +func (t *deploymentTx) CollectGenerations() error { return t.q.CollectSandboxGenerations(t.ctx) } + +func (t *deploymentTx) RecordAudit(action, installationID string) error { + return auditpg.RecordDeploymentMutation(t.ctx, t.q, action, "sandbox_deployment", installationID) +} diff --git a/services/core/internal/sandbox/deployment.go b/services/core/internal/sandbox/deployment.go index 544820913..95b146cc0 100644 --- a/services/core/internal/sandbox/deployment.go +++ b/services/core/internal/sandbox/deployment.go @@ -112,3 +112,10 @@ func (s DeploymentSpec) Digest(provider string) string { digest := sha256.Sum256(raw) return hex.EncodeToString(digest[:]) } + +// Description is what a provider registration says about a deployment of it: +// its mode, its backend namespace fingerprint and its checkpoint timing. +type Description struct { + Mode, BackendFingerprint string + IdleSeconds, RetentionSeconds int64 +} diff --git a/services/core/internal/sandbox/node/agent.go b/services/core/internal/sandbox/node/agent.go index 3d31e488b..cb86b6eac 100644 --- a/services/core/internal/sandbox/node/agent.go +++ b/services/core/internal/sandbox/node/agent.go @@ -3,7 +3,6 @@ package node import ( "context" "errors" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "math/rand/v2" "net/http" "net/url" @@ -84,9 +83,6 @@ func Run(ctx context.Context, config AgentConfig) error { if err := sandbox.ValidateProvider(config.Provider); err != nil { return err } - if sandbox.SupportsCheckpoint(config.Provider) != providers.SupportsCheckpoint(config.Identity.Provider) { - return sandbox.ErrInvalid - } } release, err := lockDirectory(config.StateDirectory) if err != nil { diff --git a/services/core/internal/sandbox/node/connection_test.go b/services/core/internal/sandbox/node/connection_test.go index ede51f5b2..30f88ecd8 100644 --- a/services/core/internal/sandbox/node/connection_test.go +++ b/services/core/internal/sandbox/node/connection_test.go @@ -11,6 +11,8 @@ import ( "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/gorilla/websocket" ) @@ -174,7 +176,7 @@ func TestCopiedIdentityCannotReplaceNodeWithInflightCreate(t *testing.T) { hub.mu.Unlock() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - proxy := hub.Proxy(id.NodeID, "docker", 1) + proxy := hub.Proxy(id.NodeID, "docker", docker.Operations(), 1) r := reference() created := make(chan error, 1) go func() { _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: r}); created <- err }() diff --git a/services/core/internal/sandbox/node/creation_settlement_test.go b/services/core/internal/sandbox/node/creation_settlement_test.go index 8f04fe7b0..02ef95248 100644 --- a/services/core/internal/sandbox/node/creation_settlement_test.go +++ b/services/core/internal/sandbox/node/creation_settlement_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) @@ -71,7 +73,7 @@ func TestNodeCarriesCreationSettlementWithoutConvertingFailureToSuccess(t *testi } }() wait(t, func() bool { return hub.Online(id.NodeID) }) - proxy := hub.Proxy(id.NodeID, "docker", 1) + proxy := hub.Proxy(id.NodeID, "docker", docker.Operations(), 1) for _, operation := range []func(context.Context) (sandbox.Info, error){ func(ctx context.Context) (sandbox.Info, error) { return proxy.Create(ctx, sandbox.Bootstrap{Reference: ref}) diff --git a/services/core/internal/sandbox/node/docker_live_test.go b/services/core/internal/sandbox/node/docker_live_test.go index d9f0de5f9..5862b308c 100644 --- a/services/core/internal/sandbox/node/docker_live_test.go +++ b/services/core/internal/sandbox/node/docker_live_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" sandboxdocker "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" "github.com/google/uuid" "github.com/moby/moby/client" @@ -75,7 +76,7 @@ func TestDockerNodeTransportLifecycle(t *testing.T) { } }() wait(t, func() bool { return hub.Online(id.NodeID) }) - proxy := hub.Proxy(id.NodeID, "docker", 1) + proxy := hub.Proxy(id.NodeID, "docker", docker.Operations(), 1) r := reference() defer func() { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) diff --git a/services/core/internal/sandbox/node/generation_connection_test.go b/services/core/internal/sandbox/node/generation_connection_test.go index f440ba333..d39b68459 100644 --- a/services/core/internal/sandbox/node/generation_connection_test.go +++ b/services/core/internal/sandbox/node/generation_connection_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) @@ -61,7 +63,7 @@ func TestGenerationWireRoutesOldOwnershipAndCurrentTargetSeparately(t *testing.T }() wait(t, func() bool { return hub.Online(id.NodeID) }) for _, generation := range []uint64{1, 17, 9} { - proxy := hub.GenerationProvider("docker", func(context.Context, sandbox.Reference) (string, uint64, error) { return id.NodeID, generation, nil }) + proxy := hub.GenerationProvider("docker", docker.Operations(), func(context.Context, sandbox.Reference) (string, uint64, error) { return id.NodeID, generation, nil }) ref := reference() if _, err := proxy.GetInfo(ctx, ref); err != nil { t.Fatal("retained generation info failed", generation, err) diff --git a/services/core/internal/sandbox/node/hub_lifetime_test.go b/services/core/internal/sandbox/node/hub_lifetime_test.go index 8d00a7451..ed5d0d379 100644 --- a/services/core/internal/sandbox/node/hub_lifetime_test.go +++ b/services/core/internal/sandbox/node/hub_lifetime_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/gorilla/websocket" ) @@ -44,7 +46,7 @@ func assertInfoResponsive(t *testing.T, hub *Hub, id Identity) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - info, err := hub.Proxy(id.NodeID, id.Provider, 1).GetInfo(ctx, reference()) + info, err := hub.Proxy(id.NodeID, id.Provider, docker.Operations(), 1).GetInfo(ctx, reference()) if err != nil || info.ProviderID != "retained" { t.Fatalf("unrelated node RPC blocked: info=%+v err=%v", info, err) } @@ -284,7 +286,10 @@ func TestHubSendQueueRespectsCallerCancellation(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) defer cancel() done := make(chan error, 1) - go func() { _, err := hub.Proxy(id.NodeID, id.Provider, 1).GetInfo(ctx, reference()); done <- err }() + go func() { + _, err := hub.Proxy(id.NodeID, id.Provider, docker.Operations(), 1).GetInfo(ctx, reference()) + done <- err + }() select { case err := <-done: if !errors.Is(err, context.DeadlineExceeded) { diff --git a/services/core/internal/sandbox/node/identity.go b/services/core/internal/sandbox/node/identity.go index 86ab0a187..79567666e 100644 --- a/services/core/internal/sandbox/node/identity.go +++ b/services/core/internal/sandbox/node/identity.go @@ -12,7 +12,6 @@ import ( "path/filepath" "syscall" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "github.com/google/uuid" ) @@ -129,7 +128,7 @@ func initIdentity(dir, coreURL string, identity Identity) (StoredIdentity, error if identity.NodeID == "" { identity.NodeID = uuid.NewString() } - if !validID(identity.NodeID) || !validID(identity.InstallationID) || !providers.IsNode(identity.Provider) || len(identity.BackendFingerprint) != 64 { + if !validID(identity.NodeID) || !validID(identity.InstallationID) || identity.Provider == "" || len(identity.BackendFingerprint) != 64 { return StoredIdentity{}, errors.New("invalid node configuration") } secret := make([]byte, 32) diff --git a/services/core/internal/sandbox/node/node_test.go b/services/core/internal/sandbox/node/node_test.go index 792bc3f48..2d375be1c 100644 --- a/services/core/internal/sandbox/node/node_test.go +++ b/services/core/internal/sandbox/node/node_test.go @@ -14,6 +14,8 @@ import ( "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/google/uuid" "github.com/gorilla/websocket" @@ -122,7 +124,7 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing t.Fatal("running node did not retain lifetime identity lock") } r := reference() - proxy := hub.Proxy(id.NodeID, "docker", 1) + proxy := hub.Proxy(id.NodeID, "docker", docker.Operations(), 1) createCtx, stopCreate := context.WithTimeout(ctx, 150*time.Millisecond) defer stopCreate() createDone := make(chan error, 1) @@ -165,7 +167,7 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing func TestOfflineIsUnknownAndDockerDoesNotAdvertiseCheckpoint(t *testing.T) { h := NewHub(HubOptions{OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) - p := h.Proxy(uuid.NewString(), "docker", 1) + p := h.Proxy(uuid.NewString(), "docker", docker.Operations(), 1) if sandbox.SupportsCheckpoint(p) { t.Fatal("docker advertised checkpoint") } diff --git a/services/core/internal/sandbox/node/observations_test.go b/services/core/internal/sandbox/node/observations_test.go index 401ab9cf1..f9b7e1656 100644 --- a/services/core/internal/sandbox/node/observations_test.go +++ b/services/core/internal/sandbox/node/observations_test.go @@ -4,13 +4,15 @@ import ( "context" "encoding/json" "errors" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "net/http/httptest" "reflect" "sync" "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/google/uuid" @@ -87,7 +89,7 @@ func TestObservationsRouteThroughAssignedNodeWithoutLifecycleCalls(t *testing.T) stopSecond := runObservationNode(t, hub, server.URL, second, b) ra, rb := reference(), reference() assignments := map[sandbox.Reference]string{ra: first.NodeID, rb: second.NodeID} - source := hub.GenerationProvider("docker", func(_ context.Context, r sandbox.Reference) (string, uint64, error) { + source := hub.GenerationProvider("docker", docker.Operations(), func(_ context.Context, r sandbox.Reference) (string, uint64, error) { if id, ok := assignments[r]; ok { return id, 1, nil } diff --git a/services/core/internal/sandbox/node/operations_test.go b/services/core/internal/sandbox/node/operations_test.go index dfafaf4c0..2cd469fda 100644 --- a/services/core/internal/sandbox/node/operations_test.go +++ b/services/core/internal/sandbox/node/operations_test.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "errors" + "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" "github.com/google/uuid" - "testing" ) func TestUnsupportedWireIsExplicitAndDoesNotInvokeProvider(t *testing.T) { @@ -35,10 +37,10 @@ func TestUnsupportedWireIsExplicitAndDoesNotInvokeProvider(t *testing.T) { } } func TestUnsupportedProxyRejectsBeforeNodeResolution(t *testing.T) { - p := &provider{kind: "docker", resolveGeneration: func(context.Context, sandbox.Reference) (string, uint64, error) { + p := (&Hub{}).GenerationProvider("docker", docker.Operations(), func(context.Context, sandbox.Reference) (string, uint64, error) { t.Fatal("unsupported call resolved a node") return "", 0, nil - }} + }).(*provider) if err := sandbox.ValidateProvider(p); err != nil { t.Fatal(err) } diff --git a/services/core/internal/sandbox/node/provider_operations_fixture_test.go b/services/core/internal/sandbox/node/provider_operations_fixture_test.go index 8d34c1511..2569a6632 100644 --- a/services/core/internal/sandbox/node/provider_operations_fixture_test.go +++ b/services/core/internal/sandbox/node/provider_operations_fixture_test.go @@ -2,6 +2,7 @@ package node import ( "context" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" diff --git a/services/core/internal/sandbox/node/proxy.go b/services/core/internal/sandbox/node/proxy.go index cf30880ea..01daa2aa1 100644 --- a/services/core/internal/sandbox/node/proxy.go +++ b/services/core/internal/sandbox/node/proxy.go @@ -3,26 +3,26 @@ package node import ( "context" "errors" + "maps" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" ) type provider struct { hub *Hub resolveGeneration func(context.Context, sandbox.Reference) (string, uint64, error) kind string + operations providercontract.Operations } var _ sandbox.SandboxProvider = (*provider)(nil) var _ sandbox.CheckpointProvider = (*provider)(nil) // Proxy binds a fixed node and deployment generation explicitly. -func (h *Hub) Proxy(id, kind string, generation uint64) sandbox.SandboxProvider { - return h.GenerationProvider(kind, func(context.Context, sandbox.Reference) (string, uint64, error) { return id, generation, nil }) - +func (h *Hub) Proxy(id, kind string, declared providercontract.Operations, generation uint64) sandbox.SandboxProvider { + return h.GenerationProvider(kind, declared, func(context.Context, sandbox.Reference) (string, uint64, error) { return id, generation, nil }) } func (p *provider) call(ctx context.Context, q request) (response, error) { if err := providercontract.Require(p, operationMethod(q.Operation)); err != nil { @@ -70,15 +70,7 @@ func creationSettled(info *sandbox.Info, ref sandbox.Reference) bool { return info.ProviderID != "" } func (p *provider) ProviderOperations() providercontract.Operations { - adapter, err := providers.Lookup(p.kind) - if err != nil { - return nil - } - ops := adapter.Operations() - ops["DiscoverSelection"] = providercontract.Support{State: providercontract.Unsupported, Reason: "node_configuration_is_core_owned"} - ops["VerifyCredential"] = providercontract.Support{State: providercontract.Unsupported, Reason: "node_credentials_are_transport_owned"} - ops["ObserveBatch"] = providercontract.Support{State: providercontract.Unsupported, Reason: "node_transport_has_no_batch_observation"} - return ops + return maps.Clone(p.operations) } func (*provider) ObserveBatch(context.Context, []runtimeobs.Target) ([]runtimeobs.BatchResult, error) { return nil, &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: "node_transport_has_no_batch_observation"} @@ -170,8 +162,14 @@ func (p *provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c san } // GenerationProvider routes every operation with allocation-owned generation, -// distinct from the request's compute generation. -func (h *Hub) GenerationProvider(kind string, resolve func(context.Context, sandbox.Reference) (string, uint64, error)) sandbox.SandboxProvider { - p := &provider{hub: h, kind: kind, resolveGeneration: resolve} - return p +// distinct from the request's compute generation. declared is the kind's +// registered operations; the transport replaces the ones it owns. +func (h *Hub) GenerationProvider(kind string, declared providercontract.Operations, resolve func(context.Context, sandbox.Reference) (string, uint64, error)) sandbox.SandboxProvider { + operations := maps.Clone(declared) + if operations != nil { + operations["DiscoverSelection"] = providercontract.Support{State: providercontract.Unsupported, Reason: "node_configuration_is_core_owned"} + operations["VerifyCredential"] = providercontract.Support{State: providercontract.Unsupported, Reason: "node_credentials_are_transport_owned"} + operations["ObserveBatch"] = providercontract.Support{State: providercontract.Unsupported, Reason: "node_transport_has_no_batch_observation"} + } + return &provider{hub: h, kind: kind, operations: operations, resolveGeneration: resolve} } diff --git a/services/core/internal/sandbox/node/recovery_test.go b/services/core/internal/sandbox/node/recovery_test.go index d22d8a270..23ba863e3 100644 --- a/services/core/internal/sandbox/node/recovery_test.go +++ b/services/core/internal/sandbox/node/recovery_test.go @@ -3,9 +3,6 @@ package node import ( "context" "errors" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/google/uuid" - "github.com/gorilla/websocket" "net/http" "net/http/httptest" "os" @@ -14,6 +11,11 @@ import ( "sync" "testing" "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" + "github.com/google/uuid" + "github.com/gorilla/websocket" ) func TestCoreRestartFencesOldConnectionAndNodeRestartKeepsIdentity(t *testing.T) { @@ -55,10 +57,10 @@ func TestCoreRestartFencesOldConnectionAndNodeRestartKeepsIdentity(t *testing.T) if first.Online(id.NodeID) { t.Fatal("old owner remained online") } - if _, err = first.Proxy(id.NodeID, "docker", 1).GetInfo(context.Background(), reference()); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { + if _, err = first.Proxy(id.NodeID, "docker", docker.Operations(), 1).GetInfo(context.Background(), reference()); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { t.Fatalf("old owner request = %v", err) } - if _, err = second.Proxy(id.NodeID, "docker", 1).GetInfo(context.Background(), reference()); err != nil { + if _, err = second.Proxy(id.NodeID, "docker", docker.Operations(), 1).GetInfo(context.Background(), reference()); err != nil { t.Fatal(err) } stop() @@ -154,7 +156,7 @@ func TestHeartbeatAcknowledgementKeepsIdleConnectionAlive(t *testing.T) { if !hub.Online(id.NodeID) { t.Fatal("idle node disconnected") } - if _, err = hub.Proxy(id.NodeID, "docker", 1).GetInfo(ctx, reference()); err != nil { + if _, err = hub.Proxy(id.NodeID, "docker", docker.Operations(), 1).GetInfo(ctx, reference()); err != nil { t.Fatal(err) } } @@ -232,7 +234,7 @@ func TestDegradedNodeRetainsObservationAndCleanup(t *testing.T) { case <-time.After(time.Second): t.Fatal("missing health") } - proxy := hub.Proxy(id.NodeID, "docker", 1) + proxy := hub.Proxy(id.NodeID, "docker", docker.Operations(), 1) r := reference() if _, err = proxy.Create(ctx, sandbox.Bootstrap{Reference: r}); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { t.Fatalf("create = %v", err) diff --git a/services/core/internal/sandbox/node/timeout_test.go b/services/core/internal/sandbox/node/timeout_test.go index b8832a6e3..c849b853d 100644 --- a/services/core/internal/sandbox/node/timeout_test.go +++ b/services/core/internal/sandbox/node/timeout_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/google/uuid" ) @@ -101,7 +103,7 @@ func TestQueuedMutationExpiresWithoutExecution(t *testing.T) { } }() wait(t, func() bool { return hub.Online(id.NodeID) }) - proxy := hub.Proxy(id.NodeID, "docker", 1) + proxy := hub.Proxy(id.NodeID, "docker", docker.Operations(), 1) r := reference() createCtx, stopCreate := context.WithTimeout(ctx, 3*time.Second) defer stopCreate() diff --git a/services/core/internal/sandbox/providers/artifacts.go b/services/core/internal/sandbox/providers/artifacts.go index e6b70b936..8db3baa16 100644 --- a/services/core/internal/sandbox/providers/artifacts.go +++ b/services/core/internal/sandbox/providers/artifacts.go @@ -14,12 +14,12 @@ var runtimeImage = providerassets.Artifact{Path: "images/runtime.tar.gz", Suffix var runtimePolicy = providerassets.Artifact{Path: "runtime/seccomp.json", Suffix: "seccomp.json", Role: "policy"} // ArtifactCatalog projects registered node requirements for Web and installers. -func ArtifactCatalog() (map[string][]providerassets.Artifact, error) { +func (r *Registry) ArtifactCatalog() (map[string][]providerassets.Artifact, error) { result := map[string][]providerassets.Artifact{} paths := map[string]providerassets.Artifact{} suffixes := map[string]string{} - for kind := range adapters { - a, err := Lookup(kind) + for kind := range r.adapters { + a, err := r.Lookup(kind) if err != nil { return nil, err } diff --git a/services/core/internal/sandbox/providers/artifacts_test.go b/services/core/internal/sandbox/providers/artifacts_test.go index 9e1fed379..0857454f5 100644 --- a/services/core/internal/sandbox/providers/artifacts_test.go +++ b/services/core/internal/sandbox/providers/artifacts_test.go @@ -14,7 +14,8 @@ import ( ) func TestArtifactProjectionsMatchRegistrations(t *testing.T) { - catalog, err := ArtifactCatalog() + registry := Builtin() + catalog, err := registry.ArtifactCatalog() if err != nil { t.Fatal(err) } @@ -44,6 +45,7 @@ func TestArtifactProjectionsMatchRegistrations(t *testing.T) { } func TestNodeArtifactRegistrationRejectsInvalidDeclarations(t *testing.T) { + registry := Builtin() for _, mutate := range []func(*Adapter){ func(a *Adapter) { a.NodeArtifacts = nil }, func(a *Adapter) { a.NodeArtifacts[0].Path = "../private" }, @@ -55,7 +57,7 @@ func TestNodeArtifactRegistrationRejectsInvalidDeclarations(t *testing.T) { func(a *Adapter) { a.NodeArtifacts[0].Role = "unknown" }, func(a *Adapter) { a.NodeArtifacts = append(a.NodeArtifacts, a.NodeArtifacts[0]) }, } { - a := adapters["docker"] + a := registry.adapters["docker"] a.NodeArtifacts = append([]providerassets.Artifact(nil), a.NodeArtifacts...) mutate(&a) if err := ValidateRegistration(a); !errors.Is(err, providercontract.ErrContract) { @@ -63,26 +65,25 @@ func TestNodeArtifactRegistrationRejectsInvalidDeclarations(t *testing.T) { } } const kind = "another-node-provider" - adapters[kind] = adapters["docker"] - defer delete(adapters, kind) - catalog, err := ArtifactCatalog() - if err != nil || !reflect.DeepEqual(catalog[kind], adapters[kind].NodeArtifacts) { + registry.adapters[kind] = registry.adapters["docker"] + catalog, err := registry.ArtifactCatalog() + if err != nil || !reflect.DeepEqual(catalog[kind], registry.adapters[kind].NodeArtifacts) { t.Fatalf("additional registration not projected: %v", err) } } func TestArtifactCatalogRejectsConflictingSharedFiles(t *testing.T) { + registry := Builtin() const kind = "conflicting-provider" - defer delete(adapters, kind) for _, mutate := range []func(*providerassets.Artifact){ func(item *providerassets.Artifact) { item.Suffix = "different" }, func(item *providerassets.Artifact) { item.Path = "native/bin/other-node" }, } { - a := adapters["docker"] + a := registry.adapters["docker"] a.NodeArtifacts = append([]providerassets.Artifact(nil), a.NodeArtifacts...) mutate(&a.NodeArtifacts[0]) - adapters[kind] = a - if _, err := ArtifactCatalog(); !errors.Is(err, providercontract.ErrContract) { + registry.adapters[kind] = a + if _, err := registry.ArtifactCatalog(); !errors.Is(err, providercontract.ErrContract) { t.Fatalf("conflicting declaration accepted: %v", err) } } diff --git a/services/core/internal/sandbox/providers/config.go b/services/core/internal/sandbox/providers/config.go index 2a0a6701c..22e789f49 100644 --- a/services/core/internal/sandbox/providers/config.go +++ b/services/core/internal/sandbox/providers/config.go @@ -76,16 +76,16 @@ type LocalOptions struct { GenerationStateDirectory string } -func Build(config Config, options LocalOptions) (*Built, func(), error) { +func (r *Registry) Build(config Config, options LocalOptions) (*Built, func(), error) { closeProvider := func() {} - if err := validateSpecification(config); err != nil { + if err := r.validateSpecification(config); err != nil { return nil, closeProvider, err } id, err := uuid.Parse(config.InstallationID) if err != nil || id == uuid.Nil || id.String() != config.InstallationID { return nil, closeProvider, errors.New("sandbox requires a canonical installation_id UUID") } - adapter, err := Lookup(config.Provider) + adapter, err := r.Lookup(config.Provider) if err != nil || adapter.BuildLocal == nil { return nil, closeProvider, errors.New("sandbox provider is not node-local") } diff --git a/services/core/internal/sandbox/providers/config_test.go b/services/core/internal/sandbox/providers/config_test.go index 2426b4538..0753377ef 100644 --- a/services/core/internal/sandbox/providers/config_test.go +++ b/services/core/internal/sandbox/providers/config_test.go @@ -24,19 +24,20 @@ func TestNodeRejectsCoreConfigurationAndUnknownProvider(t *testing.T) { } } func TestNodeSpecificationCannotBeOverridden(t *testing.T) { - if err := validateSpecification(Config{Generation: 1, Provider: "e2b", Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}, Docker: &Docker{}}); err == nil { + registry := Builtin() + if err := registry.validateSpecification(Config{Generation: 1, Provider: "e2b", Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}, Docker: &Docker{}}); err == nil { t.Fatal("accepted a cloud provider on a node") } release := sandbox.RuntimeRelease{SourceCommit: strings.Repeat("a", 40), ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64), MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64)} c := Config{Generation: 1, Provider: "docker", Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}, Runtime: &release}, Docker: &Docker{Image: release.ImageID}} for _, image := range []string{release.ImageID, release.ImageManifestDigest} { c.Docker.Image = image - if err := validateSpecification(c); err != nil { + if err := registry.validateSpecification(c); err != nil { t.Fatal(err) } } c.Docker.Image = "sha256:" + strings.Repeat("a", 64) - if err := validateSpecification(c); err == nil { + if err := registry.validateSpecification(c); err == nil { t.Fatal("accepted different Runtime") } c.Provider = "microsandbox" @@ -44,7 +45,7 @@ func TestNodeSpecificationCannotBeOverridden(t *testing.T) { c.Specification.Resources.RootDiskMiB = 8192 c.Specification.Resources.EnvironmentDiskMiB = 8192 c.Microsandbox = &Microsandbox{CPUs: 2, MemoryMiB: 2048, RootDiskMiB: 8192, EnvironmentDiskMiB: 8192, Image: release.MicrosandboxRef, RuntimeSHA256: release.RuntimeSHA256, FirmwareSHA256: release.FirmwareSHA256} - if err := validateSpecification(c); err != nil { + if err := registry.validateSpecification(c); err != nil { t.Fatal(err) } for _, change := range []func(*Microsandbox){func(m *Microsandbox) { m.CPUs = 1 }, func(m *Microsandbox) { m.MemoryMiB = 1024 }, func(m *Microsandbox) { m.EnvironmentDiskMiB = 4096 }, func(m *Microsandbox) { m.RootDiskMiB = 4096 }, func(m *Microsandbox) { m.FirmwareSHA256 = strings.Repeat("a", 64) }} { @@ -52,12 +53,12 @@ func TestNodeSpecificationCannotBeOverridden(t *testing.T) { change(©) other := c other.Microsandbox = © - if err := validateSpecification(other); err == nil { + if err := registry.validateSpecification(other); err == nil { t.Fatal("accepted local specification override") } } c.Generation = 0 - if err := validateSpecification(c); err == nil { + if err := registry.validateSpecification(c); err == nil { t.Fatal("accepted unbound node") } } diff --git a/services/core/internal/sandbox/providers/configuration.go b/services/core/internal/sandbox/providers/configuration.go index e4699998e..910a0cb4c 100644 --- a/services/core/internal/sandbox/providers/configuration.go +++ b/services/core/internal/sandbox/providers/configuration.go @@ -8,36 +8,36 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -func DecodeInput(kind string, public, credential json.RawMessage) (sandbox.Configuration, error) { - a, err := Lookup(kind) +func (r *Registry) DecodeInput(kind string, public, credential json.RawMessage) (sandbox.Configuration, error) { + a, err := r.Lookup(kind) if err != nil { return nil, err } return a.Configuration.DecodeInput(public, credential) } -func Encode(kind string, c sandbox.Configuration) (sandbox.ConfigurationRecord, error) { - a, err := Lookup(kind) +func (r *Registry) Encode(kind string, c sandbox.Configuration) (sandbox.ConfigurationRecord, error) { + a, err := r.Lookup(kind) if err != nil { return sandbox.ConfigurationRecord{}, err } return a.Configuration.Encode(c) } -func Decode(kind string, r sandbox.ConfigurationRecord) (sandbox.Configuration, error) { - a, err := Lookup(kind) +func (r *Registry) Decode(kind string, record sandbox.ConfigurationRecord) (sandbox.Configuration, error) { + a, err := r.Lookup(kind) if err != nil { return nil, err } - return a.Configuration.Decode(r) + return a.Configuration.Decode(record) } -func Equal(kind string, a, b sandbox.Configuration) (bool, error) { - adapter, err := Lookup(kind) +func (r *Registry) Equal(kind string, a, b sandbox.Configuration) (bool, error) { + adapter, err := r.Lookup(kind) if err != nil { return false, err } return adapter.Configuration.Equal(a, b) } -func UsesCredential(kind string) (bool, error) { - a, err := Lookup(kind) +func (r *Registry) UsesCredential(kind string) (bool, error) { + a, err := r.Lookup(kind) if err != nil { return false, err } @@ -46,8 +46,8 @@ func UsesCredential(kind string) (bool, error) { } return required(a.Configuration.Requirements().Credential) } -func RequiresPublicOrigin(kind string) (bool, error) { - a, err := Lookup(kind) +func (r *Registry) RequiresPublicOrigin(kind string) (bool, error) { + a, err := r.Lookup(kind) if err != nil { return false, err } @@ -66,29 +66,29 @@ func required(value sandbox.Requirement) (bool, error) { return false, providercontract.ErrContract } } -func Normalize(s sandbox.Selection) (sandbox.Selection, error) { - a, e := Lookup(s.Provider) +func (r *Registry) Normalize(s sandbox.Selection) (sandbox.Selection, error) { + a, e := r.Lookup(s.Provider) if e != nil { return s, e } return a.Configuration.Normalize(s) } -func ResolveChange(next, previous sandbox.Selection) (sandbox.Selection, error) { - a, e := Lookup(next.Provider) +func (r *Registry) ResolveChange(next, previous sandbox.Selection) (sandbox.Selection, error) { + a, e := r.Lookup(next.Provider) if e != nil { return next, e } return a.Configuration.ResolveChange(next, previous) } -func WithCredential(owner, candidate sandbox.Selection) (sandbox.Selection, error) { +func (r *Registry) WithCredential(owner, candidate sandbox.Selection) (sandbox.Selection, error) { if owner.Provider != candidate.Provider { return owner, sandbox.ErrInvalid } - a, e := Lookup(owner.Provider) + a, e := r.Lookup(owner.Provider) if e != nil { return owner, e } - needsCredential, e := UsesCredential(owner.Provider) + needsCredential, e := r.UsesCredential(owner.Provider) if e != nil { return owner, e } @@ -98,8 +98,8 @@ func WithCredential(owner, candidate sandbox.Selection) (sandbox.Selection, erro owner.Configuration, e = a.Configuration.WithCredential(owner.Configuration, candidate.Configuration) return owner, e } -func DiscoverConfiguration(ctx context.Context, kind string, input sandbox.ConfigurationDiscoveryInput, paths sandbox.ProcessPaths) (json.RawMessage, error) { - a, err := Lookup(kind) +func (r *Registry) DiscoverConfiguration(ctx context.Context, kind string, input sandbox.ConfigurationDiscoveryInput, paths sandbox.ProcessPaths) (json.RawMessage, error) { + a, err := r.Lookup(kind) if err != nil { return nil, err } diff --git a/services/core/internal/sandbox/providers/configuration_flow_test.go b/services/core/internal/sandbox/providers/configuration_flow_test.go index f335cb84c..05a5eb8b4 100644 --- a/services/core/internal/sandbox/providers/configuration_flow_test.go +++ b/services/core/internal/sandbox/providers/configuration_flow_test.go @@ -4,7 +4,13 @@ import ( "bytes" "context" "encoding/json" + "net/http/httptest" + "strings" + "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" @@ -13,9 +19,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" - "net/http/httptest" - "strings" - "testing" ) type regionalConfiguration struct { @@ -79,21 +82,35 @@ func TestAdditionalConfigurationProviderUsesCommonAPIAndStore(t *testing.T) { // The deployment identity and execution lease are database-wide. pool := pgtest.OpenIsolated(t, nil) kind := "regional-fixture" - adapter, err := providers.Lookup("docker") + adapter, err := providers.Builtin().Lookup("docker") if err != nil { t.Fatal(err) } adapter.Configuration = regionalCodec{} - providers.RegisterFixture(t, kind, adapter) + registry := providers.FixtureRegistry(t, kind, adapter) s := store.New(pool) + // The deployment reaches the registered configuration only through the + // registry it is built with. + deployments := func() *deployment.Service { + storage := deploymentpg.New(pgunit.NewPool(pool), nil) + service, err := deployment.NewService(storage, storage, registry, "") + if err != nil { + t.Fatal(err) + } + return service + } + service := deployments() lease, err := pgunit.AcquireLease(t.Context(), pool) if err != nil { t.Fatal(err) } defer lease.Close(context.Background()) - w := store.NewExecution(s, lease) + changes, err := deployment.NewExecutionOperations(service, deploymentpg.NewExecution(lease, nil)) + if err != nil { + t.Fatal(err) + } installation := uuid.NewString() - if err = w.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + if err = changes.Claim(t.Context(), installation); err != nil { t.Fatal(err) } auth, err := api.NewDeploymentAuthenticator([]string{runtimedevice.HashCredential("fixture-admin")}) @@ -116,7 +133,8 @@ func TestAdditionalConfigurationProviderUsesCommonAPIAndStore(t *testing.T) { ExecutorConnections: struct{ api.ExecutorConnections }{}, Metrics: struct{ api.Metrics }{}, RuntimeObservations: struct{ api.RuntimeObservations }{}, RuntimeHistory: struct{ api.RuntimeHistory }{}, Execution: &api.Execution{ExecutorURL: "wss://core.example/api/v1/agent-daemon/ws", Admission: s, SessionArchive: s, Workspaces: struct{ api.EnvironmentWorkspaces }{}}, - Sandboxes: &api.Sandboxes{Deployment: s, DeploymentChanges: leaseSetup{t: t, store: w, installation: installation}, ConfigurationDiscovery: struct{ api.ConfigurationDiscovery }{}}, + Sandboxes: &api.Sandboxes{Deployment: service, NodeAllocations: s, DeploymentChanges: leaseSetup{t: t, changes: changes, installation: installation}, + DeploymentReset: leaseSetup{t: t, changes: changes, installation: installation}, ConfigurationDiscovery: struct{ api.ConfigurationDiscovery }{}}, }) if err != nil { t.Fatal(err) @@ -130,8 +148,7 @@ func TestAdditionalConfigurationProviderUsesCommonAPIAndStore(t *testing.T) { if response.Code != 200 || !strings.Contains(response.Body.String(), `"zone":"west"`) { t.Fatal(response.Code, response.Body.String()) } - reopened := store.New(pool) - saved, err := reopened.GetSandboxSetup(t.Context()) + saved, err := deployments().Setup(t.Context()) if err != nil || saved.Configuration.(regionalConfiguration).Zone != "west" { t.Fatal("configuration did not roundtrip", err) } @@ -145,25 +162,25 @@ func TestAdditionalConfigurationProviderUsesCommonAPIAndStore(t *testing.T) { // The flow makes no other deployment change. type leaseSetup struct { t *testing.T - store *store.Store + changes *deployment.ExecutionOperations installation string } -func (l leaseSetup) InitializeSandboxDeployment(ctx context.Context, in store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error) { - return l.store.InitializeSandboxDeployment(ctx, l.installation, in) +func (l leaseSetup) InitializeSandboxDeployment(ctx context.Context, in sandbox.Selection) (deployment.View, error) { + return l.changes.Initialize(ctx, l.installation, in) } -func (l leaseSetup) UpdateSandboxDeployment(context.Context, store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) { +func (l leaseSetup) UpdateSandboxDeployment(context.Context, sandbox.Selection) (deployment.View, error) { l.t.Fatal("unexpected call to UpdateSandboxDeployment") - return store.RuntimeDeploymentView{}, nil + return deployment.View{}, nil } -func (l leaseSetup) StartSandboxReset(context.Context, store.SandboxResetRequest) (store.RuntimeDeploymentView, error) { +func (l leaseSetup) StartSandboxReset(context.Context, store.SandboxResetRequest) (deployment.View, error) { l.t.Fatal("unexpected call to StartSandboxReset") - return store.RuntimeDeploymentView{}, nil + return deployment.View{}, nil } -func (l leaseSetup) CancelSandboxReset(context.Context, uint64) (store.RuntimeDeploymentView, error) { +func (l leaseSetup) CancelSandboxReset(context.Context, uint64) (deployment.View, error) { l.t.Fatal("unexpected call to CancelSandboxReset") - return store.RuntimeDeploymentView{}, nil + return deployment.View{}, nil } diff --git a/services/core/internal/sandbox/providers/configuration_test.go b/services/core/internal/sandbox/providers/configuration_test.go index d032c4760..43588e61a 100644 --- a/services/core/internal/sandbox/providers/configuration_test.go +++ b/services/core/internal/sandbox/providers/configuration_test.go @@ -9,8 +9,9 @@ import ( ) func TestNodeConfigurationExplicitUnsupportedAndStrictEmptyInput(t *testing.T) { + registry := Builtin() for _, kind := range []string{"docker", "microsandbox"} { - a, err := Lookup(kind) + a, err := registry.Lookup(kind) if err != nil { t.Fatal(err) } @@ -38,7 +39,8 @@ func TestNodeConfigurationExplicitUnsupportedAndStrictEmptyInput(t *testing.T) { } func TestConfigurationRequirementsDoNotTurnLookupFailuresIntoFalse(t *testing.T) { - for _, check := range []func(string) (bool, error){UsesCredential, RequiresPublicOrigin} { + registry := Builtin() + for _, check := range []func(string) (bool, error){registry.UsesCredential, registry.RequiresPublicOrigin} { if _, err := check("missing-configuration-provider"); err == nil { t.Fatal("unknown provider treated as not required") } diff --git a/services/core/internal/sandbox/providers/deployment_contract_test.go b/services/core/internal/sandbox/providers/deployment_contract_test.go index 990c48f3c..ad8b4b414 100644 --- a/services/core/internal/sandbox/providers/deployment_contract_test.go +++ b/services/core/internal/sandbox/providers/deployment_contract_test.go @@ -10,11 +10,12 @@ import ( ) func TestInstallerDeploymentProjectionIsCurrent(t *testing.T) { + registry := Builtin() raw, err := os.ReadFile("../../../../../deploy/install/node_spec.py") if err != nil { t.Fatal(err) } - expected, err := PythonDeploymentContract() + expected, err := registry.PythonDeploymentContract() if err != nil { t.Fatal(err) } @@ -23,6 +24,7 @@ func TestInstallerDeploymentProjectionIsCurrent(t *testing.T) { } } func TestDeploymentContractFixtures(t *testing.T) { + registry := Builtin() raw, err := os.ReadFile("../testdata/deployment-contract.json") if err != nil { t.Fatal(err) @@ -42,7 +44,7 @@ func TestDeploymentContractFixtures(t *testing.T) { decoder.DisallowUnknownFields() err := decoder.Decode(&spec) if err == nil { - err = ValidateSpecification(fixture.Provider, spec) + err = registry.ValidateSpecification(fixture.Provider, spec) } if (err == nil) != fixture.Valid { t.Fatalf("validation differs: %v", err) diff --git a/services/core/internal/sandbox/providers/e2b.go b/services/core/internal/sandbox/providers/e2b.go index 041046270..bb3861f7f 100644 --- a/services/core/internal/sandbox/providers/e2b.go +++ b/services/core/internal/sandbox/providers/e2b.go @@ -14,8 +14,8 @@ type DirectConfig struct { Fence *sandbox.CallFence } -func BuildDirect(c DirectConfig) (sandbox.SandboxProvider, error) { - a, e := Lookup(c.Selection.Provider) +func (r *Registry) BuildDirect(c DirectConfig) (sandbox.SandboxProvider, error) { + a, e := r.Lookup(c.Selection.Provider) if e != nil { return nil, e } diff --git a/services/core/internal/sandbox/providers/export_test.go b/services/core/internal/sandbox/providers/export_test.go index d9f223cb0..55a8c501a 100644 --- a/services/core/internal/sandbox/providers/export_test.go +++ b/services/core/internal/sandbox/providers/export_test.go @@ -2,12 +2,14 @@ package providers import "testing" -// RegisterFixture exists only in this package's test binary, never in Core. -func RegisterFixture(t *testing.T, kind string, adapter Adapter) { +// FixtureRegistry returns the built-in registry with one more adapter. It +// exists only in this package's test binary, never in Core. +func FixtureRegistry(t *testing.T, kind string, adapter Adapter) *Registry { t.Helper() - if _, ok := adapters[kind]; ok { + registry := Builtin() + if _, ok := registry.adapters[kind]; ok { t.Fatal("fixture replaces existing registration") } - adapters[kind] = adapter - t.Cleanup(func() { delete(adapters, kind) }) + registry.adapters[kind] = adapter + return registry } diff --git a/services/core/internal/sandbox/providers/generation_test.go b/services/core/internal/sandbox/providers/generation_test.go index 35a1b7a72..58f961693 100644 --- a/services/core/internal/sandbox/providers/generation_test.go +++ b/services/core/internal/sandbox/providers/generation_test.go @@ -32,6 +32,7 @@ func generationConfig(t *testing.T) Config { } func TestMicrosandboxGenerationDirectoryOwnership(t *testing.T) { + registry := Builtin() for _, mode := range []string{"private", "public", "symlink", "file"} { t.Run(mode, func(t *testing.T) { state := t.TempDir() @@ -53,7 +54,7 @@ func TestMicrosandboxGenerationDirectoryOwnership(t *testing.T) { t.Fatal(err) } } - built, closeProvider, err := Build(generationConfig(t), LocalOptions{GenerationStateDirectory: state}) + built, closeProvider, err := registry.Build(generationConfig(t), LocalOptions{GenerationStateDirectory: state}) closeProvider() if mode == "private" { info, statErr := os.Lstat(directory) @@ -72,9 +73,10 @@ func TestMicrosandboxGenerationDirectoryOwnership(t *testing.T) { // Exercise the actual ProcessCaller boundary to prove the constructor binds the // lease to this generation, installation and specification before any helper runs. func TestMicrosandboxGenerationBindsLeaseIdentity(t *testing.T) { + registry := Builtin() config := generationConfig(t) state := t.TempDir() - built, closeProvider, err := Build(config, LocalOptions{GenerationStateDirectory: state}) + built, closeProvider, err := registry.Build(config, LocalOptions{GenerationStateDirectory: state}) if err != nil { t.Fatal(err) } @@ -144,12 +146,13 @@ func TestMicrosandboxGenerationBindsLeaseIdentity(t *testing.T) { } func TestMicrosandboxGenerationRejectsUnpinnedImage(t *testing.T) { + registry := Builtin() for _, image := range []string{"latest", "oac-runtime@sha256:bad", "oac-runtime@sha256:"} { t.Run(image, func(t *testing.T) { config := generationConfig(t) config.Microsandbox.Image = image config.Specification.Runtime.MicrosandboxRef = image - built, closeProvider, err := Build(config, LocalOptions{GenerationStateDirectory: t.TempDir()}) + built, closeProvider, err := registry.Build(config, LocalOptions{GenerationStateDirectory: t.TempDir()}) closeProvider() if err == nil || built != nil { t.Fatalf("accepted image %q", image) @@ -159,6 +162,7 @@ func TestMicrosandboxGenerationRejectsUnpinnedImage(t *testing.T) { } func TestLocalConstructionRequiresExplicitContext(t *testing.T) { + registry := Builtin() state := t.TempDir() for _, options := range []LocalOptions{ {}, @@ -166,13 +170,13 @@ func TestLocalConstructionRequiresExplicitContext(t *testing.T) { {GenerationStateDirectory: "relative"}, {GenerationStateDirectory: state + "/../node"}, } { - built, closeProvider, err := Build(generationConfig(t), options) + built, closeProvider, err := registry.Build(generationConfig(t), options) closeProvider() if !errors.Is(err, sandbox.ErrInvalid) || built != nil { t.Fatalf("accepted construction context %+v: %v", options, err) } } - built, closeProvider, err := Build(generationConfig(t), LocalOptions{Standalone: true}) + built, closeProvider, err := registry.Build(generationConfig(t), LocalOptions{Standalone: true}) closeProvider() if err != nil || built == nil { t.Fatalf("standalone construction: %v", err) @@ -180,6 +184,7 @@ func TestLocalConstructionRequiresExplicitContext(t *testing.T) { } func TestMicrosandboxConstructionSelectsGenerationReadiness(t *testing.T) { + registry := Builtin() if runtime.GOOS != "linux" { t.Skip("microsandbox requires Linux") } @@ -200,7 +205,7 @@ func TestMicrosandboxConstructionSelectsGenerationReadiness(t *testing.T) { t.Fatal(err) } for _, options := range []LocalOptions{{Standalone: true}, {GenerationStateDirectory: t.TempDir()}} { - built, closeProvider, err := Build(config, options) + built, closeProvider, err := registry.Build(config, options) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/sandbox/providers/operations_test.go b/services/core/internal/sandbox/providers/operations_test.go index b02ef060f..9e2659573 100644 --- a/services/core/internal/sandbox/providers/operations_test.go +++ b/services/core/internal/sandbox/providers/operations_test.go @@ -8,14 +8,15 @@ import ( ) func TestRegistrationRejectsMissingAndMismatchedDeclarations(t *testing.T) { + registry := Builtin() for _, adapter := range []Adapter{{}, {Operations: func() providercontract.Operations { return nil }}} { - adapters["invalid-contract-fixture"] = adapter - if _, err := Lookup("invalid-contract-fixture"); err == nil { + registry.adapters["invalid-contract-fixture"] = adapter + if _, err := registry.Lookup("invalid-contract-fixture"); err == nil { t.Fatal("invalid declaration registered") } } - delete(adapters, "invalid-contract-fixture") - if err := ValidateBinding(adapters["e2b"], &docker.Provider{}); !errors.Is(err, providercontract.ErrContract) { + delete(registry.adapters, "invalid-contract-fixture") + if err := ValidateBinding(registry.adapters["e2b"], &docker.Provider{}); !errors.Is(err, providercontract.ErrContract) { t.Fatal("registration differs from instance", err) } } diff --git a/services/core/internal/sandbox/providers/registration_configuration_test.go b/services/core/internal/sandbox/providers/registration_configuration_test.go index b48300e64..7fdcda590 100644 --- a/services/core/internal/sandbox/providers/registration_configuration_test.go +++ b/services/core/internal/sandbox/providers/registration_configuration_test.go @@ -26,11 +26,12 @@ func (a registrationConfiguration) Requirements() sandbox.ConfigurationRequireme type missingConfigurationDiscovery struct{ sandbox.ConfigurationAdapter } func TestConfigurationRegistrationRejectsNilAndMissingDiscovery(t *testing.T) { - a := adapters["docker"] + registry := Builtin() + a := registry.adapters["docker"] var typedNil *registrationConfiguration for _, configuration := range []sandbox.ConfigurationAdapter{ nil, typedNil, missingConfigurationDiscovery{a.Configuration}, - missingConfigurationDiscovery{adapters["e2b"].Configuration}, + missingConfigurationDiscovery{registry.adapters["e2b"].Configuration}, } { a.Configuration = configuration if err := ValidateRegistration(a); !errors.Is(err, providercontract.ErrContract) { @@ -40,7 +41,8 @@ func TestConfigurationRegistrationRejectsNilAndMissingDiscovery(t *testing.T) { } func TestConfigurationRequirementsRejectEachOmission(t *testing.T) { - a := adapters["docker"] + registry := Builtin() + a := registry.adapters["docker"] original := a.Configuration.Requirements() typ := reflect.TypeOf(original) for i := 0; i < typ.NumField(); i++ { @@ -56,6 +58,7 @@ func TestConfigurationRequirementsRejectEachOmission(t *testing.T) { } func TestConfigurationRequirementsRejectInvalidDeclarations(t *testing.T) { + registry := Builtin() for _, change := range []func(*sandbox.ConfigurationRequirements){ func(r *sandbox.ConfigurationRequirements) { r.Credential = "automatic" }, func(r *sandbox.ConfigurationRequirements) { r.PublicOrigin = "private" }, @@ -64,7 +67,7 @@ func TestConfigurationRequirementsRejectInvalidDeclarations(t *testing.T) { func(r *sandbox.ConfigurationRequirements) { r.Discovery.Reason = "https://private:key@host" }, func(r *sandbox.ConfigurationRequirements) { r.Discovery.State = providercontract.Supported }, } { - a := adapters["docker"] + a := registry.adapters["docker"] requirements := a.Configuration.Requirements() change(&requirements) a.Configuration = registrationConfiguration{requirements: requirements} @@ -75,25 +78,25 @@ func TestConfigurationRequirementsRejectInvalidDeclarations(t *testing.T) { } func TestConfigurationRequirementsDoNotInventDependencies(t *testing.T) { + registry := Builtin() const kind = "configuration-requirement-test" - defer delete(adapters, kind) // Required credentials are input policy. VerifyCredential is a separate // resource operation that may be Unsupported for this provider. for _, credential := range []sandbox.Requirement{sandbox.Required, sandbox.NotRequired} { for _, public := range []sandbox.Requirement{sandbox.Required, sandbox.NotRequired} { - a := adapters["docker"] + a := registry.adapters["docker"] requirements := a.Configuration.Requirements() requirements.Credential, requirements.PublicOrigin = credential, public a.Configuration = registrationConfiguration{requirements: requirements} if err := ValidateRegistration(a); err != nil { t.Fatal(err) } - adapters[kind] = a - gotCredential, err := UsesCredential(kind) + registry.adapters[kind] = a + gotCredential, err := registry.UsesCredential(kind) if err != nil || gotCredential != (credential == sandbox.Required) { t.Fatalf("credential %s: value=%v error=%v", credential, gotCredential, err) } - gotPublic, err := RequiresPublicOrigin(kind) + gotPublic, err := registry.RequiresPublicOrigin(kind) if err != nil || gotPublic != (public == sandbox.Required) { t.Fatalf("public origin %s: value=%v error=%v", public, gotPublic, err) } @@ -107,7 +110,8 @@ type futureConfigurationDiscovery interface { } func TestFutureConfigurationRequirementAndMethodNeedExplicitHandling(t *testing.T) { - original := adapters["docker"].Configuration.Requirements() + registry := Builtin() + original := registry.adapters["docker"].Configuration.Requirements() fields := make([]reflect.StructField, 0, 4) typ := reflect.TypeOf(original) for i := 0; i < typ.NumField(); i++ { @@ -128,7 +132,8 @@ func TestFutureConfigurationRequirementAndMethodNeedExplicitHandling(t *testing. } func TestUnsupportedConfigurationDiscoveryMatchesAuthoredReason(t *testing.T) { - for kind, a := range adapters { + registry := Builtin() + for kind, a := range registry.adapters { support := a.Configuration.Requirements().Discovery if support.State != providercontract.Unsupported { continue @@ -139,7 +144,7 @@ func TestUnsupportedConfigurationDiscoveryMatchesAuthoredReason(t *testing.T) { return native.DiscoverConfiguration(t.Context(), sandbox.ConfigurationDiscoveryInput{}, sandbox.ProcessPaths{}) }, func() ([]byte, error) { - return DiscoverConfiguration(t.Context(), kind, sandbox.ConfigurationDiscoveryInput{}, sandbox.ProcessPaths{}) + return registry.DiscoverConfiguration(t.Context(), kind, sandbox.ConfigurationDiscoveryInput{}, sandbox.ProcessPaths{}) }, } { result, err := read() diff --git a/services/core/internal/sandbox/providers/registration_test.go b/services/core/internal/sandbox/providers/registration_test.go index 65fc3a355..93eb5304b 100644 --- a/services/core/internal/sandbox/providers/registration_test.go +++ b/services/core/internal/sandbox/providers/registration_test.go @@ -24,8 +24,8 @@ func validRegistrationSpec() sandbox.DeploymentSpec { } func TestRegistrationRejectsBeforeCallbacksOrConstruction(t *testing.T) { + registry := Builtin() const kind = "registration-test-provider" - defer delete(adapters, kind) for _, tc := range []struct { name string mutate func(*Adapter) @@ -64,7 +64,7 @@ func TestRegistrationRejectsBeforeCallbacksOrConstruction(t *testing.T) { {"incomplete operations", func(a *Adapter) { a.Operations = func() providercontract.Operations { return nil } }}, } { t.Run(tc.name, func(t *testing.T) { - a := adapters["docker"] + a := registry.adapters["docker"] a.BuildLocal = func(Config, LocalOptions, *Built) (func(), error) { t.Fatal("called local constructor") return nil, nil @@ -73,37 +73,37 @@ func TestRegistrationRejectsBeforeCallbacksOrConstruction(t *testing.T) { a.ValidateResources = func(sandbox.Resources) error { t.Fatal("called resource validator"); return nil } a.Configuration = registrationConfiguration{requirements: a.Configuration.Requirements()} tc.mutate(&a) - adapters[kind] = a + registry.adapters[kind] = a selection := sandbox.Selection{Provider: kind, DeploymentSpec: validRegistrationSpec()} for _, entry := range []struct { name string call func() error }{ - {"lookup", func() error { _, err := Lookup(kind); return err }}, - {"credential requirement", func() error { _, err := UsesCredential(kind); return err }}, - {"public origin requirement", func() error { _, err := RequiresPublicOrigin(kind); return err }}, - {"normalize", func() error { _, err := Normalize(selection); return err }}, - {"specification", func() error { return ValidateSpecification(kind, selection.DeploymentSpec) }}, - {"resources", func() error { return ValidateResources(kind, selection.Resources) }}, - {"description", func() error { _, err := Describe(kind, uuid.NewString()); return err }}, - {"decode input", func() error { _, err := DecodeInput(kind, nil, nil); return err }}, - {"encode", func() error { _, err := Encode(kind, nil); return err }}, - {"decode", func() error { _, err := Decode(kind, sandbox.ConfigurationRecord{}); return err }}, - {"equal", func() error { _, err := Equal(kind, nil, nil); return err }}, + {"lookup", func() error { _, err := registry.Lookup(kind); return err }}, + {"credential requirement", func() error { _, err := registry.UsesCredential(kind); return err }}, + {"public origin requirement", func() error { _, err := registry.RequiresPublicOrigin(kind); return err }}, + {"normalize", func() error { _, err := registry.Normalize(selection); return err }}, + {"specification", func() error { return registry.ValidateSpecification(kind, selection.DeploymentSpec) }}, + {"resources", func() error { return registry.ValidateResources(kind, selection.Resources) }}, + {"description", func() error { _, err := registry.Describe(kind, uuid.NewString()); return err }}, + {"decode input", func() error { _, err := registry.DecodeInput(kind, nil, nil); return err }}, + {"encode", func() error { _, err := registry.Encode(kind, nil); return err }}, + {"decode", func() error { _, err := registry.Decode(kind, sandbox.ConfigurationRecord{}); return err }}, + {"equal", func() error { _, err := registry.Equal(kind, nil, nil); return err }}, {"discovery", func() error { - _, err := DiscoverConfiguration(t.Context(), kind, sandbox.ConfigurationDiscoveryInput{}, sandbox.ProcessPaths{}) + _, err := registry.DiscoverConfiguration(t.Context(), kind, sandbox.ConfigurationDiscoveryInput{}, sandbox.ProcessPaths{}) return err }}, - {"resolve change", func() error { _, err := ResolveChange(selection, selection); return err }}, - {"credential", func() error { _, err := WithCredential(selection, selection); return err }}, + {"resolve change", func() error { _, err := registry.ResolveChange(selection, selection); return err }}, + {"credential", func() error { _, err := registry.WithCredential(selection, selection); return err }}, {"local build", func() error { - _, _, err := Build(Config{Provider: kind, Generation: 1, InstallationID: uuid.NewString(), Specification: selection.DeploymentSpec}, LocalOptions{Standalone: true}) + _, _, err := registry.Build(Config{Provider: kind, Generation: 1, InstallationID: uuid.NewString(), Specification: selection.DeploymentSpec}, LocalOptions{Standalone: true}) return err }}, - {"direct build", func() error { _, err := BuildDirect(DirectConfig{Selection: selection}); return err }}, + {"direct build", func() error { _, err := registry.BuildDirect(DirectConfig{Selection: selection}); return err }}, {"binding", func() error { return ValidateBinding(a, &docker.Provider{}) }}, {"projection", func() error { - text, err := PythonDeploymentContract() + text, err := registry.PythonDeploymentContract() if text != "" { t.Fatal("partial invalid projection") } @@ -122,16 +122,16 @@ func TestRegistrationRejectsBeforeCallbacksOrConstruction(t *testing.T) { } func TestCompleteRegistrationsPreserveConstruction(t *testing.T) { - for kind, a := range adapters { + registry := Builtin() + for kind, a := range registry.adapters { if err := ValidateRegistration(a); err != nil { t.Fatalf("%s: %v", kind, err) } } const kind = "new-test-provider" - defer delete(adapters, kind) calls, closes := 0, 0 options := LocalOptions{GenerationStateDirectory: t.TempDir()} - a := adapters["docker"] + a := registry.adapters["docker"] a.BuildLocal = func(_ Config, got LocalOptions, built *Built) (func(), error) { calls++ if got != options { @@ -140,8 +140,8 @@ func TestCompleteRegistrationsPreserveConstruction(t *testing.T) { built.Provider = &docker.Provider{} return func() { closes++ }, nil } - adapters[kind] = a - built, closeProvider, err := Build(Config{Provider: kind, Generation: 1, InstallationID: uuid.NewString(), Specification: validRegistrationSpec()}, options) + registry.adapters[kind] = a + built, closeProvider, err := registry.Build(Config{Provider: kind, Generation: 1, InstallationID: uuid.NewString(), Specification: validRegistrationSpec()}, options) if err != nil || built.Provider == nil || calls != 1 { t.Fatalf("node build: %v calls=%d", err, calls) } @@ -156,12 +156,12 @@ func TestCompleteRegistrationsPreserveConstruction(t *testing.T) { calls++ return &docker.Provider{}, nil } - adapters[kind] = a - p, err := BuildDirect(DirectConfig{Selection: sandbox.Selection{Provider: kind}}) + registry.adapters[kind] = a + p, err := registry.BuildDirect(DirectConfig{Selection: sandbox.Selection{Provider: kind}}) if err != nil || p == nil || calls != 2 { t.Fatalf("credential-free direct build: %v calls=%d", err, calls) } - if _, err := PythonDeploymentContract(); err != nil { + if _, err := registry.PythonDeploymentContract(); err != nil { t.Fatal(err) } } @@ -169,6 +169,7 @@ func TestCompleteRegistrationsPreserveConstruction(t *testing.T) { // Idle time is measured before suspension, retention after suspension. Neither // duration needs to be greater than the other. func TestRegistrationCheckpointPolicy(t *testing.T) { + registry := Builtin() for _, tc := range []struct { name string kind string @@ -185,10 +186,10 @@ func TestRegistrationCheckpointPolicy(t *testing.T) { {"no suspension", "docker", 0, 0, false, true}, } { t.Run(tc.name, func(t *testing.T) { - a := adapters[tc.kind] + a := registry.adapters[tc.kind] a.IdleSeconds, a.RetentionSeconds = tc.idle, tc.retention if tc.direct { - a.Mode, a.BuildLocal, a.BuildDirect = "direct", nil, adapters["e2b"].BuildDirect + a.Mode, a.BuildLocal, a.BuildDirect = "direct", nil, registry.adapters["e2b"].BuildDirect } err := ValidateRegistration(a) if (err == nil) != tc.valid || err != nil && !errors.Is(err, providercontract.ErrContract) { diff --git a/services/core/internal/sandbox/providers/registry.go b/services/core/internal/sandbox/providers/registry.go index f579441f7..5d60ab7fc 100644 --- a/services/core/internal/sandbox/providers/registry.go +++ b/services/core/internal/sandbox/providers/registry.go @@ -3,8 +3,6 @@ package providers import ( - "crypto/sha256" - "encoding/hex" "fmt" "github.com/MiniMax-AI/OpenAgentCore/internal/providerassets" @@ -31,96 +29,117 @@ type Adapter struct { ValidateResources func(sandbox.Resources) error } -var adapters = map[string]Adapter{ - "docker": { - NodeArtifacts: []providerassets.Artifact{nodeProgram, runtimeImage, runtimePolicy}, - Policy: docker.Policy(), Operations: docker.Operations, Mode: "nodes", BuildLocal: buildDocker, - ValidateSpecification: docker.ValidateSpecification, ValidateResources: docker.ValidateResources, - Configuration: nodeConfigurationAdapter{docker.ValidateSpecification}, - }, - "microsandbox": { - NodeArtifacts: []providerassets.Artifact{nodeProgram, runtimeImage, runtimePolicy, - {Path: "native/bin/oac-microsandbox-provider", Suffix: "microsandbox-provider", Role: "runtime"}, - {Path: "native/microsandbox/msb", Suffix: "msb", Role: "runtime"}, - {Path: "native/microsandbox/libkrunfw.so.5.6.1", Suffix: "libkrunfw.so.5.6.1", Role: "runtime"}}, - Policy: microsandbox.Policy(), Operations: microsandbox.Operations, Mode: "nodes", BuildLocal: buildMicrosandbox, - IdleSeconds: 300, RetentionSeconds: 86400, - ValidateSpecification: microsandbox.ValidateSpecification, ValidateResources: microsandbox.ValidateResources, - Configuration: nodeConfigurationAdapter{microsandbox.ValidateSpecification}, - }, - "e2b": { - Policy: e2b.Policy(), Operations: e2b.Operations, Mode: "direct", BuildDirect: buildE2B, - Configuration: e2b.ConfigurationAdapter{}, - ValidateSpecification: e2b.ValidateSpecification, ValidateResources: e2b.ValidateResources, - }, +// Registry holds the registered adapters. Core and the node program each build +// one with Builtin and pass it explicitly; there is no package-level lookup. +type Registry struct{ adapters map[string]Adapter } + +// ErrUnknownProvider reports a provider kind that no adapter registers. +var ErrUnknownProvider = fmt.Errorf("%w: unsupported sandbox provider", sandbox.ErrInvalid) + +// Builtin returns a registry of the adapters built into this program. +func Builtin() *Registry { + return &Registry{adapters: map[string]Adapter{ + "docker": { + NodeArtifacts: []providerassets.Artifact{nodeProgram, runtimeImage, runtimePolicy}, + Policy: docker.Policy(), Operations: docker.Operations, Mode: "nodes", BuildLocal: buildDocker, + ValidateSpecification: docker.ValidateSpecification, ValidateResources: docker.ValidateResources, + Configuration: nodeConfigurationAdapter{docker.ValidateSpecification}, + }, + "microsandbox": { + NodeArtifacts: []providerassets.Artifact{nodeProgram, runtimeImage, runtimePolicy, + {Path: "native/bin/oac-microsandbox-provider", Suffix: "microsandbox-provider", Role: "runtime"}, + {Path: "native/microsandbox/msb", Suffix: "msb", Role: "runtime"}, + {Path: "native/microsandbox/libkrunfw.so.5.6.1", Suffix: "libkrunfw.so.5.6.1", Role: "runtime"}}, + Policy: microsandbox.Policy(), Operations: microsandbox.Operations, Mode: "nodes", BuildLocal: buildMicrosandbox, + IdleSeconds: 300, RetentionSeconds: 86400, + ValidateSpecification: microsandbox.ValidateSpecification, ValidateResources: microsandbox.ValidateResources, + Configuration: nodeConfigurationAdapter{microsandbox.ValidateSpecification}, + }, + "e2b": { + Policy: e2b.Policy(), Operations: e2b.Operations, Mode: "direct", BuildDirect: buildE2B, + Configuration: e2b.ConfigurationAdapter{}, + ValidateSpecification: e2b.ValidateSpecification, ValidateResources: e2b.ValidateResources, + }, + }} } -func Lookup(kind string) (Adapter, error) { - a, ok := adapters[kind] +// Lookup returns a registered adapter after validating its registration. +func (r *Registry) Lookup(kind string) (Adapter, error) { + a, ok := r.adapters[kind] if !ok { - return Adapter{}, fmt.Errorf("%w: unsupported sandbox provider", sandbox.ErrInvalid) + return Adapter{}, ErrUnknownProvider } if err := ValidateRegistration(a); err != nil { return Adapter{}, err } return a, nil } -func IsNode(kind string) bool { a, e := Lookup(kind); return e == nil && a.Mode == "nodes" } -func SupportsCheckpoint(kind string) bool { - a, e := Lookup(kind) - return e == nil && a.Operations()["Initial"].State == providercontract.Supported + +// IsNode reports whether the provider runs on enrolled sandbox nodes. +func (r *Registry) IsNode(kind string) (bool, error) { + a, err := r.Lookup(kind) + if err != nil { + return false, err + } + return a.Mode == "nodes", nil +} + +// SupportsCheckpoint reports whether the provider declares checkpoint suspension. +func (r *Registry) SupportsCheckpoint(kind string) (bool, error) { + a, err := r.Lookup(kind) + if err != nil { + return false, err + } + return a.Operations()["Initial"].State == providercontract.Supported, nil } // RetainedLimit keeps nodes without checkpoint support within their active capacity. -func RetainedLimit(kind string, active, retained int) int { - a, err := Lookup(kind) - if err == nil && a.Mode == "nodes" && !SupportsCheckpoint(kind) { - return active +func (r *Registry) RetainedLimit(kind string, active, retained int) (int, error) { + a, err := r.Lookup(kind) + if err != nil { + return 0, err + } + if a.Mode == "nodes" && a.Operations()["Initial"].State != providercontract.Supported { + return active, nil } - return retained + return retained, nil } -func ValidateSpecification(kind string, s sandbox.DeploymentSpec) error { - a, e := Lookup(kind) +func (r *Registry) ValidateSpecification(kind string, s sandbox.DeploymentSpec) error { + a, e := r.Lookup(kind) if e != nil { return e } return a.ValidateSpecification(s) } -func ValidateResources(kind string, s sandbox.Resources) error { - a, e := Lookup(kind) +func (r *Registry) ValidateResources(kind string, s sandbox.Resources) error { + a, e := r.Lookup(kind) if e != nil { return e } return a.ValidateResources(s) } -// Description is derived once for both preview and persistence. Its fingerprint -// identifies a namespace, never mutable capacity or a credential. -type Description struct { - Mode, BackendFingerprint string - IdleSeconds, RetentionSeconds int64 -} - -func Describe(kind, installation string) (Description, error) { - a, e := Lookup(kind) +// Describe derives the description once for both preview and persistence. +// Its fingerprint identifies a namespace, never mutable capacity or a credential. +func (r *Registry) Describe(kind, installation string) (sandbox.Description, error) { + a, e := r.Lookup(kind) if e != nil { - return Description{}, e + return sandbox.Description{}, e } namespace := a.Mode if a.Mode == "direct" { namespace = kind } - digest := sha256.Sum256([]byte(kind + "\x00" + namespace + ":" + installation)) - return Description{a.Mode, hex.EncodeToString(digest[:]), a.IdleSeconds, a.RetentionSeconds}, nil + return sandbox.Description{Mode: a.Mode, BackendFingerprint: BackendFingerprint(kind, namespace+":"+installation), IdleSeconds: a.IdleSeconds, RetentionSeconds: a.RetentionSeconds}, nil } // PythonDeploymentContract projects the same registered adapter policies into // the node installer; no second provider list exists in another language. -func PythonDeploymentContract() (string, error) { - policies := make(map[string]sandbox.DeploymentPolicy, len(adapters)) - for kind := range adapters { - a, err := Lookup(kind) +func (r *Registry) PythonDeploymentContract() (string, error) { + policies := make(map[string]sandbox.DeploymentPolicy, len(r.adapters)) + for kind := range r.adapters { + a, err := r.Lookup(kind) if err != nil { return "", err } diff --git a/services/core/internal/sandbox/providers/registry_test.go b/services/core/internal/sandbox/providers/registry_test.go index 3d3639b13..97c1539df 100644 --- a/services/core/internal/sandbox/providers/registry_test.go +++ b/services/core/internal/sandbox/providers/registry_test.go @@ -2,6 +2,7 @@ package providers import ( "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" "github.com/google/uuid" @@ -9,6 +10,7 @@ import ( ) func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { + registry := Builtin() installation := uuid.NewString() for _, tc := range []struct { kind, mode, namespace string @@ -20,24 +22,27 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { {"e2b", "direct", "e2b", 0, 0, false}, } { t.Run(tc.kind, func(t *testing.T) { - d, err := Describe(tc.kind, installation) + d, err := registry.Describe(tc.kind, installation) if err != nil || d.Mode != tc.mode || d.IdleSeconds != tc.idle || d.RetentionSeconds != tc.retention || d.BackendFingerprint != BackendFingerprint(tc.kind, tc.namespace+":"+installation) { t.Fatalf("wrong namespace or defaults: %+v %v", d, err) } - a, err := Lookup(tc.kind) - if err != nil || SupportsCheckpoint(tc.kind) != tc.checkpoint || IsNode(tc.kind) != (tc.mode == "nodes") || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { - t.Fatal("inconsistent construction/capability registration", err) + a, err := registry.Lookup(tc.kind) + checkpoint, checkpointErr := registry.SupportsCheckpoint(tc.kind) + isNode, nodeErr := registry.IsNode(tc.kind) + if err != nil || checkpointErr != nil || nodeErr != nil || checkpoint != tc.checkpoint || isNode != (tc.mode == "nodes") || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { + t.Fatal("inconsistent construction/capability registration", err, checkpointErr, nodeErr) } }) } - if _, err := Describe("unregistered", installation); !errors.Is(err, sandbox.ErrInvalid) { + if _, err := registry.Describe("unregistered", installation); !errors.Is(err, ErrUnknownProvider) || !errors.Is(err, sandbox.ErrInvalid) { t.Fatal("unknown kind accepted", err) } } func TestSelectionNormalizationAndCredentialInheritance(t *testing.T) { + registry := Builtin() input := sandbox.Selection{Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "original-key", Template: "runtime:" + uuid.NewString()}} - normalized, err := Normalize(input) + normalized, err := registry.Normalize(input) if err != nil || normalized.Configuration.(*e2b.DeploymentConfiguration).APIURL != "https://api.e2b.app" || normalized.Configuration.(*e2b.DeploymentConfiguration).Domain != "e2b.app" { t.Fatal("defaults not normalized", err) } @@ -47,12 +52,12 @@ func TestSelectionNormalizationAndCredentialInheritance(t *testing.T) { normalized.Resources = sandbox.Resources{CPUs: 4, MemoryMiB: 4096} request := input request.Configuration = &e2b.DeploymentConfiguration{Template: input.Configuration.(*e2b.DeploymentConfiguration).Template} - next, err := ResolveChange(request, normalized) + next, err := registry.ResolveChange(request, normalized) if err != nil || next.Resources != normalized.Resources || next.Configuration.(*e2b.DeploymentConfiguration).APIKey != "original-key" || next.Configuration.(*e2b.DeploymentConfiguration).APIURL != normalized.Configuration.(*e2b.DeploymentConfiguration).APIURL || next.ReplacesCredential() { t.Fatal("omitted values lost committed selection", err) } request.Configuration.(*e2b.DeploymentConfiguration).CredentialSupplied = true - if _, err := ResolveChange(request, normalized); !errors.Is(err, sandbox.ErrInvalid) { + if _, err := registry.ResolveChange(request, normalized); !errors.Is(err, sandbox.ErrInvalid) { t.Fatal("explicit empty credential silently inherited", err) } if request.Configuration.(*e2b.DeploymentConfiguration).APIKey != "" || request.Resources != (sandbox.Resources{}) { @@ -61,40 +66,61 @@ func TestSelectionNormalizationAndCredentialInheritance(t *testing.T) { } func TestNewRegistrationDoesNotNeedCoreDispatchChanges(t *testing.T) { + registry := Builtin() const kind = "contract-test-provider" // Registration is test-local: production registrations are fixed, never plugins. - adapters[kind] = adapters["docker"] - defer delete(adapters, kind) - s, err := Normalize(sandbox.Selection{Provider: kind, DeploymentSpec: validRegistrationSpec()}) - if err != nil || s.Provider != kind || !IsNode(kind) || SupportsCheckpoint(kind) { - t.Fatal("new entry did not follow shared boundary", err) + registry.adapters[kind] = registry.adapters["docker"] + s, err := registry.Normalize(sandbox.Selection{Provider: kind, DeploymentSpec: validRegistrationSpec()}) + isNode, nodeErr := registry.IsNode(kind) + checkpoint, checkpointErr := registry.SupportsCheckpoint(kind) + if err != nil || nodeErr != nil || checkpointErr != nil || s.Provider != kind || !isNode || checkpoint { + t.Fatal("new entry did not follow shared boundary", err, nodeErr, checkpointErr) } - d, err := Describe(kind, uuid.NewString()) + d, err := registry.Describe(kind, uuid.NewString()) if err != nil || d.Mode != "nodes" || d.IdleSeconds != 0 { t.Fatal(d, err) } - if _, err := Normalize(sandbox.Selection{Provider: kind, Configuration: &e2b.DeploymentConfiguration{APIKey: "wrong-provider"}}); !errors.Is(err, sandbox.ErrInvalid) { + if _, err := registry.Normalize(sandbox.Selection{Provider: kind, Configuration: &e2b.DeploymentConfiguration{APIKey: "wrong-provider"}}); !errors.Is(err, sandbox.ErrInvalid) { t.Fatal("mixed configuration admitted", err) } } func TestRetainedLimitUsesRegisteredCapabilities(t *testing.T) { + registry := Builtin() const kind = "capacity-test-provider" - defer delete(adapters, kind) for _, checkpoint := range []bool{false, true} { - adapter := adapters["docker"] + adapter := registry.adapters["docker"] if checkpoint { - adapter = adapters["microsandbox"] + adapter = registry.adapters["microsandbox"] } - adapters[kind] = adapter + registry.adapters[kind] = adapter for _, retained := range []int{0, 20} { want := 10 if checkpoint { want = retained } - if got := RetainedLimit(kind, 10, retained); got != want { - t.Fatalf("checkpoint=%v retained=%d: got %d, want %d", checkpoint, retained, got, want) + if got, err := registry.RetainedLimit(kind, 10, retained); err != nil || got != want { + t.Fatalf("checkpoint=%v retained=%d: got %d, want %d (%v)", checkpoint, retained, got, want, err) } } } } + +// Capability questions report lookup failures instead of answering false. +func TestCapabilityLookupsReportFailures(t *testing.T) { + registry := Builtin() + invalid := registry.adapters["docker"] + invalid.Operations = nil + registry.adapters["invalid-registration"] = invalid + for kind, want := range map[string]error{"unregistered": ErrUnknownProvider, "invalid-registration": providercontract.ErrContract} { + if _, err := registry.IsNode(kind); !errors.Is(err, want) { + t.Fatal(kind, "IsNode", err) + } + if _, err := registry.SupportsCheckpoint(kind); !errors.Is(err, want) { + t.Fatal(kind, "SupportsCheckpoint", err) + } + if _, err := registry.RetainedLimit(kind, 1, 2); !errors.Is(err, want) { + t.Fatal(kind, "RetainedLimit", err) + } + } +} diff --git a/services/core/internal/sandbox/providers/specification.go b/services/core/internal/sandbox/providers/specification.go index f1d887937..72b1540b2 100644 --- a/services/core/internal/sandbox/providers/specification.go +++ b/services/core/internal/sandbox/providers/specification.go @@ -9,8 +9,8 @@ import ( // Local paths belong to the node. Core owns reservation capacity, execution // resources and the immutable deployment release it enrolled with. -func validateSpecification(c Config) error { - adapter, err := Lookup(c.Provider) +func (r *Registry) validateSpecification(c Config) error { + adapter, err := r.Lookup(c.Provider) if err != nil { return err } @@ -20,12 +20,12 @@ func validateSpecification(c Config) error { if c.Generation == 0 { return errors.New("node requires a deployment generation; obtain configuration from Core") } - if err := ValidateSpecification(c.Provider, c.Specification); err != nil { + if err := r.ValidateSpecification(c.Provider, c.Specification); err != nil { return err } - r := c.Specification.Runtime + release := c.Specification.Runtime if d := c.Docker; d != nil { - if d.Image != r.ImageID && d.Image != r.ImageManifestDigest { + if d.Image != release.ImageID && d.Image != release.ImageManifestDigest { return errors.New("Docker Runtime image differs from the deployment release") } } @@ -34,7 +34,7 @@ func validateSpecification(c Config) error { if uint32(m.CPUs) != s.CPUs || m.MemoryMiB != s.MemoryMiB || m.RootDiskMiB != s.RootDiskMiB || m.EnvironmentDiskMiB != s.EnvironmentDiskMiB { return errors.New("microsandbox CPU, memory or disk limits differ from the deployment specification") } - if m.Image != r.MicrosandboxRef || m.RuntimeSHA256 != r.RuntimeSHA256 || m.FirmwareSHA256 != r.FirmwareSHA256 { + if m.Image != release.MicrosandboxRef || m.RuntimeSHA256 != release.RuntimeSHA256 || m.FirmwareSHA256 != release.FirmwareSHA256 { return errors.New("microsandbox Runtime or firmware differs from the deployment release") } } diff --git a/services/core/internal/sandbox/providers/validation_test.go b/services/core/internal/sandbox/providers/validation_test.go index 7fb0eff9c..3fe6eb981 100644 --- a/services/core/internal/sandbox/providers/validation_test.go +++ b/services/core/internal/sandbox/providers/validation_test.go @@ -8,6 +8,7 @@ import ( ) func TestDeploymentValidationFieldsPreserveMessages(t *testing.T) { + registry := Builtin() for _, tc := range []struct { provider string resources sandbox.Resources @@ -21,7 +22,7 @@ func TestDeploymentValidationFieldsPreserveMessages(t *testing.T) { {"docker", sandbox.Resources{CPUs: 1, MemoryMiB: 512, RootDiskMiB: 1}, "resources.root_disk_mib", "docker does not support independent disk capacity limits", validationBound(0), validationBound(0)}, {"e2b", sandbox.Resources{CPUs: 1, MemoryMiB: 512, EnvironmentDiskMiB: 1}, "resources.environment_disk_mib", "e2b does not support independent disk capacity limits", validationBound(0), validationBound(0)}, } { - err := ValidateResources(tc.provider, tc.resources) + err := registry.ValidateResources(tc.provider, tc.resources) var field *sandbox.ValidationError if !errors.As(err, &field) || !errors.Is(err, sandbox.ErrInvalid) || err.Error() != sandbox.ErrInvalid.Error()+": "+tc.message || field.Param != tc.param { t.Fatalf("wrong error: %#v", err) @@ -42,17 +43,17 @@ func TestDeploymentValidationFieldsPreserveMessages(t *testing.T) { {"docker", &sandbox.RuntimeRelease{}, "Runtime must reference one immutable distribution"}, {"e2b", &sandbox.RuntimeRelease{}, "E2B Runtime is selected by its immutable template build"}, } { - err := ValidateSpecification(tc.provider, sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 1, MemoryMiB: 512}, Runtime: tc.runtime}) + err := registry.ValidateSpecification(tc.provider, sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 1, MemoryMiB: 512}, Runtime: tc.runtime}) var field *sandbox.ValidationError if !errors.As(err, &field) || field.Param != "runtime" || err.Error() != sandbox.ErrInvalid.Error()+": "+tc.message || !errors.Is(err, sandbox.ErrInvalid) { t.Fatal(err) } } - if err := ValidateResources("docker", sandbox.Resources{CPUs: 255, MemoryMiB: 1048576}); err != nil { + if err := registry.ValidateResources("docker", sandbox.Resources{CPUs: 255, MemoryMiB: 1048576}); err != nil { t.Fatal(err) } var field *sandbox.ValidationError - err := ValidateResources("private-provider", sandbox.Resources{CPUs: 1, MemoryMiB: 512}) + err := registry.ValidateResources("private-provider", sandbox.Resources{CPUs: 1, MemoryMiB: 512}) if errors.As(err, &field) || err.Error() != sandbox.ErrInvalid.Error()+": unsupported sandbox provider" { t.Fatal("unknown provider reclassified", err) } diff --git a/services/core/internal/store/admin_session_archive.go b/services/core/internal/store/admin_session_archive.go index 08695dceb..7ff87f2a9 100644 --- a/services/core/internal/store/admin_session_archive.go +++ b/services/core/internal/store/admin_session_archive.go @@ -5,6 +5,8 @@ import ( "errors" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" @@ -48,18 +50,18 @@ func (s *Store) archiveManagedSession(ctx context.Context, tenantID, sessionID s var result ManagedSessionArchive err = s.withPublicSession(ctx, tenantID, sessionID, func(ctx context.Context, q *sqlc.Queries, session pgtype.UUID) error { // Session precedes deployment, matching Turn, allocation and input admission. - deployment, err := q.LockRuntimeDeployment(ctx) + current, err := q.LockRuntimeDeployment(ctx) if err != nil { return err } - if uint64(deployment.Generation) != expectedGeneration { - return &SandboxGenerationStaleError{uint64(deployment.Generation)} + if uint64(current.Generation) != expectedGeneration { + return &deployment.GenerationStaleError{CurrentGeneration: uint64(current.Generation)} } - if !deployment.WebManaged || !deployment.InstallationID.Valid { - return ErrSandboxDeploymentConflict + if !current.WebManaged || !current.InstallationID.Valid { + return deployment.ErrConflict } - if deployment.ProviderKind == "" { - return ErrSandboxNotConfigured + if current.ProviderKind == "" { + return deployment.ErrNotConfigured } environment, err := q.GetSessionEnvironment(ctx, sqlc.GetSessionEnvironmentParams{TenantID: tenant, ID: session}) if errors.Is(err, pgx.ErrNoRows) { @@ -73,10 +75,10 @@ func (s *Store) archiveManagedSession(ctx context.Context, tenantID, sessionID s return ErrInvalidInput } if resetRequestedAt != nil { - if !deployment.ResetClear.Valid || !deployment.ResetRequestedAt.Time.Equal(*resetRequestedAt) { - return ErrSandboxDeploymentConflict + if !current.ResetClear.Valid || !current.ResetRequestedAt.Time.Equal(*resetRequestedAt) { + return deployment.ErrConflict } - if deployment.ResetClear.String == "auto" { + if current.ResetClear.String == "auto" { busy, err := q.SessionBlocksAutoReset(ctx, session) if err != nil { return err @@ -89,7 +91,7 @@ func (s *Store) archiveManagedSession(ctx context.Context, tenantID, sessionID s result, err = getManagedSessionArchive(ctx, q, tenant, session) return err } - source, err := sandboxResetAudit(deployment) + source, err := sandboxResetAudit(current) if err != nil { return err } @@ -105,8 +107,8 @@ func (s *Store) archiveManagedSession(ctx context.Context, tenantID, sessionID s if err != nil && !errors.Is(err, pgx.ErrNoRows) { return err } - if allocated && allocation.RuntimeAllocation.State != "released" && allocation.RuntimeAllocation.ProviderKey != deployment.InstallationID { - return ErrSandboxDeploymentConflict + if allocated && allocation.RuntimeAllocation.State != "released" && allocation.RuntimeAllocation.ProviderKey != current.InstallationID { + return deployment.ErrConflict } if err := withEnvironmentInputActivity(ctx, q, session, func() error { if environment.Environment.Status != "failed" && environment.Environment.Status != "expired" { diff --git a/services/core/internal/store/admin_session_archive_race_test.go b/services/core/internal/store/admin_session_archive_race_test.go index c699217b3..029f57bc4 100644 --- a/services/core/internal/store/admin_session_archive_race_test.go +++ b/services/core/internal/store/admin_session_archive_race_test.go @@ -8,24 +8,29 @@ import ( "testing" "github.com/google/uuid" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) func TestManagedSessionArchiveReleasesPendingNodePlacement(t *testing.T) { s, _ := newManagedTestStore(t) w := executionWriter(t, s) installation := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + changes := deploymentExecution(t, w) + if err := changes.Claim(t.Context(), installation); err != nil { t.Fatal(err) } - if _, err := w.InitializeSandboxDeployment(t.Context(), installation, SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { + if _, err := changes.Initialize(t.Context(), installation, sandbox.Selection{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } - token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 1, MaxRetained: 1})) + nodes := deploymentService(t, s) + token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 1})) if err != nil { t.Fatal(err) } nodeID := uuid.NewString() - if _, err := s.EnrollRuntimeNode(t.Context(), token, RuntimeNodeEnrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: nodeID, Name: "Archive fixture", Provider: "docker", Credential: strings.Repeat("x", 64), BackendFingerprint: strings.Repeat("b", 64)}); err != nil { + if _, err := nodes.Enroll(t.Context(), token, deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: nodeID, Name: "Archive fixture", Provider: "docker", Credential: strings.Repeat("x", 64), BackendFingerprint: strings.Repeat("b", 64), CoreURL: s.publicURL}); err != nil { t.Fatal(err) } onlineManagerNode(t, s, nodeID) @@ -34,11 +39,11 @@ func TestManagedSessionArchiveReleasesPendingNodePlacement(t *testing.T) { if err != nil || result.State != "released" { t.Fatal(result, err) } - nodes, err := s.ListRuntimeNodes(t.Context()) - if err != nil || len(nodes) != 1 || nodes[0].Active != 0 || nodes[0].Retained != 0 || nodes[0].Reserved != 0 { - t.Fatal("unallocated archive retained placement capacity", nodes, err) + listed, err := nodes.ListNodes(t.Context()) + if err != nil || len(listed) != 1 || listed[0].Active != 0 || listed[0].Retained != 0 || listed[0].Reserved != 0 { + t.Fatal("unallocated archive retained placement capacity", listed, err) } - if err := s.RemoveRuntimeNode(t.Context(), nodeID); err != nil { + if err := nodes.RemoveNode(t.Context(), nodeID); err != nil { t.Fatal("released placement prevented node removal", err) } } @@ -81,7 +86,7 @@ func TestManagedSessionArchiveOrdersConcurrentInput(t *testing.T) { func TestManagedSessionArchiveRejectsFileManagedDeployment(t *testing.T) { s, w, _ := managerFixture(t, 1, 1) tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) - if _, err := w.ArchiveManagedSession(adminDeleteContext(t.Context(), tenant, uuid.NewString()), tenant, session.ID, 0); !errors.Is(err, ErrSandboxDeploymentConflict) { + if _, err := w.ArchiveManagedSession(adminDeleteContext(t.Context(), tenant, uuid.NewString()), tenant, session.ID, 0); !errors.Is(err, deployment.ErrConflict) { t.Fatal("archive accepted file-managed deployment", err) } } diff --git a/services/core/internal/store/admin_session_archive_test.go b/services/core/internal/store/admin_session_archive_test.go index 10740942c..61ad06459 100644 --- a/services/core/internal/store/admin_session_archive_test.go +++ b/services/core/internal/store/admin_session_archive_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/files" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -25,10 +26,11 @@ func managedArchiveFixture(t *testing.T) (*Store, *Store, string) { s := NewWithCredentialCipher(pool, cipher) w := executionWriter(t, s) installation := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + changes := deploymentExecution(t, w) + if err := changes.Claim(t.Context(), installation); err != nil { t.Fatal(err) } - if _, err := w.InitializeSandboxDeployment(t.Context(), installation, e2bSelection()); err != nil { + if _, err := changes.Initialize(t.Context(), installation, e2bSelection()); err != nil { t.Fatal(err) } return s, w, installation @@ -70,7 +72,7 @@ func TestManagedSessionArchiveUnallocatedAndGuards(t *testing.T) { t.Fatal("unallocated Session status", active, err) } for _, generation := range []uint64{0, 2, ^uint64(0)} { - if _, err := w.ArchiveManagedSession(ctx, tenant, session.ID, generation); !errors.Is(err, ErrSandboxDeploymentConflict) { + if _, err := w.ArchiveManagedSession(ctx, tenant, session.ID, generation); !errors.Is(err, deployment.ErrConflict) { t.Fatal("archive accepted wrong generation", generation, err) } } @@ -117,7 +119,7 @@ func TestManagedSessionArchiveUnallocatedAndGuards(t *testing.T) { if _, err := s.ReserveEnvironmentInput(t.Context(), tenant, session.ID, "later", []Input{{Kind: "message", Payload: json.RawMessage(`{"text":"later"}`)}}); !errors.Is(err, ErrEnvironmentUnavailable) { t.Fatal("archived Environment accepted new input", err) } - view, err := s.GetRuntimeDeployment(t.Context()) + view, err := deploymentService(t, s).View(t.Context()) if err != nil || view.Resources.Pending != 0 || view.Resources.Allocations != 0 { t.Fatal("unallocated archive still blocks switching", view, err) } diff --git a/services/core/internal/store/admin_session_archive_worker_http_test.go b/services/core/internal/store/admin_session_archive_worker_http_test.go index 19f24d208..788c2aebc 100644 --- a/services/core/internal/store/admin_session_archive_worker_http_test.go +++ b/services/core/internal/store/admin_session_archive_worker_http_test.go @@ -17,6 +17,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" @@ -37,16 +38,17 @@ func TestAdminSessionArchiveWorkerHTTPPostgres(t *testing.T) { s, db := store.NewWithCredentialCipher(pool, cipher), fixtureDB{pool: pool, cipher: cipher} installation := uuid.NewString() provider := &lifecycleProvider{resources: map[string]sandbox.Info{}} - providerConfig := func(setup store.SandboxSetup) *execution.RuntimeProvider { + deployments := fixtureDeployment(t, db) + providerConfig := func(setup deployment.Setup) *execution.RuntimeProvider { return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} } configuration := execution.NewDeferredRuntimeProvider(installation, func(ctx context.Context) (*execution.RuntimeProvider, error) { - setup, err := s.GetSandboxSetup(ctx) + setup, err := deployments.Setup(ctx) if err != nil || setup.Provider == "" { return nil, err } return providerConfig(setup), nil - }, func(_ context.Context, setup store.SandboxSetup) (execution.PreparedRuntimeDeployment, error) { + }, func(_ context.Context, setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { return execution.PreparedRuntimeDeployment{Config: providerConfig(setup)}, nil }) worker := startWorker(t, t.Context(), db, &execution.Dispatcher{Store: s, Registry: runtimegateway.NewRegistry(), ManagedRuntimes: configuration}) @@ -59,7 +61,7 @@ func TestAdminSessionArchiveWorkerHTTPPostgres(t *testing.T) { }) } t.Cleanup(stop) - selection := store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("e2b"), Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-api-key", Template: "runtime:" + uuid.NewString()}} + selection := sandbox.Selection{DeploymentSpec: store.SandboxDeploymentTestSpec("e2b"), Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-api-key", Template: "runtime:" + uuid.NewString()}} if _, err := worker.InitializeSandboxDeployment(t.Context(), selection); err != nil { t.Fatal(err) } diff --git a/services/core/internal/store/admin_validation_test.go b/services/core/internal/store/admin_validation_test.go deleted file mode 100644 index e5dc0399c..000000000 --- a/services/core/internal/store/admin_validation_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package store - -import ( - "errors" - "strings" - "testing" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" -) - -func TestRuntimeNodeValidationIdentityAndPrecedence(t *testing.T) { - for _, tc := range []struct { - name string - active, retained int - param string - }{ - {"", 0, 0, "name"}, {strings.Repeat("界", 43), 1, 1, "name"}, {"private\nname", 1, 1, "name"}, - {"node", 0, 0, "max_active"}, {"node", 1000001, 1000001, "max_active"}, {"node", 1000001, 8, "max_active"}, {"node", 2, 1, "max_retained"}, {"node", 1, 1000001, "max_retained"}, - } { - err := validateRuntimeNode(tc.name, tc.active, tc.retained) - var field *AdminValidationError - if !errors.As(err, &field) || !errors.Is(err, ErrInvalidInput) || field.Param != tc.param || err.Error() != ErrInvalidInput.Error() { - t.Fatalf("wrong capacity error: %#v", err) - } - } - if err := validateRuntimeNode("node", 1000000, 1000000); err != nil { - t.Fatal("inclusive active capacity bound changed", err) - } - // Node validation retains its historical byte bound and limited controls; - // it must not silently adopt the stricter Project/key name validator. - for _, name := range []string{strings.Repeat("a", 128), "node\tname", string([]byte{0xff})} { - if err := validateRuntimeNode(name, 1, 1000000); err != nil { - t.Fatal("node acceptance changed", err) - } - } -} - -func TestSandboxValidationWrapperPreservesClassification(t *testing.T) { - _, err := SandboxSetupForSelection("00000000-0000-4000-8000-000000000001", SandboxDeploymentSetupRequest{Provider: "docker"}) - var configuration *SandboxConfigurationError - if !errors.As(err, &configuration) || !errors.Is(err, ErrInvalidInput) || configuration.Validation == nil || configuration.Validation.Param != "resources.cpus" { - t.Fatalf("missing validation metadata: %#v", err) - } - // The wrapper historically unwraps only ErrInvalidInput. Retain that contract - // while carrying the package-owned validation metadata separately. - if errors.Is(err, sandbox.ErrInvalid) { - t.Fatal("wrapper changed sentinel identity") - } - _, err = SandboxSetupForSelection("00000000-0000-4000-8000-000000000001", SandboxDeploymentSetupRequest{Provider: "e2b", DeploymentSpec: sandbox.DeploymentSpec{Runtime: &sandbox.RuntimeRelease{}}}) - if !errors.As(err, &configuration) || configuration.Validation == nil || configuration.Validation.Param != "runtime" || err.Error() != "invalid sandbox configuration: E2B Runtime is selected by its immutable template build" { - t.Fatal("pending E2B validation order changed", err) - } -} diff --git a/services/core/internal/store/agent_execution_defaults_http_test.go b/services/core/internal/store/agent_execution_defaults_http_test.go index 38d97377d..ab5d19976 100644 --- a/services/core/internal/store/agent_execution_defaults_http_test.go +++ b/services/core/internal/store/agent_execution_defaults_http_test.go @@ -24,7 +24,7 @@ func TestAgentExecutionDefaultsPublicSnapshotAndPrecedence(t *testing.T) { auth := newTestAuthenticator(t, []testAPIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "defaults-test", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}}) deployment := &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://deployment.example/v1", APIKey: "deployment-canary"} defaultsCalls := 0 - handler, err := publicHandler(t, st, db, auth, "codex", storeExecution(t, st), managedSandboxes(t, st), withHarnesses([]string{"codex", "claude_sdk", "mcode"}), modelProviderDefaults(func(context.Context, string) (*modelconfiguration.Snapshot, error) { + handler, err := publicHandler(t, st, db, auth, "codex", storeExecution(t, st), managedSandboxes(t, st, db), withHarnesses([]string{"codex", "claude_sdk", "mcode"}), modelProviderDefaults(func(context.Context, string) (*modelconfiguration.Snapshot, error) { defaultsCalls++ copy := *deployment return &modelconfiguration.Snapshot{Model: "fixture", Provider: ©, Revision: uuid.New()}, nil diff --git a/services/core/internal/store/archive_cancellation_test.go b/services/core/internal/store/archive_cancellation_test.go index 28b8ffc96..f8dcc28f3 100644 --- a/services/core/internal/store/archive_cancellation_test.go +++ b/services/core/internal/store/archive_cancellation_test.go @@ -22,6 +22,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtime" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" @@ -44,10 +45,10 @@ func TestArchiveWaitingCancellationReceipts(t *testing.T) { } }) installation := uuid.NewString() - if err := writer.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + if err := leased.Deployment.Claim(t.Context(), installation); err != nil { t.Fatal(err) } - if _, err := writer.InitializeSandboxDeployment(t.Context(), installation, store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("e2b"), Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture", Template: "runtime:" + uuid.NewString()}}); err != nil { + if _, err := leased.Deployment.Initialize(t.Context(), installation, sandbox.Selection{DeploymentSpec: store.SandboxDeploymentTestSpec("e2b"), Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture", Template: "runtime:" + uuid.NewString()}}); err != nil { t.Fatal(err) } projectID := uuid.NewString() diff --git a/services/core/internal/store/credential_matrix_http_test.go b/services/core/internal/store/credential_matrix_http_test.go index 6af95bf4b..bad59e53e 100644 --- a/services/core/internal/store/credential_matrix_http_test.go +++ b/services/core/internal/store/credential_matrix_http_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/projectpg" @@ -29,13 +30,14 @@ import ( func TestCredentialNamespaceMatrix(t *testing.T) { s, db := newManagedTestStoreDB(t) s.SetPublicURL("https://core.example") + db.publicURL = "https://core.example" ctx := t.Context() coreKey := uuid.NewString() admin, err := api.NewDeploymentAuthenticator([]string{runtimedevice.HashCredential(coreKey)}) if err != nil { t.Fatal(err) } - handler, err := publicHandler(t, s, db, nil, "codex", storeKeys(s), storeExecution(t, s), managedSandboxes(t, s), withCoreKeys(admin)) + handler, err := publicHandler(t, s, db, nil, "codex", storeKeys(s), storeExecution(t, s), managedSandboxes(t, s, db), withCoreKeys(admin)) if err != nil { t.Fatal(err) } @@ -86,15 +88,16 @@ func TestCredentialNamespaceMatrix(t *testing.T) { created("POST", "/core/v1/projects/"+project.ID+"/environments/"+environment.ID+"/executor-credentials", coreKey, `{"key_id":"`+uuid.NewString()+`"}`, &executor) // A node credential: a Docker deployment, an enrollment token issued with the Core key, and an enrolled node. + deployments := fixtureDeployment(t, db) installation := uuid.NewString() provider := &lifecycleProvider{resources: map[string]sandbox.Info{}} runtimes := execution.NewDeferredRuntimeProvider(installation, func(ctx context.Context) (*execution.RuntimeProvider, error) { - setup, err := s.GetSandboxSetup(ctx) + setup, err := deployments.Setup(ctx) if err != nil || setup.Provider == "" { return nil, err } return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, BackendFingerprint: setup.BackendFingerprint, CoreURL: "https://core.example/api/v1", Provider: provider}, nil - }, func(_ context.Context, setup store.SandboxSetup) (execution.PreparedRuntimeDeployment, error) { + }, func(_ context.Context, setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider}}, nil }) worker := startWorker(t, ctx, db, &execution.Dispatcher{Store: s, Registry: runtimegateway.NewRegistry(), ManagedRuntimes: runtimes}) @@ -107,15 +110,15 @@ func TestCredentialNamespaceMatrix(t *testing.T) { }) }) specification := store.SandboxDeploymentTestSpec("docker") - if _, err := worker.InitializeSandboxDeployment(ctx, store.SandboxDeploymentSetupRequest{DeploymentSpec: specification, Provider: "docker"}); err != nil { + if _, err := worker.InitializeSandboxDeployment(ctx, sandbox.Selection{DeploymentSpec: specification, Provider: "docker"}); err != nil { t.Fatal(err) } var enrollment api.SandboxEnrollmentToken created("POST", "/core/v1/sandbox/enrollment-tokens", coreKey, `{}`, &enrollment) nodeID, nodeCredential := uuid.NewString(), strings.Repeat("n", 64) - enroll, _ := json.Marshal(store.RuntimeNodeEnrollment{NodeID: nodeID, Credential: nodeCredential, Name: "Matrix node", Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), + enroll, _ := json.Marshal(deployment.Enrollment{NodeID: nodeID, Credential: nodeCredential, Name: "Matrix node", Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: 1, SpecificationDigest: specification.Digest("docker"), CoreURL: "https://core.example"}) - var node store.RuntimeNodeIdentity + var node deployment.NodeIdentity created("POST", "/api/v1/sandbox-node/enroll", enrollment.Token, string(enroll), &node) // A second, unconsumed enrollment token; its only uses are enroll and configuration. diff --git a/services/core/internal/store/deployment_fixture_test.go b/services/core/internal/store/deployment_fixture_test.go new file mode 100644 index 000000000..41b0d7139 --- /dev/null +++ b/services/core/internal/store/deployment_fixture_test.go @@ -0,0 +1,42 @@ +package store + +import ( + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" +) + +// deploymentStore builds the deployment store as cmd/server does, on s's +// database and credential key. +func deploymentStore(s *Store) *deploymentpg.Store { + return deploymentpg.New(pgunit.NewPool(s.pool), s.credentialCipher) +} + +// deploymentService builds the deployment service as cmd/server does, on s's +// database and credential key, with s's current public URL. +func deploymentService(t testing.TB, s *Store) *deployment.Service { + t.Helper() + adapter := deploymentStore(s) + service, err := deployment.NewService(adapter, adapter, providers.Builtin(), s.publicURL) + if err != nil { + t.Fatal(err) + } + return service +} + +// deploymentExecution builds the deployment execution operations on w's +// execution lease, as cmd/server does for the Worker. +func deploymentExecution(t testing.TB, w *Store) *deployment.ExecutionOperations { + t.Helper() + if w.lease == nil { + t.Fatal("deployment execution operations need an execution writer") + } + operations, err := deployment.NewExecutionOperations(deploymentService(t, w), deploymentpg.NewExecution(w.lease, w.credentialCipher)) + if err != nil { + t.Fatal(err) + } + return operations +} diff --git a/services/core/internal/store/deployment_model_providers_http_test.go b/services/core/internal/store/deployment_model_providers_http_test.go index 6a1807614..ae75ac86d 100644 --- a/services/core/internal/store/deployment_model_providers_http_test.go +++ b/services/core/internal/store/deployment_model_providers_http_test.go @@ -41,7 +41,7 @@ func TestDeploymentModelProvidersHTTP(t *testing.T) { if err != nil { t.Fatal(err) } - handler, err := publicHandler(t, st, db, auth, "codex", storeExecution(t, st), managedSandboxes(t, st), withCoreKeys(admin), withHarnesses([]string{"codex", "mcode"})) + handler, err := publicHandler(t, st, db, auth, "codex", storeExecution(t, st), managedSandboxes(t, st, db), withCoreKeys(admin), withHarnesses([]string{"codex", "mcode"})) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/store/deployment_public_fixture_test.go b/services/core/internal/store/deployment_public_fixture_test.go new file mode 100644 index 000000000..7a041cc26 --- /dev/null +++ b/services/core/internal/store/deployment_public_fixture_test.go @@ -0,0 +1,47 @@ +package store_test + +import ( + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" +) + +// fixtureDeployment builds the deployment service on db, as cmd/server does. +func fixtureDeployment(t testing.TB, db fixtureDB) *deployment.Service { + t.Helper() + service, err := fixtureDeploymentService(db) + if err != nil { + t.Fatal(err) + } + return service +} + +func fixtureDeploymentService(db fixtureDB) (*deployment.Service, error) { + adapter := deploymentpg.New(pgunit.NewPool(db.pool), db.cipher) + return deployment.NewService(adapter, adapter, providers.Builtin(), db.publicURL) +} + +// fixtureOwnerEpoch reads the execution owner epoch from the deployment store, +// as cmd/server does to fence node connections. +func fixtureOwnerEpoch(t testing.TB, db fixtureDB) uint64 { + t.Helper() + epoch, err := deploymentpg.New(pgunit.NewPool(db.pool), db.cipher).OwnerEpoch(t.Context()) + if err != nil { + t.Fatal(err) + } + return epoch +} + +// fixtureDeploymentExecution builds the pooled deployment service and the +// deployment execution operations on lease, as cmd/server does for the Worker. +func fixtureDeploymentExecution(db fixtureDB, lease *pgunit.Lease) (*deployment.Service, *deployment.ExecutionOperations, error) { + service, err := fixtureDeploymentService(db) + if err != nil { + return nil, nil, err + } + changes, err := deployment.NewExecutionOperations(service, deploymentpg.NewExecution(lease, db.cipher)) + return service, changes, err +} diff --git a/services/core/internal/store/device_bootstrap_binding_test.go b/services/core/internal/store/device_bootstrap_binding_test.go index c6b4c5f21..975e9fb6d 100644 --- a/services/core/internal/store/device_bootstrap_binding_test.go +++ b/services/core/internal/store/device_bootstrap_binding_test.go @@ -5,25 +5,27 @@ import ( "strings" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/google/uuid" ) func TestDeviceCredentialCarriesPersistedAllocationNode(t *testing.T) { - s, writer, deployment := managerFixture(t, 4, 8) - token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 4, MaxRetained: 8})) + s, writer, d := managerFixture(t, 4, 8) + nodes := deploymentService(t, s) + token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 4, MaxRetained: 8})) if err != nil { t.Fatal(err) } remote := uuid.NewString() - _, err = s.EnrollRuntimeNode(t.Context(), token, RuntimeNodeEnrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: remote, Credential: strings.Repeat("x", 64), - Name: "remote", Provider: "docker", BackendFingerprint: strings.Repeat("b", 64)}) + _, err = nodes.Enroll(t.Context(), token, deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: remote, Credential: strings.Repeat("x", 64), + Name: "remote", Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), CoreURL: s.publicURL}) if err != nil { t.Fatal(err) } onlineManagerNode(t, s, remote) - for _, nodeID := range []string{deployment.LocalNodeID, remote} { + for _, nodeID := range []string{d.LocalNodeID, remote} { t.Run(nodeID, func(t *testing.T) { tenant, bearer := uuid.NewString(), uuid.NewString() session, err := createSessionOnNode(t, s, tenant, managerSessionInput(uuid.NewString()), nodeID) @@ -34,7 +36,7 @@ func TestDeviceCredentialCarriesPersistedAllocationNode(t *testing.T) { if err != nil { t.Fatal(err) } - allocation, err := writer.ReserveRuntimeAllocation(t.Context(), tenant, environment.ID, deployment.InstallationID, runtimedevice.HashCredential(bearer)) + allocation, err := writer.ReserveRuntimeAllocation(t.Context(), tenant, environment.ID, d.InstallationID, runtimedevice.HashCredential(bearer)) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/store/execution_test.go b/services/core/internal/store/execution_test.go index cf533bd4b..9c1e466ce 100644 --- a/services/core/internal/store/execution_test.go +++ b/services/core/internal/store/execution_test.go @@ -240,7 +240,6 @@ func TestPooledStoreHasNoExecutionAuthority(t *testing.T) { "ownership check": s.checkExecutionOwnership(t.Context()), "archive": archiveErr, "input expiry": expiryErr, - "deployment": s.ConfigureRuntimeDeployment(t.Context(), nil), "reconciliation": s.ReconcileEnvironmentConnections(t.Context()), "mixed batch": s.AppendTurnEvents(t.Context(), tenant, session.ID, input.TurnID, 1, subagent), "subagent only": s.AppendTurnEvents(t.Context(), tenant, session.ID, input.TurnID, 1, subagent[1:]), diff --git a/services/core/internal/store/fixture_db_test.go b/services/core/internal/store/fixture_db_test.go index a60f21d9c..6cecd4e75 100644 --- a/services/core/internal/store/fixture_db_test.go +++ b/services/core/internal/store/fixture_db_test.go @@ -2,6 +2,7 @@ package store_test import ( "context" + "errors" "testing" "github.com/jackc/pgx/v5/pgxpool" @@ -16,8 +17,9 @@ import ( // fixtureDB is the database and credential key that built the test's Store. // Cutovers build their adapters from it; add fields here, never parameters. type fixtureDB struct { - pool *pgxpool.Pool - cipher *credentialcrypto.Cipher // nil for a keyless Store + pool *pgxpool.Pool + cipher *credentialcrypto.Cipher // nil for a keyless Store + publicURL string // the value given to the Store's SetPublicURL, if any } // newTestStoreDB is store.NewTestStore with the fixtureDB that built it. @@ -62,10 +64,19 @@ func startWorkerErr(ctx context.Context, db fixtureDB, dispatcher *execution.Dis if err != nil { return nil, err } + deployments, changes, err := fixtureDeploymentExecution(db, lease) + if err != nil { + return nil, errors.Join(err, lease.Close(ctx)) + } owned := *dispatcher owned.Credentials = credentials owned.Observer = modelconfigurationpg.New(pgunit.NewPool(db.pool), db.cipher) - return execution.StartWorker(ctx, &owned, execution.Owner{Lease: lease, Store: store.NewExecution(dispatcher.Store, lease)}) + owned.Deployment = deployments + return execution.StartWorker(ctx, &owned, execution.Owner{ + Lease: lease, + Store: store.NewExecution(dispatcher.Store, lease), + Deployment: changes, + }) } // executionOwner acquires the execution lease on db and builds s's execution @@ -78,5 +89,13 @@ func executionOwner(t testing.TB, db fixtureDB, s *store.Store) execution.Owner t.Fatal(err) } t.Cleanup(func() { _ = lease.Close(context.Background()) }) - return execution.Owner{Lease: lease, Store: store.NewExecution(s, lease)} + _, changes, err := fixtureDeploymentExecution(db, lease) + if err != nil { + t.Fatal(err) + } + return execution.Owner{ + Lease: lease, + Store: store.NewExecution(s, lease), + Deployment: changes, + } } diff --git a/services/core/internal/store/initial_files_http_test.go b/services/core/internal/store/initial_files_http_test.go index 97cd3ce4b..4759a6ae3 100644 --- a/services/core/internal/store/initial_files_http_test.go +++ b/services/core/internal/store/initial_files_http_test.go @@ -24,7 +24,7 @@ func TestInitialFilesHTTPInlineLimitsAndRetry(t *testing.T) { tenant, token := uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}}) // Exercise HTTP parsing and durable storage without starting a Runtime. - handler, err := publicHandler(t, s, db, auth, "codex", storeExecution(t, s), managedSandboxes(t, s), fixtureDeploymentProvider()) + handler, err := publicHandler(t, s, db, auth, "codex", storeExecution(t, s), managedSandboxes(t, s, db), fixtureDeploymentProvider()) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/store/node_diagnostic_contract_test.go b/services/core/internal/store/node_diagnostic_contract_test.go index 610be6748..b6c059474 100644 --- a/services/core/internal/store/node_diagnostic_contract_test.go +++ b/services/core/internal/store/node_diagnostic_contract_test.go @@ -9,6 +9,8 @@ import ( "testing" "gopkg.in/yaml.v3" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" ) func TestNodeDiagnosticSchemaContract(t *testing.T) { @@ -28,7 +30,7 @@ func TestNodeDiagnosticSchemaContract(t *testing.T) { t.Errorf("%s diagnostic enum = %v, want shared fixture %v", name, values, codes) } } - for _, model := range []reflect.Type{reflect.TypeFor[RuntimeNodeHealth](), reflect.TypeFor[SandboxNodeRollout]()} { + for _, model := range []reflect.Type{reflect.TypeFor[deployment.NodeHealth](), reflect.TypeFor[deployment.NodeRollout]()} { field, ok := model.FieldByName("Diagnostic") if !ok { t.Fatalf("%s has no Diagnostic field", model.Name()) @@ -49,7 +51,7 @@ func TestNodeDiagnosticSchemaContract(t *testing.T) { if err := yaml.Unmarshal(raw, &document); err != nil { t.Fatal(err) } - for _, name := range []string{"store.RuntimeNode", "store.RuntimeNodeDetail", "store.SandboxNodeRollout"} { + for _, name := range []string{"deployment.Node", "deployment.NodeDetail", "deployment.NodeRollout"} { check(name, document.Definitions[name].Properties["diagnostic"].Enum) } } diff --git a/services/core/internal/store/node_host_history.go b/services/core/internal/store/node_host_history.go deleted file mode 100644 index 5802c402c..000000000 --- a/services/core/internal/store/node_host_history.go +++ /dev/null @@ -1,132 +0,0 @@ -package store - -import ( - "context" - "encoding/json" - "math" - "time" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/jackc/pgx/v5/pgtype" -) - -// Host observations are deployment telemetry, not sandbox capacity authority. -type RuntimeNodeHost struct { - EffectiveCPUCores *float64 `json:"effective_cpu_cores" extensions:"x-nullable"` - CPUUtilization *float64 `json:"cpu_utilization" extensions:"x-nullable"` - TotalMemoryBytes *int64 `json:"total_memory_bytes" extensions:"x-nullable"` - AvailableMemoryBytes *int64 `json:"available_memory_bytes" extensions:"x-nullable"` - AvailableDiskBytes *int64 `json:"available_disk_bytes" extensions:"x-nullable"` - ObservedAt *time.Time `json:"observed_at" extensions:"x-nullable"` -} -type runtimeNodeHealthRecord struct { - RuntimeNodeHealth - Host *RuntimeNodeHost `json:"host,omitempty"` -} -type NodeHostHistoryPoint struct { - Start time.Time `json:"start"` - CPUUtilizationMax *float64 `json:"cpu_utilization_max" extensions:"x-nullable"` - MemoryUsedBytesMax *int64 `json:"memory_used_bytes_max" extensions:"x-nullable"` - AvailableDiskBytesMin *int64 `json:"available_disk_bytes_min" extensions:"x-nullable"` -} -type NodeHostHistory struct { - ResolutionSeconds int64 `json:"resolution_seconds"` - Points []NodeHostHistoryPoint `json:"points"` -} -type RuntimeNodeDetail struct { - RuntimeNode - Host RuntimeNodeHost `json:"host"` - History NodeHostHistory `json:"history"` -} - -func validateRuntimeNodeHost(host *RuntimeNodeHost) error { - if host == nil { - return nil - } - if host.ObservedAt == nil || host.ObservedAt.IsZero() || host.ObservedAt.Year() < 1970 || host.ObservedAt.Year() > 9999 { - return ErrInvalidInput - } - for _, value := range []*float64{host.CPUUtilization, host.EffectiveCPUCores} { - if value != nil && (math.IsNaN(*value) || math.IsInf(*value, 0) || *value < 0) { - return ErrInvalidInput - } - } - if host.CPUUtilization != nil && *host.CPUUtilization > 1 || host.EffectiveCPUCores != nil && *host.EffectiveCPUCores == 0 { - return ErrInvalidInput - } - for _, value := range []*int64{host.TotalMemoryBytes, host.AvailableMemoryBytes, host.AvailableDiskBytes} { - if value != nil && (*value < 0 || *value > 1<<53-1) { - return ErrInvalidInput - } - } - if host.TotalMemoryBytes != nil && (*host.TotalMemoryBytes == 0 || host.AvailableMemoryBytes != nil && *host.AvailableMemoryBytes > *host.TotalMemoryBytes) { - return ErrInvalidInput - } - return nil -} - -// SampleNodeHostHistory runs with the existing Runtime sampler cadence. Only a -// fresh authenticated heartbeat is copied; neither reads nor offline nodes fill gaps. -func (s *Store) SampleNodeHostHistory(ctx context.Context) (int64, error) { - return s.queries.SampleNodeHostHistory(ctx) -} - -func (s *Store) GetRuntimeNodeDetail(ctx context.Context, id, name string) (RuntimeNodeDetail, error) { - nodeID, err := parseConnectionGeneration(id) - if err != nil { - return RuntimeNodeDetail{}, ErrInvalidInput - } - if name != "1h" && name != "6h" && name != "24h" { - return RuntimeNodeDetail{}, ErrInvalidInput - } - window, _ := coremetrics.Window(time.Now(), name) - rows, err := s.queries.ListRuntimeNodes(ctx, nodeID) - if err != nil { - return RuntimeNodeDetail{}, err - } - if len(rows) == 0 { - return RuntimeNodeDetail{}, ErrNotFound - } - nodes, err := runtimeNodeViews(rows) - if err != nil { - return RuntimeNodeDetail{}, err - } - var health runtimeNodeHealthRecord - if err := json.Unmarshal(rows[0].Health, &health); err != nil { - return RuntimeNodeDetail{}, err - } - value := RuntimeNodeDetail{RuntimeNode: nodes[0], History: NodeHostHistory{ResolutionSeconds: window.ResolutionSeconds, Points: make([]NodeHostHistoryPoint, 0)}} - if health.Host != nil { - value.Host = *health.Host - } - samples, err := s.queries.ListNodeHostHistory(ctx, sqlc.ListNodeHostHistoryParams{ - NodeID: nodeID, StartAt: pgtype.Timestamptz{Time: window.Start, Valid: true}, EndAt: pgtype.Timestamptz{Time: window.End, Valid: true}, - BucketWidth: pgtype.Interval{Microseconds: window.ResolutionSeconds * 1_000_000, Valid: true}, - }) - if err != nil { - return RuntimeNodeDetail{}, err - } - byStart := make(map[time.Time]NodeHostHistoryPoint, len(samples)) - for _, sample := range samples { - point := NodeHostHistoryPoint{Start: sample.Start.Time.UTC()} - if sample.CpuSamples > 0 { - point.CPUUtilizationMax = &sample.CpuUtilizationMax - } - if sample.MemorySamples > 0 { - point.MemoryUsedBytesMax = &sample.MemoryUsedBytesMax - } - if sample.DiskSamples > 0 { - point.AvailableDiskBytesMin = &sample.AvailableDiskBytesMin - } - byStart[point.Start] = point - } - for start := window.Start; start.Before(window.End); start = start.Add(time.Duration(window.ResolutionSeconds) * time.Second) { - point, ok := byStart[start] - if !ok { - point.Start = start - } - value.History.Points = append(value.History.Points, point) - } - return value, nil -} diff --git a/services/core/internal/store/node_host_history_test.go b/services/core/internal/store/node_host_history_test.go index 27ff4487f..43bdcc818 100644 --- a/services/core/internal/store/node_host_history_test.go +++ b/services/core/internal/store/node_host_history_test.go @@ -1,167 +1,23 @@ package store import ( - "encoding/json" - "errors" - "math" "testing" "time" - - "github.com/google/uuid" ) -func hostPtr[T any](value T) *T { return &value } - -func TestNodeHostHistorySamplingAndDetail(t *testing.T) { +// Retention uses the Runtime history cleanup operation and never deletes nodes. +func TestNodeHostHistoryRetentionKeepsNode(t *testing.T) { s, _, d := managerFixture(t, 2, 8) - connection := onlineManagerNode(t, s, d.LocalNodeID) - epoch := managerEpoch(t, s) now := time.Now().UTC() - host := &RuntimeNodeHost{EffectiveCPUCores: hostPtr(2.0), CPUUtilization: hostPtr(0.35), TotalMemoryBytes: hostPtr(int64(4096)), AvailableMemoryBytes: hostPtr(int64(1024)), AvailableDiskBytes: hostPtr(int64(8192)), ObservedAt: &now} - health := RuntimeNodeHealth{ProviderReady: true, Host: host} - if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, connection, epoch, health); err != nil { - t.Fatal(err) - } - for i, want := range []int64{1, 0} { - if n, err := s.SampleNodeHostHistory(t.Context()); err != nil || n != want { - t.Fatal(i, n, err) - } - } - detail, err := s.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, "1h") - if err != nil || detail.Host.TotalMemoryBytes == nil || *detail.Host.CPUUtilization != 0.35 || len(detail.History.Points) != 60 { - t.Fatal(detail, err) - } - // The current partial minute does not enter history yet. - for _, p := range detail.History.Points { - if p.CPUUtilizationMax != nil { - t.Fatal("partial bucket leaked", p) - } - } - list, err := s.ListRuntimeNodes(t.Context()) - if err != nil { - t.Fatal(err) - } - raw, _ := json.Marshal(list[0]) - var fields map[string]json.RawMessage - _ = json.Unmarshal(raw, &fields) - for _, key := range []string{"host", "history", "cpu_utilization", "total_memory_bytes", "effective_cpu_cores"} { - if _, exists := fields[key]; exists { - t.Fatal("list contract expanded", key, string(raw)) - } - } - // Sample maxima/minima differ from the latest reading. - end := time.Now().UTC().Truncate(time.Minute) - start := end.Add(-time.Minute) - for _, sample := range []struct { - at time.Time - cpu any - mem any - disk any - }{ - {start.Add(time.Second), 0.1, int64(100), int64(800)}, - {start.Add(20 * time.Second), 0.8, int64(70), int64(900)}, - {start.Add(40 * time.Second), nil, int64(500), int64(600)}, - {start.Add(-time.Minute), nil, nil, nil}, - } { - if _, err := s.pool.Exec(t.Context(), "INSERT INTO node_host_history_samples(node_id, observed_at,cpu_utilization,memory_used_bytes,available_disk_bytes) VALUES($1,$2,$3,$4,$5)", d.LocalNodeID, sample.at, sample.cpu, sample.mem, sample.disk); err != nil { + for _, at := range []time.Time{now.Add(-8 * 24 * time.Hour), now.Add(-2 * time.Minute)} { + if _, err := s.pool.Exec(t.Context(), "INSERT INTO node_host_history_samples(node_id, observed_at) VALUES($1,$2)", d.LocalNodeID, at); err != nil { t.Fatal(err) } } - detail, err = s.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, "1h") - if err != nil { - t.Fatal(err) - } - last := detail.History.Points[len(detail.History.Points)-1] - if !last.Start.Equal(start) || last.CPUUtilizationMax == nil || *last.CPUUtilizationMax != 0.8 || *last.MemoryUsedBytesMax != 500 || *last.AvailableDiskBytesMin != 600 { - t.Fatal(last) - } - previous := detail.History.Points[len(detail.History.Points)-2] - if previous.CPUUtilizationMax != nil || previous.MemoryUsedBytesMax != nil || previous.AvailableDiskBytesMin != nil { - t.Fatal(previous) - } - for name, want := range map[string]int{"6h": 72, "24h": 96} { - detail, err := s.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, name) - if err != nil || len(detail.History.Points) != want { - t.Fatal(name, detail, err) - } - } - if _, err := s.GetRuntimeNodeDetail(t.Context(), uuid.NewString(), "1h"); !errors.Is(err, ErrNotFound) { - t.Fatal(err) - } - for _, name := range []string{"", "7d", "other"} { - if _, err := s.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, name); !errors.Is(err, ErrInvalidInput) { - t.Fatal(err) - } - } - // A disconnected node cannot create another history row, even with a fresh last observation. - next := now.Add(time.Millisecond) - host.ObservedAt = &next - if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, connection, epoch, health); err != nil { - t.Fatal(err) - } - if err := s.DisconnectRuntimeNode(t.Context(), d.LocalNodeID, connection, epoch); err != nil { - t.Fatal(err) - } - if n, err := s.SampleNodeHostHistory(t.Context()); err != nil || n != 0 { - t.Fatal(n, err) - } - // A restarted service can query existing durable history without re-sampling. - fresh := New(s.pool) - restored, err := fresh.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, "1h") - if err != nil || restored.Online || restored.History.Points[59].CPUUtilizationMax == nil { - t.Fatal(restored, err) - } - if !restored.Host.ObservedAt.Equal(next) { - t.Fatal(restored.Host) - } - // Retention uses the same existing cleanup operation and does not delete nodes. - old := now.Add(-8 * 24 * time.Hour) - if _, err := s.pool.Exec(t.Context(), "INSERT INTO node_host_history_samples(node_id, observed_at) VALUES($1,$2)", d.LocalNodeID, old); err != nil { - t.Fatal(err) - } if n, err := s.PruneRuntimeHistorySamples(t.Context(), now.Add(-7*24*time.Hour).UnixNano()); err != nil || n != 1 { t.Fatal(n, err) } - if _, err := s.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, "1h"); err != nil { + if _, err := deploymentService(t, s).NodeDetail(t.Context(), d.LocalNodeID, "1h"); err != nil { t.Fatal(err) } } - -func TestNodeHostHistoryFencingAndUnknown(t *testing.T) { - s, _, d := managerFixture(t, 2, 8) - conn := onlineManagerNode(t, s, d.LocalNodeID) - epoch := managerEpoch(t, s) - for _, at := range []time.Time{time.Now().Add(-time.Minute), time.Now().Add(time.Hour)} { - if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, conn, epoch, RuntimeNodeHealth{Host: &RuntimeNodeHost{ObservedAt: &at}}); err != nil { - t.Fatal(err) - } - if n, err := s.SampleNodeHostHistory(t.Context()); err != nil || n != 0 { - t.Fatal(n, err) - } - } - now := time.Now().UTC() - health := RuntimeNodeHealth{Host: &RuntimeNodeHost{ObservedAt: &now}} - if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, uuid.NewString(), epoch, health); !errors.Is(err, ErrRuntimeNodeCredential) { - t.Fatal(err) - } - if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, conn, epoch, health); err != nil { - t.Fatal(err) - } - if n, err := s.SampleNodeHostHistory(t.Context()); err != nil || n != 1 { - t.Fatal(n, err) - } - var cpu *float64 - var memory, disk *int64 - if err := s.pool.QueryRow(t.Context(), "SELECT cpu_utilization,memory_used_bytes,available_disk_bytes FROM node_host_history_samples WHERE node_id=$1", d.LocalNodeID).Scan(&cpu, &memory, &disk); err != nil || cpu != nil || memory != nil || disk != nil { - t.Fatal(cpu, memory, disk, err) - } - for _, host := range []*RuntimeNodeHost{ - {ObservedAt: &now, CPUUtilization: hostPtr(1.1)}, {ObservedAt: &now, CPUUtilization: hostPtr(math.NaN())}, - {ObservedAt: &now, TotalMemoryBytes: hostPtr(int64(10)), AvailableMemoryBytes: hostPtr(int64(11))}, - {ObservedAt: &now, AvailableDiskBytes: hostPtr(int64(-1))}, {ObservedAt: &now, EffectiveCPUCores: hostPtr(0.0)}, {}, - } { - if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, conn, epoch, RuntimeNodeHealth{Host: host}); !errors.Is(err, ErrInvalidInput) { - t.Fatal(host, err) - } - } -} diff --git a/services/core/internal/store/provider_configuration_migration_test.go b/services/core/internal/store/provider_configuration_migration_test.go index 8513dd0d7..a6883c994 100644 --- a/services/core/internal/store/provider_configuration_migration_test.go +++ b/services/core/internal/store/provider_configuration_migration_test.go @@ -4,12 +4,13 @@ import ( "bytes" "database/sql" "encoding/json" + "os" + "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" "github.com/google/uuid" "github.com/jackc/pgx/v5/stdlib" "github.com/pressly/goose/v3" - "os" - "testing" ) func TestProviderConfigurationMigrationPreservesCiphertextAndRetainedOwnership(t *testing.T) { @@ -17,7 +18,8 @@ func TestProviderConfigurationMigrationPreservesCiphertextAndRetainedOwnership(t tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) archiveAllocation(t, w, tenant, session, view.InstallationID) input.Configuration.(*e2b.DeploymentConfiguration).Template = "next:" + uuid.NewString() - if _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}); err != nil { + input.ExpectedGeneration = 1 + if _, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, input); err != nil { t.Fatal(err) } db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) @@ -44,11 +46,11 @@ func TestProviderConfigurationMigrationPreservesCiphertextAndRetainedOwnership(t if err = db.QueryRowContext(t.Context(), "SELECT provider_credential FROM runtime_deployment").Scan(&after); err != nil || !bytes.Equal(before, after) { t.Fatal("ciphertext rewritten", err) } - restored, err := s.GetSandboxSetup(t.Context()) + restored, err := deploymentService(t, s).Setup(t.Context()) if err != nil || restored.Generation != uint64(generation) || restored.Configuration.(*e2b.DeploymentConfiguration).APIKey != input.Configuration.(*e2b.DeploymentConfiguration).APIKey { t.Fatal("ownership or credential changed", err) } - public, err := s.GetRuntimeDeployment(t.Context()) + public, err := deploymentService(t, s).View(t.Context()) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/store/provider_registration_migration_test.go b/services/core/internal/store/provider_registration_migration_test.go index 06a6074c8..86a3a7cb3 100644 --- a/services/core/internal/store/provider_registration_migration_test.go +++ b/services/core/internal/store/provider_registration_migration_test.go @@ -18,13 +18,15 @@ func TestProviderRegistrationDowngradePreservesCustomEndpoints(t *testing.T) { tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) archiveAllocation(t, w, tenant, session, view.InstallationID) input.Configuration.(*e2b.DeploymentConfiguration).APIURL, input.Configuration.(*e2b.DeploymentConfiguration).Domain = "https://api.example.test", "example.test" - if _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: view.Generation}); err != nil { + input.ExpectedGeneration = view.Generation + if _, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, input); err != nil { t.Fatal(err) } tenant, session = managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) archiveAllocation(t, w, tenant, session, view.InstallationID) input.Configuration.(*e2b.DeploymentConfiguration).Template = "next:" + uuid.NewString() - if _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 2}); err != nil { + input.ExpectedGeneration = 2 + if _, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, input); err != nil { t.Fatal(err) } db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) diff --git a/services/core/internal/store/public_handler_fixture_test.go b/services/core/internal/store/public_handler_fixture_test.go index 01566aa6c..f25d83c34 100644 --- a/services/core/internal/store/public_handler_fixture_test.go +++ b/services/core/internal/store/public_handler_fixture_test.go @@ -10,6 +10,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/agents" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmenttemplates" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/files" @@ -151,12 +152,19 @@ func executorURL(url string) func(*api.Dependencies) { return func(d *api.Dependencies) { d.Execution.ExecutorURL = url } } -// managedSandboxes enables the Store's managed sandbox deployment: its -// administration routes and openai_hosted Environments. It follows the option -// that enables Execution. -func managedSandboxes(t testing.TB, s *store.Store) func(*api.Dependencies) { +// managedSandboxes enables the managed sandbox deployment on db: its +// administration and node routes and openai_hosted Environments. Deployment +// changes, reset and discovery need the Worker and are strict stand-ins. It +// follows the option that enables Execution. +func managedSandboxes(t testing.TB, s *store.Store, db fixtureDB) func(*api.Dependencies) { return func(d *api.Dependencies) { - d.Sandboxes = &api.Sandboxes{Deployment: s, DeploymentChanges: strictStandIn{t}, ConfigurationDiscovery: strictStandIn{t}} + d.Sandboxes = &api.Sandboxes{ + Deployment: fixtureDeployment(t, db), + NodeAllocations: s, + DeploymentChanges: strictStandIn{t}, + DeploymentReset: strictStandIn{t}, + ConfigurationDiscovery: strictStandIn{t}, + } } } @@ -248,22 +256,22 @@ func (s strictStandIn) DiscoverConfiguration(context.Context, string, sandbox.Co return nil, nil } -func (s strictStandIn) InitializeSandboxDeployment(context.Context, store.SandboxDeploymentSetupRequest) (store.RuntimeDeploymentView, error) { +func (s strictStandIn) InitializeSandboxDeployment(context.Context, sandbox.Selection) (deployment.View, error) { s.unexpected("InitializeSandboxDeployment") - return store.RuntimeDeploymentView{}, nil + return deployment.View{}, nil } -func (s strictStandIn) UpdateSandboxDeployment(context.Context, store.SandboxDeploymentUpdateRequest) (store.RuntimeDeploymentView, error) { +func (s strictStandIn) UpdateSandboxDeployment(context.Context, sandbox.Selection) (deployment.View, error) { s.unexpected("UpdateSandboxDeployment") - return store.RuntimeDeploymentView{}, nil + return deployment.View{}, nil } -func (s strictStandIn) StartSandboxReset(context.Context, store.SandboxResetRequest) (store.RuntimeDeploymentView, error) { +func (s strictStandIn) StartSandboxReset(context.Context, store.SandboxResetRequest) (deployment.View, error) { s.unexpected("StartSandboxReset") - return store.RuntimeDeploymentView{}, nil + return deployment.View{}, nil } -func (s strictStandIn) CancelSandboxReset(context.Context, uint64) (store.RuntimeDeploymentView, error) { +func (s strictStandIn) CancelSandboxReset(context.Context, uint64) (deployment.View, error) { s.unexpected("CancelSandboxReset") - return store.RuntimeDeploymentView{}, nil + return deployment.View{}, nil } diff --git a/services/core/internal/store/public_url.go b/services/core/internal/store/public_url.go index 1d92dbd31..22e9beedf 100644 --- a/services/core/internal/store/public_url.go +++ b/services/core/internal/store/public_url.go @@ -2,16 +2,10 @@ package store import ( "context" - "errors" ) -// ErrSandboxPublicURLUnreachable rejects selection and admission when the provider -// requires a reachable public origin and the installation is loopback. -var ErrSandboxPublicURLUnreachable = errors.New("This sandbox provider needs a reachable HTTPS public URL before they can connect to Core.") - -// SetPublicURL records OAC_PUBLIC_URL, validated by the caller. Core -// reports it as the deployment and node configuration core_url and records it -// on each node it enrolls. Call it once, before serving requests. +// SetPublicURL records OAC_PUBLIC_URL, validated by the caller. Placement +// admits only nodes enrolled with it. Call it once, before serving requests. func (s *Store) SetPublicURL(value string) { s.publicURL = value } // AddressBindings counts what is bound to an installation address: nodes diff --git a/services/core/internal/store/runtime_adoption_test.go b/services/core/internal/store/runtime_adoption_test.go index de32a0121..ce25b5caf 100644 --- a/services/core/internal/store/runtime_adoption_test.go +++ b/services/core/internal/store/runtime_adoption_test.go @@ -5,11 +5,12 @@ import ( "reflect" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) -func legacyAdoptionFixture(t *testing.T) (*Store, *Store, RuntimeDeployment, RuntimeAllocation) { +func legacyAdoptionFixture(t *testing.T) (*Store, *Store, deployment.ProcessDeployment, RuntimeAllocation) { t.Helper() s, _ := newManagedTestStore(t) w := executionWriter(t, s) @@ -51,7 +52,7 @@ func TestHistoricalRuntimeResourcesCannotBeAdopted(t *testing.T) { if err != nil { t.Fatal(err) } - err = w.ConfigureRuntimeDeployment(t.Context(), &d) + err = deploymentExecution(t, w).ConfigureProcess(t.Context(), &d) if state == "released" && !pending { if err != nil { t.Fatal("released history blocked fresh configuration", err) diff --git a/services/core/internal/store/runtime_allocations.go b/services/core/internal/store/runtime_allocations.go index 8e7cb3043..14efb2687 100644 --- a/services/core/internal/store/runtime_allocations.go +++ b/services/core/internal/store/runtime_allocations.go @@ -11,6 +11,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" ) // RuntimeAllocation retains compute ownership, not public readiness. It survives @@ -96,18 +97,18 @@ func (s *Store) ReserveRuntimeAllocation(ctx context.Context, tenant, environmen return ErrInvalidInput } var nodeID pgtype.UUID - deployment, err := q.GetRuntimeDeployment(ctx) + active, err := q.GetRuntimeDeployment(ctx) if err != nil { return err } - generation := pgtype.Int8{Int64: deployment.Generation, Valid: true} - if deployment.Mode == "nodes" { + generation := pgtype.Int8{Int64: active.Generation, Valid: true} + if active.Mode == "nodes" { placement, err := q.GetRuntimePlacement(ctx, lookup.ID) if err != nil { return err } if placement.ReleasedAt.Valid || !placement.Available { - return ErrRuntimeNodeUnavailable + return deployment.ErrNodeUnavailable } nodeID = placement.NodeID generation = placement.DeploymentGeneration diff --git a/services/core/internal/store/runtime_deployment.go b/services/core/internal/store/runtime_deployment.go index 90cced855..809d6025c 100644 --- a/services/core/internal/store/runtime_deployment.go +++ b/services/core/internal/store/runtime_deployment.go @@ -2,93 +2,17 @@ package store import ( "context" - "encoding/hex" + "encoding/json" "fmt" - "strings" - - "github.com/jackc/pgx/v5" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" ) -// RuntimeDeployment identifies the one operator-selected installation for this database. -// Its fingerprint describes the backend namespace, never credentials or image contents. -type RuntimeDeployment struct { - ProviderKind string - LocalNodeID string - LocalCredentialSHA256 string - LocalMaxActive, LocalMaxRetained int - InstallationID string - BackendFingerprint string - AdmissionPaused bool -} - -// ConfigureRuntimeDeployment runs before Worker startup under its execution lease. -// AdmissionPaused must be committed for the old installation before any switch. -// A nil selection never forgets the previous identity or unresolved resources. -func (s *Store) ConfigureRuntimeDeployment(ctx context.Context, selected *RuntimeDeployment) error { - if err := s.checkExecutionAuthority(); err != nil { - return err - } - if selected != nil { - copy := *selected - selected = © - } - var update sqlc.SetRuntimeDeploymentParams - if selected != nil { - id, err := parseConnectionGeneration(selected.InstallationID) - if err != nil { - return err - } - digest, err := hex.DecodeString(selected.BackendFingerprint) - if err != nil || len(digest) != 32 || strings.ToLower(selected.BackendFingerprint) != selected.BackendFingerprint { - return fmt.Errorf("%w: invalid backend identity fingerprint", ErrInvalidInput) - } - update = sqlc.SetRuntimeDeploymentParams{InstallationID: id, BackendFingerprint: selected.BackendFingerprint, AdmissionPaused: selected.AdmissionPaused} - } - return s.writer.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { - q := s.queries.WithTx(tx) - previous, err := q.LockRuntimeDeployment(ctx) - if err != nil { - return err - } - if previous.WebManaged { - return ErrSandboxDeploymentConflict - } - if selected != nil && previous.InstallationID == update.InstallationID && previous.BackendFingerprint == update.BackendFingerprint && (previous.ProviderKind == "" || selected.ProviderKind == previous.ProviderKind) { - if err := q.SetRuntimeDeployment(ctx, update); err != nil { - return err - } - return configureRuntimeManager(ctx, q, previous, selected) - } - resources, err := q.CountRuntimeDeploymentResources(ctx) - if err != nil { - return err - } - if selected == nil { - if previous.InstallationID.Valid && (resources.Allocations != 0 || resources.Pending != 0) { - return fmt.Errorf("cannot disable managed sandbox provider: %d unreleased allocations (instances, retained snapshots, uncertain operations or pending cleanup) and %d pending hosted environments remain", resources.Allocations, resources.Pending) - } - return nil - } - if !previous.InstallationID.Valid { - if resources.Allocations != 0 { - return fmt.Errorf("cannot adopt sandbox installation: %d existing unreleased allocations (including retained snapshots and pending cleanup) have no verified backend identity", resources.Allocations) - } - } else { - if !previous.AdmissionPaused || !selected.AdmissionPaused { - return fmt.Errorf("cannot switch sandbox installation: persist maintenance on the previous installation and keep the new installation in maintenance") - } - if resources.Allocations != 0 || resources.Pending != 0 { - return fmt.Errorf("cannot switch sandbox installation: %d unreleased allocations (instances, retained snapshots, uncertain operations or pending cleanup) and %d pending hosted environments remain", resources.Allocations, resources.Pending) - } - } - if err := q.SetRuntimeDeployment(ctx, update); err != nil { - return err - } - return configureRuntimeManager(ctx, q, previous, selected) - }) -} +// sandboxProviders interprets the deployment's provider declarations for +// Session admission and placement until they move out of the Store. +var sandboxProviders = providers.Builtin() // New work and deployment changes share this lock. Existing receipts are checked // first, preserving idempotent retries and cleanup while maintenance is active. @@ -106,8 +30,11 @@ func checkRuntimeDeploymentAdmission(ctx context.Context, q *sqlc.Queries, insta if current.AdmissionPaused { return fmt.Errorf("%w: sandbox creation is paused for provider maintenance", ErrEnvironmentUnavailable) } - if _, err := deploymentSpecification(current); current.ProviderKind != "" && err != nil { - return fmt.Errorf("%w: sandbox creation requires a deployment specification", ErrEnvironmentUnavailable) + if current.ProviderKind != "" { + var spec sandbox.DeploymentSpec + if json.Unmarshal(current.Specification, &spec) != nil || sandboxProviders.ValidateSpecification(current.ProviderKind, spec) != nil { + return fmt.Errorf("%w: sandbox creation requires a deployment specification", ErrEnvironmentUnavailable) + } } if installation != "" { id, err := parseConnectionGeneration(installation) diff --git a/services/core/internal/store/runtime_deployment_test.go b/services/core/internal/store/runtime_deployment_test.go index 788b3646d..124ab7a3c 100644 --- a/services/core/internal/store/runtime_deployment_test.go +++ b/services/core/internal/store/runtime_deployment_test.go @@ -10,16 +10,17 @@ import ( "github.com/google/uuid" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" ) -func deploymentSelection() RuntimeDeployment { - return RuntimeDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64)} +func deploymentSelection() deployment.ProcessDeployment { + return deployment.ProcessDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64)} } -func deploymentConfigure(t *testing.T, w *Store, config *RuntimeDeployment) { +func deploymentConfigure(t *testing.T, w *Store, config *deployment.ProcessDeployment) { t.Helper() - if err := w.ConfigureRuntimeDeployment(t.Context(), config); err != nil { + if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), config); err != nil { t.Fatal(err) } if config != nil && config.ProviderKind != "" { @@ -41,52 +42,6 @@ func legacyRuntimeSpecification(t *testing.T, w *Store, provider string) { } } -func TestRuntimeDeploymentRequiresMaintenanceBeforeIdentityChange(t *testing.T) { - s, pool := newManagedTestStore(t) - w := executionWriter(t, s) - old := deploymentSelection() - deploymentConfigure(t, w, &old) - if err := s.ConfigureRuntimeDeployment(t.Context(), &old); !errors.Is(err, ErrExecutionAuthority) { - t.Fatal("unleased configuration accepted", err) - } - for _, changeID := range []bool{false, true} { - next := old - if changeID { - next.InstallationID = uuid.NewString() - } else { - next.BackendFingerprint = strings.Repeat("b", 64) - } - next.AdmissionPaused = true - if err := w.ConfigureRuntimeDeployment(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "maintenance") { - t.Fatal("identity changed before prior maintenance", err) - } - } - old.AdmissionPaused = true - deploymentConfigure(t, w, &old) - next := old - next.BackendFingerprint = strings.Repeat("b", 64) - next.AdmissionPaused = false - if err := w.ConfigureRuntimeDeployment(t.Context(), &next); err == nil { - t.Fatal("switch reopened creation in same operation") - } - next.AdmissionPaused = true - deploymentConfigure(t, w, &next) - var id, fingerprint string - var maintenance bool - if err := pool.QueryRow(t.Context(), "SELECT installation_id::text,backend_fingerprint,admission_paused FROM runtime_deployment").Scan(&id, &fingerprint, &maintenance); err != nil || id != next.InstallationID || fingerprint != next.BackendFingerprint || !maintenance { - t.Fatal("switch identity not durable", id, fingerprint, maintenance, err) - } - deploymentConfigure(t, w, nil) - // Disabling the configured adapter must not forget the old maintenance state. - next.AdmissionPaused = false - deploymentConfigure(t, w, &next) - another := deploymentSelection() - another.AdmissionPaused = true - if err := w.ConfigureRuntimeDeployment(t.Context(), &another); err == nil { - t.Fatal("nil selection erased the maintenance prerequisite") - } -} - func TestRuntimeDeploymentPendingSessionsCannotMigrate(t *testing.T) { s, _ := newManagedTestStore(t) w := executionWriter(t, s) @@ -99,10 +54,10 @@ func TestRuntimeDeploymentPendingSessionsCannotMigrate(t *testing.T) { deploymentConfigure(t, w, &old) next := deploymentSelection() next.AdmissionPaused = true - if err := w.ConfigureRuntimeDeployment(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 pending hosted") { + if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 pending hosted") { t.Fatal("pending Session migrated", err) } - if err := w.ConfigureRuntimeDeployment(t.Context(), nil); err == nil { + if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), nil); err == nil { t.Fatal("pending Session orphaned by removing provider") } if err := s.DeleteSession(t.Context(), tenant, session.ID); err != nil { @@ -121,7 +76,7 @@ func TestRuntimeDeploymentUnknownAllocationsBlockAdoptionAndSwitch(t *testing.T) if err != nil { t.Fatal(err) } - if err := w.ConfigureRuntimeDeployment(t.Context(), &old); err == nil || !strings.Contains(err.Error(), "no verified backend identity") { + if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &old); err == nil || !strings.Contains(err.Error(), "no verified backend identity") { t.Fatal("legacy allocation silently adopted", err) } if err := s.DeleteSession(t.Context(), tenant, session.ID); err != nil { @@ -133,7 +88,7 @@ func TestRuntimeDeploymentUnknownAllocationsBlockAdoptionAndSwitch(t *testing.T) if _, err := w.ReleaseRuntimeAllocation(t.Context(), owner); !errors.Is(err, ErrTurnConflict) { t.Fatal("unknown creation lost cleanup ownership", err) } - if err := w.ConfigureRuntimeDeployment(t.Context(), &old); err == nil { + if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &old); err == nil { t.Fatal("deleted unknown allocation did not block adoption") } if _, err := w.SettleRuntimeCreation(t.Context(), owner); err != nil { @@ -152,10 +107,10 @@ func TestRuntimeDeploymentUnknownAllocationsBlockAdoptionAndSwitch(t *testing.T) deploymentConfigure(t, w, &old) next := deploymentSelection() next.AdmissionPaused = true - if err := w.ConfigureRuntimeDeployment(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { + if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { t.Fatal("unknown creation did not block switch", err) } - if err := w.ConfigureRuntimeDeployment(t.Context(), nil); err == nil { + if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), nil); err == nil { t.Fatal("removing adapter orphaned unknown creation") } replay, err := w.ReserveRuntimeAllocation(t.Context(), tenant, environment.ID, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())) @@ -282,7 +237,7 @@ func TestRuntimeDeploymentRetainedResourcesBlockSwitchWithoutMutation(t *testing } next := deploymentSelection() next.AdmissionPaused = true - if err := w.ConfigureRuntimeDeployment(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { + if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { t.Fatal("retained resource allowed switch", state, err) } if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(a)::text FROM runtime_allocations a WHERE id=$1", owner.ID).Scan(&after); err != nil { @@ -325,7 +280,8 @@ func TestRuntimeDeploymentAllocationBeforeMaintenanceRetainsOwnership(t *testing maintaining := make(chan error, 1) maintenance := config maintenance.AdmissionPaused = true - go func() { maintaining <- w.ConfigureRuntimeDeployment(ctx, &maintenance) }() + changes := deploymentExecution(t, w) + go func() { maintaining <- changes.ConfigureProcess(ctx, &maintenance) }() if err := tx.Commit(ctx); err != nil { t.Fatal(err) } @@ -341,7 +297,7 @@ func TestRuntimeDeploymentAllocationBeforeMaintenanceRetainsOwnership(t *testing } next := deploymentSelection() next.AdmissionPaused = true - if err := w.ConfigureRuntimeDeployment(ctx, &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { + if err := deploymentExecution(t, w).ConfigureProcess(ctx, &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { t.Fatal("earlier in-flight allocation omitted from switch guard", err) } } diff --git a/services/core/internal/store/runtime_lifecycle_nodes.go b/services/core/internal/store/runtime_lifecycle_nodes.go index bfc3cc01a..53090d591 100644 --- a/services/core/internal/store/runtime_lifecycle_nodes.go +++ b/services/core/internal/store/runtime_lifecycle_nodes.go @@ -4,6 +4,8 @@ import ( "context" "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/jackc/pgx/v5" @@ -111,12 +113,12 @@ func (s *Store) ResolveRuntimeLifecycleNode(ctx context.Context, tenant, environ } if row.ProviderKind == "" || row.Mode == "direct" { if row.PlacementNodeID.Valid || row.AllocationNodeID.Valid { - return "", ErrRuntimeNodeUnavailable + return "", deployment.ErrNodeUnavailable } return "", nil } if !row.PlacementNodeID.Valid || (row.AllocationID.Valid && row.AllocationNodeID != row.PlacementNodeID) || (!row.AllocationID.Valid && row.ReleasedAt.Valid) { - return "", ErrRuntimeNodeUnavailable + return "", deployment.ErrNodeUnavailable } return runtimeUUID(row.PlacementNodeID), nil } diff --git a/services/core/internal/store/runtime_lifecycle_nodes_test.go b/services/core/internal/store/runtime_lifecycle_nodes_test.go index 7c4e62358..a67b061d7 100644 --- a/services/core/internal/store/runtime_lifecycle_nodes_test.go +++ b/services/core/internal/store/runtime_lifecycle_nodes_test.go @@ -6,18 +6,20 @@ import ( "strings" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) func lifecycleTestNode(t *testing.T, s *Store) string { t.Helper() - token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 100, MaxRetained: 100})) + nodes := deploymentService(t, s) + token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 100, MaxRetained: 100})) if err != nil { t.Fatal(err) } id := uuid.NewString() - _, err = s.EnrollRuntimeNode(t.Context(), token, RuntimeNodeEnrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: id, Credential: strings.Repeat("n", 64), Name: "second", Provider: "docker", BackendFingerprint: strings.Repeat("b", 64)}) + _, err = nodes.Enroll(t.Context(), token, deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: id, Credential: strings.Repeat("n", 64), Name: "second", Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), CoreURL: s.publicURL}) if err != nil { t.Fatal(err) } @@ -33,7 +35,7 @@ func lifecycleTestSession(t *testing.T, s *Store, node string) (string, Session) } return tenant, session } -func lifecycleTestAllocation(t *testing.T, s, w *Store, d RuntimeDeployment, node string) RuntimeAllocation { +func lifecycleTestAllocation(t *testing.T, s, w *Store, d deployment.ProcessDeployment, node string) RuntimeAllocation { t.Helper() tenant, session := lifecycleTestSession(t, s, node) allocation, err := w.ReserveRuntimeAllocation(t.Context(), tenant, session.Environment.ID, d.InstallationID, runtimedevice.HashCredential(uuid.NewString())) @@ -150,7 +152,7 @@ func TestRuntimeLifecycleNodeInventoryAndRouting(t *testing.T) { if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET node_id=NULL WHERE id=$1", owner.ID); err != nil { t.Fatal(err) } - checkRoute("", ErrRuntimeNodeUnavailable) + checkRoute("", deployment.ErrNodeUnavailable) if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET node_id=$1 WHERE id=$2", other, owner.ID); err != nil { t.Fatal(err) } @@ -173,7 +175,7 @@ func TestRuntimeLifecycleNodeInventoryAndRouting(t *testing.T) { if rows, err := w.ListRuntimeAllocationsForNode(t.Context(), other, ""); err != nil || len(rows) != 0 { t.Fatal("released allocation scanned", rows, err) } - if err := s.RemoveRuntimeNode(t.Context(), other); err != nil { + if err := deploymentService(t, s).RemoveNode(t.Context(), other); err != nil { t.Fatal(err) } nodes, err = w.ListRuntimeLifecycleNodes(t.Context()) @@ -193,7 +195,7 @@ func TestRuntimeLifecycleNodeRejectsMissingOrReleasedPlacement(t *testing.T) { if _, err := s.pool.Exec(t.Context(), mutation, session.Environment.ID); err != nil { t.Fatal(err) } - if _, err := w.ResolveRuntimeLifecycleNode(t.Context(), tenant, session.Environment.ID); !errors.Is(err, ErrRuntimeNodeUnavailable) { + if _, err := w.ResolveRuntimeLifecycleNode(t.Context(), tenant, session.Environment.ID); !errors.Is(err, deployment.ErrNodeUnavailable) { t.Fatal("invalid placement routed", err) } rows, err := w.ListUnallocatedHostedEnvironmentsForNode(t.Context(), d.LocalNodeID, "") diff --git a/services/core/internal/store/runtime_node_capacity_test.go b/services/core/internal/store/runtime_node_capacity_test.go index 3434ed0e6..3c8a8c643 100644 --- a/services/core/internal/store/runtime_node_capacity_test.go +++ b/services/core/internal/store/runtime_node_capacity_test.go @@ -2,121 +2,24 @@ package store import ( "errors" - "strings" "testing" - "github.com/google/uuid" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" ) -func TestRuntimeEnrollmentApprovedCapacity(t *testing.T) { +// Invalid capacity is reported before the active reset conflict. +func TestRuntimeEnrollmentCapacityPrecedesResetConflict(t *testing.T) { s, w, view, _ := webSpecificationFixture(t, "microsandbox") - for _, capacity := range []RuntimeNodeCapacity{{0, 8}, {3, 2}, {1, 1000001}} { - if _, err := s.CreateRuntimeEnrollment(t.Context(), capacity); !errors.Is(err, ErrInvalidInput) { - t.Fatal("invalid capacity accepted", err) - } - } - token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{1, 3})) - if err != nil { - t.Fatal(err) - } - config, err := s.RuntimeNodeConfiguration(t.Context(), "", token) - if err != nil || config.MaxActive != 1 || config.MaxRetained != 3 { - t.Fatal("bootstrap lost approved capacity", config, err) - } - input := RuntimeNodeEnrollment{NodeID: uuid.NewString(), Name: "approved", Credential: strings.Repeat("a", 64), Provider: "microsandbox", BackendFingerprint: strings.Repeat("b", 64), SpecificationDigest: view.SpecificationDigest, DeploymentGeneration: view.Generation} - identity, err := s.EnrollRuntimeNode(t.Context(), token, input) - if err != nil || identity.MaxActive != 1 || identity.MaxRetained != 3 { - t.Fatal("enrollment did not apply token capacity", identity, err) - } - if _, err := s.EnrollRuntimeNode(t.Context(), token, input); !errors.Is(err, ErrRuntimeNodeCredential) { - t.Fatal("consumed token reused", err) - } - if err := s.UpdateRuntimeNode(t.Context(), input.NodeID, RuntimeNodeUpdate{Name: "approved", MaxActive: 2, MaxRetained: 5}); err != nil { - t.Fatal(err) - } - // Microsandbox uses both limits, so a retained limit below the active one is rejected and nothing is written. - if err := s.UpdateRuntimeNode(t.Context(), input.NodeID, RuntimeNodeUpdate{Name: "rejected", MaxActive: 4, MaxRetained: 3}); !errors.Is(err, ErrInvalidInput) { - t.Fatal("retained limit below active accepted", err) - } - var name string - if err := s.pool.QueryRow(t.Context(), "SELECT name FROM runtime_nodes WHERE id=$1", input.NodeID).Scan(&name); err != nil || name != "approved" { - t.Fatal("rejected update was written", name, err) - } - identity, err = s.AuthenticateRuntimeNode(t.Context(), input.NodeID, input.Credential) - if err != nil || identity.MaxActive != 2 || identity.MaxRetained != 5 { - t.Fatal("credential read ignored admin update", identity, err) - } - config, err = s.RuntimeNodeConfiguration(t.Context(), input.NodeID, input.Credential) - if err != nil || config.MaxActive != 2 || config.MaxRetained != 5 { - t.Fatal("configuration read ignored admin update", config, err) - } - // Invalid capacity is reported before the active reset conflict. Enter reset - // through its owner transaction so the fixture obeys the admission invariant. - if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxResetRequest{ExpectedGeneration: view.Generation, Clear: "auto"}); err != nil { + nodes := deploymentService(t, s) + // Enter reset through its owner transaction so the fixture obeys the + // admission invariant. + if err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxResetRequest{ExpectedGeneration: view.Generation, Clear: "auto"}); err != nil { t.Fatal(err) } - if _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{1, 3}); !errors.Is(err, ErrSandboxResetInProgress) { + if _, err := nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 3}); !errors.Is(err, deployment.ErrResetInProgress) { t.Fatal("valid capacity did not reach the reset guard", err) } - if _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{3, 2}); !errors.Is(err, ErrInvalidInput) { + if _, err := nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 3, MaxRetained: 2}); !errors.Is(err, deployment.ErrInvalidInput) { t.Fatal("invalid capacity reported as a conflict", err) } } - -// Docker never suspends a sandbox, so every read reports its retained limit as its -// active limit, including for a token or node stored before Core applied that rule. -func TestDockerRetainedLimitFollowsActive(t *testing.T) { - s, _, view, _ := webSpecificationFixture(t, "docker") - node := func(name, credential string) RuntimeNodeEnrollment { - return RuntimeNodeEnrollment{NodeID: uuid.NewString(), Name: name, Credential: strings.Repeat(credential, 64), Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), SpecificationDigest: view.SpecificationDigest, DeploymentGeneration: view.Generation} - } - sql := func(query string, args ...any) { - t.Helper() - if _, err := s.pool.Exec(t.Context(), query, args...); err != nil { - t.Fatal(err) - } - } - token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 3, MaxRetained: 1})) - if err != nil { - t.Fatal(err) - } - current := node("Docker", "d") - if identity, err := s.EnrollRuntimeNode(t.Context(), token, current); err != nil || identity.MaxRetained != 3 { - t.Fatal("enrollment kept a separate Docker retained limit", identity, err) - } - legacyToken, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 8})) - if err != nil { - t.Fatal(err) - } - sql("UPDATE runtime_node_enrollments SET max_retained=8 WHERE token_sha256=$1", runtimeTokenDigest(legacyToken)) - if config, err := s.RuntimeNodeConfiguration(t.Context(), "", legacyToken); err != nil || config.MaxRetained != 2 { - t.Fatal("bootstrap reported a legacy Docker retained limit", config, err) - } - legacy := node("Legacy", "l") - if _, err := s.EnrollRuntimeNode(t.Context(), legacyToken, legacy); err != nil { - t.Fatal(err) - } - var stored int32 - if err := s.pool.QueryRow(t.Context(), "SELECT max_retained FROM runtime_nodes WHERE id=$1", legacy.NodeID).Scan(&stored); err != nil || stored != 2 { - t.Fatal("enrollment stored a legacy Docker retained limit", stored, err) - } - sql("UPDATE runtime_nodes SET max_retained=8 WHERE id=$1", legacy.NodeID) - if identity, err := s.AuthenticateRuntimeNode(t.Context(), legacy.NodeID, legacy.Credential); err != nil || identity.MaxRetained != 2 { - t.Fatal("node identity reported a legacy Docker retained limit", identity, err) - } - if config, err := s.RuntimeNodeConfiguration(t.Context(), legacy.NodeID, legacy.Credential); err != nil || config.MaxRetained != 2 { - t.Fatal("node configuration reported a legacy Docker retained limit", config, err) - } - if err := s.UpdateRuntimeNode(t.Context(), current.NodeID, RuntimeNodeUpdate{Name: "Docker", MaxActive: 5, MaxRetained: 12}); err != nil { - t.Fatal(err) - } - nodes, err := s.ListRuntimeNodes(t.Context()) - if err != nil || len(nodes) != 2 { - t.Fatal(nodes, err) - } - for _, n := range nodes { - if n.MaxRetained != n.MaxActive || (n.ID == current.NodeID && n.MaxActive != 5) { - t.Fatal("node list kept a separate Docker retained limit", n) - } - } -} diff --git a/services/core/internal/store/runtime_node_configuration.go b/services/core/internal/store/runtime_node_configuration.go deleted file mode 100644 index 2b1cdf5b9..000000000 --- a/services/core/internal/store/runtime_node_configuration.go +++ /dev/null @@ -1,104 +0,0 @@ -package store - -import ( - "context" - "math" - "time" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" - "github.com/jackc/pgx/v5/pgtype" -) - -type RuntimeNodeConfiguration struct { - MaxActive int `json:"max_active"` - MaxRetained int `json:"max_retained"` - InstallationID string `json:"installation_id"` - Provider string `json:"provider"` - CoreURL string `json:"core_url"` - Generation uint64 `json:"generation"` - Specification sandbox.DeploymentSpec `json:"specification"` - SpecificationDigest string `json:"specification_digest"` -} - -// Read-only bootstrap never consumes enrollment tokens or reveals cloud credentials. -// The deployment lock keeps authentication and the returned generation consistent. -// The credential is authenticated before any deployment state is reported. -func (s *Store) RuntimeNodeConfiguration(ctx context.Context, nodeID, token string) (RuntimeNodeConfiguration, error) { - return s.RuntimeNodeGenerationConfiguration(ctx, nodeID, token, 0) -} - -// Exact generation recovery is restricted to this node's current target, serving -// pin and unreleased ownership. It is never a general history read. -func (s *Store) RuntimeNodeGenerationConfiguration(ctx context.Context, nodeID, token string, generation uint64) (RuntimeNodeConfiguration, error) { - var result RuntimeNodeConfiguration - if generation > math.MaxInt64 { - return result, ErrInvalidInput - } - err := s.runtimeDeploymentTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { - var node *sqlc.RuntimeNode - var installation pgtype.UUID - var active, retained int32 - if nodeID == "" { - r, err := q.GetRuntimeEnrollment(ctx, runtimeTokenDigest(token)) - if err != nil || r.ConsumedAt.Valid || !r.ExpiresAt.Time.After(time.Now()) { - return ErrRuntimeNodeCredential - } - installation, active, retained = r.InstallationID, r.MaxActive, r.MaxRetained - } else { - id, err := parseConnectionGeneration(nodeID) - if err != nil { - return ErrRuntimeNodeCredential - } - n, err := q.GetRuntimeNode(ctx, id) - if err != nil || n.CredentialSha256 != runtimeTokenDigest(token) { - return ErrRuntimeNodeCredential - } - node, installation, active, retained = &n, n.InstallationID, n.MaxActive, n.MaxRetained - } - // A claimed installation rejects foreign credentials identically before - // and after initialization. - if d.InstallationID.Valid && installation != d.InstallationID { - return ErrRuntimeNodeCredential - } - if !runtimeDeploymentInitialized(d) { - return ErrRuntimeNodeUnavailable - } - if node == nil && d.ResetClear.Valid { - return ErrSandboxResetInProgress - } - if d.Mode != "nodes" { - return ErrSandboxDeploymentConflict - } - if node != nil { - if err := validateNodeEnrollmentIdentity(ctx, q, d, *node); err != nil { - return err - } - } - selected := uint64(d.Generation) - if generation != 0 { - if node == nil { - return ErrRuntimeNodeCredential - } - selected = generation - kept, err := q.NodeGenerationKept(ctx, sqlc.NodeGenerationKeptParams{NodeID: node.ID, Generation: int64(generation)}) - if err != nil { - return err - } - if !kept { - return ErrRuntimeSpecificationMismatch - } - } - spec, err := nodeGenerationSpec(ctx, q, d, selected) - if err != nil { - return err - } - if node == nil && d.AdmissionPaused { - return ErrSandboxDeploymentConflict - } - result = RuntimeNodeConfiguration{MaxActive: int(active), MaxRetained: providers.RetainedLimit(d.ProviderKind, int(active), int(retained)), InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, CoreURL: s.publicURL, Generation: selected, Specification: spec, SpecificationDigest: spec.Digest(d.ProviderKind)} - return nil - }) - return result, err -} diff --git a/services/core/internal/store/runtime_node_deployment.go b/services/core/internal/store/runtime_node_deployment.go deleted file mode 100644 index 1d2691eb9..000000000 --- a/services/core/internal/store/runtime_node_deployment.go +++ /dev/null @@ -1,70 +0,0 @@ -package store - -import ( - "context" - "errors" - "fmt" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" -) - -func configureRuntimeManager(ctx context.Context, q *sqlc.Queries, previous sqlc.RuntimeDeployment, selected *RuntimeDeployment) error { - if selected.ProviderKind == "" { - return nil - } - if !providers.IsNode(selected.ProviderKind) { - return ErrInvalidInput - } - installation, err := parseConnectionGeneration(selected.InstallationID) - if err != nil { - return err - } - if previous.ProviderKind == "" { - resources, err := q.CountRuntimeDeploymentResources(ctx) - if err != nil { - return err - } - if resources.Allocations != 0 || resources.Pending != 0 { - return fmt.Errorf("cannot adopt historical sandbox resources: keep the original Core responsible for retained resources and install this release separately") - } - } - var localNode pgtype.UUID - if selected.LocalNodeID != "" { - id, err := parseConnectionGeneration(selected.LocalNodeID) - if err != nil { - return err - } - localNode = id - if previous.LocalNodeID.Valid && previous.LocalNodeID != id { - resources, err := q.CountRuntimeDeploymentResources(ctx) - if err != nil { - return err - } - if resources.Allocations != 0 || resources.Pending != 0 || !previous.AdmissionPaused || !selected.AdmissionPaused { - return fmt.Errorf("local sandbox node identity changed: restore its original state directory; replacement requires maintenance and no retained resources") - } - } - if !validRuntimeDigest(selected.LocalCredentialSHA256) { - return ErrInvalidInput - } - if err := validateRuntimeNode("Local", selected.LocalMaxActive, selected.LocalMaxRetained); err != nil { - return err - } - n, err := q.GetRuntimeNode(ctx, id) - if errors.Is(err, pgx.ErrNoRows) { - _, err = q.InsertRuntimeNode(ctx, sqlc.InsertRuntimeNodeParams{ID: id, InstallationID: installation, Name: "Local", BackendFingerprint: selected.BackendFingerprint, CredentialSha256: selected.LocalCredentialSHA256, MaxActive: int32(selected.LocalMaxActive), MaxRetained: int32(selected.LocalMaxRetained)}) - } else if err == nil { - if n.InstallationID != installation || n.BackendFingerprint != selected.BackendFingerprint || n.CredentialSha256 != selected.LocalCredentialSHA256 { - return fmt.Errorf("local sandbox node identity does not match the retained backend") - } - _, err = q.UpdateRuntimeNode(ctx, sqlc.UpdateRuntimeNodeParams{ID: id, Name: n.Name, MaxActive: int32(selected.LocalMaxActive), MaxRetained: int32(selected.LocalMaxRetained)}) - } - if err != nil { - return err - } - } - return q.SetRuntimeManagerDeployment(ctx, sqlc.SetRuntimeManagerDeploymentParams{ProviderKind: selected.ProviderKind, LocalNodeID: localNode}) -} diff --git a/services/core/internal/store/runtime_node_generations.go b/services/core/internal/store/runtime_node_generations.go deleted file mode 100644 index e69a8f8b7..000000000 --- a/services/core/internal/store/runtime_node_generations.go +++ /dev/null @@ -1,163 +0,0 @@ -package store - -import ( - "context" - "encoding/json" - "errors" - "math" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" -) - -func nodeGenerationSpec(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, generation uint64) (sandbox.DeploymentSpec, error) { - var spec sandbox.DeploymentSpec - if generation == 0 || generation > math.MaxInt64 { - return spec, ErrInvalidInput - } - row, err := q.GetNodeGenerationSpecification(ctx, int64(generation)) - if err != nil { - return spec, err - } - if row.ProviderKind != d.ProviderKind || json.Unmarshal(row.Specification, &spec) != nil || providers.ValidateSpecification(d.ProviderKind, spec) != nil { - return spec, ErrRuntimeSpecificationMismatch - } - return spec, nil -} - -func nodeConnection(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, nodeID, connectionID string, epoch uint64) (sqlc.RuntimeNode, error) { - id, err := parseConnectionGeneration(nodeID) - if err != nil { - return sqlc.RuntimeNode{}, ErrRuntimeNodeCredential - } - n, err := q.GetRuntimeNode(ctx, id) - if err != nil { - return n, err - } - if runtimeUUID(n.ConnectionID) != connectionID || n.ConnectedEpoch != int64(epoch) || epoch == 0 || epoch > math.MaxInt64 || d.OwnerEpoch != int64(epoch) || n.InstallationID != d.InstallationID || d.Mode != "nodes" { - return n, ErrRuntimeNodeCredential - } - return n, nil -} - -// RuntimeNodeRetention is a bounded read under the same serialization as pin -// promotion and placement. A drop cannot subsequently gain fresh old ownership. -func (s *Store) RuntimeNodeRetention(ctx context.Context, nodeID, connectionID string, epoch uint64, refs []sandbox.GenerationReference) (sandbox.NodeDeployment, []sandbox.GenerationRetention, error) { - var deployment sandbox.NodeDeployment - grants := make([]sandbox.GenerationRetention, 0, len(refs)) - if len(refs) > 8 { - return deployment, nil, ErrInvalidInput - } - ctx, cancel := context.WithTimeout(ctx, pgunit.ExecutionTimeout) - defer cancel() - err := s.runtimeDeploymentTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { - n, err := nodeConnection(ctx, q, d, nodeID, connectionID, epoch) - if err != nil { - return err - } - deployment.Generation = uint64(d.Generation) - spec, err := nodeGenerationSpec(ctx, q, d, uint64(d.Generation)) - if err != nil { - return err - } - deployment.SpecificationDigest = spec.Digest(d.ProviderKind) - if n.ReadyGeneration.Valid { - pin := uint64(n.ReadyGeneration.Int64) - deployment.ServingGeneration = &pin - } - seen := map[uint64]bool{} - for _, ref := range refs { - if ref.Generation == 0 || ref.Generation > math.MaxInt64 || !validRuntimeDigest(ref.SpecificationDigest) || seen[ref.Generation] { - return ErrInvalidInput - } - seen[ref.Generation] = true - kept, err := q.NodeGenerationKept(ctx, sqlc.NodeGenerationKeptParams{NodeID: n.ID, Generation: int64(ref.Generation)}) - if err != nil { - return err - } - if kept { - spec, err := nodeGenerationSpec(ctx, q, d, ref.Generation) - if err != nil { - return err - } - if spec.Digest(d.ProviderKind) != ref.SpecificationDigest { - return ErrRuntimeSpecificationMismatch - } - } - if !kept { - if err := q.DeleteNodeGenerationStatus(ctx, sqlc.DeleteNodeGenerationStatusParams{NodeID: n.ID, Generation: int64(ref.Generation)}); err != nil { - return err - } - } - grants = append(grants, sandbox.GenerationRetention{GenerationReference: ref, Keep: kept}) - } - return nil - }) - if err != nil { - return sandbox.NodeDeployment{}, nil, err - } - return deployment, grants, nil -} - -func (s *Store) HeartbeatRuntimeNodeGenerations(ctx context.Context, nodeID, connectionID string, epoch uint64, health RuntimeNodeHealth, statuses []sandbox.GenerationStatus) error { - if len(statuses) > 8 { - return ErrInvalidInput - } - return s.heartbeatRuntimeNode(ctx, nodeID, connectionID, epoch, health, statuses, 2) -} - -func recordNodeGenerations(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, n sqlc.RuntimeNode, statuses []sandbox.GenerationStatus, protocol int32) error { - seen := map[uint64]bool{} - for _, status := range statuses { - if status.Generation == 0 || status.Generation > math.MaxInt64 || !validRuntimeDigest(status.SpecificationDigest) || seen[status.Generation] || (status.State != "ready" && status.State != "preparing" && status.State != "failed") || status.State == "ready" && status.Diagnostic != "" { - return ErrInvalidInput - } - seen[status.Generation] = true - kept, err := q.NodeGenerationKept(ctx, sqlc.NodeGenerationKeptParams{NodeID: n.ID, Generation: int64(status.Generation)}) - if err != nil { - return err - } - // A sparse report may race collection of a skipped target. It creates no - // readiness or pin; only the later correlated retention reply can drop it. - if !kept { - continue - } - spec, err := nodeGenerationSpec(ctx, q, d, status.Generation) - if err != nil { - return err - } - if spec.Digest(d.ProviderKind) != status.SpecificationDigest { - return ErrRuntimeSpecificationMismatch - } - diagnostic := sandbox.NormalizeNodeDiagnostic(status.Diagnostic) - if err := q.UpsertNodeGenerationStatus(ctx, sqlc.UpsertNodeGenerationStatusParams{NodeID: n.ID, Generation: int64(status.Generation), SpecificationDigest: status.SpecificationDigest, ConnectionID: n.ConnectionID, OwnerEpoch: d.OwnerEpoch, State: status.State, Diagnostic: diagnostic}); err != nil { - return err - } - if status.State == "ready" && status.Generation == uint64(d.Generation) { - if err := q.PromoteNodeServingGeneration(ctx, sqlc.PromoteNodeServingGenerationParams{ID: n.ID, ReadyGeneration: pgtype.Int8{Int64: int64(status.Generation), Valid: true}}); err != nil { - return err - } - } - } - return q.RefreshNodeServingReadiness(ctx, sqlc.RefreshNodeServingReadinessParams{ID: n.ID, ProtocolVersion: protocol}) -} - -// Enrollment identity survives collection of its old specification. Whenever -// that specification is still authoritative, its exact digest must agree. -func validateNodeEnrollmentIdentity(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, n sqlc.RuntimeNode) error { - if n.DeploymentGeneration <= 0 || n.DeploymentGeneration > d.Generation || !validRuntimeDigest(n.SpecificationDigest) { - return ErrRuntimeSpecificationMismatch - } - spec, err := nodeGenerationSpec(ctx, q, d, uint64(n.DeploymentGeneration)) - if errors.Is(err, pgx.ErrNoRows) && n.DeploymentGeneration < d.Generation { - return nil - } - if err != nil || spec.Digest(d.ProviderKind) != n.SpecificationDigest { - return ErrRuntimeSpecificationMismatch - } - return nil -} diff --git a/services/core/internal/store/runtime_node_generations_test.go b/services/core/internal/store/runtime_node_generations_test.go index a13cbf748..f0e2d8e00 100644 --- a/services/core/internal/store/runtime_node_generations_test.go +++ b/services/core/internal/store/runtime_node_generations_test.go @@ -1,22 +1,21 @@ package store import ( - "database/sql" "errors" - "github.com/jackc/pgx/v5/stdlib" - "github.com/pressly/goose/v3" - "os" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/google/uuid" ) -func changeNodeTarget(t *testing.T, w *Store, view RuntimeDeploymentView, input SandboxDeploymentSetupRequest) (RuntimeDeploymentView, SandboxDeploymentSetupRequest) { +func changeNodeTarget(t *testing.T, w *Store, view deployment.View, input sandbox.Selection) (deployment.View, sandbox.Selection) { t.Helper() input.Resources.CPUs++ - next, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: view.Generation}) + request := input + request.ExpectedGeneration = view.Generation + next, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, request) if err != nil { t.Fatal(err) } @@ -26,9 +25,9 @@ func changeNodeTarget(t *testing.T, w *Store, view RuntimeDeploymentView, input return next, input } -func generationHeartbeat(t *testing.T, s *Store, node RuntimeNodeEnrollment, connection string, view RuntimeDeploymentView, state string) { +func generationHeartbeat(t *testing.T, s *Store, node deployment.Enrollment, connection string, view deployment.View, state string) { t.Helper() - err := s.HeartbeatRuntimeNodeGenerations(t.Context(), node.NodeID, connection, view.OwnerEpoch, RuntimeNodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: state}}) + err := deploymentService(t, s).HeartbeatGenerations(t.Context(), node.NodeID, connection, view.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: state}}) if err != nil { t.Fatal(err) } @@ -48,6 +47,7 @@ func TestNodeGenerationsCapacityFallbackAndImmutablePending(t *testing.T) { for _, provider := range []string{"microsandbox", "docker"} { t.Run(provider, func(t *testing.T) { s, w, first, input := webSpecificationFixture(t, provider) + nodes := deploymentService(t, s) a := specificationNode(t, s, first) b := specificationNode(t, s, first) ca := onlineManagerNode(t, s, a.NodeID) @@ -58,15 +58,15 @@ func TestNodeGenerationsCapacityFallbackAndImmutablePending(t *testing.T) { t.Fatal(err) } pendingNode, pendingGeneration := placedGeneration(t, s, pending) - token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 1, MaxRetained: 2})) + token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 2})) if err != nil { t.Fatal(err) } second, input := changeNodeTarget(t, w, first, input) - if _, err = s.RuntimeNodeConfiguration(t.Context(), "", token); err != nil { + if _, err = nodes.NodeConfiguration(t.Context(), "", token, 0); err != nil { t.Fatal("update retired enrollment", err) } - if _, err = s.AuthenticateRuntimeNode(t.Context(), a.NodeID, a.Credential); err != nil { + if _, err = nodes.AuthenticateNode(t.Context(), a.NodeID, a.Credential); err != nil { t.Fatal("update retired node", err) } generationHeartbeat(t, s, a, ca, second, "preparing") @@ -99,11 +99,11 @@ func TestNodeGenerationsCapacityFallbackAndImmutablePending(t *testing.T) { third, _ := changeNodeTarget(t, w, second, input) // B finishes a superseded target late: it may describe ownership but cannot adopt it. generationHeartbeat(t, s, b, cb, second, "ready") - nodes, err := s.ListRuntimeNodes(t.Context()) + list, err := nodes.ListNodes(t.Context()) if err != nil { t.Fatal(err) } - for _, n := range nodes { + for _, n := range list { want := uint64(1) if n.ID == a.NodeID { want = 2 @@ -123,12 +123,12 @@ func TestNodeGenerationsCapacityFallbackAndImmutablePending(t *testing.T) { if err != nil || n != pendingNode || g != 1 { t.Fatal(n, g, err) } - for _, v := range []RuntimeDeploymentView{first, second, third} { + for _, v := range []deployment.View{first, second, third} { target := a if v.Generation == 1 { target = b } - got, err := s.RuntimeNodeGenerationConfiguration(t.Context(), target.NodeID, target.Credential, v.Generation) + got, err := nodes.NodeConfiguration(t.Context(), target.NodeID, target.Credential, v.Generation) if err != nil || got.SpecificationDigest != v.SpecificationDigest { t.Fatal("kept generation unrecoverable", v.Generation, err) } @@ -139,10 +139,11 @@ func TestNodeGenerationsCapacityFallbackAndImmutablePending(t *testing.T) { func TestNodeGenerationsReconnectAndV1Fallback(t *testing.T) { s, w, first, input := webSpecificationFixture(t, "docker") + service := deploymentService(t, s) node := specificationNode(t, s, first) old := onlineManagerNode(t, s, node.NodeID) second, _ := changeNodeTarget(t, w, first, input) - nodes, err := s.ListRuntimeNodes(t.Context()) + nodes, err := service.ListNodes(t.Context()) if err != nil || !nodes[0].ProviderReady || nodes[0].Rollout.State != "update_required" { t.Fatal(nodes, err) } @@ -150,20 +151,20 @@ func TestNodeGenerationsReconnectAndV1Fallback(t *testing.T) { t.Fatal("v1 lost fallback", err) } connection := uuid.NewString() - if err = s.ConnectRuntimeNode(t.Context(), node.NodeID, connection, first.OwnerEpoch); err != nil { + if err = service.ConnectNode(t.Context(), node.NodeID, connection, first.OwnerEpoch); err != nil { t.Fatal(err) } - if err = s.HeartbeatRuntimeNodeGenerations(t.Context(), node.NodeID, old, first.OwnerEpoch, RuntimeNodeHealth{}, []sandbox.GenerationStatus{{Generation: second.Generation, SpecificationDigest: second.SpecificationDigest, State: "ready"}}); !errors.Is(err, ErrRuntimeNodeCredential) { + if err = service.HeartbeatGenerations(t.Context(), node.NodeID, old, first.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: second.Generation, SpecificationDigest: second.SpecificationDigest, State: "ready"}}); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("old connection qualified", err) } - nodes, err = s.ListRuntimeNodes(t.Context()) + nodes, err = service.ListNodes(t.Context()) if err != nil || nodes[0].ProviderReady || *nodes[0].Rollout.ReadyGeneration != 1 { t.Fatal("reconnect inherited readiness or lost pin", nodes, err) } - if _, err = s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, ErrRuntimeNodeUnavailable) { + if _, err = s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, deployment.ErrNodeUnavailable) { t.Fatal("unconfirmed connection admitted", err) } - if err = s.HeartbeatRuntimeNode(t.Context(), node.NodeID, connection, first.OwnerEpoch, RuntimeNodeHealth{ProviderReady: true}); err != nil { + if err = service.Heartbeat(t.Context(), node.NodeID, connection, first.OwnerEpoch, deployment.NodeHealth{ProviderReady: true}); err != nil { t.Fatal(err) } if _, err = s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); err != nil { @@ -171,65 +172,16 @@ func TestNodeGenerationsReconnectAndV1Fallback(t *testing.T) { } } -func TestNodeGenerationDowngradePreservesServingProtocol(t *testing.T) { - for _, mode := range []string{"v2", "old_v1", "current_v1"} { - t.Run(mode, func(t *testing.T) { - s, w, first, input := webSpecificationFixture(t, "docker") - node := specificationNode(t, s, first) - switch mode { - case "v2": - generationHeartbeat(t, s, node, onlineManagerNode(t, s, node.NodeID), first, "ready") - case "old_v1": - changeNodeTarget(t, w, first, input) - } - db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) - defer db.Close() - migration, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) - if err != nil { - t.Fatal(err) - } - _, err = migration.DownTo(t.Context(), 81) - if mode == "current_v1" { - if err != nil { - t.Fatal("safe v1 downgrade refused", err) - } - } else { - if err == nil { - t.Fatal("downgrade discarded required node protocol") - } - // DownTo may have removed later, reversible migrations before the - // node protocol migration refused the downgrade. Restore the current - // schema before using this version of the Store to verify recovery. - if _, err = migration.Up(t.Context()); err != nil { - t.Fatal("refused downgrade could not restore current schema", err) - } - if _, err = s.RuntimeNodeGenerationConfiguration(t.Context(), node.NodeID, node.Credential, 1); err != nil { - t.Fatal("refused downgrade damaged retained recovery", err) - } - if err = s.RemoveRuntimeNode(t.Context(), node.NodeID); err != nil { - t.Fatal(err) - } - if _, err = migration.DownTo(t.Context(), 81); err != nil { - t.Fatal("removed node blocked downgrade", err) - } - } - if _, err = migration.Up(t.Context()); err != nil { - t.Fatal("node schema could not upgrade again", err) - } - }) - } -} - func TestNodeGenerationPreparationRefusalCreatesNoProvisionalOwnership(t *testing.T) { s, _, first, _ := webSpecificationFixture(t, "docker") node := specificationNode(t, s, first) connection := uuid.NewString() - if err := s.ConnectRuntimeNode(t.Context(), node.NodeID, connection, first.OwnerEpoch); err != nil { + if err := deploymentService(t, s).ConnectNode(t.Context(), node.NodeID, connection, first.OwnerEpoch); err != nil { t.Fatal(err) } generationHeartbeat(t, s, node, connection, first, "preparing") tenant := uuid.NewString() - if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())); !errors.Is(err, ErrSandboxNodesPreparing) { + if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())); !errors.Is(err, deployment.ErrNodesPreparing) { t.Fatal("actual preparation was not identified", err) } var sessions, placements int @@ -237,7 +189,7 @@ func TestNodeGenerationPreparationRefusalCreatesNoProvisionalOwnership(t *testin t.Fatal("refusal left provisional ownership", sessions, placements, err) } generationHeartbeat(t, s, node, connection, first, "failed") - if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())); !errors.Is(err, ErrRuntimeNodeUnavailable) { + if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())); !errors.Is(err, deployment.ErrNodeUnavailable) { t.Fatal("failed preparation advertised active work", err) } } diff --git a/services/core/internal/store/runtime_node_lifecycle_fixture_test.go b/services/core/internal/store/runtime_node_lifecycle_fixture_test.go index 77eaa43ad..66c3acae0 100644 --- a/services/core/internal/store/runtime_node_lifecycle_fixture_test.go +++ b/services/core/internal/store/runtime_node_lifecycle_fixture_test.go @@ -15,6 +15,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" @@ -76,6 +77,7 @@ func (p *nodeIsolationProvider) RunCommand(ctx context.Context, r sandbox.Refere type nodeIsolationFixture struct { t *testing.T store *store.Store + nodes *deployment.Service pool *pgxpool.Pool worker *execution.Worker provider *nodeIsolationProvider @@ -122,7 +124,7 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { cp.mu.Unlock() server.Close() }) - f := &nodeIsolationFixture{initializationCancel: cancelPreparation, t: t, store: s, pool: pool, provider: p, key: uuid.NewString(), nodeA: uuid.NewString(), nodeB: uuid.NewString()} + f := &nodeIsolationFixture{initializationCancel: cancelPreparation, t: t, store: s, nodes: fixtureDeployment(t, db), pool: pool, provider: p, key: uuid.NewString(), nodeA: uuid.NewString(), nodeB: uuid.NewString()} // Keep restored compute awake throughout the isolation assertions. // The suspension setup explicitly dates its activity two minutes in the past. policy := &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Minute, Retention: time.Hour, MaxActive: 100, MaxRetained: 100} @@ -136,10 +138,7 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { t.Fatal(err) } t.Cleanup(f.stop) - f.epoch, err = s.RuntimeOwnerEpoch(t.Context()) - if err != nil { - t.Fatal(err) - } + f.epoch = fixtureOwnerEpoch(t, db) f.enroll(f.nodeB) f.online(f.nodeA) f.online(f.nodeB) @@ -147,11 +146,11 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { } func (f *nodeIsolationFixture) enroll(id string) { f.t.Helper() - token, err := store.EnrollmentTestToken(f.store.CreateRuntimeEnrollment(f.t.Context(), store.RuntimeNodeCapacity{MaxActive: 100, MaxRetained: 100})) + token, err := store.EnrollmentTestToken(f.nodes.CreateEnrollment(f.t.Context(), deployment.Capacity{MaxActive: 100, MaxRetained: 100})) if err != nil { f.t.Fatal(err) } - _, err = f.store.EnrollRuntimeNode(f.t.Context(), token, store.RuntimeNodeEnrollment{DeploymentGeneration: 1, SpecificationDigest: store.SandboxDeploymentTestSpec("microsandbox").Digest("microsandbox"), NodeID: id, Credential: strings.Repeat("x", 64), Name: id, Provider: "microsandbox", BackendFingerprint: strings.Repeat("b", 64)}) + _, err = f.nodes.Enroll(f.t.Context(), token, deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: store.SandboxDeploymentTestSpec("microsandbox").Digest("microsandbox"), NodeID: id, Credential: strings.Repeat("x", 64), Name: id, Provider: "microsandbox", BackendFingerprint: strings.Repeat("b", 64)}) if err != nil { f.t.Fatal(err) } @@ -159,10 +158,10 @@ func (f *nodeIsolationFixture) enroll(id string) { func (f *nodeIsolationFixture) online(id string) { f.t.Helper() connection := uuid.NewString() - if err := f.store.ConnectRuntimeNode(f.t.Context(), id, connection, f.epoch); err != nil { + if err := f.nodes.ConnectNode(f.t.Context(), id, connection, f.epoch); err != nil { f.t.Fatal(err) } - if err := f.store.HeartbeatRuntimeNode(f.t.Context(), id, connection, f.epoch, store.RuntimeNodeHealth{ProviderReady: true}); err != nil { + if err := f.nodes.Heartbeat(f.t.Context(), id, connection, f.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { f.t.Fatal(err) } } @@ -197,13 +196,13 @@ func (f *nodeIsolationFixture) session(node string, initialize bool) (string, st f.t.Fatal(err) } for _, value := range others { - if err := f.store.HeartbeatRuntimeNode(f.t.Context(), value.id, value.connection, value.epoch, store.RuntimeNodeHealth{ProviderReady: false}); err != nil { + if err := f.nodes.Heartbeat(f.t.Context(), value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: false}); err != nil { f.t.Fatal(err) } } session, err := f.store.CreateSession(f.t.Context(), tenant, input) for _, value := range others { - if err := f.store.HeartbeatRuntimeNode(context.WithoutCancel(f.t.Context()), value.id, value.connection, value.epoch, store.RuntimeNodeHealth{ProviderReady: true}); err != nil { + if err := f.nodes.Heartbeat(context.WithoutCancel(f.t.Context()), value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { f.t.Fatal(err) } } diff --git a/services/core/internal/store/runtime_node_lifecycle_test.go b/services/core/internal/store/runtime_node_lifecycle_test.go index bbe8b36bc..ed9295804 100644 --- a/services/core/internal/store/runtime_node_lifecycle_test.go +++ b/services/core/internal/store/runtime_node_lifecycle_test.go @@ -142,7 +142,7 @@ func TestManagedNodesIsolateBlockedProviderAndInitialization(t *testing.T) { owner, err := f.store.GetRuntimeAllocation(t.Context(), ct, ce.ID) return err == nil && owner.State == "released", fmt.Sprintf("new node allocation=%s/%s err=%v", owner.State, owner.ComputePhase, err) }) - if err := f.store.RemoveRuntimeNode(t.Context(), nodeC); err != nil { + if err := f.nodes.RemoveNode(t.Context(), nodeC); err != nil { t.Fatal(err) } f.stop() diff --git a/services/core/internal/store/runtime_node_presence.go b/services/core/internal/store/runtime_node_presence.go deleted file mode 100644 index eb54fd2e9..000000000 --- a/services/core/internal/store/runtime_node_presence.go +++ /dev/null @@ -1,110 +0,0 @@ -package store - -import ( - "context" - "encoding/json" - "errors" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/jackc/pgx/v5" -) - -func (s *Store) ConnectRuntimeNode(ctx context.Context, nodeID, connectionID string, epoch uint64) error { - id, err := parseConnectionGeneration(nodeID) - if err != nil { - return err - } - connection, err := parseConnectionGeneration(connectionID) - if err != nil { - return err - } - // A canceled autocommit UPDATE may still finish on PostgreSQL after pgx - // returns. An explicit transaction cannot publish that late write. - return s.pooled.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { - changed, err := s.queries.WithTx(tx).ConnectRuntimeNode(ctx, sqlc.ConnectRuntimeNodeParams{ID: id, ConnectionID: connection, OwnerEpoch: int64(epoch)}) - if err != nil { - return err - } - if changed != 1 { - return ErrRuntimeNodeCredential - } - return ctx.Err() - }) -} -func (s *Store) HeartbeatRuntimeNode(ctx context.Context, nodeID, connectionID string, epoch uint64, health RuntimeNodeHealth) error { - return s.heartbeatRuntimeNode(ctx, nodeID, connectionID, epoch, health, nil, 1) -} -func (s *Store) heartbeatRuntimeNode(ctx context.Context, nodeID, connectionID string, epoch uint64, health RuntimeNodeHealth, statuses []sandbox.GenerationStatus, protocol int32) error { - id, err := parseConnectionGeneration(nodeID) - if err != nil { - return err - } - connection, err := parseConnectionGeneration(connectionID) - if err != nil { - return err - } - // A diagnostic explains unreadiness only. Unknown node values, including - // arbitrary text, are stored as provider_unavailable; empty stays empty. - health.Diagnostic = sandbox.NormalizeNodeDiagnostic(health.Diagnostic) - if health.ProviderReady { - health.Diagnostic = "" - } - for _, value := range []*int64{health.CPUCount, health.AvailableMemoryBytes, health.AvailableDiskBytes} { - if value != nil && *value < 0 { - return ErrInvalidInput - } - } - if err := validateRuntimeNodeHost(health.Host); err != nil { - return err - } - raw, err := json.Marshal(runtimeNodeHealthRecord{RuntimeNodeHealth: health, Host: health.Host}) - if err != nil { - return err - } - return s.runtimeDeploymentTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { - n, err := nodeConnection(ctx, q, d, nodeID, connectionID, epoch) - if err != nil { - return err - } - changed, err := q.HeartbeatRuntimeNode(ctx, sqlc.HeartbeatRuntimeNodeParams{ID: id, ConnectionID: connection, OwnerEpoch: int64(epoch), ProviderReady: health.ProviderReady, Health: raw}) - if err != nil { - return err - } - if changed != 1 { - return ErrRuntimeNodeCredential - } - if protocol == 1 { - state := "failed" - if health.ProviderReady { - state = "ready" - } - statuses = []sandbox.GenerationStatus{{Generation: uint64(n.DeploymentGeneration), SpecificationDigest: n.SpecificationDigest, State: state, Diagnostic: health.Diagnostic}} - } - return recordNodeGenerations(ctx, q, d, n, statuses, protocol) - }) -} -func (s *Store) DisconnectRuntimeNode(ctx context.Context, nodeID, connectionID string, epoch uint64) error { - id, err := parseConnectionGeneration(nodeID) - if err != nil { - return err - } - connection, err := parseConnectionGeneration(connectionID) - if err != nil { - return err - } - return s.pooled.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { - q := s.queries.WithTx(tx) - // A Connect COMMIT may be uncertain. Wait on the node regardless of - // the visible connection, then fence cleanup in a fresh statement snapshot. - if _, err := q.LockRuntimeNodePresence(ctx, id); errors.Is(err, pgx.ErrNoRows) { - return nil - } else if err != nil { - return err - } - if err := q.DisconnectRuntimeNode(ctx, sqlc.DisconnectRuntimeNodeParams{ID: id, ConnectionID: connection, OwnerEpoch: int64(epoch)}); err != nil { - return err - } - return ctx.Err() - }) -} diff --git a/services/core/internal/store/runtime_node_presence_test.go b/services/core/internal/store/runtime_node_presence_test.go deleted file mode 100644 index de00dd178..000000000 --- a/services/core/internal/store/runtime_node_presence_test.go +++ /dev/null @@ -1,233 +0,0 @@ -package store - -import ( - "context" - "errors" - "strings" - "testing" - "time" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" - "github.com/jackc/pgx/v5/pgxpool" -) - -func runtimePresenceContext(t *testing.T) context.Context { - t.Helper() - ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) - t.Cleanup(cancel) - return ctx -} - -func runtimePresenceOtherNode(t *testing.T, s *Store) string { - t.Helper() - token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 1, MaxRetained: 1})) - if err != nil { - t.Fatal(err) - } - input := RuntimeNodeEnrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: uuid.NewString(), Credential: strings.Repeat("x", 64), Name: "presence-other", Provider: "docker", BackendFingerprint: strings.Repeat("b", 64)} - if _, err := s.EnrollRuntimeNode(t.Context(), token, input); err != nil { - t.Fatal(err) - } - return input.NodeID -} - -func assertRuntimePresence(t *testing.T, s *Store, node, want string) { - t.Helper() - var actual pgtype.UUID - // Wait for any canceled transaction to roll back before reading its outcome. - if err := s.pool.QueryRow(runtimePresenceContext(t), "SELECT connection_id FROM runtime_nodes WHERE id=$1 FOR UPDATE", node).Scan(&actual); err != nil { - t.Fatal(err) - } - if runtimeUUID(actual) != want { - t.Fatalf("presence = %q, want %q", runtimeUUID(actual), want) - } -} - -func TestRuntimeNodePresenceCanceledBlockedConnect(t *testing.T) { - s, _, d := managerFixture(t, 1, 1) - other := runtimePresenceOtherNode(t, s) - epoch := managerEpoch(t, s) - ctx := runtimePresenceContext(t) - runtimeSuspensionSQL(t, s.pool, "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.LocalNodeID) - lock, err := s.pool.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer lock.Rollback(ctx) - var blocker int32 - if err := lock.QueryRow(ctx, "SELECT pg_backend_pid() FROM runtime_nodes WHERE id=$1 FOR UPDATE", d.LocalNodeID).Scan(&blocker); err != nil { - t.Fatal(err) - } - connecting, cancel := context.WithCancel(ctx) - defer cancel() - connection := uuid.NewString() - done := make(chan error, 1) - go func() { done <- s.ConnectRuntimeNode(connecting, d.LocalNodeID, connection, epoch) }() - runtimeSuspensionWaitBlocked(t, ctx, s.pool, blocker, done) - var writer int32 - if err := s.pool.QueryRow(ctx, "SELECT pid FROM pg_stat_activity WHERE $1=ANY(pg_blocking_pids(pid)) LIMIT 1", blocker).Scan(&writer); err != nil { - t.Fatal(err) - } - // Another node does not share the blocked presence transaction's lock. - otherConnection := uuid.NewString() - if err := s.ConnectRuntimeNode(ctx, other, otherConnection, epoch); err != nil { - t.Fatal(err) - } - if err := s.DisconnectRuntimeNode(ctx, other, otherConnection, epoch); err != nil { - t.Fatal(err) - } - cancel() - select { - case err := <-done: - if !errors.Is(err, context.Canceled) { - t.Fatal("blocked connect cancellation", err) - } - case <-ctx.Done(): - t.Fatal("blocked connect did not cancel") - } - if err := lock.Rollback(ctx); err != nil { - t.Fatal(err) - } - // pgx cancellation can return before PostgreSQL stops the original statement. - // Observe backend exit so a late autocommit cannot escape the assertion. - for { - var gone bool - if err := s.pool.QueryRow(ctx, "SELECT NOT EXISTS (SELECT 1 FROM pg_stat_activity WHERE pid=$1)", writer).Scan(&gone); err != nil { - t.Fatal(err) - } - if gone { - break - } - select { - case <-ctx.Done(): - t.Fatal("canceled backend did not finish") - case <-time.After(time.Millisecond): - } - } - assertRuntimePresence(t, s, d.LocalNodeID, "") -} - -type cancelPresenceAfterUpdate struct{ cancel context.CancelFunc } - -func (trace cancelPresenceAfterUpdate) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { - return ctx -} -func (trace cancelPresenceAfterUpdate) TraceQueryEnd(_ context.Context, _ *pgx.Conn, result pgx.TraceQueryEndData) { - if result.Err == nil && result.CommandTag.Update() { - trace.cancel() - } -} - -func TestRuntimeNodePresenceCanceledBeforeCommit(t *testing.T) { - s, _, d := managerFixture(t, 1, 1) - epoch := managerEpoch(t, s) - runtimeSuspensionSQL(t, s.pool, "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.LocalNodeID) - ctx, cancel := context.WithCancel(runtimePresenceContext(t)) - defer cancel() - cfg := s.pool.Config() - // Cancel after PostgreSQL acknowledges UPDATE, before Store can commit it. - cfg.ConnConfig.Tracer = cancelPresenceAfterUpdate{cancel: cancel} - pool, err := pgxpool.NewWithConfig(t.Context(), cfg) - if err != nil { - t.Fatal(err) - } - defer pool.Close() - if err := New(pool).ConnectRuntimeNode(ctx, d.LocalNodeID, uuid.NewString(), epoch); !errors.Is(err, context.Canceled) { - t.Fatal("canceled UPDATE published presence", err) - } - assertRuntimePresence(t, s, d.LocalNodeID, "") -} - -func TestRuntimeNodePresenceDisconnectWaitsForCommit(t *testing.T) { - for _, guard := range []string{"matching", "newer_connection", "newer_epoch"} { - t.Run(guard, func(t *testing.T) { - s, _, d := managerFixture(t, 1, 1) - other := runtimePresenceOtherNode(t, s) - epoch := managerEpoch(t, s) - ctx := runtimePresenceContext(t) - runtimeSuspensionSQL(t, s.pool, "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.LocalNodeID) - pending, err := s.pool.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer pending.Rollback(ctx) - current, cleanup := uuid.NewString(), "" - cleanup = current - cleanupEpoch := epoch - want := "" - if guard == "newer_connection" { - cleanup = uuid.NewString() - want = current - } - if guard == "newer_epoch" { - cleanupEpoch-- - want = current - } - if _, err := pending.Exec(ctx, "UPDATE runtime_nodes SET connection_id=$2,connected_epoch=$3 WHERE id=$1", d.LocalNodeID, current, epoch); err != nil { - t.Fatal(err) - } - var blocker int32 - if err := pending.QueryRow(ctx, "SELECT pg_backend_pid()").Scan(&blocker); err != nil { - t.Fatal(err) - } - done := make(chan error, 1) - go func() { done <- s.DisconnectRuntimeNode(ctx, d.LocalNodeID, cleanup, cleanupEpoch) }() - runtimeSuspensionWaitBlocked(t, ctx, s.pool, blocker, done) - otherConnection := uuid.NewString() - if err := s.ConnectRuntimeNode(ctx, other, otherConnection, epoch); err != nil { - t.Fatal(err) - } - if err := s.DisconnectRuntimeNode(ctx, other, otherConnection, epoch); err != nil { - t.Fatal(err) - } - if err := pending.Commit(ctx); err != nil { - t.Fatal(err) - } - select { - case err := <-done: - if err != nil { - t.Fatal(err) - } - case <-ctx.Done(): - t.Fatal("cleanup did not settle after commit") - } - assertRuntimePresence(t, s, d.LocalNodeID, want) - }) - } -} - -func TestRuntimeNodeDiagnosticReachesListAndDetail(t *testing.T) { - s, _, d := managerFixture(t, 1, 1) - connection := onlineManagerNode(t, s, d.LocalNodeID) - epoch := managerEpoch(t, s) - for _, tc := range []struct { - reported, want string - ready bool - }{ - {reported: "docker_unavailable", want: "docker_unavailable"}, - {reported: "capacity_insufficient", want: "capacity_insufficient"}, - // Older nodes send provider_unavailable or nothing; unknown text is never stored. - {reported: "provider_unavailable", want: "provider_unavailable"}, - {reported: "", want: ""}, - {reported: "dial unix /var/run/docker.sock: permission denied", want: "provider_unavailable"}, - {reported: "kvm_unavailable", want: "", ready: true}, - } { - if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, connection, epoch, RuntimeNodeHealth{ProviderReady: tc.ready, Diagnostic: tc.reported}); err != nil { - t.Fatal(tc.reported, err) - } - list, err := s.ListRuntimeNodes(t.Context()) - if err != nil || len(list) != 1 || list[0].Diagnostic != tc.want { - t.Fatal(tc.reported, list, err) - } - detail, err := s.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, "1h") - if err != nil || detail.Diagnostic != tc.want { - t.Fatal(tc.reported, detail.Diagnostic, err) - } - var stored string - if err := s.pool.QueryRow(t.Context(), "SELECT health::text FROM runtime_nodes WHERE id=$1", d.LocalNodeID).Scan(&stored); err != nil || strings.Contains(stored, "/var/run") { - t.Fatal("raw diagnostic stored", stored, err) - } - } -} diff --git a/services/core/internal/store/runtime_node_status.go b/services/core/internal/store/runtime_node_status.go deleted file mode 100644 index 580e38281..000000000 --- a/services/core/internal/store/runtime_node_status.go +++ /dev/null @@ -1,27 +0,0 @@ -package store - -import "context" - -// RuntimeNodeStatus reports only the authenticated node's Core-owned presence. -type RuntimeNodeStatus struct { - RuntimeNodeIdentity - Connected bool `json:"connected"` - ProviderReady bool `json:"provider_ready"` -} - -func (s *Store) RuntimeNodeStatus(ctx context.Context, nodeID, credential string) (RuntimeNodeStatus, error) { - identity, err := s.AuthenticateRuntimeNode(ctx, nodeID, credential) - if err != nil { - return RuntimeNodeStatus{}, err - } - nodes, err := s.ListRuntimeNodes(ctx) - if err != nil { - return RuntimeNodeStatus{}, err - } - for _, node := range nodes { - if node.ID == identity.NodeID { - return RuntimeNodeStatus{RuntimeNodeIdentity: identity, Connected: node.Online, ProviderReady: node.Online && node.ProviderReady}, nil - } - } - return RuntimeNodeStatus{}, ErrRuntimeNodeCredential -} diff --git a/services/core/internal/store/runtime_node_status_test.go b/services/core/internal/store/runtime_node_status_test.go deleted file mode 100644 index abbf3c372..000000000 --- a/services/core/internal/store/runtime_node_status_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package store - -import ( - "encoding/json" - "errors" - "strings" - "testing" -) - -func TestRuntimeNodeStatusUsesAuthenticatedFreshPresence(t *testing.T) { - s, _, d := managerFixture(t, 1, 4) - status := func(connected, ready bool) { - t.Helper() - got, err := s.RuntimeNodeStatus(t.Context(), d.LocalNodeID, "local-node-credential") - if err != nil || got.NodeID != d.LocalNodeID || got.Connected != connected || got.ProviderReady != ready { - t.Fatal(got, err) - } - raw, err := json.Marshal(got) - if err != nil || strings.Contains(string(raw), "credential") || strings.Contains(string(raw), "backend_fingerprint") { - t.Fatal("private identity leaked", string(raw), err) - } - } - status(true, true) - if _, err := s.RuntimeNodeStatus(t.Context(), d.LocalNodeID, "different-credential"); !errors.Is(err, ErrRuntimeNodeCredential) { - t.Fatal("status admitted another credential", err) - } - connection := onlineManagerNode(t, s, d.LocalNodeID) - if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, connection, managerEpoch(t, s), RuntimeNodeHealth{ProviderReady: false}); err != nil { - t.Fatal(err) - } - status(true, false) - if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, connection, managerEpoch(t, s), RuntimeNodeHealth{ProviderReady: true}); err != nil { - t.Fatal(err) - } - status(true, true) - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET last_seen_at=clock_timestamp()-interval '46 seconds' WHERE id=$1", d.LocalNodeID); err != nil { - t.Fatal(err) - } - status(false, false) - onlineManagerNode(t, s, d.LocalNodeID) - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_deployment SET owner_epoch=owner_epoch+1 WHERE singleton=true"); err != nil { - t.Fatal(err) - } - status(false, false) -} diff --git a/services/core/internal/store/runtime_node_types.go b/services/core/internal/store/runtime_node_types.go deleted file mode 100644 index bb0e934cc..000000000 --- a/services/core/internal/store/runtime_node_types.go +++ /dev/null @@ -1,158 +0,0 @@ -package store - -import ( - "encoding/json" - "errors" - "time" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" -) - -var ( - ErrSandboxNodesPreparing = errors.New("sandbox nodes are preparing the target generation") - ErrRuntimeNodeUnavailable = errors.New("sandbox node unavailable") - ErrRuntimeNodeInUse = errors.New("sandbox node retains resources") - ErrRuntimeNodeCredential = errors.New("invalid sandbox node credential") - ErrRuntimeLocalNodeConfigured = errors.New("local sandbox node is enabled in deployment configuration") - // ErrRuntimeNodeAddressMismatch rejects an enrollment whose Core address is not - // the installation public URL. The token stays unconsumed. - ErrRuntimeNodeAddressMismatch = errors.New("sandbox node Core address differs from the public URL") -) - -type RuntimeNodeIdentity struct { - SpecificationDigest string `json:"specification_digest"` - DeploymentGeneration uint64 `json:"deployment_generation"` - NodeID string `json:"node_id"` - InstallationID string `json:"installation_id"` - Provider string `json:"provider"` - BackendFingerprint string `json:"-"` - MaxActive int `json:"max_active"` - MaxRetained int `json:"max_retained"` -} - -// RuntimeNodeEnrollmentToken is an issued one-use node enrollment token. ID is a -// public, non-secret handle that never authenticates; the node the token registers -// reports it as enrollment_id. -type RuntimeNodeEnrollmentToken struct { - Token string - ExpiresAt time.Time - ID string -} -type RuntimeNodeCapacity struct { - MaxActive int `json:"max_active"` - MaxRetained int `json:"max_retained"` -} - -type RuntimeNodeEnrollment struct { - SpecificationDigest string `json:"specification_digest"` - DeploymentGeneration uint64 `json:"deployment_generation"` - NodeID string `json:"node_id"` - Credential string `json:"credential"` - Name string `json:"name"` - Provider string `json:"provider"` - BackendFingerprint string `json:"backend_fingerprint"` - // The Core origin this node stores and connects to, such as https://core.example. It must equal the installation public URL; otherwise enrollment gets 409 sandbox_node_address_mismatch and the token stays unused. - CoreURL string `json:"core_url"` -} -type RuntimeNodeHealth struct { - Host *RuntimeNodeHost `json:"-"` - // Fixed reason for the last reported unreadiness; absent while the provider is ready. Clients treat an unknown value as provider_unavailable. - Diagnostic string `json:"diagnostic,omitempty" enums:"provider_unavailable,docker_unavailable,docker_limits_unsupported,runtime_download_failed,runtime_image_unavailable,kvm_unavailable,microsandbox_artifacts_unavailable,capacity_insufficient"` - ProviderReady bool `json:"provider_ready"` - CPUCount *int64 `json:"cpu_count"` - AvailableMemoryBytes *int64 `json:"available_memory_bytes"` - AvailableDiskBytes *int64 `json:"available_disk_bytes"` -} -type RuntimeNode struct { - Rollout SandboxNodeRollout `json:"rollout"` - RuntimeNodeHealth - Running int64 `json:"running"` - Snapshots int64 `json:"snapshots"` - ID string `json:"id"` - Name string `json:"name"` - Provider string `json:"provider"` - Online bool `json:"online"` - LastSeenAt *time.Time `json:"last_seen_at"` - MaxActive int `json:"max_active"` - MaxRetained int `json:"max_retained"` - Active int64 `json:"active"` - Reserved int64 `json:"reserved"` - Retained int64 `json:"retained"` - CleanupPending int64 `json:"cleanup_pending"` - CreatedAt time.Time `json:"created_at"` - // The Core address this node enrolled with. A node whose address differs - // from the installation public URL receives no new sandboxes; re-add it. - CoreURL string `json:"core_url"` - // The enrollment_id of the command that registered this node (POST /core/v1/sandbox/enrollment-tokens); null for nodes enrolled before Core recorded it. - EnrollmentID *string `json:"enrollment_id" extensions:"x-nullable"` -} -type RuntimeNodeUpdate struct { - Name string `json:"name"` - MaxActive int `json:"max_active"` - // Docker never suspends, so Core replaces this with max_active; microsandbox uses both limits. - MaxRetained int `json:"max_retained"` -} -type SandboxDeploymentResources struct { - Allocations int64 `json:"allocations"` - Pending int64 `json:"pending"` -} - -// SandboxSuspensionView is the idle suspension policy. Only microsandbox -// suspends sandboxes; Docker and E2B deployments return null. -type SandboxSuspensionView struct { - IdleSeconds int64 `json:"idle_seconds"` - RetentionSeconds int64 `json:"retention_seconds"` -} -type SandboxNodeRollout struct { - // Target preparation, independent of an old pin's serving readiness. - State string `json:"state" enums:"ready,preparing,failed,update_required,unknown"` - // Durable serving-generation pin; online and provider_ready still gate placement. - ReadyGeneration *uint64 `json:"ready_generation" extensions:"x-nullable"` - Diagnostic string `json:"diagnostic,omitempty" enums:"provider_unavailable,docker_unavailable,docker_limits_unsupported,runtime_download_failed,runtime_image_unavailable,kvm_unavailable,microsandbox_artifacts_unavailable,capacity_insufficient"` -} -type SandboxRolloutNodes struct { - Ready int64 `json:"ready"` - Preparing int64 `json:"preparing"` - Failed int64 `json:"failed"` - UpdateRequired int64 `json:"update_required"` - Unknown int64 `json:"unknown"` -} -type SandboxRollout struct { - State string `json:"state" enums:"settled,preparing"` - PreviousGenerationSandboxes int64 `json:"previous_generation_sandboxes"` - Nodes *SandboxRolloutNodes `json:"nodes" extensions:"x-nullable"` -} -type RuntimeDeploymentView struct { - Rollout SandboxRollout `json:"rollout"` - Specification *sandbox.DeploymentSpec `json:"specification,omitempty"` - SpecificationDigest string `json:"specification_digest,omitempty"` - Generation uint64 `json:"generation"` - Mode string `json:"mode"` - Resources SandboxDeploymentResources `json:"resources"` - Configuration json.RawMessage `json:"configuration,omitempty" swaggertype:"object"` - Metadata json.RawMessage `json:"metadata,omitempty" swaggertype:"object"` - CredentialConfigured bool `json:"credential_configured"` - // Idle suspension policy; microsandbox only, otherwise null. - Suspension *SandboxSuspensionView `json:"suspension" extensions:"x-nullable"` - InstallationID string `json:"installation_id"` - Provider string `json:"provider"` - Reset *SandboxResetView `json:"reset" extensions:"x-nullable"` - OwnerEpoch uint64 `json:"owner_epoch"` - // Read-only: the installation public URL (OAC_PUBLIC_URL), which nodes and sandboxes use to reach Core. The deployment API does not accept it. - CoreURL string `json:"core_url"` -} -type RuntimeNodeAllocation struct { - DeploymentGeneration uint64 `json:"deployment_generation"` - Diagnostic string `json:"diagnostic"` - ID string `json:"id"` - NodeID string `json:"node_id"` - TenantID string `json:"tenant_id"` - SessionID string `json:"session_id"` - EnvironmentID string `json:"environment_id"` - State string `json:"state"` - ComputePhase string `json:"compute_phase"` - // The time the allocation entered its current compute_phase, or null when unknown; an allocation that existed before Core recorded it reports null until its next phase change. For a suspended microsandbox allocation, this time plus the deployment's snapshot retention tells roughly when Core reclaims it. - ComputePhaseChangedAt *time.Time `json:"compute_phase_changed_at" extensions:"x-nullable"` - Initialization string `json:"initialization"` - CreatedAt time.Time `json:"created_at"` -} diff --git a/services/core/internal/store/runtime_nodes.go b/services/core/internal/store/runtime_nodes.go index a1e43cc93..605d2c833 100644 --- a/services/core/internal/store/runtime_nodes.go +++ b/services/core/internal/store/runtime_nodes.go @@ -2,104 +2,30 @@ package store import ( "context" - "crypto/rand" - "crypto/sha256" - "encoding/hex" - "encoding/json" "errors" - "strings" "time" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" ) -func runtimeTokenDigest(token string) string { - digest := sha256.Sum256([]byte(token)) - return hex.EncodeToString(digest[:]) -} -func validRuntimeDigest(value string) bool { - decoded, err := hex.DecodeString(value) - return err == nil && len(decoded) == 32 && hex.EncodeToString(decoded) == value +type RuntimeNodeAllocation struct { + DeploymentGeneration uint64 `json:"deployment_generation"` + Diagnostic string `json:"diagnostic"` + ID string `json:"id"` + NodeID string `json:"node_id"` + TenantID string `json:"tenant_id"` + SessionID string `json:"session_id"` + EnvironmentID string `json:"environment_id"` + State string `json:"state"` + ComputePhase string `json:"compute_phase"` + // The time the allocation entered its current compute_phase, or null when unknown; an allocation that existed before Core recorded it reports null until its next phase change. For a suspended microsandbox allocation, this time plus the deployment's snapshot retention tells roughly when Core reclaims it. + ComputePhaseChangedAt *time.Time `json:"compute_phase_changed_at" extensions:"x-nullable"` + Initialization string `json:"initialization"` + CreatedAt time.Time `json:"created_at"` } -func validateRuntimeNode(name string, active, retained int) error { - if strings.TrimSpace(name) == "" || len(name) > 128 || strings.ContainsAny(name, "\x00\r\n") { - return &AdminValidationError{Code: "invalid_name", Param: "name", MaxLength: 128, message: ErrInvalidInput.Error()} - } - if active < 1 || active > 1000000 { - return &AdminValidationError{Code: "invalid_node_capacity", Param: "max_active", message: ErrInvalidInput.Error()} - } - if retained < active || retained > 1000000 { - return &AdminValidationError{Code: "invalid_node_capacity", Param: "max_retained", message: ErrInvalidInput.Error()} - } - return nil -} -func (s *Store) runtimeManagerTransaction(ctx context.Context, apply func(*sqlc.Queries, sqlc.RuntimeDeployment) error) error { - return s.runtimeDeploymentTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { - if !runtimeDeploymentInitialized(d) { - return ErrRuntimeNodeUnavailable - } - return apply(q, d) - }) -} - -// runtimeDeploymentTransaction locks the deployment whether or not it is -// initialized. Node machine routes use it to authenticate their credential -// before reporting any deployment state, including an uninitialized one. -func (s *Store) runtimeDeploymentTransaction(ctx context.Context, apply func(*sqlc.Queries, sqlc.RuntimeDeployment) error) error { - return s.pooled.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { - q := s.queries.WithTx(tx) - deployment, err := q.LockRuntimeDeployment(ctx) - if err != nil { - return err - } - return apply(q, deployment) - }) -} - -func runtimeDeploymentInitialized(d sqlc.RuntimeDeployment) bool { - return d.InstallationID.Valid && d.ProviderKind != "" -} -func (s *Store) GetRuntimeDeployment(ctx context.Context) (RuntimeDeploymentView, error) { - return s.deploymentView(ctx, s.queries) -} - -// deploymentView reports the public URL as the deployment's read-only core_url. -func (s *Store) deploymentView(ctx context.Context, q *sqlc.Queries) (RuntimeDeploymentView, error) { - row, err := q.GetSandboxDeploymentSnapshot(ctx) - if err != nil { - return RuntimeDeploymentView{}, err - } - d := row.RuntimeDeployment - result, err := runtimeDeploymentView(d, s.publicURL) - if err != nil { - return RuntimeDeploymentView{}, err - } - if err := json.Unmarshal(row.Rollout, &result.Rollout); err != nil { - return RuntimeDeploymentView{}, err - } - result.Resources = SandboxDeploymentResources{Allocations: row.Allocations, Pending: row.Pending} - if d.ResetClear.Valid { - remaining := SandboxResetRemaining{} - if err := json.Unmarshal(row.Remaining, &remaining); err != nil { - return RuntimeDeploymentView{}, err - } - result.Reset = &SandboxResetView{Clear: d.ResetClear.String, RequestedAt: d.ResetRequestedAt.Time, - DeadlineAt: resetTimestamp(d.ResetDeadlineAt), ForcedAt: resetTimestamp(d.ResetForcedAt), Remaining: remaining} - } - return result, nil -} - -func (s *Store) RuntimeOwnerEpoch(ctx context.Context) (uint64, error) { - d, err := s.queries.GetRuntimeDeployment(ctx) - return uint64(d.OwnerEpoch), err -} func runtimeUUID(id pgtype.UUID) string { if !id.Valid { return "" @@ -107,227 +33,6 @@ func runtimeUUID(id pgtype.UUID) string { return uuid.UUID(id.Bytes).String() } -func optionalUUID(id pgtype.UUID) *string { - if !id.Valid { - return nil - } - value := runtimeUUID(id) - return &value -} -func (s *Store) ListRuntimeNodes(ctx context.Context) ([]RuntimeNode, error) { - rows, err := s.queries.ListRuntimeNodes(ctx, pgtype.UUID{}) - if err != nil { - return nil, err - } - return runtimeNodeViews(rows) -} - -func runtimeNodeViews(rows []sqlc.ListRuntimeNodesRow) ([]RuntimeNode, error) { - out := make([]RuntimeNode, 0, len(rows)) - for _, n := range rows { - var seen *time.Time - if n.LastSeenAt.Valid { - value := n.LastSeenAt.Time - seen = &value - } - var health RuntimeNodeHealth - if err := json.Unmarshal(n.Health, &health); err != nil { - return nil, err - } - health.ProviderReady = n.Online && n.ServingReady - out = append(out, RuntimeNode{Rollout: nodeRollout(n), RuntimeNodeHealth: health, Running: n.Running, Snapshots: n.Snapshots, ID: runtimeUUID(n.ID), Name: n.Name, CoreURL: n.CoreUrl, EnrollmentID: optionalUUID(n.EnrollmentID), Provider: n.ProviderKind, Online: n.Online, LastSeenAt: seen, MaxActive: int(n.MaxActive), MaxRetained: providers.RetainedLimit(n.ProviderKind, int(n.MaxActive), int(n.MaxRetained)), Active: n.Active, Reserved: n.Reserved, Retained: n.Retained, CleanupPending: n.CleanupPending, CreatedAt: n.CreatedAt.Time}) - } - return out, nil -} - -// CreateRuntimeEnrollment issues a one-use enrollment token. Its ID is a public, -// non-secret handle: the node the token registers reports it as enrollment_id. -func (s *Store) CreateRuntimeEnrollment(ctx context.Context, capacity RuntimeNodeCapacity) (RuntimeNodeEnrollmentToken, error) { - // The retained limit depends on the provider, so the transaction checks it. - if err := validateRuntimeNode("enrollment", capacity.MaxActive, capacity.MaxActive); err != nil { - return RuntimeNodeEnrollmentToken{}, err - } - var bytes [32]byte - if _, err := rand.Read(bytes[:]); err != nil { - return RuntimeNodeEnrollmentToken{}, err - } - id := uuid.New() - result := RuntimeNodeEnrollmentToken{Token: hex.EncodeToString(bytes[:]), ID: id.String()} - err := s.runtimeManagerTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { - capacity.MaxRetained = providers.RetainedLimit(d.ProviderKind, capacity.MaxActive, capacity.MaxRetained) - if err := validateRuntimeNode("enrollment", capacity.MaxActive, capacity.MaxRetained); err != nil { - return err - } - if d.ResetClear.Valid { - return ErrSandboxResetInProgress - } - if d.Mode != "nodes" || d.AdmissionPaused { - return ErrSandboxDeploymentConflict - } - if _, err := deploymentSpecification(d); err != nil { - return ErrSandboxDeploymentConflict - } - if err := q.CreateRuntimeEnrollment(ctx, sqlc.CreateRuntimeEnrollmentParams{ID: pgtype.UUID{Bytes: id, Valid: true}, TokenSha256: runtimeTokenDigest(result.Token), InstallationID: d.InstallationID, MaxActive: int32(capacity.MaxActive), MaxRetained: int32(capacity.MaxRetained)}); err != nil { - return err - } - row, err := q.GetRuntimeEnrollment(ctx, runtimeTokenDigest(result.Token)) - result.ExpiresAt = row.ExpiresAt.Time - return err - }) - return result, err -} -func (s *Store) EnrollRuntimeNode(ctx context.Context, token string, input RuntimeNodeEnrollment) (RuntimeNodeIdentity, error) { - id, err := parseConnectionGeneration(input.NodeID) - if err != nil || len(input.Credential) < 32 || len(input.Credential) > 256 || strings.ContainsAny(input.Credential, " \t\r\n") || !validRuntimeDigest(input.BackendFingerprint) { - return RuntimeNodeIdentity{}, ErrInvalidInput - } - if err := validateRuntimeNode(input.Name, 1, 1); err != nil { - return RuntimeNodeIdentity{}, err - } - var result RuntimeNodeIdentity - err = s.runtimeDeploymentTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { - receipt, err := q.GetRuntimeEnrollment(ctx, runtimeTokenDigest(token)) - if errors.Is(err, pgx.ErrNoRows) { - return ErrRuntimeNodeCredential - } - if err != nil { - return ErrRuntimeNodeUnavailable - } - if receipt.ConsumedAt.Valid || !receipt.ExpiresAt.Time.After(time.Now()) { - return ErrRuntimeNodeCredential - } - if d.InstallationID.Valid && receipt.InstallationID != d.InstallationID { - return ErrRuntimeNodeCredential - } - if !runtimeDeploymentInitialized(d) { - return ErrRuntimeNodeUnavailable - } - if d.ResetClear.Valid { - return ErrSandboxResetInProgress - } - if d.Mode != "nodes" || d.AdmissionPaused || input.Provider != d.ProviderKind { - return ErrInvalidInput - } - spec, err := deploymentSpecification(d) - if err != nil || input.DeploymentGeneration != uint64(d.Generation) || input.SpecificationDigest != spec.Digest(d.ProviderKind) { - return ErrRuntimeSpecificationMismatch - } - // The node must use the address Core advertises now. It read that address - // from its configuration, but the public URL may have changed since, or an - // operator may have registered by hand with another origin. - if input.CoreURL != s.publicURL { - return ErrRuntimeNodeAddressMismatch - } - if _, err := q.GetRuntimeNode(ctx, id); err == nil { - return ErrIdempotencyConflict - } else if !errors.Is(err, pgx.ErrNoRows) { - return err - } - row, err := q.InsertRuntimeNode(ctx, sqlc.InsertRuntimeNodeParams{ID: id, InstallationID: d.InstallationID, Name: input.Name, BackendFingerprint: input.BackendFingerprint, CredentialSha256: runtimeTokenDigest(input.Credential), MaxActive: receipt.MaxActive, MaxRetained: int32(providers.RetainedLimit(d.ProviderKind, int(receipt.MaxActive), int(receipt.MaxRetained))), SpecificationDigest: input.SpecificationDigest, DeploymentGeneration: int64(input.DeploymentGeneration), CoreUrl: input.CoreURL, EnrollmentID: receipt.ID}) - if err != nil { - return err - } - changed, err := q.ConsumeRuntimeEnrollment(ctx, sqlc.ConsumeRuntimeEnrollmentParams{TokenSha256: runtimeTokenDigest(token), NodeID: id}) - if err != nil { - return err - } - if changed != 1 { - return ErrRuntimeNodeCredential - } - result = nodeIdentity(row, d.ProviderKind) - return nil - }) - return result, err -} -func nodeIdentity(n sqlc.RuntimeNode, kind string) RuntimeNodeIdentity { - return RuntimeNodeIdentity{SpecificationDigest: n.SpecificationDigest, DeploymentGeneration: uint64(n.DeploymentGeneration), NodeID: runtimeUUID(n.ID), InstallationID: runtimeUUID(n.InstallationID), Provider: kind, BackendFingerprint: n.BackendFingerprint, MaxActive: int(n.MaxActive), MaxRetained: providers.RetainedLimit(kind, int(n.MaxActive), int(n.MaxRetained))} -} -func (s *Store) AuthenticateRuntimeNode(ctx context.Context, nodeID, credential string) (RuntimeNodeIdentity, error) { - id, err := parseConnectionGeneration(nodeID) - if err != nil { - return RuntimeNodeIdentity{}, ErrRuntimeNodeCredential - } - n, err := s.queries.GetRuntimeNode(ctx, id) - if errors.Is(err, pgx.ErrNoRows) { - return RuntimeNodeIdentity{}, ErrRuntimeNodeCredential - } - if err != nil { - return RuntimeNodeIdentity{}, ErrRuntimeNodeUnavailable - } - if n.CredentialSha256 != runtimeTokenDigest(credential) { - return RuntimeNodeIdentity{}, ErrRuntimeNodeCredential - } - d, err := s.queries.GetRuntimeDeployment(ctx) - if err != nil { - return RuntimeNodeIdentity{}, ErrRuntimeNodeUnavailable - } - if n.InstallationID != d.InstallationID || d.Mode != "nodes" { - return RuntimeNodeIdentity{}, ErrRuntimeNodeCredential - } - if err := validateNodeEnrollmentIdentity(ctx, s.queries, d, n); err != nil { - return RuntimeNodeIdentity{}, err - } - // Reset retires nodes before another backend lineage can be selected. - // Enrollment generation and digest remain immutable identity history; the - // current target and per-generation readiness do not replace that history. - if n.DeploymentGeneration <= 0 || !validRuntimeDigest(n.SpecificationDigest) { - return RuntimeNodeIdentity{}, ErrRuntimeSpecificationMismatch - } - return nodeIdentity(n, d.ProviderKind), nil -} -func (s *Store) UpdateRuntimeNode(ctx context.Context, nodeID string, input RuntimeNodeUpdate) error { - // The retained limit depends on the provider, so the transaction checks it. - if err := validateRuntimeNode(input.Name, input.MaxActive, input.MaxActive); err != nil { - return err - } - id, err := parseConnectionGeneration(nodeID) - if err != nil { - return err - } - return s.runtimeManagerTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { - input.MaxRetained = providers.RetainedLimit(d.ProviderKind, input.MaxActive, input.MaxRetained) - if err := validateRuntimeNode(input.Name, input.MaxActive, input.MaxRetained); err != nil { - return err - } - n, err := q.GetRuntimeNode(ctx, id) - if errors.Is(err, pgx.ErrNoRows) { - return ErrNotFound - } - if err != nil { - return err - } - if n.InstallationID != d.InstallationID { - return ErrNotFound - } - _, err = q.UpdateRuntimeNode(ctx, sqlc.UpdateRuntimeNodeParams{ID: id, Name: input.Name, MaxActive: int32(input.MaxActive), MaxRetained: int32(input.MaxRetained)}) - return err - }) -} -func (s *Store) RemoveRuntimeNode(ctx context.Context, nodeID string) error { - id, err := parseConnectionGeneration(nodeID) - if err != nil { - return err - } - return s.runtimeManagerTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { - rows, err := q.ListRuntimeNodes(ctx, pgtype.UUID{}) - if err != nil { - return err - } - for _, n := range rows { - if n.ID != id { - continue - } - if n.Retained != 0 || n.CleanupPending != 0 { - return ErrRuntimeNodeInUse - } - if d.LocalNodeID == id { - return ErrRuntimeLocalNodeConfigured - } - return q.RemoveRuntimeNode(ctx, id) - } - return ErrNotFound - }) -} func (s *Store) ListNodeRuntimeAllocations(ctx context.Context, nodeID string) ([]RuntimeNodeAllocation, error) { id, err := parseConnectionGeneration(nodeID) if err != nil { @@ -353,26 +58,3 @@ func (s *Store) ListNodeRuntimeAllocations(ctx context.Context, nodeID string) ( } return out, nil } - -func nodeRollout(n sqlc.ListRuntimeNodesRow) SandboxNodeRollout { - out := SandboxNodeRollout{State: "unknown"} - if n.ReadyGeneration.Valid { - generation := uint64(n.ReadyGeneration.Int64) - out.ReadyGeneration = &generation - } - if !n.Online { - return out - } - if n.ProtocolVersion == 1 && n.DeploymentGeneration != n.TargetGeneration { - out.State = "update_required" - return out - } - switch n.TargetState { - case "ready", "preparing", "failed": - out.State = n.TargetState - } - if out.State == "failed" && n.TargetDiagnostic != "" { - out.Diagnostic = sandbox.NormalizeNodeDiagnostic(n.TargetDiagnostic) - } - return out -} diff --git a/services/core/internal/store/runtime_nodes_test.go b/services/core/internal/store/runtime_nodes_test.go index fbf73b66f..28c129cc3 100644 --- a/services/core/internal/store/runtime_nodes_test.go +++ b/services/core/internal/store/runtime_nodes_test.go @@ -9,12 +9,13 @@ import ( "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" "github.com/jackc/pgx/v5" ) -func managerFixture(t *testing.T, active, retained int) (*Store, *Store, RuntimeDeployment) { +func managerFixture(t *testing.T, active, retained int) (*Store, *Store, deployment.ProcessDeployment) { t.Helper() s, _ := newManagedTestStore(t) w := executionWriter(t, s) @@ -31,10 +32,11 @@ func managerFixture(t *testing.T, active, retained int) (*Store, *Store, Runtime func onlineManagerNode(t *testing.T, s *Store, id string) string { t.Helper() connection := uuid.NewString() - if err := s.ConnectRuntimeNode(t.Context(), id, connection, managerEpoch(t, s)); err != nil { + nodes := deploymentService(t, s) + if err := nodes.ConnectNode(t.Context(), id, connection, managerEpoch(t, s)); err != nil { t.Fatal(err) } - if err := s.HeartbeatRuntimeNode(t.Context(), id, connection, managerEpoch(t, s), RuntimeNodeHealth{ProviderReady: true}); err != nil { + if err := nodes.Heartbeat(t.Context(), id, connection, managerEpoch(t, s), deployment.NodeHealth{ProviderReady: true}); err != nil { t.Fatal(err) } return connection @@ -70,14 +72,15 @@ func createSessionOnNode(t *testing.T, s *Store, tenant string, input CreateSess if err := rows.Err(); err != nil { t.Fatal(err) } + nodes := deploymentService(t, s) for _, value := range others { - if err := s.HeartbeatRuntimeNode(t.Context(), value.id, value.connection, value.epoch, RuntimeNodeHealth{ProviderReady: false}); err != nil { + if err := nodes.Heartbeat(t.Context(), value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: false}); err != nil { t.Fatal(err) } } defer func() { for _, value := range others { - if err := s.HeartbeatRuntimeNode(context.WithoutCancel(t.Context()), value.id, value.connection, value.epoch, RuntimeNodeHealth{ProviderReady: true}); err != nil { + if err := nodes.Heartbeat(context.WithoutCancel(t.Context()), value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { t.Fatal(err) } } @@ -131,7 +134,7 @@ func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { for err := range failures { if err == nil { successes++ - } else if !errors.Is(err, ErrRuntimeNodeUnavailable) { + } else if !errors.Is(err, deployment.ErrNodeUnavailable) { t.Fatal(err) } } @@ -144,11 +147,12 @@ func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { retained = session } } - nodes, err := s.ListRuntimeNodes(t.Context()) + service := deploymentService(t, s) + nodes, err := service.ListNodes(t.Context()) if err != nil || len(nodes) != 1 || nodes[0].Active != 1 || nodes[0].Retained != 1 || nodes[0].Reserved != 1 { t.Fatal(nodes, err) } - if err := s.RemoveRuntimeNode(t.Context(), d.LocalNodeID); !errors.Is(err, ErrRuntimeNodeInUse) { + if err := service.RemoveNode(t.Context(), d.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("removed pending placement", err) } if err := s.DeleteSession(t.Context(), tenant, retained.ID); err != nil { @@ -176,30 +180,31 @@ func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { } func TestRuntimeNodesEnrollmentAndEpoch(t *testing.T) { s, w, d := managerFixture(t, 2, 4) - token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4})) + nodes := deploymentService(t, s) + token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 2, MaxRetained: 4})) if err != nil { t.Fatal(err) } - input := RuntimeNodeEnrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: uuid.NewString(), Credential: strings.Repeat("x", 64), Name: "remote", Provider: "microsandbox", BackendFingerprint: strings.Repeat("b", 64)} - if _, err := s.EnrollRuntimeNode(t.Context(), token, input); !errors.Is(err, ErrInvalidInput) { + input := deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: uuid.NewString(), Credential: strings.Repeat("x", 64), Name: "remote", Provider: "microsandbox", BackendFingerprint: strings.Repeat("b", 64), CoreURL: s.publicURL} + if _, err := nodes.Enroll(t.Context(), token, input); !errors.Is(err, deployment.ErrInvalidInput) { t.Fatal("mixed provider accepted", err) } input.Provider = "docker" - enrolled, err := s.EnrollRuntimeNode(t.Context(), token, input) + enrolled, err := nodes.Enroll(t.Context(), token, input) if err != nil || enrolled.InstallationID != d.InstallationID { t.Fatal(enrolled, err) } - if _, err := s.EnrollRuntimeNode(t.Context(), token, input); !errors.Is(err, ErrRuntimeNodeCredential) { + if _, err := nodes.Enroll(t.Context(), token, input); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("enrollment token reused", err) } - if _, err := s.AuthenticateRuntimeNode(t.Context(), input.NodeID, input.Credential); err != nil { + if _, err := nodes.AuthenticateNode(t.Context(), input.NodeID, input.Credential); err != nil { t.Fatal("lost response cannot recover", err) } connection := onlineManagerNode(t, s, input.NodeID) - if err := s.DisconnectRuntimeNode(t.Context(), input.NodeID, uuid.NewString(), managerEpoch(t, s)); err != nil { + if err := nodes.DisconnectNode(t.Context(), input.NodeID, uuid.NewString(), managerEpoch(t, s)); err != nil { t.Fatal(err) } - current, err := s.ListRuntimeNodes(t.Context()) + current, err := nodes.ListNodes(t.Context()) if err != nil { t.Fatal(err) } @@ -208,31 +213,28 @@ func TestRuntimeNodesEnrollmentAndEpoch(t *testing.T) { t.Fatal("stale disconnect fenced current connection") } } - epoch, err := s.RuntimeOwnerEpoch(t.Context()) - if err != nil { - t.Fatal(err) - } + epoch := managerEpoch(t, s) deploymentConfigure(t, w, &d) - next, err := s.RuntimeOwnerEpoch(t.Context()) - if err != nil || next != epoch+1 { - t.Fatal(next, err) + if next := managerEpoch(t, s); next != epoch+1 { + t.Fatal(next) } - if err := s.HeartbeatRuntimeNode(t.Context(), input.NodeID, connection, epoch, RuntimeNodeHealth{ProviderReady: true}); !errors.Is(err, ErrRuntimeNodeCredential) { + if err := nodes.Heartbeat(t.Context(), input.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("old epoch heartbeat revived node", err) } - if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput("stale")); !errors.Is(err, ErrRuntimeNodeUnavailable) { + if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput("stale")); !errors.Is(err, deployment.ErrNodeUnavailable) { t.Fatal("stale node admitted", err) } onlineManagerNode(t, s, d.LocalNodeID) - if err := s.RemoveRuntimeNode(t.Context(), input.NodeID); err != nil { + if err := nodes.RemoveNode(t.Context(), input.NodeID); err != nil { t.Fatal(err) } - if _, err := s.AuthenticateRuntimeNode(t.Context(), input.NodeID, input.Credential); !errors.Is(err, ErrRuntimeNodeCredential) { + if _, err := nodes.AuthenticateNode(t.Context(), input.NodeID, input.Credential); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("removed node credential accepted", err) } } func TestRuntimeNodesRetention(t *testing.T) { s, w, next := managerFixture(t, 2, 2) + nodes := deploymentService(t, s) tenant := uuid.NewString() first, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())) if err != nil { @@ -246,7 +248,7 @@ func TestRuntimeNodesRetention(t *testing.T) { if err != nil { t.Fatal(err) } - if err := s.RemoveRuntimeNode(t.Context(), next.LocalNodeID); !errors.Is(err, ErrRuntimeNodeInUse) { + if err := nodes.RemoveNode(t.Context(), next.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal(err) } if err := s.DeleteSession(t.Context(), tenant, pending.ID); err != nil { @@ -259,7 +261,7 @@ func TestRuntimeNodesRetention(t *testing.T) { if err != nil { t.Fatal(err) } - if err := s.RemoveRuntimeNode(t.Context(), next.LocalNodeID); !errors.Is(err, ErrRuntimeNodeInUse) { + if err := nodes.RemoveNode(t.Context(), next.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("unknown cleanup released node", err) } retained, err = w.SettleRuntimeCreation(t.Context(), retained) @@ -269,10 +271,10 @@ func TestRuntimeNodesRetention(t *testing.T) { if _, err := w.ReleaseRuntimeAllocation(t.Context(), retained); err != nil { t.Fatal(err) } - if err := s.RemoveRuntimeNode(t.Context(), next.LocalNodeID); !errors.Is(err, ErrRuntimeLocalNodeConfigured) { + if err := nodes.RemoveNode(t.Context(), next.LocalNodeID); !errors.Is(err, deployment.ErrLocalNodeConfigured) { t.Fatal("configured local node was removed", err) } - if _, err := s.AuthenticateRuntimeNode(t.Context(), next.LocalNodeID, "local-node-credential"); err != nil { + if _, err := nodes.AuthenticateNode(t.Context(), next.LocalNodeID, "local-node-credential"); err != nil { t.Fatal("rejected removal changed local credentials", err) } next.AdmissionPaused = true @@ -283,7 +285,7 @@ func TestRuntimeNodesRetention(t *testing.T) { detached.LocalMaxActive, detached.LocalMaxRetained = 0, 0 detached.BackendFingerprint = strings.Repeat("b", 64) deploymentConfigure(t, w, &detached) - if err := s.RemoveRuntimeNode(t.Context(), next.LocalNodeID); err != nil { + if err := nodes.RemoveNode(t.Context(), next.LocalNodeID); err != nil { t.Fatal("detached resolved node cannot be removed", err) } } @@ -324,14 +326,14 @@ func TestRuntimeNodesRestoreAndCreationShareCapacity(t *testing.T) { err := <-results if err == nil { success++ - } else if !errors.Is(err, ErrRuntimeNodeUnavailable) { + } else if !errors.Is(err, deployment.ErrNodeUnavailable) { t.Fatal(err) } } if success != 1 { t.Fatal("restore and creation overbooked", success) } - nodes, err := s.ListRuntimeNodes(t.Context()) + nodes, err := deploymentService(t, s).ListNodes(t.Context()) if err != nil || nodes[0].Active != 1 { t.Fatal(nodes, err) } @@ -339,7 +341,7 @@ func TestRuntimeNodesRestoreAndCreationShareCapacity(t *testing.T) { func managerEpoch(t *testing.T, s *Store) uint64 { t.Helper() - epoch, err := s.RuntimeOwnerEpoch(t.Context()) + epoch, err := deploymentStore(s).OwnerEpoch(t.Context()) if err != nil { t.Fatal(err) } @@ -370,12 +372,12 @@ func TestRuntimeNodesLongOfflineRetainsExactAllocation(t *testing.T) { if err != nil || offline.Expired || offline.State != "running" { t.Fatal("offline treated as destructive expiry", offline, err) } - if err := s.RemoveRuntimeNode(t.Context(), d.LocalNodeID); !errors.Is(err, ErrRuntimeNodeInUse) { + if err := deploymentService(t, s).RemoveNode(t.Context(), d.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("offline ownership discarded", err) } changed := d changed.LocalNodeID = uuid.NewString() - if err := w.ConfigureRuntimeDeployment(t.Context(), &changed); err == nil { + if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &changed); err == nil { t.Fatal("lost local state created replacement identity") } onlineManagerNode(t, s, d.LocalNodeID) diff --git a/services/core/internal/store/runtime_observation_test.go b/services/core/internal/store/runtime_observation_test.go index 4052fc8f4..9ec0a46fb 100644 --- a/services/core/internal/store/runtime_observation_test.go +++ b/services/core/internal/store/runtime_observation_test.go @@ -4,6 +4,7 @@ import ( "errors" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) @@ -30,7 +31,7 @@ func TestRuntimeNodeObservationRetainsResourcesAndFencesStaleResults(t *testing. if err != nil || retained.State != "running" || retained.ID != owner.ID || retained.ObservationError != "node_unavailable" { t.Fatal(retained, err) } - if err := s.RemoveRuntimeNode(t.Context(), d.LocalNodeID); !errors.Is(err, ErrRuntimeNodeInUse) { + if err := deploymentService(t, s).RemoveNode(t.Context(), d.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("diagnostic released resource", err) } // A new lifecycle observation must not be erased by an earlier result. @@ -58,22 +59,3 @@ func TestRuntimeNodeObservationRetainsResourcesAndFencesStaleResults(t *testing. t.Fatal("raw diagnostics accepted", err) } } -func TestRuntimeNodeStaleEpochCannotReplaceCurrentConnection(t *testing.T) { - s, w, d := managerFixture(t, 1, 4) - epoch := managerEpoch(t, s) - deploymentConfigure(t, w, &d) - connection := onlineManagerNode(t, s, d.LocalNodeID) - if err := s.ConnectRuntimeNode(t.Context(), d.LocalNodeID, uuid.NewString(), epoch); !errors.Is(err, ErrRuntimeNodeCredential) { - t.Fatal("old Core replaced new connection", err) - } - if err := s.DisconnectRuntimeNode(t.Context(), d.LocalNodeID, connection, epoch); err != nil { - t.Fatal(err) - } - if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, connection, epoch, RuntimeNodeHealth{ProviderReady: false}); !errors.Is(err, ErrRuntimeNodeCredential) { - t.Fatal("old Core rewrote health", err) - } - nodes, err := s.ListRuntimeNodes(t.Context()) - if err != nil || !nodes[0].Online || !nodes[0].ProviderReady { - t.Fatal("stale callback changed current epoch", nodes, err) - } -} diff --git a/services/core/internal/store/runtime_placements.go b/services/core/internal/store/runtime_placements.go index b3c4cb524..bf35cbf3f 100644 --- a/services/core/internal/store/runtime_placements.go +++ b/services/core/internal/store/runtime_placements.go @@ -3,8 +3,9 @@ package store import ( "context" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/jackc/pgx/v5/pgtype" @@ -24,30 +25,30 @@ func reserveRuntimePlacement(ctx context.Context, q *sqlc.Queries, session pgtyp // A changed installation address cannot admit guests that require a public // origin. Existing owned resources remain available for cleanup. if d.ProviderKind != "" { - publicOrigin, err := providers.RequiresPublicOrigin(d.ProviderKind) + publicOrigin, err := sandboxProviders.RequiresPublicOrigin(d.ProviderKind) if err != nil { return err } - if publicOrigin && LoopbackOrigin(publicURL) { - return ErrSandboxPublicURLUnreachable + if publicOrigin && deployment.LoopbackOrigin(publicURL) { + return deployment.ErrPublicURLUnreachable } } if d.Mode == "direct" { if d.AdmissionPaused { - return ErrRuntimeNodeUnavailable + return deployment.ErrNodeUnavailable } return nil } if d.ProviderKind == "" { if d.WebManaged { - return ErrRuntimeNodeUnavailable + return deployment.ErrNodeUnavailable } return nil } if d.AdmissionPaused { - return ErrRuntimeNodeUnavailable + return deployment.ErrNodeUnavailable } rows, err := q.ListRuntimeNodes(ctx, pgtype.UUID{}) if err != nil { @@ -69,9 +70,9 @@ func reserveRuntimePlacement(ctx context.Context, q *sqlc.Queries, session pgtyp } if chosen == nil { if preparing { - return ErrSandboxNodesPreparing + return deployment.ErrNodesPreparing } - return ErrRuntimeNodeUnavailable + return deployment.ErrNodeUnavailable } return q.CreateSessionRuntimePlacement(ctx, sqlc.CreateSessionRuntimePlacementParams{SessionID: session, NodeID: chosen.ID, Generation: chosen.ReadyGeneration.Int64}) } @@ -83,7 +84,7 @@ func (s *Store) ResolveRuntimeNode(ctx context.Context, tenant, environment stri return "", err } if allocation.NodeID == "" { - return "", ErrRuntimeNodeUnavailable + return "", deployment.ErrNodeUnavailable } return allocation.NodeID, nil } @@ -105,12 +106,12 @@ func reserveRuntimeRestore(ctx context.Context, q *sqlc.Queries, node pgtype.UUI return err } if !generation.Valid || !n.Online || !ready || n.Active >= int64(n.MaxActive) { - return ErrRuntimeNodeUnavailable + return deployment.ErrNodeUnavailable } return nil } } - return ErrRuntimeNodeUnavailable + return deployment.ErrNodeUnavailable } // ResolveRuntimeGeneration includes deleted Sessions and never substitutes the target. @@ -120,7 +121,7 @@ func (s *Store) ResolveRuntimeGeneration(ctx context.Context, ref sandbox.Refere return "", 0, err } if a.State == "released" || a.ID != ref.AllocationID || a.NodeID == "" || a.DeploymentGeneration == 0 { - return "", 0, ErrRuntimeNodeUnavailable + return "", 0, deployment.ErrNodeUnavailable } return a.NodeID, a.DeploymentGeneration, nil } diff --git a/services/core/internal/store/sandbox_credential_allocations.go b/services/core/internal/store/sandbox_credential_allocations.go new file mode 100644 index 000000000..879dd6929 --- /dev/null +++ b/services/core/internal/store/sandbox_credential_allocations.go @@ -0,0 +1,28 @@ +package store + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +// SandboxCredentialAllocationPage reads owned receipts without granting execution authority. +func (s *Store) SandboxCredentialAllocationPage(ctx context.Context, after string) ([]RuntimeAllocation, error) { + id := pgtype.UUID{Valid: true} + if after != "" { + var err error + id, err = parseID(after) + if err != nil { + return nil, err + } + } + rows, err := s.queries.ListRuntimeAllocations(ctx, id) + if err != nil { + return nil, err + } + out := make([]RuntimeAllocation, 0, len(rows)) + for _, r := range rows { + out = append(out, runtimeAllocationFromRow(r.RuntimeAllocation, r.SessionID, r.TenantID, r.DeletedAt, r.Expired)) + } + return out, nil +} diff --git a/services/core/internal/store/sandbox_deployment_mutations.go b/services/core/internal/store/sandbox_deployment_mutations.go deleted file mode 100644 index 93e4a4c54..000000000 --- a/services/core/internal/store/sandbox_deployment_mutations.go +++ /dev/null @@ -1,253 +0,0 @@ -package store - -import ( - "context" - "encoding/json" - "errors" - "math" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" - "github.com/jackc/pgx/v5" -) - -var ErrSandboxCredentialUnavailable = errors.New("sandbox credential encryption is unavailable") - -type SandboxDeploymentUpdateRequest struct { - SandboxDeploymentSetupRequest - ExpectedGeneration uint64 `json:"expected_generation"` -} - -func validateSandboxSelection(input SandboxDeploymentSetupRequest) error { - _, err := providers.Normalize(input) - if err != nil { - return sandboxConfigurationError(err) - } - return nil -} - -func (s *Store) sandboxSelectionEqual(d sqlc.RuntimeDeployment, input SandboxDeploymentSetupRequest) (bool, error) { - if d.ProviderKind != input.Provider { - return false, nil - } - previous, err := s.sandboxSetup(d) - if err != nil { - return false, err - } - normalized, err := providers.Normalize(input) - if err != nil { - return false, sandboxConfigurationError(err) - } - if previous.Specification.Digest(d.ProviderKind) != normalized.DeploymentSpec.Digest(input.Provider) { - return false, nil - } - return providers.Equal(input.Provider, previous.Configuration, normalized.Configuration) -} - -func configurationJSON(raw json.RawMessage) json.RawMessage { - if len(raw) == 0 { - return json.RawMessage(`{}`) - } - return raw -} -func (s *Store) saveSandboxSelection(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, input SandboxDeploymentSetupRequest) error { - if err := providers.ValidateSpecification(input.Provider, input.DeploymentSpec); err != nil { - return sandboxConfigurationError(err) - } - if d.Generation == math.MaxInt64 { - return ErrSandboxDeploymentConflict - } - generation := d.Generation + 1 - description, err := providers.Describe(input.Provider, runtimeUUID(d.InstallationID)) - if err != nil { - return sandboxConfigurationError(err) - } - input, err = providers.Normalize(input) - if err != nil { - return sandboxConfigurationError(err) - } - record, err := providers.Encode(input.Provider, input.Configuration) - if err != nil { - return sandboxConfigurationError(err) - } - params := sqlc.InitializeSandboxDeploymentParams{ProviderKind: input.Provider, BackendFingerprint: description.BackendFingerprint, Generation: generation, Mode: description.Mode, IdleSeconds: description.IdleSeconds, RetentionSeconds: description.RetentionSeconds, ProviderConfig: configurationJSON(record.Public), ProviderMetadata: configurationJSON(record.Metadata)} - params.Specification, _ = json.Marshal(input.DeploymentSpec) - if len(record.Secret) > 0 { - params.ProviderCredential, err = s.credentialCipher.SealSandboxDeployment(record.Secret, runtimeUUID(d.InstallationID), uint64(generation)) - if err != nil { - return ErrSandboxCredentialUnavailable - } - } - return q.InitializeSandboxDeployment(ctx, params) -} - -func recordConfigurationMetadata(ctx context.Context, q *sqlc.Queries, input SandboxDeploymentSetupRequest) error { - record, err := providers.Encode(input.Provider, input.Configuration) - if err != nil { - return sandboxConfigurationError(err) - } - if len(record.Metadata) == 0 { - return nil - } - return q.RecordSandboxConfigurationMetadata(ctx, record.Metadata) -} - -func (s *Store) InitializeSandboxDeployment(ctx context.Context, installationID string, input SandboxDeploymentSetupRequest) (RuntimeDeploymentView, error) { - if err := s.checkExecutionAuthority(); err != nil { - return RuntimeDeploymentView{}, err - } - if err := validateSandboxSelection(input); err != nil { - return RuntimeDeploymentView{}, err - } - id, err := parseConnectionGeneration(installationID) - if err != nil { - return RuntimeDeploymentView{}, err - } - var result RuntimeDeploymentView - err = s.writer.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { - q := s.queries.WithTx(tx) - d, err := q.LockRuntimeDeployment(ctx) - if err != nil { - return err - } - if err := checkSandboxGeneration(d, installationID, input.ExpectedGeneration); err != nil { - return err - } - if d.ResetClear.Valid { - return ErrSandboxResetInProgress - } - if d.InstallationID != id { - return ErrSandboxDeploymentConflict - } - if d.ProviderKind != "" { - equal, err := s.sandboxSelectionEqual(d, input) - if err != nil { - return err - } - if !equal { - return ErrSandboxDeploymentConflict - } - if err := recordConfigurationMetadata(ctx, q, input); err != nil { - return err - } - } else if err := s.saveSandboxSelection(ctx, q, d, input); err != nil { - return err - } - result, err = s.deploymentView(ctx, q) - return err - }) - return result, err -} - -// CheckSandboxDeploymentSwitch is a preliminary check only. The mutation repeats -// it in the committing transaction; no database lock spans provider work. -func (s *Store) CheckSandboxDeploymentSwitch(ctx context.Context, installation string, input SandboxDeploymentUpdateRequest) error { - if err := s.checkExecutionAuthority(); err != nil { - return err - } - if err := validateSandboxSelection(input.SandboxDeploymentSetupRequest); err != nil { - return err - } - return s.writer.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { - q := s.queries.WithTx(tx) - d, err := q.LockRuntimeDeployment(ctx) - if err != nil { - return err - } - return checkSandboxSwitch(ctx, q, d, installation, input) - }) -} -func checkSandboxSwitch(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment, installation string, input SandboxDeploymentUpdateRequest) error { - if err := checkSandboxGeneration(d, installation, input.ExpectedGeneration); err != nil { - return err - } - if d.ResetClear.Valid { - return ErrSandboxResetInProgress - } - if d.ProviderKind == "" { - return ErrSandboxNotConfigured - } - if _, err := deploymentSpecification(d); err != nil { - return err - } - if d.ProviderKind != input.Provider { - return &SandboxResetRequiredError{CurrentProvider: d.ProviderKind, RequestedProvider: input.Provider} - } - - return nil -} - -func (s *Store) UpdateSandboxDeployment(ctx context.Context, installation string, input SandboxDeploymentUpdateRequest) (RuntimeDeploymentView, error) { - if err := s.checkExecutionAuthority(); err != nil { - return RuntimeDeploymentView{}, err - } - if err := validateSandboxSelection(input.SandboxDeploymentSetupRequest); err != nil { - return RuntimeDeploymentView{}, err - } - var result RuntimeDeploymentView - err := s.writer.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { - q := s.queries.WithTx(tx) - d, err := q.LockRuntimeDeployment(ctx) - if err != nil { - return err - } - if err := checkSandboxSwitch(ctx, q, d, installation, input); err != nil { - return err - } - equal, err := s.sandboxSelectionEqual(d, input.SandboxDeploymentSetupRequest) - if err != nil { - return err - } - if !equal || input.ReplacesCredential() { - if err := q.RetainSandboxGeneration(ctx); err != nil { - return err - } - if err := s.saveSandboxSelection(ctx, q, d, input.SandboxDeploymentSetupRequest); err != nil { - return err - } - if err := q.CollectSandboxGenerations(ctx); err != nil { - return err - } - { - action := "change" - if input.ReplacesCredential() { - action = "replace_credential" - } - if err := auditpg.RecordDeploymentMutation(ctx, q, action, "sandbox_deployment", installation); err != nil { - return err - } - } - } else if err := recordConfigurationMetadata(ctx, q, input.SandboxDeploymentSetupRequest); err != nil { - return err - } - result, err = s.deploymentView(ctx, q) - return err - }) - return result, err -} - -// CheckSandboxDeploymentSetup rejects stale/reset state before provider preparation; -// InitializeSandboxDeployment repeats the check in its committing transaction. -func (s *Store) CheckSandboxDeploymentSetup(ctx context.Context, installation string, input SandboxDeploymentSetupRequest) error { - return s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { - if err := checkSandboxGeneration(d, installation, input.ExpectedGeneration); err != nil { - return err - } - if d.ResetClear.Valid { - return ErrSandboxResetInProgress - } - if err := validateSandboxSelection(input); err != nil { - return err - } - if d.ProviderKind != "" { - if _, err := deploymentSpecification(d); err != nil { - return err - } - } - if d.ProviderKind != "" && d.ProviderKind != input.Provider { - return &SandboxResetRequiredError{CurrentProvider: d.ProviderKind, RequestedProvider: input.Provider} - } - return nil - }) -} diff --git a/services/core/internal/store/sandbox_deployment_resources_test.go b/services/core/internal/store/sandbox_deployment_resources_test.go index e8cc68700..34f546bcf 100644 --- a/services/core/internal/store/sandbox_deployment_resources_test.go +++ b/services/core/internal/store/sandbox_deployment_resources_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) @@ -17,12 +18,13 @@ func TestSandboxDeploymentMutationViewsIncludeActualResources(t *testing.T) { } s := NewWithCredentialCipher(pool, cipher) w := executionWriter(t, s) + changes := deploymentExecution(t, w) installation := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + if err := changes.Claim(t.Context(), installation); err != nil { t.Fatal(err) } selection := e2bSelection() - if _, err := w.InitializeSandboxDeployment(t.Context(), installation, selection); err != nil { + if _, err := changes.Initialize(t.Context(), installation, selection); err != nil { t.Fatal(err) } tenant := uuid.NewString() @@ -36,30 +38,31 @@ func TestSandboxDeploymentMutationViewsIncludeActualResources(t *testing.T) { if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())); err != nil { t.Fatal(err) } - want := SandboxDeploymentResources{Allocations: 1, Pending: 1} + want := deployment.Resources{Allocations: 1, Pending: 1} // Replaying setup must report the current resources rather than initial zeros. selection.ExpectedGeneration = 1 - replay, err := w.InitializeSandboxDeployment(t.Context(), installation, selection) + replay, err := changes.Initialize(t.Context(), installation, selection) if err != nil || replay.Resources != want { t.Fatalf("setup replay resources = %+v, error = %v", replay.Resources, err) } + // Reset changes return no view; the view read after each commit reports + // the current resources. for _, maintenance := range []bool{true, false} { - var response RuntimeDeploymentView var err error if maintenance { - response, err = w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + err = w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) } else { - response, err = w.CancelSandboxReset(SandboxResetTestContext(t.Context()), installation, 1) + err = w.CancelSandboxReset(SandboxResetTestContext(t.Context()), installation, 1) } if err != nil { t.Fatal(err) } - current, err := s.GetRuntimeDeployment(t.Context()) + current, err := deploymentService(t, s).View(t.Context()) if err != nil { t.Fatal(err) } - if (response.Reset != nil) != maintenance || response.Resources != want || response.Resources != current.Resources { - t.Fatalf("maintenance %v response = %+v, current resources = %+v", maintenance, response, current.Resources) + if (current.Reset != nil) != maintenance || current.Resources != want { + t.Fatalf("maintenance %v view = %+v", maintenance, current) } } } diff --git a/services/core/internal/store/sandbox_deployment_setup.go b/services/core/internal/store/sandbox_deployment_setup.go deleted file mode 100644 index 8191cb5e1..000000000 --- a/services/core/internal/store/sandbox_deployment_setup.go +++ /dev/null @@ -1,196 +0,0 @@ -package store - -import ( - "context" - "encoding/json" - "errors" - "net" - "net/url" - "strconv" - "strings" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" - "github.com/jackc/pgx/v5" -) - -var ErrSandboxDeploymentConflict = errors.New("sandbox deployment is already configured differently") - -type SandboxDeploymentSetupRequest = sandbox.Selection - -// SandboxSetup is the immutable configuration selected by the deployment admin. -// An empty Provider means that Web setup has not yet selected an adapter. -type SandboxSetup struct { - Specification sandbox.DeploymentSpec - InstallationID, Provider, BackendFingerprint string - Generation uint64 - Mode string - AdmissionPaused bool - Configuration sandbox.Configuration `json:"-"` - IdleSeconds, RetentionSeconds int64 -} - -func (s *Store) GetSandboxSetup(ctx context.Context) (SandboxSetup, error) { - ctx, cancel := context.WithTimeout(ctx, pgunit.ExecutionTimeout) - defer cancel() - d, err := s.queries.GetRuntimeDeployment(ctx) - if err != nil { - return SandboxSetup{}, err - } - return s.sandboxSetup(d) -} - -func (s *Store) sandboxSetup(d sqlc.RuntimeDeployment) (SandboxSetup, error) { - if !d.WebManaged { - return SandboxSetup{}, ErrSandboxDeploymentConflict - } - result := SandboxSetup{InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, BackendFingerprint: d.BackendFingerprint, IdleSeconds: d.IdleSeconds, RetentionSeconds: d.RetentionSeconds, Generation: uint64(d.Generation), Mode: d.Mode, AdmissionPaused: d.AdmissionPaused} - if err := json.Unmarshal(d.Specification, &result.Specification); err != nil { - return SandboxSetup{}, err - } - if d.ProviderKind != "" { - if err := providers.ValidateSpecification(d.ProviderKind, result.Specification); err != nil { - return SandboxSetup{}, err - } - } - if d.ProviderKind != "" { - var secret []byte - if len(d.ProviderCredential) > 0 { - var err error - secret, err = s.credentialCipher.OpenSandboxDeployment(d.ProviderCredential, result.InstallationID, result.Generation) - if err != nil { - return SandboxSetup{}, ErrSandboxCredentialUnavailable - } - } - var err error - result.Configuration, err = providers.Decode(d.ProviderKind, sandbox.ConfigurationRecord{Public: d.ProviderConfig, Metadata: d.ProviderMetadata, Secret: secret}) - if err != nil { - return SandboxSetup{}, ErrSandboxDeploymentConflict - } - } - - return result, nil -} - -// ClaimWebSandboxDeployment runs exactly once per execution-owner startup. It -// reserves the installation before selection and fences previous node presence. -func (s *Store) ClaimWebSandboxDeployment(ctx context.Context, installationID string) error { - if err := s.checkExecutionAuthority(); err != nil { - return err - } - id, err := parseConnectionGeneration(installationID) - if err != nil { - return ErrInvalidInput - } - return s.writer.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { - q := s.queries.WithTx(tx) - d, err := q.LockRuntimeDeployment(ctx) - if err != nil { - return err - } - if d.InstallationID.Valid { - if !d.WebManaged || d.InstallationID != id { - return ErrSandboxDeploymentConflict - } - } else { - resources, err := q.CountRuntimeDeploymentResources(ctx) - if err != nil { - return err - } - if resources.Allocations != 0 || resources.Pending != 0 { - return ErrSandboxDeploymentConflict - } - } - if d.ProviderKind != "" { - if _, err := deploymentSpecification(d); err != nil { - return err - } - } - return q.ClaimWebSandboxDeployment(ctx, id) - }) -} - -// ValidateSandboxCoreURL accepts a canonical public origin, never a path or -// credential. Plain HTTP is reserved for explicit loopback development hosts. -// OAC_PUBLIC_URL must pass it. -func ValidateSandboxCoreURL(value string) error { - u, err := url.Parse(value) - if err != nil || u.Hostname() == "" || u.User != nil || u.Path != "" || u.RawPath != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || u.RawFragment != "" || u.Opaque != "" || u.String() != value || u.Host != strings.ToLower(u.Host) { - return ErrInvalidInput - } - if strings.ContainsAny(u.Host, "\\% \t\r\n") || strings.HasSuffix(u.Host, ":") { - return ErrInvalidInput - } - if port := u.Port(); port != "" { - n, err := strconv.Atoi(port) - if err != nil || n < 1 || n > 65535 || strconv.Itoa(n) != port { - return ErrInvalidInput - } - } - loopback := u.Hostname() == "localhost" - if ip := net.ParseIP(u.Hostname()); ip != nil { - loopback = ip.IsLoopback() - } else { - if len(u.Hostname()) > 253 || strings.ContainsAny(u.Host, "[]") { - return ErrInvalidInput - } - for _, label := range strings.Split(u.Hostname(), ".") { - if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { - return ErrInvalidInput - } - for _, char := range label { - if (char < 'a' || char > 'z') && (char < '0' || char > '9') && char != '-' { - return ErrInvalidInput - } - } - } - } - if u.Scheme != "https" && !(u.Scheme == "http" && loopback) { - return ErrInvalidInput - } - return nil -} - -// LoopbackOrigin reports whether a validated origin names a loopback host, which -// nothing outside the Core host can reach. -func LoopbackOrigin(value string) bool { - u, err := url.Parse(value) - if err != nil { - return false - } - if ip := net.ParseIP(u.Hostname()); ip != nil { - return ip.IsLoopback() - } - return u.Hostname() == "localhost" -} - -func runtimeDeploymentView(d sqlc.RuntimeDeployment, publicURL string) (RuntimeDeploymentView, error) { - result := RuntimeDeploymentView{InstallationID: runtimeUUID(d.InstallationID), Provider: d.ProviderKind, CoreURL: publicURL, OwnerEpoch: uint64(d.OwnerEpoch), Generation: uint64(d.Generation), Mode: d.Mode} - if len(d.Specification) > 0 && string(d.Specification) != "{}" { - var spec sandbox.DeploymentSpec - if json.Unmarshal(d.Specification, &spec) == nil { - result.Specification = &spec - result.SpecificationDigest = spec.Digest(d.ProviderKind) - } - } - if d.ProviderKind != "" { - value, err := providers.Decode(d.ProviderKind, sandbox.ConfigurationRecord{Public: d.ProviderConfig, Metadata: d.ProviderMetadata}) - if err != nil { - return RuntimeDeploymentView{}, ErrSandboxDeploymentConflict - } - record, err := providers.Encode(d.ProviderKind, value) - if err != nil { - return RuntimeDeploymentView{}, ErrSandboxDeploymentConflict - } - result.Configuration = configurationJSON(record.Public) - result.Metadata = configurationJSON(record.Metadata) - result.CredentialConfigured = len(d.ProviderCredential) > 0 - } - - if providers.SupportsCheckpoint(d.ProviderKind) { - result.Suspension = &SandboxSuspensionView{IdleSeconds: d.IdleSeconds, RetentionSeconds: d.RetentionSeconds} - } - return result, nil -} diff --git a/services/core/internal/store/sandbox_deployment_setup_test.go b/services/core/internal/store/sandbox_deployment_setup_test.go index 73f20b911..5e1c39435 100644 --- a/services/core/internal/store/sandbox_deployment_setup_test.go +++ b/services/core/internal/store/sandbox_deployment_setup_test.go @@ -2,69 +2,49 @@ package store import ( "context" + "encoding/hex" "errors" - "fmt" "reflect" - "sync" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" - + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/google/uuid" ) -func TestSandboxCoreURLValidation(t *testing.T) { - // deploy/install/test_install.py checks the installer's valid_core_origin against the same cases. - for _, value := range []string{"https://core.example", "https://core.example:8443", "http://localhost:8091", "http://127.0.0.2:8091", "http://[::1]:8091", "https://[2001:db8::1]"} { - if err := ValidateSandboxCoreURL(value); err != nil { - t.Errorf("rejected %q: %v", value, err) - } - } - for _, value := range []string{"", "http://core.example", "http://core:8091", "http://host.localhost", "https://core.example/", "https://user:secret@core.example", "https://core.example/path", "https://core.example?", "https://core.example?q=x", "https://core.example#x", "https://core.example#", "https://CORE.example", "https://core.example:", "https://core.example:0", "https://core.example:65536", "https://core.example:0080", "https://core.example\\evil"} { - if err := ValidateSandboxCoreURL(value); !errors.Is(err, ErrInvalidInput) { - t.Errorf("accepted %q: %v", value, err) - } - } - for _, value := range []string{"https://[not-an-ip]", "https://-core.example", "https://core..example", "https://core_example", "https://core.example.", "https://bücher.example", "https://core.example:0443"} { - if err := ValidateSandboxCoreURL(value); !errors.Is(err, ErrInvalidInput) { - t.Errorf("accepted invalid hostname %q: %v", value, err) - } - } -} - func TestSandboxDeploymentSetupPersistsWithoutExecution(t *testing.T) { s, pool := newManagedTestStore(t) s.SetPublicURL("https://core.example") w := executionWriter(t, s) id := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + if err := deploymentExecution(t, w).Claim(t.Context(), id); err != nil { t.Fatal(err) } - before, err := s.GetRuntimeDeployment(t.Context()) + before, err := deploymentService(t, s).View(t.Context()) if err != nil || before.InstallationID != id || before.Provider != "" || before.CoreURL != "https://core.example" { t.Fatal(before, err) } - if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, ErrRuntimeNodeUnavailable) { + if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, deployment.ErrNodeUnavailable) { t.Fatal("uninitialized hosted admission", err) } - input := SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("microsandbox"), Provider: "microsandbox"} - selected, err := w.InitializeSandboxDeployment(t.Context(), id, input) + input := sandbox.Selection{DeploymentSpec: SandboxDeploymentTestSpec("microsandbox"), Provider: "microsandbox"} + selected, err := deploymentExecution(t, w).Initialize(t.Context(), id, input) if err != nil || selected.Provider != input.Provider || selected.CoreURL != "https://core.example" || selected.OwnerEpoch != before.OwnerEpoch { t.Fatal(selected, err) } input.ExpectedGeneration = selected.Generation - replay, err := w.InitializeSandboxDeployment(t.Context(), id, input) + replay, err := deploymentExecution(t, w).Initialize(t.Context(), id, input) if err != nil || !reflect.DeepEqual(replay, selected) { t.Fatal("identical retry changed selection", replay, err) } - for _, changed := range []SandboxDeploymentSetupRequest{{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}} { - if _, err := w.InitializeSandboxDeployment(t.Context(), id, changed); !errors.Is(err, ErrSandboxDeploymentConflict) { + for _, changed := range []sandbox.Selection{{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}} { + if _, err := deploymentExecution(t, w).Initialize(t.Context(), id, changed); !errors.Is(err, deployment.ErrConflict) { t.Fatal("changed selection accepted", err) } } - setup, err := s.GetSandboxSetup(t.Context()) - if err != nil || setup.IdleSeconds != 300 || setup.RetentionSeconds != 86400 || !validRuntimeDigest(setup.BackendFingerprint) { + setup, err := deploymentService(t, s).Setup(t.Context()) + if err != nil || setup.Suspension == nil || setup.Suspension.IdleSeconds != 300 || setup.Suspension.RetentionSeconds != 86400 || !sha256Hex(setup.BackendFingerprint) { t.Fatal(setup, err) } var sideEffects int @@ -77,95 +57,24 @@ func TestSandboxDeploymentSetupPersistsWithoutExecution(t *testing.T) { } awaitRelease() restarted := executionWriter(t, s) - if err := restarted.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + if err := deploymentExecution(t, restarted).Claim(t.Context(), id); err != nil { t.Fatal(err) } - after, err := s.GetSandboxSetup(t.Context()) + after, err := deploymentService(t, s).Setup(t.Context()) if err != nil || !reflect.DeepEqual(after, setup) { t.Fatal("restart lost configuration", after, err) } - epoch, err := s.RuntimeOwnerEpoch(t.Context()) + epoch, err := deploymentStore(s).OwnerEpoch(t.Context()) if err != nil || epoch != before.OwnerEpoch+1 { t.Fatal("restart did not fence node presence", epoch, err) } - if err := restarted.ClaimWebSandboxDeployment(t.Context(), uuid.NewString()); !errors.Is(err, ErrSandboxDeploymentConflict) { + if err := deploymentExecution(t, restarted).Claim(t.Context(), uuid.NewString()); !errors.Is(err, deployment.ErrConflict) { t.Fatal("installation replacement accepted", err) } } -func TestSandboxDeploymentSetupConcurrentSelection(t *testing.T) { - s, _ := newManagedTestStore(t) - w := executionWriter(t, s) - id := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { - t.Fatal(err) - } - var wg sync.WaitGroup - results := make(chan RuntimeDeploymentView, 20) - errorsFound := make(chan error, 20) - for i := range 20 { - wg.Add(1) - go func() { - defer wg.Done() - provider := "docker" - if i%2 == 1 { - provider = "microsandbox" - } - value, err := w.InitializeSandboxDeployment(t.Context(), id, SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec(provider), Provider: provider}) - results <- value - errorsFound <- err - }() - } - wg.Wait() - close(results) - close(errorsFound) - conflicts := 0 - for err := range errorsFound { - if errors.Is(err, ErrSandboxDeploymentConflict) { - conflicts++ - } else if err != nil { - t.Fatal(err) - } - } - if conflicts != 19 { - t.Fatal("both provider selections won", conflicts) - } - winner, err := s.GetSandboxSetup(t.Context()) - if err != nil { - t.Fatal(err) - } - for result := range results { - if result.Provider != "" && result.Provider != winner.Provider { - t.Fatal("inconsistent selection", result) - } - } -} - -func TestSandboxDeploymentSetupRejectsFileManagedAndUnleasedWrites(t *testing.T) { - s, w, selection := managerFixture(t, 4, 16) - input := SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"} - if _, err := s.InitializeSandboxDeployment(t.Context(), selection.InstallationID, input); !errors.Is(err, ErrExecutionAuthority) { - t.Fatal("unleased setup accepted", err) - } - if _, err := w.InitializeSandboxDeployment(t.Context(), selection.InstallationID, input); !errors.Is(err, ErrSandboxDeploymentConflict) { - t.Fatal("file-managed deployment changed", err) - } - if err := w.ClaimWebSandboxDeployment(t.Context(), selection.InstallationID); !errors.Is(err, ErrSandboxDeploymentConflict) { - t.Fatal("file-managed deployment adopted", err) - } -} - -func TestSandboxSelectionRejectsWhitespaceInE2BCredential(t *testing.T) { - for _, separator := range []string{" ", "\t", "\r", "\n", "\x00", "\u00a0", "\u2003", "\u3000"} { - t.Run(fmt.Sprintf("U+%04X", []rune(separator)[0]), func(t *testing.T) { - selection := e2bSelection() - selection.Configuration.(*e2b.DeploymentConfiguration).APIKey = "prefix" + separator + "suffix" - if err := validateSandboxSelection(selection); !errors.Is(err, ErrInvalidInput) { - t.Fatalf("credential containing whitespace or NUL accepted: %v", err) - } - }) - } - if err := validateSandboxSelection(e2bSelection()); err != nil { - t.Fatal(err) - } +// sha256Hex accepts a lowercase hexadecimal SHA-256 digest. +func sha256Hex(value string) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == 32 && hex.EncodeToString(decoded) == value } diff --git a/services/core/internal/store/sandbox_deployment_switch_test.go b/services/core/internal/store/sandbox_deployment_switch_test.go index 9e137dc8f..6bac6e3c7 100644 --- a/services/core/internal/store/sandbox_deployment_switch_test.go +++ b/services/core/internal/store/sandbox_deployment_switch_test.go @@ -2,53 +2,32 @@ package store import ( "bytes" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "strings" "sync" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" ) -func e2bSelection() SandboxDeploymentSetupRequest { - return SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("e2b"), Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-private-api-key", Template: "runtime:" + uuid.NewString()}} +func e2bSelection() sandbox.Selection { + return sandbox.Selection{DeploymentSpec: SandboxDeploymentTestSpec("e2b"), Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-private-api-key", Template: "runtime:" + uuid.NewString()}} } -func TestSandboxE2BEndpointPersistenceAndOnlineSwitch(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) - w := executionWriter(t, s) - id := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { - t.Fatal(err) - } - input := e2bSelection() - input.Configuration.(*e2b.DeploymentConfiguration).APIURL, input.Configuration.(*e2b.DeploymentConfiguration).Domain = "https://sandbox-test.sandbase.ai", "sandbox-test.sandbase.ai" - view, err := w.InitializeSandboxDeployment(t.Context(), id, input) - if err != nil || view.Configuration == nil || e2bPublicConfiguration(t, view).APIURL != input.Configuration.(*e2b.DeploymentConfiguration).APIURL || e2bPublicConfiguration(t, view).Domain != input.Configuration.(*e2b.DeploymentConfiguration).Domain { - t.Fatal("custom endpoint was not returned", view, err) - } - setup, err := s.GetSandboxSetup(t.Context()) - if err != nil || setup.Configuration == nil || setup.Configuration.(*e2b.DeploymentConfiguration).APIURL != input.Configuration.(*e2b.DeploymentConfiguration).APIURL || setup.Configuration.(*e2b.DeploymentConfiguration).Domain != input.Configuration.(*e2b.DeploymentConfiguration).Domain { - t.Fatal("custom endpoint was not persisted", setup, err) - } - change := e2bSelection() - change.Configuration.(*e2b.DeploymentConfiguration).Template = input.Configuration.(*e2b.DeploymentConfiguration).Template - update := SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: change, ExpectedGeneration: view.Generation} - changed, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), id, update) - if err != nil || changed.Generation != view.Generation+1 || changed.Configuration == nil || e2bPublicConfiguration(t, changed).APIURL != "https://api.e2b.app" || e2bPublicConfiguration(t, changed).Domain != "e2b.app" { - t.Fatal("online endpoint switch failed", changed, err) - } +// enrollmentTokenDigest is the stored form of an enrollment token. +func enrollmentTokenDigest(token string) string { + digest := sha256.Sum256([]byte(token)) + return hex.EncodeToString(digest[:]) } func TestSandboxResetClearsCustomE2BEndpoint(t *testing.T) { @@ -59,22 +38,30 @@ func TestSandboxResetClearsCustomE2BEndpoint(t *testing.T) { } s := NewWithCredentialCipher(pool, cipher) w := executionWriter(t, s) + changes := deploymentExecution(t, w) installation := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + if err := changes.Claim(t.Context(), installation); err != nil { t.Fatal(err) } input := e2bSelection() input.Configuration.(*e2b.DeploymentConfiguration).APIURL, input.Configuration.(*e2b.DeploymentConfiguration).Domain = "https://sandbox-test.sandbase.ai", "sandbox-test.sandbase.ai" - configured, err := w.InitializeSandboxDeployment(t.Context(), installation, input) + configured, err := changes.Initialize(t.Context(), installation, input) if err != nil { t.Fatal(err) } ctx := SandboxResetTestContext(t.Context()) - reset, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: configured.Generation, Clear: "force"}) - if err != nil { + if err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: configured.Generation, Clear: "force"}); err != nil { t.Fatal(err) } - empty, err := w.CompleteSandboxReset(ctx, installation, configured.Generation, reset.Reset.RequestedAt) + reset, err := deploymentService(t, s).View(ctx) + if err != nil || reset.Reset == nil { + t.Fatal("reset did not start", reset, err) + } + generation, err := w.CompleteSandboxReset(ctx, installation, configured.Generation, reset.Reset.RequestedAt) + if err != nil || generation != configured.Generation+1 { + t.Fatal("custom endpoint blocked reset completion", generation, err) + } + empty, err := deploymentService(t, s).View(ctx) if err != nil || empty.Provider != "" || empty.Reset != nil || empty.Generation != configured.Generation+1 { t.Fatal("custom endpoint blocked reset completion", empty, err) } @@ -92,12 +79,13 @@ func TestSandboxDirectDeploymentOwnershipAndCleanSwitch(t *testing.T) { } s := NewWithCredentialCipher(pool, cipher) w := executionWriter(t, s) + changes := deploymentExecution(t, w) id := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + if err := changes.Claim(t.Context(), id); err != nil { t.Fatal(err) } input := e2bSelection() - view, err := w.InitializeSandboxDeployment(t.Context(), id, input) + view, err := changes.Initialize(t.Context(), id, input) if err != nil || view.Generation != 1 || view.Mode != "direct" || view.Configuration == nil || !view.CredentialConfigured { t.Fatal("direct setup", view, err) } @@ -109,11 +97,11 @@ func TestSandboxDirectDeploymentOwnershipAndCleanSwitch(t *testing.T) { if err := pool.QueryRow(t.Context(), "SELECT provider_credential FROM runtime_deployment").Scan(&ciphertext); err != nil || bytes.Contains(ciphertext, []byte(input.Configuration.(*e2b.DeploymentConfiguration).APIKey)) { t.Fatal("credential not encrypted", err) } - setup, err := s.GetSandboxSetup(t.Context()) + setup, err := deploymentService(t, s).Setup(t.Context()) if err != nil || setup.Configuration.(*e2b.DeploymentConfiguration).APIKey != input.Configuration.(*e2b.DeploymentConfiguration).APIKey { t.Fatal("internal credential unavailable", err) } - if _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 8}); !errors.Is(err, ErrSandboxDeploymentConflict) { + if _, err := deploymentService(t, s).CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 2, MaxRetained: 8}); !errors.Is(err, deployment.ErrConflict) { t.Fatal("cloud enrolled a machine", err) } tenant := uuid.NewString() @@ -141,11 +129,11 @@ func TestSandboxDirectDeploymentOwnershipAndCleanSwitch(t *testing.T) { if err != nil || observed.Expired { t.Fatal("cloud inherited legacy node-less expiry", err) } - if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { + if err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { t.Fatal(err) } - update := SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}, ExpectedGeneration: 1} - if _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), id, update); !errors.Is(err, ErrSandboxResetInProgress) { + update := sandbox.Selection{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker", ExpectedGeneration: 1} + if _, err := changes.Update(SandboxResetTestContext(t.Context()), id, update); !errors.Is(err, deployment.ErrResetInProgress) { t.Fatal("reset allowed switch", err) } if _, err := w.RequestRuntimeCleanup(t.Context(), owner); err != nil { @@ -160,11 +148,11 @@ func TestSandboxDirectDeploymentOwnershipAndCleanSwitch(t *testing.T) { if _, err := w.ReleaseRuntimeAllocation(t.Context(), owner); err != nil { t.Fatal(err) } - changed, err := resetAndSelect(t, w, id, update.ExpectedGeneration, update.SandboxDeploymentSetupRequest) - if err != nil || changed.Generation != 3 || changed.Mode != "nodes" || changed.Reset != nil || string(changed.Configuration) != "{}" || changed.Resources != (SandboxDeploymentResources{}) { + changed, err := resetAndSelect(t, w, id, update.ExpectedGeneration, update) + if err != nil || changed.Generation != 3 || changed.Mode != "nodes" || changed.Reset != nil || string(changed.Configuration) != "{}" || changed.Resources != (deployment.Resources{}) { t.Fatal("clean switch", changed, err) } - if _, err := w.CancelSandboxReset(SandboxResetTestContext(t.Context()), id, 1); !errors.Is(err, ErrSandboxDeploymentConflict) { + if err := w.CancelSandboxReset(SandboxResetTestContext(t.Context()), id, 1); !errors.Is(err, deployment.ErrConflict) { t.Fatal("stale resume accepted", err) } if _, err := s.GetSession(t.Context(), tenant, session.ID); err != nil { @@ -177,41 +165,43 @@ func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{5}, 32)) s := NewWithCredentialCipher(pool, cipher) w := executionWriter(t, s) + changes := deploymentExecution(t, w) + nodes := deploymentService(t, s) id := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + if err := changes.Claim(t.Context(), id); err != nil { t.Fatal(err) } - if _, err := w.InitializeSandboxDeployment(t.Context(), id, SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { + if _, err := changes.Initialize(t.Context(), id, sandbox.Selection{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } - token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4})) + token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 2, MaxRetained: 4})) if err != nil { t.Fatal(err) } - node := RuntimeNodeEnrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: uuid.NewString(), Name: "Machine", Provider: "docker", Credential: strings.Repeat("c", 64), BackendFingerprint: strings.Repeat("b", 64)} - if _, err := s.EnrollRuntimeNode(t.Context(), token, node); err != nil { + node := deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: uuid.NewString(), Name: "Machine", Provider: "docker", Credential: strings.Repeat("c", 64), BackendFingerprint: strings.Repeat("b", 64)} + if _, err := nodes.Enroll(t.Context(), token, node); err != nil { t.Fatal(err) } - unused, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 8})) + unused, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 2, MaxRetained: 8})) if err != nil { t.Fatal(err) } - if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { + if err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { t.Fatal(err) } // AdmissionPaused rejects a valid enrollment without consuming it. Authentication // still precedes deployment details for invalid or retired credentials. spareNode := node spareNode.NodeID = uuid.NewString() - if _, err := s.EnrollRuntimeNode(t.Context(), unused, spareNode); !errors.Is(err, ErrSandboxResetInProgress) { + if _, err := nodes.Enroll(t.Context(), unused, spareNode); !errors.Is(err, deployment.ErrResetInProgress) { t.Fatal("reset accepted enrollment", err) } var consumed bool - if err := pool.QueryRow(t.Context(), "SELECT consumed_at IS NOT NULL FROM runtime_node_enrollments WHERE token_sha256=$1", runtimeTokenDigest(unused)).Scan(&consumed); err != nil || consumed { + if err := pool.QueryRow(t.Context(), "SELECT consumed_at IS NOT NULL FROM runtime_node_enrollments WHERE token_sha256=$1", enrollmentTokenDigest(unused)).Scan(&consumed); err != nil || consumed { t.Fatal("maintenance consumed enrollment", err) } spareNode.Provider = "microsandbox" - if _, err := s.EnrollRuntimeNode(t.Context(), strings.Repeat("invalid", 8), spareNode); !errors.Is(err, ErrRuntimeNodeCredential) { + if _, err := nodes.Enroll(t.Context(), strings.Repeat("invalid", 8), spareNode); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("invalid token disclosed deployment validation", err) } spareNode.Provider = "docker" @@ -220,27 +210,27 @@ func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { t.Fatal(err) } - if _, err := s.EnrollRuntimeNode(t.Context(), unused, spareNode); !errors.Is(err, ErrRuntimeNodeCredential) { + if _, err := nodes.Enroll(t.Context(), unused, spareNode); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("retired token did not reject before cloud deployment validation", err) } - if _, err := s.AuthenticateRuntimeNode(t.Context(), node.NodeID, node.Credential); !errors.Is(err, ErrRuntimeNodeCredential) { + if _, err := nodes.AuthenticateNode(t.Context(), node.NodeID, node.Credential); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("old node credential survived", err) } - if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 3}); err != nil { + if err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 3}); err != nil { t.Fatal(err) } - if _, err := resetAndSelect(t, w, id, 3, SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { + if _, err := resetAndSelect(t, w, id, 3, sandbox.Selection{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } - if _, err := w.CancelSandboxReset(SandboxResetTestContext(t.Context()), id, 5); err != nil { + if err := w.CancelSandboxReset(SandboxResetTestContext(t.Context()), id, 5); err != nil { t.Fatal(err) } node.NodeID = uuid.NewString() - if _, err := s.EnrollRuntimeNode(t.Context(), unused, node); !errors.Is(err, ErrRuntimeNodeCredential) { + if _, err := nodes.Enroll(t.Context(), unused, node); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("old enrollment survived roundtrip", err) } - nodes, err := s.ListRuntimeNodes(t.Context()) - if err != nil || len(nodes) != 0 { + listed, err := nodes.ListNodes(t.Context()) + if err != nil || len(listed) != 0 { t.Fatal("retired nodes reappeared", err) } } @@ -250,12 +240,13 @@ func TestSandboxResetSerializesFreshDirectSessions(t *testing.T) { cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{6}, 32)) s := NewWithCredentialCipher(pool, cipher) w := executionWriter(t, s) + changes := deploymentExecution(t, w) id := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + if err := changes.Claim(t.Context(), id); err != nil { t.Fatal(err) } input := e2bSelection() - if _, err := w.InitializeSandboxDeployment(t.Context(), id, input); err != nil { + if _, err := changes.Initialize(t.Context(), id, input); err != nil { t.Fatal(err) } var wg sync.WaitGroup @@ -264,12 +255,12 @@ func TestSandboxResetSerializesFreshDirectSessions(t *testing.T) { go func() { defer wg.Done() _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())) - if err != nil && !errors.Is(err, ErrSandboxResetAdmission) && !errors.Is(err, ErrRuntimeNodeUnavailable) { + if err != nil && !errors.Is(err, ErrSandboxResetAdmission) && !errors.Is(err, deployment.ErrNodeUnavailable) { t.Error(err) } }() } - if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { + if err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), id, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { t.Fatal(err) } wg.Wait() @@ -278,12 +269,12 @@ func TestSandboxResetSerializesFreshDirectSessions(t *testing.T) { t.Fatal("fresh creation bypassed reset", err) } } - view, err := s.GetRuntimeDeployment(t.Context()) + view, err := deploymentService(t, s).View(t.Context()) if err != nil { t.Fatal(err) } - _, err = w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}, ExpectedGeneration: 1}) - if view.Resources.Pending > 0 && !errors.Is(err, ErrSandboxResetInProgress) { + _, err = changes.Update(SandboxResetTestContext(t.Context()), id, sandbox.Selection{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker", ExpectedGeneration: 1}) + if view.Resources.Pending > 0 && !errors.Is(err, deployment.ErrResetInProgress) { t.Fatal("committed pending Session bypassed switch guard", err) } } @@ -293,12 +284,13 @@ func TestSandboxSwitchPreservesReleasedAllocationAndItemHistory(t *testing.T) { cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{8}, 32)) s := NewWithCredentialCipher(pool, cipher) w := executionWriter(t, s) + changes := deploymentExecution(t, w) installation := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), installation); err != nil { + if err := changes.Claim(t.Context(), installation); err != nil { t.Fatal(err) } selection := e2bSelection() - if _, err := w.InitializeSandboxDeployment(t.Context(), installation, selection); err != nil { + if _, err := changes.Initialize(t.Context(), installation, selection); err != nil { t.Fatal(err) } tenant := uuid.NewString() @@ -340,10 +332,10 @@ func TestSandboxSwitchPreservesReleasedAllocationAndItemHistory(t *testing.T) { if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(a) FROM runtime_allocations a WHERE id=$1", owner.ID).Scan(&allocationBefore); err != nil { t.Fatal(err) } - if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { + if err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { t.Fatal(err) } - if _, err := resetAndSelect(t, w, installation, 1, SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { + if _, err := resetAndSelect(t, w, installation, 1, sandbox.Selection{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } items, err = s.ListItems(t.Context(), tenant, history.ID, "", 100, true) @@ -377,21 +369,23 @@ func TestUnspecifiedNodeDeploymentRejectedWithoutMutation(t *testing.T) { cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) s := NewWithCredentialCipher(pool, cipher) w := executionWriter(t, s) + changes := deploymentExecution(t, w) + nodes := deploymentService(t, s) id := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + if err := changes.Claim(t.Context(), id); err != nil { t.Fatal(err) } spec := SandboxDeploymentTestSpec("docker") - selection := SandboxDeploymentSetupRequest{DeploymentSpec: spec, Provider: "docker"} - if _, err := w.InitializeSandboxDeployment(t.Context(), id, selection); err != nil { + selection := sandbox.Selection{DeploymentSpec: spec, Provider: "docker"} + if _, err := changes.Initialize(t.Context(), id, selection); err != nil { t.Fatal(err) } - token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4})) + token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 2, MaxRetained: 4})) if err != nil { t.Fatal(err) } - node := RuntimeNodeEnrollment{DeploymentGeneration: 1, SpecificationDigest: spec.Digest("docker"), NodeID: uuid.NewString(), Name: "Legacy", Provider: "docker", Credential: strings.Repeat("l", 64), BackendFingerprint: strings.Repeat("b", 64)} - if _, err := s.EnrollRuntimeNode(t.Context(), token, node); err != nil { + node := deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: spec.Digest("docker"), NodeID: uuid.NewString(), Name: "Legacy", Provider: "docker", Credential: strings.Repeat("l", 64), BackendFingerprint: strings.Repeat("b", 64)} + if _, err := nodes.Enroll(t.Context(), token, node); err != nil { t.Fatal(err) } if _, err := pool.Exec(t.Context(), "UPDATE runtime_deployment SET specification='{}'"); err != nil { @@ -401,24 +395,24 @@ func TestUnspecifiedNodeDeploymentRejectedWithoutMutation(t *testing.T) { t.Fatal(err) } - if _, err := s.GetSandboxSetup(t.Context()); err == nil { + if _, err := nodes.Setup(t.Context()); err == nil { t.Fatal("missing deployment specification accepted") } - if _, err := s.AuthenticateRuntimeNode(t.Context(), node.NodeID, node.Credential); !errors.Is(err, ErrRuntimeSpecificationMismatch) { + if _, err := nodes.AuthenticateNode(t.Context(), node.NodeID, node.Credential); !errors.Is(err, deployment.ErrSpecificationMismatch) { t.Fatal("unspecified node authenticated", err) } - if _, err := s.RuntimeNodeConfiguration(t.Context(), node.NodeID, node.Credential); !errors.Is(err, ErrRuntimeSpecificationMismatch) { + if _, err := nodes.NodeConfiguration(t.Context(), node.NodeID, node.Credential, 0); !errors.Is(err, deployment.ErrSpecificationMismatch) { t.Fatal("node configuration served without a specification", err) } if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, ErrEnvironmentUnavailable) { t.Fatal("unspecified deployment admitted a fresh sandbox", err) } - if _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4}); !errors.Is(err, ErrSandboxDeploymentConflict) { + if _, err := nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 2, MaxRetained: 4}); !errors.Is(err, deployment.ErrConflict) { t.Fatal("unspecified deployment issued an enrollment token", err) } epoch := managerEpoch(t, s) - if err := w.ClaimWebSandboxDeployment(t.Context(), id); err == nil { + if err := changes.Claim(t.Context(), id); err == nil { t.Fatal("unsupported installation claimed") } if managerEpoch(t, s) != epoch { diff --git a/services/core/internal/store/sandbox_deployment_switch_worker_test.go b/services/core/internal/store/sandbox_deployment_switch_worker_test.go index 7d4224079..09f78cbc5 100644 --- a/services/core/internal/store/sandbox_deployment_switch_worker_test.go +++ b/services/core/internal/store/sandbox_deployment_switch_worker_test.go @@ -13,6 +13,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -24,17 +25,18 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { _, pool := store.NewManagedTestStore(t) cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) s, db := store.NewWithCredentialCipher(pool, cipher), fixtureDB{pool: pool, cipher: cipher} + deployments := fixtureDeployment(t, db) id := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}} var fail atomic.Bool var preparations atomic.Int32 configuration := execution.NewDeferredRuntimeProvider(id, func(ctx context.Context) (*execution.RuntimeProvider, error) { - setup, err := s.GetSandboxSetup(ctx) + setup, err := deployments.Setup(ctx) if err != nil || setup.Provider == "" { return nil, err } return &execution.RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}, nil - }, func(ctx context.Context, setup store.SandboxSetup) (execution.PreparedRuntimeDeployment, error) { + }, func(ctx context.Context, setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { preparations.Add(1) if fail.Load() { return execution.PreparedRuntimeDeployment{}, errors.New("fixture provider unavailable") @@ -54,45 +56,44 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { } }) fail.Store(true) - if _, err := w.InitializeSandboxDeployment(t.Context(), store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err == nil { + if _, err := w.InitializeSandboxDeployment(t.Context(), sandbox.Selection{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err == nil { t.Fatal("rejected initial provider configuration was committed") } - empty, err := s.GetRuntimeDeployment(t.Context()) - if err != nil || empty.Provider != "" || empty.Generation != 0 || empty.Specification != nil || empty.Resources != (store.SandboxDeploymentResources{}) { + empty, err := deployments.View(t.Context()) + if err != nil || empty.Provider != "" || empty.Generation != 0 || empty.Specification != nil || empty.Resources != (deployment.Resources{}) { t.Fatal("failed initial candidate changed the deployment", err) } fail.Store(false) - if _, err := w.InitializeSandboxDeployment(t.Context(), store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { + if _, err := w.InitializeSandboxDeployment(t.Context(), sandbox.Selection{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } prepared := preparations.Load() - if _, err := w.InitializeSandboxDeployment(t.Context(), store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err == nil || preparations.Load() != prepared { + if _, err := w.InitializeSandboxDeployment(t.Context(), sandbox.Selection{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err == nil || preparations.Load() != prepared { t.Fatal("stale identical POST reached provider preparation", err) } - enrollment, err := store.EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), store.RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4})) + enrollment, err := store.EnrollmentTestToken(deployments.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 2, MaxRetained: 4})) if err != nil { t.Fatal(err) } - node := store.RuntimeNodeEnrollment{NodeID: uuid.NewString(), Name: "retained candidate fixture", Provider: "docker", Credential: strings.Repeat("n", 64), BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: 1, SpecificationDigest: store.SandboxDeploymentTestSpec("docker").Digest("docker")} - if _, err := s.EnrollRuntimeNode(t.Context(), enrollment, node); err != nil { + node := deployment.Enrollment{NodeID: uuid.NewString(), Name: "retained candidate fixture", Provider: "docker", Credential: strings.Repeat("n", 64), BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: 1, SpecificationDigest: store.SandboxDeploymentTestSpec("docker").Digest("docker")} + if _, err := deployments.Enroll(t.Context(), enrollment, node); err != nil { t.Fatal(err) } - previous, err := s.GetRuntimeDeployment(t.Context()) + previous, err := deployments.View(t.Context()) if err != nil { t.Fatal(err) } - changed := store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"} - changed.Resources.CPUs++ - input := store.SandboxDeploymentUpdateRequest{ExpectedGeneration: 1, SandboxDeploymentSetupRequest: changed} + input := sandbox.Selection{ExpectedGeneration: 1, DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"} + input.Resources.CPUs++ fail.Store(true) if _, err := w.UpdateSandboxDeployment(store.SandboxResetTestContext(t.Context()), input); err == nil { t.Fatal("failed activation reported success") } - view, err := s.GetRuntimeDeployment(t.Context()) + view, err := deployments.View(t.Context()) if err != nil || !reflect.DeepEqual(view, previous) { t.Fatal("failed candidate changed committed configuration", view, err) } - if _, err := s.AuthenticateRuntimeNode(t.Context(), node.NodeID, node.Credential); err != nil { + if _, err := deployments.AuthenticateNode(t.Context(), node.NodeID, node.Credential); err != nil { t.Fatal("rejected candidate retired the previous node", err) } fail.Store(false) @@ -110,20 +111,20 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { if err := w.ReconcileManagedRuntimes(t.Context()); err != nil { t.Fatal("failed commit left manager barrier closed", err) } - if view, err := s.GetRuntimeDeployment(t.Context()); err != nil || view.Generation != 1 || view.Provider != "docker" { + if view, err := deployments.View(t.Context()); err != nil || view.Generation != 1 || view.Provider != "docker" { t.Fatal("failed commit replaced provider", view, err) } if _, err := w.UpdateSandboxDeployment(store.SandboxResetTestContext(t.Context()), input); err != nil { t.Fatal(err) } - reset := func(generation uint64) store.RuntimeDeploymentView { + reset := func(generation uint64) deployment.View { t.Helper() if _, err := w.StartSandboxReset(store.SandboxResetTestContext(t.Context()), store.SandboxResetRequest{ExpectedGeneration: generation, Clear: "force"}); err != nil { t.Fatal(err) } deadline := time.Now().Add(12 * time.Second) for time.Now().Before(deadline) { - view, err := s.GetRuntimeDeployment(t.Context()) + view, err := deployments.View(t.Context()) if err != nil { t.Fatal(err) } @@ -137,10 +138,10 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { } } t.Fatal("reset completion blocked (including possible manager self-drain)") - return store.RuntimeDeploymentView{} + return deployment.View{} } empty = reset(2) - cloud := store.SandboxDeploymentSetupRequest{ExpectedGeneration: empty.Generation, DeploymentSpec: store.SandboxDeploymentTestSpec("e2b"), Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-api-key", Template: "runtime:" + uuid.NewString()}} + cloud := sandbox.Selection{ExpectedGeneration: empty.Generation, DeploymentSpec: store.SandboxDeploymentTestSpec("e2b"), Provider: "e2b", Configuration: &e2b.DeploymentConfiguration{APIKey: "fixture-api-key", Template: "runtime:" + uuid.NewString()}} if _, err := w.InitializeSandboxDeployment(t.Context(), cloud); err != nil { t.Fatal("setup after unconfigured publication", err) } @@ -149,8 +150,8 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { if err != nil || allocation.NodeID != "" || allocation.State != "running" { t.Fatal("direct provider not available after resume", allocation, err) } - next := store.SandboxDeploymentUpdateRequest{ExpectedGeneration: 4, SandboxDeploymentSetupRequest: store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}} - if _, err := w.UpdateSandboxDeployment(store.SandboxResetTestContext(t.Context()), next); !errors.Is(err, store.ErrSandboxDeploymentConflict) { + next := sandbox.Selection{ExpectedGeneration: 4, DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"} + if _, err := w.UpdateSandboxDeployment(store.SandboxResetTestContext(t.Context()), next); !errors.Is(err, deployment.ErrConflict) { t.Fatal("dirty switch accepted", err) } if err := s.DeleteSession(t.Context(), tenant, session.ID); err != nil { @@ -158,8 +159,8 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { } reconcileManagedState(t, w, s, tenant, environment.ID, "released") empty = reset(4) - next.SandboxDeploymentSetupRequest.ExpectedGeneration = empty.Generation - if _, err := w.InitializeSandboxDeployment(t.Context(), next.SandboxDeploymentSetupRequest); err != nil { + next.ExpectedGeneration = empty.Generation + if _, err := w.InitializeSandboxDeployment(t.Context(), next); err != nil { t.Fatal("clean setup after reset", err) } select { diff --git a/services/core/internal/store/sandbox_deployment_view_test.go b/services/core/internal/store/sandbox_deployment_view_test.go index 7626d2a0e..6414b5ceb 100644 --- a/services/core/internal/store/sandbox_deployment_view_test.go +++ b/services/core/internal/store/sandbox_deployment_view_test.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/google/uuid" ) @@ -21,20 +22,21 @@ func TestSandboxDeploymentViewRecordsTemplateBuildAndSuspension(t *testing.T) { } s := NewWithCredentialCipher(pool, cipher) w := executionWriter(t, s) + changes := deploymentExecution(t, w) id := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + if err := changes.Claim(t.Context(), id); err != nil { t.Fatal(err) } input := e2bSelection() input.Resources = sandbox.Resources{} - var invalid *SandboxConfigurationError - if _, err := w.InitializeSandboxDeployment(t.Context(), id, input); !errors.As(err, &invalid) { + var invalid *deployment.ConfigurationError + if _, err := changes.Initialize(t.Context(), id, input); !errors.As(err, &invalid) { t.Fatal("omitted E2B resources were stored without a validated build", err) } disk := int32(24063) input.Resources = sandbox.Resources{CPUs: 2, MemoryMiB: 2048} input.Configuration.(*e2b.DeploymentConfiguration).TemplateBuild = &e2b.DeploymentBuild{Status: "ready", CPUs: 2, MemoryMiB: 2048, RootDiskMiB: &disk} - view, err := w.InitializeSandboxDeployment(t.Context(), id, input) + view, err := changes.Initialize(t.Context(), id, input) if err != nil { t.Fatal(err) } @@ -58,32 +60,30 @@ func TestSandboxDeploymentViewRecordsTemplateBuildAndSuspension(t *testing.T) { } recorded := []byte(`"template_build":{"status":"ready","resources":{"cpus":2,"memory_mib":2048,"root_disk_mib":24063}}`) forget() - view, err = s.GetRuntimeDeployment(t.Context()) + view, err = deploymentService(t, s).View(t.Context()) raw, _ = json.Marshal(view) if err != nil || !bytes.Contains(raw, []byte(`"metadata":{}`)) { t.Fatalf("unknown build was not null: %s %v", raw, err) } input.ExpectedGeneration = 1 - view, err = w.InitializeSandboxDeployment(t.Context(), id, input) + view, err = changes.Initialize(t.Context(), id, input) raw, _ = json.Marshal(view) if err != nil || view.Generation != 1 || !bytes.Contains(raw, recorded) { t.Fatalf("identical POST did not record the build: %s %v", raw, err) } forget() - view, err = w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), id, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}) + view, err = changes.Update(SandboxResetTestContext(t.Context()), id, input) raw, _ = json.Marshal(view) if err != nil || view.Generation != 1 || !bytes.Contains(raw, recorded) { t.Fatalf("identical PUT did not record the build: %s %v", raw, err) } - update := SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: SandboxDeploymentSetupRequest{ - DeploymentSpec: SandboxDeploymentTestSpec("microsandbox"), Provider: "microsandbox"}, ExpectedGeneration: 1} - view, err = resetAndSelect(t, w, id, update.ExpectedGeneration, update.SandboxDeploymentSetupRequest) + view, err = resetAndSelect(t, w, id, 1, sandbox.Selection{DeploymentSpec: SandboxDeploymentTestSpec("microsandbox"), Provider: "microsandbox"}) if err != nil || string(view.Configuration) != "{}" || view.Suspension == nil || view.Suspension.IdleSeconds != 300 || view.Suspension.RetentionSeconds != 86400 { t.Fatalf("microsandbox suspension view = %+v %v", view, err) } } -func e2bPublicConfiguration(t *testing.T, v RuntimeDeploymentView) *e2b.DeploymentConfiguration { +func e2bPublicConfiguration(t *testing.T, v deployment.View) *e2b.DeploymentConfiguration { t.Helper() c, err := (e2b.ConfigurationAdapter{}).Decode(sandbox.ConfigurationRecord{Public: v.Configuration, Metadata: v.Metadata}) if err != nil { diff --git a/services/core/internal/store/sandbox_deployment_worker_test.go b/services/core/internal/store/sandbox_deployment_worker_test.go index a8bd14f2f..47f176bd9 100644 --- a/services/core/internal/store/sandbox_deployment_worker_test.go +++ b/services/core/internal/store/sandbox_deployment_worker_test.go @@ -8,6 +8,7 @@ import ( "sync" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -17,15 +18,16 @@ import ( func TestSandboxDeploymentWorkerActivatesWithoutRestart(t *testing.T) { s, db := newManagedTestStoreDB(t) + deployments := fixtureDeployment(t, db) id := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}} configuration := execution.NewDeferredRuntimeProvider(id, func(ctx context.Context) (*execution.RuntimeProvider, error) { - setup, err := s.GetSandboxSetup(ctx) + setup, err := deployments.Setup(ctx) if err != nil || setup.Provider == "" { return nil, err } return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, BackendFingerprint: setup.BackendFingerprint, CoreURL: "https://core.example/api/v1", Provider: p}, nil - }, func(ctx context.Context, setup store.SandboxSetup) (execution.PreparedRuntimeDeployment, error) { + }, func(ctx context.Context, setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil }) @@ -44,31 +46,28 @@ func TestSandboxDeploymentWorkerActivatesWithoutRestart(t *testing.T) { if _, err := w.CreateSession(t.Context(), uuid.NewString(), input); !errors.Is(err, execution.ErrExecutionUnavailable) { t.Fatal("uninitialized worker admitted hosted Session", err) } - if _, err := w.InitializeSandboxDeployment(t.Context(), store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { + if _, err := w.InitializeSandboxDeployment(t.Context(), sandbox.Selection{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } - if _, err := s.CreateSession(t.Context(), uuid.NewString(), input); !errors.Is(err, store.ErrRuntimeNodeUnavailable) { + if _, err := s.CreateSession(t.Context(), uuid.NewString(), input); !errors.Is(err, deployment.ErrNodeUnavailable) { t.Fatal("zero-node deployment admitted Session", err) } - token, err := store.EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), store.RuntimeNodeCapacity{MaxActive: 4, MaxRetained: 16})) + token, err := store.EnrollmentTestToken(deployments.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 4, MaxRetained: 16})) if err != nil { t.Fatal(err) } nodeID := uuid.NewString() - if _, err := s.EnrollRuntimeNode(t.Context(), token, store.RuntimeNodeEnrollment{DeploymentGeneration: 1, SpecificationDigest: store.SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: nodeID, Name: "Remote", Provider: "docker", Credential: strings.Repeat("x", 64), BackendFingerprint: strings.Repeat("b", 64)}); err != nil { + if _, err := deployments.Enroll(t.Context(), token, deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: store.SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: nodeID, Name: "Remote", Provider: "docker", Credential: strings.Repeat("x", 64), BackendFingerprint: strings.Repeat("b", 64)}); err != nil { t.Fatal(err) } connect := func() { t.Helper() - epoch, err := s.RuntimeOwnerEpoch(t.Context()) - if err != nil { - t.Fatal(err) - } + epoch := fixtureOwnerEpoch(t, db) connection := uuid.NewString() - if err := s.ConnectRuntimeNode(t.Context(), nodeID, connection, epoch); err != nil { + if err := deployments.ConnectNode(t.Context(), nodeID, connection, epoch); err != nil { t.Fatal(err) } - if err := s.HeartbeatRuntimeNode(t.Context(), nodeID, connection, epoch, store.RuntimeNodeHealth{ProviderReady: true}); err != nil { + if err := deployments.Heartbeat(t.Context(), nodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { t.Fatal(err) } } diff --git a/services/core/internal/store/sandbox_generations.go b/services/core/internal/store/sandbox_generations.go deleted file mode 100644 index fd5742fb8..000000000 --- a/services/core/internal/store/sandbox_generations.go +++ /dev/null @@ -1,159 +0,0 @@ -package store - -import ( - "context" - "encoding/json" - "errors" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" -) - -// ClassifySandboxDeploymentChange resolves an omitted credential before validation. -// Explicit key submission is a verified mutation even when its bytes are unchanged. -func (s *Store) ClassifySandboxDeploymentChange(ctx context.Context, installation string, input SandboxDeploymentUpdateRequest) (SandboxDeploymentUpdateRequest, bool, error) { - var unchanged bool - err := s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { - if err := checkSandboxSwitch(ctx, q, d, installation, input); err != nil { - return err - } - previous, err := s.sandboxSetup(d) - if err != nil { - return err - } - resolved, err := providers.ResolveChange(input.SandboxDeploymentSetupRequest, sandbox.Selection{Provider: previous.Provider, DeploymentSpec: previous.Specification, Configuration: previous.Configuration}) - if err != nil { - return sandboxConfigurationError(err) - } - input.SandboxDeploymentSetupRequest = resolved - if err := validateSandboxSelection(input.SandboxDeploymentSetupRequest); err != nil { - return err - } - equal, err := s.sandboxSelectionEqual(d, input.SandboxDeploymentSetupRequest) - unchanged = equal && !input.ReplacesCredential() - return err - }) - return input, unchanged, err -} - -// GetSandboxAllocationSetup reads immutable ownership and the current credential -// in one snapshot. A released receipt remains historical but is never rebound. -func (s *Store) GetSandboxAllocationSetup(ctx context.Context, ref sandbox.Reference) (SandboxSetup, error) { - var result SandboxSetup - err := s.pooled.Snapshot(ctx, func(ctx context.Context, tx pgx.Tx) error { - q := s.queries.WithTx(tx) - lookup, err := deviceLookup(ref.TenantID, ref.EnvironmentID) - if err != nil { - return err - } - a, err := q.GetRuntimeAllocation(ctx, sqlc.GetRuntimeAllocationParams{TenantID: lookup.TenantID, EnvironmentID: lookup.ID}) - if err != nil { - return err - } - if runtimeUUID(a.RuntimeAllocation.ID) != ref.AllocationID || !a.RuntimeAllocation.DeploymentGeneration.Valid || a.RuntimeAllocation.State == "released" { - return ErrInvalidInput - } - d, err := q.GetRuntimeDeployment(ctx) - if err != nil { - return err - } - if a.RuntimeAllocation.ProviderKey != d.InstallationID { - return sandbox.ErrOwnership - } - result, err = s.sandboxSetup(d) - if err != nil { - return err - } - generation := a.RuntimeAllocation.DeploymentGeneration.Int64 - if generation != d.Generation { - g, err := q.GetSandboxGeneration(ctx, generation) - if errors.Is(err, pgx.ErrNoRows) { - return ErrSandboxDeploymentConflict - } - if err != nil { - return err - } - if g.ProviderKind != d.ProviderKind { - return ErrSandboxDeploymentConflict - } - result.Generation = uint64(generation) - if err = json.Unmarshal(g.Specification, &result.Specification); err != nil { - return err - } - retained, err := providers.Decode(g.ProviderKind, sandbox.ConfigurationRecord{Public: g.ProviderConfig, Metadata: g.ProviderMetadata}) - if err != nil { - return ErrSandboxDeploymentConflict - } - needsCredential, err := providers.UsesCredential(g.ProviderKind) - if err != nil { - return err - } - if needsCredential { - composed, err := providers.WithCredential(sandbox.Selection{Provider: g.ProviderKind, Configuration: retained}, sandbox.Selection{Provider: d.ProviderKind, Configuration: result.Configuration}) - if err != nil { - return ErrSandboxDeploymentConflict - } - retained = composed.Configuration - } - result.Configuration = retained - } - return nil - }) - if err != nil { - return SandboxSetup{}, err - } - return result, nil -} - -// SandboxGenerationPage is bounded; callers retain a deadline over the full scan. -func (s *Store) SandboxGenerationPage(ctx context.Context, after int64) ([]SandboxSetup, error) { - rows, err := s.queries.ListSandboxGenerations(ctx, after) - if err != nil { - return nil, err - } - result := make([]SandboxSetup, 0, len(rows)) - for _, r := range rows { - v := SandboxSetup{Generation: uint64(r.Generation), Provider: r.ProviderKind} - if err := json.Unmarshal(r.Specification, &v.Specification); err != nil { - return nil, err - } - v.Configuration, err = providers.Decode(r.ProviderKind, sandbox.ConfigurationRecord{Public: r.ProviderConfig, Metadata: r.ProviderMetadata}) - if err != nil { - return nil, ErrSandboxDeploymentConflict - } - - result = append(result, v) - } - return result, nil -} - -// CollectSandboxGenerations shares the same serialization as admission and PUT. -func (s *Store) CollectSandboxGenerations(ctx context.Context) error { - return s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, _ sqlc.RuntimeDeployment) error { - return q.CollectSandboxGenerations(ctx) - }) -} - -// SandboxCredentialAllocationPage reads owned receipts without granting execution authority. -func (s *Store) SandboxCredentialAllocationPage(ctx context.Context, after string) ([]RuntimeAllocation, error) { - id := pgtype.UUID{Valid: true} - if after != "" { - var err error - id, err = parseID(after) - if err != nil { - return nil, err - } - } - rows, err := s.queries.ListRuntimeAllocations(ctx, id) - if err != nil { - return nil, err - } - out := make([]RuntimeAllocation, 0, len(rows)) - for _, r := range rows { - out = append(out, runtimeAllocationFromRow(r.RuntimeAllocation, r.SessionID, r.TenantID, r.DeletedAt, r.Expired)) - } - return out, nil -} diff --git a/services/core/internal/store/sandbox_generations_test.go b/services/core/internal/store/sandbox_generations_test.go index f3a0e8714..75c3d6e2c 100644 --- a/services/core/internal/store/sandbox_generations_test.go +++ b/services/core/internal/store/sandbox_generations_test.go @@ -2,14 +2,12 @@ package store import ( "database/sql" - "errors" "os" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/stdlib" "github.com/pressly/goose/v3" @@ -21,38 +19,41 @@ func TestE2BGenerationsRetainOwnershipAndUseCurrentCredential(t *testing.T) { s, w, view, input := webSpecificationFixture(t, "e2b") tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) owner := archiveAllocation(t, w, tenant, session, view.InstallationID) + changes, deployments := deploymentExecution(t, w), deploymentService(t, s) ctx := SandboxResetTestContext(t.Context()) oldTemplate := input.Configuration.(*e2b.DeploymentConfiguration).Template input.Configuration.(*e2b.DeploymentConfiguration).Template = "next:" + uuid.NewString() input.Configuration.(*e2b.DeploymentConfiguration).APIURL, input.Configuration.(*e2b.DeploymentConfiguration).Domain = "https://sandbox.example.com", "sandbox.example.com" input.Resources.CPUs++ - changed, err := w.UpdateSandboxDeployment(ctx, view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}) + input.ExpectedGeneration = 1 + changed, err := changes.Update(ctx, view.InstallationID, input) if err != nil || changed.Generation != 2 || changed.OwnerEpoch != view.OwnerEpoch || changed.Rollout.PreviousGenerationSandboxes != 1 || changed.Rollout.State != "settled" { t.Fatal(changed, err) } assertSandboxSnapshotEquivalent(t, s.pool) ref := sandbox.Reference{TenantID: tenant, EnvironmentID: owner.EnvironmentID, AllocationID: owner.ID} - retained, err := s.GetSandboxAllocationSetup(t.Context(), ref) + retained, err := deployments.AllocationSetup(t.Context(), ref) if err != nil || retained.Generation != 1 || retained.Configuration.(*e2b.DeploymentConfiguration).Template != oldTemplate || retained.Configuration.(*e2b.DeploymentConfiguration).APIURL != "https://api.e2b.app" || retained.Specification.Resources.CPUs == input.Resources.CPUs { t.Fatal(retained, err) } input.Configuration.(*e2b.DeploymentConfiguration).APIKey = "replacement-secret" input.Configuration.(*e2b.DeploymentConfiguration).CredentialSupplied = true - changed, err = w.UpdateSandboxDeployment(ctx, view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 2}) + input.ExpectedGeneration = 2 + changed, err = changes.Update(ctx, view.InstallationID, input) if err != nil || changed.Generation != 3 || changed.OwnerEpoch != view.OwnerEpoch { t.Fatal(changed, err) } - retained, err = s.GetSandboxAllocationSetup(t.Context(), ref) + retained, err = deployments.AllocationSetup(t.Context(), ref) if err != nil || retained.Generation != 1 || retained.Configuration.(*e2b.DeploymentConfiguration).APIKey != input.Configuration.(*e2b.DeploymentConfiguration).APIKey || retained.Configuration.(*e2b.DeploymentConfiguration).Template != oldTemplate || retained.Configuration.(*e2b.DeploymentConfiguration).APIURL != "https://api.e2b.app" { t.Fatal("old generation did not use committed key", err) } if _, err = s.pool.Exec(t.Context(), `UPDATE runtime_allocations SET deployment_generation=3 WHERE id=$1`, owner.ID); err == nil { t.Fatal("ownership generation was mutable") } - if err = w.CollectSandboxGenerations(t.Context()); err != nil { + if err = changes.CollectGenerations(t.Context()); err != nil { t.Fatal(err) } - generations, err := s.SandboxGenerationPage(t.Context(), -1) + generations, err := deployments.GenerationPage(t.Context(), -1) if err != nil || len(generations) != 1 || generations[0].Generation != 1 || generations[0].Configuration.(*e2b.DeploymentConfiguration).APIURL != "https://api.e2b.app" { t.Fatal(generations, err) } @@ -65,10 +66,10 @@ func TestE2BGenerationsRetainOwnershipAndUseCurrentCredential(t *testing.T) { if _, err = w.ReleaseRuntimeAllocation(t.Context(), owner); err != nil { t.Fatal(err) } - if err = w.CollectSandboxGenerations(t.Context()); err != nil { + if err = changes.CollectGenerations(t.Context()); err != nil { t.Fatal(err) } - generations, err = s.SandboxGenerationPage(t.Context(), -1) + generations, err = deployments.GenerationPage(t.Context(), -1) if err != nil || len(generations) != 0 { t.Fatal(generations, err) } @@ -80,106 +81,33 @@ func TestE2BGenerationsRetainOwnershipAndUseCurrentCredential(t *testing.T) { func TestE2BRetainedCustomEndpointAfterOnlineSwitch(t *testing.T) { s, w, view, input := webSpecificationFixture(t, "e2b") + changes, deployments := deploymentExecution(t, w), deploymentService(t, s) ctx := SandboxResetTestContext(t.Context()) input.Configuration.(*e2b.DeploymentConfiguration).APIURL, input.Configuration.(*e2b.DeploymentConfiguration).Domain = "https://sandbox.example.com", "sandbox.example.com" - custom, err := w.UpdateSandboxDeployment(ctx, view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: view.Generation}) + input.ExpectedGeneration = view.Generation + custom, err := changes.Update(ctx, view.InstallationID, input) if err != nil || custom.Generation != 2 { t.Fatal(custom, err) } tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) owner := archiveAllocation(t, w, tenant, session, view.InstallationID) input.Configuration.(*e2b.DeploymentConfiguration).APIURL, input.Configuration.(*e2b.DeploymentConfiguration).Domain = "", "" - current, err := w.UpdateSandboxDeployment(ctx, view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: custom.Generation}) + input.ExpectedGeneration = custom.Generation + current, err := changes.Update(ctx, view.InstallationID, input) if err != nil || current.Generation != 3 || e2bPublicConfiguration(t, current).APIURL != "https://api.e2b.app" { t.Fatal(current, err) } ref := sandbox.Reference{TenantID: tenant, EnvironmentID: owner.EnvironmentID, AllocationID: owner.ID} - retained, err := s.GetSandboxAllocationSetup(t.Context(), ref) + retained, err := deployments.AllocationSetup(t.Context(), ref) if err != nil || retained.Generation != 2 || retained.Configuration.(*e2b.DeploymentConfiguration).APIURL != "https://sandbox.example.com" || retained.Configuration.(*e2b.DeploymentConfiguration).Domain != "sandbox.example.com" { t.Fatal(retained, err) } - generations, err := s.SandboxGenerationPage(t.Context(), -1) + generations, err := deployments.GenerationPage(t.Context(), -1) if err != nil || len(generations) != 1 || generations[0].Configuration.(*e2b.DeploymentConfiguration).APIURL != "https://sandbox.example.com" { t.Fatal(generations, err) } } -func TestE2BChangeClassifierOmittedKeyAndExplicitSameKey(t *testing.T) { - _, w, view, input := webSpecificationFixture(t, "e2b") - key := input.Configuration.(*e2b.DeploymentConfiguration).APIKey - input.Configuration.(*e2b.DeploymentConfiguration).APIKey = "" - input.Resources = sandbox.Resources{} - resolved, noOp, err := w.ClassifySandboxDeploymentChange(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}) - if err != nil || !noOp || resolved.Configuration.(*e2b.DeploymentConfiguration).APIKey != key || resolved.Resources.CPUs == 0 { - t.Fatal(noOp, err) - } - input.Configuration.(*e2b.DeploymentConfiguration).APIKey = key - input.Configuration.(*e2b.DeploymentConfiguration).CredentialSupplied = true - _, noOp, err = w.ClassifySandboxDeploymentChange(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}) - if err != nil || noOp { - t.Fatal("explicit same key skipped verification", err) - } - invalid := input - invalid.Runtime = &sandbox.RuntimeRelease{} - if _, _, err := w.ClassifySandboxDeploymentChange(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: invalid, ExpectedGeneration: 1}); err == nil { - t.Fatal("omitted resources erased forbidden Runtime input") - } - _, _, err = w.ClassifySandboxDeploymentChange(t.Context(), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 0}) - var stale *SandboxGenerationStaleError - if !errors.As(err, &stale) { - t.Fatal("stale did not precede no-op", err) - } -} - -func TestGenerationPinRetainsOfflineZeroResourceFallback(t *testing.T) { - s, w, view, _ := webSpecificationFixture(t, "docker") - node := specificationNode(t, s, view) - if _, err := s.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, node.NodeID); err != nil { - t.Fatal(err) - } - // PR-N will make this transition through its generation protocol. This fixture - // isolates the persistence invariant without enabling node online PUT in PR-G. - tx, err := s.pool.Begin(t.Context()) - if err != nil { - t.Fatal(err) - } - defer tx.Rollback(t.Context()) - q := s.queries.WithTx(tx) - if _, err = q.LockRuntimeDeployment(t.Context()); err != nil { - t.Fatal(err) - } - if err = q.RetainSandboxGeneration(t.Context()); err != nil { - t.Fatal(err) - } - if _, err = tx.Exec(t.Context(), `UPDATE runtime_deployment SET generation=2`); err != nil { - t.Fatal(err) - } - if err = tx.Commit(t.Context()); err != nil { - t.Fatal(err) - } - if err = w.CollectSandboxGenerations(t.Context()); err != nil { - t.Fatal(err) - } - rows, err := s.SandboxGenerationPage(t.Context(), -1) - if err != nil || len(rows) != 1 { - t.Fatal("offline pin was collected", rows, err) - } - nodes, err := s.ListRuntimeNodes(t.Context()) - if err != nil || len(nodes) != 1 || nodes[0].Rollout.State != "unknown" || nodes[0].Rollout.ReadyGeneration == nil || *nodes[0].Rollout.ReadyGeneration != 1 { - t.Fatal(nodes, err) - } - if _, err = s.pool.Exec(t.Context(), `UPDATE runtime_nodes SET removed_at=clock_timestamp(),ready_generation=NULL WHERE id=$1`, node.NodeID); err != nil { - t.Fatal(err) - } - if err = w.CollectSandboxGenerations(t.Context()); err != nil { - t.Fatal(err) - } - rows, err = s.SandboxGenerationPage(t.Context(), -1) - if err != nil || len(rows) != 0 { - t.Fatal(rows, err) - } -} - func TestPendingPlacementGenerationSurvivesRepeatedUpdates(t *testing.T) { s, w, view, _ := webSpecificationFixture(t, "docker") node := specificationNode(t, s, view) @@ -208,10 +136,10 @@ func TestPendingPlacementGenerationSurvivesRepeatedUpdates(t *testing.T) { if owner.DeploymentGeneration != 1 || owner.NodeID != node.NodeID { t.Fatal("pending allocation rebound to newest generation", owner) } - if err := w.CollectSandboxGenerations(t.Context()); err != nil { + if err := deploymentExecution(t, w).CollectGenerations(t.Context()); err != nil { t.Fatal(err) } - rows, err := s.SandboxGenerationPage(t.Context(), -1) + rows, err := deploymentService(t, s).GenerationPage(t.Context(), -1) if err != nil || len(rows) != 1 || rows[0].Generation != 1 { t.Fatal(rows, err) } @@ -285,7 +213,8 @@ func TestGenerationDowngradeRefusesOldAllocation(t *testing.T) { tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) owner := archiveAllocation(t, w, tenant, session, view.InstallationID) input.Configuration.(*e2b.DeploymentConfiguration).Template = "next:" + uuid.NewString() - if _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: 1}); err != nil { + input.ExpectedGeneration = 1 + if _, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, input); err != nil { t.Fatal(err) } db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) @@ -318,14 +247,16 @@ func TestGenerationDowngradeRefusesOldAllocation(t *testing.T) { func TestGenerationUpdateSerializesWithAllocationAdmission(t *testing.T) { s, w, view, input := webSpecificationFixture(t, "e2b") + changes, deployments := deploymentExecution(t, w), deploymentService(t, s) for generation := uint64(1); generation <= 6; generation++ { tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) input.Configuration.(*e2b.DeploymentConfiguration).Template = "next:" + uuid.NewString() + input.ExpectedGeneration = generation start := make(chan struct{}) changed := make(chan error, 1) go func() { <-start - _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{SandboxDeploymentSetupRequest: input, ExpectedGeneration: generation}) + _, err := changes.Update(SandboxResetTestContext(t.Context()), view.InstallationID, input) changed <- err }() close(start) @@ -336,62 +267,47 @@ func TestGenerationUpdateSerializesWithAllocationAdmission(t *testing.T) { if owner.DeploymentGeneration != generation && owner.DeploymentGeneration != generation+1 { t.Fatal("allocation bound unrelated generation", owner.DeploymentGeneration) } - if err := w.CollectSandboxGenerations(t.Context()); err != nil { + if err := changes.CollectGenerations(t.Context()); err != nil { t.Fatal(err) } - setup, err := s.GetSandboxAllocationSetup(t.Context(), sandbox.Reference{TenantID: tenant, EnvironmentID: owner.EnvironmentID, AllocationID: owner.ID}) + setup, err := deployments.AllocationSetup(t.Context(), sandbox.Reference{TenantID: tenant, EnvironmentID: owner.EnvironmentID, AllocationID: owner.ID}) if err != nil || setup.Generation != owner.DeploymentGeneration { t.Fatal("committed allocation lost immutable routing", err) } } } -func TestNodeRolloutSeparatesOfflinePinAndTargetReadiness(t *testing.T) { - for _, tc := range []struct { - name string - online, ready bool - enrolled, pin, target int64 - targetState, diagnostic, state string - }{ - {"offline pin", false, true, 1, 1, 2, "ready", "", "unknown"}, - {"old serving", true, true, 1, 1, 2, "ready", "", "update_required"}, - {"current serving", true, true, 2, 2, 2, "ready", "", "ready"}, - {"current failed", true, false, 2, 2, 2, "failed", "kvm_unavailable", "failed"}, - {"current unknown", true, false, 2, 2, 2, "", "", "unknown"}, - } { - t.Run(tc.name, func(t *testing.T) { - got := nodeRollout(sqlc.ListRuntimeNodesRow{Online: tc.online, ProviderReady: tc.ready, DeploymentGeneration: tc.enrolled, ReadyGeneration: pgtype.Int8{Int64: tc.pin, Valid: true}, TargetGeneration: tc.target, ProtocolVersion: 1, TargetState: tc.targetState, TargetDiagnostic: tc.diagnostic}) - if got.State != tc.state || got.Diagnostic != tc.diagnostic || got.ReadyGeneration == nil || *got.ReadyGeneration != uint64(tc.pin) { - t.Fatal(got) - } - }) - } -} - func TestSandboxSnapshotRolloutEquivalence(t *testing.T) { s, w, view, input := webSpecificationFixture(t, "docker") node := specificationNode(t, s, view) + deployments := deploymentService(t, s) + heartbeat := func(connection, state string) { + t.Helper() + if err := deployments.HeartbeatGenerations(t.Context(), node.NodeID, connection, view.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: state}}); err != nil { + t.Fatal(err) + } + } for _, state := range []string{"ready", "preparing", "failed", "unconfirmed", "offline", "update_required"} { t.Run(state, func(t *testing.T) { connection := onlineManagerNode(t, s, node.NodeID) - want := SandboxRolloutNodes{} + want := deployment.RolloutNodes{} wantState := "settled" switch state { case "ready": want.Ready = 1 case "preparing": - generationHeartbeat(t, s, node, connection, view, "preparing") + heartbeat(connection, "preparing") want.Preparing = 1 wantState = "preparing" case "failed": - generationHeartbeat(t, s, node, connection, view, "failed") + heartbeat(connection, "failed") want.Failed = 1 case "unconfirmed": connection = uuid.NewString() - if err := s.ConnectRuntimeNode(t.Context(), node.NodeID, connection, view.OwnerEpoch); err != nil { + if err := deployments.ConnectNode(t.Context(), node.NodeID, connection, view.OwnerEpoch); err != nil { t.Fatal(err) } - if err := s.HeartbeatRuntimeNodeGenerations(t.Context(), node.NodeID, connection, view.OwnerEpoch, RuntimeNodeHealth{}, nil); err != nil { + if err := deployments.HeartbeatGenerations(t.Context(), node.NodeID, connection, view.OwnerEpoch, deployment.NodeHealth{}, nil); err != nil { t.Fatal(err) } want.Unknown = 1 @@ -399,11 +315,19 @@ func TestSandboxSnapshotRolloutEquivalence(t *testing.T) { runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET last_seen_at=clock_timestamp()-interval '46 seconds',health='{"diagnostic":"provider_unavailable"}' WHERE id=$1`, node.NodeID) want.Unknown = 1 case "update_required": - changeNodeTarget(t, w, view, input) + // Change the target on the same backend so the node's enrolled + // generation falls behind. + change := input + change.Resources.CPUs++ + change.ExpectedGeneration = view.Generation + next, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, change) + if err != nil || next.Generation != view.Generation+1 || next.OwnerEpoch != view.OwnerEpoch { + t.Fatal("target change replaced execution ownership", next, err) + } want.UpdateRequired = 1 } assertSandboxSnapshotEquivalent(t, s.pool) - got, err := s.GetRuntimeDeployment(t.Context()) + got, err := deployments.View(t.Context()) if err != nil || got.Rollout.Nodes == nil || *got.Rollout.Nodes != want || got.Rollout.State != wantState { t.Fatal("rollout classification changed", got.Rollout, err) } diff --git a/services/core/internal/store/sandbox_node_auth_order_http_test.go b/services/core/internal/store/sandbox_node_auth_order_http_test.go index 8bc3ca6d7..17837417d 100644 --- a/services/core/internal/store/sandbox_node_auth_order_http_test.go +++ b/services/core/internal/store/sandbox_node_auth_order_http_test.go @@ -8,8 +8,8 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" ) @@ -22,7 +22,7 @@ func TestSandboxNodeRoutesAuthenticateBeforeDeploymentState(t *testing.T) { if err != nil { t.Fatal(err) } - handler, err := publicHandler(t, s, db, nil, "codex", storeKeys(s), storeExecution(t, s), managedSandboxes(t, s), withCoreKeys(admin)) + handler, err := publicHandler(t, s, db, nil, "codex", storeKeys(s), storeExecution(t, s), managedSandboxes(t, s, db), withCoreKeys(admin)) if err != nil { t.Fatal(err) } @@ -36,7 +36,7 @@ func TestSandboxNodeRoutesAuthenticateBeforeDeploymentState(t *testing.T) { token, claimedToken, claimed := strings.Repeat("e", 64), strings.Repeat("c", 64), uuid.NewString() enrollment(token, uuid.NewString()) enrollment(claimedToken, claimed) - enroll, _ := json.Marshal(store.RuntimeNodeEnrollment{NodeID: uuid.NewString(), Credential: strings.Repeat("n", 64), Name: "Early node", Provider: "docker", + enroll, _ := json.Marshal(deployment.Enrollment{NodeID: uuid.NewString(), Credential: strings.Repeat("n", 64), Name: "Early node", Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: 1, SpecificationDigest: strings.Repeat("d", 64), CoreURL: "https://core.example"}) nodeID := uuid.NewString() type check struct { diff --git a/services/core/internal/store/sandbox_reset.go b/services/core/internal/store/sandbox_reset.go index 1346d9080..c7bf36874 100644 --- a/services/core/internal/store/sandbox_reset.go +++ b/services/core/internal/store/sandbox_reset.go @@ -7,6 +7,8 @@ import ( "math" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" @@ -15,29 +17,14 @@ import ( ) var ErrSandboxResetAdmission = errors.New("hosted admission is paused for a sandbox reset") -var ErrSandboxResetInProgress = errors.New("a sandbox reset is in progress") -var ErrSandboxNotConfigured = errors.New("the sandbox deployment is not configured") - -type SandboxGenerationStaleError struct{ CurrentGeneration uint64 } - -func (e *SandboxGenerationStaleError) Error() string { - return "the sandbox deployment generation changed" -} -func (e *SandboxGenerationStaleError) Unwrap() error { return ErrSandboxDeploymentConflict } -type SandboxResetRequiredError struct{ CurrentProvider, RequestedProvider string } - -func (e *SandboxResetRequiredError) Error() string { - return "reset the sandbox deployment before changing its backend" -} -func (e *SandboxResetRequiredError) Unwrap() error { return ErrSandboxDeploymentConflict } - -type SandboxInUseError struct{ Resources SandboxDeploymentResources } +// SandboxInUseError rejects reset completion while hosted resources remain. +type SandboxInUseError struct{ Resources deployment.Resources } func (e *SandboxInUseError) Error() string { return "hosted sandbox resources still belong to this deployment" } -func (e *SandboxInUseError) Unwrap() error { return ErrSandboxDeploymentConflict } +func (e *SandboxInUseError) Unwrap() error { return deployment.ErrConflict } type SandboxResetRequest struct { ExpectedGeneration uint64 `json:"expected_generation" binding:"required" minimum:"0"` @@ -47,10 +34,10 @@ type SandboxResetRequest struct { func checkSandboxGeneration(d sqlc.RuntimeDeployment, installation string, generation uint64) error { if uint64(d.Generation) != generation { - return &SandboxGenerationStaleError{uint64(d.Generation)} + return &deployment.GenerationStaleError{CurrentGeneration: uint64(d.Generation)} } if !d.WebManaged || runtimeUUID(d.InstallationID) != installation { - return ErrSandboxDeploymentConflict + return deployment.ErrConflict } return nil } @@ -69,29 +56,30 @@ func (s *Store) resetTransaction(ctx context.Context, apply func(context.Context }) } -func (s *Store) StartSandboxReset(ctx context.Context, installation string, input SandboxResetRequest) (RuntimeDeploymentView, error) { +// StartSandboxReset starts or escalates the reset. It returns no view: the +// caller reads the deployment after commit. +func (s *Store) StartSandboxReset(ctx context.Context, installation string, input SandboxResetRequest) error { if input.Clear != "auto" && input.Clear != "force" { - return RuntimeDeploymentView{}, ErrInvalidInput + return ErrInvalidInput } deadline := int32(3600) if input.DeadlineSeconds != nil { if input.Clear != "auto" || *input.DeadlineSeconds < 300 || *input.DeadlineSeconds > 86400 { - return RuntimeDeploymentView{}, ErrInvalidInput + return ErrInvalidInput } deadline = *input.DeadlineSeconds } - var result RuntimeDeploymentView - err := s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { + return s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { if err := checkSandboxGeneration(d, installation, input.ExpectedGeneration); err != nil { return err } if d.ProviderKind == "" { - return ErrSandboxNotConfigured + return deployment.ErrNotConfigured } if d.ResetClear.Valid { if d.ResetClear.String != input.Clear { if input.Clear != "force" { - return ErrSandboxResetInProgress + return deployment.ErrResetInProgress } if err := q.ForceSandboxReset(ctx); err != nil { return err @@ -114,20 +102,16 @@ func (s *Store) StartSandboxReset(ctx context.Context, installation string, inpu if err := q.StartSandboxReset(ctx, sqlc.StartSandboxResetParams{Clear: pgtype.Text{String: input.Clear, Valid: true}, DeadlineSeconds: deadline, Audit: audit}); err != nil { return err } - if err := auditpg.RecordDeploymentMutation(ctx, q, "reset_start", "sandbox_deployment", installation); err != nil { - return err - } + return auditpg.RecordDeploymentMutation(ctx, q, "reset_start", "sandbox_deployment", installation) } - var err error - result, err = s.deploymentView(ctx, q) - return err + return nil }) - return result, err } -func (s *Store) CancelSandboxReset(ctx context.Context, installation string, generation uint64) (RuntimeDeploymentView, error) { - var result RuntimeDeploymentView - err := s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { +// CancelSandboxReset cancels a running reset. It returns no view: the caller +// reads the deployment after commit. +func (s *Store) CancelSandboxReset(ctx context.Context, installation string, generation uint64) error { + return s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { if err := checkSandboxGeneration(d, installation, generation); err != nil { return err } @@ -135,15 +119,10 @@ func (s *Store) CancelSandboxReset(ctx context.Context, installation string, gen if err := q.CancelSandboxReset(ctx); err != nil { return err } - if err := auditpg.RecordDeploymentMutation(ctx, q, "reset_cancel", "sandbox_deployment", installation); err != nil { - return err - } + return auditpg.RecordDeploymentMutation(ctx, q, "reset_cancel", "sandbox_deployment", installation) } - var err error - result, err = s.deploymentView(ctx, q) - return err + return nil }) - return result, err } // AdvanceSandboxResetDeadline makes force escalation durable before selecting @@ -174,24 +153,25 @@ func sandboxResetAudit(d sqlc.RuntimeDeployment) (adminaudit.Source, error) { // CompleteSandboxReset must run after the manager has drained. It does not take // Session locks: archive and admission always lock Session before deployment. -func (s *Store) CompleteSandboxReset(ctx context.Context, installation string, generation uint64, requestedAt time.Time) (RuntimeDeploymentView, error) { - var result RuntimeDeploymentView +// It returns the committed, unconfigured generation. +func (s *Store) CompleteSandboxReset(ctx context.Context, installation string, generation uint64, requestedAt time.Time) (uint64, error) { + var committed uint64 err := s.resetTransaction(ctx, func(ctx context.Context, q *sqlc.Queries, d sqlc.RuntimeDeployment) error { if err := checkSandboxGeneration(d, installation, generation); err != nil { return err } if !d.ResetClear.Valid || !d.ResetRequestedAt.Time.Equal(requestedAt) { - return ErrSandboxDeploymentConflict + return deployment.ErrConflict } if d.Generation == math.MaxInt64 || d.OwnerEpoch == math.MaxInt64 { - return ErrSandboxDeploymentConflict + return deployment.ErrConflict } resources, err := q.CountRuntimeDeploymentResources(ctx) if err != nil { return err } if resources.Allocations != 0 || resources.Pending != 0 { - return &SandboxInUseError{SandboxDeploymentResources{Allocations: resources.Allocations, Pending: resources.Pending}} + return &SandboxInUseError{deployment.Resources{Allocations: resources.Allocations, Pending: resources.Pending}} } source, err := sandboxResetAudit(d) if err != nil { @@ -200,6 +180,8 @@ func (s *Store) CompleteSandboxReset(ctx context.Context, installation string, g if err := q.CompleteSandboxReset(ctx); err != nil { return err } + // The locked row advances by exactly one generation. + committed = uint64(d.Generation) + 1 if err := q.RetireSandboxNodes(ctx); err != nil { return err } @@ -209,41 +191,12 @@ func (s *Store) CompleteSandboxReset(ctx context.Context, installation string, g if err := q.ClearSandboxGenerations(ctx); err != nil { return err } - if err := auditpg.RecordDeploymentMutation(adminaudit.WithSource(ctx, source), q, "reset_complete", "sandbox_deployment", installation); err != nil { - return err - } - result, err = s.deploymentView(ctx, q) - return err + return auditpg.RecordDeploymentMutation(adminaudit.WithSource(ctx, source), q, "reset_complete", "sandbox_deployment", installation) }) - return result, err -} - -// SandboxResetView contains only durable state and a single-snapshot resource partition. -type SandboxResetView struct { - Clear string `json:"clear"` - RequestedAt time.Time `json:"requested_at"` - DeadlineAt *time.Time `json:"deadline_at" extensions:"x-nullable"` - ForcedAt *time.Time `json:"forced_at" extensions:"x-nullable"` - Remaining SandboxResetRemaining `json:"remaining"` -} -type SandboxResetRemaining struct { - Busy int64 `json:"busy"` - Idle int64 `json:"idle"` - Cleanup int64 `json:"cleanup"` - OnOfflineNodes int64 `json:"on_offline_nodes"` - OfflineNodes []SandboxResetOfflineNode `json:"offline_nodes"` -} -type SandboxResetOfflineNode struct { - NodeID string `json:"node_id"` - Name string `json:"name"` - Resources int64 `json:"resources"` -} - -func resetTimestamp(value pgtype.Timestamptz) *time.Time { - if !value.Valid { - return nil + if err != nil { + return 0, err } - return &value.Time + return committed, nil } type SandboxResetSession struct{ SessionID, TenantID string } diff --git a/services/core/internal/store/sandbox_reset_test.go b/services/core/internal/store/sandbox_reset_test.go index ed2ab71b7..d6f077649 100644 --- a/services/core/internal/store/sandbox_reset_test.go +++ b/services/core/internal/store/sandbox_reset_test.go @@ -10,6 +10,8 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" ) @@ -17,22 +19,25 @@ import ( func SandboxResetTestContext(ctx context.Context) context.Context { return adminaudit.WithSource(ctx, adminaudit.Source{CredentialID: "reset-fixture", ActorLabel: "operator", RequestID: "reset-request", TraceID: "reset-trace"}) } -func resetAndSelect(t *testing.T, w *Store, installation string, generation uint64, input SandboxDeploymentSetupRequest) (RuntimeDeploymentView, error) { +func resetAndSelect(t *testing.T, w *Store, installation string, generation uint64, input sandbox.Selection) (deployment.View, error) { t.Helper() ctx := SandboxResetTestContext(t.Context()) - reset, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{Clear: "auto", ExpectedGeneration: generation}) + if err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{Clear: "auto", ExpectedGeneration: generation}); err != nil { + return deployment.View{}, err + } + reset, err := deploymentService(t, w).View(ctx) if err != nil { - return RuntimeDeploymentView{}, err + return deployment.View{}, err } empty, err := w.CompleteSandboxReset(ctx, installation, generation, reset.Reset.RequestedAt) if err != nil { - return RuntimeDeploymentView{}, err + return deployment.View{}, err } - input.ExpectedGeneration = empty.Generation - return w.InitializeSandboxDeployment(ctx, installation, input) + input.ExpectedGeneration = empty + return deploymentExecution(t, w).Initialize(ctx, installation, input) } -func assertResetPartition(t *testing.T, view RuntimeDeploymentView) { +func assertResetPartition(t *testing.T, view deployment.View) { t.Helper() if view.Reset == nil { t.Fatal("missing reset") @@ -50,6 +55,16 @@ func assertResetPartition(t *testing.T, view RuntimeDeploymentView) { } } +// startReset starts or escalates the reset and returns the deployment view +// read after it commits. +func startReset(t *testing.T, ctx context.Context, w *Store, installation string, input SandboxResetRequest) (deployment.View, error) { + t.Helper() + if err := w.StartSandboxReset(ctx, installation, input); err != nil { + return deployment.View{}, err + } + return deploymentService(t, w).View(ctx) +} + func TestSandboxResetAutoUsesStartedWorkAndLockedRecheck(t *testing.T) { for _, kind := range []string{"idle", "queued", "in_progress", "waiting", "subagent_queued", "subagent_in_progress", "subagent_waiting", "file_write", "suspended", "pending"} { t.Run(kind, func(t *testing.T) { @@ -74,7 +89,7 @@ func TestSandboxResetAutoUsesStartedWorkAndLockedRecheck(t *testing.T) { case "suspended": runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_allocations SET compute_phase='suspended', compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, owner.ID) } - reset, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + reset, err := startReset(t, SandboxResetTestContext(t.Context()), w, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) if err != nil { t.Fatal(err) } @@ -92,7 +107,7 @@ func TestSandboxResetAutoUsesStartedWorkAndLockedRecheck(t *testing.T) { if !errors.Is(err, ErrSandboxResetSessionBusy) { t.Fatal("auto cut active work", err) } - if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { + if err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { t.Fatal(err) } _, err = w.ArchiveSandboxResetSession(t.Context(), tenant, session.ID, 1, reset.Reset.RequestedAt) @@ -116,7 +131,7 @@ func TestSandboxResetCancellationABADeadlineAndGeneration(t *testing.T) { s, w, installation := managedArchiveFixture(t) tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) ctx := SandboxResetTestContext(t.Context()) - first, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + first, err := startReset(t, ctx, w, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) if err != nil { t.Fatal(err) } @@ -124,48 +139,52 @@ func TestSandboxResetCancellationABADeadlineAndGeneration(t *testing.T) { t.Fatal("default deadline", first) } shorter := int32(300) - replay, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto", DeadlineSeconds: &shorter}) + replay, err := startReset(t, ctx, w, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto", DeadlineSeconds: &shorter}) if err != nil || !replay.Reset.RequestedAt.Equal(first.Reset.RequestedAt) || !replay.Reset.DeadlineAt.Equal(*first.Reset.DeadlineAt) { t.Fatal("retry moved durable deadline", replay, err) } - if _, err = w.CancelSandboxReset(ctx, installation, 1); err != nil { + if err = w.CancelSandboxReset(ctx, installation, 1); err != nil { t.Fatal(err) } - second, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + second, err := startReset(t, ctx, w, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) if err != nil { t.Fatal(err) } - if _, err = w.ArchiveSandboxResetSession(t.Context(), tenant, session.ID, 1, first.Reset.RequestedAt); !errors.Is(err, ErrSandboxDeploymentConflict) { + if _, err = w.ArchiveSandboxResetSession(t.Context(), tenant, session.ID, 1, first.Reset.RequestedAt); !errors.Is(err, deployment.ErrConflict) { t.Fatal("cancelled reset archived successor work", err) } runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_deployment SET reset_deadline_at=clock_timestamp()-interval '1 second'`) if err := w.AdvanceSandboxResetDeadline(t.Context()); err != nil { t.Fatal(err) } - forced, err := s.GetRuntimeDeployment(t.Context()) + forced, err := deploymentService(t, s).View(t.Context()) if err != nil || forced.Reset.Clear != "force" || forced.Reset.ForcedAt == nil || !forced.Reset.RequestedAt.Equal(second.Reset.RequestedAt) { t.Fatal("deadline not durable", forced, err) } - if _, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}); !errors.Is(err, ErrSandboxResetInProgress) { + if err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}); !errors.Is(err, deployment.ErrResetInProgress) { t.Fatal("force downgraded", err) } - if _, err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: 0, Clear: "force"}); !errors.Is(err, ErrSandboxDeploymentConflict) { + if err := w.StartSandboxReset(ctx, installation, SandboxResetRequest{ExpectedGeneration: 0, Clear: "force"}); !errors.Is(err, deployment.ErrConflict) { t.Fatal("stale reset precedence", err) } if _, err := w.ArchiveSandboxResetSession(t.Context(), tenant, session.ID, 1, second.Reset.RequestedAt); err != nil { t.Fatal(err) } - if _, err := w.CompleteSandboxReset(ctx, installation, 1, first.Reset.RequestedAt); !errors.Is(err, ErrSandboxDeploymentConflict) { + if _, err := w.CompleteSandboxReset(ctx, installation, 1, first.Reset.RequestedAt); !errors.Is(err, deployment.ErrConflict) { t.Fatal("cancelled reset finalized successor", err) } - empty, err := w.CompleteSandboxReset(ctx, installation, 1, second.Reset.RequestedAt) + committed, err := w.CompleteSandboxReset(ctx, installation, 1, second.Reset.RequestedAt) + if err != nil || committed != 2 { + t.Fatal("reset commit", committed, err) + } + empty, err := deploymentService(t, s).View(t.Context()) if err != nil || empty.Provider != "" || empty.Generation != 2 || empty.Reset != nil || empty.InstallationID != installation || empty.Configuration != nil || empty.Specification != nil { t.Fatal("reset commit", empty, err) } - if _, err := w.CancelSandboxReset(ctx, installation, 1); !errors.Is(err, ErrSandboxDeploymentConflict) { + if err := w.CancelSandboxReset(ctx, installation, 1); !errors.Is(err, deployment.ErrConflict) { t.Fatal("stale cancel", err) } - if _, err := w.CancelSandboxReset(ctx, installation, 2); err != nil { + if err := w.CancelSandboxReset(ctx, installation, 2); err != nil { t.Fatal("cancel without reset", err) } } @@ -174,7 +193,7 @@ func TestSandboxResetAutoRechecksTurnStartedAfterListing(t *testing.T) { s, w, installation := managedArchiveFixture(t) tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) archiveAllocation(t, w, tenant, session, installation) - reset, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + reset, err := startReset(t, SandboxResetTestContext(t.Context()), w, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) if err != nil { t.Fatal(err) } @@ -197,7 +216,7 @@ func TestSandboxResetAutoRechecksTurnStartedAfterListing(t *testing.T) { if !reflect.DeepEqual(before, after) { t.Fatal("rejected admission left provisional rows") } - if _, err := w.CancelSandboxReset(SandboxResetTestContext(t.Context()), installation, 1); err != nil { + if err := w.CancelSandboxReset(SandboxResetTestContext(t.Context()), installation, 1); err != nil { t.Fatal(err) } if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())); err != nil { @@ -209,14 +228,14 @@ func TestSandboxResetAuditFailureRollsBackPauseAndCompletion(t *testing.T) { s, w, installation := managedArchiveFixture(t) rejectAdminAuditInsert(t, s) rejected := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "reset-fixture", ActorLabel: "operator", RequestID: rejectedAdminRequest, TraceID: "reset-trace"}) - if _, err := w.StartSandboxReset(rejected, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err == nil { + if err := w.StartSandboxReset(rejected, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err == nil { t.Fatal("reset committed without audit") } - view, err := s.GetRuntimeDeployment(t.Context()) + view, err := deploymentService(t, s).View(t.Context()) if err != nil || view.Reset != nil || view.Generation != 1 { t.Fatal("failed audit left reset state", view, err) } - reset, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}) + reset, err := startReset(t, SandboxResetTestContext(t.Context()), w, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}) if err != nil { t.Fatal(err) } @@ -224,37 +243,37 @@ func TestSandboxResetAuditFailureRollsBackPauseAndCompletion(t *testing.T) { if _, err := w.CompleteSandboxReset(t.Context(), installation, 1, reset.Reset.RequestedAt); err == nil { t.Fatal("completion committed without audit") } - view, err = s.GetRuntimeDeployment(t.Context()) + view, err = deploymentService(t, s).View(t.Context()) if err != nil || view.Reset == nil || view.Provider != "e2b" || view.Generation != 1 || view.OwnerEpoch != reset.OwnerEpoch { t.Fatal("failed completion destroyed committed provider", view, err) } } func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { - s, w, deployment := managerFixture(t, 10, 10) + s, w, process := managerFixture(t, 10, 10) // Reuse the real placement fixture, then adopt its selection as Web-managed. runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_deployment SET web_managed=true,local_node_id=NULL`) runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET deployment_generation=1,specification_digest=$1`, SandboxDeploymentTestSpec("docker").Digest("docker")) _, pending := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) tenant, suspended := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) - allocation := archiveAllocation(t, w, tenant, suspended, deployment.InstallationID) + allocation := archiveAllocation(t, w, tenant, suspended, process.InstallationID) runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_allocations SET compute_phase='suspended',compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, allocation.ID) tenant, deleted := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) - archiveAllocation(t, w, tenant, deleted, deployment.InstallationID) + archiveAllocation(t, w, tenant, deleted, process.InstallationID) if err := s.DeleteSession(t.Context(), tenant, deleted.ID); err != nil { t.Fatal(err) } - reset, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), deployment.InstallationID, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + reset, err := startReset(t, SandboxResetTestContext(t.Context()), w, process.InstallationID, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) if err != nil { t.Fatal(err) } assertResetPartition(t, reset) - if reset.Resources != (SandboxDeploymentResources{Allocations: 2, Pending: 1}) || reset.Reset.Remaining.Idle != 2 || reset.Reset.Remaining.Cleanup != 1 { + if reset.Resources != (deployment.Resources{Allocations: 2, Pending: 1}) || reset.Reset.Remaining.Idle != 2 || reset.Reset.Remaining.Cleanup != 1 { t.Fatal("duplicated placement or missing deleted receipt", reset) } for _, state := range []string{"preparing", "stale", "epoch", "disconnected"} { runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET connected_epoch=(SELECT owner_epoch FROM runtime_deployment)`) - onlineManagerNode(t, s, deployment.LocalNodeID) + onlineManagerNode(t, s, process.LocalNodeID) switch state { case "preparing": runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=false`) @@ -265,7 +284,7 @@ func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { case "disconnected": runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET connection_id=NULL`) } - view, err := s.GetRuntimeDeployment(t.Context()) + view, err := deploymentService(t, s).View(t.Context()) if err != nil { t.Fatal(err) } @@ -278,14 +297,14 @@ func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { if view.Reset.Remaining.OnOfflineNodes != want { t.Fatalf("%s presence: %+v", state, view.Reset.Remaining) } - if want > 0 && (len(view.Reset.Remaining.OfflineNodes) != 1 || view.Reset.Remaining.OfflineNodes[0].NodeID != deployment.LocalNodeID || view.Reset.Remaining.OfflineNodes[0].Resources != 3) { + if want > 0 && (len(view.Reset.Remaining.OfflineNodes) != 1 || view.Reset.Remaining.OfflineNodes[0].NodeID != process.LocalNodeID || view.Reset.Remaining.OfflineNodes[0].Resources != 3) { t.Fatal("offline ownership projection", view.Reset.Remaining) } } - if _, err := w.CompleteSandboxReset(t.Context(), deployment.InstallationID, 1, reset.Reset.RequestedAt); err == nil { + if _, err := w.CompleteSandboxReset(t.Context(), process.InstallationID, 1, reset.Reset.RequestedAt); err == nil { t.Fatal("offline resources were treated as cleaned") } - if err := s.RemoveRuntimeNode(t.Context(), deployment.LocalNodeID); !errors.Is(err, ErrRuntimeNodeInUse) { + if err := deploymentService(t, s).RemoveNode(t.Context(), process.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("removed node with reset resources", err) } if row, err := s.GetEnvironment(t.Context(), pending.TenantID, pending.Environment.ID); err == nil && row.Status == "expired" { @@ -312,7 +331,7 @@ func TestSandboxResetPaginationSkipsBusyPrefixAndPreservesSelfHosted(t *testing. } } selfTenant, self := managedArchiveSession(t, s, environmentInput(uuid.NewString(), "self_hosted", "/workspace")) - reset, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + reset, err := startReset(t, SandboxResetTestContext(t.Context()), w, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) if err != nil { t.Fatal(err) } @@ -323,7 +342,7 @@ func TestSandboxResetPaginationSkipsBusyPrefixAndPreservesSelfHosted(t *testing. if _, err := w.ArchiveSandboxResetSession(t.Context(), page[0].TenantID, page[0].SessionID, 1, reset.Reset.RequestedAt); err != nil { t.Fatal(err) } - if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { + if err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "force"}); err != nil { t.Fatal(err) } first, err := w.ListSandboxResetSessions(t.Context(), "", true) @@ -346,7 +365,7 @@ func TestSandboxResetPaginationSkipsBusyPrefixAndPreservesSelfHosted(t *testing. func TestSandboxResetOwnerRestartRetainsDeadlineAndProvenance(t *testing.T) { s, w, installation := managedArchiveFixture(t) tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) - reset, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + reset, err := startReset(t, SandboxResetTestContext(t.Context()), w, installation, SandboxResetRequest{ExpectedGeneration: 1, Clear: "auto"}) if err != nil { t.Fatal(err) } @@ -357,11 +376,11 @@ func TestSandboxResetOwnerRestartRetainsDeadlineAndProvenance(t *testing.T) { t.Fatal(stopped, err) } successor := executionWriter(t, s) - current, err := s.GetRuntimeDeployment(t.Context()) + current, err := deploymentService(t, s).View(t.Context()) if err != nil || !current.Reset.RequestedAt.Equal(reset.Reset.RequestedAt) || !current.Reset.DeadlineAt.Equal(*reset.Reset.DeadlineAt) { t.Fatal("restart moved reset deadline", current, err) } - if _, err := w.CancelSandboxReset(SandboxResetTestContext(t.Context()), installation, 1); err == nil { + if err := w.CancelSandboxReset(SandboxResetTestContext(t.Context()), installation, 1); err == nil { t.Fatal("detached writer cancelled successor reset") } runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_deployment SET reset_deadline_at=clock_timestamp()-interval '1 second'`) @@ -371,7 +390,11 @@ func TestSandboxResetOwnerRestartRetainsDeadlineAndProvenance(t *testing.T) { if _, err := successor.ArchiveSandboxResetSession(t.Context(), tenant, session.ID, 1, reset.Reset.RequestedAt); err != nil { t.Fatal(err) } - empty, err := successor.CompleteSandboxReset(t.Context(), installation, 1, reset.Reset.RequestedAt) + committed, err := successor.CompleteSandboxReset(t.Context(), installation, 1, reset.Reset.RequestedAt) + if err != nil || committed != 2 { + t.Fatal(committed, err) + } + empty, err := deploymentService(t, s).View(t.Context()) if err != nil || empty.Generation != 2 || empty.Provider != "" { t.Fatal(empty, err) } diff --git a/services/core/internal/store/sandbox_specification.go b/services/core/internal/store/sandbox_specification.go deleted file mode 100644 index ba8f8a38e..000000000 --- a/services/core/internal/store/sandbox_specification.go +++ /dev/null @@ -1,55 +0,0 @@ -package store - -import ( - "encoding/json" - "errors" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" -) - -// SandboxConfigurationError contains only validated, non-secret configuration diagnostics. -type SandboxConfigurationError struct { - Message string - Validation *sandbox.ValidationError -} - -func sandboxConfigurationError(err error) *SandboxConfigurationError { - var validation *sandbox.ValidationError - errors.As(err, &validation) - return &SandboxConfigurationError{Message: err.Error(), Validation: validation} -} - -func (e *SandboxConfigurationError) Error() string { return e.Message } -func (e *SandboxConfigurationError) Unwrap() error { return ErrInvalidInput } - -var ErrRuntimeSpecificationMismatch = errors.New("node does not match the deployment specification") - -// SandboxSetupForSelection prepares metadata without writing or allocating resources. -func SandboxSetupForSelection(installationID string, input SandboxDeploymentSetupRequest) (SandboxSetup, error) { - if _, err := parseConnectionGeneration(installationID); err != nil { - return SandboxSetup{}, err - } - if err := validateSandboxSelection(input); err != nil { - return SandboxSetup{}, err - } - normalized, err := providers.Normalize(input) - if err != nil { - return SandboxSetup{}, sandboxConfigurationError(err) - } - description, err := providers.Describe(input.Provider, installationID) - if err != nil { - return SandboxSetup{}, sandboxConfigurationError(err) - } - result := SandboxSetup{InstallationID: installationID, Provider: input.Provider, Mode: description.Mode, Specification: normalized.DeploymentSpec, Configuration: normalized.Configuration, BackendFingerprint: description.BackendFingerprint, IdleSeconds: description.IdleSeconds, RetentionSeconds: description.RetentionSeconds} - return result, nil -} - -func deploymentSpecification(d sqlc.RuntimeDeployment) (sandbox.DeploymentSpec, error) { - var spec sandbox.DeploymentSpec - if json.Unmarshal(d.Specification, &spec) != nil || providers.ValidateSpecification(d.ProviderKind, spec) != nil { - return spec, ErrRuntimeSpecificationMismatch - } - return spec, nil -} diff --git a/services/core/internal/store/sandbox_specification_store_test.go b/services/core/internal/store/sandbox_specification_store_test.go index 95e9a6f3c..2e5f3e047 100644 --- a/services/core/internal/store/sandbox_specification_store_test.go +++ b/services/core/internal/store/sandbox_specification_store_test.go @@ -2,20 +2,20 @@ package store import ( "bytes" - "encoding/json" "errors" "reflect" "strings" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) -func webSpecificationFixture(t *testing.T, provider string) (*Store, *Store, RuntimeDeploymentView, SandboxDeploymentSetupRequest) { +func webSpecificationFixture(t *testing.T, provider string) (*Store, *Store, deployment.View, sandbox.Selection) { t.Helper() _, pool := newManagedTestStore(t) cipher, err := credentialcrypto.New(bytes.Repeat([]byte{13}, 32)) @@ -25,111 +25,78 @@ func webSpecificationFixture(t *testing.T, provider string) (*Store, *Store, Run s := NewWithCredentialCipher(pool, cipher) w := executionWriter(t, s) id := uuid.NewString() - if err := w.ClaimWebSandboxDeployment(t.Context(), id); err != nil { + changes := deploymentExecution(t, w) + if err := changes.Claim(t.Context(), id); err != nil { t.Fatal(err) } - input := SandboxDeploymentSetupRequest{Provider: provider, DeploymentSpec: SandboxDeploymentTestSpec(provider)} + input := sandbox.Selection{Provider: provider, DeploymentSpec: SandboxDeploymentTestSpec(provider)} if provider == "e2b" { input = e2bSelection() } - view, err := w.InitializeSandboxDeployment(t.Context(), id, input) + view, err := changes.Initialize(t.Context(), id, input) if err != nil { t.Fatal(err) } return s, w, view, input } -func specificationNode(t *testing.T, s *Store, view RuntimeDeploymentView) RuntimeNodeEnrollment { +func specificationNode(t *testing.T, s *Store, view deployment.View) deployment.Enrollment { t.Helper() - token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 4, MaxRetained: 16})) + nodes := deploymentService(t, s) + token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 4, MaxRetained: 16})) if err != nil { t.Fatal(err) } - input := RuntimeNodeEnrollment{NodeID: uuid.NewString(), Name: "specification fixture", Credential: strings.Repeat("n", 64), Provider: view.Provider, + input := deployment.Enrollment{NodeID: uuid.NewString(), Name: "specification fixture", Credential: strings.Repeat("n", 64), Provider: view.Provider, BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest, CoreURL: s.publicURL} - if _, err := s.EnrollRuntimeNode(t.Context(), token, input); err != nil { + if _, err := nodes.Enroll(t.Context(), token, input); err != nil { t.Fatal(err) } onlineManagerNode(t, s, input.NodeID) return input } -func TestSandboxSpecificationRoundTripAndFileConfigurationCannotOverride(t *testing.T) { - for _, provider := range []string{"docker", "microsandbox", "e2b"} { - t.Run(provider, func(t *testing.T) { - s, w, view, input := webSpecificationFixture(t, provider) - setup, err := s.GetSandboxSetup(t.Context()) - if err != nil || !reflect.DeepEqual(setup.Specification, input.DeploymentSpec) || view.Specification == nil || !reflect.DeepEqual(*view.Specification, input.DeploymentSpec) || view.SpecificationDigest != input.DeploymentSpec.Digest(provider) { - t.Fatal("saved deployment lost its resources or Runtime provenance", err) - } - preview, err := SandboxSetupForSelection(view.InstallationID, input) - if err != nil || preview.Mode != setup.Mode || preview.BackendFingerprint != setup.BackendFingerprint || preview.IdleSeconds != setup.IdleSeconds || preview.RetentionSeconds != setup.RetentionSeconds || !reflect.DeepEqual(preview.Configuration, setup.Configuration) { - t.Fatal("preview and persisted normalized deployment disagree", err) - } - input.ExpectedGeneration = view.Generation - retry, err := w.InitializeSandboxDeployment(t.Context(), view.InstallationID, input) - if err != nil || !reflect.DeepEqual(retry, view) { - t.Fatal("identical specification changed the generation", err) - } - changed := input - changed.Resources.CPUs++ - if _, err := w.InitializeSandboxDeployment(t.Context(), view.InstallationID, changed); !errors.Is(err, ErrSandboxDeploymentConflict) { - t.Fatal("initial setup silently resized a configured deployment", err) - } - file := RuntimeDeployment{InstallationID: view.InstallationID, BackendFingerprint: setup.BackendFingerprint, ProviderKind: provider, AdmissionPaused: true} - for _, candidate := range []*RuntimeDeployment{nil, &file} { - if err := w.ConfigureRuntimeDeployment(t.Context(), candidate); !errors.Is(err, ErrSandboxDeploymentConflict) { - t.Fatal("file configuration replaced database ownership", err) - } - } - after, err := s.GetRuntimeDeployment(t.Context()) - if err != nil || !reflect.DeepEqual(after, view) { - t.Fatal("rejected writes changed the committed specification", err) - } - }) - } -} - func TestSandboxSpecificationBootstrapReadDoesNotConsumeEnrollment(t *testing.T) { s, w, view, input := webSpecificationFixture(t, "docker") - token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4})) + nodes := deploymentService(t, s) + token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 2, MaxRetained: 4})) if err != nil { t.Fatal(err) } for range 2 { - config, err := s.RuntimeNodeConfiguration(t.Context(), "", token) + config, err := nodes.NodeConfiguration(t.Context(), "", token, 0) if err != nil || config.Generation != view.Generation || config.InstallationID != view.InstallationID || !reflect.DeepEqual(config.Specification, input.DeploymentSpec) || config.SpecificationDigest != view.SpecificationDigest { t.Fatal("bootstrap did not return the saved configuration", err) } } - if _, err := s.RuntimeNodeConfiguration(t.Context(), "", "invalid-token"); !errors.Is(err, ErrRuntimeNodeCredential) { + if _, err := nodes.NodeConfiguration(t.Context(), "", "invalid-token", 0); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("unauthenticated configuration read", err) } - node := RuntimeNodeEnrollment{NodeID: uuid.NewString(), Name: "bootstrap", Credential: strings.Repeat("n", 64), Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest} - for _, change := range []func(*RuntimeNodeEnrollment){ - func(n *RuntimeNodeEnrollment) { n.DeploymentGeneration++ }, - func(n *RuntimeNodeEnrollment) { n.SpecificationDigest = strings.Repeat("c", 64) }, + node := deployment.Enrollment{NodeID: uuid.NewString(), Name: "bootstrap", Credential: strings.Repeat("n", 64), Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest} + for _, change := range []func(*deployment.Enrollment){ + func(n *deployment.Enrollment) { n.DeploymentGeneration++ }, + func(n *deployment.Enrollment) { n.SpecificationDigest = strings.Repeat("c", 64) }, } { wrong := node change(&wrong) - if _, err := s.EnrollRuntimeNode(t.Context(), token, wrong); !errors.Is(err, ErrRuntimeSpecificationMismatch) { + if _, err := nodes.Enroll(t.Context(), token, wrong); !errors.Is(err, deployment.ErrSpecificationMismatch) { t.Fatal("mismatched node configuration enrolled", err) } } - if _, err := s.EnrollRuntimeNode(t.Context(), token, node); err != nil { + if _, err := nodes.Enroll(t.Context(), token, node); err != nil { t.Fatal("read or mismatch consumed the enrollment", err) } - if _, err := s.RuntimeNodeConfiguration(t.Context(), "", token); !errors.Is(err, ErrRuntimeNodeCredential) { + if _, err := nodes.NodeConfiguration(t.Context(), "", token, 0); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("consumed enrollment still authorized bootstrap", err) } - if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxResetRequest{Clear: "auto", ExpectedGeneration: view.Generation}); err != nil { + if err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxResetRequest{Clear: "auto", ExpectedGeneration: view.Generation}); err != nil { t.Fatal(err) } - config, err := s.RuntimeNodeConfiguration(t.Context(), node.NodeID, node.Credential) + config, err := nodes.NodeConfiguration(t.Context(), node.NodeID, node.Credential, 0) if err != nil || config.SpecificationDigest != view.SpecificationDigest { t.Fatal("retained node identity could not recover configuration in maintenance", err) } - if _, err := s.RuntimeNodeConfiguration(t.Context(), node.NodeID, "invalid-credential"); !errors.Is(err, ErrRuntimeNodeCredential) { + if _, err := nodes.NodeConfiguration(t.Context(), node.NodeID, "invalid-credential", 0); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("retained identity bypassed credential validation", err) } for _, field := range []string{"deployment_generation", "specification_digest"} { @@ -144,10 +111,10 @@ func TestSandboxSpecificationBootstrapReadDoesNotConsumeEnrollment(t *testing.T) if _, err := s.pool.Exec(t.Context(), query, args...); err != nil { t.Fatal(err) } - if _, err := s.AuthenticateRuntimeNode(t.Context(), node.NodeID, node.Credential); !errors.Is(err, ErrRuntimeSpecificationMismatch) { + if _, err := nodes.AuthenticateNode(t.Context(), node.NodeID, node.Credential); !errors.Is(err, deployment.ErrSpecificationMismatch) { t.Fatal("stale persisted node authenticated", field, err) } - if _, err := s.RuntimeNodeConfiguration(t.Context(), node.NodeID, node.Credential); !errors.Is(err, ErrRuntimeSpecificationMismatch) { + if _, err := nodes.NodeConfiguration(t.Context(), node.NodeID, node.Credential, 0); !errors.Is(err, deployment.ErrSpecificationMismatch) { t.Fatal("stale persisted node received configuration", field, err) } if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET deployment_generation=$2,specification_digest=$3 WHERE id=$1", node.NodeID, view.Generation, view.SpecificationDigest); err != nil { @@ -160,6 +127,7 @@ func TestSandboxSpecificationChangesPreserveEveryRetainedResource(t *testing.T) for _, state := range []string{"pending", "creating", "running", "stopped", "snapshot", "cleanup_pending"} { t.Run(state, func(t *testing.T) { s, w, view, input := webSpecificationFixture(t, "microsandbox") + changes, deployments := deploymentExecution(t, w), deploymentService(t, s) node := specificationNode(t, s, view) tenant := uuid.NewString() session, err := createSessionOnNode(t, s, tenant, managerSessionInput(uuid.NewString()), node.NodeID) @@ -198,7 +166,7 @@ func TestSandboxSpecificationChangesPreserveEveryRetainedResource(t *testing.T) } } } - before, err := s.GetRuntimeDeployment(t.Context()) + before, err := deployments.View(t.Context()) if err != nil { t.Fatal(err) } @@ -220,17 +188,17 @@ func TestSandboxSpecificationChangesPreserveEveryRetainedResource(t *testing.T) runtime.SourceCommit = strings.Repeat("1", 40) changed.Runtime = &runtime } - update := SandboxDeploymentUpdateRequest{ExpectedGeneration: view.Generation, SandboxDeploymentSetupRequest: changed} - if err := w.CheckSandboxDeploymentSwitch(t.Context(), view.InstallationID, update); err != nil { + changed.ExpectedGeneration = view.Generation + if _, _, err := changes.ClassifyChange(t.Context(), view.InstallationID, changed); err != nil { t.Fatal(field, err) } - next, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, update) + next, err := changes.Update(SandboxResetTestContext(t.Context()), view.InstallationID, changed) if err != nil || next.Generation != view.Generation+1 || next.OwnerEpoch != view.OwnerEpoch { t.Fatal(field, next, err) } view = next } - after, err := s.GetRuntimeDeployment(t.Context()) + after, err := deployments.View(t.Context()) if err != nil || before.Resources != after.Resources { t.Fatal("online specification change altered ownership", err) } @@ -261,33 +229,18 @@ func TestSandboxSpecificationChangesPreserveEveryRetainedResource(t *testing.T) runtime := *input.Runtime runtime.SourceCommit = strings.Repeat("2", 40) changed.Runtime = &runtime - committed, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{ExpectedGeneration: view.Generation, SandboxDeploymentSetupRequest: changed}) - if err != nil || committed.Generation != view.Generation+1 || committed.Reset != nil || committed.Resources != (SandboxDeploymentResources{}) || committed.Specification == nil || !reflect.DeepEqual(*committed.Specification, changed.DeploymentSpec) { + changed.ExpectedGeneration = view.Generation + committed, err := changes.Update(SandboxResetTestContext(t.Context()), view.InstallationID, changed) + if err != nil || committed.Generation != view.Generation+1 || committed.Reset != nil || committed.Resources != (deployment.Resources{}) || committed.Specification == nil || !reflect.DeepEqual(*committed.Specification, changed.DeploymentSpec) { t.Fatal("completed cleanup did not permit the replacement", err) } - if _, err := s.AuthenticateRuntimeNode(t.Context(), node.NodeID, node.Credential); err != nil { + if _, err := deployments.AuthenticateNode(t.Context(), node.NodeID, node.Credential); err != nil { t.Fatal("online change retired serving identity", err) } }) } } -func TestSandboxSpecificationInitialCredentialRemainsPrivate(t *testing.T) { - s, _, view, input := webSpecificationFixture(t, "e2b") - raw, err := json.Marshal(view) - if err != nil || bytes.Contains(raw, []byte(input.Configuration.(*e2b.DeploymentConfiguration).APIKey)) || bytes.Contains(raw, []byte("api_key")) { - t.Fatal("public deployment serialized a private credential", err) - } - var stored []byte - if err := s.pool.QueryRow(t.Context(), "SELECT provider_credential FROM runtime_deployment").Scan(&stored); err != nil || len(stored) == 0 || bytes.Contains(stored, []byte(input.Configuration.(*e2b.DeploymentConfiguration).APIKey)) { - t.Fatal("private credential was not encrypted", err) - } - // The credential is rejected before the cloud deployment mode is reported. - if _, err := s.RuntimeNodeConfiguration(t.Context(), "", input.Configuration.(*e2b.DeploymentConfiguration).APIKey); !errors.Is(err, ErrRuntimeNodeCredential) { - t.Fatal("cloud key authorized node bootstrap", err) - } -} - func TestSandboxSpecificationAllocationRaceWithMaintenance(t *testing.T) { s, w, view, input := webSpecificationFixture(t, "e2b") tenant := uuid.NewString() @@ -316,8 +269,7 @@ func TestSandboxSpecificationAllocationRaceWithMaintenance(t *testing.T) { } go func() { <-start - _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxResetRequest{Clear: "auto", ExpectedGeneration: view.Generation}) - maintenance <- err + maintenance <- w.StartSandboxReset(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxResetRequest{Clear: "auto", ExpectedGeneration: view.Generation}) }() close(start) if err := <-maintenance; err != nil { @@ -341,12 +293,13 @@ func TestSandboxSpecificationAllocationRaceWithMaintenance(t *testing.T) { } } } - after, err := s.GetRuntimeDeployment(t.Context()) + after, err := deploymentService(t, s).View(t.Context()) if err != nil || after.Reset == nil || after.Generation != view.Generation || after.Resources.Allocations != allocated || after.Resources.Pending != int64(len(sessions))-allocated { t.Fatal("concurrent maintenance lost resource accounting", after.Resources, err) } input.Resources.CPUs++ - if _, err := w.UpdateSandboxDeployment(SandboxResetTestContext(t.Context()), view.InstallationID, SandboxDeploymentUpdateRequest{ExpectedGeneration: view.Generation, SandboxDeploymentSetupRequest: input}); !errors.Is(err, ErrSandboxResetInProgress) { + input.ExpectedGeneration = view.Generation + if _, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, input); !errors.Is(err, deployment.ErrResetInProgress) { t.Fatal("allocation race bypassed replacement guard", err) } } @@ -357,12 +310,12 @@ func TestNodeBoundToAnotherPublicURLGetsNoNewSandboxes(t *testing.T) { s, _, view, _ := webSpecificationFixture(t, "docker") s.SetPublicURL("https://old.example") node := specificationNode(t, s, view) - nodes, err := s.ListRuntimeNodes(t.Context()) + nodes, err := deploymentService(t, s).ListNodes(t.Context()) if err != nil || len(nodes) != 1 || nodes[0].ID != node.NodeID || nodes[0].CoreURL != "https://old.example" { t.Fatal("enrollment did not record the node's address", nodes, err) } s.SetPublicURL("https://new.example") - if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, ErrRuntimeNodeUnavailable) { + if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, deployment.ErrNodeUnavailable) { t.Fatal("placed a new sandbox on a node bound to the old address", err) } bindings, err := s.AddressBindings(t.Context()) @@ -375,62 +328,15 @@ func TestNodeBoundToAnotherPublicURLGetsNoNewSandboxes(t *testing.T) { } } -// Enrollment records the command's public ID and the node's Core address. A node -// using another address is refused without consuming the token, and a node -// enrolled with a token issued before Core recorded IDs reports none. -func TestEnrollmentRecordsItsIDAndRefusesAnotherAddress(t *testing.T) { - s, _, view, _ := webSpecificationFixture(t, "docker") - s.SetPublicURL("https://core.example") - issued, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 2}) - if err != nil || uuid.Validate(issued.ID) != nil || issued.Token == "" { - t.Fatal(issued.ID, err) - } - input := RuntimeNodeEnrollment{NodeID: uuid.NewString(), Name: "addressed", Credential: strings.Repeat("a", 64), Provider: "docker", - BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest, CoreURL: "https://other.example"} - if _, err := s.EnrollRuntimeNode(t.Context(), issued.Token, input); !errors.Is(err, ErrRuntimeNodeAddressMismatch) { - t.Fatal("enrolled a node that uses another Core address", err) - } - input.CoreURL = "https://core.example" - if _, err := s.EnrollRuntimeNode(t.Context(), issued.Token, input); err != nil { - t.Fatal("the refused enrollment consumed its token", err) - } - earlier := strings.Repeat("e", 64) - if _, err := s.pool.Exec(t.Context(), "INSERT INTO runtime_node_enrollments(token_sha256,installation_id,expires_at) VALUES($1,$2,clock_timestamp()+interval '10 minutes')", - runtimeTokenDigest(earlier), view.InstallationID); err != nil { - t.Fatal(err) - } - older := input - older.NodeID, older.Credential = uuid.NewString(), strings.Repeat("o", 64) - if _, err := s.EnrollRuntimeNode(t.Context(), earlier, older); err != nil { - t.Fatal(err) - } - nodes, err := s.ListRuntimeNodes(t.Context()) - if err != nil || len(nodes) != 2 { - t.Fatal(nodes, err) - } - for _, node := range nodes { - switch node.ID { - case input.NodeID: - if node.EnrollmentID == nil || *node.EnrollmentID != issued.ID || node.CoreURL != "https://core.example" { - t.Fatal("the node did not record its enrollment", node.EnrollmentID, node.CoreURL) - } - case older.NodeID: - if node.EnrollmentID != nil { - t.Fatal("a token without an ID reported one", *node.EnrollmentID) - } - } - } -} - // An E2B selection saved before the public URL became loopback admits no new // Session, and its configuration stays readable for cleanup. func TestE2BAdmitsNothingWhileThePublicURLIsLoopback(t *testing.T) { s, _, _, _ := webSpecificationFixture(t, "e2b") s.SetPublicURL("http://127.0.0.1:8091") - if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, ErrSandboxPublicURLUnreachable) { + if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, deployment.ErrPublicURLUnreachable) { t.Fatal("admitted an E2B Session that could not reach Core", err) } - if setup, err := s.GetSandboxSetup(t.Context()); err != nil || setup.Provider != "e2b" { + if setup, err := deploymentService(t, s).Setup(t.Context()); err != nil || setup.Provider != "e2b" { t.Fatal("the saved E2B selection became unreadable", err) } s.SetPublicURL("https://core.example") @@ -438,25 +344,3 @@ func TestE2BAdmitsNothingWhileThePublicURLIsLoopback(t *testing.T) { t.Fatal(err) } } - -func TestDatabaseDoesNotEnumerateProviderRegistrations(t *testing.T) { - s, w, view, input := webSpecificationFixture(t, "docker") - tx, err := s.pool.Begin(t.Context()) - if err != nil { - t.Fatal(err) - } - defer tx.Rollback(t.Context()) - if _, err = tx.Exec(t.Context(), "UPDATE runtime_deployment SET provider_kind='new-adapter' WHERE singleton=true"); err != nil { - t.Fatal("database enumerated provider implementations", err) - } - // Roll back before calling the serialized Store, which still rejects unknown - // registrations even though persistence can represent a new adapter. - if err = tx.Rollback(t.Context()); err != nil { - t.Fatal(err) - } - input.Provider = "new-adapter" - input.ExpectedGeneration = view.Generation - if _, err = w.InitializeSandboxDeployment(t.Context(), view.InstallationID, input); !errors.Is(err, ErrInvalidInput) { - t.Fatal("unknown adapter reached persistence", err) - } -} diff --git a/services/core/internal/store/sandbox_specification_test.go b/services/core/internal/store/sandbox_specification_test.go index 3bfcf557b..ffa0e1dc4 100644 --- a/services/core/internal/store/sandbox_specification_test.go +++ b/services/core/internal/store/sandbox_specification_test.go @@ -1,8 +1,10 @@ package store import ( - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "strings" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) func SandboxDeploymentTestSpec(provider string) sandbox.DeploymentSpec { @@ -19,6 +21,6 @@ func SandboxDeploymentTestSpec(provider string) sandbox.DeploymentSpec { } // EnrollmentTestToken keeps only the secret token of an issued node enrollment. -func EnrollmentTestToken(issued RuntimeNodeEnrollmentToken, err error) (string, error) { +func EnrollmentTestToken(issued deployment.EnrollmentToken, err error) (string, error) { return issued.Token, err } diff --git a/services/core/internal/store/session_deletion_test.go b/services/core/internal/store/session_deletion_test.go index a18cc0b26..4d8edad55 100644 --- a/services/core/internal/store/session_deletion_test.go +++ b/services/core/internal/store/session_deletion_test.go @@ -390,7 +390,7 @@ func TestSessionDeletionKeepsProvisioningInputPlacementUntilSettled(t *testing.T WHERE s.id=$1 AND p.node_id=$2`, session.ID, d.LocalNodeID).Scan(¤t.deleted, ¤t.released); err != nil { t.Fatal("missing placement", err) } - nodes, err := s.ListRuntimeNodes(ctx) + nodes, err := deploymentService(t, s).ListNodes(ctx) if err != nil || len(nodes) != 1 { t.Fatal(nodes, err) } diff --git a/services/core/internal/store/session_model_execution_http_test.go b/services/core/internal/store/session_model_execution_http_test.go index 86ee0f461..c50a0338d 100644 --- a/services/core/internal/store/session_model_execution_http_test.go +++ b/services/core/internal/store/session_model_execution_http_test.go @@ -19,7 +19,7 @@ func TestModelExecutionHTTPWriteOnlyAndStrictAdmission(t *testing.T) { st, db := store.NewWithCredentialCipher(pool, cipher), fixtureDB{pool: pool, cipher: cipher} tenant, token := uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "catalog-test", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}}) - handler, err := publicHandler(t, st, db, auth, "codex", storeExecution(t, st), managedSandboxes(t, st)) + handler, err := publicHandler(t, st, db, auth, "codex", storeExecution(t, st), managedSandboxes(t, st, db)) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/store/template_composition_public_test.go b/services/core/internal/store/template_composition_public_test.go index c809364ce..fb08725f3 100644 --- a/services/core/internal/store/template_composition_public_test.go +++ b/services/core/internal/store/template_composition_public_test.go @@ -40,7 +40,7 @@ func TestTemplateCompositionOfficialClientPostgres(t *testing.T) { serve := func(current *store.Store) *httptest.Server { t.Helper() // Hosted admission and freezing use the real Store; no Runtime or model runs. - h, err := publicHandler(t, current, db, auth, "codex", storeExecution(t, current), managedSandboxes(t, current), fixtureDeploymentProvider()) + h, err := publicHandler(t, current, db, auth, "codex", storeExecution(t, current), managedSandboxes(t, current, db), fixtureDeploymentProvider()) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/store/template_null_selection_public_test.go b/services/core/internal/store/template_null_selection_public_test.go index e8f292070..523785b26 100644 --- a/services/core/internal/store/template_null_selection_public_test.go +++ b/services/core/internal/store/template_null_selection_public_test.go @@ -41,7 +41,7 @@ func TestTemplateNullSelectionOfficialClientPostgres(t *testing.T) { }) serve := func(current *store.Store) *httptest.Server { t.Helper() - h, err := publicHandler(t, current, db, auth, "codex", storeExecution(t, current), managedSandboxes(t, current), fixtureDeploymentProvider()) + h, err := publicHandler(t, current, db, auth, "codex", storeExecution(t, current), managedSandboxes(t, current, db), fixtureDeploymentProvider()) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/store/unified_model_configuration_http_test.go b/services/core/internal/store/unified_model_configuration_http_test.go index cfed60a7f..4d17474f2 100644 --- a/services/core/internal/store/unified_model_configuration_http_test.go +++ b/services/core/internal/store/unified_model_configuration_http_test.go @@ -21,7 +21,7 @@ func TestUnifiedModelConfigurationHTTP(t *testing.T) { if err != nil { t.Fatal(err) } - handler, err := publicHandler(t, st, db, auth, "codex", storeExecution(t, st), managedSandboxes(t, st), withCoreKeys(admin), withHarnesses([]string{"codex", "claude_sdk"})) + handler, err := publicHandler(t, st, db, auth, "codex", storeExecution(t, st), managedSandboxes(t, st, db), withCoreKeys(admin), withHarnesses([]string{"codex", "claude_sdk"})) if err != nil { t.Fatal(err) }