diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e4e64c332..0c6ae3415 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,7 +38,10 @@ Go tests, Core contract/client/service tests, Core Web and TypeScript client checks (including fixture-only Playwright acceptance), a real dedicated PostgreSQL test database, byte-for-byte sqlc regeneration checks, standalone API builds, Claude SDK tests and packaging, MiniMax companion checks, and Rust filesystem-helper -tests/format/Clippy. It intentionally has no product Web/server/installer gates. The full gate fails when the database variable is missing. +tests/format/Clippy. It intentionally has no product Web/server/installer gates. The full gate fails when the database variable is missing. The test database role +needs CREATE DATABASE permission: managed-provider tests create and drop isolated +`parsar_agents_api_*_tests` databases because provider identity is deployment-wide. +Tests must not bypass the production provider-switch guard. Use Go from `go.mod`, Node 22, pnpm 10.30.3, Python 3.9+, Rust 1.95.0 with rustfmt and Clippy, and Linux OpenSSL development libraries. `make sqlc-generate` owns only @@ -227,7 +230,7 @@ forwarding. The explicit daemon-executor decision supersedes the previous native executor interoperability requirement. The superseded execution route is removed; retain reusable filesystem helpers, necessary regressions and historical evidence without a compatibility layer. -The private daemon wire protocol is 0.4.0. Initial, prepared and active input use +The private daemon wire protocol is 0.5.0. Initial, prepared and active input use the same ordered MessageInput contract, replacing scalar prompts and attachments. User-message boundaries and text/image order remain intact through Core and the Runtime wire; adapters own native conversion and receipt aggregation. Text-only @@ -414,14 +417,15 @@ batches. Resource reads need only tenant authorization, not a live Runtime. See the [Template coverage and unresolved semantics](contracts/agents-api/environment-templates.md). SandboxProvider has five operations: Create, GetInfo, Renew, Kill and RunCommand. -Use maintained provider SDKs and thin adapters. The current hosted offering uses Docker. +Use maintained provider SDKs and thin adapters. Hosted deployments select Docker or +the optional single-host microsandbox profile. Provider initialization creates the sandbox and starts its daemon/harness; RunCommand is for initialization only. Daily execution and Files use Runtime and native or bounded local capabilities. Docker's lack of a native renewable lease does not remove service-owned hosted expiry and cleanup requirements. The official `openai_hosted` discriminator means hosting by this independent Core -service, using Docker V1. Keep the public value unchanged; `parsar_hosted` is not +service, using its configured hosted Provider. Keep the public value unchanged; `parsar_hosted` is not a new API type. Public Environment Templates apply only to this hosted path. E2B onboarding follows the application-managed `self_hosted` resource workflow: the application owns sandbox provisioning and cleanup, and our daemon connects @@ -453,6 +457,97 @@ The [managed Runtime build and operator configuration](services/agents-api/deplo defines the explicit opt-in for basic hosted admission. Building an image alone does not qualify its isolation or enable public creation. +### Optional single-host sandbox suspension + +Each Core deployment enables exactly one sandbox provider, selected at setup: +Docker or microsandbox. Keep both adapters but reject multiple provider entries, +legacy default-provider maps and engine-based placement. Harness selection is +independent. The configuration has one installation UUID, one provider kind and +one backend object. No compatibility parser or parallel provider route remains. + +The execution database pins the selected installation and backend namespace. +Under the existing execution lease, startup validates that identity before +reconciling work. Changing an installation or backend requires a previous startup +of the old configuration in maintenance, with new configuration still in +maintenance. Maintenance prevents fresh hosted Sessions and fresh allocations, +while retaining known receipts, existing Session use and explicit cleanup. +Unreleased allocation receipts include live/stopped compute, snapshots, uncertain +operations and pending cleanup; all must be released before a switch. Pending +hosted Environments that have not yet received an allocation also block a switch. +A failed check reports why and performs no resource deletion or provider change. +After a successful switch, restart the same configuration with maintenance off to +admit new sandboxes. Retain immutable historical allocation ownership; never +migrate an existing Session to another provider or recreate a released allocation. +Fresh adoption of a deployment with unverified retained allocations fails closed. + +The common +`services/agents-api/internal/sandbox` contract owns the five base operations +(Create, GetInfo, Renew, Kill, RunCommand) and the optional CheckpointProvider +capability. Core orchestration must not import an adapter or SDK. Exact compute +identity, generation construction, inspection, full snapshot capture, restore, +thaw, command execution and owned artifact cleanup use that common capability. +Provider-specific names and snapshot identities are opaque to Core. Self-hosted +compute and providers without checkpoint support keep their existing behavior. + +The [microsandbox deployment profile](services/agents-api/deploy/microsandbox/README.md) +pins the SDK, runtime, firmware and image. Core remains a pure-Go binary. The +one-shot native Linux helper contains the SDK/FFI and runs under the same private +service namespace; it is not another scheduler or network control plane. Ordinary +pause does not release RAM. Suspension captures and verifies a full snapshot, +stops the exact source, and removes its writable compute closure only after the +artifact is durably identified. Explicit network policy applies on create and +restore. Do not inherit undeclared host resources. + +Suspend only after at least one Turn is terminal, no queued/in-progress/waiting +root or subagent Turn, pending input/file operation or initialization remains, +and real activity has been idle for the configured interval. Heartbeats do not +reset activity. The daemon must close admission and drain native cleanup, output +receipts and file work before acknowledging planned suspension. Never change a +harness or keep an agent process alive across Turns solely to meet this feature. +The acceptance boundary is a next Turn in the same Session with history, files +and configuration intact, without replaying an earlier request. + +The existing Worker lease, Session lock and lifecycle gate own both providers. +New Turn claims, file-write intents and capture admission serialize under the +Session lock. Turn and file-write admission share the same compute-phase check; +existing receipts remain readable. New pending work cancels capture and wakes +the same source. Normal preparation waits for the +compute phase to be running, after the authenticated resume handshake; a pending +input remains pending if its promotion conflicts with a lifecycle transition. +Private compute phases and revision-checked JSON receipts live on the existing +allocation. Persist quiesce/capture/restore intent before effects; only the fresh +receipt performs a capture or restore. Recovery observes the exact attempt and +never retries an unknown creation, capture or restore. A consumed snapshot cannot +roll a running generation back. Deletion, revocation and retention expiry take +precedence over wake, including at the final database compare-and-swap. Retain +unknown cleanup identities until owned resources are confirmed absent. + +Fixed guest CPU/memory/disk settings, max_active reservations, max_retained +allocation count and snapshot retention bound the single host. Unknown operations +retain capacity reservations. Source teardown must be confirmed before releasing +active capacity. Delete consumed artifacts and old compute closures; do not grow +a chain of old writable disks across suspension cycles. No Kubernetes, distributed +scheduler or snapshot replication belongs in this V1 profile. + +Queued work and live Environment file access request wake. History and published +artifact reads do not. Planned suspension uses private daemon wire 0.5.0 with an +Environment and suspension token; a PID/start-time fenced local control signal +wakes the parked daemon, which reauthenticates before admitting new work. A +transient disconnect before confirmation retries the same armed suspension with +bounded attempts and backoff; permanent authentication or protocol rejection +still closes it. Snapshot +lifetime has no daemon wall-clock timer: Core owns its retention deadline. A lost +quiesce acknowledgement may thaw the same source using explicit rollback control; +it does not authorize capturing it. Ordinary disconnect keeps the existing +conservative shutdown behavior. Authentication rejection cannot create a new +Runtime or replay a request. + +All normal `make check` gates still apply. Linux qualification additionally runs +the pinned helper module tests/build through `check-microsandbox-provider`, a real +KVM full-snapshot/reclamation probe, and idle-to-next-Turn integration acceptance. +Synthetic process-memory probes support the backend claim only; they do not prove +agent continuity. Independent blind review uses the clarified idle-only scope. + Managed Runtime allocation, dedicated daemon credential hash and exact Session binding commit atomically before Provider.Create, using the existing execution lease and Session lock. Only the fresh allocation receipt permits Create; retries @@ -491,7 +586,8 @@ Allocation state is private compute ownership, separate from public Environment connection/native readiness. Adapters qualify bootstrap completion; Core does not infer it from an engine or provider name. Connected, observed compute receives service keepalives between Turns. Keepalives cannot revive a one-hour lapse or a -cleanup request. Idle alone never requests shutdown. A stopped/missing container +cleanup request. The Docker provider keeps its current idle behavior; only an +explicit checkpoint policy may suspend completed, idle work as described below. A stopped/missing container does not authorize discarding retained workspace or history. Session deletion or expiry requests cleanup, revokes the scoped device and cancels pending work before Provider.Kill; the existing Worker serializes these lifecycle operations and drains diff --git a/Makefile b/Makefile index 0ae2fd788..f9142df47 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SWAG_VERSION ?= v1.16.4 help: @printf '%s\n' 'make build-agents-api Build standalone Core commands' 'make build-daemon Build the execution daemon' 'make check Run Core, persistence and runtime checks' 'See README.md for runtime prerequisites and deployment.' -check: check-database check-sqlc check-go check-agents-api check-claude-sdk check-web check-mcode-harness check-agents-executor +check: check-database check-sqlc check-go check-microsandbox-provider check-agents-api check-claude-sdk check-web check-mcode-harness check-agents-executor @printf 'Parsar Core checks passed.\n' check-database: @@ -94,3 +94,20 @@ build-mcode-harness: build-mcode-runtime: ./scripts/build-mcode-runtime.sh + +.PHONY: build-microsandbox-provider check-microsandbox-provider +build-microsandbox-provider: + @test "$$(go env GOOS)" = linux || { echo 'The microsandbox provider helper requires Linux' >&2; exit 1; } + @set -e; output="$${PARSAR_HOME:-$$HOME/.parsar}/build/microsandbox-provider"; \ + [[ "$$output" == /* ]] || { echo 'Provider output directory must be absolute' >&2; exit 1; }; \ + mkdir -p "$$output"; \ + cd services/agents-api/tools/microsandbox-provider; \ + GOWORK=off CGO_ENABLED=1 go build -mod=readonly -trimpath -o "$$output/agents-api-microsandbox-provider" . + +check-microsandbox-provider: + go test -mod=readonly ./services/agents-api/internal/sandbox/microsandbox/... -count=1 + @if [[ "$$(go env GOOS)" == linux ]]; then \ + cd services/agents-api/tools/microsandbox-provider && GOWORK=off CGO_ENABLED=1 go test -mod=readonly ./... -count=1; \ + else \ + printf 'Skipping the Linux-only microsandbox SDK helper tests; the full Linux gate is required before release.\n'; \ + fi diff --git a/apps/parsar-daemon/internal/cli/connect.go b/apps/parsar-daemon/internal/cli/connect.go index 6e78ab5a5..391db6ea0 100644 --- a/apps/parsar-daemon/internal/cli/connect.go +++ b/apps/parsar-daemon/internal/cli/connect.go @@ -295,6 +295,13 @@ func mainLoopRemote(rc *runContext, profile string, prof auth.Profile, agentCLIs registry := agent.NewRegistry() registerAgentKinds(registry, agentCLIs, prof.ServerURL) + control, err := newSuspendControl() + if err != nil { + return err + } + if control != nil { + defer control.Close() + } dial := func(ctx context.Context) (*transport.Conn, error) { conn, err := transport.Dial(ctx, transport.DialOptions{ WSURL: wsURL, @@ -315,6 +322,9 @@ func mainLoopRemote(rc *runContext, profile string, prof auth.Profile, agentCLIs return conn, err } + if control != nil { + return runSuspendLoop(rootCtx, dial, registry, boot, agentCLIs, control) + } for { if err := rootCtx.Err(); err != nil { return nil diff --git a/apps/parsar-daemon/internal/cli/connect_suspend.go b/apps/parsar-daemon/internal/cli/connect_suspend.go new file mode 100644 index 000000000..38a391252 --- /dev/null +++ b/apps/parsar-daemon/internal/cli/connect_suspend.go @@ -0,0 +1,282 @@ +package cli + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/authoring" + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/dispatch" + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/localworkspace" + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/transport" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + obslog "github.com/MiniMax-AI-Dev/parsar/internal/obs/log" +) + +const suspendReconnectTimeout = 30 * time.Second + +// The bridge and router share this sender across the one explicitly planned +// reconnect. Quiesce has drained every prior send before replace is called. +type reconnectSender struct { + mu sync.RWMutex + conn *transport.Conn +} + +func (s *reconnectSender) Send(ctx context.Context, env proto.Envelope) error { + s.mu.RLock() + conn := s.conn + s.mu.RUnlock() + return conn.Send(ctx, env) +} +func (s *reconnectSender) replace(conn *transport.Conn) { s.mu.Lock(); s.conn = conn; s.mu.Unlock() } + +type suspendedRouter struct { + router *dispatch.Router + bridge *authoring.Bridge + sender *reconnectSender + registry *agent.Registry + local *localworkspace.Binding +} + +func newSuspendedRouter(conn *transport.Conn, registry *agent.Registry) (*suspendedRouter, error) { + local, err := localworkspace.Load() + if err != nil { + return nil, err + } + sender := &reconnectSender{conn: conn} + bridge := authoring.New(sender) + wrapped := authoringRegistry(registry, bridge) + router, err := dispatch.New(dispatch.Config{Registry: wrapped, Sender: sender, Log: obslog.Bg(), LocalWorkspace: local}) + if err != nil { + return nil, err + } + return &suspendedRouter{router: router, bridge: bridge, sender: sender, registry: wrapped, local: local}, nil +} + +func (s *suspendedRouter) shutdown() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return s.router.Shutdown(ctx) +} + +// runSuspendLoop uses the ordinary connection authentication and dispatch chain. +// Only an acknowledged, fully drained suspension retains a Router across sockets. +func runSuspendLoop(ctx context.Context, dial transport.DialFn, registry *agent.Registry, boot *transport.BootstrapResponse, discovery agentCLIDiscovery, control *suspendControl) error { + for ctx.Err() == nil { + conn, err := transport.Reconnect(ctx, dial, transport.DefaultBackoff, nil) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + state, err := newSuspendedRouter(conn, registry) + if err != nil { + _ = conn.Close() + return err + } + err = serveSuspendLifecycle(ctx, conn, dial, state, boot, discovery, control) + closeErr := state.shutdown() + if closeErr != nil { + return errors.Join(err, closeErr) + } + if ctx.Err() != nil { + return nil + } + if errors.Is(err, transport.ErrPermanent) { + return err + } + if err != nil { + obslog.Bg().Warn("hosted connection ended", "err", err) + } + if err := transport.Sleep(ctx, time.Second); err != nil { + return nil + } + } + return nil +} + +func serveSuspendLifecycle(ctx context.Context, conn *transport.Conn, dial transport.DialFn, state *suspendedRouter, boot *transport.BootstrapResponse, discovery agentCLIDiscovery, control *suspendControl) error { + defer func() { _ = conn.Close() }() + for { + request, err := state.pump(ctx, conn, boot, discovery, control) + _ = conn.Close() + if err != nil || request == nil { + return err + } + // There is deliberately no pre-snapshot wall-clock deadline here. A clock + // jump on restore must not end the park before the host's wake command. + // Only this exact armed suspension can park; Core owns its snapshot TTL. + if err := control.Wait(ctx); err != nil { + return err + } + next, err := state.reconnectSuspension(ctx, dial, *request, control, suspendReconnectTimeout) + if err != nil { + if next != nil { + _ = next.Close() + } + // Once Resume commits, a failed acknowledgement is an ordinary + // disconnect. Keep only its receipt for the fresh authenticated Router. + if control.lastResumed != nil && control.lastResumed.SameSuspension(*request) { + return errors.Join(err, control.Disarm()) + } + return err + } + conn = next + if err := control.Disarm(); err != nil { + return err + } + } +} + +// Each attempt has a deadline, but transient transport failures cannot discard +// the armed suspension. Only matching Core confirmation opens admission again. +func (s *suspendedRouter) reconnectSuspension(ctx context.Context, dial transport.DialFn, request proto.EnvironmentSuspendPayload, control *suspendControl, timeout time.Duration) (*transport.Conn, error) { + for attempt := 1; ; attempt++ { + recovery, cancel := context.WithTimeout(ctx, timeout) + conn, err := transport.Reconnect(recovery, dial, transport.DefaultBackoff, nil) + if err == nil { + err = s.resume(recovery, conn, request, control) + } + cancel() + if err == nil { + return conn, nil + } + if conn != nil { + _ = conn.Close() + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + if errors.Is(err, transport.ErrPermanent) || (control.lastResumed != nil && control.lastResumed.SameSuspension(request)) { + return nil, err + } + if err := transport.Sleep(ctx, transport.DefaultBackoff.Delay(attempt)); err != nil { + return nil, err + } + } +} + +func (s *suspendedRouter) heartbeats(ctx context.Context, conn *transport.Conn, boot *transport.BootstrapResponse, discovery agentCLIDiscovery) { + conn.StartHeartbeats(ctx, boot.HeartbeatInterval(), func() proto.HeartbeatPayload { + kinds := s.registry.SupportedAgentKinds() + for i := range kinds { + caps := &kinds[i].Capabilities + caps.WorkspaceOutputExport = s.local.CanExport() && caps.LocalEnvironment && caps.WorkspaceReadPreparation + } + return proto.HeartbeatPayload{Timestamp: time.Now().Unix(), ActiveRequests: s.router.ActiveRuns(), DaemonVersion: Version, ClaudeAvailable: discovery.ClaudeCode.Available, SupportedAgentKinds: kinds} + }, obslog.Bg()) +} + +func (s *suspendedRouter) pump(ctx context.Context, conn *transport.Conn, boot *transport.BootstrapResponse, discovery agentCLIDiscovery, control *suspendControl) (*proto.EnvironmentSuspendPayload, error) { + s.heartbeats(ctx, conn, boot, discovery) + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-conn.Done(): + return nil, conn.Err() + case env, ok := <-conn.Recv(): + if !ok { + return nil, conn.Err() + } + if env.Type == proto.TypeAuthoringResponse { + s.bridge.Deliver(env) + continue + } + if env.Type == proto.TypeEnvironmentResume { + request := rejectedResumeRequest(env) + code := "not_suspended" + valid := env.ID != "" && len(env.ID) <= 128 && request.EnvironmentID == control.identity.EnvironmentID && request.SuspendID != "" && len(request.SuspendID) <= 128 + if valid && ((control.lastResumed != nil && control.lastResumed.SameSuspension(request)) || request.Rollback) { + code = "" + control.lastResumed = &request + } + if err := sendSuspendResult(ctx, conn, env, proto.TypeEnvironmentResumed, request, code); err != nil { + return nil, err + } + continue + } + if env.Type == proto.TypeEnvironmentQuiesce { + var request proto.EnvironmentSuspendPayload + if env.ID == "" || len(env.ID) > 128 || env.DecodePayload(&request) != nil || request.EnvironmentID != control.identity.EnvironmentID || request.SuspendID == "" || request.Rollback { + if err := sendSuspendResult(ctx, conn, env, proto.TypeEnvironmentQuiesced, request, "invalid_request"); err != nil { + return nil, err + } + continue + } + quiet, cancel := context.WithTimeout(ctx, 5*time.Second) + err := s.router.Quiesce(quiet, request) + cancel() + if err != nil { + if sendErr := sendSuspendResult(ctx, conn, env, proto.TypeEnvironmentQuiesced, request, "resource_busy"); sendErr != nil { + return nil, sendErr + } + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return nil, err + } + continue + } + if err := control.Arm(request); err != nil { + return nil, err + } + if err := sendSuspendResult(ctx, conn, env, proto.TypeEnvironmentQuiesced, request, ""); err != nil { + return nil, err + } + return &request, nil + } + if err := s.router.Handle(ctx, env); err != nil { + obslog.Bg().Error("router.Handle failed", "type", env.Type, "err", err) + } + } + } +} + +func (s *suspendedRouter) resume(ctx context.Context, conn *transport.Conn, request proto.EnvironmentSuspendPayload, control *suspendControl) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-conn.Done(): + if err := conn.Err(); err != nil { + return err + } + return transport.ErrConnClosed + case env, ok := <-conn.Recv(): + if !ok { + if err := conn.Err(); err != nil { + return err + } + return transport.ErrConnClosed + } + var echoed proto.EnvironmentSuspendPayload + if env.Type != proto.TypeEnvironmentResume || env.ID == "" || env.DecodePayload(&echoed) != nil || !echoed.SameSuspension(request) { + return errors.Join(transport.ErrPermanent, errors.New("connect: expected authenticated suspension resume")) + } + s.sender.replace(conn) + if err := s.router.Resume(echoed, s.sender); err != nil { + return errors.Join(transport.ErrPermanent, err) + } + control.lastResumed = &request + return sendSuspendResult(ctx, conn, env, proto.TypeEnvironmentResumed, request, "") + } +} + +func sendSuspendResult(ctx context.Context, conn *transport.Conn, env proto.Envelope, kind string, request proto.EnvironmentSuspendPayload, code string) error { + result, err := proto.NewEnvelope(kind, env.ID, proto.EnvironmentSuspendResultPayload{EnvironmentID: request.EnvironmentID, SuspendID: request.SuspendID, Accepted: code == "", ErrorCode: code}) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + return conn.Send(ctx, result) +} + +func rejectedResumeRequest(env proto.Envelope) proto.EnvironmentSuspendPayload { + var request proto.EnvironmentSuspendPayload + if len(env.Payload) <= 1024 { + _ = env.DecodePayload(&request) + } + return request +} diff --git a/apps/parsar-daemon/internal/cli/connect_suspend_test.go b/apps/parsar-daemon/internal/cli/connect_suspend_test.go new file mode 100644 index 000000000..5ad36ef5d --- /dev/null +++ b/apps/parsar-daemon/internal/cli/connect_suspend_test.go @@ -0,0 +1,334 @@ +package cli + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/transport" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/gorilla/websocket" +) + +func TestPlannedReconnectRequiresAuthenticatedMatchingResume(t *testing.T) { + for _, scenario := range []string{"resume", "rollback", "reconnect", "revoked", "wrong_operation"} { + t.Run(scenario, func(t *testing.T) { + t.Setenv("PARSAR_RUNTIME_WORKSPACE", "") + var connections atomic.Int32 + peers := make(chan *websocket.Conn, 3) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer credential" { + http.Error(w, "unauthorized", 401) + return + } + attempt := connections.Add(1) + if scenario == "revoked" && attempt > 1 { + http.Error(w, "revoked", 401) + return + } + peer, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err == nil { + peers <- peer + } + })) + defer server.Close() + dial := func(ctx context.Context) (*transport.Conn, error) { + return transport.Dial(ctx, transport.DialOptions{WSURL: "ws" + strings.TrimPrefix(server.URL, "http"), DeviceID: "device", Credential: "credential", DaemonVersion: proto.Version}) + } + registry := agent.NewRegistry() + control := &suspendControl{path: filepath.Join(t.TempDir(), "suspend.json"), identity: suspendIdentity{EnvironmentID: "env"}, signal: make(chan os.Signal, 1)} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + done <- runSuspendLoop(ctx, dial, registry, &transport.BootstrapResponse{HeartbeatSeconds: 1}, agentCLIDiscovery{}, control) + }() + var first *websocket.Conn + select { + case first = <-peers: + case <-time.After(3 * time.Second): + t.Fatal("initial connection missing") + } + defer first.Close() + request := proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "planned"} + sendLifecycleFrame(t, first, proto.TypeEnvironmentQuiesce, request) + result := readLifecycleResult(t, first, proto.TypeEnvironmentQuiesced) + if !result.Accepted { + t.Fatalf("quiesce rejected: %+v", result) + } + // The daemon deliberately closes its old socket. Neither that error nor a + // heartbeat interval may exit the planned park while awaiting host wake. + _ = first.SetReadDeadline(time.Now().Add(time.Second)) + for { + var env proto.Envelope + if first.ReadJSON(&env) != nil { + break + } + } + select { + case err := <-done: + t.Fatalf("park exited without wake: %v", err) + case <-time.After(25 * time.Millisecond): + } + control.signal <- syscall.SIGUSR1 + if scenario == "revoked" { + select { + case err := <-done: + if !errors.Is(err, transport.ErrPermanent) { + t.Fatalf("revoke error=%v", err) + } + case <-time.After(3 * time.Second): + t.Fatal("revoked reconnect did not terminate") + } + } else { + var second *websocket.Conn + select { + case second = <-peers: + case <-time.After(3 * time.Second): + t.Fatal("resume connection missing") + } + defer second.Close() + if scenario == "rollback" { + request.Rollback = true + } + if scenario == "wrong_operation" { + request.SuspendID = "stale" + } + sendLifecycleFrame(t, second, proto.TypeEnvironmentResume, request) + if scenario == "resume" || scenario == "rollback" || scenario == "reconnect" { + result = readLifecycleResult(t, second, proto.TypeEnvironmentResumed) + if !result.Accepted { + t.Fatalf("resume failed: %+v", result) + } + // A lost response may be retried on this same authenticated socket. + sendLifecycleFrame(t, second, proto.TypeEnvironmentResume, request) + if repeated := readLifecycleResult(t, second, proto.TypeEnvironmentResumed); !repeated.Accepted { + t.Fatal("resume receipt was not idempotent") + } + if scenario == "reconnect" { + // Core can lose the receipt before committing waking -> running. + // The next ordinary connection has a fresh Router but must confirm + // this consumed token without restoring or retaining old work. + _ = second.Close() + var third *websocket.Conn + select { + case third = <-peers: + case <-time.After(3 * time.Second): + t.Fatal("ordinary reconnect missing") + } + defer third.Close() + sendLifecycleFrame(t, third, proto.TypeEnvironmentResume, request) + if repeated := readLifecycleResult(t, third, proto.TypeEnvironmentResumed); !repeated.Accepted { + t.Fatal("receipt lost across ordinary reconnect") + } + request.SuspendID = "unrelated" + sendLifecycleFrame(t, third, proto.TypeEnvironmentResume, request) + if unrelated := readLifecycleResult(t, third, proto.TypeEnvironmentResumed); unrelated.Accepted { + t.Fatal("unrelated resume accepted by fresh Router") + } + } + cancel() + } + select { + case err := <-done: + if scenario == "wrong_operation" && err == nil { + t.Fatal("mismatched resume accepted") + } + case <-time.After(3 * time.Second): + t.Fatal("lifecycle failed to terminate") + } + } + }) + } +} + +func sendLifecycleFrame(t *testing.T, peer *websocket.Conn, kind string, payload proto.EnvironmentSuspendPayload) { + t.Helper() + env, err := proto.NewEnvelope(kind, "operation", payload) + if err != nil { + t.Fatal(err) + } + if err := peer.WriteJSON(env); err != nil { + t.Fatal(err) + } +} +func readLifecycleResult(t *testing.T, peer *websocket.Conn, kind string) proto.EnvironmentSuspendResultPayload { + t.Helper() + _ = peer.SetReadDeadline(time.Now().Add(3 * time.Second)) + for { + var env proto.Envelope + if err := peer.ReadJSON(&env); err != nil { + t.Fatal(err) + } + if env.Type != kind { + continue + } + var result proto.EnvironmentSuspendResultPayload + if err := env.DecodePayload(&result); err != nil { + t.Fatal(err) + } + return result + } +} + +func TestRunningSourceOnlyAcceptsExplicitRollbackOrItsReceipt(t *testing.T) { + t.Setenv("PARSAR_RUNTIME_WORKSPACE", "") + peers := make(chan *websocket.Conn, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + peer, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err == nil { + peers <- peer + } + })) + defer server.Close() + dial := func(ctx context.Context) (*transport.Conn, error) { + return transport.Dial(ctx, transport.DialOptions{WSURL: "ws" + strings.TrimPrefix(server.URL, "http"), DeviceID: "device", Credential: "credential", DaemonVersion: proto.Version}) + } + control := &suspendControl{path: filepath.Join(t.TempDir(), "control.json"), identity: suspendIdentity{EnvironmentID: "env"}, signal: make(chan os.Signal, 1)} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + done <- runSuspendLoop(ctx, dial, agent.NewRegistry(), &transport.BootstrapResponse{HeartbeatSeconds: 60}, agentCLIDiscovery{}, control) + }() + var peer *websocket.Conn + select { + case peer = <-peers: + case <-time.After(time.Second): + t.Fatal("no connection") + } + defer peer.Close() + for _, test := range []struct { + request proto.EnvironmentSuspendPayload + accepted bool + }{ + {proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "cold_restore"}, false}, + {proto.EnvironmentSuspendPayload{EnvironmentID: "wrong", SuspendID: "rollback", Rollback: true}, false}, + {proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "rollback", Rollback: true}, true}, + {proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "rollback"}, true}, + {proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "cold_restore"}, false}, + } { + sendLifecycleFrame(t, peer, proto.TypeEnvironmentResume, test.request) + got := readLifecycleResult(t, peer, proto.TypeEnvironmentResumed) + if got.Accepted != test.accepted { + t.Fatalf("request=%+v result=%+v", test.request, got) + } + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("shutdown did not settle") + } +} + +func TestSuspensionReconnectBeforeConfirmation(t *testing.T) { + for _, scenario := range []string{"disconnect", "timeout", "revoked", "cancelled", "deleted"} { + t.Run(scenario, func(t *testing.T) { + t.Setenv("PARSAR_RUNTIME_WORKSPACE", "") + var attempts atomic.Int32 + peers := make(chan *websocket.Conn, 2) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempt := attempts.Add(1) + if scenario == "revoked" && attempt > 1 { + http.Error(w, "revoked", http.StatusUnauthorized) + return + } + peer, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err == nil { + peers <- peer + } + })) + defer server.Close() + dial := func(ctx context.Context) (*transport.Conn, error) { + return transport.Dial(ctx, transport.DialOptions{WSURL: "ws" + strings.TrimPrefix(server.URL, "http"), DeviceID: "device", Credential: "credential", DaemonVersion: proto.Version}) + } + state, err := newSuspendedRouter(nil, agent.NewRegistry()) + if err != nil { + t.Fatal(err) + } + defer state.shutdown() + request := proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "pending-confirmation"} + if err := state.router.Quiesce(t.Context(), request); err != nil { + t.Fatal(err) + } + control := &suspendControl{path: filepath.Join(t.TempDir(), "control.json"), identity: suspendIdentity{EnvironmentID: "env"}, signal: make(chan os.Signal, 1)} + if err := control.Arm(request); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { + conn, err := state.reconnectSuspension(ctx, dial, request, control, 100*time.Millisecond) + if conn != nil { + _ = conn.Close() + } + done <- err + }() + var first *websocket.Conn + select { + case first = <-peers: + case <-time.After(3 * time.Second): + t.Fatal("initial recovery connection missing") + } + defer first.Close() + switch scenario { + case "cancelled": + cancel() + case "deleted": + if err := first.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(4001, "runtime deleted"), time.Now().Add(time.Second)); err != nil { + t.Fatal(err) + } + case "timeout": + // Leave the connection open without confirmation. The short injected + // attempt deadline must cause a new authenticated connection. + default: + _ = first.Close() + } + if scenario == "disconnect" || scenario == "timeout" { + var second *websocket.Conn + select { + case second = <-peers: + case err := <-done: + t.Fatalf("transient failure discarded suspension: %v", err) + case <-time.After(3 * time.Second): + t.Fatal("recovery retry missing") + } + defer second.Close() + sendLifecycleFrame(t, second, proto.TypeEnvironmentResume, request) + if result := readLifecycleResult(t, second, proto.TypeEnvironmentResumed); !result.Accepted { + t.Fatal("matching resume rejected after retry") + } + } + select { + case err := <-done: + switch scenario { + case "revoked", "deleted": + if !errors.Is(err, transport.ErrPermanent) { + t.Fatalf("permanent rejection=%v", err) + } + case "cancelled": + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation=%v", err) + } + default: + if err != nil || control.lastResumed == nil || !control.lastResumed.SameSuspension(request) { + t.Fatalf("recovery did not preserve suspension: %v", err) + } + } + case <-time.After(3 * time.Second): + t.Fatal("recovery failed to settle") + } + }) + } +} diff --git a/apps/parsar-daemon/internal/cli/root.go b/apps/parsar-daemon/internal/cli/root.go index 2719d2d36..50c6fd073 100644 --- a/apps/parsar-daemon/internal/cli/root.go +++ b/apps/parsar-daemon/internal/cli/root.go @@ -33,6 +33,7 @@ func defaultRunContext() *runContext { // commands lists subcommands in --help render order: the user's // likely flow connect → status → stop / logs → logout. var commands = []command{ + {name: "resume", summary: "Wake one planned hosted suspension", run: runResume}, {name: "runtime-capabilities", summary: "Install frozen capabilities in the packaged Runtime", run: runRuntimeCapabilities}, {name: "runtime-mcp-exec", summary: "Execute installed MCP inside the packaged Runtime sandbox", run: runRuntimeMCP}, {name: "placement", summary: "Enroll or retire an explicitly managed local execution placement", run: runPlacement}, diff --git a/apps/parsar-daemon/internal/cli/root_test.go b/apps/parsar-daemon/internal/cli/root_test.go index ffa12cdbf..9cf3d8bd7 100644 --- a/apps/parsar-daemon/internal/cli/root_test.go +++ b/apps/parsar-daemon/internal/cli/root_test.go @@ -60,6 +60,7 @@ func TestSubcommandsAreRegistered(t *testing.T) { // Guards against dropping a subcommand off the commands slice — // the public CLI surface is the shipped contract. want := map[string]bool{ + "resume": false, "runtime-capabilities": false, "runtime-mcp-exec": false, "placement": false, diff --git a/apps/parsar-daemon/internal/cli/suspend_control.go b/apps/parsar-daemon/internal/cli/suspend_control.go new file mode 100644 index 000000000..ec6a9ce20 --- /dev/null +++ b/apps/parsar-daemon/internal/cli/suspend_control.go @@ -0,0 +1,144 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/signal" + "path/filepath" + "syscall" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +const suspendControlEnv = "PARSAR_DAEMON_SUSPEND_PID_FILE" + +type suspendIdentity struct { + PID int `json:"pid"` + StartTime string `json:"start_time"` + EnvironmentID string `json:"environment_id"` + SuspendID string `json:"suspend_id"` +} + +type suspendControl struct { + // Keep one consumed token across ordinary socket failures so Core can + // retry its acknowledgement. Arm replaces it before the next suspension. + lastResumed *proto.EnvironmentSuspendPayload + path string + identity suspendIdentity + signal chan os.Signal +} + +func newSuspendControl() (*suspendControl, error) { + path := os.Getenv(suspendControlEnv) + if path == "" { + return nil, nil + } + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return nil, errors.New("connect: absolute suspend control file required") + } + info, err := os.Lstat(filepath.Dir(path)) + if err != nil || !info.IsDir() || info.Mode().Perm()&0077 != 0 { + return nil, errors.New("connect: private suspend control directory required") + } + st, ok := info.Sys().(*syscall.Stat_t) + if !ok || st.Uid != uint32(os.Getuid()) { + return nil, errors.New("connect: owned suspend control directory required") + } + environment := os.Getenv("PARSAR_RUNTIME_ENVIRONMENT_ID") + if environment == "" { + return nil, errors.New("connect: suspend control requires an Environment binding") + } + start, err := suspendProcessStart(os.Getpid()) + if err != nil { + return nil, err + } + c := &suspendControl{path: path, identity: suspendIdentity{PID: os.Getpid(), StartTime: start, EnvironmentID: environment}, signal: make(chan os.Signal, 1)} + // Refuse a second owner. A new microVM must not adopt another live daemon. + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0600) + if err != nil { + return nil, fmt.Errorf("connect: create suspend control: %w", err) + } + err = json.NewEncoder(f).Encode(c.identity) + closeErr := f.Close() + if err != nil || closeErr != nil { + _ = os.Remove(path) + return nil, errors.New("connect: write suspend control failed") + } + signal.Notify(c.signal, syscall.SIGUSR1) + return c, nil +} + +func (c *suspendControl) Arm(request proto.EnvironmentSuspendPayload) error { + if request.EnvironmentID != c.identity.EnvironmentID || request.SuspendID == "" { + return errors.New("suspension identity mismatch") + } + // Signals delivered while serving cannot authorize a future suspension. + for { + select { + case <-c.signal: + continue + default: + } + break + } + c.lastResumed = nil + c.identity.SuspendID = request.SuspendID + return c.save() +} + +func (c *suspendControl) save() error { + f, err := os.CreateTemp(filepath.Dir(c.path), ".suspend-*") + if err != nil { + return err + } + defer os.Remove(f.Name()) + err = json.NewEncoder(f).Encode(c.identity) + if err == nil { + err = f.Sync() + } + closeErr := f.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + return os.Rename(f.Name(), c.path) +} + +func (c *suspendControl) Wait(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-c.signal: + return nil + } +} + +func (c *suspendControl) Disarm() error { c.identity.SuspendID = ""; return c.save() } +func (c *suspendControl) Close() { signal.Stop(c.signal); _ = os.Remove(c.path) } + +func runResume(_ *runContext, args []string) error { + flags := newFlagSet("resume") + path := flags.String("control-file", "", "absolute hosted daemon control file") + environment := flags.String("environment-id", "", "expected Environment identity") + suspension := flags.String("suspend-id", "", "expected suspension identity") + if err := flags.Parse(args); err != nil { + return err + } + if flags.NArg() != 0 || *environment == "" || *suspension == "" { + return errors.New("resume: Environment and suspension identities required") + } + raw, err := readEnvironmentPrivateFile(*path) + if err != nil { + return errors.New("resume: private control file unavailable") + } + var identity suspendIdentity + if decodeEnvironmentJSON(raw, &identity) != nil || identity.PID <= 0 || identity.StartTime == "" || identity.EnvironmentID != *environment || identity.SuspendID != *suspension { + return errors.New("resume: suspension identity mismatch") + } + return signalSuspendedProcess(identity) +} diff --git a/apps/parsar-daemon/internal/cli/suspend_control_linux_test.go b/apps/parsar-daemon/internal/cli/suspend_control_linux_test.go new file mode 100644 index 000000000..3829f949f --- /dev/null +++ b/apps/parsar-daemon/internal/cli/suspend_control_linux_test.go @@ -0,0 +1,67 @@ +//go:build linux + +package cli + +import ( + "context" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +func TestResumeControlRejectsStaleIdentityAndDiscardsPreParkSignals(t *testing.T) { + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "control.json") + t.Setenv(suspendControlEnv, path) + t.Setenv("PARSAR_RUNTIME_ENVIRONMENT_ID", "env") + control, err := newSuspendControl() + if err != nil { + t.Fatal(err) + } + defer control.Close() + control.signal <- syscall.SIGUSR1 + if err := control.Arm(proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "current"}); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + if err := control.Wait(ctx); err == nil { + t.Fatal("stale signal woke future suspension") + } + cancel() + if err := runResume(nil, []string{"--control-file", path, "--environment-id", "env", "--suspend-id", "stale"}); err == nil { + t.Fatal("stale suspension accepted") + } + saved := control.identity.StartTime + control.identity.StartTime = "wrong" + if err := control.save(); err != nil { + t.Fatal(err) + } + if err := runResume(nil, []string{"--control-file", path, "--environment-id", "env", "--suspend-id", "current"}); err == nil { + t.Fatal("reused PID accepted") + } + control.identity.StartTime = saved + if err := control.save(); err != nil { + t.Fatal(err) + } + if err := runResume(nil, []string{"--control-file", path, "--environment-id", "env", "--suspend-id", "current"}); err != nil { + t.Fatal(err) + } + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := control.Wait(ctx); err != nil { + t.Fatal("validated pidfd signal did not wake owner", err) + } + if err := control.Disarm(); err != nil { + t.Fatal(err) + } + if err := runResume(nil, []string{"--control-file", path, "--environment-id", "env", "--suspend-id", "current"}); err == nil { + t.Fatal("completed suspension accepted another wake") + } +} diff --git a/apps/parsar-daemon/internal/cli/suspend_process_linux.go b/apps/parsar-daemon/internal/cli/suspend_process_linux.go new file mode 100644 index 000000000..556172408 --- /dev/null +++ b/apps/parsar-daemon/internal/cli/suspend_process_linux.go @@ -0,0 +1,56 @@ +//go:build linux + +package cli + +import ( + "errors" + "fmt" + "os" + "strings" + + "golang.org/x/sys/unix" +) + +func suspendProcessStart(pid int) (string, error) { + raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)) + if err != nil { + return "", errors.New("suspend process identity unavailable") + } + // comm may contain whitespace and parentheses; fields after its last closing + // parenthesis begin at the process state (field 3), starttime is field 22. + end := strings.LastIndexByte(string(raw), ')') + if end < 0 { + return "", errors.New("invalid process identity") + } + fields := strings.Fields(string(raw[end+1:])) + if len(fields) <= 19 { + return "", errors.New("invalid process identity") + } + return fields[19], nil +} + +func signalSuspendedProcess(identity suspendIdentity) error { + // Pin the process before reading its identity, eliminating PID reuse between + // validation and signal delivery. A dead pidfd can never target its successor. + fd, err := unix.PidfdOpen(identity.PID, 0) + if err != nil { + return errors.New("resume: daemon unavailable") + } + defer unix.Close(fd) + start, err := suspendProcessStart(identity.PID) + if err != nil || start != identity.StartTime { + return errors.New("resume: process identity mismatch") + } + own, err := os.Stat("/proc/self/exe") + if err != nil { + return errors.New("resume: executable identity unavailable") + } + target, err := os.Stat(fmt.Sprintf("/proc/%d/exe", identity.PID)) + if err != nil || !os.SameFile(own, target) { + return errors.New("resume: executable identity mismatch") + } + if err := unix.PidfdSendSignal(fd, unix.SIGUSR1, nil, 0); err != nil { + return errors.New("resume: signal failed") + } + return nil +} diff --git a/apps/parsar-daemon/internal/cli/suspend_process_other.go b/apps/parsar-daemon/internal/cli/suspend_process_other.go new file mode 100644 index 000000000..cdf1bc1f4 --- /dev/null +++ b/apps/parsar-daemon/internal/cli/suspend_process_other.go @@ -0,0 +1,12 @@ +//go:build !linux + +package cli + +import "errors" + +func suspendProcessStart(int) (string, error) { + return "", errors.New("hosted suspension requires Linux") +} +func signalSuspendedProcess(suspendIdentity) error { + return errors.New("hosted suspension requires Linux") +} diff --git a/apps/parsar-daemon/internal/dispatch/preparation.go b/apps/parsar-daemon/internal/dispatch/preparation.go index f5d892feb..acfe25be6 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation.go +++ b/apps/parsar-daemon/internal/dispatch/preparation.go @@ -175,7 +175,7 @@ func (r *Router) handleExecutionRelease(_ context.Context, env proto.Envelope) e func (r *Router) releasePreparation(p *preparationState, state, code string, publish, retryHandoff bool) { r.mu.Lock() - if r.closed { + if r.closed || r.suspension != nil { r.mu.Unlock() return } @@ -252,7 +252,7 @@ func (r *Router) publishPreparation(p *preparationState, status proto.Preparatio return } } - if r.closed { + if r.closed || r.suspension != nil { r.mu.Unlock() return } diff --git a/apps/parsar-daemon/internal/dispatch/router.go b/apps/parsar-daemon/internal/dispatch/router.go index 468a4d45e..61c97f360 100644 --- a/apps/parsar-daemon/internal/dispatch/router.go +++ b/apps/parsar-daemon/internal/dispatch/router.go @@ -34,6 +34,8 @@ type Router struct { sender Sender log *slog.Logger + admission sync.RWMutex + suspension *proto.EnvironmentSuspendPayload mu sync.Mutex sessions map[string]*sessionState // RunID → state idle map[string]map[*sessionState]struct{} @@ -41,8 +43,8 @@ type Router struct { askIndex map[string]string // askID → RunID applied map[string]appliedInteractionDecision shutdownAttempt *shutdownAttempt - shutdownCh chan struct{} // closed by Shutdown - shutdownWG sync.WaitGroup // waits for all pump goroutines + shutdownCh chan struct{} // closed by Shutdown + shutdownWG dispatchWork // waits for all pump goroutines idleTimeout time.Duration closed bool preparations map[string]*preparationState @@ -147,11 +149,17 @@ func New(cfg Config) (*Router, error) { // Adopts env.Trace into ctx so every downstream log under it inherits // the same trace_id, making a single grep cover both sides. func (r *Router) Handle(ctx context.Context, env proto.Envelope) error { + r.admission.RLock() + defer r.admission.RUnlock() r.mu.Lock() if r.closed { r.mu.Unlock() return ErrRouterClosed } + if r.suspension != nil && env.Type != proto.TypeDeviceShutdown { + r.mu.Unlock() + return ErrRouterQuiesced + } r.mu.Unlock() ctx = adoptEnvelopeTrace(ctx, env) @@ -275,10 +283,12 @@ func (r *Router) scheduleIdleLocked(state *sessionState) { func (r *Router) expireIdle(state *sessionState, lease uint64) { r.mu.Lock() states := r.idle[state.stateKey] - if _, ok := states[state]; !ok || state.idleLease != lease { + if _, ok := states[state]; !ok || state.idleLease != lease || r.suspension != nil { r.mu.Unlock() return } + r.shutdownWG.Add(1) + defer r.shutdownWG.Done() delete(states, state) if len(states) == 0 { delete(r.idle, state.stateKey) diff --git a/apps/parsar-daemon/internal/dispatch/suspend.go b/apps/parsar-daemon/internal/dispatch/suspend.go new file mode 100644 index 000000000..924fe237e --- /dev/null +++ b/apps/parsar-daemon/internal/dispatch/suspend.go @@ -0,0 +1,98 @@ +package dispatch + +import ( + "context" + "errors" + "strings" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +var ErrRouterQuiesced = errors.New("dispatch: router quiesced") +var ErrRouterBusy = errors.New("dispatch: router has unsettled work") + +// Quiesce serializes against admission, then drains every admitted output and +// receipt before acknowledging suspension. Busy rejection leaves admission open; +// a drain timeout keeps it closed until the caller shuts the connection down. +func (r *Router) Quiesce(ctx context.Context, request proto.EnvironmentSuspendPayload) error { + if strings.TrimSpace(request.EnvironmentID) == "" || strings.TrimSpace(request.SuspendID) == "" || len(request.SuspendID) > 128 { + return errors.New("dispatch: invalid suspension identity") + } + if !r.admission.TryLock() { + return ErrRouterBusy + } + defer r.admission.Unlock() + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return ErrRouterClosed + } + if r.suspension != nil { + r.mu.Unlock() + return ErrRouterQuiesced + } + if len(r.sessions) != 0 || len(r.workspaceReads) != 0 || r.workspaceWrite != nil || r.workspaceExport != nil || len(r.permIndex) != 0 || len(r.askIndex) != 0 { + r.mu.Unlock() + return ErrRouterBusy + } + for _, p := range r.preparations { + if p.owns || p.busy { + r.mu.Unlock() + return ErrRouterBusy + } + } + for _, states := range r.idle { + for state := range states { + if state.environmentID != request.EnvironmentID || state.steerBusy || len(state.pendingIDs) != 0 || len(state.pendingAsks) != 0 { + r.mu.Unlock() + return ErrRouterBusy + } + } + } + r.suspension = &request + for _, p := range r.preparations { + if p.timer != nil { + p.timer.Stop() + } + } + for _, states := range r.idle { + for state := range states { + state.idleLease++ + if state.idleTimer != nil { + state.idleTimer.Stop() + } + } + } + r.mu.Unlock() + err := r.shutdownWG.waitContext(ctx) + r.mu.Lock() + if r.closed { + err = ErrRouterClosed + } + + r.mu.Unlock() + return err +} + +// Resume opens admission only after the caller authenticated a new connection +// and Core confirmed the exact suspension identity on that connection. +func (r *Router) Resume(request proto.EnvironmentSuspendPayload, sender Sender) error { + r.admission.Lock() + defer r.admission.Unlock() + r.mu.Lock() + defer r.mu.Unlock() + if r.closed { + return ErrRouterClosed + } + if sender == nil || r.suspension == nil || !r.suspension.SameSuspension(request) { + return errors.New("dispatch: suspension identity mismatch") + } + r.sender = sender + r.suspension = nil + for _, states := range r.idle { + for state := range states { + r.scheduleIdleLocked(state) + } + } + return nil +} diff --git a/apps/parsar-daemon/internal/dispatch/suspend_test.go b/apps/parsar-daemon/internal/dispatch/suspend_test.go new file mode 100644 index 000000000..e03347a74 --- /dev/null +++ b/apps/parsar-daemon/internal/dispatch/suspend_test.go @@ -0,0 +1,174 @@ +package dispatch + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +type suspendSender func(context.Context, proto.Envelope) error + +func (s suspendSender) Send(ctx context.Context, env proto.Envelope) error { return s(ctx, env) } + +type suspendedSession struct{ cancelled atomic.Int32 } + +func (s *suspendedSession) Cancel(context.Context) error { s.cancelled.Add(1); return nil } + +func suspensionRouter(t *testing.T, sender Sender) *Router { + t.Helper() + r, err := New(Config{Registry: agent.NewRegistry(), Sender: sender, IdleTimeout: time.Hour}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := r.Shutdown(context.Background()); err != nil { + t.Error(err) + } + }) + return r +} + +func TestQuiesceRejectsEveryUnsettledResource(t *testing.T) { + cases := map[string]func(*Router){ + "active": func(r *Router) { r.sessions["run"] = &sessionState{ctxCancel: func() {}} }, + "preparing": func(r *Router) { r.preparations["p"] = &preparationState{owns: true} }, + "receipt": func(r *Router) { r.preparations["p"] = &preparationState{busy: true} }, + "read": func(r *Router) { r.workspaceReads = map[string]struct{}{"read": {}} }, + "write": func(r *Router) { r.workspaceWrite = &workspaceUpload{} }, + "export": func(r *Router) { r.workspaceExport = &workspaceExport{} }, + "permission": func(r *Router) { r.permIndex["permission"] = "run" }, + "choice": func(r *Router) { r.askIndex["choice"] = "run" }, + } + for name, setup := range cases { + t.Run(name, func(t *testing.T) { + r := suspensionRouter(t, suspendSender(func(context.Context, proto.Envelope) error { return nil })) + setup(r) + err := r.Quiesce(context.Background(), proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"}) + if !errors.Is(err, ErrRouterBusy) { + t.Fatalf("quiesce=%v", err) + } + // These synthetic resources own no goroutine; remove them before cleanup. + r.sessions = map[string]*sessionState{} + r.preparations = map[string]*preparationState{} + r.workspaceReads = nil + r.workspaceWrite = nil + r.workspaceExport = nil + }) + } +} + +func TestQuiesceDrainsPendingReceiptAndFencesConcurrentAdmission(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + r := suspensionRouter(t, suspendSender(func(context.Context, proto.Envelope) error { close(entered); <-release; return nil })) + // Rejection receipts run independently of the preparation resource map. + _ = r.Handle(context.Background(), proto.Envelope{Type: proto.TypeExecutionPrepare, ID: "invalid"}) + <-entered + request := proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"} + quiet := make(chan error, 1) + go func() { quiet <- r.Quiesce(context.Background(), request) }() + deadline := time.After(time.Second) + for { + r.mu.Lock() + parked := r.suspension != nil + r.mu.Unlock() + if parked { + break + } + select { + case <-deadline: + t.Fatal("quiesce did not fence admission") + default: + time.Sleep(time.Millisecond) + } + } + admitted := make(chan error, 1) + go func() { + admitted <- r.Handle(context.Background(), proto.Envelope{Type: proto.TypePromptRequest, ID: "late"}) + }() + select { + case err := <-quiet: + t.Fatalf("acknowledged before receipt settled: %v", err) + case <-time.After(20 * time.Millisecond): + } + close(release) + if err := <-quiet; err != nil { + t.Fatal(err) + } + if err := <-admitted; !errors.Is(err, ErrRouterQuiesced) { + t.Fatalf("new admission = %v", err) + } +} + +func TestQuiescePreservesIdleOwnerAgainstExpiredTimerAndRequiresExactResume(t *testing.T) { + sender := suspendSender(func(context.Context, proto.Envelope) error { return nil }) + r := suspensionRouter(t, sender) + session := &suspendedSession{} + state := &sessionState{runID: "run", environmentID: "env", stateKey: "state", session: session, ctxCancel: func() {}, retain: true} + r.mu.Lock() + r.idle["state"] = map[*sessionState]struct{}{state: {}} + r.scheduleIdleLocked(state) + oldLease := state.idleLease + r.mu.Unlock() + request := proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"} + if err := r.Quiesce(context.Background(), request); err != nil { + t.Fatal(err) + } + r.expireIdle(state, oldLease) + if session.cancelled.Load() != 0 { + t.Fatal("pre-snapshot timer killed retained owner") + } + wrong := request + wrong.SuspendID = "obsolete" + if err := r.Resume(wrong, sender); err == nil { + t.Fatal("stale operation reopened admission") + } + if err := r.Resume(request, sender); err != nil { + t.Fatal(err) + } + r.mu.Lock() + newLease := state.idleLease + r.mu.Unlock() + r.expireIdle(state, newLease) + if session.cancelled.Load() != 1 { + t.Fatal("normal idle expiration was not restored") + } +} + +func TestShutdownDestroysQuiescedOwnerAndCannotResume(t *testing.T) { + sender := suspendSender(func(context.Context, proto.Envelope) error { return nil }) + r := suspensionRouter(t, sender) + request := proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"} + if err := r.Quiesce(context.Background(), request); err != nil { + t.Fatal(err) + } + if err := r.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } + if !errors.Is(r.Resume(request, sender), ErrRouterClosed) { + t.Fatal("closed Router resurrected") + } +} + +func TestQuiesceDrainDeadlineCannotReopenAdmission(t *testing.T) { + r := suspensionRouter(t, suspendSender(func(context.Context, proto.Envelope) error { return nil })) + r.shutdownWG.Add(1) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := r.Quiesce(ctx, proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"}); !errors.Is(err, context.Canceled) { + t.Fatalf("quiesce=%v", err) + } + if err := r.Handle(context.Background(), proto.Envelope{Type: proto.TypePromptRequest, ID: "late"}); !errors.Is(err, ErrRouterQuiesced) { + t.Fatalf("deadline reopened admission: %v", err) + } + r.shutdownWG.Done() + // Starting cleanup after the drain observer timed out must not reuse a + // sync.WaitGroup while an abandoned waiter is still returning from Wait. + if err := r.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } +} diff --git a/apps/parsar-daemon/internal/dispatch/work.go b/apps/parsar-daemon/internal/dispatch/work.go new file mode 100644 index 000000000..3cba37c47 --- /dev/null +++ b/apps/parsar-daemon/internal/dispatch/work.go @@ -0,0 +1,49 @@ +package dispatch + +import ( + "context" + "sync" +) + +// dispatchWork tracks admitted goroutines. Unlike sync.WaitGroup, its zero +// notification can be observed with a deadline without leaving a stale waiter +// behind when shutdown starts another cleanup attempt. Admission owns the rule +// that no new work may begin after a successful quiesce observation. +type dispatchWork struct { + mu sync.Mutex + count int + settled chan struct{} +} + +func (w *dispatchWork) Add(delta int) { + w.mu.Lock() + defer w.mu.Unlock() + next := w.count + delta + if next < 0 { + panic("dispatch: negative work count") + } + if w.count == 0 && next > 0 { + w.settled = make(chan struct{}) + } + w.count = next + if next == 0 && w.settled != nil { + close(w.settled) + w.settled = nil + } +} +func (w *dispatchWork) Done() { w.Add(-1) } +func (w *dispatchWork) Wait() { _ = w.waitContext(context.Background()) } +func (w *dispatchWork) waitContext(ctx context.Context) error { + w.mu.Lock() + settled := w.settled + w.mu.Unlock() + if settled == nil { + return ctx.Err() + } + select { + case <-settled: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/apps/web/e2e/vault-credentials.spec.ts b/apps/web/e2e/vault-credentials.spec.ts index 59c557187..25d9a354d 100644 --- a/apps/web/e2e/vault-credentials.spec.ts +++ b/apps/web/e2e/vault-credentials.spec.ts @@ -142,6 +142,8 @@ test("shows a fixed 503 storage error, clears the token, and does not retry the const vaultCard = page.locator(".vault-card").filter({ hasText: "Unavailable storage" }); await vaultCard.getByRole("button", { name: "Credential", exact: true }).click(); const credentialDialog = page.getByRole("dialog", { name: "Add static bearer Credential" }); + // Let the modal finish its initial focus before filling another field. + await expect(credentialDialog.getByLabel("Name")).toBeFocused(); await credentialDialog.getByLabel("Name").fill("Unavailable MCP"); await credentialDialog.getByLabel("Exact MCP server URL").fill("https://mcp.example/unavailable"); await fillWriteOnlyBearer(credentialDialog); diff --git a/contracts/agents-api/harness-selection.md b/contracts/agents-api/harness-selection.md index 843c945a9..fa7287ea9 100644 --- a/contracts/agents-api/harness-selection.md +++ b/contracts/agents-api/harness-selection.md @@ -37,46 +37,41 @@ resources and native histories. Agent edits do not change accepted Sessions. `AGENTS_API_ENGINE` selects the default engine. `AGENTS_API_HARNESSES` explicitly adds comma-separated deployment-supported engines, for example `codex,claude_sdk,mcode`, without requiring a managed Provider. The default engine -and configured managed engine profiles remain enabled; unknown names fail startup. +remains enabled; unknown names fail startup. This setting does not install a harness or qualify a native deployment. -Existing `default_provider` deployments keep their default behavior. A deployment enabling multiple hosted harnesses adds `engine_providers` -to `AGENTS_API_MANAGED_RUNTIMES_FILE`: +Hosted placement uses one provider per deployment. Set `provider` to `docker` or +`microsandbox` in `AGENTS_API_MANAGED_RUNTIMES_FILE`, with a stable +`installation_id`, a `maintenance` boolean and one matching backend object. +For example: ```json { "core_url": "http://core:8091/api/v1", - "default_provider": "11111111-1111-4111-8111-111111111111", - "engine_providers": { - "codex": "11111111-1111-4111-8111-111111111111", - "claude_sdk": "22222222-2222-4222-8222-222222222222" - }, + "provider": "docker", + "installation_id": "11111111-1111-4111-8111-111111111111", + "maintenance": false, "docker": { - "11111111-1111-4111-8111-111111111111": { - "host": "unix:///var/run/docker.sock", - "image": "sha256:", - "network": "bridge", - "seccomp_file": "/private/seccomp.json" - }, - "22222222-2222-4222-8222-222222222222": { - "host": "unix:///var/run/docker.sock", - "image": "sha256:", - "network": "bridge", - "seccomp_file": "/private/seccomp.json", - "nested_sandbox": true - } + "host": "unix:///var/run/docker.sock", + "image": "sha256:", + "network": "bridge", + "seccomp_file": "/private/seccomp.json", + "nested_sandbox": true } } ``` -Each reference must name an existing qualified provider entry. The default engine -inherits `default_provider` when no explicit mapping exists. Other engines have no -fallback. Admission and initial allocation use the same engine mapping. Once an -allocation exists, its persisted provider identity remains authoritative across -restarts and configuration changes. Keep retained provider entries for cleanup; -changing a provider target requires a new key. No image is qualified merely by -being named in this map. Existing provider security and native capability checks -still apply. This change adds no provider or native harness implementation. +There is no engine-to-provider routing or mixed-provider configuration. Enabling +another harness does not choose another image or backend; the selected Runtime +image must contain and qualify each enabled harness. Capability checks still +apply. Legacy `default_provider`, `engine_providers` and provider maps are rejected. + +Before changing provider or backend namespace, restart the original configuration +with `maintenance: true`, explicitly settle or delete its hosted resources and +confirm cleanup is complete. Restart the new provider configuration in maintenance +to validate the switch, then disable maintenance without changing its identity. +Switching never migrates Sessions or deletes resources automatically. See the +[deployment procedure](../../services/agents-api/deploy/microsandbox/README.md#change-the-deployment-provider). For multiple engines, use an exclusive `by_harness` object in the private `AGENTS_API_EXECUTION_OPTIONS_FILE`, with one existing adapter-options object per diff --git a/contracts/agents-api/message-input.md b/contracts/agents-api/message-input.md index 4e5b4ae79..4c5a9bc1f 100644 --- a/contracts/agents-api/message-input.md +++ b/contracts/agents-api/message-input.md @@ -36,7 +36,7 @@ image `detail` or file-ID extension. ## Runtime boundary -Private wire 0.4.0 uses `MessageInput` for initial requests, prepared start and +Private wire 0.5.0 uses `MessageInput` for initial requests, prepared start and active steering. Each message contains ordered `InputContent` parts, also reused by function-result content. Core does not download, transcode or repair media. The separate Claude bridge reports protocol 2; daemon readiness rejects protocol 1. diff --git a/internal/agentdaemon/gateway/session.go b/internal/agentdaemon/gateway/session.go index 47fd77dc5..c0cd7572f 100644 --- a/internal/agentdaemon/gateway/session.go +++ b/internal/agentdaemon/gateway/session.go @@ -108,6 +108,8 @@ type Session struct { preparations map[string]*preparationSubscription workspaceWriteMu sync.Mutex workspaceWrites map[string]chan proto.Envelope + suspendMu sync.Mutex + suspendReplies map[string]chan proto.Envelope workspaceReadMu sync.Mutex workspaceReads map[string]chan proto.Envelope workspaceExportMu sync.Mutex @@ -262,6 +264,7 @@ func (s *Session) Close(reason string) { s.reg.Deregister(s) s.closePreparations() s.closeWorkspaceReads() + s.closeSuspendReplies() s.closeWorkspaceWrites() s.closeWorkspaceExports() s.markOfflineOnClose() @@ -589,6 +592,8 @@ func (s *Session) dispatch(env proto.Envelope) { case proto.TypeWorkspaceReadResult: s.dispatchWorkspaceRead(env) return + case proto.TypeEnvironmentQuiesced, proto.TypeEnvironmentResumed: + s.dispatchSuspendReply(env) case proto.TypePreparationStatus: s.dispatchPreparation(env) return diff --git a/internal/agentdaemon/gateway/suspension.go b/internal/agentdaemon/gateway/suspension.go new file mode 100644 index 000000000..ab507cd00 --- /dev/null +++ b/internal/agentdaemon/gateway/suspension.go @@ -0,0 +1,90 @@ +package gateway + +import ( + "context" + "errors" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/google/uuid" +) + +// SuspendControl correlates one lifecycle operation on this authenticated socket. +// A closed socket is not an acknowledgement; callers retain their durable intent. +func (s *Session) SuspendControl(ctx context.Context, kind string, request proto.EnvironmentSuspendPayload) (proto.EnvironmentSuspendResultPayload, error) { + var result proto.EnvironmentSuspendResultPayload + expected := proto.TypeEnvironmentQuiesced + if kind == proto.TypeEnvironmentResume { + expected = proto.TypeEnvironmentResumed + } else if kind != proto.TypeEnvironmentQuiesce { + return result, errors.New("invalid suspension operation") + } + if request.EnvironmentID == "" || request.SuspendID == "" { + return result, errors.New("invalid suspension identity") + } + id := uuid.NewString() + envelope, err := proto.NewEnvelope(kind, id, request) + if err != nil { + return result, err + } + s.suspendMu.Lock() + if s.IsClosed() { + s.suspendMu.Unlock() + return result, ErrSessionClosed + } + if len(s.suspendReplies) >= 2 { + s.suspendMu.Unlock() + return result, errors.New("suspension request capacity") + } + if s.suspendReplies == nil { + s.suspendReplies = make(map[string]chan proto.Envelope) + } + replies := make(chan proto.Envelope, 1) + s.suspendReplies[id] = replies + s.suspendMu.Unlock() + defer func() { s.suspendMu.Lock(); delete(s.suspendReplies, id); s.suspendMu.Unlock() }() + if err := s.Send(ctx, envelope); err != nil { + return result, err + } + // The daemon closes after quiesced. Read the buffered result before treating + // connection closure as unknown, including when both channels become ready. + decode := func(reply proto.Envelope, ok bool) (proto.EnvironmentSuspendResultPayload, error) { + if !ok { + return result, ErrSessionClosed + } + if reply.Type != expected || reply.DecodePayload(&result) != nil || result.EnvironmentID != request.EnvironmentID || result.SuspendID != request.SuspendID || (result.Accepted && result.ErrorCode != "") { + return proto.EnvironmentSuspendResultPayload{}, errors.New("invalid suspension acknowledgement") + } + return result, nil + } + select { + case reply, ok := <-replies: + return decode(reply, ok) + case <-ctx.Done(): + return result, ctx.Err() + case <-s.closed: + select { + case reply, ok := <-replies: + return decode(reply, ok) + default: + return result, ErrSessionClosed + } + } +} + +func (s *Session) dispatchSuspendReply(env proto.Envelope) { + s.suspendMu.Lock() + defer s.suspendMu.Unlock() + if replies := s.suspendReplies[env.ID]; replies != nil { + select { + case replies <- env: + default: + } + } +} +func (s *Session) closeSuspendReplies() { + s.suspendMu.Lock() + defer s.suspendMu.Unlock() + for id, replies := range s.suspendReplies { + close(replies) + delete(s.suspendReplies, id) + } +} diff --git a/internal/agentdaemon/gateway/suspension_test.go b/internal/agentdaemon/gateway/suspension_test.go new file mode 100644 index 000000000..c9e8b0f53 --- /dev/null +++ b/internal/agentdaemon/gateway/suspension_test.go @@ -0,0 +1,75 @@ +package gateway + +import ( + "context" + "errors" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "testing" +) + +func TestSuspendAcknowledgementSurvivesImmediateConnectionClose(t *testing.T) { + for range 100 { + s := NewSession(newFakeConn(), "device", "tenant", "test", nil, nil) + done := make(chan error, 1) + go func() { + result, err := s.SuspendControl(t.Context(), proto.TypeEnvironmentQuiesce, proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"}) + if err == nil && !result.Accepted { + err = errors.New("ack lost") + } + done <- err + }() + request := <-s.sendCh + reply, _ := proto.NewEnvelope(proto.TypeEnvironmentQuiesced, request.ID, proto.EnvironmentSuspendResultPayload{EnvironmentID: "env", SuspendID: "attempt", Accepted: true}) + s.dispatch(reply) + s.Close("parked") + if err := <-done; err != nil { + t.Fatal(err) + } + } +} +func TestSuspendControlRejectsForeignIdentityAndDoesNotReplay(t *testing.T) { + for _, result := range []proto.EnvironmentSuspendResultPayload{ + {EnvironmentID: "other", SuspendID: "attempt", Accepted: true}, + {EnvironmentID: "env", SuspendID: "old", Accepted: true}, + {EnvironmentID: "env", SuspendID: "attempt", Accepted: true, ErrorCode: "resource_busy"}, + } { + s := NewSession(newFakeConn(), "device", "tenant", "test", nil, nil) + done := make(chan error, 1) + go func() { + _, err := s.SuspendControl(t.Context(), proto.TypeEnvironmentResume, proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"}) + done <- err + }() + request := <-s.sendCh + reply, _ := proto.NewEnvelope(proto.TypeEnvironmentResumed, request.ID, result) + s.dispatch(reply) + if err := <-done; err == nil { + t.Fatal("foreign receipt accepted") + } + select { + case <-s.sendCh: + t.Fatal("unexpected replay") + default: + } + s.Close("test") + } +} +func TestSuspendObserverCancellationRetainsUnknownOutcome(t *testing.T) { + s := NewSession(newFakeConn(), "device", "tenant", "test", nil, nil) + defer s.Close("test") + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { + _, err := s.SuspendControl(ctx, proto.TypeEnvironmentQuiesce, proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"}) + done <- err + }() + <-s.sendCh + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + select { + case <-s.sendCh: + t.Fatal("cancellation replayed operation") + default: + } +} diff --git a/internal/agentdaemon/proto/envelope_test.go b/internal/agentdaemon/proto/envelope_test.go index 41bdf8cf8..8aace4d66 100644 --- a/internal/agentdaemon/proto/envelope_test.go +++ b/internal/agentdaemon/proto/envelope_test.go @@ -88,8 +88,8 @@ func TestVersionCompatible(t *testing.T) { ok bool }{ {Version, true}, // exact match - {"0.4.99", true}, // patch drift OK - {"0.2.99", false}, // minor drift NOT OK + {"0.5.99", true}, // patch drift OK + {"0.4.99", false}, // minor drift NOT OK {"1.0.0", false}, // major drift NOT OK {"", false}, // missing {"garbage", false}, // unparseable diff --git a/internal/agentdaemon/proto/suspend.go b/internal/agentdaemon/proto/suspend.go new file mode 100644 index 000000000..053b6fa46 --- /dev/null +++ b/internal/agentdaemon/proto/suspend.go @@ -0,0 +1,30 @@ +package proto + +// Suspension is private hosted lifecycle traffic. Envelope.ID correlates each +// request; SuspendID fences one persisted suspension attempt across connections. +const ( + TypeEnvironmentQuiesce = "environment_quiesce" + TypeEnvironmentQuiesced = "environment_quiesced" + TypeEnvironmentResume = "environment_resume" + TypeEnvironmentResumed = "environment_resumed" +) + +type EnvironmentSuspendPayload struct { + // Rollback permits a no-op resume on the still-running source after an + // uncertain quiesce. It never authorizes cold replacement on snapshot restore. + Rollback bool `json:"rollback,omitempty"` + EnvironmentID string `json:"environment_id"` + SuspendID string `json:"suspend_id"` +} + +type EnvironmentSuspendResultPayload struct { + EnvironmentID string `json:"environment_id"` + SuspendID string `json:"suspend_id"` + Accepted bool `json:"accepted"` + ErrorCode string `json:"error_code,omitempty"` +} + +// SameSuspension compares ownership, independently of the requested recovery mode. +func (p EnvironmentSuspendPayload) SameSuspension(other EnvironmentSuspendPayload) bool { + return p.EnvironmentID == other.EnvironmentID && p.SuspendID == other.SuspendID +} diff --git a/internal/agentdaemon/proto/version.go b/internal/agentdaemon/proto/version.go index 641f511bf..a030854b8 100644 --- a/internal/agentdaemon/proto/version.go +++ b/internal/agentdaemon/proto/version.go @@ -5,7 +5,7 @@ package proto // upgrade query (`version=`) and in the bootstrap HTTP // response; mismatches fail closed at WS upgrade. const ( - Version = "0.4.0" + Version = "0.5.0" ) // VersionCompatible returns true when clientVersion's "X.Y" prefix diff --git a/services/agents-api/HOSTED-RELEASE.md b/services/agents-api/HOSTED-RELEASE.md index 56a35787b..7127aa514 100644 --- a/services/agents-api/HOSTED-RELEASE.md +++ b/services/agents-api/HOSTED-RELEASE.md @@ -54,20 +54,20 @@ export AGENTS_API_MANAGED_RUNTIMES_FILE="$PARSAR_HOME/managed-runtimes.json" export AGENTS_API_EXECUTION_OPTIONS_FILE="$PARSAR_HOME/execution-options.json" ``` -Create the private managed configuration with a fresh stable provider UUID. Replace -both occurrences of ``, the gateway and the absolute package path: +Create the private managed configuration with a fresh stable installation UUID. Replace +``, the gateway and the absolute package path: ```json { "core_url": "http://:8091/api/v1", - "default_provider": "", + "provider": "docker", + "installation_id": "", + "maintenance": false, "docker": { - "": { - "host": "unix:///var/run/docker.sock", - "image": "@RUNTIME_IMAGE@", - "network": "bridge", - "seccomp_file": "/runtime/seccomp.json" - } + "host": "unix:///var/run/docker.sock", + "image": "@RUNTIME_IMAGE@", + "network": "bridge", + "seccomp_file": "/runtime/seccomp.json" } } ``` @@ -163,9 +163,17 @@ request or delete native history to make a retry succeed. When finished, `client.beta.agents.sessions.delete(session.id)` requests owned Runtime cleanup. Core must remain running with the original Provider configured until its labelled container and volumes are gone. Public deletion acknowledgment -is not physical cleanup confirmation. Do not use broad Docker pruning. An empty -`default_provider` disables new hosted admission while keeping existing Session -controls and cleanup available. Back up the independent PostgreSQL database +is not physical cleanup confirmation. Do not use broad Docker pruning. + +To change providers, restart the old configuration with `maintenance: true`, +explicitly handle or delete old hosted resources and verify cleanup. Restart the +new provider configuration in maintenance to validate the empty deployment, then +restart the same identity with `maintenance: false`. Maintenance blocks new compute +without deleting retained resources. One deployment runs either Docker or +[microsandbox](deploy/microsandbox/README.md), with no mixed configuration, +engine-to-provider routing or automatic Session migration. + +Back up the independent PostgreSQL database (including large objects) and retained Runtime state together under an operator recovery plan; the archive itself contains no deployment data. diff --git a/services/agents-api/cmd/server/harnesses.go b/services/agents-api/cmd/server/harnesses.go index 86e7518f6..a3dd76970 100644 --- a/services/agents-api/cmd/server/harnesses.go +++ b/services/agents-api/cmd/server/harnesses.go @@ -7,18 +7,12 @@ import ( "strings" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/engine" - "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" ) // Engine selection is independent of compute ownership. Operators may enable // user-managed Runtime profiles without configuring a managed Provider. -func enabledHarnesses(defaultEngine string, managed *execution.RuntimeProviders) ([]string, error) { +func enabledHarnesses(defaultEngine string) ([]string, error) { kinds := []string{defaultEngine} - if managed != nil { - for kind := range managed.EngineProviders { - kinds = append(kinds, kind) - } - } if value := os.Getenv("AGENTS_API_HARNESSES"); value != "" { kinds = append(kinds, strings.Split(value, ",")...) } diff --git a/services/agents-api/cmd/server/harnesses_test.go b/services/agents-api/cmd/server/harnesses_test.go index 1e6296730..6acc1f3a9 100644 --- a/services/agents-api/cmd/server/harnesses_test.go +++ b/services/agents-api/cmd/server/harnesses_test.go @@ -4,12 +4,12 @@ import "testing" func TestUserManagedHarnessSelectionNeedsNoProvider(t *testing.T) { t.Setenv("AGENTS_API_HARNESSES", "codex,claude_sdk,mcode") - kinds, err := enabledHarnesses("codex", nil) + kinds, err := enabledHarnesses("codex") if err != nil || len(kinds) != 3 { t.Fatal(kinds, err) } t.Setenv("AGENTS_API_HARNESSES", "unqualified") - if _, err = enabledHarnesses("codex", nil); err == nil { + if _, err = enabledHarnesses("codex"); err == nil { t.Fatal("unqualified harness enabled") } } diff --git a/services/agents-api/cmd/server/main.go b/services/agents-api/cmd/server/main.go index 757c2be80..607b5c01f 100644 --- a/services/agents-api/cmd/server/main.go +++ b/services/agents-api/cmd/server/main.go @@ -119,12 +119,12 @@ func run() error { } }() options = append(options, api.WithExecution(worker), api.WithEnvironmentDirectoryReader(worker), api.WithEnvironmentFileWriter(worker)) - kinds, err := enabledHarnesses(engine, managed) + kinds, err := enabledHarnesses(engine) if err != nil { return err } options = append(options, api.WithHarnesses(kinds)) - if managed != nil && (managed.DefaultProvider != "" || len(managed.EngineProviders) > 0) { + if managed != nil { options = append(options, api.WithHostedEnvironments()) } } diff --git a/services/agents-api/cmd/server/managed_microsandbox.go b/services/agents-api/cmd/server/managed_microsandbox.go new file mode 100644 index 000000000..a306a8a9f --- /dev/null +++ b/services/agents-api/cmd/server/managed_microsandbox.go @@ -0,0 +1,70 @@ +package main + +import ( + "errors" + "path/filepath" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + sandboxmicro "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/microsandbox" +) + +// Single-host providers pin the helper, runtime, firmware, image and resource +// limits explicitly. The helper owns local paths; no ambient backend is selected. +type managedMicrosandboxConfig struct { + HelperPath string `json:"helper_path"` + RuntimeHome string `json:"runtime_home"` + RuntimePath string `json:"runtime_path"` + FirmwarePath string `json:"firmware_path"` + RuntimeSHA256 string `json:"runtime_sha256"` + FirmwareSHA256 string `json:"firmware_sha256"` + Image string `json:"image"` + MemoryMiB uint32 `json:"memory_mib"` + CPUs uint8 `json:"cpus"` + RootDiskMiB uint32 `json:"root_disk_mib"` + Network managedMicrosandboxNetwork `json:"network"` + IdleSeconds int64 `json:"idle_seconds"` + RetentionSeconds int64 `json:"retention_seconds"` + MaxActive int `json:"max_active"` + MaxRetained int `json:"max_retained"` +} + +type managedMicrosandboxNetwork struct { + DefaultEgress string `json:"default_egress"` + DefaultIngress string `json:"default_ingress"` + Rules []managedMicrosandboxRule `json:"rules"` +} + +type managedMicrosandboxRule struct { + Action string `json:"action"` + Direction string `json:"direction"` + Destination string `json:"destination"` + Protocol string `json:"protocol"` + Port string `json:"port"` +} + +func configureManagedMicrosandbox(entry managedMicrosandboxConfig, result *execution.RuntimeProvider) error { + const maxSeconds = int64((1<<63 - 1) / time.Second) + if entry.IdleSeconds <= 0 || entry.IdleSeconds > maxSeconds || entry.RetentionSeconds <= 0 || entry.RetentionSeconds > maxSeconds || entry.MaxActive <= 0 || entry.MaxRetained < entry.MaxActive { + return errors.New("managed microsandbox requires positive bounded idle_seconds, retention_seconds and max_active, with max_retained >= max_active") + } + if !filepath.IsAbs(entry.RuntimeHome) || filepath.Clean(entry.RuntimeHome) != entry.RuntimeHome { + return errors.New("managed microsandbox runtime_home must be a canonical absolute path") + } + network := sandboxmicro.NetworkPolicy{DefaultEgress: entry.Network.DefaultEgress, DefaultIngress: entry.Network.DefaultIngress} + for _, rule := range entry.Network.Rules { + network.Rules = append(network.Rules, sandboxmicro.NetworkRule{Action: rule.Action, Direction: rule.Direction, Destination: rule.Destination, Protocol: rule.Protocol, Port: rule.Port}) + } + provider, err := sandboxmicro.New(sandboxmicro.Config{ + InstallationID: result.InstallationID, HelperPath: entry.HelperPath, RuntimeHome: entry.RuntimeHome, RuntimePath: entry.RuntimePath, FirmwarePath: entry.FirmwarePath, + RuntimeSHA256: entry.RuntimeSHA256, FirmwareSHA256: entry.FirmwareSHA256, Image: entry.Image, + MemoryMiB: entry.MemoryMiB, CPUs: entry.CPUs, RootDiskMiB: entry.RootDiskMiB, Network: network, + }) + if err != nil { + return errors.New("invalid managed microsandbox provider configuration") + } + result.Provider = provider + result.BackendFingerprint = managedBackendFingerprint("microsandbox", entry.RuntimeHome) + result.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(entry.IdleSeconds) * time.Second, Retention: time.Duration(entry.RetentionSeconds) * time.Second, MaxActive: entry.MaxActive, MaxRetained: entry.MaxRetained} + return nil +} diff --git a/services/agents-api/cmd/server/managed_microsandbox_test.go b/services/agents-api/cmd/server/managed_microsandbox_test.go new file mode 100644 index 000000000..ca0ba2f2e --- /dev/null +++ b/services/agents-api/cmd/server/managed_microsandbox_test.go @@ -0,0 +1,135 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/google/uuid" +) + +func managedMicrosandboxFixture(t *testing.T) (managedRuntimeConfig, string, func(managedRuntimeConfig)) { + t.Helper() + key := uuid.NewString() + config := managedRuntimeConfig{ + CoreURL: "http://core.example/api/v1", Provider: "microsandbox", InstallationID: key, + Microsandbox: &managedMicrosandboxConfig{ + HelperPath: "/opt/parsar/microsandbox-provider", RuntimeHome: "/var/lib/parsar/microsandbox", RuntimePath: "/opt/parsar/msb", FirmwarePath: "/opt/parsar/libkrunfw.so", + RuntimeSHA256: strings.Repeat("a", 64), FirmwareSHA256: strings.Repeat("b", 64), Image: "registry.example/parsar-runtime@sha256:" + strings.Repeat("c", 64), + MemoryMiB: 1024, CPUs: 1, RootDiskMiB: 4096, + Network: managedMicrosandboxNetwork{DefaultEgress: "deny", DefaultIngress: "deny", Rules: []managedMicrosandboxRule{{Action: "allow", Direction: "egress", Destination: "core.example", Protocol: "tcp", Port: "443"}}}, + IdleSeconds: 300, RetentionSeconds: 86400, MaxActive: 4, MaxRetained: 8, + }, + } + file := filepath.Join(t.TempDir(), "providers.json") + t.Setenv("AGENTS_API_MANAGED_RUNTIMES_FILE", file) + t.Setenv("AGENTS_API_DAEMON_WS_URL", "ws://core.example/api/v1/agent-daemon/ws") + t.Setenv("AGENTS_API_ENGINE", "codex") + write := func(c managedRuntimeConfig) { + raw, err := json.Marshal(c) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, raw, 0600); err != nil { + t.Fatal(err) + } + } + write(config) + return config, key, write +} + +func TestManagedMicrosandboxConfigurationAndPolicy(t *testing.T) { + _, key, _ := managedMicrosandboxFixture(t) + result, close, err := managedRuntimes() + if err != nil { + t.Fatal(err) + } + defer close() + if result.InstallationID != key || result.Provider == nil || result.Maintenance { + t.Fatal("microsandbox identity lost") + } + if _, ok := result.Provider.(sandbox.CheckpointProvider); !ok { + t.Fatal("microsandbox checkpoint capability missing") + } + policy := result.Suspension + if policy == nil || policy.IdleTimeout != 5*time.Minute || policy.Retention != 24*time.Hour || policy.MaxActive != 4 || policy.MaxRetained != 8 { + t.Fatalf("incorrect suspension policy: %+v", policy) + } +} + +func TestManagedMicrosandboxRejectsUnboundedOrImplicitConfiguration(t *testing.T) { + config, _, write := managedMicrosandboxFixture(t) + cases := map[string]func(*managedMicrosandboxConfig){ + "retained_missing": func(c *managedMicrosandboxConfig) { c.MaxRetained = 0 }, + "retained_below_active": func(c *managedMicrosandboxConfig) { c.MaxRetained = c.MaxActive - 1 }, + "retained_negative": func(c *managedMicrosandboxConfig) { c.MaxRetained = -1 }, + "idle_missing": func(c *managedMicrosandboxConfig) { c.IdleSeconds = 0 }, + "retention_missing": func(c *managedMicrosandboxConfig) { c.RetentionSeconds = 0 }, + "capacity_missing": func(c *managedMicrosandboxConfig) { c.MaxActive = 0 }, + "negative_capacity": func(c *managedMicrosandboxConfig) { c.MaxActive = -1 }, + "negative_idle": func(c *managedMicrosandboxConfig) { c.IdleSeconds = -1 }, + "idle_overflow": func(c *managedMicrosandboxConfig) { c.IdleSeconds = 1<<63 - 1 }, + "retention_overflow": func(c *managedMicrosandboxConfig) { c.RetentionSeconds = 1<<63 - 1 }, + "memory_missing": func(c *managedMicrosandboxConfig) { c.MemoryMiB = 0 }, + "cpus_missing": func(c *managedMicrosandboxConfig) { c.CPUs = 0 }, + "disk_missing": func(c *managedMicrosandboxConfig) { c.RootDiskMiB = 0 }, + "relative_helper": func(c *managedMicrosandboxConfig) { c.HelperPath = "./helper" }, + "unclean_home": func(c *managedMicrosandboxConfig) { c.RuntimeHome = "/private/state/../msb" }, + "relative_home": func(c *managedMicrosandboxConfig) { c.RuntimeHome = ".cache" }, + "runtime_missing": func(c *managedMicrosandboxConfig) { c.RuntimePath = "" }, + "firmware_missing": func(c *managedMicrosandboxConfig) { c.FirmwarePath = "" }, + "runtime_hash_missing": func(c *managedMicrosandboxConfig) { c.RuntimeSHA256 = "" }, + "firmware_hash_missing": func(c *managedMicrosandboxConfig) { c.FirmwareSHA256 = "" }, + "image_unpinned": func(c *managedMicrosandboxConfig) { c.Image = "registry.example/parsar-runtime:latest" }, + "network_missing": func(c *managedMicrosandboxConfig) { c.Network = managedMicrosandboxNetwork{} }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + changed := config + entry := *config.Microsandbox + mutate(&entry) + changed.Microsandbox = &entry + write(changed) + if _, close, err := managedRuntimes(); err == nil { + close() + t.Fatal("invalid provider configuration accepted") + } + }) + } +} + +func TestManagedMicrosandboxFingerprintPinsOnlyBackendNamespace(t *testing.T) { + config, _, write := managedMicrosandboxFixture(t) + original, close, err := managedRuntimes() + if err != nil { + t.Fatal(err) + } + close() + config.Maintenance = true + config.Microsandbox.Image = "registry.example/parsar-runtime@sha256:" + strings.Repeat("d", 64) + config.Microsandbox.MaxActive++ + config.Microsandbox.IdleSeconds++ + write(config) + changed, close, err := managedRuntimes() + if err != nil { + t.Fatal(err) + } + close() + if !changed.Maintenance || changed.BackendFingerprint != original.BackendFingerprint { + t.Fatal("policy/image change replaced backend identity") + } + config.Microsandbox.RuntimeHome = "/var/lib/parsar/another-installation" + write(config) + changed, close, err = managedRuntimes() + if err != nil { + t.Fatal(err) + } + close() + if changed.BackendFingerprint == original.BackendFingerprint || len(changed.BackendFingerprint) != 64 { + t.Fatal("backend namespace change was not fenced") + } +} diff --git a/services/agents-api/cmd/server/managed_runtimes.go b/services/agents-api/cmd/server/managed_runtimes.go index cc69d240b..db2f8b054 100644 --- a/services/agents-api/cmd/server/managed_runtimes.go +++ b/services/agents-api/cmd/server/managed_runtimes.go @@ -2,24 +2,28 @@ package main import ( "bytes" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "io" + "net/url" "os" - "strings" + "path/filepath" - "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/engine" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" - "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" sandboxdocker "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/docker" + "github.com/google/uuid" "github.com/moby/moby/client" ) type managedRuntimeConfig struct { - CoreURL string `json:"core_url"` - EngineProviders map[string]string `json:"engine_providers"` - DefaultProvider string `json:"default_provider"` - Docker map[string]managedDockerConfig `json:"docker"` + CoreURL string `json:"core_url"` + Provider string `json:"provider"` + InstallationID string `json:"installation_id"` + Maintenance bool `json:"maintenance"` + Docker *managedDockerConfig `json:"docker,omitempty"` + Microsandbox *managedMicrosandboxConfig `json:"microsandbox,omitempty"` } type managedDockerConfig struct { @@ -31,81 +35,81 @@ type managedDockerConfig struct { NestedSandbox bool `json:"nested_sandbox"` } -// Each provider key pins an explicit backend, independently of ambient -// DOCKER_HOST. Retained entries keep their cleanup backend when the default changes. -func managedRuntimes() (*execution.RuntimeProviders, func(), error) { +// One deployment owns one explicit backend. Maintenance keeps that adapter +// available for existing resources while Core blocks new compute admission. +func managedRuntimes() (*execution.RuntimeProvider, func(), error) { + closeProvider := func() {} file := os.Getenv("AGENTS_API_MANAGED_RUNTIMES_FILE") - closeAll := func() {} if file == "" { - return nil, closeAll, nil + return nil, closeProvider, nil } if os.Getenv("AGENTS_API_DAEMON_WS_URL") == "" { - return nil, closeAll, errors.New("managed Runtime configuration requires AGENTS_API_DAEMON_WS_URL") + return nil, closeProvider, errors.New("managed Runtime configuration requires AGENTS_API_DAEMON_WS_URL") } raw, err := os.ReadFile(file) if err != nil { - return nil, closeAll, errors.New("cannot read AGENTS_API_MANAGED_RUNTIMES_FILE") + return nil, closeProvider, errors.New("cannot read AGENTS_API_MANAGED_RUNTIMES_FILE") } var config managedRuntimeConfig decoder := json.NewDecoder(bytes.NewReader(raw)) decoder.DisallowUnknownFields() - if decoder.Decode(&config) != nil || decoder.Decode(new(any)) != io.EOF || len(config.Docker) == 0 { - return nil, closeAll, errors.New("invalid managed Runtime configuration") + if decoder.Decode(&config) != nil || decoder.Decode(new(any)) != io.EOF { + return nil, closeProvider, errors.New("invalid managed Runtime configuration") } - if config.DefaultProvider != "" { - _, dockerOK := config.Docker[config.DefaultProvider] - if !dockerOK { - return nil, closeAll, errors.New("managed default provider is not configured") - } - } - if config.EngineProviders == nil { - config.EngineProviders = map[string]string{} + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil { + return nil, closeProvider, errors.New("invalid managed Runtime configuration") } - defaultEngine := os.Getenv("AGENTS_API_ENGINE") - if defaultEngine == "" { - defaultEngine = "codex" + _, hasDocker := fields["docker"] + _, hasMicrosandbox := fields["microsandbox"] + if maintenance, exists := fields["maintenance"]; exists && bytes.Equal(bytes.TrimSpace(maintenance), []byte("null")) { + return nil, closeProvider, errors.New("managed maintenance must be a boolean") } - if config.DefaultProvider != "" && config.EngineProviders[defaultEngine] == "" { - config.EngineProviders[defaultEngine] = config.DefaultProvider + id, err := uuid.Parse(config.InstallationID) + if err != nil || id == uuid.Nil || id.String() != config.InstallationID { + return nil, closeProvider, errors.New("managed Runtime requires a canonical installation_id UUID") } - for kind, key := range config.EngineProviders { - _, qualified := (engine.Catalog{}).Lookup(kind) - _, dockerOK := config.Docker[key] - if !qualified || key == "" || (!dockerOK) { - return nil, closeAll, errors.New("invalid managed engine provider mapping") + result := &execution.RuntimeProvider{CoreURL: config.CoreURL, InstallationID: config.InstallationID, Maintenance: config.Maintenance} + switch config.Provider { + case "docker": + if config.Docker == nil || !hasDocker || hasMicrosandbox { + return nil, closeProvider, errors.New("managed Docker requires only the docker configuration object") } - } - var clients []*client.Client - closeAll = func() { - for _, c := range clients { - _ = c.Close() - } - } - result := &execution.RuntimeProviders{EngineProviders: config.EngineProviders, CoreURL: config.CoreURL, DefaultProvider: config.DefaultProvider, Providers: map[string]sandbox.Provider{}} - for key, entry := range config.Docker { - // V1 qualifies a local Docker daemon. Remote executor/provider transports are - // separate work; do not silently inherit a different backend from the shell. - if !strings.HasPrefix(entry.Host, "unix:///") || len(entry.Host) <= 8 { - closeAll() - return nil, func() {}, errors.New("managed Docker host must be an explicit unix socket") + entry := config.Docker + host, err := url.Parse(entry.Host) + if err != nil || host.Scheme != "unix" || host.Host != "" || host.User != nil || host.RawQuery != "" || host.Fragment != "" || host.RawPath != "" || host.Path == "/" || !filepath.IsAbs(host.Path) || filepath.Clean(host.Path) != host.Path || entry.Host != "unix://"+host.Path { + return nil, closeProvider, errors.New("managed Docker host must be an explicit canonical unix socket") } seccomp, err := os.ReadFile(entry.SeccompFile) if err != nil || !json.Valid(seccomp) { - closeAll() - return nil, func() {}, errors.New("cannot read managed Docker seccomp JSON") + return nil, closeProvider, errors.New("cannot read managed Docker seccomp JSON") } c, err := client.New(client.WithHost(entry.Host)) if err != nil { - closeAll() - return nil, func() {}, errors.New("invalid managed Docker endpoint") + return nil, closeProvider, errors.New("invalid managed Docker endpoint") } - clients = append(clients, c) - provider, err := sandboxdocker.New(c, sandboxdocker.Config{InstallationID: key, Image: entry.Image, Network: entry.Network, Seccomp: string(seccomp), ExtraHosts: entry.ExtraHosts, NestedSandbox: entry.NestedSandbox}) + closeProvider = func() { _ = c.Close() } + provider, err := sandboxdocker.New(c, sandboxdocker.Config{InstallationID: config.InstallationID, Image: entry.Image, Network: entry.Network, Seccomp: string(seccomp), ExtraHosts: entry.ExtraHosts, NestedSandbox: entry.NestedSandbox}) if err != nil { - closeAll() + closeProvider() return nil, func() {}, errors.New("invalid managed Docker provider configuration") } - result.Providers[key] = provider + result.Provider = provider + result.BackendFingerprint = managedBackendFingerprint(config.Provider, entry.Host) + case "microsandbox": + if config.Microsandbox == nil || !hasMicrosandbox || hasDocker { + return nil, closeProvider, errors.New("managed microsandbox requires only the microsandbox configuration object") + } + if err := configureManagedMicrosandbox(*config.Microsandbox, result); err != nil { + return nil, closeProvider, err + } + default: + return nil, closeProvider, errors.New("managed provider must be docker or microsandbox") } - return result, closeAll, nil + return result, closeProvider, nil +} + +func managedBackendFingerprint(kind, namespace string) string { + digest := sha256.Sum256([]byte(kind + "\x00" + namespace)) + return hex.EncodeToString(digest[:]) } diff --git a/services/agents-api/cmd/server/managed_runtimes_test.go b/services/agents-api/cmd/server/managed_runtimes_test.go index 361c94da4..b322bffbf 100644 --- a/services/agents-api/cmd/server/managed_runtimes_test.go +++ b/services/agents-api/cmd/server/managed_runtimes_test.go @@ -17,25 +17,14 @@ func TestManagedRuntimeOperatorConfigurationIsExplicit(t *testing.T) { } else { close() } - root := t.TempDir() - seccomp := filepath.Join(root, "seccomp.json") - file := filepath.Join(root, "providers.json") + seccomp := filepath.Join(t.TempDir(), "seccomp.json") if err := os.WriteFile(seccomp, []byte(`{"defaultAction":"SCMP_ACT_ERRNO","syscalls":[]}`), 0600); err != nil { t.Fatal(err) } - key := uuid.NewString() - config := managedRuntimeConfig{CoreURL: "http://core.example/api/v1", DefaultProvider: key, Docker: map[string]managedDockerConfig{key: {Host: "unix:///var/run/docker.sock", Image: "sha256:" + strings.Repeat("a", 64), Network: "bridge", SeccompFile: seccomp}}} - write := func(c managedRuntimeConfig) { - raw, err := json.Marshal(c) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(file, raw, 0600); err != nil { - t.Fatal(err) - } - } + config, _, write := managedMicrosandboxFixture(t) + config.Provider, config.Microsandbox = "docker", nil + config.Docker = &managedDockerConfig{Host: "unix:///var/run/docker.sock", Image: "sha256:" + strings.Repeat("a", 64), Network: "bridge", SeccompFile: seccomp} write(config) - t.Setenv("AGENTS_API_MANAGED_RUNTIMES_FILE", file) t.Setenv("AGENTS_API_DAEMON_WS_URL", "") if _, close, err := managedRuntimes(); err == nil { close() @@ -43,22 +32,26 @@ func TestManagedRuntimeOperatorConfigurationIsExplicit(t *testing.T) { } t.Setenv("AGENTS_API_DAEMON_WS_URL", "ws://core.example/api/v1/agent-daemon/ws") t.Setenv("DOCKER_HOST", "not-a-valid-ambient-endpoint") + t.Setenv("AGENTS_API_ENGINE", "claude_sdk") result, close, err := managedRuntimes() if err != nil { t.Fatal(err) } - defer close() - if result.DefaultProvider != key || len(result.Providers) != 1 || result.Providers[key] == nil { - t.Fatal("provider identity lost") + close() + if result.InstallationID != config.InstallationID || result.Provider == nil || result.Suspension != nil || result.Maintenance { + t.Fatal("provider identity or admission policy lost") } for _, mutate := range []func(*managedRuntimeConfig){ - func(c *managedRuntimeConfig) { c.DefaultProvider = uuid.NewString() }, - func(c *managedRuntimeConfig) { e := c.Docker[key]; e.Host = "tcp://remote:2375"; c.Docker[key] = e }, - func(c *managedRuntimeConfig) { e := c.Docker[key]; e.Image = "latest"; c.Docker[key] = e }, - func(c *managedRuntimeConfig) { e := c.Docker[key]; e.Network = "host"; c.Docker[key] = e }, + func(c *managedRuntimeConfig) { c.InstallationID = "invalid" }, + func(c *managedRuntimeConfig) { c.Provider = "unknown" }, + func(c *managedRuntimeConfig) { c.Docker.Host = "tcp://remote:2375" }, + func(c *managedRuntimeConfig) { c.Docker.Host = "unix:///var/run/../docker.sock" }, + func(c *managedRuntimeConfig) { c.Docker.Host = "unix:///docker.sock?other=backend" }, + func(c *managedRuntimeConfig) { c.Docker.Image = "latest" }, + func(c *managedRuntimeConfig) { c.Docker.Network = "host" }, } { - changed := config - changed.Docker = map[string]managedDockerConfig{key: config.Docker[key]} + changed, entry := config, *config.Docker + changed.Docker = &entry mutate(&changed) write(changed) if _, close, err := managedRuntimes(); err == nil { @@ -66,14 +59,76 @@ func TestManagedRuntimeOperatorConfigurationIsExplicit(t *testing.T) { t.Fatal("unqualified managed configuration accepted") } } - config.DefaultProvider = "" + config.Maintenance = true + config.Docker.Image = "sha256:" + strings.Repeat("b", 64) + write(config) + maintenance, close, err := managedRuntimes() + if err != nil { + t.Fatal(err) + } + close() + if !maintenance.Maintenance || maintenance.Provider == nil || maintenance.BackendFingerprint != result.BackendFingerprint { + t.Fatal("maintenance lost the original cleanup adapter") + } + config.Docker.Host = "unix:///another/docker.sock" write(config) - result, close, err = managedRuntimes() + changed, close, err := managedRuntimes() if err != nil { t.Fatal(err) } - defer close() - if result.DefaultProvider != "" || len(result.Providers) != 1 { - t.Fatal("cleanup-only provider configuration lost") + close() + if changed.BackendFingerprint == result.BackendFingerprint { + t.Fatal("Docker endpoint change retained backend identity") + } +} + +func TestManagedRuntimeRejectsMixedAndLegacyConfiguration(t *testing.T) { + config, _, _ := managedMicrosandboxFixture(t) + raw, err := json.Marshal(config) + if err != nil { + t.Fatal(err) + } + cases := map[string]func(map[string]any){ + "default_provider": func(c map[string]any) { c["default_provider"] = config.InstallationID }, + "engine_providers": func(c map[string]any) { c["engine_providers"] = map[string]string{"codex": config.InstallationID} }, + "mixed_null": func(c map[string]any) { c["docker"] = nil }, + "selected_null": func(c map[string]any) { c["microsandbox"] = nil }, + "maintenance_null": func(c map[string]any) { c["maintenance"] = nil }, + "mixed": func(c map[string]any) { c["docker"] = map[string]any{} }, + "old_micro_map": func(c map[string]any) { c["microsandbox"] = map[string]any{config.InstallationID: c["microsandbox"]} }, + "old_docker_map": func(c map[string]any) { + c["provider"] = "docker" + delete(c, "microsandbox") + c["docker"] = map[string]any{uuid.NewString(): map[string]any{}} + }, + "missing_backend": func(c map[string]any) { delete(c, "microsandbox") }, + "missing_choice": func(c map[string]any) { delete(c, "provider") }, + "wrong_backend": func(c map[string]any) { c["provider"] = "docker" }, + "missing_identity": func(c map[string]any) { delete(c, "installation_id") }, + "misspelled_policy": func(c map[string]any) { + m := c["microsandbox"].(map[string]any) + m["idle_timeout"] = m["idle_seconds"] + delete(m, "idle_seconds") + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + var changed map[string]any + if err := json.Unmarshal(raw, &changed); err != nil { + t.Fatal(err) + } + mutate(changed) + data, err := json.Marshal(changed) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(os.Getenv("AGENTS_API_MANAGED_RUNTIMES_FILE"), data, 0600); err != nil { + t.Fatal(err) + } + if _, close, err := managedRuntimes(); err == nil { + close() + t.Fatal("mixed, legacy or incomplete configuration accepted") + } + }) } } diff --git a/services/agents-api/deploy/codex/README.md b/services/agents-api/deploy/codex/README.md index 8abad602a..223e174c1 100644 --- a/services/agents-api/deploy/codex/README.md +++ b/services/agents-api/deploy/codex/README.md @@ -149,22 +149,31 @@ Set `AGENTS_API_DAEMON_WS_URL` to the outward URL reachable from the Runtime and ```json { "core_url": "https://core.example/api/v1", - "default_provider": "11111111-1111-4111-8111-111111111111", + "provider": "docker", + "installation_id": "11111111-1111-4111-8111-111111111111", + "maintenance": false, "docker": { - "11111111-1111-4111-8111-111111111111": { - "host": "unix:///var/run/docker.sock", - "image": "sha256:", - "network": "bridge", - "seccomp_file": "/absolute/path/to/seccomp.json" - } + "host": "unix:///var/run/docker.sock", + "image": "sha256:", + "network": "bridge", + "seccomp_file": "/absolute/path/to/seccomp.json" } } ``` -The provider key identifies this Docker installation permanently. Keep its entry -while any allocation needs cleanup; changing the endpoint requires a new key. -An empty default disables new hosted admission/bootstrap while preserving existing -Session controls, input retry outcomes and cleanup of retained allocations. +Choose Docker or [microsandbox](../microsandbox/README.md) at setup. A Core +deployment accepts one provider and one backend object; legacy provider maps and +engine-to-provider routing are rejected. The installation ID and explicit socket +identify this backend. Harness selection does not choose a different provider. + +Set `maintenance: true` and restart with the old configuration before changing +providers. Explicitly handle or delete old hosted Sessions and resources, then +verify cleanup has completed. Configure the new provider and fresh installation +ID with maintenance still enabled, restart to validate the switch, then restart +with `maintenance: false` to resume new compute. See the +[provider switch procedure](../microsandbox/README.md#change-the-deployment-provider). +No resources are automatically deleted and no Sessions migrate between providers. + V1 supports explicit local Unix Docker sockets, ignoring ambient `DOCKER_HOST`. Optional `extra_hosts` is trusted operator configuration. No Docker socket is mounted in a Runtime. User-managed enrollment remains separate work. @@ -176,7 +185,7 @@ read at startup, copied for each execution and never persisted as public Session configuration. Omit it for the adapter's existing model configuration. Changes require a Core restart. Do not place credentials in public requests or images. -With the qualified Codex image and default provider configured, create an idle or +With the qualified Codex image and Docker provider configured, create an idle or initial-text Session using `environment: {"type": "openai_hosted"}`. Core commits its identity before automatically provisioning it. Queries expose durable connection status; execution separately prepares the native harness. Session diff --git a/services/agents-api/deploy/microsandbox/README.md b/services/agents-api/deploy/microsandbox/README.md new file mode 100644 index 000000000..abaac2d0b --- /dev/null +++ b/services/agents-api/deploy/microsandbox/README.md @@ -0,0 +1,176 @@ +# Single-host idle microVM suspension + +Run Core and its microsandbox helper natively on one Linux amd64 host. PostgreSQL +can remain in Docker. A hosted Session gets a dedicated microVM with the existing +daemon, native harness and workspace. Core suspends it only after a completed Turn +has remained idle and all admitted work has settled. The next Turn restores the +same Session history, files and configuration. Native harness startup and shutdown +keep their existing behavior. + +This profile uses microsandbox v0.7.2. The helper runs one bounded operation at a +time and exits; Core owns scheduling and durable recovery. See the +[provider contract](../../tools/microsandbox-provider/README.md) for snapshot +identity, uncertain operations and cleanup rules. This document specifies setup; +real-model continuation and isolation still require deployment acceptance. + +## Choose a hosted provider + +Choose either the [Docker deployment](../codex/README.md#standalone-operator-configuration) +or this microsandbox deployment during setup. Both adapters implement the same +Core Provider contract. One Core deployment uses one provider, one installation +identity and one backend configuration. Snapshot suspension is available only on +microsandbox. Docker remains supported with its ordinary lifecycle. + +Set `provider` to `docker` or `microsandbox` and include only that configuration +object. Provider selection applies to the entire deployment, independently of +harness selection. Legacy provider maps, `default_provider` and `engine_providers` +are rejected. The setup remains a private configuration file and a Core restart; +there is no separate setup service or automatic migration. + +## Change the deployment provider + +1. Keep the old provider, installation ID and backend path configured. Set + `maintenance: true` and restart Core. Maintenance blocks new compute; the old + adapter remains available for observing and explicitly cleaning up resources. +2. Handle or delete the old hosted Sessions and resources explicitly. Confirm all + retained allocations, including snapshots and pending hosted Sessions, are + gone. A public deletion acknowledgement alone does not prove physical cleanup. +3. Configure the new provider with a fresh `installation_id`, its backend object + and `maintenance: true`, then restart Core. Core validates that the old + deployment is empty before accepting the new identity. +4. Keep the new identity unchanged, set `maintenance: false` and restart Core to + allow new compute. + +The installation ID and backend namespace are persisted. Changing the Docker +socket or microsandbox `runtime_home` counts as a provider switch even if the UUID +is reused. Image, resource limits and idle policy do not change that identity. +Maintenance itself never initiates deletion; ordinary expiry, revocation and +explicit deletion keep their existing cleanup behavior. A switch requires both +the persisted old configuration and incoming configuration to be in maintenance. +Switching never deletes resources automatically or migrates Sessions between +providers. Do not repoint a configured backend path to another installation. + +When upgrading a database that has retained allocations but no recorded provider +identity, Core cannot verify their original backend and refuses initial adoption. +Keep the previous Core and its configuration available to finish cleanup before +starting this profile. An empty legacy deployment can select its first provider; +previously unallocated Sessions have no provider ownership to migrate. + +## Host and binaries + +Use a dedicated service account with access to `/dev/kvm`, a C compiler and the +repository's Go version. The helper requires glibc and the standard Linux dynamic +libraries. The existing Core binary remains a CGO-disabled build. The current +Core `distroless/static` image cannot run this helper. + +Build from the repository root: + +```sh +make build-agents-api build-daemon build-microsandbox-provider +make check-microsandbox-provider +``` + +The helper is written to +`~/.parsar/build/microsandbox-provider/agents-api-microsandbox-provider`. +Its separate Go module pins the published SDK and embeds its matching FFI library. +`make check` runs the pure-Go provider tests on every supported host. On Linux it +also runs the SDK helper module; other hosts print an explicit skip for that +Linux-only module. A full Linux check is required before publishing this profile. + +Install the Linux x86_64 archive from the official +[v0.7.2 release](https://github.com/superradcompany/microsandbox/releases/tag/v0.7.2) +into a fresh private directory under `~/.parsar/runtime/`. Verify the release +checksum before extraction. The qualified archive is +`microsandbox-linux-x86_64.tar.gz`, SHA256 +`47c223e3ef5298abf05f47ed9f87981106e400d99bb3f1d042d4d6881346b18b`. +It supplies `msb` and `libkrunfw.so.5.6.1`. Record each extracted file's SHA256 in +the provider configuration. The helper verifies both files on every invocation; +it does not install or upgrade them. + +Create a private, short runtime state path, for example +`~/.parsar/msb`, with mode 0700. Keep it on persistent local storage and reserve it +for this installation. Unix socket path limits apply. The directory holds +confidential disks, memory snapshots and SDK state; retain it together with the +Core database when recovering the host. + +## Runtime image + +Build the existing [Codex Runtime](../codex/README.md#managed-runtime-image-and-docker-adapter) +with the daemon from this branch, then publish it to an operator-controlled OCI +registry. Configure its immutable `repository@sha256:...` digest. The image must +be available to the local microsandbox installation before provisioning. + +The provider performs the existing Runtime bootstrap, starts the daemon as +uid/gid 1000 and creates `/run/parsar` as a private control directory. No model, +Core or tenant credential belongs in the image. The ordinary Runtime initializer +and native isolation profile still apply; snapshot restore does not rerun setup +commands or initial file writes. + +## Core configuration + +Copy [managed-runtimes.example.json](managed-runtimes.example.json) to a private +file under the service account's `~/.parsar/` directory and set mode 0600. Replace +all placeholder paths, hashes, image digest and hostnames. Generate a fresh +installation UUID for `installation_id`. Keep that identity and its original +backend while any allocation needs cleanup. Set `provider: "microsandbox"` and +include the single `microsandbox` object; do not include a `docker` object. + +Set VM memory, CPU and disk limits explicitly. `max_active` bounds active compute. +`max_retained` bounds all retained allocations, including suspended snapshots, and +must be at least `max_active`. `idle_seconds` is the sustained idle interval before +suspension. +`retention_seconds` bounds retained snapshots. Each value must be positive. +Choose capacity and retention for the host's RAM and disk budget. + +The example denies network access except HTTPS to the named Core and model +endpoints. Adapt those explicit rules to the deployment, including required +package registries if initialization uses them. Both creation and restore apply +the same host network policy. Native tool-network policy remains separate and +continues to use the existing Runtime controls. + +Use the existing [standalone Core setup](../../README.md) for the database, +migrations, API keys and encryption key. Add these variables to its private +service environment: + +```sh +export AGENTS_API_MANAGED_RUNTIMES_FILE="$HOME/.parsar/managed-runtimes.json" +export AGENTS_API_DAEMON_WS_URL='wss://core.example/api/v1/agent-daemon/ws' +export AGENTS_API_ENGINE=codex +"$HOME/.parsar/build/agents-api/agents-api" +``` + +The outward Core URL must be reachable from the guest. `localhost` inside the +microVM refers to the guest. Model provider settings continue to use the existing +private `AGENTS_API_EXECUTION_OPTIONS_FILE` contract. + +## Park, restore and verification + +Core first reserves the idle allocation. The daemon rejects suspension while +Turns, preparations, input receipts or file operations remain unsettled. After +its quiesce acknowledgement, it closes the old socket and parks. Core captures +and verifies the full snapshot before stopping the source VM and reclaiming RAM. + +Restore creates the precommitted next compute generation. Core invokes the +existing daemon binary inside that exact VM: + +```sh +parsar-daemon resume --control-file /run/parsar/daemon-suspend.json \ + --environment-id ENVIRONMENT_ID --suspend-id SUSPENSION_ID +``` + +This is Core's recovery action, not a second service. The command validates the +protected Environment/suspension identity and the daemon's PID/start time before +signalling through a Linux pidfd. The daemon reauthenticates and requires Core's +matching resume confirmation before admitting work. Each reconnect attempt is +limited to 30 seconds; transient failures retry the same suspension with backoff. +Authentication or protocol rejection terminates recovery. Ordinary disconnects +after resume retain the existing cleanup behavior. A failed capture may roll back the still-running +source through the same controlled wake path. + +For acceptance, complete a real-model Turn that writes a known file, wait for +suspension and confirm the source VM has stopped and released RAM. Submit the next +Turn in the same Session and verify the prior history, exact file bytes and frozen +configuration. Confirm setup ran once and no prior input or tool operation was +replayed. Exercise Core restart during capture/restore, pending-work exclusion, +credential revocation and Session deletion with both live and suspended compute. +Keep the resulting evidence separate from unit-test results. diff --git a/services/agents-api/deploy/microsandbox/managed-runtimes.example.json b/services/agents-api/deploy/microsandbox/managed-runtimes.example.json new file mode 100644 index 000000000..eb6aa69d1 --- /dev/null +++ b/services/agents-api/deploy/microsandbox/managed-runtimes.example.json @@ -0,0 +1,42 @@ +{ + "core_url": "https://core.example/api/v1", + "provider": "microsandbox", + "installation_id": "11111111-1111-4111-8111-111111111111", + "maintenance": false, + "microsandbox": { + "helper_path": "/home/parsar/.parsar/build/microsandbox-provider/agents-api-microsandbox-provider", + "runtime_home": "/home/parsar/.parsar/msb", + "runtime_path": "/home/parsar/.parsar/runtime/microsandbox-v0.7.2/msb", + "firmware_path": "/home/parsar/.parsar/runtime/microsandbox-v0.7.2/libkrunfw.so.5.6.1", + "runtime_sha256": "", + "firmware_sha256": "", + "image": "registry.example/parsar-codex-runtime@sha256:<64 lowercase hexadecimal digits>", + "memory_mib": 4096, + "cpus": 2, + "root_disk_mib": 8192, + "network": { + "default_egress": "deny", + "default_ingress": "deny", + "rules": [ + { + "action": "allow", + "direction": "egress", + "destination": "core.example", + "protocol": "tcp", + "port": "443" + }, + { + "action": "allow", + "direction": "egress", + "destination": "model.example", + "protocol": "tcp", + "port": "443" + } + ] + }, + "idle_seconds": 300, + "retention_seconds": 86400, + "max_active": 4, + "max_retained": 16 + } +} diff --git a/services/agents-api/internal/db/queries/runtime_allocations.sql b/services/agents-api/internal/db/queries/runtime_allocations.sql index 4c7e97418..248be3e56 100644 --- a/services/agents-api/internal/db/queries/runtime_allocations.sql +++ b/services/agents-api/internal/db/queries/runtime_allocations.sql @@ -3,14 +3,14 @@ INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, in VALUES ($1, $2, $3, $4, CASE WHEN EXISTS (SELECT 1 FROM initial_environment_files f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) OR EXISTS (SELECT 1 FROM environment_setups f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) THEN 'pending' ELSE 'complete' END) RETURNING *; -- name: GetRuntimeAllocation :one -SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (a.kept_at <= clock_timestamp() - interval '1 hour') AS expired +SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = $1 AND a.environment_id = $2; -- name: ListRuntimeAllocations :many -SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (a.kept_at <= clock_timestamp() - interval '1 hour') AS expired +SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id @@ -42,7 +42,7 @@ UPDATE runtime_allocations SET state = 'released', released_at = clock_timestamp WHERE id = $1 AND state = 'cleanup_pending' AND create_settled RETURNING *; -- name: ListUnallocatedHostedEnvironments :many -SELECT e.id, s.tenant_id, s.engine +SELECT e.id, s.tenant_id FROM environments e JOIN sessions s ON s.id = e.session_id WHERE e.id > $1 AND s.deleted_at IS NULL AND e.status = 'pending' AND s.configuration->'environment'->>'type' = 'openai_hosted' diff --git a/services/agents-api/internal/db/queries/runtime_deployment.sql b/services/agents-api/internal/db/queries/runtime_deployment.sql new file mode 100644 index 000000000..c23546b24 --- /dev/null +++ b/services/agents-api/internal/db/queries/runtime_deployment.sql @@ -0,0 +1,14 @@ +-- name: LockRuntimeDeployment :one +SELECT * FROM runtime_deployment WHERE singleton = true FOR UPDATE; + +-- name: SetRuntimeDeployment :exec +UPDATE runtime_deployment SET installation_id = $1, backend_fingerprint = $2, +maintenance = $3, updated_at = clock_timestamp() WHERE singleton = true; + +-- name: CountRuntimeDeploymentResources :one +SELECT +(SELECT count(*) FROM runtime_allocations WHERE state <> 'released')::bigint AS allocations, +(SELECT count(*) FROM environments e JOIN sessions s ON s.id = e.session_id + WHERE s.deleted_at IS NULL AND e.status = 'pending' + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id))::bigint AS pending; diff --git a/services/agents-api/internal/db/queries/runtime_suspension.sql b/services/agents-api/internal/db/queries/runtime_suspension.sql new file mode 100644 index 000000000..8040e41fd --- /dev/null +++ b/services/agents-api/internal/db/queries/runtime_suspension.sql @@ -0,0 +1,52 @@ +-- name: SetRuntimeCompute :one +UPDATE runtime_allocations +SET compute_phase = sqlc.arg(phase), compute_state = sqlc.arg(state)::jsonb, + compute_revision = compute_revision + 1, + compute_retained_until = sqlc.narg(retained_until), + kept_at = CASE WHEN sqlc.arg(phase)::text = 'running' THEN clock_timestamp() ELSE kept_at END +WHERE id = sqlc.arg(id) AND compute_revision = sqlc.arg(revision) + AND state = 'running' AND initialization = 'complete' + AND ((compute_phase IN ('disabled','running') AND kept_at > clock_timestamp() - interval '1 hour') + OR (compute_phase NOT IN ('disabled','running') AND compute_retained_until > clock_timestamp())) +RETURNING *; + +-- name: TouchRuntimeActivity :exec +UPDATE runtime_allocations a +SET compute_activity_at = clock_timestamp(), compute_wake_requested = true +FROM environments e JOIN sessions s ON s.id = e.session_id +WHERE a.environment_id = e.id AND s.tenant_id = sqlc.arg(tenant_id) + AND e.id = sqlc.arg(environment_id) AND s.deleted_at IS NULL + AND a.state = 'running' AND a.compute_phase <> 'disabled'; + +-- name: ClearRuntimeWake :exec +UPDATE runtime_allocations +SET compute_wake_requested = false +WHERE id = $1 AND compute_phase = 'running' AND compute_activity_at <= $2; + +-- name: GetRuntimeActivity :one +SELECT GREATEST(a.compute_activity_at, + COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at), + (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id), + (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, + (EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) + OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) + OR EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = e.session_id AND r.state = 'pending') + OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = e.id AND f.state = 'pending'))::boolean AS busy, + a.compute_wake_requested, + EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.completed_at IS NOT NULL)::boolean AS has_completed_turn +FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id +WHERE a.id = $1; + +-- name: CountRuntimeComputeReservations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released' AND compute_phase <> 'suspended'; + +-- name: CountRuntimeRetainedAllocations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released'; + +-- name: RuntimeComputeBlocksAdmission :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id + WHERE e.session_id = $1 AND a.compute_phase NOT IN ('disabled', 'running') +)::boolean; diff --git a/services/agents-api/internal/db/sqlc/initial_environment_files.sql.go b/services/agents-api/internal/db/sqlc/initial_environment_files.sql.go index 3f9fa3ef6..2f0659e30 100644 --- a/services/agents-api/internal/db/sqlc/initial_environment_files.sql.go +++ b/services/agents-api/internal/db/sqlc/initial_environment_files.sql.go @@ -15,7 +15,7 @@ const claimRuntimeInitialization = `-- name: ClaimRuntimeInitialization :one UPDATE runtime_allocations SET initialization = 'running' WHERE id = $1 AND initialization = 'pending' AND state = 'running' AND create_settled AND kept_at > clock_timestamp() - interval '1 hour' -RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization +RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until ` func (q *Queries) ClaimRuntimeInitialization(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -32,6 +32,12 @@ func (q *Queries) ClaimRuntimeInitialization(ctx context.Context, id pgtype.UUID &i.KeptAt, &i.ReleasedAt, &i.Initialization, + &i.ComputePhase, + &i.ComputeRevision, + &i.ComputeState, + &i.ComputeActivityAt, + &i.ComputeWakeRequested, + &i.ComputeRetainedUntil, ) return i, err } @@ -40,7 +46,7 @@ const completeRuntimeInitialization = `-- name: CompleteRuntimeInitialization :o UPDATE runtime_allocations SET initialization = 'complete' WHERE id = $1 AND initialization = 'running' AND state = 'running' AND create_settled AND kept_at > clock_timestamp() - interval '1 hour' -RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization +RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until ` func (q *Queries) CompleteRuntimeInitialization(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -57,6 +63,12 @@ func (q *Queries) CompleteRuntimeInitialization(ctx context.Context, id pgtype.U &i.KeptAt, &i.ReleasedAt, &i.Initialization, + &i.ComputePhase, + &i.ComputeRevision, + &i.ComputeState, + &i.ComputeActivityAt, + &i.ComputeWakeRequested, + &i.ComputeRetainedUntil, ) return i, err } diff --git a/services/agents-api/internal/db/sqlc/models.go b/services/agents-api/internal/db/sqlc/models.go index ceb478f18..2a3811d95 100644 --- a/services/agents-api/internal/db/sqlc/models.go +++ b/services/agents-api/internal/db/sqlc/models.go @@ -143,16 +143,30 @@ type PublicExecutionTurn struct { } type RuntimeAllocation struct { - ID pgtype.UUID `json:"id"` - EnvironmentID pgtype.UUID `json:"environment_id"` - DeviceID pgtype.UUID `json:"device_id"` - ProviderKey pgtype.UUID `json:"provider_key"` - State string `json:"state"` - CreateSettled bool `json:"create_settled"` - CreatedAt pgtype.Timestamptz `json:"created_at"` - KeptAt pgtype.Timestamptz `json:"kept_at"` - ReleasedAt pgtype.Timestamptz `json:"released_at"` - Initialization string `json:"initialization"` + ID pgtype.UUID `json:"id"` + EnvironmentID pgtype.UUID `json:"environment_id"` + DeviceID pgtype.UUID `json:"device_id"` + ProviderKey pgtype.UUID `json:"provider_key"` + State string `json:"state"` + CreateSettled bool `json:"create_settled"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + KeptAt pgtype.Timestamptz `json:"kept_at"` + ReleasedAt pgtype.Timestamptz `json:"released_at"` + Initialization string `json:"initialization"` + ComputePhase string `json:"compute_phase"` + ComputeRevision int64 `json:"compute_revision"` + ComputeState []byte `json:"compute_state"` + ComputeActivityAt pgtype.Timestamptz `json:"compute_activity_at"` + ComputeWakeRequested bool `json:"compute_wake_requested"` + ComputeRetainedUntil pgtype.Timestamptz `json:"compute_retained_until"` +} + +type RuntimeDeployment struct { + Singleton bool `json:"singleton"` + InstallationID pgtype.UUID `json:"installation_id"` + BackendFingerprint string `json:"backend_fingerprint"` + Maintenance bool `json:"maintenance"` + UpdatedAt pgtype.Timestamptz `json:"updated_at"` } type RuntimeDeviceAuthority struct { diff --git a/services/agents-api/internal/db/sqlc/runtime_allocations.sql.go b/services/agents-api/internal/db/sqlc/runtime_allocations.sql.go index 7406ba900..b6ae080bf 100644 --- a/services/agents-api/internal/db/sqlc/runtime_allocations.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_allocations.sql.go @@ -13,7 +13,7 @@ import ( const createRuntimeAllocation = `-- name: CreateRuntimeAllocation :one INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, initialization) -VALUES ($1, $2, $3, $4, CASE WHEN EXISTS (SELECT 1 FROM initial_environment_files f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) OR EXISTS (SELECT 1 FROM environment_setups f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) THEN 'pending' ELSE 'complete' END) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization +VALUES ($1, $2, $3, $4, CASE WHEN EXISTS (SELECT 1 FROM initial_environment_files f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) OR EXISTS (SELECT 1 FROM environment_setups f JOIN environments e ON e.session_id = f.session_id WHERE e.id = $2) THEN 'pending' ELSE 'complete' END) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until ` type CreateRuntimeAllocationParams struct { @@ -42,12 +42,18 @@ func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntime &i.KeptAt, &i.ReleasedAt, &i.Initialization, + &i.ComputePhase, + &i.ComputeRevision, + &i.ComputeState, + &i.ComputeActivityAt, + &i.ComputeWakeRequested, + &i.ComputeRetainedUntil, ) return i, err } const getRuntimeAllocation = `-- name: GetRuntimeAllocation :one -SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, e.session_id, s.tenant_id, s.deleted_at, (a.kept_at <= clock_timestamp() - interval '1 hour') AS expired +SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id @@ -81,6 +87,12 @@ func (q *Queries) GetRuntimeAllocation(ctx context.Context, arg GetRuntimeAlloca &i.RuntimeAllocation.KeptAt, &i.RuntimeAllocation.ReleasedAt, &i.RuntimeAllocation.Initialization, + &i.RuntimeAllocation.ComputePhase, + &i.RuntimeAllocation.ComputeRevision, + &i.RuntimeAllocation.ComputeState, + &i.RuntimeAllocation.ComputeActivityAt, + &i.RuntimeAllocation.ComputeWakeRequested, + &i.RuntimeAllocation.ComputeRetainedUntil, &i.SessionID, &i.TenantID, &i.DeletedAt, @@ -93,7 +105,7 @@ const keepRuntimeAllocation = `-- name: KeepRuntimeAllocation :one UPDATE runtime_allocations SET kept_at = clock_timestamp() WHERE id = $1 AND state = 'running' AND kept_at > clock_timestamp() - interval '1 hour' -RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization +RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until ` func (q *Queries) KeepRuntimeAllocation(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -110,12 +122,18 @@ func (q *Queries) KeepRuntimeAllocation(ctx context.Context, id pgtype.UUID) (Ru &i.KeptAt, &i.ReleasedAt, &i.Initialization, + &i.ComputePhase, + &i.ComputeRevision, + &i.ComputeState, + &i.ComputeActivityAt, + &i.ComputeWakeRequested, + &i.ComputeRetainedUntil, ) return i, err } const listRuntimeAllocations = `-- name: ListRuntimeAllocations :many -SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, e.session_id, s.tenant_id, s.deleted_at, (a.kept_at <= clock_timestamp() - interval '1 hour') AS expired +SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.initialization, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id @@ -151,6 +169,12 @@ func (q *Queries) ListRuntimeAllocations(ctx context.Context, id pgtype.UUID) ([ &i.RuntimeAllocation.KeptAt, &i.RuntimeAllocation.ReleasedAt, &i.RuntimeAllocation.Initialization, + &i.RuntimeAllocation.ComputePhase, + &i.RuntimeAllocation.ComputeRevision, + &i.RuntimeAllocation.ComputeState, + &i.RuntimeAllocation.ComputeActivityAt, + &i.RuntimeAllocation.ComputeWakeRequested, + &i.RuntimeAllocation.ComputeRetainedUntil, &i.SessionID, &i.TenantID, &i.DeletedAt, @@ -167,7 +191,7 @@ func (q *Queries) ListRuntimeAllocations(ctx context.Context, id pgtype.UUID) ([ } const listUnallocatedHostedEnvironments = `-- name: ListUnallocatedHostedEnvironments :many -SELECT e.id, s.tenant_id, s.engine +SELECT e.id, s.tenant_id FROM environments e JOIN sessions s ON s.id = e.session_id WHERE e.id > $1 AND s.deleted_at IS NULL AND e.status = 'pending' AND s.configuration->'environment'->>'type' = 'openai_hosted' @@ -178,7 +202,6 @@ ORDER BY e.id LIMIT 32 type ListUnallocatedHostedEnvironmentsRow struct { ID pgtype.UUID `json:"id"` TenantID pgtype.UUID `json:"tenant_id"` - Engine string `json:"engine"` } func (q *Queries) ListUnallocatedHostedEnvironments(ctx context.Context, id pgtype.UUID) ([]ListUnallocatedHostedEnvironmentsRow, error) { @@ -190,7 +213,7 @@ func (q *Queries) ListUnallocatedHostedEnvironments(ctx context.Context, id pgty items := []ListUnallocatedHostedEnvironmentsRow{} for rows.Next() { var i ListUnallocatedHostedEnvironmentsRow - if err := rows.Scan(&i.ID, &i.TenantID, &i.Engine); err != nil { + if err := rows.Scan(&i.ID, &i.TenantID); err != nil { return nil, err } items = append(items, i) @@ -205,7 +228,7 @@ const observeRuntimeRunning = `-- name: ObserveRuntimeRunning :one UPDATE runtime_allocations SET state = 'running', create_settled = true WHERE id = $1 AND state IN ('creating', 'running') AND kept_at > clock_timestamp() - interval '1 hour' -RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization +RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until ` func (q *Queries) ObserveRuntimeRunning(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -222,13 +245,19 @@ func (q *Queries) ObserveRuntimeRunning(ctx context.Context, id pgtype.UUID) (Ru &i.KeptAt, &i.ReleasedAt, &i.Initialization, + &i.ComputePhase, + &i.ComputeRevision, + &i.ComputeState, + &i.ComputeActivityAt, + &i.ComputeWakeRequested, + &i.ComputeRetainedUntil, ) return i, err } const releaseRuntimeAllocation = `-- name: ReleaseRuntimeAllocation :one UPDATE runtime_allocations SET state = 'released', released_at = clock_timestamp() -WHERE id = $1 AND state = 'cleanup_pending' AND create_settled RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization +WHERE id = $1 AND state = 'cleanup_pending' AND create_settled RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until ` func (q *Queries) ReleaseRuntimeAllocation(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -245,13 +274,19 @@ func (q *Queries) ReleaseRuntimeAllocation(ctx context.Context, id pgtype.UUID) &i.KeptAt, &i.ReleasedAt, &i.Initialization, + &i.ComputePhase, + &i.ComputeRevision, + &i.ComputeState, + &i.ComputeActivityAt, + &i.ComputeWakeRequested, + &i.ComputeRetainedUntil, ) return i, err } const requestRuntimeCleanup = `-- name: RequestRuntimeCleanup :one UPDATE runtime_allocations SET state = 'cleanup_pending' -WHERE id = $1 AND state <> 'released' RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization +WHERE id = $1 AND state <> 'released' RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until ` func (q *Queries) RequestRuntimeCleanup(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -268,13 +303,19 @@ func (q *Queries) RequestRuntimeCleanup(ctx context.Context, id pgtype.UUID) (Ru &i.KeptAt, &i.ReleasedAt, &i.Initialization, + &i.ComputePhase, + &i.ComputeRevision, + &i.ComputeState, + &i.ComputeActivityAt, + &i.ComputeWakeRequested, + &i.ComputeRetainedUntil, ) return i, err } const settleRuntimeCreation = `-- name: SettleRuntimeCreation :one UPDATE runtime_allocations SET create_settled = true -WHERE id = $1 AND state <> 'released' RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization +WHERE id = $1 AND state <> 'released' RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until ` func (q *Queries) SettleRuntimeCreation(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -291,6 +332,12 @@ func (q *Queries) SettleRuntimeCreation(ctx context.Context, id pgtype.UUID) (Ru &i.KeptAt, &i.ReleasedAt, &i.Initialization, + &i.ComputePhase, + &i.ComputeRevision, + &i.ComputeState, + &i.ComputeActivityAt, + &i.ComputeWakeRequested, + &i.ComputeRetainedUntil, ) return i, err } diff --git a/services/agents-api/internal/db/sqlc/runtime_deployment.sql.go b/services/agents-api/internal/db/sqlc/runtime_deployment.sql.go new file mode 100644 index 000000000..a7385d66a --- /dev/null +++ b/services/agents-api/internal/db/sqlc/runtime_deployment.sql.go @@ -0,0 +1,66 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: runtime_deployment.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const countRuntimeDeploymentResources = `-- name: CountRuntimeDeploymentResources :one +SELECT +(SELECT count(*) FROM runtime_allocations WHERE state <> 'released')::bigint AS allocations, +(SELECT count(*) FROM environments e JOIN sessions s ON s.id = e.session_id + WHERE s.deleted_at IS NULL AND e.status = 'pending' + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id))::bigint AS pending +` + +type CountRuntimeDeploymentResourcesRow struct { + Allocations int64 `json:"allocations"` + Pending int64 `json:"pending"` +} + +func (q *Queries) CountRuntimeDeploymentResources(ctx context.Context) (CountRuntimeDeploymentResourcesRow, error) { + row := q.db.QueryRow(ctx, countRuntimeDeploymentResources) + var i CountRuntimeDeploymentResourcesRow + err := row.Scan(&i.Allocations, &i.Pending) + return i, err +} + +const lockRuntimeDeployment = `-- name: LockRuntimeDeployment :one +SELECT singleton, installation_id, backend_fingerprint, maintenance, updated_at FROM runtime_deployment WHERE singleton = true FOR UPDATE +` + +func (q *Queries) LockRuntimeDeployment(ctx context.Context) (RuntimeDeployment, error) { + row := q.db.QueryRow(ctx, lockRuntimeDeployment) + var i RuntimeDeployment + err := row.Scan( + &i.Singleton, + &i.InstallationID, + &i.BackendFingerprint, + &i.Maintenance, + &i.UpdatedAt, + ) + return i, err +} + +const setRuntimeDeployment = `-- name: SetRuntimeDeployment :exec +UPDATE runtime_deployment SET installation_id = $1, backend_fingerprint = $2, +maintenance = $3, updated_at = clock_timestamp() WHERE singleton = true +` + +type SetRuntimeDeploymentParams struct { + InstallationID pgtype.UUID `json:"installation_id"` + BackendFingerprint string `json:"backend_fingerprint"` + Maintenance bool `json:"maintenance"` +} + +func (q *Queries) SetRuntimeDeployment(ctx context.Context, arg SetRuntimeDeploymentParams) error { + _, err := q.db.Exec(ctx, setRuntimeDeployment, arg.InstallationID, arg.BackendFingerprint, arg.Maintenance) + return err +} diff --git a/services/agents-api/internal/db/sqlc/runtime_suspension.sql.go b/services/agents-api/internal/db/sqlc/runtime_suspension.sql.go new file mode 100644 index 000000000..c817d959c --- /dev/null +++ b/services/agents-api/internal/db/sqlc/runtime_suspension.sql.go @@ -0,0 +1,170 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: runtime_suspension.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const clearRuntimeWake = `-- name: ClearRuntimeWake :exec +UPDATE runtime_allocations +SET compute_wake_requested = false +WHERE id = $1 AND compute_phase = 'running' AND compute_activity_at <= $2 +` + +type ClearRuntimeWakeParams struct { + ID pgtype.UUID `json:"id"` + ComputeActivityAt pgtype.Timestamptz `json:"compute_activity_at"` +} + +func (q *Queries) ClearRuntimeWake(ctx context.Context, arg ClearRuntimeWakeParams) error { + _, err := q.db.Exec(ctx, clearRuntimeWake, arg.ID, arg.ComputeActivityAt) + return err +} + +const countRuntimeComputeReservations = `-- name: CountRuntimeComputeReservations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released' AND compute_phase <> 'suspended' +` + +func (q *Queries) CountRuntimeComputeReservations(ctx context.Context, providerKey pgtype.UUID) (int64, error) { + row := q.db.QueryRow(ctx, countRuntimeComputeReservations, providerKey) + var count int64 + err := row.Scan(&count) + return count, err +} + +const countRuntimeRetainedAllocations = `-- name: CountRuntimeRetainedAllocations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released' +` + +func (q *Queries) CountRuntimeRetainedAllocations(ctx context.Context, providerKey pgtype.UUID) (int64, error) { + row := q.db.QueryRow(ctx, countRuntimeRetainedAllocations, providerKey) + var count int64 + err := row.Scan(&count) + return count, err +} + +const getRuntimeActivity = `-- name: GetRuntimeActivity :one +SELECT GREATEST(a.compute_activity_at, + COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at), + (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id), + (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, + (EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) + OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) + OR EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = e.session_id AND r.state = 'pending') + OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = e.id AND f.state = 'pending'))::boolean AS busy, + a.compute_wake_requested, + EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.completed_at IS NOT NULL)::boolean AS has_completed_turn +FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id +WHERE a.id = $1 +` + +type GetRuntimeActivityRow struct { + LastActivity pgtype.Timestamptz `json:"last_activity"` + Busy bool `json:"busy"` + ComputeWakeRequested bool `json:"compute_wake_requested"` + HasCompletedTurn bool `json:"has_completed_turn"` +} + +func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRuntimeActivityRow, error) { + row := q.db.QueryRow(ctx, getRuntimeActivity, id) + var i GetRuntimeActivityRow + err := row.Scan( + &i.LastActivity, + &i.Busy, + &i.ComputeWakeRequested, + &i.HasCompletedTurn, + ) + return i, err +} + +const runtimeComputeBlocksAdmission = `-- name: RuntimeComputeBlocksAdmission :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id + WHERE e.session_id = $1 AND a.compute_phase NOT IN ('disabled', 'running') +)::boolean +` + +func (q *Queries) RuntimeComputeBlocksAdmission(ctx context.Context, sessionID pgtype.UUID) (bool, error) { + row := q.db.QueryRow(ctx, runtimeComputeBlocksAdmission, sessionID) + var column_1 bool + err := row.Scan(&column_1) + return column_1, err +} + +const setRuntimeCompute = `-- name: SetRuntimeCompute :one +UPDATE runtime_allocations +SET compute_phase = $1, compute_state = $2::jsonb, + compute_revision = compute_revision + 1, + compute_retained_until = $3, + kept_at = CASE WHEN $1::text = 'running' THEN clock_timestamp() ELSE kept_at END +WHERE id = $4 AND compute_revision = $5 + AND state = 'running' AND initialization = 'complete' + AND ((compute_phase IN ('disabled','running') AND kept_at > clock_timestamp() - interval '1 hour') + OR (compute_phase NOT IN ('disabled','running') AND compute_retained_until > clock_timestamp())) +RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, initialization, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until +` + +type SetRuntimeComputeParams struct { + Phase string `json:"phase"` + State []byte `json:"state"` + RetainedUntil pgtype.Timestamptz `json:"retained_until"` + ID pgtype.UUID `json:"id"` + Revision int64 `json:"revision"` +} + +func (q *Queries) SetRuntimeCompute(ctx context.Context, arg SetRuntimeComputeParams) (RuntimeAllocation, error) { + row := q.db.QueryRow(ctx, setRuntimeCompute, + arg.Phase, + arg.State, + arg.RetainedUntil, + arg.ID, + arg.Revision, + ) + var i RuntimeAllocation + err := row.Scan( + &i.ID, + &i.EnvironmentID, + &i.DeviceID, + &i.ProviderKey, + &i.State, + &i.CreateSettled, + &i.CreatedAt, + &i.KeptAt, + &i.ReleasedAt, + &i.Initialization, + &i.ComputePhase, + &i.ComputeRevision, + &i.ComputeState, + &i.ComputeActivityAt, + &i.ComputeWakeRequested, + &i.ComputeRetainedUntil, + ) + return i, err +} + +const touchRuntimeActivity = `-- name: TouchRuntimeActivity :exec +UPDATE runtime_allocations a +SET compute_activity_at = clock_timestamp(), compute_wake_requested = true +FROM environments e JOIN sessions s ON s.id = e.session_id +WHERE a.environment_id = e.id AND s.tenant_id = $1 + AND e.id = $2 AND s.deleted_at IS NULL + AND a.state = 'running' AND a.compute_phase <> 'disabled' +` + +type TouchRuntimeActivityParams struct { + TenantID pgtype.UUID `json:"tenant_id"` + EnvironmentID pgtype.UUID `json:"environment_id"` +} + +func (q *Queries) TouchRuntimeActivity(ctx context.Context, arg TouchRuntimeActivityParams) error { + _, err := q.db.Exec(ctx, touchRuntimeActivity, arg.TenantID, arg.EnvironmentID) + return err +} diff --git a/services/agents-api/internal/execution/dispatcher.go b/services/agents-api/internal/execution/dispatcher.go index e2427eee6..3d9c98e03 100644 --- a/services/agents-api/internal/execution/dispatcher.go +++ b/services/agents-api/internal/execution/dispatcher.go @@ -35,7 +35,7 @@ type Dispatcher struct { // Options resolves transient engine credentials; they are never stored here. Options func(context.Context, store.Session) (map[string]any, error) // ManagedRuntimes is optional internal provisioning; it does not admit hosted API requests. - ManagedRuntimes *RuntimeProviders + ManagedRuntimes *RuntimeProvider } type Result struct { diff --git a/services/agents-api/internal/execution/environment_admission.go b/services/agents-api/internal/execution/environment_admission.go index f818c3551..eadda58d7 100644 --- a/services/agents-api/internal/execution/environment_admission.go +++ b/services/agents-api/internal/execution/environment_admission.go @@ -54,9 +54,6 @@ func (w *Worker) validateCreation(ctx context.Context, input store.CreateSession if err := json.Unmarshal(input.Configuration, &snapshot); err != nil { return store.ErrInvalidInput } - if snapshot.Environment.Type == "openai_hosted" && w.runtimes.config.ProviderForEngine(input.Engine) == "" { - return ErrExecutionUnavailable - } if len(input.InitialInputs) == 0 && snapshot.Environment.Type == "self_hosted" { return nil } diff --git a/services/agents-api/internal/execution/environment_directory.go b/services/agents-api/internal/execution/environment_directory.go index 205dd4f65..eb00489bc 100644 --- a/services/agents-api/internal/execution/environment_directory.go +++ b/services/agents-api/internal/execution/environment_directory.go @@ -34,6 +34,9 @@ func (w *Worker) ReadEnvironmentDirectory(ctx context.Context, environment store if current.SessionID != environment.SessionID { return proto.WorkspaceDirectoryResult{}, store.ErrNotFound } + if err := w.waitRuntimeAwake(ctx, current); err != nil { + return proto.WorkspaceDirectoryResult{}, err + } request := directoryReadRequest{ctx: ctx, environment: current, path: path, result: make(chan directoryReadResult, 1)} select { case w.directoryReads <- request: @@ -54,6 +57,16 @@ func (w *Worker) ReadEnvironmentDirectory(ctx context.Context, environment store func (w *Worker) runDirectoryRead(owner context.Context, request directoryReadRequest, reserved bool) (result directoryReadResult) { result.err = ErrExecutionUnavailable + defer func() { + if w.runtimes == nil { + return + } + touch, stop := context.WithTimeout(context.WithoutCancel(owner), 5*time.Second) + defer stop() + if err := w.dispatcher.Store.TouchRuntimeActivity(touch, request.environment.TenantID, request.environment.ID); err != nil { + result.err = err + } + }() check, cancel := context.WithTimeout(owner, 5*time.Second) defer cancel() if w.CheckOwnership(check) != nil { diff --git a/services/agents-api/internal/execution/environment_file_write.go b/services/agents-api/internal/execution/environment_file_write.go index 7cdb1aee3..c28e58ce7 100644 --- a/services/agents-api/internal/execution/environment_file_write.go +++ b/services/agents-api/internal/execution/environment_file_write.go @@ -30,6 +30,9 @@ func (w *Worker) WriteEnvironmentFile(ctx context.Context, environment store.Env if len(data) > proto.WorkspaceWriteMaxBytes { return 0, store.ErrInvalidInput } + if err := w.waitRuntimeAwake(ctx, environment); err != nil { + return 0, err + } request := fileWriteRequest{ctx: ctx, environment: environment, path: path, data: data, result: make(chan fileWriteResult, 1)} select { case w.fileWrites <- request: diff --git a/services/agents-api/internal/execution/prepared_dispatch.go b/services/agents-api/internal/execution/prepared_dispatch.go index ed12c8295..acf024e96 100644 --- a/services/agents-api/internal/execution/prepared_dispatch.go +++ b/services/agents-api/internal/execution/prepared_dispatch.go @@ -3,6 +3,7 @@ package execution import ( "context" "encoding/json" + "errors" "strings" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" @@ -85,7 +86,12 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, tenantID, sessionI if err := d.messageInputSupport(peer, session.Engine, snapshot, messages); err != nil { return run, err } - run.Reservation, err = d.Store.PromoteEnvironmentInput(owner, tenantID, sessionID, reservationID) + promoted, err := d.Store.PromoteEnvironmentInput(owner, tenantID, sessionID, reservationID) + if errors.Is(err, store.ErrTurnConflict) { + // A rejected claim leaves the reservation pending for a later attempt. + return run, err + } + run.Reservation = promoted if err != nil || run.Reservation.State != store.EnvironmentInputAdmitted { return run, err } diff --git a/services/agents-api/internal/execution/runtime_compute.go b/services/agents-api/internal/execution/runtime_compute.go new file mode 100644 index 000000000..eced36aa1 --- /dev/null +++ b/services/agents-api/internal/execution/runtime_compute.go @@ -0,0 +1,272 @@ +package execution + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +// RuntimeSuspensionPolicy applies only to an explicitly qualified single-host +// provider. Fixed guest sizing plus MaxActive bounds reserved CPU and memory. +type RuntimeSuspensionPolicy struct { + IdleTimeout time.Duration + Retention time.Duration + MaxActive int + MaxRetained int +} + +type runtimeCompute struct { + Current sandbox.Compute `json:"current"` + Target *sandbox.Compute `json:"target,omitempty"` + Snapshot *sandbox.SnapshotIdentity `json:"snapshot,omitempty"` + SuspendID string `json:"suspend_id,omitempty"` + RestoreID string `json:"restore_id,omitempty"` + Rollback bool `json:"rollback,omitempty"` +} + +func (r *runtimeLifecycle) computeCapacity(ctx context.Context, key string) error { + policy := r.config.Suspension + if key != r.config.InstallationID { + return sandbox.ErrOwnership + } + if policy == nil { + return nil + } + count, err := r.store.CountRuntimeComputeReservations(ctx, key) + if err != nil { + return err + } + if count >= int64(policy.MaxActive) { + return ErrExecutionUnavailable + } + return nil +} +func (r *runtimeLifecycle) saveCompute(ctx context.Context, owner store.RuntimeAllocation, phase string, state runtimeCompute, until *time.Time) (store.RuntimeAllocation, error) { + raw, err := json.Marshal(state) + if err != nil { + return owner, err + } + idleBefore := time.Time{} + if phase == "quiescing" { + policy := r.config.Suspension + if policy == nil { + return owner, sandbox.ErrInvalid + } + idleBefore = time.Now().Add(-policy.IdleTimeout) + } + return r.store.SetRuntimeCompute(ctx, owner, phase, raw, until, idleBefore) +} +func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner store.RuntimeAllocation) error { + p, ok := r.config.Provider.(sandbox.CheckpointProvider) + if !ok { + return sandbox.ErrInvalid + } + state, err := p.GetCompute(ctx, runtimeReference(owner), p.Initial(runtimeReference(owner))) + if err != nil { + return err + } + if state.Status != "running" || !state.BootstrapComplete || state.Compute.ID == "" { + return sandbox.ErrComputeUnconfirmed + } + _, err = r.saveCompute(ctx, owner, "running", runtimeCompute{Current: state.Compute}, nil) + return err +} + +func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner store.RuntimeAllocation) error { + p, ok := r.config.Provider.(sandbox.CheckpointProvider) + if !ok { + return sandbox.ErrInvalid + } + var state runtimeCompute + if json.Unmarshal(owner.ComputeState, &state) != nil || state.Current.ID == "" { + return sandbox.ErrOwnership + } + if owner.SessionDeleted || owner.Expired || owner.State == "cleanup_pending" { + return r.cleanupCompute(ctx, p, owner, state) + } + if err := r.store.CheckExecutionOwnership(ctx); err != nil { + return err + } + switch owner.ComputePhase { + case "running": + return r.idleCompute(ctx, p, owner, state) + case "quiescing": + // A lost quiesce acknowledgement never authorizes a snapshot. Wake the + // original source and retain its files; no user request is sent again. + state.Rollback = true + next, err := r.saveCompute(ctx, owner, "waking", state, owner.ComputeRetainedUntil) + if err != nil { + return err + } + return r.wakeCompute(ctx, p, next, state) + case "suspending": + return r.captureCompute(ctx, p, owner, state, true) + case "suspended": + return r.restoreIdleCompute(ctx, p, owner, state) + case "restoring": + return r.restoreCompute(ctx, p, owner, state, true) + case "waking": + return r.wakeCompute(ctx, p, owner, state) + default: + return sandbox.ErrInvalid + } +} + +func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute) error { + compute, err := p.GetCompute(ctx, runtimeReference(owner), state.Current) + if err != nil { + return err + } + if compute.Status != "running" || !compute.BootstrapComplete { + return sandbox.ErrComputeUnconfirmed + } + peer, err := authorizedRuntimePeer(ctx, r.store, r.registry, owner.DeviceID) + if err != nil { + return err + } + if _, err := r.store.KeepRuntimeAllocation(ctx, owner); err != nil { + return err + } + activity, err := r.store.RuntimeActivity(ctx, owner) + if err != nil { + return err + } + if activity.WakeRequested { + // Clearing only the observed timestamp cannot consume a newer live request. + return r.store.ClearRuntimeWake(ctx, owner, owner.ComputeActivityAt) + } + policy := r.config.Suspension + if policy == nil || activity.Busy || !activity.HasCompletedTurn || time.Since(activity.LastActivity) < policy.IdleTimeout { + return nil + } + state.SuspendID, state.RestoreID, state.Rollback = uuid.NewString(), "", false + until := time.Now().Add(policy.Retention) + next, err := r.saveCompute(ctx, owner, "quiescing", state, &until) + if err != nil { + return err + } + result, err := peer.SuspendControl(ctx, proto.TypeEnvironmentQuiesce, proto.EnvironmentSuspendPayload{EnvironmentID: owner.EnvironmentID, SuspendID: state.SuspendID}) + if err != nil { + return err + } + if !result.Accepted { + // A rejected request did not park the daemon. Reset its idle clock so a + // continuing file operation is not immediately interrupted by another try. + if err := r.store.TouchRuntimeActivity(ctx, owner.TenantID, owner.EnvironmentID); err != nil { + return err + } + _, err = r.saveCompute(ctx, next, "running", runtimeCompute{Current: state.Current}, nil) + return err + } + // Publish disconnected only after receiving the daemon's receipt barrier. + if err := observeRuntimeConnection(ctx, r.store, r.connections, owner.TenantID, owner.EnvironmentID, nil, false); err != nil { + return err + } + // The Session-locked phase commit checks pending work and wake requests. + // A competing request keeps its queue position and resumes this source. + suspending, err := r.saveCompute(ctx, next, "suspending", state, &until) + if errors.Is(err, store.ErrTurnConflict) { + state.Rollback = true + next, err = r.saveCompute(ctx, next, "waking", state, &until) + if err != nil { + return err + } + return r.wakeCompute(ctx, p, next, state) + } + if err != nil { + return err + } + return r.captureCompute(ctx, p, suspending, state, false) +} + +func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute, observeOnly bool) error { + result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Snapshot: state.Snapshot, ObserveOnly: observeOnly}) + if err != nil { + return err + } + if result.Compute.ID != state.Current.ID { + return sandbox.ErrOwnership + } + if result.Snapshot == nil { + if !observeOnly || result.SourceStopped || (result.Status != "running" && result.Status != "paused") { + return sandbox.ErrComputeUnconfirmed + } + state.Rollback = true + next, err := r.saveCompute(ctx, owner, "waking", state, owner.ComputeRetainedUntil) + if err != nil { + return err + } + return r.wakeCompute(ctx, p, next, state) + } + state.Snapshot = result.Snapshot + // Store the verified artifact before any recovery-path kill. Snapshot failure + // or an unknown result cannot silently fall back to a cold Environment. + next, err := r.saveCompute(ctx, owner, "suspending", state, owner.ComputeRetainedUntil) + if err != nil { + return err + } + if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { + return err + } + _, err = r.saveCompute(ctx, next, "suspended", state, next.ComputeRetainedUntil) + return err +} + +func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute) error { + activity, err := r.store.RuntimeActivity(ctx, owner) + if err != nil { + return err + } + if !activity.Busy && !activity.WakeRequested { + return nil + } + if err := r.computeCapacity(ctx, owner.ProviderKey); err != nil { + return err + } + if state.Snapshot == nil || state.Target != nil { + return sandbox.ErrOwnership + } + target, err := p.NewCompute(runtimeReference(owner), state.Current.Generation+1, state.Snapshot) + if err != nil { + return err + } + state.Target, state.RestoreID = &target, uuid.NewString() + next, err := r.saveCompute(ctx, owner, "restoring", state, owner.ComputeRetainedUntil) + if err != nil { + return err + } + return r.restoreCompute(ctx, p, next, state, false) +} +func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute, observeOnly bool) error { + if state.Target == nil || state.Snapshot == nil || state.Rollback { + return sandbox.ErrOwnership + } + result, err := p.Resume(ctx, sandbox.ResumeRequest{Reference: runtimeReference(owner), OperationID: state.RestoreID, Snapshot: *state.Snapshot, Target: *state.Target, ObserveOnly: observeOnly}) + if err != nil { + return err + } + if result.Status != "running" || result.Compute.ID == "" { + return sandbox.ErrComputeUnconfirmed + } + state.Current, state.Target = result.Compute, nil + // This commit consumes the snapshot before opening daemon admission. Recovery + // from waking can only reconnect this generation; it cannot restore again. + next, err := r.saveCompute(ctx, owner, "waking", state, owner.ComputeRetainedUntil) + if err != nil { + return err + } + return r.wakeCompute(ctx, p, next, state) +} + +func ignoreComputeAbsent(err error) error { + if errors.Is(err, sandbox.ErrNotFound) { + return nil + } + return err +} diff --git a/services/agents-api/internal/execution/runtime_compute_wake.go b/services/agents-api/internal/execution/runtime_compute_wake.go new file mode 100644 index 000000000..e1e9f9712 --- /dev/null +++ b/services/agents-api/internal/execution/runtime_compute_wake.go @@ -0,0 +1,147 @@ +package execution + +import ( + "context" + "errors" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute) error { + if state.Rollback { + if _, err := p.ResumeCompute(ctx, runtimeReference(owner), state.Current); err != nil { + return err + } + } + peer, err := authorizedRuntimePeer(ctx, r.store, r.registry, owner.DeviceID) + if err != nil { + if !errors.Is(err, store.ErrNotFound) && !errors.Is(err, gateway.ErrDeviceNotRegistered) && !errors.Is(err, gateway.ErrSessionClosed) { + return err + } + // This idempotent control signal is fenced by guest PID/start time and the + // suspension token. It cannot execute or replay an agent request. + result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"parsar-daemon", "resume", "--control-file", "/run/parsar/daemon-suspend.json", "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) + if err != nil { + return err + } + if result.ExitCode != 0 { + return sandbox.ErrComputeUnconfirmed + } + timer := time.NewTicker(100 * time.Millisecond) + defer timer.Stop() + for { + peer, err = authorizedRuntimePeer(ctx, r.store, r.registry, owner.DeviceID) + if err == nil { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + } + } + result, err := peer.SuspendControl(ctx, proto.TypeEnvironmentResume, proto.EnvironmentSuspendPayload{EnvironmentID: owner.EnvironmentID, SuspendID: state.SuspendID, Rollback: state.Rollback}) + if err != nil { + return err + } + if !result.Accepted { + return sandbox.ErrComputeUnconfirmed + } + // The artifact has been consumed. Never restore it after this generation + // admits work, even if garbage collection or the final database commit fails. + if state.Snapshot != nil { + if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { + return err + } + } + next, err := r.saveCompute(ctx, owner, "running", runtimeCompute{Current: state.Current}, nil) + if err != nil { + return err + } + if err := r.store.ClearRuntimeWake(ctx, next, owner.ComputeActivityAt); err != nil { + return err + } + return r.observeConnection(ctx, next) +} + +func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.CheckpointProvider, owner store.RuntimeAllocation, state runtimeCompute) error { + if err := r.store.CheckExecutionOwnership(ctx); err != nil { + return err + } + // An uncommitted artifact is found by its persisted attempt, never a directory + // glob. The helper's allocation lock also waits for an earlier unknown call. + if owner.ComputePhase == "suspending" && state.Snapshot == nil { + result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, ObserveOnly: true}) + if err != nil && !errors.Is(err, sandbox.ErrNotFound) { + return err + } + if err == nil { + state.Snapshot = result.Snapshot + } + } + if state.Target != nil { + if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), *state.Target)); err != nil { + return err + } + } + if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { + return err + } + if state.Snapshot != nil { + if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { + return err + } + } + _, err := r.store.ReleaseRuntimeAllocation(ctx, owner) + return err +} + +// waitRuntimeAwake is called only for live Environment file operations, before +// entering the Worker's work queues. Persisted history/artifact reads bypass it. +func (w *Worker) waitRuntimeAwake(ctx context.Context, environment store.Environment) error { + if w.runtimes == nil { + return nil + } + owner, err := w.admission.GetRuntimeAllocation(ctx, environment.TenantID, environment.ID) + if errors.Is(err, store.ErrNotFound) { + return nil + } + if err != nil { + return err + } + if owner.SessionDeleted || owner.Expired || owner.State == "cleanup_pending" || owner.State == "released" { + return ErrExecutionUnavailable + } + if owner.ComputePhase == "disabled" { + return nil + } + if err := w.admission.TouchRuntimeActivity(ctx, environment.TenantID, environment.ID); err != nil { + return err + } + timer := time.NewTicker(100 * time.Millisecond) + defer timer.Stop() + for { + owner, err = w.admission.GetRuntimeAllocation(ctx, environment.TenantID, environment.ID) + if err != nil { + return err + } + if owner.SessionDeleted || owner.Expired || owner.State != "running" { + return ErrExecutionUnavailable + } + if owner.ComputePhase == "running" { + return nil + } + select { + case <-ctx.Done(): + return ErrExecutionUnavailable + case <-w.stopped: + return ErrExecutionUnavailable + case <-timer.C: + } + } +} diff --git a/services/agents-api/internal/execution/runtime_initialization.go b/services/agents-api/internal/execution/runtime_initialization.go index 36f8ee8fc..e90212e65 100644 --- a/services/agents-api/internal/execution/runtime_initialization.go +++ b/services/agents-api/internal/execution/runtime_initialization.go @@ -102,10 +102,10 @@ func (r *runtimeLifecycle) advanceInitialization(ctx context.Context) error { var body []byte file, body, err = r.store.ReadInitialEnvironmentFile(operation, owner.TenantID, owner.SessionID, active.next) if err == nil { - err = installInitialFile(operation, r.config.Providers[owner.ProviderKey], runtimeReference(owner), file, body) + err = installInitialFile(operation, r.config.Provider, runtimeReference(owner), file, body) } } else { - err = runRuntimeSetup(operation, r.config.Providers[owner.ProviderKey], runtimeReference(owner), active.operations[active.next-active.files]) + err = runRuntimeSetup(operation, r.config.Provider, runtimeReference(owner), active.operations[active.next-active.files]) } if err != nil { // Clearing the in-memory owner makes the next observation request cleanup, diff --git a/services/agents-api/internal/execution/runtime_lifecycle.go b/services/agents-api/internal/execution/runtime_lifecycle.go index fb306b9f3..559910fbb 100644 --- a/services/agents-api/internal/execution/runtime_lifecycle.go +++ b/services/agents-api/internal/execution/runtime_lifecycle.go @@ -17,20 +17,21 @@ import ( "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" ) -// RuntimeProviders is trusted operator wiring, not public Environment input. -// Each stable key identifies one provider backend/installation across restarts; -// changing that target requires a new key, preserving the old cleanup adapter. -type RuntimeProviders struct { - CoreURL string - DefaultProvider string - EngineProviders map[string]string - Providers map[string]sandbox.Provider +// RuntimeProvider binds one deployment to one sandbox installation. +// BackendFingerprint identifies its namespace independently of mutable sizing. +type RuntimeProvider struct { + CoreURL string + InstallationID string + BackendFingerprint string + Provider sandbox.Provider + Maintenance bool + Suspension *RuntimeSuspensionPolicy } type runtimeLifecycle struct { store *store.Store registry *gateway.Registry - config RuntimeProviders + config RuntimeProvider gate chan struct{} ctx context.Context stop context.CancelFunc @@ -40,30 +41,26 @@ type runtimeLifecycle struct { initializing *runtimeInitialization } -func newRuntimeLifecycle(s *store.Store, registry *gateway.Registry, config *RuntimeProviders) (*runtimeLifecycle, error) { +func newRuntimeLifecycle(s *store.Store, registry *gateway.Registry, config *RuntimeProvider) (*runtimeLifecycle, error) { if config == nil { return nil, nil } u, err := url.Parse(config.CoreURL) - if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" || u.Fragment != "" || len(config.Providers) == 0 || registry == nil { + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" || u.Fragment != "" || config.Provider == nil || registry == nil { return nil, sandbox.ErrInvalid } - copied := RuntimeProviders{EngineProviders: make(map[string]string, len(config.EngineProviders)), CoreURL: config.CoreURL, DefaultProvider: config.DefaultProvider, Providers: make(map[string]sandbox.Provider, len(config.Providers))} - for key, provider := range config.Providers { - id, err := uuid.Parse(key) - if err != nil || id == uuid.Nil || id.String() != key || provider == nil { - return nil, sandbox.ErrInvalid - } - copied.Providers[key] = provider + id, err := uuid.Parse(config.InstallationID) + fingerprint, fingerprintErr := hex.DecodeString(config.BackendFingerprint) + if err != nil || id == uuid.Nil || id.String() != config.InstallationID || fingerprintErr != nil || len(fingerprint) != 32 || hex.EncodeToString(fingerprint) != config.BackendFingerprint { + return nil, sandbox.ErrInvalid } - for kind, key := range config.EngineProviders { - if key == "" || copied.Providers[key] == nil { + copied := *config + if config.Suspension != nil { + policy := *config.Suspension + if _, ok := config.Provider.(sandbox.CheckpointProvider); !ok || policy.IdleTimeout < time.Second || policy.Retention < time.Second || policy.MaxActive < 1 || policy.MaxRetained < policy.MaxActive { return nil, sandbox.ErrInvalid } - copied.EngineProviders[kind] = key - } - if copied.DefaultProvider != "" && copied.Providers[copied.DefaultProvider] == nil { - return nil, sandbox.ErrInvalid + copied.Suspension = &policy } ctx, stop := context.WithCancel(context.Background()) return &runtimeLifecycle{store: s, registry: registry, config: copied, gate: make(chan struct{}, 1), ctx: ctx, stop: stop, connections: make(map[string]*runtimeConnection)}, nil @@ -107,8 +104,8 @@ func (w *Worker) ProvisionEnvironment(ctx context.Context, tenant, environment, // provision runs under the lifecycle gate and uses the durable one-shot receipt. func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, providerKey string) (store.RuntimeAllocation, error) { - provider := r.config.Providers[providerKey] - if provider == nil { + provider := r.config.Provider + if providerKey != r.config.InstallationID { return store.RuntimeAllocation{}, sandbox.ErrInvalid } environmentValue, err := r.store.GetEnvironment(ctx, tenant, environment) @@ -119,6 +116,25 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p if err != nil || placement.Type != "openai_hosted" { return store.RuntimeAllocation{}, sandbox.ErrInvalid } + if _, err := r.store.GetRuntimeAllocation(ctx, tenant, environment); errors.Is(err, store.ErrNotFound) { + if r.config.Maintenance { + return store.RuntimeAllocation{}, ErrExecutionUnavailable + } + if err := r.computeCapacity(ctx, providerKey); err != nil { + return store.RuntimeAllocation{}, err + } + if policy := r.config.Suspension; policy != nil { + count, err := r.store.CountRuntimeRetainedAllocations(ctx, providerKey) + if err != nil { + return store.RuntimeAllocation{}, err + } + if count >= int64(policy.MaxRetained) { + return store.RuntimeAllocation{}, ErrExecutionUnavailable + } + } + } else if err != nil { + return store.RuntimeAllocation{}, err + } secret := make([]byte, 32) if _, err := rand.Read(secret); err != nil { return store.RuntimeAllocation{}, err @@ -199,6 +215,9 @@ func (w *Worker) ReconcileManagedRuntimes(ctx context.Context) error { } func (r *runtimeLifecycle) observe(ctx context.Context, owner store.RuntimeAllocation) error { + if owner.ProviderKey != r.config.InstallationID { + return sandbox.ErrOwnership + } if owner.Initialization == "running" && (r.initializing == nil || r.initializing.owner.ID != owner.ID) { var err error owner, err = r.store.RequestRuntimeCleanup(ctx, owner) @@ -213,11 +232,16 @@ func (r *runtimeLifecycle) observe(ctx context.Context, owner store.RuntimeAlloc return err } r.clearRuntimeState(owner) - } else if err := r.observeConnection(ctx, owner); err != nil { - return err + } else if owner.ComputePhase == "disabled" || owner.ComputePhase == "running" { + if err := r.observeConnection(ctx, owner); err != nil { + return err + } } - provider := r.config.Providers[owner.ProviderKey] - if provider == nil { + if owner.ComputePhase != "disabled" { + return r.observeCompute(ctx, owner) + } + provider := r.config.Provider + if owner.ProviderKey != r.config.InstallationID { return sandbox.ErrInvalid } if err := r.store.CheckExecutionOwnership(ctx); err != nil { @@ -267,6 +291,9 @@ func (r *runtimeLifecycle) observe(ctx context.Context, owner store.RuntimeAlloc if err := r.observeInitialization(ctx, owner); err != nil { return err } + if r.config.Suspension != nil && owner.Initialization == "complete" { + return r.enableCompute(ctx, owner) + } if peer, err := r.registry.LookupDevice(owner.DeviceID); err != nil || peer.IsClosed() { return nil } diff --git a/services/agents-api/internal/execution/runtime_pending.go b/services/agents-api/internal/execution/runtime_pending.go index b66078333..d148860fa 100644 --- a/services/agents-api/internal/execution/runtime_pending.go +++ b/services/agents-api/internal/execution/runtime_pending.go @@ -10,7 +10,7 @@ import ( // provisionPending shares the existing lifecycle owner and serial gate. This // also recovers idle Session creation interrupted after its database commit. func (r *runtimeLifecycle) provisionPending(ctx context.Context) error { - if r.config.DefaultProvider == "" && len(r.config.EngineProviders) == 0 { + if r.config.Maintenance { return nil } rows, err := r.store.ListUnallocatedHostedEnvironments(ctx, r.pendingCursor) @@ -23,10 +23,7 @@ func (r *runtimeLifecycle) provisionPending(ctx context.Context) error { } for _, environment := range rows { r.pendingCursor = environment.ID - provider := r.config.ProviderForEngine(environment.Engine) - if provider == "" { - continue - } + provider := r.config.InstallationID operation, cancel := context.WithTimeout(ctx, 30*time.Second) _, err := r.provision(operation, environment.TenantID, environment.ID, provider) cancel() diff --git a/services/agents-api/internal/execution/runtime_provider_selection.go b/services/agents-api/internal/execution/runtime_provider_selection.go deleted file mode 100644 index b893f39f4..000000000 --- a/services/agents-api/internal/execution/runtime_provider_selection.go +++ /dev/null @@ -1,8 +0,0 @@ -package execution - -func (r RuntimeProviders) ProviderForEngine(kind string) string { - if len(r.EngineProviders) == 0 { - return r.DefaultProvider - } - return r.EngineProviders[kind] -} diff --git a/services/agents-api/internal/execution/runtime_provider_selection_test.go b/services/agents-api/internal/execution/runtime_provider_selection_test.go deleted file mode 100644 index 6a757c5a1..000000000 --- a/services/agents-api/internal/execution/runtime_provider_selection_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package execution - -import "testing" - -func TestRuntimeProviderSelectionDoesNotFallBack(t *testing.T) { - config := RuntimeProviders{DefaultProvider: "original", EngineProviders: map[string]string{"codex": "codex-image", "mcode": "mcode-image"}} - if config.ProviderForEngine("mcode") != "mcode-image" || config.ProviderForEngine("claude_sdk") != "" { - t.Fatal("selected an unrelated runtime") - } - if (RuntimeProviders{DefaultProvider: "original"}).ProviderForEngine("codex") != "original" { - t.Fatal("changed legacy default") - } -} diff --git a/services/agents-api/internal/execution/worker.go b/services/agents-api/internal/execution/worker.go index 903ecf92e..ed087264e 100644 --- a/services/agents-api/internal/execution/worker.go +++ b/services/agents-api/internal/execution/worker.go @@ -37,6 +37,18 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher) (*Worker, error) { _ = lease.Close(context.Background()) return nil, err } + var deployment *store.RuntimeDeployment + if worker.runtimes != nil { + config := worker.runtimes.config + deployment = &store.RuntimeDeployment{InstallationID: config.InstallationID, BackendFingerprint: config.BackendFingerprint, Maintenance: config.Maintenance} + } + if err := owned.Store.ConfigureRuntimeDeployment(ctx, deployment); err != nil { + if worker.runtimes != nil { + worker.runtimes.stop() + } + _ = lease.Close(context.Background()) + return nil, err + } if err := owned.Store.ReconcileEnvironmentConnections(ctx); err != nil { if worker.runtimes != nil { worker.runtimes.stop() diff --git a/services/agents-api/internal/execution/worker_device.go b/services/agents-api/internal/execution/worker_device.go index d8f26cada..3bd43d538 100644 --- a/services/agents-api/internal/execution/worker_device.go +++ b/services/agents-api/internal/execution/worker_device.go @@ -80,6 +80,18 @@ func (w *Worker) bindSessionDevice(ctx context.Context, session store.Session, r if _, err := parseEnvironmentPlacement(environment.Configuration); err != nil { return false, nil } + allocation, err := w.dispatcher.Store.GetRuntimeAllocation(ctx, session.TenantID, environment.ID) + if errors.Is(err, store.ErrNotFound) { + if snapshot.Environment.Type == "openai_hosted" { + return false, nil + } + } else if err != nil { + return false, err + } else if allocation.ComputePhase != "disabled" && allocation.ComputePhase != "running" { + // A reconnect authenticates transport before the retained Environment + // resumes. Its first control frame must remain the lifecycle's Resume. + return false, nil + } bound, err := w.dispatcher.Store.GetSessionDevice(ctx, session.TenantID, session.ID) if errors.Is(err, store.ErrNotFound) { return false, nil diff --git a/services/agents-api/internal/sandbox/microsandbox/identity.go b/services/agents-api/internal/sandbox/microsandbox/identity.go new file mode 100644 index 000000000..dd3e89b4b --- /dev/null +++ b/services/agents-api/internal/sandbox/microsandbox/identity.go @@ -0,0 +1,138 @@ +package microsandbox + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "net/url" + "path/filepath" + "strings" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentnetwork" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/google/uuid" +) + +func validID(s string) bool { + v, e := uuid.Parse(s) + return e == nil && v != uuid.Nil && v.String() == s +} +func ValidReference(r sandbox.Reference) bool { + return validID(r.TenantID) && validID(r.EnvironmentID) && validID(r.AllocationID) +} +func validHash(s string) bool { + b, e := hex.DecodeString(s) + return e == nil && len(b) == 32 && strings.ToLower(s) == s +} +func (c Config) Validate() error { + if !validID(c.InstallationID) || !filepath.IsAbs(c.HelperPath) || !filepath.IsAbs(c.RuntimeHome) || !filepath.IsAbs(c.RuntimePath) || !filepath.IsAbs(c.FirmwarePath) || !validHash(c.RuntimeSHA256) || !validHash(c.FirmwareSHA256) || c.MemoryMiB == 0 || c.CPUs == 0 || c.RootDiskMiB == 0 { + return sandbox.ErrInvalid + } + imageName, digest, pinned := strings.Cut(c.Image, "@sha256:") + if !pinned || imageName == "" || strings.Contains(imageName, "@") || !validHash(digest) { + return sandbox.ErrInvalid + } + if c.Network.DefaultEgress != "allow" && c.Network.DefaultEgress != "deny" { + return sandbox.ErrInvalid + } + if c.Network.DefaultIngress != "allow" && c.Network.DefaultIngress != "deny" { + return sandbox.ErrInvalid + } + for _, r := range c.Network.Rules { + if (r.Action != "allow" && r.Action != "deny") || (r.Direction != "egress" && r.Direction != "ingress") || r.Destination == "" { + return sandbox.ErrInvalid + } + } + return nil +} +func Name(c Config, r sandbox.Reference, g uint64) string { + h := sha256.Sum256([]byte(c.InstallationID + ":" + r.TenantID + ":" + r.EnvironmentID + ":" + r.AllocationID)) + return fmt.Sprintf("parsar-%x-g%d", h[:16], g) +} +func SnapshotReference(c Config, r sandbox.Reference, operation string) string { + return Name(c, r, 0) + ":s-" + operation +} +func Labels(c Config, r sandbox.Reference) map[string]string { + return map[string]string{"io.parsar.installation": c.InstallationID, "io.parsar.tenant": r.TenantID, "io.parsar.environment": r.EnvironmentID, "io.parsar.allocation": r.AllocationID} +} +func ValidateCompute(c Config, r sandbox.Reference, v Compute) error { + if !ValidReference(r) || v.Name != Name(c, r, v.Generation) { + return sandbox.ErrInvalid + } + if v.Generation == 0 { + if v.RestoredFrom != nil { + return sandbox.ErrInvalid + } + } else { + if v.RestoredFrom == nil || ValidateSnapshot(c, r, *v.RestoredFrom) != nil || v.RestoredFrom.SourceGeneration >= v.Generation { + return sandbox.ErrInvalid + } + } + return nil +} +func ValidateSnapshot(c Config, r sandbox.Reference, s SnapshotIdentity) error { + if !ValidReference(r) || !validID(s.OperationID) || s.Reference != SnapshotReference(c, r, s.OperationID) || s.SourceName != Name(c, r, s.SourceGeneration) || s.SourceID == "" || s.ID == "" || s.Digest == "" || s.CheckpointID == "" || s.CheckpointRoot == "" { + return sandbox.ErrInvalid + } + return nil +} +func ValidateBootstrap(b sandbox.Bootstrap) error { + u, e := url.Parse(b.CoreURL) + if !ValidReference(b.Reference) || !validID(b.SessionID) || !validID(b.DeviceID) || e != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" || u.Fragment != "" || strings.TrimSpace(b.Credential) == "" { + return sandbox.ErrInvalid + } + return (agentnetwork.Policy{Access: b.NetworkAccess, AllowedDomains: b.AllowedDomains}).Validate() +} +func ValidateCommand(c sandbox.Command) error { + if len(c.Args) == 0 || c.Args[0] == "" || len(c.Stdin) > sandbox.MaxCommandInputBytes || (c.Directory != "" && !filepath.IsAbs(c.Directory)) { + return sandbox.ErrInvalid + } + for _, a := range c.Args { + if strings.IndexByte(a, 0) >= 0 { + return sandbox.ErrInvalid + } + } + return nil +} +func ValidateRequest(q Request) error { + if q.Version != ProtocolVersion || q.Config.Validate() != nil || !ValidReference(q.Reference) || q.Deadline.IsZero() { + return sandbox.ErrInvalid + } + switch q.Operation { + case "create": + if q.Bootstrap == nil || q.Bootstrap.Reference != q.Reference || ValidateBootstrap(*q.Bootstrap) != nil { + return sandbox.ErrInvalid + } + case "inspect", "kill", "resume_compute": + if q.Operation == "resume_compute" && q.Compute.ID == "" { + return sandbox.ErrInvalid + } + return ValidateCompute(q.Config, q.Reference, q.Compute) + case "command": + if q.Command == nil || ValidateCommand(*q.Command) != nil { + return sandbox.ErrInvalid + } + return ValidateCompute(q.Config, q.Reference, q.Compute) + case "suspend": + s := q.Suspend + if s == nil || s.Reference != q.Reference || !validID(s.OperationID) || s.Source.ID == "" || ValidateCompute(q.Config, q.Reference, s.Source) != nil { + return sandbox.ErrInvalid + } + if s.Snapshot != nil && (ValidateSnapshot(q.Config, q.Reference, *s.Snapshot) != nil || s.Snapshot.OperationID != s.OperationID || s.Snapshot.SourceID != s.Source.ID || s.Snapshot.SourceName != s.Source.Name || s.Snapshot.SourceGeneration != s.Source.Generation) { + return sandbox.ErrInvalid + } + case "resume": + v := q.Resume + if v == nil || v.Reference != q.Reference || !validID(v.OperationID) || ValidateSnapshot(q.Config, q.Reference, v.Snapshot) != nil || ValidateCompute(q.Config, q.Reference, v.Target) != nil || v.Target.RestoredFrom == nil || *v.Target.RestoredFrom != v.Snapshot { + return sandbox.ErrInvalid + } + case "delete_snapshot": + if q.Snapshot == nil { + return sandbox.ErrInvalid + } + return ValidateSnapshot(q.Config, q.Reference, *q.Snapshot) + default: + return sandbox.ErrInvalid + } + return nil +} diff --git a/services/agents-api/internal/sandbox/microsandbox/process.go b/services/agents-api/internal/sandbox/microsandbox/process.go new file mode 100644 index 000000000..54ccba087 --- /dev/null +++ b/services/agents-api/internal/sandbox/microsandbox/process.go @@ -0,0 +1,84 @@ +package microsandbox + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "strings" +) + +// ProcessCaller never kills a mutating helper on a Core response timeout. +// The helper retains its allocation flock until the SDK mutation settles, +// including after Core exits. Output is still drained and bounded by this waiter. +type ProcessCaller struct{} + +func (*ProcessCaller) Call(ctx context.Context, q Request) (Response, error) { + data, e := json.Marshal(q) + if e != nil || len(data) > MaxRequestBytes { + return Response{}, errors.New("invalid helper request") + } + cmd := exec.Command(q.Config.HelperPath) + cmd.Stdin = bytes.NewReader(data) + cmd.Env = HelperEnvironment(os.Environ(), q.Config) + stdout := &limitBuffer{limit: MaxResponseBytes} + stderr := &limitBuffer{limit: MaxOutputBytes} + cmd.Stdout, cmd.Stderr = stdout, stderr + if e := cmd.Start(); e != nil { + return Response{}, errors.New("helper unavailable") + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case <-ctx.Done(): + return Response{}, ctx.Err() + case err := <-done: + if err != nil || stdout.exceeded || stderr.exceeded { + return Response{}, errors.New("helper result unconfirmed") + } + } + var out Response + decoder := json.NewDecoder(bytes.NewReader(stdout.Bytes())) + decoder.DisallowUnknownFields() + if e := decoder.Decode(&out); e != nil { + return Response{}, errors.New("invalid helper response") + } + var extra any + if e := decoder.Decode(&extra); e != io.EOF { + return Response{}, errors.New("trailing helper response") + } + return out, nil +} + +// Strip native selector and credential-bearing inherited configuration. Operator +// proxy settings are not passed into guest environments by this adapter. +func HelperEnvironment(env []string, c Config) []string { + out := []string{} + for _, v := range env { + key, _, _ := strings.Cut(v, "=") + if key == "HOME" || key == "PATH" || key == "TMPDIR" || key == "LANG" || key == "SSL_CERT_FILE" || key == "SSL_CERT_DIR" { + out = append(out, v) + } + } + return append(out, "MSB_HOME="+c.RuntimeHome, "MSB_PATH="+c.RuntimePath, "MSB_LIBKRUNFW_PATH="+c.FirmwarePath, "MSB_BACKEND=local", "RUST_LOG=off", "NO_COLOR=1") +} + +type limitBuffer struct { + bytes.Buffer + limit int + exceeded bool +} + +func (b *limitBuffer) Write(v []byte) (int, error) { + n := len(v) + left := b.limit - b.Len() + if n > left { + b.exceeded = true + v = v[:left] + } + _, _ = b.Buffer.Write(v) + return n, nil // Drain excess output without unbounded retention or pipe deadlock. +} diff --git a/services/agents-api/internal/sandbox/microsandbox/process_test.go b/services/agents-api/internal/sandbox/microsandbox/process_test.go new file mode 100644 index 000000000..4da81b2ac --- /dev/null +++ b/services/agents-api/internal/sandbox/microsandbox/process_test.go @@ -0,0 +1,82 @@ +package microsandbox + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestMain(m *testing.M) { + if filepath.Base(os.Args[0]) == "microsandbox-test-helper" { + var q Request + if json.NewDecoder(os.Stdin).Decode(&q) != nil { + os.Exit(2) + } + switch q.Operation { + case "delayed": + time.Sleep(150 * time.Millisecond) + if os.WriteFile(q.Config.RuntimePath, []byte("settled"), 0600) != nil { + os.Exit(3) + } + case "oversize": + fmt.Print(strings.Repeat("x", MaxResponseBytes+1)) + os.Exit(0) + case "trailing": + fmt.Print("{\"Version\":1}{}") + os.Exit(0) + } + _ = json.NewEncoder(os.Stdout).Encode(Response{Version: ProtocolVersion}) + os.Exit(0) + } + os.Exit(m.Run()) +} +func processConfig(t *testing.T) Config { + t.Helper() + c := testConfig() + directory := t.TempDir() + executable, e := os.Executable() + if e != nil { + t.Fatal(e) + } + c.HelperPath = filepath.Join(directory, "microsandbox-test-helper") + if e = os.Symlink(executable, c.HelperPath); e != nil { + t.Fatal(e) + } + c.RuntimePath = filepath.Join(directory, "settled") + return c +} +func TestResponseTimeoutDoesNotKillLifecycleOwner(t *testing.T) { + c := processConfig(t) + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond) + defer cancel() + _, e := (&ProcessCaller{}).Call(ctx, Request{Operation: "delayed", Config: c}) + if !errors.Is(e, context.DeadlineExceeded) { + t.Fatalf("want timeout: %v", e) + } + until := time.Now().Add(3 * time.Second) + for time.Now().Before(until) { + if data, e := os.ReadFile(c.RuntimePath); e == nil && string(data) == "settled" { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("helper was killed before settlement") +} +func TestInvalidOrUnboundedHelperOutputIsUnconfirmed(t *testing.T) { + for _, mode := range []string{"oversize", "trailing"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _, e := (&ProcessCaller{}).Call(ctx, Request{Operation: mode, Config: processConfig(t)}) + if e == nil { + t.Fatal("invalid helper output accepted") + } + }) + } +} diff --git a/services/agents-api/internal/sandbox/microsandbox/provider.go b/services/agents-api/internal/sandbox/microsandbox/provider.go new file mode 100644 index 000000000..3b74f72cd --- /dev/null +++ b/services/agents-api/internal/sandbox/microsandbox/provider.go @@ -0,0 +1,165 @@ +package microsandbox + +import ( + "context" + "errors" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" +) + +var ErrUnconfirmed = sandbox.ErrComputeUnconfirmed + +type Provider struct { + config Config + caller Caller +} + +var _ sandbox.Provider = (*Provider)(nil) +var _ sandbox.CheckpointProvider = (*Provider)(nil) + +func New(c Config) (*Provider, error) { return NewWithCaller(c, &ProcessCaller{}) } +func NewWithCaller(c Config, caller Caller) (*Provider, error) { + if c.Validate() != nil || caller == nil { + return nil, sandbox.ErrInvalid + } + c.Network.Rules = append([]NetworkRule(nil), c.Network.Rules...) + return &Provider{config: c, caller: caller}, nil +} +func (p *Provider) Initial(r sandbox.Reference) Compute { return Compute{Name: Name(p.config, r, 0)} } +func (p *Provider) call(ctx context.Context, q Request) (Response, error) { + deadline, ok := ctx.Deadline() + if !ok { + return Response{}, sandbox.ErrInvalid + } + q.Version, q.Config, q.Deadline = ProtocolVersion, p.config, deadline + if err := ValidateRequest(q); err != nil { + return Response{}, err + } + out, err := p.caller.Call(ctx, q) + if err != nil { + if q.Operation == "command" { + return out, errors.Join(sandbox.ErrCommandUnconfirmed, err) + } + return out, errors.Join(ErrUnconfirmed, err) + } + if out.Version != ProtocolVersion { + return out, ErrUnconfirmed + } + switch out.ErrorCode { + case "": + return out, nil + case "invalid": + return out, sandbox.ErrInvalid + case "ownership": + return out, sandbox.ErrOwnership + case "exists": + return out, sandbox.ErrExists + case "not_found": + return out, sandbox.ErrNotFound + case "command_unconfirmed": + return out, sandbox.ErrCommandUnconfirmed + default: + return out, ErrUnconfirmed + } +} +func (p *Provider) state(ctx context.Context, q Request) (State, error) { + out, e := p.call(ctx, q) + if e != nil { + return State{}, e + } + if out.State == nil || ValidateCompute(p.config, q.Reference, out.State.Compute) != nil || out.State.Compute.ID == "" { + return State{}, ErrUnconfirmed + } + want := q.Compute + if q.Operation == "create" { + want = p.Initial(q.Reference) + } + if q.Operation == "suspend" { + want = q.Suspend.Source + } + if q.Operation == "resume" { + want = q.Resume.Target + } + got := out.State.Compute + if (got.RestoredFrom == nil) != (want.RestoredFrom == nil) || (want.RestoredFrom != nil && *got.RestoredFrom != *want.RestoredFrom) { + return State{}, sandbox.ErrOwnership + } + if got.Name != want.Name || got.Generation != want.Generation || (want.ID != "" && got.ID != want.ID) { + return State{}, sandbox.ErrOwnership + } + if q.Operation == "suspend" { + s := out.State.Snapshot + if s == nil && q.Suspend.ObserveOnly && out.State.BootstrapComplete && !out.State.SourceStopped && (out.State.Status == "running" || out.State.Status == "paused") { + return *out.State, nil + } + if s == nil || ValidateSnapshot(p.config, q.Reference, *s) != nil || s.OperationID != q.Suspend.OperationID || s.SourceID != want.ID || (!q.Suspend.ObserveOnly && (!out.State.SourceStopped || out.State.Status != "suspended")) { + return State{}, ErrUnconfirmed + } + } + return *out.State, nil +} +func info(r sandbox.Reference, s State) sandbox.Info { + return sandbox.Info{Reference: r, ProviderID: s.Compute.ID, State: s.Status, BootstrapComplete: s.BootstrapComplete} +} +func (p *Provider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { + s, e := p.state(ctx, Request{Operation: "create", Reference: b.Reference, Bootstrap: &b}) + return info(b.Reference, s), e +} +func (p *Provider) GetInfo(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { + s, e := p.GetCompute(ctx, r, p.Initial(r)) + return info(r, s), e +} +func (p *Provider) Renew(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { + return p.GetInfo(ctx, r) +} +func (p *Provider) Kill(ctx context.Context, r sandbox.Reference) error { + return p.KillCompute(ctx, r, p.Initial(r)) +} +func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { + return p.RunCommandCompute(ctx, r, p.Initial(r), c) +} +func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { + return p.state(ctx, Request{Operation: "inspect", Reference: r, Compute: c}) +} +func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c Compute) error { + _, e := p.call(ctx, Request{Operation: "kill", Reference: r, Compute: c}) + return e +} +func (p *Provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s SnapshotIdentity) error { + _, e := p.call(ctx, Request{Operation: "delete_snapshot", Reference: r, Snapshot: &s}) + return e +} +func (p *Provider) Suspend(ctx context.Context, q SuspendRequest) (State, error) { + return p.state(ctx, Request{Operation: "suspend", Reference: q.Reference, Suspend: &q}) +} +func (p *Provider) Resume(ctx context.Context, q ResumeRequest) (State, error) { + return p.state(ctx, Request{Operation: "resume", Reference: q.Reference, Resume: &q}) +} +func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c Compute, command sandbox.Command) (sandbox.CommandResult, error) { + out, e := p.call(ctx, Request{Operation: "command", Reference: r, Compute: c, Command: &command}) + if e != nil { + return sandbox.CommandResult{}, e + } + if out.Command == nil || len(out.Command.Stdout) > MaxOutputBytes || len(out.Command.Stderr) > MaxOutputBytes { + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + } + return *out.Command, nil +} + +// ResumeCompute thaws the exact resident source after an aborted suspension. +// It never starts stopped compute or restores a checkpoint. +func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { + return p.state(ctx, Request{Operation: "resume_compute", Reference: r, Compute: c}) +} + +func (p *Provider) NewCompute(r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { + c := Compute{Generation: generation, Name: Name(p.config, r, generation)} + if snapshot != nil { + value := *snapshot + c.RestoredFrom = &value + } + if e := ValidateCompute(p.config, r, c); e != nil { + return Compute{}, e + } + return c, nil +} diff --git a/services/agents-api/internal/sandbox/microsandbox/provider_test.go b/services/agents-api/internal/sandbox/microsandbox/provider_test.go new file mode 100644 index 000000000..046aa2d2d --- /dev/null +++ b/services/agents-api/internal/sandbox/microsandbox/provider_test.go @@ -0,0 +1,164 @@ +package microsandbox + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" +) + +type callerFunc func(context.Context, Request) (Response, error) + +func (f callerFunc) Call(c context.Context, q Request) (Response, error) { return f(c, q) } +func testConfig() Config { + return Config{ + InstallationID: "11111111-1111-4111-8111-111111111111", HelperPath: "/helper", RuntimeHome: "/private/msb", RuntimePath: "/private/bin/msb", FirmwarePath: "/private/lib/libkrunfw.so", + RuntimeSHA256: strings.Repeat("a", 64), FirmwareSHA256: strings.Repeat("b", 64), Image: "registry/runtime@sha256:" + strings.Repeat("c", 64), + MemoryMiB: 2048, CPUs: 2, RootDiskMiB: 4096, Network: NetworkPolicy{DefaultEgress: "deny", DefaultIngress: "deny", Rules: []NetworkRule{{Action: "allow", Direction: "egress", Destination: "host"}}}, + } +} +func testRef() sandbox.Reference { + return sandbox.Reference{TenantID: "22222222-2222-4222-8222-222222222222", EnvironmentID: "33333333-3333-4333-8333-333333333333", AllocationID: "44444444-4444-4444-8444-444444444444"} +} +func testSnapshot() SnapshotIdentity { + c, r := testConfig(), testRef() + op := "55555555-5555-4555-8555-555555555555" + return SnapshotIdentity{Reference: SnapshotReference(c, r, op), ID: "snap_exact", Digest: "digest", CheckpointID: "checkpoint", CheckpointRoot: "root", OperationID: op, SourceName: Name(c, r, 0), SourceID: "local:4"} +} +func deadline(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + t.Cleanup(cancel) + return ctx +} +func TestRejectsChangedComputeIdentity(t *testing.T) { + c, r := testConfig(), testRef() + p, e := NewWithCaller(c, callerFunc(func(_ context.Context, q Request) (Response, error) { + got := q.Compute + got.ID = "local:foreign" + return Response{Version: ProtocolVersion, State: &State{Compute: got, Status: "running", BootstrapComplete: true}}, nil + })) + if e != nil { + t.Fatal(e) + } + _, e = p.GetCompute(deadline(t), r, Compute{Name: Name(c, r, 0), ID: "local:4"}) + if !errors.Is(e, sandbox.ErrOwnership) { + t.Fatalf("foreign identity accepted: %v", e) + } +} +func TestRestoreMustRetainExactProvenance(t *testing.T) { + c, r, s := testConfig(), testRef(), testSnapshot() + target := Compute{Generation: 1, Name: Name(c, r, 1), RestoredFrom: &s} + p, _ := NewWithCaller(c, callerFunc(func(_ context.Context, q Request) (Response, error) { + v := *q.Resume.Target.RestoredFrom + v.ID = "snap_foreign" + got := q.Resume.Target + got.ID = "local:5" + got.RestoredFrom = &v + return Response{Version: ProtocolVersion, State: &State{Compute: got, Status: "running", BootstrapComplete: true}}, nil + })) + _, e := p.Resume(deadline(t), ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: target}) + if !errors.Is(e, sandbox.ErrOwnership) { + t.Fatalf("different snapshot accepted: %v", e) + } +} +func TestRejectsSnapshotPathBeforeHelper(t *testing.T) { + c, r, s := testConfig(), testRef(), testSnapshot() + s.Reference = "/foreign/checkpoint" + calls := 0 + p, _ := NewWithCaller(c, callerFunc(func(context.Context, Request) (Response, error) { calls++; return Response{}, nil })) + if e := p.DeleteSnapshot(deadline(t), r, s); !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { + t.Fatalf("e=%v calls=%d", e, calls) + } +} +func TestObserveOnlyPreservesCapturedButResidentState(t *testing.T) { + c, r, s := testConfig(), testRef(), testSnapshot() + source := Compute{Name: Name(c, r, 0), ID: s.SourceID} + p, _ := NewWithCaller(c, callerFunc(func(_ context.Context, q Request) (Response, error) { + if q.Suspend == nil || !q.Suspend.ObserveOnly { + t.Fatal("observation became mutation") + } + return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: "paused", Snapshot: &s}}, nil + })) + got, e := p.Suspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) + if e != nil || got.SourceStopped || got.Status != "paused" { + t.Fatalf("state=%+v error=%v", got, e) + } +} +func TestCommandUncertaintyAndResultLimits(t *testing.T) { + for _, tc := range []struct { + name string + response Response + failure error + }{ + {"transport", Response{}, errors.New("lost result")}, + {"missing", Response{Version: ProtocolVersion}, nil}, + {"too_large", Response{Version: ProtocolVersion, Command: &sandbox.CommandResult{Stdout: strings.Repeat("x", MaxOutputBytes+1)}}, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + p, _ := NewWithCaller(testConfig(), callerFunc(func(context.Context, Request) (Response, error) { return tc.response, tc.failure })) + _, e := p.RunCommand(deadline(t), testRef(), sandbox.Command{Args: []string{"/bin/true"}}) + if !errors.Is(e, sandbox.ErrCommandUnconfirmed) { + t.Fatalf("uncertainty lost: %v", e) + } + }) + } +} +func TestNoDeadlineOrForeignAllocationNeverCallsHelper(t *testing.T) { + calls := 0 + p, _ := NewWithCaller(testConfig(), callerFunc(func(context.Context, Request) (Response, error) { calls++; return Response{}, nil })) + _, e := p.GetInfo(context.Background(), testRef()) + if !errors.Is(e, sandbox.ErrInvalid) { + t.Fatal(e) + } + r := testRef() + foreign := r + foreign.EnvironmentID = "77777777-7777-4777-8777-777777777777" + _, e = p.GetCompute(deadline(t), foreign, p.Initial(r)) + if !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { + t.Fatalf("e=%v calls=%d", e, calls) + } +} +func TestHelperEnvironmentDropsCredentialsAndBackendOverrides(t *testing.T) { + got := strings.Join(HelperEnvironment([]string{"HOME=/home/test", "PATH=/bin", "MSB_BACKEND=cloud", "MSB_CONFIG_PATH=/foreign", "MICROSANDBOX_FFI_PATH=/foreign.so", "OPENAI_API_KEY=secret", "HTTPS_PROXY=secret"}, testConfig()), "\n") + for _, bad := range []string{"secret", "cloud", "foreign"} { + if strings.Contains(got, bad) { + t.Fatalf("ambient override retained: %s", got) + } + } + if !strings.Contains(got, "MSB_BACKEND=local") { + t.Fatal("missing local backend") + } +} + +func TestMissingSnapshotObservationAllowsOnlyIntactSourceRollback(t *testing.T) { + for _, status := range []string{"running", "paused", "stopped", "draining", "unknown"} { + t.Run(status, func(t *testing.T) { + c, r, s := testConfig(), testRef(), testSnapshot() + source := Compute{Name: Name(c, r, 0), ID: s.SourceID} + p, _ := NewWithCaller(c, callerFunc(func(context.Context, Request) (Response, error) { + return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: status, BootstrapComplete: true}}, nil + })) + _, e := p.Suspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) + if status == "running" || status == "paused" { + if e != nil { + t.Fatal(e) + } + } else if e == nil { + t.Fatal("unrecoverable source accepted") + } + }) + } +} +func TestImageMustUseCompletePinnedOCIManifestDigest(t *testing.T) { + for _, image := range []string{"image:latest", "image@sha256:garbage", "sha256:" + strings.Repeat("a", 64), "image@sha256:" + strings.Repeat("A", 64)} { + c := testConfig() + c.Image = image + if c.Validate() == nil { + t.Fatalf("unqualified image admitted: %s", image) + } + } +} diff --git a/services/agents-api/internal/sandbox/microsandbox/types.go b/services/agents-api/internal/sandbox/microsandbox/types.go new file mode 100644 index 000000000..5a1b7a3f7 --- /dev/null +++ b/services/agents-api/internal/sandbox/microsandbox/types.go @@ -0,0 +1,73 @@ +// Package microsandbox is the pure-Go client for the colocated SDK helper. +// Core owns all durable state; the helper owns no database or background service. +package microsandbox + +import ( + "context" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" +) + +const ProtocolVersion = 1 +const SDKVersion = "v0.7.2" +const MaxOutputBytes = 1024 * 1024 +const MaxRequestBytes = 72 * 1024 * 1024 +const MaxResponseBytes = 16 * 1024 * 1024 + +// Config is trusted deployment configuration. Paths and hashes refer to one +// immutable, qualified installation. Network is explicitly used on create and restore. +type Config struct { + InstallationID string + HelperPath string + RuntimeHome string + RuntimePath string + FirmwarePath string + RuntimeSHA256 string + FirmwareSHA256 string + Image string + MemoryMiB uint32 + CPUs uint8 + RootDiskMiB uint32 + Network NetworkPolicy +} + +type NetworkPolicy struct { + DefaultEgress string + DefaultIngress string + Rules []NetworkRule +} +type NetworkRule struct{ Action, Direction, Destination, Protocol, Port string } + +// These aliases keep the helper wire private while Core uses provider-neutral types. +type Compute = sandbox.Compute +type SnapshotIdentity = sandbox.SnapshotIdentity +type State = sandbox.ComputeState +type SuspendRequest = sandbox.SuspendRequest +type ResumeRequest = sandbox.ResumeRequest + +// Request and Response are the finite, private helper boundary. Confidential +// Bootstrap and Command bytes travel only through stdin and are never logged. +type Request struct { + Version int + Operation string + Config Config + Reference sandbox.Reference + Compute Compute + Bootstrap *sandbox.Bootstrap + Command *sandbox.Command + Suspend *SuspendRequest + Resume *ResumeRequest + Snapshot *SnapshotIdentity + Deadline time.Time +} +type Response struct { + Version int + State *State + Command *sandbox.CommandResult + ErrorCode string +} + +type Caller interface { + Call(context.Context, Request) (Response, error) +} diff --git a/services/agents-api/internal/sandbox/suspension.go b/services/agents-api/internal/sandbox/suspension.go new file mode 100644 index 000000000..e30a2e3bd --- /dev/null +++ b/services/agents-api/internal/sandbox/suspension.go @@ -0,0 +1,74 @@ +package sandbox + +import ( + "context" + "errors" +) + +// ErrComputeUnconfirmed requires observation of the retained operation identity; +// it does not authorize another Create, capture, restore, or cold start. +var ErrComputeUnconfirmed = errors.New("sandbox lifecycle outcome unconfirmed") + +// Compute identifies one incarnation of an allocation. Name is provider-derived. +// ID is empty only until the original create or restore result is observed. +type Compute struct { + Generation uint64 + Name string + ID string + RestoredFrom *SnapshotIdentity +} + +// SnapshotIdentity is provider evidence from a verified full snapshot. Core +// persists it unchanged and records consumption separately; it never invents +// paths, checksums, native checkpoint fields, or source identity. +type SnapshotIdentity struct { + Reference string + ID string + Digest string + CheckpointID string + CheckpointRoot string + OperationID string + SourceGeneration uint64 + SourceName string + SourceID string +} + +type ComputeState struct { + Compute Compute + Status string + BootstrapComplete bool + Snapshot *SnapshotIdentity + SourceStopped bool +} +type SuspendRequest struct { + Reference Reference + OperationID string + Source Compute + Snapshot *SnapshotIdentity + // Recovery observes the previous attempt and never starts a new capture. + ObserveOnly bool +} +type ResumeRequest struct { + Reference Reference + OperationID string + Snapshot SnapshotIdentity + Target Compute + // Recovery observes the previous target and never starts a new restore. + ObserveOnly bool +} + +// CheckpointProvider is optional. It supplements the existing provider with +// exact-incarnation operations; Worker and Store remain the lifecycle owner. +type CheckpointProvider interface { + Provider + Initial(Reference) Compute + NewCompute(Reference, uint64, *SnapshotIdentity) (Compute, error) + GetCompute(context.Context, Reference, Compute) (ComputeState, error) + Suspend(context.Context, SuspendRequest) (ComputeState, error) + Resume(context.Context, ResumeRequest) (ComputeState, error) + KillCompute(context.Context, Reference, Compute) error + DeleteSnapshot(context.Context, Reference, SnapshotIdentity) error + RunCommandCompute(context.Context, Reference, Compute, Command) (CommandResult, error) + // ResumeCompute thaws only the same resident instance after an aborted pause. + ResumeCompute(context.Context, Reference, Compute) (ComputeState, error) +} diff --git a/services/agents-api/internal/store/environment_file_writes.go b/services/agents-api/internal/store/environment_file_writes.go index 2c56a96f9..b3a305b00 100644 --- a/services/agents-api/internal/store/environment_file_writes.go +++ b/services/agents-api/internal/store/environment_file_writes.go @@ -85,6 +85,9 @@ func (s *Store) ReserveEnvironmentFileWrite(ctx context.Context, tenant, environ if uuid.UUID(device.ID.Bytes).String() != key.DeviceID || device.EnvironmentID != lookup.EnvironmentID { return ErrDeviceBindingConflict } + if err := checkRuntimeComputeAdmission(ctx, q, session); err != nil { + return err + } if err := environmentInputMayStart(ctx, q, session); err != nil { return err } diff --git a/services/agents-api/internal/store/local_environment_worker_test.go b/services/agents-api/internal/store/local_environment_worker_test.go index dee150d24..948a8fc95 100644 --- a/services/agents-api/internal/store/local_environment_worker_test.go +++ b/services/agents-api/internal/store/local_environment_worker_test.go @@ -16,6 +16,10 @@ import ( func localWorker(t *testing.T, scoped, execute bool) (*dispatchHarness, *execution.Worker, store.Environment) { t.Helper() h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}`), scoped) + if scoped { + _, pool := store.NewTestStore(t) + insertWorkerRuntimeAllocation(t, pool, h, "disabled") + } environment, err := h.s.GetSessionEnvironment(t.Context(), h.tenant, h.session.ID) if err != nil { t.Fatal(err) diff --git a/services/agents-api/internal/store/managed_test_store_test.go b/services/agents-api/internal/store/managed_test_store_test.go new file mode 100644 index 000000000..bfb2b8c7e --- /dev/null +++ b/services/agents-api/internal/store/managed_test_store_test.go @@ -0,0 +1,58 @@ +package store + +import ( + "context" + "database/sql" + "os" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" +) + +// Deployment identity belongs to a whole database, so managed fixtures cannot +// share the ordinary Store fixture database or bypass production startup checks. +func newManagedTestStore(t *testing.T) (*Store, *pgxpool.Pool) { + t.Helper() + _, admin := testStore(t) + name := "parsar_agents_api_m_" + uuid.NewString()[:8] + "_tests" + quoted := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(t.Context(), "CREATE DATABASE "+quoted); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := admin.Exec(ctx, "DROP DATABASE "+quoted+" WITH (FORCE)"); err != nil { + t.Error(err) + } + }) + cfg := admin.Config().Copy() + cfg.ConnConfig.Database = name + db := sql.OpenDB(stdlib.GetConnector(*cfg.ConnConfig)) + provider, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) + if err != nil { + _ = db.Close() + t.Fatal(err) + } + _, err = provider.Up(t.Context()) + _ = db.Close() + if err != nil { + t.Fatal(err) + } + pool, err := pgxpool.NewWithConfig(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + return New(pool), pool +} + +func NewManagedTestStore(t *testing.T) (*Store, *pgxpool.Pool) { + t.Helper() + return newManagedTestStore(t) +} diff --git a/services/agents-api/internal/store/runtime_allocations.go b/services/agents-api/internal/store/runtime_allocations.go index 7d37e99fd..dd643d28e 100644 --- a/services/agents-api/internal/store/runtime_allocations.go +++ b/services/agents-api/internal/store/runtime_allocations.go @@ -16,6 +16,12 @@ import ( // RuntimeAllocation retains compute ownership, not public readiness. It survives // Session deletion until cleanup is confirmed. No bootstrap secret is retained. type RuntimeAllocation struct { + ComputePhase string + ComputeRevision int64 + ComputeState json.RawMessage + ComputeActivityAt time.Time + ComputeWakeRequested bool + ComputeRetainedUntil *time.Time ID, EnvironmentID, SessionID, TenantID, DeviceID, ProviderKey string Initialization string State string @@ -65,6 +71,9 @@ func (s *Store) ReserveRuntimeAllocation(ctx context.Context, tenant, environmen if !errors.Is(err, pgx.ErrNoRows) { return err } + if err := checkRuntimeDeploymentAdmission(ctx, q, providerKey); err != nil { + return err + } current, err := q.GetSessionEnvironment(ctx, sqlc.GetSessionEnvironmentParams{TenantID: lookup.TenantID, ID: session}) if err != nil { return err @@ -131,7 +140,14 @@ func (s *Store) ListRuntimeAllocations(ctx context.Context, after string) ([]Run } func runtimeAllocationFromRow(row sqlc.RuntimeAllocation, session, tenant pgtype.UUID, deleted pgtype.Timestamptz, expired bool) RuntimeAllocation { + var retainedUntil *time.Time + if row.ComputeRetainedUntil.Valid { + value := row.ComputeRetainedUntil.Time + retainedUntil = &value + } return RuntimeAllocation{ + ComputePhase: row.ComputePhase, ComputeRevision: row.ComputeRevision, ComputeState: row.ComputeState, + ComputeActivityAt: row.ComputeActivityAt.Time, ComputeWakeRequested: row.ComputeWakeRequested, ComputeRetainedUntil: retainedUntil, ID: uuid.UUID(row.ID.Bytes).String(), EnvironmentID: uuid.UUID(row.EnvironmentID.Bytes).String(), SessionID: uuid.UUID(session.Bytes).String(), TenantID: uuid.UUID(tenant.Bytes).String(), DeviceID: uuid.UUID(row.DeviceID.Bytes).String(), ProviderKey: uuid.UUID(row.ProviderKey.Bytes).String(), @@ -143,7 +159,7 @@ func runtimeAllocationFromRow(row sqlc.RuntimeAllocation, session, tenant pgtype // UnallocatedHostedEnvironment is a committed resource awaiting service bootstrap. // A missing allocation is distinct from an unknown outcome of an existing Create. type UnallocatedHostedEnvironment struct { - ID, TenantID, Engine string + ID, TenantID string } func (s *Store) ListUnallocatedHostedEnvironments(ctx context.Context, after string) ([]UnallocatedHostedEnvironment, error) { @@ -164,7 +180,7 @@ func (s *Store) ListUnallocatedHostedEnvironments(ctx context.Context, after str } result := make([]UnallocatedHostedEnvironment, 0, len(rows)) for _, row := range rows { - result = append(result, UnallocatedHostedEnvironment{Engine: row.Engine, ID: uuid.UUID(row.ID.Bytes).String(), TenantID: uuid.UUID(row.TenantID.Bytes).String()}) + result = append(result, UnallocatedHostedEnvironment{ID: uuid.UUID(row.ID.Bytes).String(), TenantID: uuid.UUID(row.TenantID.Bytes).String()}) } return result, nil } diff --git a/services/agents-api/internal/store/runtime_compute_lifecycle_test.go b/services/agents-api/internal/store/runtime_compute_lifecycle_test.go new file mode 100644 index 000000000..a7f16c0af --- /dev/null +++ b/services/agents-api/internal/store/runtime_compute_lifecycle_test.go @@ -0,0 +1,539 @@ +package store_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" + "github.com/gorilla/websocket" + "github.com/jackc/pgx/v5/pgxpool" +) + +// The controlled provider records external effects independently of DB phases. +// Lost replies retain those effects so recovery must use observation, not replay. +type fakeCheckpointProvider struct { + lifecycleProvider + computes map[string]sandbox.ComputeState + snapshots map[string]sandbox.SnapshotIdentity + bootstraps map[string]sandbox.Bootstrap + peers map[string]*websocket.Conn + registry *gateway.Registry + endpoint string + captures, restores, captureObservations, restoreObservations int + computeKills, snapshotDeletes, wakeCommands int + promptFrames atomic.Int32 + quiesces, resumes atomic.Int32 + loseCapture, loseRestore, rejectQuiesce bool + beforeQuiesce func() +} + +func (p *fakeCheckpointProvider) Initial(r sandbox.Reference) sandbox.Compute { + return sandbox.Compute{Name: r.AllocationID + "-g0"} +} +func (p *fakeCheckpointProvider) NewCompute(r sandbox.Reference, generation uint64, parent *sandbox.SnapshotIdentity) (sandbox.Compute, error) { + return sandbox.Compute{Generation: generation, Name: fmt.Sprintf("%s-g%d", r.AllocationID, generation), RestoredFrom: parent}, nil +} +func (p *fakeCheckpointProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { + info, err := p.lifecycleProvider.Create(ctx, b) + p.mu.Lock() + defer p.mu.Unlock() + current := p.Initial(b.Reference) + current.ID = uuid.NewString() + p.computes[current.Name] = sandbox.ComputeState{Compute: current, Status: "running", BootstrapComplete: true} + p.bootstraps[b.AllocationID] = b + return info, err +} +func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + p.mu.Lock() + defer p.mu.Unlock() + state, ok := p.computes[c.Name] + if !ok { + return sandbox.ComputeState{}, sandbox.ErrNotFound + } + if c.ID != "" && state.Compute.ID != c.ID { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + return state, nil +} +func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + p.mu.Lock() + defer p.mu.Unlock() + state, ok := p.computes[q.Source.Name] + if !ok { + return sandbox.ComputeState{}, sandbox.ErrNotFound + } + if state.Compute.ID != q.Source.ID { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + if q.ObserveOnly { + p.captureObservations++ + } else { + p.captures++ + if _, exists := p.snapshots[q.OperationID]; exists { + return sandbox.ComputeState{}, errors.New("capture replayed") + } + p.snapshots[q.OperationID] = sandbox.SnapshotIdentity{Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Digest: "verified", CheckpointID: "checkpoint", CheckpointRoot: "private", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} + state.Status = "paused" + p.computes[q.Source.Name] = state + if p.loseCapture { + p.loseCapture = false + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + } + if snapshot, exists := p.snapshots[q.OperationID]; exists { + state.Snapshot = &snapshot + } + return state, nil +} +func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + p.mu.Lock() + defer p.mu.Unlock() + if q.ObserveOnly { + p.restoreObservations++ + state, ok := p.computes[q.Target.Name] + if !ok { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return state, nil + } + p.restores++ + if _, exists := p.computes[q.Target.Name]; exists { + return sandbox.ComputeState{}, errors.New("restore replayed") + } + target := q.Target + target.ID = uuid.NewString() + state := sandbox.ComputeState{Compute: target, Status: "running", BootstrapComplete: true} + p.computes[target.Name] = state + if p.loseRestore { + p.loseRestore = false + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return state, nil +} +func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) error { + p.mu.Lock() + defer p.mu.Unlock() + state, ok := p.computes[c.Name] + if !ok { + return sandbox.ErrNotFound + } + if c.ID != "" && c.ID != state.Compute.ID { + return sandbox.ErrOwnership + } + p.computeKills++ + delete(p.computes, c.Name) + return nil +} +func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Reference, s sandbox.SnapshotIdentity) error { + p.mu.Lock() + defer p.mu.Unlock() + old, ok := p.snapshots[s.OperationID] + if !ok { + return sandbox.ErrNotFound + } + if old != s { + return sandbox.ErrOwnership + } + p.snapshotDeletes++ + delete(p.snapshots, s.OperationID) + return nil +} +func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + state, err := p.GetCompute(ctx, r, c) + if err != nil { + return state, err + } + p.mu.Lock() + defer p.mu.Unlock() + state.Status = "running" + p.computes[c.Name] = state + return state, nil +} +func (p *fakeCheckpointProvider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { + if _, err := p.GetCompute(ctx, r, c); err != nil { + return sandbox.CommandResult{}, err + } + if len(command.Args) != 8 || command.Args[0] != "parsar-daemon" || command.Args[1] != "resume" || command.Args[5] != r.EnvironmentID { + return sandbox.CommandResult{}, errors.New("unexpected wake command") + } + p.mu.Lock() + p.wakeCommands++ + b := p.bootstraps[r.AllocationID] + p.mu.Unlock() + return sandbox.CommandResult{}, p.connect(ctx, b) +} +func (p *fakeCheckpointProvider) connect(ctx context.Context, b sandbox.Bootstrap) error { + header := http.Header{"Authorization": []string{"Bearer " + b.Credential}} + conn, _, err := websocket.DefaultDialer.DialContext(ctx, p.endpoint+"?device_id="+b.DeviceID+"&version="+proto.Version, header) + if err != nil { + return err + } + p.mu.Lock() + p.peers[b.AllocationID] = conn + p.mu.Unlock() + if _, err = p.registry.WaitForDevice(ctx, b.DeviceID, time.Second); err != nil { + conn.Close() + return err + } + go func() { + defer conn.Close() + for { + var env proto.Envelope + if conn.ReadJSON(&env) != nil { + return + } + if env.Type == proto.TypeDeviceShutdown { + return + } + if env.Type != proto.TypeEnvironmentQuiesce && env.Type != proto.TypeEnvironmentResume { + p.mu.Lock() + p.promptFrames.Add(1) + p.mu.Unlock() + continue + } + var request proto.EnvironmentSuspendPayload + if env.DecodePayload(&request) != nil { + return + } + p.mu.Lock() + reject := p.rejectQuiesce && env.Type == proto.TypeEnvironmentQuiesce + if env.Type == proto.TypeEnvironmentQuiesce { + p.quiesces.Add(1) + } else { + p.resumes.Add(1) + } + beforeQuiesce := p.beforeQuiesce + p.mu.Unlock() + if env.Type == proto.TypeEnvironmentQuiesce && beforeQuiesce != nil { + beforeQuiesce() + } + kind := proto.TypeEnvironmentResumed + if env.Type == proto.TypeEnvironmentQuiesce { + kind = proto.TypeEnvironmentQuiesced + } + reply, _ := proto.NewEnvelope(kind, env.ID, proto.EnvironmentSuspendResultPayload{EnvironmentID: request.EnvironmentID, SuspendID: request.SuspendID, Accepted: !reject}) + if conn.WriteJSON(reply) != nil { + return + } + if env.Type == proto.TypeEnvironmentQuiesce && !reject { + return + } + } + }() + return nil +} + +type computeLifecycleFixture struct { + t *testing.T + store *store.Store + pool *pgxpool.Pool + provider *fakeCheckpointProvider + worker *execution.Worker + stop func() + key string + policy execution.RuntimeSuspensionPolicy +} + +func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *computeLifecycleFixture { + t.Helper() + s, pool := store.NewManagedTestStore(t) + registry := gateway.NewRegistry() + p := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} + handler := gateway.NewHandler(gateway.HandlerConfig{Authenticator: gateway.NewAuthenticator(s), Registry: registry}) + server := httptest.NewServer(http.HandlerFunc(handler.WS)) + p.endpoint = "ws" + strings.TrimPrefix(server.URL, "http") + t.Cleanup(func() { + p.mu.Lock() + defer p.mu.Unlock() + for _, peer := range p.peers { + peer.Close() + } + server.Close() + }) + f := &computeLifecycleFixture{t: t, store: s, pool: pool, provider: p, key: uuid.NewString(), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour, MaxActive: maxActive, MaxRetained: maxRetained}} + f.start() + return f +} +func (f *computeLifecycleFixture) start() { + t := f.t + t.Helper() + w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: f.store, Registry: f.provider.registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: f.key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: f.provider, Suspension: &f.policy}}) + if err != nil { + t.Fatal(err) + } + var once sync.Once + stop := func() { + once.Do(func() { ctx, cancel := context.WithCancel(context.Background()); cancel(); _ = w.Run(ctx) }) + } + f.worker, f.stop = w, stop + t.Cleanup(stop) +} +func (f *computeLifecycleFixture) sql(query string, args ...any) { + f.t.Helper() + if _, err := f.pool.Exec(f.t.Context(), query, args...); err != nil { + f.t.Fatal(err) + } +} +func (f *computeLifecycleFixture) create() (string, store.Session, store.Environment, store.RuntimeAllocation) { + t := f.t + t.Helper() + tenant, session, environment := managedSession(t, f.store) + owner, err := f.worker.ProvisionEnvironment(t.Context(), tenant, environment.ID, f.key) + if err != nil { + t.Fatal(err) + } + f.provider.mu.Lock() + b := f.provider.bootstraps[owner.ID] + f.provider.mu.Unlock() + if err := f.provider.connect(t.Context(), b); err != nil { + t.Fatal(err) + } + owner = f.phase(tenant, environment.ID, "running") + return tenant, session, environment, owner +} +func (f *computeLifecycleFixture) phase(tenant, environment, phase string) store.RuntimeAllocation { + f.t.Helper() + var owner store.RuntimeAllocation + for range 100 { + ctx, cancel := context.WithTimeout(f.t.Context(), 3*time.Second) + err := f.worker.ReconcileManagedRuntimes(ctx) + cancel() + if err != nil { + f.t.Fatal(err) + } + owner, err = f.store.GetRuntimeAllocation(f.t.Context(), tenant, environment) + if err != nil { + f.t.Fatal(err) + } + if owner.ComputePhase == phase { + return owner + } + } + f.t.Fatalf("compute phase=%s, want %s state=%s", owner.ComputePhase, phase, owner.ComputeState) + return owner +} +func (f *computeLifecycleFixture) complete(owner store.RuntimeAllocation) string { + id := uuid.NewString() + f.sql(`INSERT INTO turns(id,session_id,status,completed_at) VALUES($1,$2,'completed',clock_timestamp()-interval '2 minutes')`, id, owner.SessionID) + f.sql(`UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 minutes' WHERE id=$1`, owner.ID) + return id +} +func (f *computeLifecycleFixture) queued(owner store.RuntimeAllocation) string { + id := uuid.NewString() + f.sql(`INSERT INTO turns(id,session_id,status) VALUES($1,$2,'queued')`, id, owner.SessionID) + return id +} + +func TestRuntimeComputeLifecycleIdleSuspendAndQueuedSameSessionWake(t *testing.T) { + f := newComputeLifecycleFixture(t, 2, 4) + tenant, session, env, owner := f.create() + for range 3 { + f.phase(tenant, env.ID, "running") + } + if f.provider.captures != 0 { + t.Fatal("never-used Session suspended") + } + completed := f.complete(owner) + suspended := f.phase(tenant, env.ID, "suspended") + if f.provider.captures != 1 || f.provider.computeKills != 1 || len(f.provider.computes) != 0 || len(f.provider.snapshots) != 1 { + t.Fatal("capture did not release source compute") + } + queued := f.queued(suspended) + awake := f.phase(tenant, env.ID, "running") + if awake.SessionID != session.ID || awake.ID != owner.ID || awake.DeviceID != owner.DeviceID || f.provider.creates != 1 || f.provider.restores != 1 || f.provider.snapshotDeletes != 1 { + t.Fatal("wake replaced Session or replayed allocation") + } + var completedCount, queuedCount int + if err := f.pool.QueryRow(t.Context(), `SELECT count(*) FILTER(WHERE id=$2 AND status='completed'),count(*) FILTER(WHERE id=$3 AND status='queued') FROM turns WHERE session_id=$1`, session.ID, completed, queued).Scan(&completedCount, &queuedCount); err != nil || completedCount != 1 || queuedCount != 1 { + t.Fatal("wake replayed/consumed prior or next Turn", err) + } + if got, err := f.store.GetSession(t.Context(), tenant, session.ID); err != nil || string(got.Configuration) != string(session.Configuration) { + t.Fatal("configuration changed during restore", err) + } + if f.provider.promptFrames.Load() != 0 { + t.Fatal("lifecycle sent native execution input") + } +} + +func TestRuntimeComputeLifecycleLostCaptureAndRestoreObserveWithoutReplay(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + f.complete(owner) + f.provider.loseCapture = true + uncertain := f.phase(tenant, env.ID, "suspending") + if f.provider.captures != 1 || f.provider.computeKills != 0 { + t.Fatal("uncertain capture was reclaimed early") + } + captureIntent := append([]byte(nil), uncertain.ComputeState...) + f.stop() + f.start() + suspended := f.phase(tenant, env.ID, "suspended") + if f.provider.captures != 1 || f.provider.captureObservations == 0 || f.provider.computeKills != 1 { + t.Fatal("capture recovery replayed effects") + } + var original, mapState map[string]json.RawMessage + _ = json.Unmarshal(captureIntent, &original) + _ = json.Unmarshal(suspended.ComputeState, &mapState) + if string(original["suspend_id"]) != string(mapState["suspend_id"]) { + t.Fatal("capture operation identity changed") + } + f.provider.loseRestore = true + f.queued(suspended) + restoring := f.phase(tenant, env.ID, "restoring") + f.stop() + f.start() + f.phase(tenant, env.ID, "running") + if f.provider.restores != 1 || f.provider.restoreObservations == 0 || f.provider.creates != 1 { + t.Fatal("restore recovery repeated native operation") + } + if restoring.ComputeRevision <= uncertain.ComputeRevision { + t.Fatal("recovery lost durable progress") + } +} + +func TestRuntimeComputeLifecycleQuiesceRejectionAndUnknownIntentRollback(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + f.complete(owner) + f.provider.mu.Lock() + f.provider.rejectQuiesce = true + f.provider.mu.Unlock() + for range 100 { + f.phase(tenant, env.ID, "running") + if f.provider.quiesces.Load() != 0 { + break + } + } + if f.provider.quiesces.Load() != 1 || f.provider.captures != 0 { + t.Fatalf("quiesce rejection did not settle: requests=%d captures=%d", f.provider.quiesces.Load(), f.provider.captures) + } + // A restart with persisted quiescing but no acknowledgement must resume the + // source. Inject only the durable phase, never a fake snapshot or new source. + current, err := f.store.GetRuntimeAllocation(t.Context(), tenant, env.ID) + if err != nil { + t.Fatal(err) + } + var state map[string]any + if err := json.Unmarshal(current.ComputeState, &state); err != nil { + t.Fatal(err) + } + state["suspend_id"] = uuid.NewString() + raw, _ := json.Marshal(state) + f.sql(`UPDATE runtime_allocations SET compute_phase='quiescing',compute_state=$2,compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, owner.ID, raw) + f.phase(tenant, env.ID, "running") + if f.provider.captures != 0 || f.provider.restores != 0 || f.provider.resumes.Load() != 1 { + t.Fatal("unknown quiesce did not roll back source") + } +} + +func TestRuntimeComputeLifecycleWakeDuringQuiesceResumesSource(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + f.complete(owner) + wakeResult := make(chan error, 1) + f.provider.mu.Lock() + f.provider.beforeQuiesce = func() { + wakeResult <- f.store.TouchRuntimeActivity(t.Context(), tenant, env.ID) + } + f.provider.mu.Unlock() + for range 100 { + f.phase(tenant, env.ID, "running") + if f.provider.resumes.Load() != 0 { + break + } + } + select { + case err := <-wakeResult: + if err != nil { + t.Fatal(err) + } + default: + t.Fatal("quiesce did not reach competing wake") + } + if f.provider.resumes.Load() != 1 || f.provider.captures != 0 || f.provider.restores != 0 || f.provider.computeKills != 0 { + t.Fatal("competing wake did not resume the original source before capture") + } +} + +func TestRuntimeComputeLifecycleSuspendedDeletionAndExpiryCleanup(t *testing.T) { + for _, kind := range []string{"deleted", "expired"} { + t.Run(kind, func(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, session, env, owner := f.create() + f.complete(owner) + f.phase(tenant, env.ID, "suspended") + if kind == "deleted" { + if err := f.store.DeleteSession(t.Context(), tenant, session.ID); err != nil { + t.Fatal(err) + } + } else { + f.sql(`UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID) + } + reconcileManagedState(t, f.worker, f.store, tenant, env.ID, "released") + if len(f.provider.computes) != 0 || len(f.provider.snapshots) != 0 || f.provider.snapshotDeletes != 1 { + t.Fatal("retained snapshot survived cleanup") + } + if _, ok, err := f.store.GetDeviceCredential(t.Context(), owner.DeviceID); err != nil || ok { + t.Fatal("cleanup retained daemon authority", err) + } + }) + } +} + +func TestRuntimeComputeLifecycleCapacityBoundsActiveAndRetained(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + tenant2, _, env2 := managedSession(t, f.store) + if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant2, env2.ID, f.key); !errors.Is(err, execution.ErrExecutionUnavailable) { + t.Fatalf("active capacity ignored: %v", err) + } + if f.provider.creates != 1 { + t.Fatal("capacity rejection allocated compute") + } + f.complete(owner) + f.phase(tenant, env.ID, "suspended") + second, err := f.worker.ProvisionEnvironment(t.Context(), tenant2, env2.ID, f.key) + if err != nil { + t.Fatal(err) + } + pending := f.queued(owner) + for range 4 { + f.worker.ReconcileManagedRuntimes(t.Context()) + } + first, err := f.store.GetRuntimeAllocation(t.Context(), tenant, env.ID) + if err != nil || first.ComputePhase != "suspended" || f.provider.restores != 0 { + t.Fatal("wake exceeded active capacity", err) + } + f.sql(`UPDATE turns SET status='cancelled',completed_at=clock_timestamp() WHERE id=$1`, pending) + f.provider.mu.Lock() + b := f.provider.bootstraps[second.ID] + f.provider.mu.Unlock() + if err := f.provider.connect(t.Context(), b); err != nil { + t.Fatal(err) + } + second = f.phase(tenant2, env2.ID, "running") + f.complete(second) + f.phase(tenant2, env2.ID, "suspended") + // Both retained allocations count even when their source VMs are gone. + tenant3, _, env3 := managedSession(t, f.store) + if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant3, env3.ID, f.key); !errors.Is(err, execution.ErrExecutionUnavailable) { + t.Fatalf("retained capacity ignored: %v", err) + } + if f.provider.creates != 2 { + t.Fatal("retained limit created a third allocation") + } +} diff --git a/services/agents-api/internal/store/runtime_connection_test.go b/services/agents-api/internal/store/runtime_connection_test.go index c5c451036..a07c42b1b 100644 --- a/services/agents-api/internal/store/runtime_connection_test.go +++ b/services/agents-api/internal/store/runtime_connection_test.go @@ -20,7 +20,7 @@ import ( ) func TestManagedRuntimeConnectionTracksAuthenticatedSocket(t *testing.T) { - s, _ := store.NewTestStore(t) + s, _ := store.NewManagedTestStore(t) tenant, session, environment := managedSession(t, s) server := httptest.NewUnstartedServer(nil) wsURL := "ws://" + server.Listener.Addr().String() + "/api/v1/agent-daemon/ws" @@ -34,7 +34,7 @@ func TestManagedRuntimeConnectionTracksAuthenticatedSocket(t *testing.T) { p := &lifecycleProvider{resources: map[string]sandbox.Info{}} key := uuid.NewString() start := func() *execution.Worker { - w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: s, Registry: registry, ManagedRuntimes: &execution.RuntimeProviders{CoreURL: server.URL + "/api/v1", Providers: map[string]sandbox.Provider{key: p}}}) + w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: s, Registry: registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: server.URL + "/api/v1", InstallationID: key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: p}}) if err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/runtime_deployment.go b/services/agents-api/internal/store/runtime_deployment.go new file mode 100644 index 000000000..13673b4c8 --- /dev/null +++ b/services/agents-api/internal/store/runtime_deployment.go @@ -0,0 +1,96 @@ +package store + +import ( + "context" + "encoding/hex" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" +) + +// RuntimeDeployment identifies the one operator-selected installation for this database. +// Its fingerprint describes the backend namespace, never credentials or image contents. +type RuntimeDeployment struct { + InstallationID string + BackendFingerprint string + Maintenance bool +} + +// ConfigureRuntimeDeployment runs before Worker startup under its execution lease. +// Maintenance must be committed for the old installation before any switch. +// A nil selection never forgets the previous identity or unresolved resources. +func (s *Store) ConfigureRuntimeDeployment(ctx context.Context, selected *RuntimeDeployment) error { + if s.executionLease == nil { + return ErrInvalidInput + } + var update sqlc.SetRuntimeDeploymentParams + if selected != nil { + id, err := parseConnectionGeneration(selected.InstallationID) + if err != nil { + return err + } + digest, err := hex.DecodeString(selected.BackendFingerprint) + if err != nil || len(digest) != 32 || strings.ToLower(selected.BackendFingerprint) != selected.BackendFingerprint { + return fmt.Errorf("%w: invalid backend identity fingerprint", ErrInvalidInput) + } + update = sqlc.SetRuntimeDeploymentParams{InstallationID: id, BackendFingerprint: selected.BackendFingerprint, Maintenance: selected.Maintenance} + } + return s.executionLease.transaction(ctx, func(tx pgx.Tx) error { + q := s.queries.WithTx(tx) + previous, err := q.LockRuntimeDeployment(ctx) + if err != nil { + return err + } + if selected != nil && previous.InstallationID == update.InstallationID && previous.BackendFingerprint == update.BackendFingerprint { + return q.SetRuntimeDeployment(ctx, update) + } + resources, err := q.CountRuntimeDeploymentResources(ctx) + if err != nil { + return err + } + if selected == nil { + if previous.InstallationID.Valid && (resources.Allocations != 0 || resources.Pending != 0) { + return fmt.Errorf("cannot disable managed sandbox provider: %d unreleased allocations (instances, retained snapshots, uncertain operations or pending cleanup) and %d pending hosted environments remain", resources.Allocations, resources.Pending) + } + return nil + } + if !previous.InstallationID.Valid { + if resources.Allocations != 0 { + return fmt.Errorf("cannot adopt sandbox installation: %d existing unreleased allocations (including retained snapshots and pending cleanup) have no verified backend identity", resources.Allocations) + } + } else { + if !previous.Maintenance || !selected.Maintenance { + return fmt.Errorf("cannot switch sandbox installation: persist maintenance on the previous installation and keep the new installation in maintenance") + } + if resources.Allocations != 0 || resources.Pending != 0 { + return fmt.Errorf("cannot switch sandbox installation: %d unreleased allocations (instances, retained snapshots, uncertain operations or pending cleanup) and %d pending hosted environments remain", resources.Allocations, resources.Pending) + } + } + return q.SetRuntimeDeployment(ctx, update) + }) +} + +// New work and deployment changes share this lock. Existing receipts are checked +// first, preserving idempotent retries and cleanup while maintenance is active. +func checkRuntimeDeploymentAdmission(ctx context.Context, q *sqlc.Queries, installation string) error { + current, err := q.LockRuntimeDeployment(ctx) + if err != nil { + return err + } + if !current.InstallationID.Valid { + return nil + } + if current.Maintenance { + return fmt.Errorf("%w: sandbox creation is paused for provider maintenance", ErrEnvironmentUnavailable) + } + if installation != "" { + id, err := parseConnectionGeneration(installation) + if err != nil || id != current.InstallationID { + return fmt.Errorf("%w: sandbox installation does not match deployment", ErrEnvironmentUnavailable) + } + } + return nil +} diff --git a/services/agents-api/internal/store/runtime_deployment_test.go b/services/agents-api/internal/store/runtime_deployment_test.go new file mode 100644 index 000000000..98c7c4b9f --- /dev/null +++ b/services/agents-api/internal/store/runtime_deployment_test.go @@ -0,0 +1,330 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" +) + +func deploymentSelection() RuntimeDeployment { + return RuntimeDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64)} +} + +func deploymentConfigure(t *testing.T, w *Store, config *RuntimeDeployment) { + t.Helper() + if err := w.ConfigureRuntimeDeployment(t.Context(), config); err != nil { + t.Fatal(err) + } +} + +func TestRuntimeDeploymentRequiresMaintenanceBeforeIdentityChange(t *testing.T) { + s, pool := newManagedTestStore(t) + w := executionLease(t, s).Store() + old := deploymentSelection() + deploymentConfigure(t, w, &old) + if err := s.ConfigureRuntimeDeployment(t.Context(), &old); !errors.Is(err, ErrInvalidInput) { + t.Fatal("unleased configuration accepted", err) + } + for _, changeID := range []bool{false, true} { + next := old + if changeID { + next.InstallationID = uuid.NewString() + } else { + next.BackendFingerprint = strings.Repeat("b", 64) + } + next.Maintenance = true + if err := w.ConfigureRuntimeDeployment(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "maintenance") { + t.Fatal("identity changed before prior maintenance", err) + } + } + old.Maintenance = true + deploymentConfigure(t, w, &old) + next := old + next.BackendFingerprint = strings.Repeat("b", 64) + next.Maintenance = false + if err := w.ConfigureRuntimeDeployment(t.Context(), &next); err == nil { + t.Fatal("switch reopened creation in same operation") + } + next.Maintenance = true + deploymentConfigure(t, w, &next) + var id, fingerprint string + var maintenance bool + if err := pool.QueryRow(t.Context(), "SELECT installation_id::text,backend_fingerprint,maintenance FROM runtime_deployment").Scan(&id, &fingerprint, &maintenance); err != nil || id != next.InstallationID || fingerprint != next.BackendFingerprint || !maintenance { + t.Fatal("switch identity not durable", id, fingerprint, maintenance, err) + } + deploymentConfigure(t, w, nil) + // Disabling the configured adapter must not forget the old maintenance state. + next.Maintenance = false + deploymentConfigure(t, w, &next) + another := deploymentSelection() + another.Maintenance = true + if err := w.ConfigureRuntimeDeployment(t.Context(), &another); err == nil { + t.Fatal("nil selection erased the maintenance prerequisite") + } +} + +func TestRuntimeDeploymentPendingSessionsCannotMigrate(t *testing.T) { + s, _ := newManagedTestStore(t) + w := executionLease(t, s).Store() + tenant := uuid.NewString() + session, _ := localEnvironment(t, s, tenant) + old := deploymentSelection() + // First selection is allowed for work that has never had an installation. + deploymentConfigure(t, w, &old) + old.Maintenance = true + deploymentConfigure(t, w, &old) + next := deploymentSelection() + next.Maintenance = true + if err := w.ConfigureRuntimeDeployment(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 pending hosted") { + t.Fatal("pending Session migrated", err) + } + if err := w.ConfigureRuntimeDeployment(t.Context(), nil); err == nil { + t.Fatal("pending Session orphaned by removing provider") + } + if err := s.DeleteSession(t.Context(), tenant, session.ID); err != nil { + t.Fatal(err) + } + deploymentConfigure(t, w, &next) +} + +func TestRuntimeDeploymentUnknownAllocationsBlockAdoptionAndSwitch(t *testing.T) { + s, _ := newManagedTestStore(t) + w := executionLease(t, s).Store() + old := deploymentSelection() + tenant := uuid.NewString() + session, environment := localEnvironment(t, s, tenant) + owner, err := w.ReserveRuntimeAllocation(t.Context(), tenant, environment.ID, old.InstallationID, device.HashCredential(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + if err := w.ConfigureRuntimeDeployment(t.Context(), &old); err == nil || !strings.Contains(err.Error(), "no verified backend identity") { + t.Fatal("legacy allocation silently adopted", err) + } + if err := s.DeleteSession(t.Context(), tenant, session.ID); err != nil { + t.Fatal(err) + } + if _, err := w.RequestRuntimeCleanup(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err := w.ReleaseRuntimeAllocation(t.Context(), owner); !errors.Is(err, ErrTurnConflict) { + t.Fatal("unknown creation lost cleanup ownership", err) + } + if err := w.ConfigureRuntimeDeployment(t.Context(), &old); err == nil { + t.Fatal("deleted unknown allocation did not block adoption") + } + if _, err := w.SettleRuntimeCreation(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err := w.ReleaseRuntimeAllocation(t.Context(), owner); err != nil { + t.Fatal(err) + } + deploymentConfigure(t, w, &old) + _, environment = localEnvironment(t, s, tenant) + owner, err = w.ReserveRuntimeAllocation(t.Context(), tenant, environment.ID, old.InstallationID, device.HashCredential(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + old.Maintenance = true + deploymentConfigure(t, w, &old) + next := deploymentSelection() + next.Maintenance = true + if err := w.ConfigureRuntimeDeployment(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { + t.Fatal("unknown creation did not block switch", err) + } + if err := w.ConfigureRuntimeDeployment(t.Context(), nil); err == nil { + t.Fatal("removing adapter orphaned unknown creation") + } + replay, err := w.ReserveRuntimeAllocation(t.Context(), tenant, environment.ID, old.InstallationID, device.HashCredential(uuid.NewString())) + if err != nil || !replay.Replayed || replay.ID != owner.ID { + t.Fatal("maintenance blocked receipt replay", replay, err) + } + if _, err := w.RequestRuntimeCleanup(t.Context(), owner); err != nil { + t.Fatal("maintenance blocked cleanup", err) + } +} + +func TestRuntimeDeploymentMaintenancePreservesCreationRetriesAndOtherPlacements(t *testing.T) { + s, pool := newManagedTestStore(t) + w := executionLease(t, s).Store() + old := deploymentSelection() + deploymentConfigure(t, w, &old) + tenant := uuid.NewString() + input := CreateSessionInput{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"openai_hosted"}}`)} + existing, err := s.CreateSession(t.Context(), tenant, input) + if err != nil { + t.Fatal(err) + } + old.Maintenance = true + deploymentConfigure(t, w, &old) + replay, err := s.CreateSession(t.Context(), tenant, input) + if err != nil || replay.ID != existing.ID { + t.Fatal("creation retry lost identity", err) + } + input.IdempotencyKey = uuid.NewString() + if _, err := s.CreateSession(t.Context(), tenant, input); !errors.Is(err, ErrEnvironmentUnavailable) { + t.Fatal("maintenance created hosted Session", err) + } + var count int + if err := pool.QueryRow(t.Context(), "SELECT count(*) FROM sessions WHERE tenant_id=$1", tenant).Scan(&count); err != nil || count != 1 { + t.Fatal("rejection left partial Session", count, err) + } + if _, err := w.ReserveRuntimeAllocation(t.Context(), tenant, existing.Environment.ID, old.InstallationID, device.HashCredential(uuid.NewString())); !errors.Is(err, ErrEnvironmentUnavailable) { + t.Fatal("maintenance reserved new allocation", err) + } + for _, kind := range []string{"none", "self_hosted"} { + input.IdempotencyKey = uuid.NewString() + input.Configuration = json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"` + kind + `"}}`) + if _, err := s.CreateSession(t.Context(), tenant, input); err != nil { + t.Fatal("maintenance blocked unrelated placement", kind, err) + } + } + old.Maintenance = false + deploymentConfigure(t, w, &old) + if _, err := w.ReserveRuntimeAllocation(t.Context(), tenant, existing.Environment.ID, uuid.NewString(), device.HashCredential(uuid.NewString())); !errors.Is(err, ErrEnvironmentUnavailable) { + t.Fatal("wrong installation reserved resource", err) + } + if _, err := w.ReserveRuntimeAllocation(t.Context(), tenant, existing.Environment.ID, old.InstallationID, device.HashCredential(uuid.NewString())); err != nil { + t.Fatal("resume did not reopen allocation", err) + } +} + +func TestRuntimeDeploymentMaintenanceSerializesHostedCreation(t *testing.T) { + s, pool := newManagedTestStore(t) + w := executionLease(t, s).Store() + config := deploymentSelection() + deploymentConfigure(t, w, &config) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + var blocker int32 + if err := tx.QueryRow(ctx, "SELECT pg_backend_pid() FROM runtime_deployment FOR UPDATE").Scan(&blocker); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + tenant := uuid.NewString() + go func() { + _, err := s.CreateSession(ctx, tenant, CreateSessionInput{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"environment":{"type":"openai_hosted"}}`)}) + done <- err + }() + runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, done) + if _, err := tx.Exec(ctx, "UPDATE runtime_deployment SET maintenance=true"); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err := <-done; !errors.Is(err, ErrEnvironmentUnavailable) { + t.Fatal("creation bypassed committed maintenance", err) + } + var count int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM sessions WHERE tenant_id=$1", tenant).Scan(&count); err != nil || count != 0 { + t.Fatal("racing creation left partial work", count, err) + } +} + +func TestRuntimeDeploymentRetainedResourcesBlockSwitchWithoutMutation(t *testing.T) { + for _, state := range []string{"running", "suspended", "cleanup_pending"} { + t.Run(state, func(t *testing.T) { + s, pool := newManagedTestStore(t) + w := executionLease(t, s).Store() + old := deploymentSelection() + deploymentConfigure(t, w, &old) + tenant := uuid.NewString() + _, environment := localEnvironment(t, s, tenant) + owner, err := w.ReserveRuntimeAllocation(t.Context(), tenant, environment.ID, old.InstallationID, device.HashCredential(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + if state == "suspended" { + _, err = pool.Exec(t.Context(), `UPDATE runtime_allocations SET state='running',create_settled=true,compute_phase='suspended',compute_state='{"snapshot":{"id":"retained-test-snapshot"}}' WHERE id=$1`, owner.ID) + } else { + _, err = pool.Exec(t.Context(), "UPDATE runtime_allocations SET state=$2,create_settled=true WHERE id=$1", owner.ID, state) + } + if err != nil { + t.Fatal(err) + } + old.Maintenance = true + deploymentConfigure(t, w, &old) + var before, after, oldIdentity, newIdentity string + if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(a)::text FROM runtime_allocations a WHERE id=$1", owner.ID).Scan(&before); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(d)::text FROM runtime_deployment d").Scan(&oldIdentity); err != nil { + t.Fatal(err) + } + next := deploymentSelection() + next.Maintenance = true + if err := w.ConfigureRuntimeDeployment(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { + t.Fatal("retained resource allowed switch", state, err) + } + if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(a)::text FROM runtime_allocations a WHERE id=$1", owner.ID).Scan(&after); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(d)::text FROM runtime_deployment d").Scan(&newIdentity); err != nil { + t.Fatal(err) + } + if before != after || oldIdentity != newIdentity { + t.Fatal("refused switch changed existing ownership") + } + }) + } +} + +func TestRuntimeDeploymentAllocationBeforeMaintenanceRetainsOwnership(t *testing.T) { + s, pool := newManagedTestStore(t) + w := executionLease(t, s).Store() + config := deploymentSelection() + deploymentConfigure(t, w, &config) + tenant := uuid.NewString() + _, environment := localEnvironment(t, s, tenant) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + var blocker int32 + if err := tx.QueryRow(ctx, "SELECT pg_backend_pid() FROM runtime_deployment FOR UPDATE").Scan(&blocker); err != nil { + t.Fatal(err) + } + allocated := make(chan error, 1) + go func() { + _, err := w.ReserveRuntimeAllocation(ctx, tenant, environment.ID, config.InstallationID, device.HashCredential(uuid.NewString())) + allocated <- err + }() + runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, allocated) + maintaining := make(chan error, 1) + maintenance := config + maintenance.Maintenance = true + go func() { maintaining <- w.ConfigureRuntimeDeployment(ctx, &maintenance) }() + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err := <-allocated; err != nil { + t.Fatal("earlier allocation lost ownership", err) + } + if err := <-maintaining; err != nil { + t.Fatal(err) + } + owner, err := w.GetRuntimeAllocation(ctx, tenant, environment.ID) + if err != nil || owner.State != "creating" || owner.CreateSettled { + t.Fatal("maintenance changed uncertain receipt", owner, err) + } + next := deploymentSelection() + next.Maintenance = true + if err := w.ConfigureRuntimeDeployment(ctx, &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { + t.Fatal("earlier in-flight allocation omitted from switch guard", err) + } +} diff --git a/services/agents-api/internal/store/runtime_deployment_worker_test.go b/services/agents-api/internal/store/runtime_deployment_worker_test.go new file mode 100644 index 000000000..e50ccc67c --- /dev/null +++ b/services/agents-api/internal/store/runtime_deployment_worker_test.go @@ -0,0 +1,56 @@ +package store_test + +import ( + "strings" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +func TestManagedDeploymentStartupRejectsSwitchBeforeBackendAccess(t *testing.T) { + s, _ := store.NewManagedTestStore(t) + key := uuid.NewString() + old := &lifecycleProvider{resources: map[string]sandbox.Info{}} + worker, stop := managedWorker(t, s, key, old) + tenant, _, environment := managedSession(t, s) + owner, err := worker.ProvisionEnvironment(t.Context(), tenant, environment.ID, key) + if err != nil { + t.Fatal(err) + } + stop() + replacement := &lifecycleProvider{resources: map[string]sandbox.Info{}} + config := &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("b", 64), Provider: replacement, Maintenance: true} + start := func(config *execution.RuntimeProvider) error { + _, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: s, Registry: gateway.NewRegistry(), ManagedRuntimes: config}) + return err + } + if err := start(config); err == nil || !strings.Contains(err.Error(), "maintenance") { + t.Fatal("startup switched active deployment", err) + } + worker, stop = managedWorkerMode(t, s, key, old, true) + replay, err := worker.ProvisionEnvironment(t.Context(), tenant, environment.ID, key) + if err != nil || !replay.Replayed || replay.ID != owner.ID { + t.Fatal("maintenance interrupted existing allocation", replay, err) + } + stop() + if err := start(config); err == nil || !strings.Contains(err.Error(), "unreleased allocations") { + t.Fatal("startup abandoned retained allocation", err) + } + if err := start(nil); err == nil || !strings.Contains(err.Error(), "unreleased allocations") { + t.Fatal("omitted configuration abandoned deployment", err) + } + if replacement.creates != 0 || replacement.kills != 0 { + t.Fatal("rejected startup touched new backend") + } + got, err := s.GetRuntimeAllocation(t.Context(), tenant, environment.ID) + if err != nil || got.ID != owner.ID || got.ProviderKey != key { + t.Fatal("rejected startup rewrote resource owner", got, err) + } + // Failed startup relinquishes its lease, so the original backend can resume. + _, stop = managedWorker(t, s, key, old) + stop() +} diff --git a/services/agents-api/internal/store/runtime_file_admission_test.go b/services/agents-api/internal/store/runtime_file_admission_test.go new file mode 100644 index 000000000..a9f4e255e --- /dev/null +++ b/services/agents-api/internal/store/runtime_file_admission_test.go @@ -0,0 +1,126 @@ +package store + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" +) + +func runtimeFileWriteKey(owner RuntimeAllocation) FileWriteIdentity { + return FileWriteIdentity{ID: uuid.NewString(), DeviceID: owner.DeviceID, RequestSHA256: strings.Repeat("a", 64)} +} + +func TestRuntimeFileWriteRequiresRunningComputeBeforeNewIntent(t *testing.T) { + for _, phase := range []string{"disabled", "running", "quiescing", "suspending", "suspended", "restoring", "waking"} { + t.Run(phase, func(t *testing.T) { + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_phase=$2,compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, owner.ID, phase) + key := runtimeFileWriteKey(owner) + write, err := w.ReserveEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, key) + blocked := phase != "disabled" && phase != "running" + if blocked { + if !errors.Is(err, ErrTurnConflict) { + t.Fatal("suspended compute admitted a new write", write, err) + } + if _, err := s.GetEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, key.ID); !errors.Is(err, ErrNotFound) { + t.Fatal("rejected admission retained a blocking intent", err) + } + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_phase='running',compute_retained_until=NULL WHERE id=$1`, owner.ID) + write, err = w.ReserveEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, key) + } + if err != nil || write.Replayed || write.State != "pending" { + t.Fatal("running compute could not admit original request", write, err) + } + if _, err := w.SettleEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, key, "committed"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestRuntimeFileWritePhaseFencePreservesExistingReceipts(t *testing.T) { + _, w, pool, owner := runtimeSuspensionFixture(t) + key := runtimeFileWriteKey(owner) + first, err := w.ReserveEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, key) + if err != nil { + t.Fatal(err) + } + // A retry observes retained evidence even if its Runtime phase has changed; + // it never grants permission to send the unknown write again. + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_phase='waking',compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, owner.ID) + for _, state := range []string{"pending", "committed"} { + got, err := w.ReserveEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, key) + if err != nil || !got.Replayed || got.State != state || got.Identity != key || !got.CreatedAt.Equal(first.CreatedAt) { + t.Fatal("phase fence changed the existing receipt", got, err) + } + changed := key + changed.RequestSHA256 = strings.Repeat("b", 64) + if _, err := w.ReserveEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, changed); !errors.Is(err, ErrIdempotencyConflict) { + t.Fatal("phase fence masked changed retry identity", err) + } + if _, err := w.ReserveEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, runtimeFileWriteKey(owner)); !errors.Is(err, ErrTurnConflict) { + t.Fatal("receipt authorized a successor while waking", err) + } + if state == "pending" { + if _, err := w.SettleEnvironmentFileWrite(t.Context(), owner.TenantID, owner.EnvironmentID, key, "committed"); err != nil { + t.Fatal("phase fence prevented exact receipt settlement", err) + } + } + } +} + +func TestRuntimeFileWriteAndQuiesceSerializeBothOrders(t *testing.T) { + for _, first := range []string{"quiesce", "write"} { + t.Run(first, func(t *testing.T) { + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) + key := runtimeFileWriteKey(owner) + until := time.Now().Add(time.Hour) + ctx, tx, blocker := runtimeSuspensionLockedSession(t, pool, owner.SessionID) + done := make(chan error, 1) + go func() { + var err error + if first == "quiesce" { + _, err = w.ReserveEnvironmentFileWrite(ctx, owner.TenantID, owner.EnvironmentID, key) + } else { + _, err = w.SetRuntimeCompute(ctx, owner, "quiescing", json.RawMessage(`{}`), &until, time.Now()) + } + done <- err + }() + runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, done) + // Commit the first owner's durable change while the competitor waits + // for the same Session lock, fixing the race order without sleeps. + var err error + if first == "quiesce" { + _, err = tx.Exec(ctx, `UPDATE runtime_allocations SET compute_phase='quiescing',compute_revision=compute_revision+1,compute_retained_until=$2 WHERE id=$1`, owner.ID, until) + } else { + _, err = tx.Exec(ctx, `INSERT INTO environment_file_writes(id,environment_id,device_id,request_sha256) VALUES($1,$2,$3,$4)`, key.ID, owner.EnvironmentID, key.DeviceID, key.RequestSHA256) + } + if err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err := <-done; !errors.Is(err, ErrTurnConflict) { + t.Fatal("competing durable owner bypassed the phase fence", err) + } + allocation, err := s.GetRuntimeAllocation(ctx, owner.TenantID, owner.EnvironmentID) + if err != nil { + t.Fatal(err) + } + write, err := s.GetEnvironmentFileWrite(ctx, owner.TenantID, owner.EnvironmentID, key.ID) + if first == "quiesce" { + if allocation.ComputePhase != "quiescing" || !errors.Is(err, ErrNotFound) { + t.Fatal("late write survived quiesce", allocation.ComputePhase, write, err) + } + } else if allocation.ComputePhase != "running" || err != nil || write.State != "pending" { + t.Fatal("quiesce overtook pending write", allocation.ComputePhase, write, err) + } + }) + } +} diff --git a/services/agents-api/internal/store/runtime_initialization_test.go b/services/agents-api/internal/store/runtime_initialization_test.go index 16b0c3a44..c01da1502 100644 --- a/services/agents-api/internal/store/runtime_initialization_test.go +++ b/services/agents-api/internal/store/runtime_initialization_test.go @@ -60,7 +60,7 @@ func TestManagedInitialFilesGateFairnessCompletionAndRestart(t *testing.T) { setupOnly := strings.HasPrefix(mode, "setup-") mode = strings.TrimPrefix(mode, "setup-") expectedSteps := 2 - _, pool := store.NewTestStore(t) + _, pool := store.NewManagedTestStore(t) cipher, err := credentialcrypto.New(bytes.Repeat([]byte{9}, 32)) if err != nil { t.Fatal(err) diff --git a/services/agents-api/internal/store/runtime_input_admission_test.go b/services/agents-api/internal/store/runtime_input_admission_test.go index 182f822f3..a1c1c658b 100644 --- a/services/agents-api/internal/store/runtime_input_admission_test.go +++ b/services/agents-api/internal/store/runtime_input_admission_test.go @@ -7,13 +7,12 @@ import ( "github.com/google/uuid" - "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" ) -func TestManagedRuntimeDisabledAdmissionPreservesCancelAndRetry(t *testing.T) { - s, _ := store.NewTestStore(t) +func TestManagedRuntimeMaintenancePreservesCancelAndRetry(t *testing.T) { + s, _ := store.NewManagedTestStore(t) tenant, session, _ := managedSession(t, s) inputs := []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"accepted work"}`)}} accepted, err := s.SubmitInputs(t.Context(), tenant, session.ID, "work", inputs) @@ -21,10 +20,10 @@ func TestManagedRuntimeDisabledAdmissionPreservesCancelAndRetry(t *testing.T) { t.Fatal(err) } p := &lifecycleProvider{resources: map[string]sandbox.Info{}} - w, stop := managedWorker(t, s, uuid.NewString(), p) + w, stop := managedWorkerMode(t, s, uuid.NewString(), p, true) defer stop() - if _, err := w.CreateSession(t.Context(), tenant, store.CreateSessionInput{Creator: store.FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: session.Configuration}); !errors.Is(err, execution.ErrExecutionUnavailable) { - t.Fatal("disabled admission accepted new hosted Session", err) + if _, err := w.CreateSession(t.Context(), tenant, store.CreateSessionInput{Creator: store.FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: session.Configuration}); !errors.Is(err, store.ErrEnvironmentUnavailable) { + t.Fatal("maintenance accepted new hosted Session", err) } cancel := []store.Input{{Kind: "cancel", Payload: json.RawMessage(`{}`)}} first, err := w.SubmitInputs(t.Context(), tenant, session.ID, "cancel", cancel) @@ -40,7 +39,7 @@ func TestManagedRuntimeDisabledAdmissionPreservesCancelAndRetry(t *testing.T) { t.Fatal("matching input retry lost its accepted outcome", retry, err) } if _, err := w.SubmitInputs(t.Context(), uuid.NewString(), session.ID, "cancel", cancel); !errors.Is(err, store.ErrNotFound) { - t.Fatal("disabled admission weakened tenant isolation", err) + t.Fatal("maintenance weakened tenant isolation", err) } if p.creates != 0 { t.Fatal("existing controls provisioned a new Runtime") diff --git a/services/agents-api/internal/store/runtime_lifecycle_test.go b/services/agents-api/internal/store/runtime_lifecycle_test.go index 85b6222f2..648c14060 100644 --- a/services/agents-api/internal/store/runtime_lifecycle_test.go +++ b/services/agents-api/internal/store/runtime_lifecycle_test.go @@ -70,7 +70,12 @@ func (p *lifecycleProvider) RunCommand(context.Context, sandbox.Reference, sandb func managedWorker(t *testing.T, s *store.Store, key string, p sandbox.Provider) (*execution.Worker, func()) { t.Helper() - w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: s, Registry: gateway.NewRegistry(), ManagedRuntimes: &execution.RuntimeProviders{CoreURL: "http://core.invalid/api/v1", Providers: map[string]sandbox.Provider{key: p}}}) + return managedWorkerMode(t, s, key, p, false) +} + +func managedWorkerMode(t *testing.T, s *store.Store, key string, p sandbox.Provider, maintenance bool) (*execution.Worker, func()) { + t.Helper() + w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: s, Registry: gateway.NewRegistry(), ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: p, Maintenance: maintenance}}) if err != nil { t.Fatal(err) } @@ -114,7 +119,7 @@ func reconcileManagedState(t *testing.T, w *execution.Worker, s *store.Store, te } func TestManagedRuntimeLostCreateRestartAndDeletion(t *testing.T) { - s, _ := store.NewTestStore(t) + s, _ := store.NewManagedTestStore(t) tenant, session, env := managedSession(t, s) key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}, loseCreate: true} @@ -153,7 +158,7 @@ func TestManagedRuntimeLostCreateRestartAndDeletion(t *testing.T) { } func TestManagedRuntimeUnknownCreationRetainsCleanup(t *testing.T) { - s, _ := store.NewTestStore(t) + s, _ := store.NewManagedTestStore(t) tenant, session, env := managedSession(t, s) key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}, loseCreate: true, absent: true} @@ -183,7 +188,7 @@ func TestManagedRuntimeUnknownCreationRetainsCleanup(t *testing.T) { } func TestManagedRuntimeExpiryRevokesWhenProviderUnavailable(t *testing.T) { - s, pool := store.NewTestStore(t) + s, pool := store.NewManagedTestStore(t) tenant, _, env := managedSession(t, s) key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}} @@ -210,7 +215,7 @@ func TestManagedRuntimeExpiryRevokesWhenProviderUnavailable(t *testing.T) { } func TestManagedRuntimeStoppedComputeDoesNotRequestCleanup(t *testing.T) { - s, _ := store.NewTestStore(t) + s, _ := store.NewManagedTestStore(t) tenant, _, env := managedSession(t, s) key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}} diff --git a/services/agents-api/internal/store/runtime_pending_test.go b/services/agents-api/internal/store/runtime_pending_test.go index f5e00f183..c7251aee1 100644 --- a/services/agents-api/internal/store/runtime_pending_test.go +++ b/services/agents-api/internal/store/runtime_pending_test.go @@ -13,7 +13,7 @@ import ( ) func TestManagedRuntimeAutomaticBootstrapRecoversCommittedSessions(t *testing.T) { - s, _ := store.NewTestStore(t) + s, _ := store.NewManagedTestStore(t) tenant, idle, idleEnvironment := managedSession(t, s) initial, err := s.CreateSession(t.Context(), tenant, store.CreateSessionInput{Creator: store.FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"openai_hosted"}}`), InitialInputs: []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"hello"}`)}}}) if err != nil { @@ -26,7 +26,7 @@ func TestManagedRuntimeAutomaticBootstrapRecoversCommittedSessions(t *testing.T) key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}} start := func() *execution.Worker { - w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: s, Registry: gateway.NewRegistry(), ManagedRuntimes: &execution.RuntimeProviders{CoreURL: "http://core.invalid/api/v1", DefaultProvider: key, Providers: map[string]sandbox.Provider{key: p}}}) + w, err := execution.StartWorker(t.Context(), &execution.Dispatcher{Store: s, Registry: gateway.NewRegistry(), ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: p}}) if err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/runtime_suspension.go b/services/agents-api/internal/store/runtime_suspension.go new file mode 100644 index 000000000..a1365f9da --- /dev/null +++ b/services/agents-api/internal/store/runtime_suspension.go @@ -0,0 +1,167 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +// RuntimeActivity separates real work from the connection keepalive. +type RuntimeActivity struct { + LastActivity time.Time + Busy, WakeRequested, HasCompletedTurn bool +} + +// SetRuntimeCompute commits an operation phase before its external effects. +// Revision and the existing Session lock fence a stale lifecycle observation. +func (s *Store) SetRuntimeCompute(ctx context.Context, owner RuntimeAllocation, phase string, state json.RawMessage, retainedUntil *time.Time, idleBefore time.Time) (RuntimeAllocation, error) { + if !runtimeComputeTransition(owner.ComputePhase, phase) || !json.Valid(state) || (owner.ComputePhase == "running" && phase == "quiescing" && idleBefore.IsZero()) { + return RuntimeAllocation{}, ErrInvalidInput + } + if phase != "running" && (retainedUntil == nil || retainedUntil.IsZero()) { + return RuntimeAllocation{}, ErrInvalidInput + } + var object map[string]json.RawMessage + if json.Unmarshal(state, &object) != nil || object == nil { + return RuntimeAllocation{}, ErrInvalidInput + } + return s.mutateRuntimeAllocation(ctx, owner, true, func(ctx context.Context, q *sqlc.Queries, row sqlc.RuntimeAllocation) (sqlc.RuntimeAllocation, error) { + if row.ComputeRevision != owner.ComputeRevision || row.ComputePhase != owner.ComputePhase { + return sqlc.RuntimeAllocation{}, ErrTurnConflict + } + if (phase == "quiescing" && row.ComputePhase == "running") || (phase == "suspending" && row.ComputePhase == "quiescing") { + activity, err := q.GetRuntimeActivity(ctx, row.ID) + if err != nil { + return sqlc.RuntimeAllocation{}, err + } + if activity.Busy || activity.ComputeWakeRequested { + return sqlc.RuntimeAllocation{}, ErrTurnConflict + } + if phase == "quiescing" && (!activity.HasCompletedTurn || activity.LastActivity.Time.After(idleBefore) || row.ComputeActivityAt.Time.After(owner.ComputeActivityAt)) { + return sqlc.RuntimeAllocation{}, ErrTurnConflict + } + } + until := pgtype.Timestamptz{} + if retainedUntil != nil { + until = pgtype.Timestamptz{Time: *retainedUntil, Valid: true} + } + return q.SetRuntimeCompute(ctx, sqlc.SetRuntimeComputeParams{ + ID: row.ID, Revision: row.ComputeRevision, Phase: phase, State: state, RetainedUntil: until, + }) + }) +} + +func runtimeComputeTransition(from, to string) bool { + if from == to { + return from != "disabled" + } + switch from { + case "disabled": + return to == "running" + case "running": + return to == "quiescing" + case "quiescing": + return to == "suspending" || to == "waking" || to == "running" + case "suspending": + return to == "suspended" || to == "waking" + case "suspended": + return to == "restoring" + case "restoring": + return to == "waking" + case "waking": + return to == "running" + } + return false +} + +func (s *Store) RuntimeActivity(ctx context.Context, owner RuntimeAllocation) (RuntimeActivity, error) { + id, err := parseID(owner.ID) + if err != nil { + return RuntimeActivity{}, err + } + if err := s.CheckExecutionOwnership(ctx); err != nil { + return RuntimeActivity{}, err + } + row, err := s.queries.GetRuntimeActivity(ctx, id) + if errors.Is(err, pgx.ErrNoRows) { + return RuntimeActivity{}, ErrNotFound + } + if err != nil { + return RuntimeActivity{}, err + } + return RuntimeActivity{LastActivity: row.LastActivity.Time, Busy: row.Busy, WakeRequested: row.ComputeWakeRequested, HasCompletedTurn: row.HasCompletedTurn}, nil +} + +// TouchRuntimeActivity is used only by operations requiring live compute. +// Public history and published-artifact reads do not call it. +func (s *Store) TouchRuntimeActivity(ctx context.Context, tenant, environment string) error { + lookup, err := deviceLookup(tenant, environment) + if err != nil { + return err + } + owned, err := s.GetEnvironment(ctx, tenant, environment) + if err != nil { + return err + } + return s.withPublicSession(ctx, tenant, owned.SessionID, func(ctx context.Context, q *sqlc.Queries, session pgtype.UUID) error { + current, err := q.GetEnvironment(ctx, sqlc.GetEnvironmentParams{TenantID: lookup.TenantID, ID: lookup.ID}) + if errors.Is(err, pgx.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return err + } + if current.Environment.SessionID != session { + return ErrNotFound + } + return q.TouchRuntimeActivity(ctx, sqlc.TouchRuntimeActivityParams{TenantID: lookup.TenantID, EnvironmentID: lookup.ID}) + }) +} + +func (s *Store) ClearRuntimeWake(ctx context.Context, owner RuntimeAllocation, observedActivity time.Time) error { + id, err := parseID(owner.ID) + if err != nil { + return err + } + if err := s.CheckExecutionOwnership(ctx); err != nil { + return err + } + return s.queries.ClearRuntimeWake(ctx, sqlc.ClearRuntimeWakeParams{ID: id, ComputeActivityAt: pgtype.Timestamptz{Time: observedActivity, Valid: true}}) +} + +func (s *Store) CountRuntimeComputeReservations(ctx context.Context, provider string) (int64, error) { + id, err := parseID(provider) + if err != nil { + return 0, err + } + if err := s.CheckExecutionOwnership(ctx); err != nil { + return 0, err + } + return s.queries.CountRuntimeComputeReservations(ctx, id) +} + +func (s *Store) CountRuntimeRetainedAllocations(ctx context.Context, provider string) (int64, error) { + id, err := parseID(provider) + if err != nil { + return 0, err + } + if err := s.CheckExecutionOwnership(ctx); err != nil { + return 0, err + } + return s.queries.CountRuntimeRetainedAllocations(ctx, id) +} + +// checkRuntimeComputeAdmission runs under the Session lock before a new durable +// execution owner is created. Existing receipts remain readable in every phase. +func checkRuntimeComputeAdmission(ctx context.Context, q *sqlc.Queries, session pgtype.UUID) error { + blocked, err := q.RuntimeComputeBlocksAdmission(ctx, session) + if err == nil && blocked { + return ErrTurnConflict + } + return err +} diff --git a/services/agents-api/internal/store/runtime_suspension_concurrency_test.go b/services/agents-api/internal/store/runtime_suspension_concurrency_test.go new file mode 100644 index 000000000..aa5bd4db8 --- /dev/null +++ b/services/agents-api/internal/store/runtime_suspension_concurrency_test.go @@ -0,0 +1,218 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func runtimeSuspensionLockedSession(t *testing.T, pool *pgxpool.Pool, session string) (context.Context, pgx.Tx, int32) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + t.Cleanup(cancel) + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = tx.Rollback(context.Background()) }) + var blocker int32 + if err := tx.QueryRow(ctx, "SELECT pg_backend_pid() FROM sessions WHERE id=$1 FOR UPDATE", session).Scan(&blocker); err != nil { + t.Fatal(err) + } + return ctx, tx, blocker +} + +func runtimeSuspensionWaitBlocked(t *testing.T, ctx context.Context, pool *pgxpool.Pool, blocker int32, done <-chan error) { + t.Helper() + for { + var blocked bool + if err := pool.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE $1=ANY(pg_blocking_pids(pid)))", blocker).Scan(&blocked); err != nil { + t.Fatal(err) + } + if blocked { + return + } + select { + case err := <-done: + t.Fatal("operation bypassed Session lock", err) + case <-ctx.Done(): + t.Fatal("Session lock wait not observed") + case <-time.After(5 * time.Millisecond): + } + } +} + +func TestRuntimeSuspensionClaimReadsPhaseAfterSessionLock(t *testing.T) { + for _, phase := range []string{"disabled", "running", "quiescing", "suspending", "suspended", "restoring", "waking"} { + t.Run(phase, func(t *testing.T) { + s, w, pool, owner := runtimeSuspensionFixture(t) + turn := uuid.NewString() + runtimeSuspensionSQL(t, pool, `INSERT INTO turns(id,session_id,status) VALUES($1,$2,'queued')`, turn, owner.SessionID) + ctx, tx, blocker := runtimeSuspensionLockedSession(t, pool, owner.SessionID) + done := make(chan error, 1) + go func() { + _, err := w.TransitionTurn(ctx, owner.TenantID, owner.SessionID, turn, TurnTransition{ExpectedStatus: TurnQueued, Status: TurnInProgress}) + done <- err + }() + runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, done) + // The lifecycle commits while a stale dispatcher is waiting to claim. + if _, err := tx.Exec(ctx, `UPDATE runtime_allocations SET compute_phase=$2,compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, owner.ID, phase); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + err := <-done + blocked := phase != "disabled" && phase != "running" + if blocked && !errors.Is(err, ErrTurnConflict) || !blocked && err != nil { + t.Fatal("incorrect claim outcome", phase, err) + } + got, err := s.GetTurn(ctx, owner.TenantID, owner.SessionID, turn) + if err != nil { + t.Fatal(err) + } + var startedEvents int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM session_events WHERE session_id=$1 AND payload->'event'->>'type'='agent.session.turn.in_progress'`, owner.SessionID).Scan(&startedEvents); err != nil { + t.Fatal(err) + } + if blocked && (got.Status != TurnQueued || !got.StartedAt.IsZero() || startedEvents != 0) { + t.Fatal("blocked claim changed Turn or projected start", got, startedEvents) + } + if !blocked && (got.Status != TurnInProgress || got.StartedAt.IsZero() || startedEvents != 1) { + t.Fatal("ordinary compute no longer starts work", got, startedEvents) + } + }) + } +} + +func TestRuntimeSuspensionPromotionRetainsPendingInput(t *testing.T) { + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_phase='waking',compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, owner.ID) + pending := reserveEnvironmentInput(t, s, owner.TenantID, owner.SessionID, "during-wake") + if _, err := w.PromoteEnvironmentInput(t.Context(), owner.TenantID, owner.SessionID, pending.ID); !errors.Is(err, ErrTurnConflict) { + t.Fatal("waking allocation promoted model input", err) + } + got, err := s.GetEnvironmentInputReservation(t.Context(), owner.TenantID, owner.SessionID, pending.ID) + if err != nil || got.State != EnvironmentInputPending || got.SettledAt != nil || len(got.Receipts) != 0 { + t.Fatal("blocked promotion partially committed", got, err) + } + environmentInputHistory(t, pool, owner.SessionID, 0, 0) + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_phase='running',compute_retained_until=NULL WHERE id=$1`, owner.ID) + got, err = w.PromoteEnvironmentInput(t.Context(), owner.TenantID, owner.SessionID, pending.ID) + if err != nil || got.State != EnvironmentInputAdmitted || len(got.Receipts) != 2 { + t.Fatal("pending request could not resume once running", got, err) + } +} + +func TestRuntimeSuspensionCaptureRechecksNewPendingWork(t *testing.T) { + for _, kind := range []string{"queued", "input", "file_write", "wake"} { + t.Run(kind, func(t *testing.T) { + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) + until := time.Now().Add(time.Hour) + owner = runtimeSuspensionStep(t, w, owner, "quiescing", &until) + ctx, tx, blocker := runtimeSuspensionLockedSession(t, pool, owner.SessionID) + done := make(chan error, 1) + go func() { + _, err := w.SetRuntimeCompute(ctx, owner, "suspending", json.RawMessage(`{}`), &until, time.Time{}) + done <- err + }() + runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, done) + var err error + switch kind { + case "queued": + _, err = tx.Exec(ctx, `INSERT INTO turns(id,session_id,status) VALUES($1,$2,'queued')`, uuid.NewString(), owner.SessionID) + case "input": + _, err = tx.Exec(ctx, `INSERT INTO environment_input_reservations(id,session_id,idempotency_key,batch,created_at,deadline) VALUES($1,$2,'pending','[{}]',clock_timestamp(),clock_timestamp()+interval '1 minute')`, uuid.NewString(), owner.SessionID) + case "file_write": + _, err = tx.Exec(ctx, `INSERT INTO environment_file_writes(id,environment_id,device_id,request_sha256) VALUES($1,$2,$3,$4)`, uuid.NewString(), owner.EnvironmentID, owner.DeviceID, strings.Repeat("a", 64)) + case "wake": + _, err = tx.Exec(ctx, `UPDATE runtime_allocations SET compute_wake_requested=true,compute_activity_at=clock_timestamp() WHERE id=$1`, owner.ID) + } + if err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err := <-done; !errors.Is(err, ErrTurnConflict) { + t.Fatal("capture ignored work admitted after quiesce", err) + } + got, err := s.GetRuntimeAllocation(ctx, owner.TenantID, owner.EnvironmentID) + if err != nil || got.ComputePhase != "quiescing" || got.ComputeRevision != owner.ComputeRevision { + t.Fatal("failed capture CAS changed its owner", got, err) + } + }) + } +} + +func TestRuntimeSuspensionWakeUsesSessionLock(t *testing.T) { + for _, deleted := range []bool{false, true} { + t.Run(map[bool]string{false: "phase_commit", true: "deletion"}[deleted], func(t *testing.T) { + s, _, pool, owner := runtimeSuspensionFixture(t) + ctx, tx, blocker := runtimeSuspensionLockedSession(t, pool, owner.SessionID) + done := make(chan error, 1) + go func() { done <- s.TouchRuntimeActivity(ctx, owner.TenantID, owner.EnvironmentID) }() + runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, done) + if deleted { + if _, err := tx.Exec(ctx, `UPDATE sessions SET deleted_at=clock_timestamp() WHERE id=$1`, owner.SessionID); err != nil { + t.Fatal(err) + } + } else if _, err := tx.Exec(ctx, `UPDATE runtime_allocations SET compute_phase='suspending',compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, owner.ID); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + err := <-done + if deleted && !errors.Is(err, ErrNotFound) || !deleted && err != nil { + t.Fatal("wake did not observe locked state", err) + } + got, err := s.GetRuntimeAllocation(ctx, owner.TenantID, owner.EnvironmentID) + if err != nil || got.ComputeWakeRequested == deleted { + t.Fatal("wake was lost or crossed deletion", got, err) + } + }) + } +} + +func TestRuntimeSuspensionQuiesceCannotOvertakeClaim(t *testing.T) { + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) + until := time.Now().Add(time.Hour) + ctx, tx, blocker := runtimeSuspensionLockedSession(t, pool, owner.SessionID) + done := make(chan error, 1) + go func() { + _, err := w.SetRuntimeCompute(ctx, owner, "quiescing", json.RawMessage(`{}`), &until, time.Now()) + done <- err + }() + runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, done) + turn := uuid.NewString() + if _, err := tx.Exec(ctx, `INSERT INTO turns(id,session_id,status) VALUES($1,$2,'queued')`, turn, owner.SessionID); err != nil { + t.Fatal(err) + } + params, err := turnLookup(owner.TenantID, owner.SessionID, turn) + if err != nil { + t.Fatal(err) + } + if _, err := transitionTurn(ctx, w.queries.WithTx(tx), params, TurnTransition{ExpectedStatus: TurnQueued, Status: TurnInProgress, Outcome: json.RawMessage(`{}`)}); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err := <-done; !errors.Is(err, ErrTurnConflict) { + t.Fatal("quiesce overtook an admitted Turn", err) + } + got, err := s.GetRuntimeAllocation(ctx, owner.TenantID, owner.EnvironmentID) + if err != nil || got.ComputePhase != "running" || got.ComputeRevision != owner.ComputeRevision { + t.Fatal("active work was quiesced", got, err) + } +} diff --git a/services/agents-api/internal/store/runtime_suspension_test.go b/services/agents-api/internal/store/runtime_suspension_test.go new file mode 100644 index 000000000..5c80e91be --- /dev/null +++ b/services/agents-api/internal/store/runtime_suspension_test.go @@ -0,0 +1,424 @@ +package store + +import ( + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +func runtimeSuspensionFixture(t *testing.T) (*Store, *Store, *pgxpool.Pool, RuntimeAllocation) { + t.Helper() + s, pool := testStore(t) + w := executionLease(t, s).Store() + tenant := uuid.NewString() + _, environment := localEnvironment(t, s, tenant) + owner, err := w.ReserveRuntimeAllocation(t.Context(), tenant, environment.ID, uuid.NewString(), device.HashCredential(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + owner, err = w.ObserveRuntimeRunning(t.Context(), owner) + if err != nil { + t.Fatal(err) + } + owner, err = w.SetRuntimeCompute(t.Context(), owner, "running", json.RawMessage(`{"instance":"original"}`), nil, time.Time{}) + if err != nil { + t.Fatal(err) + } + return s, w, pool, owner +} + +func runtimeSuspensionSQL(t *testing.T, pool *pgxpool.Pool, query string, args ...any) { + t.Helper() + if _, err := pool.Exec(t.Context(), query, args...); err != nil { + t.Fatal(err) + } +} + +func runtimeSuspensionCompleted(t *testing.T, pool *pgxpool.Pool, owner RuntimeAllocation) string { + t.Helper() + id := uuid.NewString() + runtimeSuspensionSQL(t, pool, `INSERT INTO turns(id,session_id,status,completed_at) VALUES($1,$2,'completed',clock_timestamp()-interval '10 minutes')`, id, owner.SessionID) + return id +} + +func runtimeSuspensionStep(t *testing.T, w *Store, owner RuntimeAllocation, phase string, until *time.Time) RuntimeAllocation { + t.Helper() + idleBefore := time.Time{} + if owner.ComputePhase == "running" && phase == "quiescing" { + idleBefore = time.Now() + } + next, err := w.SetRuntimeCompute(t.Context(), owner, phase, json.RawMessage(`{"instance":"original","snapshot":"qualified"}`), until, idleBefore) + if err != nil { + t.Fatalf("%s -> %s: %v", owner.ComputePhase, phase, err) + } + return next +} + +func TestRuntimeSuspensionRequiresCompletedIdleAndNoPendingWork(t *testing.T) { + cases := []string{"no_completed_turn", "queued", "in_progress", "waiting", "subagent_queued", "subagent_in_progress", "subagent_waiting", "input_reservation", "file_write", "idle"} + for _, kind := range cases { + t.Run(kind, func(t *testing.T) { + _, w, pool, owner := runtimeSuspensionFixture(t) + completed := "" + if kind != "no_completed_turn" { + completed = runtimeSuspensionCompleted(t, pool, owner) + } + switch kind { + case "queued", "in_progress", "waiting": + runtimeSuspensionSQL(t, pool, `INSERT INTO turns(id,session_id,status) VALUES($1,$2,$3)`, uuid.NewString(), owner.SessionID, kind) + case "subagent_queued", "subagent_in_progress", "subagent_waiting": + child := uuid.NewString() + runtimeSuspensionSQL(t, pool, `INSERT INTO turn_events(session_id,turn_id,ordinal,kind,payload) VALUES($1,$2,1,'subagent','{}')`, owner.SessionID, completed) + runtimeSuspensionSQL(t, pool, `INSERT INTO subagent_identities(id,session_id,device_id,engine,native_id,parent_native_id,native_created_at,first_turn_id,first_event_ordinal) VALUES($1,$2,$3,'codex','child','root',1,$4,1)`, child, owner.SessionID, owner.DeviceID, completed) + runtimeSuspensionSQL(t, pool, `INSERT INTO subagent_turns(id,session_id,subagent_id,native_id,status,created_at) VALUES($1,$2,$3,'child-turn',$4,clock_timestamp())`, uuid.NewString(), owner.SessionID, child, strings.TrimPrefix(kind, "subagent_")) + case "input_reservation": + runtimeSuspensionSQL(t, pool, `INSERT INTO environment_input_reservations(id,session_id,idempotency_key,batch,created_at,deadline) VALUES($1,$2,'pending','[{}]',clock_timestamp(),clock_timestamp()+interval '1 minute')`, uuid.NewString(), owner.SessionID) + case "file_write": + runtimeSuspensionSQL(t, pool, `INSERT INTO environment_file_writes(id,environment_id,device_id,request_sha256) VALUES($1,$2,$3,$4)`, uuid.NewString(), owner.EnvironmentID, owner.DeviceID, strings.Repeat("a", 64)) + } + activity, err := w.RuntimeActivity(t.Context(), owner) + if err != nil { + t.Fatal(err) + } + wantBusy := kind != "idle" && kind != "no_completed_turn" + if activity.Busy != wantBusy || activity.HasCompletedTurn != (kind != "no_completed_turn") { + t.Fatalf("activity lost pending work: %+v", activity) + } + until := time.Now().Add(time.Hour) + _, err = w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, time.Now()) + if kind == "idle" { + if err != nil { + t.Fatal(err) + } + } else if !errors.Is(err, ErrTurnConflict) { + t.Fatalf("unsafe quiesce admitted: %v", err) + } + }) + } +} + +func TestRuntimeSuspensionCASAndActivityFence(t *testing.T) { + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) + until := time.Now().Add(time.Hour) + var wg sync.WaitGroup + results := make(chan error, 2) + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + _, err := w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{"operation":"same-observation"}`), &until, time.Now()) + results <- err + }() + } + wg.Wait() + close(results) + winners, conflicts := 0, 0 + for err := range results { + if err == nil { + winners++ + } else if errors.Is(err, ErrTurnConflict) { + conflicts++ + } else { + t.Fatal(err) + } + } + if winners != 1 || conflicts != 1 { + t.Fatalf("CAS admitted competing owners: wins=%d conflicts=%d", winners, conflicts) + } + current, err := s.GetRuntimeAllocation(t.Context(), owner.TenantID, owner.EnvironmentID) + if err != nil { + t.Fatal(err) + } + if current.ComputeRevision != owner.ComputeRevision+1 || current.ComputePhase != "quiescing" { + t.Fatal("operation intent not durable", current) + } + if _, err := w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{"stale":true}`), &until, time.Now()); !errors.Is(err, ErrTurnConflict) { + t.Fatal("stale phase overwrite", err) + } + current = runtimeSuspensionStep(t, w, current, "running", nil) + observed := current + if err := s.TouchRuntimeActivity(t.Context(), owner.TenantID, owner.EnvironmentID); err != nil { + t.Fatal(err) + } + // Clearing an observed wake cannot let an older observation authorize sleep. + latest, err := s.GetRuntimeAllocation(t.Context(), owner.TenantID, owner.EnvironmentID) + if err != nil { + t.Fatal(err) + } + if err := w.ClearRuntimeWake(t.Context(), latest, latest.ComputeActivityAt); err != nil { + t.Fatal(err) + } + if _, err := w.SetRuntimeCompute(t.Context(), observed, "quiescing", json.RawMessage(`{}`), &until, time.Now()); !errors.Is(err, ErrTurnConflict) { + t.Fatal("newer activity was swallowed", err) + } + for _, invalid := range []json.RawMessage{json.RawMessage(`[]`), json.RawMessage(`null`), json.RawMessage(`false`), json.RawMessage(`{`)} { + if _, err := w.SetRuntimeCompute(t.Context(), latest, "running", invalid, nil, time.Time{}); !errors.Is(err, ErrInvalidInput) { + t.Fatal("non-object compute state accepted", string(invalid), err) + } + } + if _, err := w.SetRuntimeCompute(t.Context(), latest, "suspended", json.RawMessage(`{}`), &until, time.Time{}); !errors.Is(err, ErrInvalidInput) { + t.Fatal("running skipped snapshot protocol", err) + } +} + +func TestRuntimeSuspensionWakeDoesNotLoseNewerWork(t *testing.T) { + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) + before, err := w.RuntimeActivity(t.Context(), owner) + if err != nil { + t.Fatal(err) + } + if _, err := w.KeepRuntimeAllocation(t.Context(), owner); err != nil { + t.Fatal(err) + } + if _, err := s.GetSession(t.Context(), owner.TenantID, owner.SessionID); err != nil { + t.Fatal(err) + } + quiet, err := w.RuntimeActivity(t.Context(), owner) + if err != nil || quiet.WakeRequested || !quiet.LastActivity.Equal(before.LastActivity) { + t.Fatal("heartbeat or history read touched compute activity", quiet, err) + } + if err := s.TouchRuntimeActivity(t.Context(), uuid.NewString(), owner.EnvironmentID); !errors.Is(err, ErrNotFound) { + t.Fatal(err) + } + foreign, err := w.RuntimeActivity(t.Context(), owner) + if err != nil || foreign.WakeRequested { + t.Fatal("foreign tenant woke environment", foreign, err) + } + if err := s.TouchRuntimeActivity(t.Context(), owner.TenantID, owner.EnvironmentID); err != nil { + t.Fatal(err) + } + first, err := s.GetRuntimeAllocation(t.Context(), owner.TenantID, owner.EnvironmentID) + if err != nil { + t.Fatal(err) + } + if err := s.TouchRuntimeActivity(t.Context(), owner.TenantID, owner.EnvironmentID); err != nil { + t.Fatal(err) + } + second, err := s.GetRuntimeAllocation(t.Context(), owner.TenantID, owner.EnvironmentID) + if err != nil || !second.ComputeActivityAt.After(first.ComputeActivityAt) { + t.Fatal("Touch did not record newer activity", err) + } + if err := w.ClearRuntimeWake(t.Context(), first, first.ComputeActivityAt); err != nil { + t.Fatal(err) + } + activity, err := w.RuntimeActivity(t.Context(), owner) + if err != nil || !activity.WakeRequested { + t.Fatal("old wake receipt swallowed newer work", activity, err) + } + if err := w.ClearRuntimeWake(t.Context(), second, second.ComputeActivityAt); err != nil { + t.Fatal(err) + } + activity, err = w.RuntimeActivity(t.Context(), owner) + if err != nil || activity.WakeRequested { + t.Fatal("current wake receipt did not settle", activity, err) + } +} + +func TestRuntimeSuspensionRetentionAndDeletedSession(t *testing.T) { + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) + until := time.Now().Add(time.Hour) + for _, phase := range []string{"quiescing", "suspending", "suspended"} { + owner = runtimeSuspensionStep(t, w, owner, phase, &until) + } + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '2 hours' WHERE id=$1`, owner.ID) + retained, err := s.GetRuntimeAllocation(t.Context(), owner.TenantID, owner.EnvironmentID) + if err != nil || retained.Expired { + t.Fatal("suspended snapshot expired by disconnected heartbeat", retained, err) + } + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID) + expired, err := s.GetRuntimeAllocation(t.Context(), owner.TenantID, owner.EnvironmentID) + if err != nil || !expired.Expired { + t.Fatal("snapshot retention expiry not observed", expired, err) + } + // Use the earlier unexpired observation to exercise expiry at the database CAS. + if _, err := w.SetRuntimeCompute(t.Context(), retained, "restoring", json.RawMessage(`{}`), &until, time.Time{}); !errors.Is(err, ErrTurnConflict) { + t.Fatal("expired snapshot restored from stale observation", err) + } + if err := s.DeleteSession(t.Context(), owner.TenantID, owner.SessionID); err != nil { + t.Fatal(err) + } + if err := s.TouchRuntimeActivity(t.Context(), owner.TenantID, owner.EnvironmentID); !errors.Is(err, ErrNotFound) { + t.Fatal(err) + } + deleted, err := s.GetRuntimeAllocation(t.Context(), owner.TenantID, owner.EnvironmentID) + if err != nil || !deleted.SessionDeleted || deleted.ComputeWakeRequested { + t.Fatal("deleted session was woken", deleted, err) + } + if _, err := w.SetRuntimeCompute(t.Context(), deleted, "restoring", json.RawMessage(`{}`), &until, time.Time{}); !errors.Is(err, ErrNotFound) { + t.Fatal("deleted session restored", err) + } +} + +func TestRuntimeSuspensionCountsUncertainCapacityUntilReleased(t *testing.T) { + s, pool := testStore(t) + w := executionLease(t, s).Store() + provider := uuid.NewString() + cases := []struct { + state, phase string + count bool + }{ + {"creating", "disabled", true}, {"running", "running", true}, {"running", "quiescing", true}, {"running", "suspending", true}, {"running", "suspended", false}, {"running", "restoring", true}, {"running", "waking", true}, {"cleanup_pending", "restoring", true}, {"released", "running", false}, + } + want := int64(0) + wantRetained := int64(0) + for _, item := range cases { + tenant := uuid.NewString() + _, env := localEnvironment(t, s, tenant) + owner, err := w.ReserveRuntimeAllocation(t.Context(), tenant, env.ID, provider, device.HashCredential(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET state=$2,compute_phase=$3,create_settled=($2<>'creating'),released_at=CASE WHEN $2='released' THEN clock_timestamp() END WHERE id=$1`, owner.ID, item.state, item.phase) + if item.count { + want++ + } + if item.state != "released" { + wantRetained++ + } + retained, err := w.CountRuntimeRetainedAllocations(t.Context(), provider) + if err != nil || retained != wantRetained { + t.Fatalf("retained capacity state=%s phase=%s got=%d want=%d err=%v", item.state, item.phase, retained, wantRetained, err) + } + got, err := w.CountRuntimeComputeReservations(t.Context(), provider) + if err != nil || got != want { + t.Fatalf("capacity state=%s phase=%s got=%d want=%d err=%v", item.state, item.phase, got, want, err) + } + } + got, err := w.CountRuntimeComputeReservations(t.Context(), uuid.NewString()) + if err != nil || got != 0 { + t.Fatal("capacity crossed installation boundary", got, err) + } +} + +func TestRuntimeSuspensionIdleStartsAfterLastCompletion(t *testing.T) { + _, w, pool, owner := runtimeSuspensionFixture(t) + turn := runtimeSuspensionCompleted(t, pool, owner) + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 hours' WHERE id=$1`, owner.ID) + var completed time.Time + if err := pool.QueryRow(t.Context(), `SELECT completed_at FROM turns WHERE id=$1`, turn).Scan(&completed); err != nil { + t.Fatal(err) + } + activity, err := w.RuntimeActivity(t.Context(), owner) + if err != nil || !activity.LastActivity.Equal(completed) { + t.Fatal("long Turn completion did not restart idle interval", activity, completed, err) + } +} + +func TestRuntimeSuspensionWakeRemainsUntilRunning(t *testing.T) { + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) + until := time.Now().Add(time.Hour) + for _, phase := range []string{"quiescing", "suspending", "suspended"} { + owner = runtimeSuspensionStep(t, w, owner, phase, &until) + } + if err := s.TouchRuntimeActivity(t.Context(), owner.TenantID, owner.EnvironmentID); err != nil { + t.Fatal(err) + } + observed, err := s.GetRuntimeAllocation(t.Context(), owner.TenantID, owner.EnvironmentID) + if err != nil { + t.Fatal(err) + } + if err := w.ClearRuntimeWake(t.Context(), owner, observed.ComputeActivityAt); err != nil { + t.Fatal(err) + } + activity, err := w.RuntimeActivity(t.Context(), owner) + if err != nil || !activity.WakeRequested { + t.Fatal("wake cleared before compute was running", activity, err) + } + for _, phase := range []string{"restoring", "waking"} { + owner = runtimeSuspensionStep(t, w, owner, phase, &until) + } + owner = runtimeSuspensionStep(t, w, owner, "running", nil) + if owner.ComputeRetainedUntil != nil { + t.Fatal("running retained snapshot expiration") + } + if err := w.ClearRuntimeWake(t.Context(), owner, observed.ComputeActivityAt); err != nil { + t.Fatal(err) + } + activity, err = w.RuntimeActivity(t.Context(), owner) + if err != nil || activity.WakeRequested { + t.Fatal("running compute could not settle wake", activity, err) + } +} + +func TestRuntimeSuspensionExpiredRunningAndLostWriterAreFenced(t *testing.T) { + _, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) + until := time.Now().Add(time.Hour) + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '2 hours' WHERE id=$1`, owner.ID) + if _, err := w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, time.Now()); !errors.Is(err, ErrTurnConflict) { + t.Fatal("expired running allocation entered checkpoint", err) + } + if err := w.executionLease.Close(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := w.RuntimeActivity(t.Context(), owner); err == nil { + t.Fatal("lost writer read execution activity") + } + if err := w.ClearRuntimeWake(t.Context(), owner, owner.ComputeActivityAt); err == nil { + t.Fatal("lost writer changed wake receipt") + } + if _, err := w.CountRuntimeComputeReservations(t.Context(), owner.ProviderKey); err == nil { + t.Fatal("lost writer admitted running capacity") + } + if _, err := w.CountRuntimeRetainedAllocations(t.Context(), owner.ProviderKey); err == nil { + t.Fatal("lost writer admitted retained capacity") + } +} + +func TestRuntimeSuspensionRechecksCompletionAgainstIdleCutoff(t *testing.T) { + for _, kind := range []string{"root", "subagent", "file_committed", "file_rejected"} { + t.Run(kind, func(t *testing.T) { + s, w, pool, owner := runtimeSuspensionFixture(t) + turn := runtimeSuspensionCompleted(t, pool, owner) + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '20 minutes' WHERE id=$1`, owner.ID) + var err error + owner, err = s.GetRuntimeAllocation(t.Context(), owner.TenantID, owner.EnvironmentID) + if err != nil { + t.Fatal(err) + } + idleBefore := time.Now().Add(-time.Minute) + observed, err := w.RuntimeActivity(t.Context(), owner) + if err != nil || observed.Busy || observed.LastActivity.After(idleBefore) { + t.Fatal("fixture is not initially idle", observed, err) + } + // A completion can arrive after the lifecycle's idle observation without + // changing the allocation revision or its explicit wake timestamp. + switch kind { + case "root": + runtimeSuspensionSQL(t, pool, `UPDATE turns SET completed_at=clock_timestamp() WHERE id=$1`, turn) + case "subagent": + child := uuid.NewString() + runtimeSuspensionSQL(t, pool, `INSERT INTO turn_events(session_id,turn_id,ordinal,kind,payload) VALUES($1,$2,1,'subagent','{}')`, owner.SessionID, turn) + runtimeSuspensionSQL(t, pool, `INSERT INTO subagent_identities(id,session_id,device_id,engine,native_id,parent_native_id,native_created_at,first_turn_id,first_event_ordinal) VALUES($1,$2,$3,'codex','child','root',1,$4,1)`, child, owner.SessionID, owner.DeviceID, turn) + runtimeSuspensionSQL(t, pool, `INSERT INTO subagent_turns(id,session_id,subagent_id,native_id,status,created_at,completed_at) VALUES($1,$2,$3,'child-turn','completed',clock_timestamp()-interval '10 minutes',clock_timestamp())`, uuid.NewString(), owner.SessionID, child) + default: + runtimeSuspensionSQL(t, pool, `INSERT INTO environment_file_writes(id,environment_id,device_id,request_sha256,state,created_at,settled_at) VALUES($1,$2,$3,$4,$5,clock_timestamp()-interval '10 minutes',clock_timestamp())`, uuid.NewString(), owner.EnvironmentID, owner.DeviceID, strings.Repeat("a", 64), strings.TrimPrefix(kind, "file_")) + } + until := time.Now().Add(time.Hour) + if _, err := w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, idleBefore); !errors.Is(err, ErrTurnConflict) { + t.Fatal("completion after idle observation did not fence quiesce", err) + } + activity, err := w.RuntimeActivity(t.Context(), owner) + if err != nil || activity.Busy || activity.WakeRequested || !activity.LastActivity.After(idleBefore) { + t.Fatal("last completion did not restart idle interval", activity, err) + } + if _, err := w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, time.Time{}); !errors.Is(err, ErrInvalidInput) { + t.Fatal("missing idle cutoff accepted", err) + } + if _, err := w.SetRuntimeCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, activity.LastActivity); err != nil { + t.Fatal("fully elapsed idle boundary rejected", err) + } + }) + } +} diff --git a/services/agents-api/internal/store/runtime_worker_recovery_test.go b/services/agents-api/internal/store/runtime_worker_recovery_test.go new file mode 100644 index 000000000..b99dbb6ed --- /dev/null +++ b/services/agents-api/internal/store/runtime_worker_recovery_test.go @@ -0,0 +1,168 @@ +package store_test + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +func insertWorkerRuntimeAllocation(t *testing.T, pool *pgxpool.Pool, h *dispatchHarness, phase string) { + t.Helper() + _, err := pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,initialization,compute_phase,compute_retained_until) + VALUES($1,$2,$3,$4,'running',true,'complete',$5,clock_timestamp()+interval '1 hour')`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase) + if err != nil { + t.Fatal(err) + } +} + +func runtimeWorkerHarness(t *testing.T) (*dispatchHarness, *pgxpool.Pool) { + t.Helper() + h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}`), true) + enableWorkerEnvironment(t, h) + _, pool := store.NewTestStore(t) + insertWorkerRuntimeAllocation(t, pool, h, "waking") + return h, pool +} + +func TestPreparedDispatchKeepsPendingReservationAfterComputeConflict(t *testing.T) { + h, _ := runtimeWorkerHarness(t) + lease, err := h.s.AcquireExecutionLease(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = lease.Close(context.Background()) }) + h.d.Store = lease.Store() + pending, err := h.s.ReserveEnvironmentInput(t.Context(), h.tenant, h.session.ID, "pending", []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"first"}`)}}) + if err != nil { + t.Fatal(err) + } + result := runPreparedDispatch(h, t.Context(), pending) + frame := h.read(proto.TypeExecutionPrepare) + handle := acknowledgePreparation(h, frame.ID) + h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 2, State: "ready"}) + got := awaitPreparedDispatch(t, result) + if !errors.Is(got.err, store.ErrTurnConflict) || got.run.Reservation.ID != pending.ID || got.run.Reservation.State != store.EnvironmentInputPending { + t.Fatal("rejected promotion lost its pending owner", got) + } + assertPreparationReleased(t, h, frame.ID, handle) + session, err := h.s.GetSession(t.Context(), h.tenant, h.session.ID) + if err != nil || session.LastTurn != nil { + t.Fatal("blocked promotion started a Turn", session, err) + } +} + +func TestWorkerWaitsForComputeAndSurvivesPromotionConflict(t *testing.T) { + h, pool := runtimeWorkerHarness(t) + pending, err := h.s.ReserveEnvironmentInput(t.Context(), h.tenant, h.session.ID, "pending", []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"first"}`)}}) + if err != nil { + t.Fatal(err) + } + worker, err := execution.StartWorker(t.Context(), h.d) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- worker.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(15 * time.Second): + t.Error("worker did not stop") + } + }) + frames := workerFrames(t, h) + environment, err := h.s.GetSessionEnvironment(t.Context(), h.tenant, h.session.ID) + if err != nil { + t.Fatal(err) + } + // This enters the same binding gate as scheduled input and returns without + // a native preparation, despite the socket already advertising capabilities. + if _, err := worker.ReadEnvironmentDirectory(t.Context(), environment, ""); !errors.Is(err, execution.ErrExecutionUnavailable) { + t.Fatal("waking connection was treated as work-ready", err) + } + select { + case frame := <-frames: + t.Fatal("work preceded lifecycle resume", frame.Type) + default: + } + setPhase := func(phase string) { + t.Helper() + if _, err := pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase=$2 WHERE environment_id=$1`, environment.ID, phase); err != nil { + t.Fatal(err) + } + } + setPhase("running") + prepare := nextWorkerFrame(t, frames, proto.TypeExecutionPrepare) + handle := acknowledgePreparation(h, prepare.ID) + // Force the final authoritative Store fence after the readiness observation. + setPhase("waking") + h.write(prepare.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 2, State: "ready"}) + release := nextWorkerFrame(t, frames, proto.TypeExecutionRelease) + if release.ID != prepare.ID { + t.Fatal("conflicted preparation was not released") + } + stored, err := h.s.GetEnvironmentInputReservation(t.Context(), h.tenant, h.session.ID, pending.ID) + if err != nil || stored.State != store.EnvironmentInputPending || len(stored.Receipts) != 0 { + t.Fatal("conflict consumed queued input", stored, err) + } + setPhase("running") + prepare = nextWorkerFrame(t, frames, proto.TypeExecutionPrepare) + handle = acknowledgePreparation(h, prepare.ID) + h.write(prepare.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 2, State: "ready"}) + frame := nextWorkerFrame(t, frames, proto.TypeExecutionStart) + var start proto.ExecutionStartPayload + if frame.DecodePayload(&start) != nil || start.Handle != handle || inputTextForTest(t, start.Input) != "first" { + t.Fatal("retry changed pending work", start) + } + h.write(prepare.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 3, State: "started", RunID: start.RunID}) + h.write(start.RunID, proto.TypeDone, proto.DonePayload{Content: "completed once"}) + completeEmptyArtifactExport(t, h, frames) + if release := nextWorkerFrame(t, frames, proto.TypeExecutionRelease); release.ID != prepare.ID { + t.Fatal("completed preparation was not released") + } + awaitDaemonRemoteCondition(t, t.Context(), 5*time.Second, "original input completed once", func() bool { + turn, err := h.s.GetTurn(t.Context(), h.tenant, h.session.ID, start.RunID) + return err == nil && turn.Status == store.TurnCompleted + }) + var turns int + if err := pool.QueryRow(t.Context(), `SELECT count(*) FROM turns WHERE session_id=$1`, h.session.ID).Scan(&turns); err != nil || turns != 1 { + t.Fatal("retry duplicated the Turn", turns, err) + } + select { + case err := <-done: + done <- err + t.Fatal("recoverable promotion conflict stopped the worker", err) + default: + } +} + +func TestWorkerRestartPreservesQueuedTurnWhileComputeWakes(t *testing.T) { + h, pool := runtimeWorkerHarness(t) + turn := uuid.NewString() + if _, err := pool.Exec(t.Context(), `INSERT INTO turns(id,session_id,status) VALUES($1,$2,'queued')`, turn, h.session.ID); err != nil { + t.Fatal(err) + } + worker, err := execution.StartWorker(t.Context(), h.d) + if err != nil { + t.Fatal("queued wake blocked Core startup", err) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := worker.Run(ctx); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + got, err := h.s.GetTurn(t.Context(), h.tenant, h.session.ID, turn) + if err != nil || got.Status != store.TurnQueued || !got.StartedAt.IsZero() { + t.Fatal("startup consumed queued work before restore", got, err) + } +} diff --git a/services/agents-api/internal/store/session_initial_input.go b/services/agents-api/internal/store/session_initial_input.go index b124d6145..9a958e376 100644 --- a/services/agents-api/internal/store/session_initial_input.go +++ b/services/agents-api/internal/store/session_initial_input.go @@ -34,6 +34,19 @@ func (s *Store) createSessionResources(ctx context.Context, tenant string, param return err } if row.ID == params.ID { + var placement struct { + Environment struct { + Type string `json:"type"` + } `json:"environment"` + } + if err := json.Unmarshal(row.Configuration, &placement); err != nil { + return err + } + if placement.Environment.Type == "openai_hosted" { + if err := checkRuntimeDeploymentAdmission(ctx, q, ""); err != nil { + return err + } + } setup, err = s.freezeEnvironmentSkills(ctx, q, tenant, setup) if err != nil { return err diff --git a/services/agents-api/internal/store/turns.go b/services/agents-api/internal/store/turns.go index 5edc6f243..b37367bc0 100644 --- a/services/agents-api/internal/store/turns.go +++ b/services/agents-api/internal/store/turns.go @@ -110,6 +110,11 @@ func transitionTurn(ctx context.Context, q *sqlc.Queries, params sqlc.GetTurnPar } else if err != nil { return sqlc.Turn{}, err } + if input.ExpectedStatus == TurnQueued && input.Status == TurnInProgress { + if err := checkRuntimeComputeAdmission(ctx, q, params.SessionID); err != nil { + return sqlc.Turn{}, err + } + } row, err := q.TransitionTurn(ctx, sqlc.TransitionTurnParams{ ID: params.ID, SessionID: params.SessionID, ExpectedStatus: input.ExpectedStatus, NewStatus: input.Status, Outcome: input.Outcome, diff --git a/services/agents-api/migrations/000052_runtime_suspension.sql b/services/agents-api/migrations/000052_runtime_suspension.sql new file mode 100644 index 000000000..d99902ee0 --- /dev/null +++ b/services/agents-api/migrations/000052_runtime_suspension.sql @@ -0,0 +1,27 @@ +-- +goose Up +ALTER TABLE runtime_allocations + ADD COLUMN compute_phase text NOT NULL DEFAULT 'disabled' + CHECK (compute_phase IN ('disabled', 'running', 'quiescing', 'suspending', 'suspended', 'restoring', 'waking')), + ADD COLUMN compute_revision bigint NOT NULL DEFAULT 0 CHECK (compute_revision >= 0), + ADD COLUMN compute_state jsonb NOT NULL DEFAULT '{}'::jsonb CHECK (jsonb_typeof(compute_state) = 'object'), + ADD COLUMN compute_activity_at timestamptz NOT NULL DEFAULT clock_timestamp(), + ADD COLUMN compute_wake_requested boolean NOT NULL DEFAULT false, + ADD COLUMN compute_retained_until timestamptz; + +-- +goose Down +LOCK TABLE runtime_allocations IN ACCESS EXCLUSIVE MODE; +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM runtime_allocations WHERE compute_phase <> 'disabled') THEN + RAISE EXCEPTION 'Cannot discard retained compute and snapshot ownership'; + END IF; +END $$; +-- +goose StatementEnd +ALTER TABLE runtime_allocations + DROP COLUMN compute_retained_until, + DROP COLUMN compute_wake_requested, + DROP COLUMN compute_activity_at, + DROP COLUMN compute_state, + DROP COLUMN compute_revision, + DROP COLUMN compute_phase; diff --git a/services/agents-api/migrations/000053_runtime_deployment.sql b/services/agents-api/migrations/000053_runtime_deployment.sql new file mode 100644 index 000000000..017ae351c --- /dev/null +++ b/services/agents-api/migrations/000053_runtime_deployment.sql @@ -0,0 +1,22 @@ +-- +goose Up +CREATE TABLE runtime_deployment ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + installation_id uuid, + backend_fingerprint text NOT NULL DEFAULT '', + maintenance boolean NOT NULL DEFAULT false, + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CHECK ((installation_id IS NULL AND backend_fingerprint = '') OR + (installation_id IS NOT NULL AND backend_fingerprint ~ '^[0-9a-f]{64}$')) +); +INSERT INTO runtime_deployment (singleton) VALUES (true); + +-- +goose Down +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM runtime_deployment WHERE installation_id IS NOT NULL) THEN + RAISE EXCEPTION 'Cannot discard persisted sandbox installation identity'; + END IF; +END $$; +-- +goose StatementEnd +DROP TABLE runtime_deployment; diff --git a/services/agents-api/tools/microsandbox-provider/README.md b/services/agents-api/tools/microsandbox-provider/README.md new file mode 100644 index 000000000..484a44c0a --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/README.md @@ -0,0 +1,165 @@ +# microsandbox provider helper + +This Linux-only, one-operation helper links the maintained microsandbox Go SDK +v0.7.2. Core stays a pure-Go binary and uses the provider-neutral +`sandbox.CheckpointProvider` interface. There is no helper daemon, local lifecycle +database, native agent adapter, or additional scheduler. + +The supported first deployment profile is native Linux Core plus this local +helper, under one dedicated service user with KVM access. PostgreSQL may continue +to run in Docker. The current Core distroless/static image cannot execute this +glibc helper; copying the binary into that image is not a supported deployment. + +## Build and installation + +From this directory, using the repository Go version: + +```sh +GOWORK=off CGO_ENABLED=1 go build -mod=readonly -trimpath -o "$HOME/.parsar/bin/agents-api-microsandbox-provider" . +GOWORK=off go test ./... +``` + +The relative replacement for the parent Core module refers to this checkout. +The SDK is the real published module, pinned in go.mod and go.sum; it has no +local-source replacement. The normal build embeds its matching FFI library. +Do not build production with the SDK's development `microsandbox_ffi_path` tag. + +Install the matching v0.7.2 msb runtime and firmware from checksum-verified release +artifacts. Supply absolute helper/runtime/firmware paths and expected SHA256 +values in the trusted provider Config. The helper checks the runtime and firmware +hashes, the ELF runtime version section, the SDK module version, and the resolved +local backend. It never auto-installs or upgrades these artifacts. The installed +paths must remain immutable for the lifetime of the provider key. + +Use a dedicated, private (0700), short MSB_HOME on local persistent storage. +The upstream runtime uses Unix sockets, so a short path such as +`/var/lib/parsar-msb` avoids pathname limits. Never share that home with another +installation, cloud profile, or manual lifecycle controller. Its VM disks, +snapshots, SDK state and credentials are confidential execution-service data. +All managed sandbox lifecycle mutations must go through the provider. + +An immutable OCI image must already be available as `repository@sha256:<64 lowercase hex>`. Bare Docker image IDs and mutable tags are rejected. +For offline archives, `msb image load --tag repository@sha256:` must register +the digest reference explicitly; loading a mutable tag alone does not create it. +Use the manifest digest from `image inspect`, not the Docker image config ID. +The qualified image contains our existing daemon, Python 3, native harness and +shared Runtime helpers. Provider Config explicitly sets VM resources and the +complete host network policy. This adapter does not install registry credentials. + +## Bootstrap and network + +VM creation does not implicitly run the OCI ENTRYPOINT. Before admitting native +work, the helper uses confidential stdin to install the existing private +`auth.json` format, create Runtime directories, bind the same workspace at +`/workspace`, and invoke the existing daemon's `connect --profile default -b` +mode as uid/gid 1000. The final provider-owned bootstrap label confirms only +completion of these writes and launch, not authentication or native readiness. +The receipt label uses the supported next-start modification policy to update +persisted metadata without restarting the guest. v0.7.2 cannot update active labels. + +The private daemon control directory is `/run/parsar` (0700, uid/gid 1000). +`PARSAR_DAEMON_SUSPEND_PID_FILE=/run/parsar/daemon-suspend.json` enables the +daemon's separately owned idle park/wake control. RunCommandCompute can execute +the exact daemon resume command authorized by Core; it never uses pkill. + +Creation and full restore both receive the same explicit, trusted host policy. +Upstream restore defaults to Public rather than inheriting source host access. +Native tool-network policy remains the existing Runtime responsibility. No +default external mount inheritance, missing-resource allowance, or policy widening +is used. + +## Lifecycle contract + +Core persists operation IDs, source/target generations, exact identities and +snapshot evidence before depending on them. `Initial` and `NewCompute` only +construct references; they allocate nothing. Names are installation/allocation/ +generation-derived and must never be reused for another incarnation. + +Suspend pauses the exact VM, captures a full snapshot under the persisted +operation's derived group/member, verifies its complete checkpoint closure, then +force-stops the source. Pausing alone is not suspension or memory reclamation. +A completed matching artifact is inspected rather than captured again. + +After a lost response, Core uses `ObserveOnly`. This never starts capture or +restore, and a suspended-operation observation never kills its source. Core can +persist recovered snapshot evidence before KillCompute. If the artifact is absent but the exact source is still running or paused with settled bootstrap, observation returns that intact source with no snapshot, allowing Core to abort suspension and thaw/wake it. Missing state never +authorizes replay of the original operation. + +Restore verifies the exact artifact and creates the precommitted target name. +Existing targets are adopted only when their immutable ID (if known) and +persisted `snapshot_parent` agree. Restored labels are not synthesized. +Unfinished restore intent is not bootstrap completion. There is no ordinary +Start, replacement, disk-only restore, or cold-boot fallback. + +`ResumeCompute` only thaws the same resident source after an aborted suspension. +The pinned Go handle method is name-based; the allocation flock plus ID checks +before and after the call fence every managed replacement. External manual +lifecycle changes in the managed namespace are unsupported. + +A helper holds an allocation flock until its lifecycle SDK call actually settles. +Core's response deadline does not kill that helper or cancel its FFI wait, since +cancelling the wait does not prove the native mutation stopped. On timeout Core +retains an unknown operation and observes it; a subsequent helper cannot race +past the surviving lock holder. A stuck owner needs operator investigation, +not automatic lock deletion or another create. + +KillCompute checks the precise incarnation before stopping and removing its writable +disks. Core calls it after persisting the verified snapshot, even if Suspend +already stopped the source. GetCompute, commands, cleanup and the next suspension +verify restored provenance from persisted VM config after the consumed artifact +has been deleted. +DeleteSnapshot accepts only the derived operation selector and matching full +artifact identity, not an arbitrary path. Core owns retention, consumed snapshot +generations and cleanup ordering. A checkpoint must never roll back work admitted +after its first restore. + +RunCommand/RunCommandCompute carry stdin on anonymous pipes, fix the guest user +to 1000, impose a deadline and 1 MiB per-stream output limits, and require both +successful stdin completion and an explicit guest exit before returning a result. +Timeouts, output overflow and missing receipts return ErrCommandUnconfirmed. +Closing an SDK exec handle alone does not prove the guest process exited. + +## Acceptance boundary + +The feature is idle-only: Core must reserve a terminal Session with no pending +work before parking its daemon and suspending compute. The next Turn uses the +same Session history, files and configuration without replaying initialization. +This adapter does not promise that a native agent process persists across Turns. + +SDK feasibility separately qualified exact IDs, full snapshot verification, +snapshot_parent, source memory release, tmpfs/RAM restoration and stale-handle +pause rejection. The published Go module's SDK sources and Linux amd64 FFI were +compared byte-for-byte with that qualification source/runtime. The FFI SHA256 was +`ed04ca4788c1c400e1b67040e964fbfcb3743d31d969d1b426dfb95afe7271d8`. + +Production helper qualification completed two full capture/restore cycles, removing +each source before restore and deleting each consumed artifact before the next +cycle. Workspace bind identity, uid 1000, private auth permissions and file content +survived. The real Core/Codex synthetic-model test separately completed two Turns +across idle suspension and a Core restart. Synthetic responses establish the +control flow, not real-model acceptance. + +The 2026-09-22 Linux amd64 live acceptance used microsandbox/Go SDK v0.7.2, +libkrunfw 5.6.1, Codex CLI 0.153.4 and the official Kimi K3 Responses API. The +unchanged native harness completed two real Turns in one Session across automatic +idle suspension, source removal and a Core restart. Generation 1 restored the +exact full snapshot; the consumed artifact was removed. The model's shell tool +read the original random file marker, retained environment configuration and one +initialization record, then correctly recalled the first request from history. +Retrying the second public input with the same idempotency key created no extra +Turn or tool side effect. A subsequent public file upload/list woke generation 2 +without starting another Turn. Public Session deletion released its allocation; +all qualification VMs and snapshots and temporary credentials were removed. +This qualifies that model/profile, not every provider or broader isolation guarantees. + +The source VM was observed at 334304 KiB RSS before suspension and with zero RSS +and no executable after exit. The qualification container's PID 1 left a zombie +PID; it retained no VM memory. The separate backend RAM probe also checked live +anonymous memory and tmpfs contents after full restore. A paused resident VM alone +would not satisfy either memory-release check. + +Private qualification evidence is grouped as `ram-03` (backend), `provider-01` +(production helper), `core-01` (synthetic Core) and `live-core-03` (real model). +These are bounded single-host checks. They do not establish cold-image download +latency, tail latency, production capacity, or general native-process persistence +across Turns. diff --git a/services/agents-api/tools/microsandbox-provider/backend.go b/services/agents-api/tools/microsandbox-provider/backend.go new file mode 100644 index 000000000..9379631fb --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/backend.go @@ -0,0 +1,128 @@ +//go:build linux + +package main + +import ( + "context" + "errors" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + wire "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/microsandbox" + sdk "github.com/superradcompany/microsandbox/sdk/go" +) + +const bootstrapLabel = "io.parsar.bootstrap" + +type backend struct{ q wire.Request } + +func (b backend) run(ctx context.Context) (wire.Response, error) { + switch b.q.Operation { + case "create": + s, e := b.create(ctx) + return wire.Response{State: &s}, e + case "inspect": + _, s, e := b.inspect(ctx, b.q.Compute) + return wire.Response{State: &s}, e + case "kill": + return wire.Response{}, b.kill(ctx, b.q.Compute) + case "resume_compute": + s, e := b.resumeCompute(ctx, b.q.Compute) + return wire.Response{State: &s}, e + case "command": + h, _, e := b.inspect(ctx, b.q.Compute) + if e != nil { + return wire.Response{}, e + } + live, e := h.Connect(ctx) + if e != nil { + return wire.Response{}, e + } + defer live.Detach(context.Background()) + commandCtx, cancel := context.WithDeadline(ctx, b.q.Deadline) + defer cancel() + result, e := runCommand(commandCtx, live, *b.q.Command, "1000:1000") + return wire.Response{Command: &result}, e + case "suspend": + s, e := b.suspend(ctx, *b.q.Suspend) + return wire.Response{State: &s}, e + case "resume": + s, e := b.resume(ctx, *b.q.Resume) + return wire.Response{State: &s}, e + case "delete_snapshot": + _, e := b.verifiedSnapshot(ctx, *b.q.Snapshot) + if sdk.IsKind(e, sdk.ErrSnapshotNotFound) { + return wire.Response{}, nil + } + if e != nil { + return wire.Response{}, e + } + return wire.Response{}, sdk.Snapshot.Remove(ctx, b.q.Snapshot.Reference, false) + } + return wire.Response{}, sandbox.ErrInvalid +} +func (b backend) inspect(ctx context.Context, c wire.Compute) (*sdk.SandboxHandle, wire.State, error) { + state := wire.State{Compute: c} + h, e := sdk.GetSandbox(ctx, c.Name) + if e != nil { + return nil, state, e + } + state, e = qualifyCompute(b.q.Config, b.q.Reference, c, h.ID(), string(h.Status()), h.ConfigJSON()) + return h, state, e +} +func (b backend) kill(ctx context.Context, c wire.Compute) error { + h, _, e := b.inspect(ctx, c) + if sdk.IsKind(e, sdk.ErrSandboxNotFound) { + return nil + } + if e != nil { + return e + } + if e = h.Kill(ctx); e != nil { + return e + } + if e = h.Remove(ctx); e != nil { + return e + } + _, e = sdk.GetSandbox(ctx, c.Name) + if sdk.IsKind(e, sdk.ErrSandboxNotFound) { + return nil + } + if e == nil { + return errors.New("compute removal unconfirmed") + } + return e +} +func (b backend) network() *sdk.NetworkConfig { + n := b.q.Config.Network + result := &sdk.NetworkConfig{DefaultEgress: sdk.PolicyAction(n.DefaultEgress), DefaultIngress: sdk.PolicyAction(n.DefaultIngress)} + for _, r := range n.Rules { + result.Rules = append(result.Rules, sdk.PolicyRule{Action: sdk.PolicyAction(r.Action), Direction: sdk.PolicyDirection(r.Direction), Destination: r.Destination, Protocol: sdk.PolicyProtocol(r.Protocol), Port: r.Port}) + } + return result +} + +func (b backend) resumeCompute(ctx context.Context, c wire.Compute) (wire.State, error) { + h, state, e := b.inspect(ctx, c) + if e != nil { + return state, e + } + if h.Status() == sdk.SandboxStatusRunning { + return state, nil + } + if h.Status() != sdk.SandboxStatusPaused { + return state, wire.ErrUnconfirmed + } + // The allocation flock excludes every managed replacement. The pinned Go + // handle Resume is name-based, so check identity before and after it. + if e = h.Resume(ctx); e != nil { + return state, e + } + if _, e = h.Refresh(ctx); e != nil { + return state, e + } + _, state, e = b.inspect(ctx, c) + if e == nil && state.Status != "running" { + e = wire.ErrUnconfirmed + } + return state, e +} diff --git a/services/agents-api/tools/microsandbox-provider/bootstrap.go b/services/agents-api/tools/microsandbox-provider/bootstrap.go new file mode 100644 index 000000000..db856f5bf --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/bootstrap.go @@ -0,0 +1,92 @@ +//go:build linux + +package main + +import ( + "context" + "encoding/json" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentnetwork" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + wire "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/microsandbox" + sdk "github.com/superradcompany/microsandbox/sdk/go" +) + +// The existing auth profile and daemon's own background mode are reused. +// All credential bytes enter the guest on stdin before any native work is admitted. +const bootstrapScript = ` +import ctypes,json,os,stat,subprocess,sys +b=json.load(sys.stdin) +for p in ['/home/runtime','/home/runtime/.parsar','/home/runtime/.parsar/parsar-daemon','/home/runtime/.parsar/parsar-daemon/default','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages','/run/parsar']: + os.makedirs(p,mode=0o700,exist_ok=True) + if not stat.S_ISDIR(os.lstat(p).st_mode): raise RuntimeError('invalid bootstrap directory') + os.chmod(p,0o700);os.chown(p,1000,1000) +p='/home/runtime/.parsar/parsar-daemon/default/auth.json' +fd=os.open(p,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) +with os.fdopen(fd,'w') as f: + json.dump({'server_url':b['CoreURL'],'runtime_id':b['DeviceID'],'runner_credential':b['Credential']},f) + f.flush();os.fsync(f.fileno());os.fchown(f.fileno(),1000,1000) +if not stat.S_ISDIR(os.lstat('/workspace').st_mode): raise RuntimeError('invalid workspace alias') +libc=ctypes.CDLL(None,use_errno=True) +if libc.mount(b'/environment/workspace',b'/workspace',None,4096,None)!=0: + raise OSError(ctypes.get_errno(),'workspace bind mount failed') +def runtime_user(): + os.setgroups([]);os.setgid(1000);os.setuid(1000) +subprocess.run(['/usr/local/bin/parsar-daemon','connect','--profile','default','-b'], + stdin=subprocess.DEVNULL,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL, + cwd='/environment/workspace',preexec_fn=runtime_user,check=True) +` + +func (b backend) create(ctx context.Context) (wire.State, error) { + c := wire.Compute{Name: wire.Name(b.q.Config, b.q.Reference, 0)} + if _, _, e := b.inspect(ctx, c); e == nil { + return wire.State{}, sandbox.ErrExists + } else if !sdk.IsKind(e, sdk.ErrSandboxNotFound) { + return wire.State{}, e + } + bootstrap := *b.q.Bootstrap + policy := agentnetwork.Policy{Access: bootstrap.NetworkAccess, AllowedDomains: bootstrap.AllowedDomains} + domains, _ := json.Marshal(policy.Hosts()) + labels := wire.Labels(b.q.Config, b.q.Reference) + labels[bootstrapLabel] = "pending" + live, e := sdk.CreateSandbox(ctx, c.Name, + sdk.WithImage(b.q.Config.Image), sdk.WithMemory(b.q.Config.MemoryMiB), sdk.WithCPUs(b.q.Config.CPUs), + sdk.WithRootDisk(sdk.RootDisk.Managed(b.q.Config.RootDiskMiB)), sdk.WithUser("1000:1000"), + sdk.WithLabels(labels), sdk.WithDetached(), sdk.WithQuietLogs(), sdk.WithNetwork(b.network()), + sdk.WithEnv(map[string]string{ + "HOME": "/home/runtime", "PARSAR_HOME": "/home/runtime/.parsar", + "PARSAR_RUNTIME_ENVIRONMENT_ID": bootstrap.EnvironmentID, "PARSAR_RUNTIME_SESSION_ID": bootstrap.SessionID, + "PARSAR_RUNTIME_NETWORK_ACCESS": policy.Access, "PARSAR_RUNTIME_ALLOWED_DOMAINS": string(domains), + "PARSAR_DAEMON_SUSPEND_PID_FILE": "/run/parsar/daemon-suspend.json", + })) + if e != nil { + return wire.State{}, e + } + defer live.Detach(context.Background()) + c.ID = live.ID() + data, e := json.Marshal(bootstrap) + if e != nil { + return wire.State{}, e + } + initialization, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + result, e := runCommand(initialization, live, sandbox.Command{Args: []string{"/usr/bin/python3", "-I", "-S", "-c", bootstrapScript}, Stdin: data}, "0:0") + if e != nil { + return wire.State{}, e + } + if result.ExitCode != 0 { + return wire.State{}, sandbox.ErrCommandUnconfirmed + } + // Persist the final bootstrap receipt without restarting the live guest. + // v0.7.2 cannot update active labels; ownership reads persisted config. + _, e = live.Modify(ctx, sdk.ModifyOptions{Labels: map[string]string{bootstrapLabel: "complete"}, Policy: sdk.ModificationPolicyNextStart}) + if e != nil { + return wire.State{}, e + } + _, state, e := b.inspect(ctx, c) + if e == nil && !state.BootstrapComplete { + return wire.State{}, sandbox.ErrCommandUnconfirmed + } + return state, e +} diff --git a/services/agents-api/tools/microsandbox-provider/command.go b/services/agents-api/tools/microsandbox-provider/command.go new file mode 100644 index 000000000..551681511 --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/command.go @@ -0,0 +1,105 @@ +//go:build linux + +package main + +import ( + "bytes" + "context" + "errors" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + wire "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/microsandbox" + sdk "github.com/superradcompany/microsandbox/sdk/go" +) + +func runCommand(ctx context.Context, live *sdk.Sandbox, c sandbox.Command, user string) (sandbox.CommandResult, error) { + var result sandbox.CommandResult + deadline, ok := ctx.Deadline() + if !ok || wire.ValidateCommand(c) != nil { + return result, sandbox.ErrInvalid + } + timeout := time.Until(deadline) + if timeout <= 0 { + return result, sandbox.ErrCommandUnconfirmed + } + options := []sdk.ExecOption{sdk.WithExecUser(user), sdk.WithExecCwd(c.Directory), sdk.WithExecTimeout(timeout), sdk.WithExecStdinPipe()} + handle, e := live.ExecStream(ctx, c.Args[0], c.Args[1:], options...) + if e != nil { + return result, errors.Join(sandbox.ErrCommandUnconfirmed, e) + } + defer handle.Close() + sink := handle.TakeStdin() + if sink == nil { + return result, sandbox.ErrCommandUnconfirmed + } + // Write and receive concurrently to avoid full-pipe deadlocks. + written := make(chan error, 1) + go func() { + data := c.Stdin + for len(data) > 0 { + n := len(data) + if n > 65536 { + n = 65536 + } + count, err := sink.WriteCtx(ctx, data[:n]) + if err != nil { + written <- err + return + } + if count != n { + written <- errors.New("short command input") + return + } + data = data[n:] + } + written <- sink.Close() + }() + return collectCommand(ctx, handle.Recv, written) +} + +func collectCommand(ctx context.Context, receive func(context.Context) (*sdk.ExecEvent, error), written <-chan error) (sandbox.CommandResult, error) { + var result sandbox.CommandResult + var stdout, stderr bytes.Buffer + exited := false + for { + event, err := receive(ctx) + if err != nil { + return result, errors.Join(sandbox.ErrCommandUnconfirmed, err) + } + switch event.Kind { + case sdk.ExecEventStdout: + if stdout.Len()+len(event.Data) > wire.MaxOutputBytes { + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + } + _, _ = stdout.Write(event.Data) + case sdk.ExecEventStderr: + if stderr.Len()+len(event.Data) > wire.MaxOutputBytes { + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + } + _, _ = stderr.Write(event.Data) + case sdk.ExecEventExited: + if exited { + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + } + exited = true + result.ExitCode = event.ExitCode + case sdk.ExecEventStdinError, sdk.ExecEventFailed: + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + case sdk.ExecEventDone: + if !exited { + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + } + select { + case err := <-written: + if err != nil { + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + } + result.Stdout, result.Stderr = stdout.String(), stderr.String() + return result, nil + case <-ctx.Done(): + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + } + } + } +} diff --git a/services/agents-api/tools/microsandbox-provider/command_test.go b/services/agents-api/tools/microsandbox-provider/command_test.go new file mode 100644 index 000000000..90bd28b18 --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/command_test.go @@ -0,0 +1,53 @@ +//go:build linux + +package main + +import ( + "context" + "errors" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + wire "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/microsandbox" + sdk "github.com/superradcompany/microsandbox/sdk/go" + "strings" + "testing" +) + +func TestCommandRequiresExitAndSuccessfulStdin(t *testing.T) { + for _, tc := range []struct { + name string + events []sdk.ExecEvent + inputError error + uncertain bool + }{ + {"confirmed", []sdk.ExecEvent{{Kind: sdk.ExecEventStdout, Data: []byte("ok")}, {Kind: sdk.ExecEventExited, ExitCode: 7}, {Kind: sdk.ExecEventDone}}, nil, false}, + {"missing_exit", []sdk.ExecEvent{{Kind: sdk.ExecEventDone}}, nil, true}, + {"failed_stdin", []sdk.ExecEvent{{Kind: sdk.ExecEventExited}, {Kind: sdk.ExecEventDone}}, errors.New("lost stdin"), true}, + {"duplicate_exit", []sdk.ExecEvent{{Kind: sdk.ExecEventExited}, {Kind: sdk.ExecEventExited}, {Kind: sdk.ExecEventDone}}, nil, true}, + {"stdin_event", []sdk.ExecEvent{{Kind: sdk.ExecEventStdinError}, {Kind: sdk.ExecEventExited}, {Kind: sdk.ExecEventDone}}, nil, true}, + {"overflow", []sdk.ExecEvent{{Kind: sdk.ExecEventStdout, Data: []byte(strings.Repeat("x", wire.MaxOutputBytes+1))}}, nil, true}, + } { + t.Run(tc.name, func(t *testing.T) { + index := 0 + recv := func(context.Context) (*sdk.ExecEvent, error) { + if index >= len(tc.events) { + return nil, errors.New("stream lost") + } + event := tc.events[index] + index++ + return &event, nil + } + input := make(chan error, 1) + input <- tc.inputError + got, e := collectCommand(context.Background(), recv, input) + if tc.uncertain { + if !errors.Is(e, sandbox.ErrCommandUnconfirmed) { + t.Fatalf("outcome confirmed: %+v %v", got, e) + } + return + } + if e != nil || got.ExitCode != 7 || got.Stdout != "ok" { + t.Fatalf("nonzero guest exit lost: %+v %v", got, e) + } + }) + } +} diff --git a/services/agents-api/tools/microsandbox-provider/go.mod b/services/agents-api/tools/microsandbox-provider/go.mod new file mode 100644 index 000000000..9de6b567f --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/go.mod @@ -0,0 +1,13 @@ +module github.com/MiniMax-AI-Dev/parsar/services/agents-api/tools/microsandbox-provider + +go 1.25.13 + +require ( + github.com/MiniMax-AI-Dev/parsar v0.0.0 + github.com/superradcompany/microsandbox/sdk/go v0.7.2 +) + +require github.com/google/uuid v1.6.0 // indirect + +// The helper ships from this repository; the SDK itself is a released module. +replace github.com/MiniMax-AI-Dev/parsar => ../../../.. diff --git a/services/agents-api/tools/microsandbox-provider/go.sum b/services/agents-api/tools/microsandbox-provider/go.sum new file mode 100644 index 000000000..e28f0957e --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/go.sum @@ -0,0 +1,4 @@ +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/superradcompany/microsandbox/sdk/go v0.7.2 h1:aO70srBRgu50rNZTtzQTOO3xxPSthkZLkq3kjWvmsqg= +github.com/superradcompany/microsandbox/sdk/go v0.7.2/go.mod h1:p7Tm/p9zkO7sHO3N8A1fiTVeLB2w/fQoKdYlpN/5neA= diff --git a/services/agents-api/tools/microsandbox-provider/installation_test.go b/services/agents-api/tools/microsandbox-provider/installation_test.go new file mode 100644 index 000000000..6810d9130 --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/installation_test.go @@ -0,0 +1,40 @@ +//go:build linux + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestIsolatedConfigReusesOnlyPrivateEmptyObject(t *testing.T) { + home := t.TempDir() + path, err := isolatedConfig(home) + if err != nil { + t.Fatal(err) + } + if again, err := isolatedConfig(home); err != nil || again != path { + t.Fatalf("reuse: %s %v", again, err) + } + if err := os.WriteFile(path, []byte(`{"default_profile":"other"}`), 0600); err != nil { + t.Fatal(err) + } + if _, err := isolatedConfig(home); err == nil { + t.Fatal("ambient profile accepted") + } +} + +func TestIsolatedConfigRejectsSymlink(t *testing.T) { + home := t.TempDir() + target := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(target, []byte("{}\n"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(home, "parsar-sdk-config.json")); err != nil { + t.Fatal(err) + } + if _, err := isolatedConfig(home); err == nil { + t.Fatal("config symlink accepted") + } +} diff --git a/services/agents-api/tools/microsandbox-provider/lock_test.go b/services/agents-api/tools/microsandbox-provider/lock_test.go new file mode 100644 index 000000000..7d4b4a3eb --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/lock_test.go @@ -0,0 +1,48 @@ +//go:build linux + +package main + +import ( + "context" + "errors" + wire "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/microsandbox" + "os" + "path/filepath" + "testing" + "time" +) + +func TestAllocationLockSurvivesCallerDeadlineUntilExplicitSettlement(t *testing.T) { + home := t.TempDir() + q := wire.Request{Config: wire.Config{RuntimeHome: home}, Deadline: time.Now().Add(time.Second)} + release, e := allocationLock(q) + if e != nil { + t.Fatal(e) + } + waiting := q + waiting.Deadline = time.Now().Add(40 * time.Millisecond) + if next, e := allocationLock(waiting); !errors.Is(e, context.DeadlineExceeded) { + if next != nil { + next() + } + t.Fatalf("second lifecycle entered: %v", e) + } + release() + q.Deadline = time.Now().Add(time.Second) + release, e = allocationLock(q) + if e != nil { + t.Fatal(e) + } + release() +} +func TestLockDirectoryCannotRedirectIntoAnotherHome(t *testing.T) { + home := t.TempDir() + foreign := t.TempDir() + if e := os.Symlink(foreign, filepath.Join(home, "parsar-locks")); e != nil { + t.Fatal(e) + } + _, e := allocationLock(wire.Request{Config: wire.Config{RuntimeHome: home}, Deadline: time.Now().Add(time.Second)}) + if e == nil { + t.Fatal("symlink lock directory accepted") + } +} diff --git a/services/agents-api/tools/microsandbox-provider/main.go b/services/agents-api/tools/microsandbox-provider/main.go new file mode 100644 index 000000000..250eb421d --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/main.go @@ -0,0 +1,257 @@ +//go:build linux + +// The helper performs one finite Provider operation. Only Core owns durable +// lifecycle intent, allocation generations, scheduling and recovery policy. +package main + +import ( + "context" + "crypto/sha256" + "debug/elf" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "runtime/debug" + "strings" + "syscall" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + wire "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/microsandbox" + sdk "github.com/superradcompany/microsandbox/sdk/go" +) + +func main() { + response := serve(os.Stdin) + // Errors and upstream diagnostics can contain secrets: expose only stable codes. + _ = json.NewEncoder(os.Stdout).Encode(response) +} +func serve(input io.Reader) wire.Response { + out := wire.Response{Version: wire.ProtocolVersion} + decoder := json.NewDecoder(io.LimitReader(input, wire.MaxRequestBytes+1)) + decoder.DisallowUnknownFields() + var q wire.Request + if decoder.Decode(&q) != nil || wire.ValidateRequest(q) != nil { + out.ErrorCode = "invalid" + return out + } + var extra any + if decoder.Decode(&extra) != io.EOF { + out.ErrorCode = "invalid" + return out + } + if err := checkInstallation(q.Config); err != nil { + out.ErrorCode = "unconfirmed" + return out + } + release, err := allocationLock(q) + if err != nil { + out.ErrorCode = "unconfirmed" + return out + } + defer release() + // Lifecycle calls deliberately retain the flock until the SDK has settled. + // Cancelling an FFI wait does not prove the underlying operation stopped. + b := backend{q: q} + out, err = b.run(context.Background()) + out.Version = wire.ProtocolVersion + if err != nil { + out.ErrorCode = code(err) + } + return out +} +func code(err error) string { + switch { + case errors.Is(err, sandbox.ErrInvalid): + return "invalid" + case errors.Is(err, sandbox.ErrOwnership), sdk.IsKind(err, sdk.ErrSandboxReplaced): + return "ownership" + case errors.Is(err, sandbox.ErrExists), sdk.IsKind(err, sdk.ErrSandboxAlreadyExists): + return "exists" + case errors.Is(err, sandbox.ErrNotFound), sdk.IsKind(err, sdk.ErrSandboxNotFound): + return "not_found" + case errors.Is(err, sandbox.ErrCommandUnconfirmed): + return "command_unconfirmed" + default: + return "unconfirmed" + } +} +func allocationLock(q wire.Request) (func(), error) { + dir := filepath.Join(q.Config.RuntimeHome, "parsar-locks") + if e := os.MkdirAll(dir, 0700); e != nil { + return nil, e + } + st, e := os.Lstat(dir) + if e != nil { + return nil, e + } + if !st.IsDir() || st.Mode().Perm() != 0700 { + return nil, sandbox.ErrOwnership + } + name := filepath.Join(dir, wire.Name(q.Config, q.Reference, 0)+".lock") + fd, e := syscall.Open(name, syscall.O_CREAT|syscall.O_RDWR|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0600) + if e != nil { + return nil, e + } + f := os.NewFile(uintptr(fd), name) + for { + if !time.Now().Before(q.Deadline) { + f.Close() + return nil, context.DeadlineExceeded + } + e = syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB) + if e == nil { + return func() { _ = syscall.Flock(fd, syscall.LOCK_UN); _ = f.Close() }, nil + } + if e != syscall.EWOULDBLOCK && e != syscall.EAGAIN { + f.Close() + return nil, e + } + time.Sleep(20 * time.Millisecond) + } +} +func checkInstallation(c wire.Config) error { + build, ok := debug.ReadBuildInfo() + if !ok { + return sandbox.ErrInvalid + } + matched := false + for _, d := range build.Deps { + if d.Path == "github.com/superradcompany/microsandbox/sdk/go" { + matched = d.Version == wire.SDKVersion && d.Replace == nil + } + } + if !matched { + return sandbox.ErrInvalid + } + for _, v := range []struct{ path, hash string }{{c.RuntimePath, c.RuntimeSHA256}, {c.FirmwarePath, c.FirmwareSHA256}} { + f, e := os.Open(v.path) + if e != nil { + return e + } + h := sha256.New() + _, e = io.Copy(h, f) + _ = f.Close() + if e != nil || hex.EncodeToString(h.Sum(nil)) != v.hash { + return sandbox.ErrInvalid + } + } + if e := runtimeVersion(c.RuntimePath); e != nil { + return e + } + if e := os.MkdirAll(c.RuntimeHome, 0700); e != nil { + return e + } + st, e := os.Lstat(c.RuntimeHome) + if e != nil { + return e + } + if !st.IsDir() || st.Mode().Perm() != 0700 { + return sandbox.ErrOwnership + } + // Explicit paths and a local backend avoid ambient cloud/profile selection. + for _, v := range wire.HelperEnvironment(nil, c) { + key, value, ok := cutEnv(v) + if ok { + if e := os.Setenv(key, value); e != nil { + return e + } + } + } + configPath, e := isolatedConfig(c.RuntimeHome) + if e != nil { + return e + } + if e := os.Setenv("MSB_CONFIG_PATH", configPath); e != nil { + return e + } + runtime, e := sdk.ResolveRuntime(sdk.RuntimeConfig{Home: c.RuntimeHome, MSBPath: c.RuntimePath, LibkrunfwPath: c.FirmwarePath}) + if e != nil { + return e + } + if runtime.MSBPath != c.RuntimePath || runtime.LibkrunfwPath != c.FirmwarePath { + return sandbox.ErrOwnership + } + selected, e := sdk.DefaultBackendInfo() + if e != nil { + return e + } + if selected.Kind != sdk.BackendLocal { + return sandbox.ErrOwnership + } + return nil +} +func cutEnv(s string) (string, string, bool) { + for i := range s { + if s[i] == '=' { + return s[:i], s[i+1:], true + } + } + return "", "", false +} + +func runtimeVersion(path string) error { + binary, e := elf.Open(path) + if e != nil { + return e + } + defer binary.Close() + section := binary.Section(".msbver") + if section == nil { + return sandbox.ErrInvalid + } + version, e := section.Data() + if e != nil { + return e + } + if strings.TrimRight(string(version), "\x00") != strings.TrimPrefix(wire.SDKVersion, "v") { + return sandbox.ErrInvalid + } + return nil +} + +// An empty JSON object prevents ambient SDK profiles without invalid empty input. +func isolatedConfig(home string) (string, error) { + path := filepath.Join(home, "parsar-sdk-config.json") + fd, err := syscall.Open(path, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_EXCL|syscall.O_NOFOLLOW, 0600) + if err == nil { + f := os.NewFile(uintptr(fd), path) + _, err = f.WriteString("{}\n") + if err == nil { + err = f.Sync() + } + closeErr := f.Close() + if err != nil { + return "", err + } + if closeErr != nil { + return "", closeErr + } + } else if !os.IsExist(err) { + return "", err + } + fd, err = syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW, 0) + if err != nil { + return "", err + } + f := os.NewFile(uintptr(fd), path) + defer f.Close() + st, err := f.Stat() + if err != nil { + return "", err + } + if !st.Mode().IsRegular() || st.Mode().Perm() != 0600 { + return "", sandbox.ErrOwnership + } + data, err := io.ReadAll(io.LimitReader(f, 4)) + if err != nil { + return "", err + } + if string(data) != "{}\n" { + return "", sandbox.ErrOwnership + } + return path, nil +} diff --git a/services/agents-api/tools/microsandbox-provider/observed.go b/services/agents-api/tools/microsandbox-provider/observed.go new file mode 100644 index 000000000..93e7698a7 --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/observed.go @@ -0,0 +1,43 @@ +//go:build linux + +package main + +import ( + "encoding/json" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + wire "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/microsandbox" +) + +func qualifyCompute(deployment wire.Config, reference sandbox.Reference, c wire.Compute, actualID, status, raw string) (wire.State, error) { + state := wire.State{Compute: c} + if c.ID != "" && actualID != c.ID { + return state, sandbox.ErrOwnership + } + var config struct { + Labels map[string]string `json:"labels"` + SnapshotParent string `json:"snapshot_parent"` + CheckpointRestore json.RawMessage `json:"checkpoint_restore"` + } + if json.Unmarshal([]byte(raw), &config) != nil { + return state, sandbox.ErrOwnership + } + if c.RestoredFrom == nil { + for key, value := range wire.Labels(deployment, reference) { + if config.Labels[key] != value { + return state, sandbox.ErrOwnership + } + } + if config.SnapshotParent != "" { + return state, sandbox.ErrOwnership + } + state.BootstrapComplete = config.Labels[bootstrapLabel] == "complete" + } else { + if config.SnapshotParent != c.RestoredFrom.ID { + return state, sandbox.ErrOwnership + } + state.BootstrapComplete = len(config.CheckpointRestore) == 0 || string(config.CheckpointRestore) == "null" + } + state.Compute.ID = actualID + state.Status = status + return state, nil +} diff --git a/services/agents-api/tools/microsandbox-provider/observed_test.go b/services/agents-api/tools/microsandbox-provider/observed_test.go new file mode 100644 index 000000000..b5fbce8f6 --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/observed_test.go @@ -0,0 +1,73 @@ +//go:build linux + +package main + +import ( + "encoding/json" + "errors" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + wire "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/microsandbox" + sdk "github.com/superradcompany/microsandbox/sdk/go" + "testing" +) + +func TestRestoredOwnershipUsesExactParentWithoutLabels(t *testing.T) { + c := wire.Compute{ID: "local:5", RestoredFrom: &wire.SnapshotIdentity{ID: "snap_exact"}} + for _, tc := range []struct { + name, raw string + complete bool + rejected bool + }{ + {"exact", `{"labels":{},"snapshot_parent":"snap_exact"}`, true, false}, + {"other", `{"labels":{},"snapshot_parent":"snap_foreign"}`, false, true}, + {"unfinished", `{"snapshot_parent":"snap_exact","checkpoint_restore":{"checkpoint_id":"x"}}`, false, false}, + {"absent", `{"labels":{}}`, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + state, e := qualifyCompute(wire.Config{RuntimeHome: t.TempDir()}, sandbox.Reference{}, c, "local:5", "running", tc.raw) + if tc.rejected { + if !errors.Is(e, sandbox.ErrOwnership) { + t.Fatalf("foreign accepted: %v", e) + } + return + } + if e != nil || state.BootstrapComplete != tc.complete { + t.Fatalf("state=%+v error=%v", state, e) + } + }) + } +} +func TestInitialIdentityNeedsAllLabelsAndBootstrapReceipt(t *testing.T) { + config := wire.Config{InstallationID: "installation"} + ref := sandbox.Reference{TenantID: "tenant", EnvironmentID: "environment", AllocationID: "allocation"} + labels := wire.Labels(config, ref) + labels[bootstrapLabel] = "pending" + raw, _ := json.Marshal(map[string]any{"labels": labels}) + state, e := qualifyCompute(config, ref, wire.Compute{ID: "local:4"}, "local:4", "running", string(raw)) + if e != nil || state.BootstrapComplete { + t.Fatalf("running implied bootstrap completion: %+v %v", state, e) + } + labels[bootstrapLabel] = "complete" + delete(labels, "io.parsar.tenant") + raw, _ = json.Marshal(map[string]any{"labels": labels}) + _, e = qualifyCompute(config, ref, wire.Compute{ID: "local:4"}, "local:4", "running", string(raw)) + if !errors.Is(e, sandbox.ErrOwnership) { + t.Fatalf("missing owner accepted: %v", e) + } +} +func TestSameNameReplacementCannotBeAdopted(t *testing.T) { + _, e := qualifyCompute(wire.Config{}, sandbox.Reference{}, wire.Compute{ID: "local:old"}, "local:new", "running", `{}`) + if !errors.Is(e, sandbox.ErrOwnership) { + t.Fatalf("replacement accepted: %v", e) + } +} +func TestOnlyObservedTerminalStatesProveSourceStopped(t *testing.T) { + for _, s := range []sdk.SandboxStatus{sdk.SandboxStatusCreated, sdk.SandboxStatusStarting, sdk.SandboxStatusRunning, sdk.SandboxStatusDraining, sdk.SandboxStatusPaused, "unknown"} { + if terminal(s) { + t.Fatalf("%s falsely proves memory release", s) + } + } + if !terminal(sdk.SandboxStatusStopped) || !terminal(sdk.SandboxStatusCrashed) { + t.Fatal("terminal states not recognized") + } +} diff --git a/services/agents-api/tools/microsandbox-provider/snapshot.go b/services/agents-api/tools/microsandbox-provider/snapshot.go new file mode 100644 index 000000000..b92879a00 --- /dev/null +++ b/services/agents-api/tools/microsandbox-provider/snapshot.go @@ -0,0 +1,182 @@ +//go:build linux + +package main + +import ( + "context" + "fmt" + "strconv" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + wire "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/microsandbox" + sdk "github.com/superradcompany/microsandbox/sdk/go" +) + +func (b backend) snapshotLabels(operation string, source wire.Compute) map[string]string { + labels := wire.Labels(b.q.Config, b.q.Reference) + labels["io.parsar.operation"] = operation + labels["io.parsar.source_id"] = source.ID + labels["io.parsar.source_generation"] = strconv.FormatUint(source.Generation, 10) + return labels +} + +// inspectSnapshot opens only an allocation/operation-derived selector. +func (b backend) inspectSnapshot(ctx context.Context, operation string, source wire.Compute) (*sdk.SnapshotArtifact, wire.SnapshotIdentity, error) { + selector := wire.SnapshotReference(b.q.Config, b.q.Reference, operation) + artifact, e := sdk.Snapshot.Open(ctx, selector) + if e != nil { + return nil, wire.SnapshotIdentity{}, e + } + if artifact.Scope() != sdk.SnapshotScopeFull || artifact.SourceSandbox() == nil || *artifact.SourceSandbox() != source.Name { + return nil, wire.SnapshotIdentity{}, sandbox.ErrOwnership + } + for key, value := range b.snapshotLabels(operation, source) { + if artifact.Labels()[key] != value { + return nil, wire.SnapshotIdentity{}, sandbox.ErrOwnership + } + } + report, e := artifact.Verify(ctx) + if e != nil { + return nil, wire.SnapshotIdentity{}, e + } + state := artifact.State() + if report.Digest != artifact.Digest() || report.Checkpoint == nil || report.Checkpoint.Root == "" || report.Checkpoint.Kind == "" || state.Checkpoint == nil || state.Checkpoint.CheckpointID == "" { + return nil, wire.SnapshotIdentity{}, sandbox.ErrOwnership + } + identity := wire.SnapshotIdentity{Reference: selector, ID: artifact.ID(), Digest: artifact.Digest(), CheckpointID: state.Checkpoint.CheckpointID, CheckpointRoot: report.Checkpoint.Root, OperationID: operation, SourceName: source.Name, SourceID: source.ID, SourceGeneration: source.Generation} + if wire.ValidateSnapshot(b.q.Config, b.q.Reference, identity) != nil { + return nil, wire.SnapshotIdentity{}, sandbox.ErrOwnership + } + return artifact, identity, nil +} +func (b backend) verifiedSnapshot(ctx context.Context, want wire.SnapshotIdentity) (*sdk.SnapshotArtifact, error) { + a, got, e := b.inspectSnapshot(ctx, want.OperationID, wire.Compute{Name: want.SourceName, ID: want.SourceID, Generation: want.SourceGeneration}) + if e != nil { + return nil, e + } + if got != want { + return nil, sandbox.ErrOwnership + } + return a, nil +} +func (b backend) suspend(ctx context.Context, q wire.SuspendRequest) (wire.State, error) { + // The host allocation flock spans pause, capture, verification and source kill. + // A surviving completed artifact is observed, never overwritten or recaptured. + _, snap, e := b.inspectSnapshot(ctx, q.OperationID, q.Source) + if sdk.IsKind(e, sdk.ErrSnapshotNotFound) { + if q.ObserveOnly { + if q.Snapshot != nil { + return wire.State{}, wire.ErrUnconfirmed + } + _, state, observeErr := b.inspect(ctx, q.Source) + if observeErr != nil { + return wire.State{}, observeErr + } + if !state.BootstrapComplete || (state.Status != "running" && state.Status != "paused") { + return wire.State{}, wire.ErrUnconfirmed + } + return state, nil + } + if q.Snapshot != nil { + return wire.State{}, wire.ErrUnconfirmed + } + source, state, err := b.inspect(ctx, q.Source) + if err != nil { + return wire.State{}, err + } + if !state.BootstrapComplete { + return wire.State{}, wire.ErrUnconfirmed + } + if err = source.PauseWithGuestFlush(ctx, sdk.GuestFlushSkip); err != nil { + return wire.State{}, err + } + _, err = sdk.Snapshot.Create(ctx, sdk.SnapshotCreateOptions{ + Name: "s-" + q.OperationID, Group: wire.Name(b.q.Config, b.q.Reference, 0), FromSandbox: q.Source.Name, + Full: true, RecordIntegrity: true, GuestFlush: sdk.GuestFlushSkip, Labels: b.snapshotLabels(q.OperationID, q.Source)}) + if err != nil { + return wire.State{}, err + } + _, snap, e = b.inspectSnapshot(ctx, q.OperationID, q.Source) + } + if e != nil { + return wire.State{}, e + } + if q.Snapshot != nil && snap != *q.Snapshot { + return wire.State{}, sandbox.ErrOwnership + } + h, _, e := b.inspect(ctx, q.Source) + if e != nil && !sdk.IsKind(e, sdk.ErrSandboxNotFound) { + return wire.State{}, e + } + if q.ObserveOnly { + stopped := sdk.IsKind(e, sdk.ErrSandboxNotFound) || (e == nil && terminal(h.Status())) + status := "suspended" + if !stopped { + status = string(h.Status()) + } + return wire.State{Compute: q.Source, Status: status, BootstrapComplete: true, Snapshot: &snap, SourceStopped: stopped}, nil + } + if e == nil { + // Graceful stop could run the captured source after the checkpoint. + if e = h.Kill(ctx); e != nil { + return wire.State{}, e + } + observed, err := h.Refresh(ctx) + if err != nil { + return wire.State{}, err + } + if !terminal(observed.Status()) { + return wire.State{}, fmt.Errorf("source termination unconfirmed") + } + } + return wire.State{Compute: q.Source, Status: "suspended", BootstrapComplete: true, Snapshot: &snap, SourceStopped: true}, nil +} +func (b backend) resume(ctx context.Context, q wire.ResumeRequest) (wire.State, error) { + // Verify the exact full artifact before either adopting or creating a target. + artifact, e := b.verifiedSnapshot(ctx, q.Snapshot) + if e != nil { + return wire.State{}, e + } + _, state, e := b.inspect(ctx, q.Target) + if e == nil { + if state.Status != "running" || !state.BootstrapComplete { + return wire.State{}, wire.ErrUnconfirmed + } + return state, nil + } + if !sdk.IsKind(e, sdk.ErrSandboxNotFound) { + return wire.State{}, e + } + if q.ObserveOnly || q.Target.ID != "" { + return wire.State{}, wire.ErrUnconfirmed + } + source, err := sdk.GetSandbox(ctx, q.Snapshot.SourceName) + if err == nil { + if source.ID() != q.Snapshot.SourceID { + return wire.State{}, sandbox.ErrOwnership + } + if !terminal(source.Status()) { + return wire.State{}, wire.ErrUnconfirmed + } + } else if !sdk.IsKind(err, sdk.ErrSandboxNotFound) { + return wire.State{}, err + } + live, e := sdk.RestoreSandbox(ctx, artifact, q.Target.Name, sdk.WithRestoreNetworkPolicy(b.network())) + if e != nil { + return wire.State{}, e + } + target := q.Target + target.ID = live.ID() + if e = live.Detach(context.Background()); e != nil { + return wire.State{}, e + } + _, state, e = b.inspect(ctx, target) + if e == nil && (state.Status != "running" || !state.BootstrapComplete) { + return wire.State{}, wire.ErrUnconfirmed + } + return state, e +} + +func terminal(s sdk.SandboxStatus) bool { + return s == sdk.SandboxStatusStopped || s == sdk.SandboxStatusCrashed +}