diff --git a/cmd/ateom-gvisor/runsc.go b/cmd/ateom-gvisor/runsc.go index 2901c1d2e..1ff0baa81 100644 --- a/cmd/ateom-gvisor/runsc.go +++ b/cmd/ateom-gvisor/runsc.go @@ -27,11 +27,23 @@ import ( "github.com/agent-substrate/substrate/internal/ateompath" ) +// runsc drives one actor's gVisor sandbox with the exact runtime binary chosen +// by atelet. SandboxConfig pins each architecture's runsc asset by URL and +// SHA-256. Atelet verifies the digest, caches the binary at a content-addressed +// path, and passes that path to ateom. Checkpoint manifests retain the same +// asset pin so a restore uses the runsc version that created the checkpoint. +// +// All containers for an actor share a runsc root directory. The "pause" +// container is the sandbox root; application containers join that sandbox. type runsc struct { path string actorUID string } +// cmdCreate creates a stopped container from the OCI bundle prepared by +// atelet. Creating "pause" creates the sandbox; subsequent application +// containers join it. additionalArgs carries restore-specific create flags, +// such as the data-only filesystem image. func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName string, additionalArgs []string) error { reapLock.RLock() defer reapLock.RUnlock() @@ -70,6 +82,8 @@ func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName stri return nil } +// cmdStart starts a container previously created by cmdCreate. Connected +// sockets are allowed to survive a later full checkpoint. func (r *runsc) cmdStart(ctx context.Context, out io.Writer, containerName string) error { reapLock.RLock() defer reapLock.RUnlock() @@ -102,6 +116,9 @@ func (r *runsc) cmdStart(ctx context.Context, out io.Writer, containerName strin return nil } +// cmdCheckpoint writes a full process, sentry, and filesystem checkpoint for +// the shared sandbox. Callers invoke it only for the root "pause" container; +// that captures every container in the sandbox. func (r *runsc) cmdCheckpoint(ctx context.Context, containerName, checkpointPath string) error { reapLock.RLock() defer reapLock.RUnlock() @@ -132,6 +149,9 @@ func (r *runsc) cmdCheckpoint(ctx context.Context, containerName, checkpointPath return nil } +// cmdFsCheckpoint writes a data-only checkpoint of the listed durable +// directories. It deliberately excludes process state and other rootfs +// changes, so restore will cold-start the containers around the saved data. func (r *runsc) cmdFsCheckpoint(ctx context.Context, containerName, checkpointPath string, durableDirMounts []string) error { reapLock.RLock() defer reapLock.RUnlock() @@ -171,8 +191,10 @@ func (r *runsc) cmdFsCheckpoint(ctx context.Context, containerName, checkpointPa return nil } -// We take a checkpoint only of the root container of the sandbox, but we need -// to call restore on each container, using the same checkpoint. +// cmdRestore restores one container from a full sandbox checkpoint and leaves +// it running in the background. Although cmdCheckpoint runs only against the +// root container, restore must be called for the root and every application +// container using the same checkpoint. func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, checkpointPath string) error { reapLock.RLock() defer reapLock.RUnlock() @@ -207,6 +229,8 @@ func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, ch return nil } +// cmdDelete forcibly removes a container after its sandbox has been +// checkpointed. Ateom deletes application containers before the root. func (r *runsc) cmdDelete(ctx context.Context, containerName string) error { reapLock.RLock() defer reapLock.RUnlock() @@ -236,6 +260,9 @@ func (r *runsc) cmdDelete(ctx context.Context, containerName string) error { return nil } +// cmdState asks runsc to reconcile and report container state before deletion. +// This mirrors containerd's cleanup sequence and avoids intermittent failures +// from deleting immediately after checkpoint. func (r *runsc) cmdState(ctx context.Context, containerName string) error { reapLock.RLock() defer reapLock.RUnlock() diff --git a/internal/proto/ateompb/ateom.proto b/internal/proto/ateompb/ateom.proto index a282d81cb..b804ef1b0 100644 --- a/internal/proto/ateompb/ateom.proto +++ b/internal/proto/ateompb/ateom.proto @@ -18,32 +18,39 @@ package ateom; option go_package = "github.com/agent-substrate/substrate/internal/proto/ateompb"; -// Ateom is the interface to control a single gVisor (or, in the future microVM) -// guest inside a worker pod. +// Ateom is the interface used by atelet to control one sandbox guest inside a +// worker pod. Ateom serves gRPC on a per-worker Unix domain socket in the +// filesystem shared with atelet. Before calling an RPC, atelet prepares the OCI +// bundles, checkpoint files, and content-addressed runtime assets on that +// shared filesystem; the request carries their local paths. Calls for a given +// ateom must not overlap. // // Each ateom server has two main states, "available" and "executing". // -// When the ateom is "available", the substrate control plane is free to either -// boot a new workload (using RunWorkload), or restore an existing workload from -// a checkpoint (using RestoreWorkload). These calls move the ateom into -// "executing" state. +// When the ateom is "available", atelet can either boot a new workload (using +// RunWorkload), or restore an existing workload from a checkpoint (using +// RestoreWorkload). These calls move the ateom into "executing" state. // -// When the ateom is "executing", the substrate control plane can checkpoint the -// running workload (with CheckpointWorkload). This moves the ateom back to -// "free" state. +// When the ateom is "executing", atelet can checkpoint the running workload +// (with CheckpointWorkload). This moves the ateom back to "available" state. service Ateom { - // RunWorkload tells ateom to begin running a new workload (one or more - // containers, potentially with shared filesystems). + // RunWorkload requires an available ateom. It creates and starts the sandbox + // root plus all application containers from atelet-prepared OCI bundles, + // waits for configured readiness checks, and leaves the ateom executing. rpc RunWorkload(RunWorkloadRequest) returns (RunWorkloadResponse) {} - // CheckpointWorkload tells ateom to save the current state of the running - // workload to object storage, and then completely reset itself to a blank - // state (back to "available" state.) + // CheckpointWorkload requires an executing ateom. It writes either the full + // sandbox state or the requested durable-directory data to the shared + // checkpoint directory, tears down the sandbox, and returns the relative + // names of the files for atelet to persist. On success the ateom is + // available again. rpc CheckpointWorkload(CheckpointWorkloadRequest) returns (CheckpointWorkloadResponse) {} - // RestoreWorkload restores a workload from checkpoint that was previously - // written by CheckpointWorkload. Ateom will handle downloading the correct - // gVisor / runsc version to match the checkpoint. + // RestoreWorkload requires an available ateom. It restores from checkpoint + // files and matching runtime assets that atelet placed on the shared + // filesystem, waits for configured readiness checks, and leaves the ateom + // executing. A full snapshot restores process state; a data-only snapshot + // starts new processes with the durable-directory data restored. rpc RestoreWorkload(RestoreWorkloadRequest) returns (RestoreWorkloadResponse) {} } diff --git a/internal/proto/ateompb/ateom_grpc.pb.go b/internal/proto/ateompb/ateom_grpc.pb.go index 68d5bd57a..6152037d0 100644 --- a/internal/proto/ateompb/ateom_grpc.pb.go +++ b/internal/proto/ateompb/ateom_grpc.pb.go @@ -42,30 +42,37 @@ const ( // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // -// Ateom is the interface to control a single gVisor (or, in the future microVM) -// guest inside a worker pod. +// Ateom is the interface used by atelet to control one sandbox guest inside a +// worker pod. Ateom serves gRPC on a per-worker Unix domain socket in the +// filesystem shared with atelet. Before calling an RPC, atelet prepares the OCI +// bundles, checkpoint files, and content-addressed runtime assets on that +// shared filesystem; the request carries their local paths. Calls for a given +// ateom must not overlap. // // Each ateom server has two main states, "available" and "executing". // -// When the ateom is "available", the substrate control plane is free to either -// boot a new workload (using RunWorkload), or restore an existing workload from -// a checkpoint (using RestoreWorkload). These calls move the ateom into -// "executing" state. +// When the ateom is "available", atelet can either boot a new workload (using +// RunWorkload), or restore an existing workload from a checkpoint (using +// RestoreWorkload). These calls move the ateom into "executing" state. // -// When the ateom is "executing", the substrate control plane can checkpoint the -// running workload (with CheckpointWorkload). This moves the ateom back to -// "free" state. +// When the ateom is "executing", atelet can checkpoint the running workload +// (with CheckpointWorkload). This moves the ateom back to "available" state. type AteomClient interface { - // RunWorkload tells ateom to begin running a new workload (one or more - // containers, potentially with shared filesystems). + // RunWorkload requires an available ateom. It creates and starts the sandbox + // root plus all application containers from atelet-prepared OCI bundles, + // waits for configured readiness checks, and leaves the ateom executing. RunWorkload(ctx context.Context, in *RunWorkloadRequest, opts ...grpc.CallOption) (*RunWorkloadResponse, error) - // CheckpointWorkload tells ateom to save the current state of the running - // workload to object storage, and then completely reset itself to a blank - // state (back to "available" state.) + // CheckpointWorkload requires an executing ateom. It writes either the full + // sandbox state or the requested durable-directory data to the shared + // checkpoint directory, tears down the sandbox, and returns the relative + // names of the files for atelet to persist. On success the ateom is + // available again. CheckpointWorkload(ctx context.Context, in *CheckpointWorkloadRequest, opts ...grpc.CallOption) (*CheckpointWorkloadResponse, error) - // RestoreWorkload restores a workload from checkpoint that was previously - // written by CheckpointWorkload. Ateom will handle downloading the correct - // gVisor / runsc version to match the checkpoint. + // RestoreWorkload requires an available ateom. It restores from checkpoint + // files and matching runtime assets that atelet placed on the shared + // filesystem, waits for configured readiness checks, and leaves the ateom + // executing. A full snapshot restores process state; a data-only snapshot + // starts new processes with the durable-directory data restored. RestoreWorkload(ctx context.Context, in *RestoreWorkloadRequest, opts ...grpc.CallOption) (*RestoreWorkloadResponse, error) } @@ -111,30 +118,37 @@ func (c *ateomClient) RestoreWorkload(ctx context.Context, in *RestoreWorkloadRe // All implementations must embed UnimplementedAteomServer // for forward compatibility. // -// Ateom is the interface to control a single gVisor (or, in the future microVM) -// guest inside a worker pod. +// Ateom is the interface used by atelet to control one sandbox guest inside a +// worker pod. Ateom serves gRPC on a per-worker Unix domain socket in the +// filesystem shared with atelet. Before calling an RPC, atelet prepares the OCI +// bundles, checkpoint files, and content-addressed runtime assets on that +// shared filesystem; the request carries their local paths. Calls for a given +// ateom must not overlap. // // Each ateom server has two main states, "available" and "executing". // -// When the ateom is "available", the substrate control plane is free to either -// boot a new workload (using RunWorkload), or restore an existing workload from -// a checkpoint (using RestoreWorkload). These calls move the ateom into -// "executing" state. +// When the ateom is "available", atelet can either boot a new workload (using +// RunWorkload), or restore an existing workload from a checkpoint (using +// RestoreWorkload). These calls move the ateom into "executing" state. // -// When the ateom is "executing", the substrate control plane can checkpoint the -// running workload (with CheckpointWorkload). This moves the ateom back to -// "free" state. +// When the ateom is "executing", atelet can checkpoint the running workload +// (with CheckpointWorkload). This moves the ateom back to "available" state. type AteomServer interface { - // RunWorkload tells ateom to begin running a new workload (one or more - // containers, potentially with shared filesystems). + // RunWorkload requires an available ateom. It creates and starts the sandbox + // root plus all application containers from atelet-prepared OCI bundles, + // waits for configured readiness checks, and leaves the ateom executing. RunWorkload(context.Context, *RunWorkloadRequest) (*RunWorkloadResponse, error) - // CheckpointWorkload tells ateom to save the current state of the running - // workload to object storage, and then completely reset itself to a blank - // state (back to "available" state.) + // CheckpointWorkload requires an executing ateom. It writes either the full + // sandbox state or the requested durable-directory data to the shared + // checkpoint directory, tears down the sandbox, and returns the relative + // names of the files for atelet to persist. On success the ateom is + // available again. CheckpointWorkload(context.Context, *CheckpointWorkloadRequest) (*CheckpointWorkloadResponse, error) - // RestoreWorkload restores a workload from checkpoint that was previously - // written by CheckpointWorkload. Ateom will handle downloading the correct - // gVisor / runsc version to match the checkpoint. + // RestoreWorkload requires an available ateom. It restores from checkpoint + // files and matching runtime assets that atelet placed on the shared + // filesystem, waits for configured readiness checks, and leaves the ateom + // executing. A full snapshot restores process state; a data-only snapshot + // starts new processes with the durable-directory data restored. RestoreWorkload(context.Context, *RestoreWorkloadRequest) (*RestoreWorkloadResponse, error) mustEmbedUnimplementedAteomServer() } diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 9d21b2295..82f5b5ca4 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -38,17 +38,41 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// Status describes the Actor's control-plane lifecycle state. The normal +// successful transitions are: +// +// CreateActor: -> SUSPENDED +// ResumeActor: SUSPENDED or PAUSED -> RESUMING -> RUNNING +// SuspendActor: RUNNING -> SUSPENDING -> SUSPENDED +// PauseActor: RUNNING -> PAUSING -> PAUSED +// DeleteActor: SUSPENDED or CRASHED -> deleted +// +// An unrecoverable activation or checkpoint failure can move an Actor to +// CRASHED. Transitional states are persisted so a retry of the same RPC can +// continue an interrupted workflow. type Actor_Status int32 const ( + // Invalid or unknown lifecycle state. Actor_STATUS_UNSPECIFIED Actor_Status = 0 - Actor_STATUS_RESUMING Actor_Status = 1 - Actor_STATUS_RUNNING Actor_Status = 2 - Actor_STATUS_SUSPENDING Actor_Status = 3 - Actor_STATUS_SUSPENDED Actor_Status = 4 - Actor_STATUS_PAUSING Actor_Status = 5 - Actor_STATUS_PAUSED Actor_Status = 6 - Actor_STATUS_CRASHED Actor_Status = 7 + // A Worker is assigned and the workload is being booted or restored. + Actor_STATUS_RESUMING Actor_Status = 1 + // The workload is executing on its assigned Worker. + Actor_STATUS_RUNNING Actor_Status = 2 + // The workload is being checkpointed to external storage. + Actor_STATUS_SUSPENDING Actor_Status = 3 + // The Actor has no assigned Worker and is eligible for ResumeActor or + // DeleteActor. A newly created Actor starts in this state without its own + // snapshot. + Actor_STATUS_SUSPENDED Actor_Status = 4 + // The workload is being checkpointed to node-local storage. + Actor_STATUS_PAUSING Actor_Status = 5 + // The Actor has no assigned Worker and its latest snapshot remains on one + // or more recorded node VMs. + Actor_STATUS_PAUSED Actor_Status = 6 + // The workload or its lifecycle workflow failed irrecoverably. Its Worker + // has been released; the Actor can be inspected or deleted. + Actor_STATUS_CRASHED Actor_Status = 7 ) // Enum value maps for Actor_Status. diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 0d87ef7f5..1b2ef0b77 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -27,22 +27,40 @@ service Control { // Get an Actor. rpc GetActor(GetActorRequest) returns (Actor) {} - // Create a new Actor deriving from a given ActorTemplate. + // Create a new Actor deriving from a given ActorTemplate. The referenced + // Atespace and ActorTemplate must already exist, and the Actor's + // (atespace, name) identity must be unused. The server initializes the Actor + // in SUSPENDED state without assigning a Worker. rpc CreateActor(CreateActorRequest) returns (Actor) {} // Update mutable fields on an existing Actor. rpc UpdateActor(UpdateActorRequest) returns (UpdateActorResponse) {} - // Suspend a given actor to a new snapshot. + // Idempotently transition a RUNNING Actor through SUSPENDING to SUSPENDED. + // The configured on-commit state is checkpointed to external storage, the + // Worker is released, and the returned Actor references the new snapshot. + // A retry continues an interrupted SUSPENDING workflow; an already fully + // SUSPENDED Actor is returned unchanged. rpc SuspendActor(SuspendActorRequest) returns (SuspendActorResponse) {} - // Pause a given actor and keep its snapshots on node VM. + // Idempotently transition a RUNNING Actor through PAUSING to PAUSED. The + // configured on-pause state is checkpointed on the node VM, the Worker is + // released, and the returned Actor records which node holds the snapshot. + // A retry continues an interrupted PAUSING workflow; an already fully PAUSED + // Actor is returned unchanged. rpc PauseActor(PauseActorRequest) returns (PauseActorResponse) {} - // Resume an actor from its latest snapshot. + // Idempotently transition a SUSPENDED or PAUSED Actor through RESUMING to + // RUNNING. The server claims an eligible Worker and restores the Actor's + // latest snapshot. If the Actor has no snapshot, it restores the + // ActorTemplate's golden snapshot unless boot is true; otherwise it boots + // from the ActorTemplate spec. The call returns after configured readiness + // checks pass. A retry continues an interrupted RESUMING workflow; an + // already RUNNING Actor is returned unchanged. rpc ResumeActor(ResumeActorRequest) returns (ResumeActorResponse) {} - // Delete an actor. Only suspended actors can be deleted. + // Delete a SUSPENDED or CRASHED Actor and return its final record. Deletion + // removes the control-plane record; it does not remove snapshot files. rpc DeleteActor(DeleteActorRequest) returns (Actor) {} // List Workers. @@ -121,14 +139,45 @@ message Actor { string actor_template_namespace = 2; string actor_template_name = 3; + // Status describes the Actor's control-plane lifecycle state. The normal + // successful transitions are: + // + // CreateActor: -> SUSPENDED + // ResumeActor: SUSPENDED or PAUSED -> RESUMING -> RUNNING + // SuspendActor: RUNNING -> SUSPENDING -> SUSPENDED + // PauseActor: RUNNING -> PAUSING -> PAUSED + // DeleteActor: SUSPENDED or CRASHED -> deleted + // + // An unrecoverable activation or checkpoint failure can move an Actor to + // CRASHED. Transitional states are persisted so a retry of the same RPC can + // continue an interrupted workflow. enum Status { + // Invalid or unknown lifecycle state. STATUS_UNSPECIFIED = 0; + + // A Worker is assigned and the workload is being booted or restored. STATUS_RESUMING = 1; + + // The workload is executing on its assigned Worker. STATUS_RUNNING = 2; + + // The workload is being checkpointed to external storage. STATUS_SUSPENDING = 3; + + // The Actor has no assigned Worker and is eligible for ResumeActor or + // DeleteActor. A newly created Actor starts in this state without its own + // snapshot. STATUS_SUSPENDED = 4; + + // The workload is being checkpointed to node-local storage. STATUS_PAUSING = 5; + + // The Actor has no assigned Worker and its latest snapshot remains on one + // or more recorded node VMs. STATUS_PAUSED = 6; + + // The workload or its lifecycle workflow failed irrecoverably. Its Worker + // has been released; the Actor can be inspected or deleted. STATUS_CRASHED = 7; } Status status = 4; diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index 80b80091a..4f20d82bf 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -58,17 +58,35 @@ const ( type ControlClient interface { // Get an Actor. GetActor(ctx context.Context, in *GetActorRequest, opts ...grpc.CallOption) (*Actor, error) - // Create a new Actor deriving from a given ActorTemplate. + // Create a new Actor deriving from a given ActorTemplate. The referenced + // Atespace and ActorTemplate must already exist, and the Actor's + // (atespace, name) identity must be unused. The server initializes the Actor + // in SUSPENDED state without assigning a Worker. CreateActor(ctx context.Context, in *CreateActorRequest, opts ...grpc.CallOption) (*Actor, error) // Update mutable fields on an existing Actor. UpdateActor(ctx context.Context, in *UpdateActorRequest, opts ...grpc.CallOption) (*UpdateActorResponse, error) - // Suspend a given actor to a new snapshot. + // Idempotently transition a RUNNING Actor through SUSPENDING to SUSPENDED. + // The configured on-commit state is checkpointed to external storage, the + // Worker is released, and the returned Actor references the new snapshot. + // A retry continues an interrupted SUSPENDING workflow; an already fully + // SUSPENDED Actor is returned unchanged. SuspendActor(ctx context.Context, in *SuspendActorRequest, opts ...grpc.CallOption) (*SuspendActorResponse, error) - // Pause a given actor and keep its snapshots on node VM. + // Idempotently transition a RUNNING Actor through PAUSING to PAUSED. The + // configured on-pause state is checkpointed on the node VM, the Worker is + // released, and the returned Actor records which node holds the snapshot. + // A retry continues an interrupted PAUSING workflow; an already fully PAUSED + // Actor is returned unchanged. PauseActor(ctx context.Context, in *PauseActorRequest, opts ...grpc.CallOption) (*PauseActorResponse, error) - // Resume an actor from its latest snapshot. + // Idempotently transition a SUSPENDED or PAUSED Actor through RESUMING to + // RUNNING. The server claims an eligible Worker and restores the Actor's + // latest snapshot. If the Actor has no snapshot, it restores the + // ActorTemplate's golden snapshot unless boot is true; otherwise it boots + // from the ActorTemplate spec. The call returns after configured readiness + // checks pass. A retry continues an interrupted RESUMING workflow; an + // already RUNNING Actor is returned unchanged. ResumeActor(ctx context.Context, in *ResumeActorRequest, opts ...grpc.CallOption) (*ResumeActorResponse, error) - // Delete an actor. Only suspended actors can be deleted. + // Delete a SUSPENDED or CRASHED Actor and return its final record. Deletion + // removes the control-plane record; it does not remove snapshot files. DeleteActor(ctx context.Context, in *DeleteActorRequest, opts ...grpc.CallOption) (*Actor, error) // List Workers. ListWorkers(ctx context.Context, in *ListWorkersRequest, opts ...grpc.CallOption) (*ListWorkersResponse, error) @@ -230,17 +248,35 @@ func (c *controlClient) DeleteAtespace(ctx context.Context, in *DeleteAtespaceRe type ControlServer interface { // Get an Actor. GetActor(context.Context, *GetActorRequest) (*Actor, error) - // Create a new Actor deriving from a given ActorTemplate. + // Create a new Actor deriving from a given ActorTemplate. The referenced + // Atespace and ActorTemplate must already exist, and the Actor's + // (atespace, name) identity must be unused. The server initializes the Actor + // in SUSPENDED state without assigning a Worker. CreateActor(context.Context, *CreateActorRequest) (*Actor, error) // Update mutable fields on an existing Actor. UpdateActor(context.Context, *UpdateActorRequest) (*UpdateActorResponse, error) - // Suspend a given actor to a new snapshot. + // Idempotently transition a RUNNING Actor through SUSPENDING to SUSPENDED. + // The configured on-commit state is checkpointed to external storage, the + // Worker is released, and the returned Actor references the new snapshot. + // A retry continues an interrupted SUSPENDING workflow; an already fully + // SUSPENDED Actor is returned unchanged. SuspendActor(context.Context, *SuspendActorRequest) (*SuspendActorResponse, error) - // Pause a given actor and keep its snapshots on node VM. + // Idempotently transition a RUNNING Actor through PAUSING to PAUSED. The + // configured on-pause state is checkpointed on the node VM, the Worker is + // released, and the returned Actor records which node holds the snapshot. + // A retry continues an interrupted PAUSING workflow; an already fully PAUSED + // Actor is returned unchanged. PauseActor(context.Context, *PauseActorRequest) (*PauseActorResponse, error) - // Resume an actor from its latest snapshot. + // Idempotently transition a SUSPENDED or PAUSED Actor through RESUMING to + // RUNNING. The server claims an eligible Worker and restores the Actor's + // latest snapshot. If the Actor has no snapshot, it restores the + // ActorTemplate's golden snapshot unless boot is true; otherwise it boots + // from the ActorTemplate spec. The call returns after configured readiness + // checks pass. A retry continues an interrupted RESUMING workflow; an + // already RUNNING Actor is returned unchanged. ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) - // Delete an actor. Only suspended actors can be deleted. + // Delete a SUSPENDED or CRASHED Actor and return its final record. Deletion + // removes the control-plane record; it does not remove snapshot files. DeleteActor(context.Context, *DeleteActorRequest) (*Actor, error) // List Workers. ListWorkers(context.Context, *ListWorkersRequest) (*ListWorkersResponse, error)