diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b0586071f1..e68c7aaf94 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -107,6 +107,16 @@ task cli:build:dev ### Desktop Development +In renderer code, use `$lib/...` for imports across renderer library modules and +`$shared/...` for shared modules. Apply this to type imports, re-exports, and +dynamic imports as well. Keep `./...` for sibling files and relative paths for +renderer entry points without an existing alias. Avoid parent-directory imports +(`../...`) when one of these aliases addresses the module. + +Main-process, preload, and shared code use relative imports: the renderer aliases +are not configured in the main/preload bundlers. Preserve the existing `.js` +extension on TypeScript module specifiers and `.svelte` on component imports. + ```bash # Install dependencies cd desktop diff --git a/cmd/internal/agent.go b/cmd/internal/agent.go index acbdbf27ed..5a6d416c66 100644 --- a/cmd/internal/agent.go +++ b/cmd/internal/agent.go @@ -25,6 +25,7 @@ func NewAgentCmd(globalFlags *flags.GlobalFlags) *cobra.Command { agentCmd.AddCommand(agentworkspace.NewWorkspaceCmd(globalFlags)) agentCmd.AddCommand(agentcontainer.NewContainerCmd(globalFlags)) agentCmd.AddCommand(NewDaemonCmd(globalFlags)) + agentCmd.AddCommand(NewDaemonDiagnosticsCmd(globalFlags)) agentCmd.AddCommand(NewContainerTunnelCmd(globalFlags)) agentCmd.AddCommand(NewGitCredentialsCmd(globalFlags)) agentCmd.AddCommand(NewGitSSHSignatureCmd(globalFlags)) diff --git a/cmd/internal/agent_daemon.go b/cmd/internal/agent_daemon.go index 1f18a205a0..63f1badaac 100644 --- a/cmd/internal/agent_daemon.go +++ b/cmd/internal/agent_daemon.go @@ -3,7 +3,6 @@ package cmdinternal import ( "bytes" "context" - "errors" "fmt" "os" "path/filepath" @@ -14,11 +13,13 @@ import ( "github.com/devsy-org/devsy/pkg/agent" "github.com/devsy-org/devsy/pkg/client/clientimplementation" agentconfig "github.com/devsy-org/devsy/pkg/config" + agentdaemon "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/driver/custom" cliflags "github.com/devsy-org/devsy/pkg/flags" "github.com/devsy-org/devsy/pkg/flags/names" "github.com/devsy-org/devsy/pkg/log" + "github.com/devsy-org/devsy/pkg/machinediagnostics" provider2 "github.com/devsy-org/devsy/pkg/provider" "github.com/spf13/cobra" ) @@ -28,11 +29,25 @@ const ( busyGracePeriod = 20 * time.Minute ) +var daemonRuntimeLockPath = machinediagnostics.DefaultRuntimeLockPath + type DaemonCmd struct { *flags.GlobalFlags - Interval string - ShutdownAction string + Interval string + ShutdownAction string + StateRoot string + StateLayout string + DiagnosticsReaderUID int + DiagnosticsReaderGID int + + recorder machinediagnostics.Recorder + startedAt time.Time + lastSuccessfulPatrolAt *time.Time + lastPatrolAt *time.Time + lastDiagnosticError *machinediagnostics.DiagnosticError + knownWorkspaceIDs map[string]struct{} + knownInvalidWorkspaceIDs map[string]struct{} } func NewDaemonCmd(flags *flags.GlobalFlags) *cobra.Command { @@ -54,30 +69,142 @@ func NewDaemonCmd(flags *flags.GlobalFlags) *cobra.Command { "", "The shutdown action (none, stopContainer, or stopCompose)", ), + cliflags.String(&cmd.StateRoot, names.StateRoot, "", "The daemon workspace state root"), + cliflags.String( + &cmd.StateLayout, + names.StateLayout, + "", + "The daemon workspace state layout", + ), + cliflags.Int( + &cmd.DiagnosticsReaderUID, + names.DiagnosticsReaderUID, + -1, + "The diagnostics reader UID", + ), + cliflags.Int( + &cmd.DiagnosticsReaderGID, + names.DiagnosticsReaderGID, + -1, + "The diagnostics reader GID", + ), ) + _ = daemonCmd.Flags().MarkHidden(names.StateRoot) + _ = daemonCmd.Flags().MarkHidden(names.StateLayout) + _ = daemonCmd.Flags().MarkHidden(names.DiagnosticsReaderUID) + _ = daemonCmd.Flags().MarkHidden(names.DiagnosticsReaderGID) return daemonCmd } func (cmd *DaemonCmd) Run(ctx context.Context) error { - // The daemon runs only container/machine-side; the host never invokes it. - if agent.IsHostAgentInvocation(cmd.AgentDir) { - return errors.New( - "`devsy internal agent daemon` is only valid inside the workspace container or machine", - ) + location, err := cmd.stateLocation() + if err != nil { + return err } - - logDir, err := agent.GetAgentDaemonLogDir(cmd.AgentDir) + runtimeLockPath := daemonRuntimeLockPath + if fallbackPath := os.Getenv(machinediagnostics.RuntimeLockPathEnv); fallbackPath != "" { + runtimeLockPath = fallbackPath + } + runtimeLock, err := machinediagnostics.AcquireRuntimeLock(runtimeLockPath) if err != nil { return err } + defer func() { _ = runtimeLock.Close() }() + cmd.startedAt = time.Now().UTC() + recorder, err := machinediagnostics.NewRecorder(machinediagnostics.Options{ + Dir: machinediagnostics.DiagnosticsDir(location.Root), + Reader: cmd.diagnosticsReader(), + PatrolInterval: cmd.pollInterval(), + ErrorReporter: func(err error) { + log.Warnf("write machine diagnostics: %v", err) + }, + }) + if err != nil { + log.Warnf("initialize machine diagnostics: %v", err) + cmd.recorder = machinediagnostics.Nop() + } else { + cmd.recorder = recorder + if err := machinediagnostics.WriteLocator( + machinediagnostics.DefaultLocatorPath, + machinediagnostics.Locator{ + SessionID: recorder.SessionID(), + DiagnosticsDir: machinediagnostics.DiagnosticsDir(location.Root), + StateLayout: string(location.Layout), + StartedAt: cmd.startedAt, + }, + ); err != nil { + log.Warnf("write machine diagnostics locator: %v", err) + } + } + cmd.recordEvent( + machinediagnostics.Event{ + Type: machinediagnostics.EventDaemonStarted, + Level: machinediagnostics.LevelInfo, + Message: "Devsy machine daemon started.", + }, + ) + cmd.updateDiagnostics( + diagnosticsUpdate{ + state: machinediagnostics.DaemonStarting, + health: machinediagnostics.DaemonHealthy, + }, + ) + defer func() { + cmd.recordEvent( + machinediagnostics.Event{ + Type: machinediagnostics.EventDaemonStopping, + Level: machinediagnostics.LevelInfo, + Message: "Devsy machine daemon stopping.", + }, + ) + cmd.updateDiagnostics( + diagnosticsUpdate{ + state: machinediagnostics.DaemonStopping, + health: machinediagnostics.DaemonHealthy, + }, + ) + _ = cmd.recorder.Close() + }() - log.Infof("starting Devsy daemon patrol at %s", logDir) + log.Infof("starting Devsy daemon patrol at %s", location.Root) cmd.patrol(ctx) return nil } +func (cmd *DaemonCmd) stateLocation() (agentdaemon.StateLocation, error) { + if cmd.StateRoot != "" || cmd.StateLayout != "" { + if cmd.StateRoot == "" || cmd.StateLayout == "" { + return agentdaemon.StateLocation{}, fmt.Errorf( + "daemon state root and state layout must be provided together", + ) + } + location := agentdaemon.StateLocation{ + Root: filepath.Clean(cmd.StateRoot), + Layout: agentdaemon.StateLayout(cmd.StateLayout), + } + return location, location.Validate() + } + + if cmd.AgentDir != "" { + location := agentdaemon.StateLocation{ + Root: filepath.Clean(cmd.AgentDir), + Layout: agentdaemon.StateLayoutAgentHome, + } + return location, location.Validate() + } + + return agentdaemon.StateLocation{}, fmt.Errorf("daemon state location is missing") +} + func (cmd *DaemonCmd) patrol(ctx context.Context) { cmd.initialTouch() + cmd.recordEvent( + machinediagnostics.Event{ + Type: machinediagnostics.EventDaemonReady, + Level: machinediagnostics.LevelInfo, + Message: "Devsy machine daemon is ready.", + }, + ) ticker := time.NewTicker(cmd.pollInterval()) defer ticker.Stop() @@ -91,6 +218,62 @@ func (cmd *DaemonCmd) patrol(ctx context.Context) { } } +func (cmd *DaemonCmd) diagnosticsReader() machinediagnostics.ReaderIdentity { + if cmd.DiagnosticsReaderUID >= 0 && cmd.DiagnosticsReaderGID >= 0 { + return machinediagnostics.ReaderIdentity{ + UID: cmd.DiagnosticsReaderUID, + GID: cmd.DiagnosticsReaderGID, + } + } + return agentdaemon.DiagnosticsReaderIdentity() +} + +func (cmd *DaemonCmd) recordEvent(event machinediagnostics.Event) { + if cmd.recorder != nil { + cmd.recorder.Record(event) + } +} + +type diagnosticsUpdate struct { + state machinediagnostics.DaemonState + health machinediagnostics.DaemonHealth + diagnosticErr *machinediagnostics.DiagnosticError + workspaces []machinediagnostics.WorkspaceStatus + candidate *machinediagnostics.ShutdownCandidate +} + +func (cmd *DaemonCmd) updateDiagnostics(update diagnosticsUpdate) { + if cmd.recorder == nil { + return + } + now := time.Now().UTC() + if update.diagnosticErr != nil { + cmd.lastDiagnosticError = update.diagnosticErr + } + if update.state == machinediagnostics.DaemonRunning { + cmd.lastPatrolAt = &now + } + if update.health == machinediagnostics.DaemonHealthy && + update.state == machinediagnostics.DaemonRunning { + cmd.lastSuccessfulPatrolAt = &now + } + cmd.recorder.Update( + machinediagnostics.Status{ + StartedAt: cmd.startedAt, + UpdatedAt: now, + State: update.state, + Health: update.health, + PatrolInterval: cmd.pollInterval().String(), + LastPatrolAt: cmd.lastPatrolAt, + LastSuccessAt: cmd.lastSuccessfulPatrolAt, + LastError: cmd.lastDiagnosticError, + WorkspaceCount: len(update.workspaces), + Workspaces: update.workspaces, + ShutdownCandidate: update.candidate, + }, + ) +} + func (cmd *DaemonCmd) pollInterval() time.Duration { if cmd.Interval == "" { return defaultPatrolInterval @@ -108,47 +291,125 @@ func (cmd *DaemonCmd) pollInterval() time.Duration { } func (cmd *DaemonCmd) workspaceConfigs() (baseDir string, configs []string, err error) { - baseDir, err = agent.FindAgentHomeDir(cmd.AgentDir) + location, err := cmd.stateLocation() + if err != nil { + return "", nil, err + } + pattern, err := location.WorkspaceConfigPattern() if err != nil { return "", nil, err } - pattern := filepath.Join( - baseDir, - "contexts", - "*", - "workspaces", - "*", - provider2.WorkspaceConfigFile, - ) configs, err = filepath.Glob(pattern) if err != nil { return "", nil, fmt.Errorf("glob %s: %w", pattern, err) } - return baseDir, configs, nil + return location.Root, configs, nil } func (cmd *DaemonCmd) patrolOnce(ctx context.Context) { baseDir, configs, err := cmd.workspaceConfigs() if err != nil { log.Errorf("list workspace configs: %v", err) + cmd.recordEvent( + machinediagnostics.Event{ + Type: machinediagnostics.EventPatrolFailed, + Level: machinediagnostics.LevelError, + Message: "Machine daemon could not discover workspace state.", + ErrorCode: "patrol_failed", + }, + ) + cmd.updateDiagnostics(diagnosticsUpdate{ + state: machinediagnostics.DaemonRunning, + health: machinediagnostics.DaemonDegraded, + diagnosticErr: &machinediagnostics.DiagnosticError{ + Code: "patrol_failed", + Message: "Machine daemon could not discover workspace state.", + Timestamp: time.Now().UTC(), + }, + }) return } + evaluation := evaluateMachineInactivity( + configs, + activityHeartbeat(), + cmd.ShutdownAction, + time.Now(), + ) + cmd.recordWorkspaceTransitions(evaluation.statuses) + cmd.updateDiagnostics( + diagnosticsUpdate{ + state: machinediagnostics.DaemonRunning, + health: machinediagnostics.DaemonHealthy, + workspaces: evaluation.statuses, + candidate: evaluation.candidate, + }, + ) + if evaluation.workspace == nil { + log.Infof("no machine shutdown candidate in %q: %s", baseDir, evaluation.reason) + return + } + if err := cmd.shutdownWorkspace(ctx, evaluation.workspace); err != nil { + cmd.updateDiagnostics(diagnosticsUpdate{ + state: machinediagnostics.DaemonRunning, + health: machinediagnostics.DaemonDegraded, + workspaces: evaluation.statuses, + candidate: evaluation.candidate, + diagnosticErr: &machinediagnostics.DiagnosticError{ + Code: "shutdown_failed", + Message: "Machine shutdown action failed.", + Timestamp: time.Now().UTC(), + }, + }) + } +} - latestActivity, workspace := findLatestActivity(configs) - if latestActivity == nil { - if len(configs) == 0 { - log.Infof("no workspaces found in %q", baseDir) - } else { - log.Infof( - "%d workspaces found in %q, but none had auto-stop configured or were still running", - len(configs), - baseDir, +func (cmd *DaemonCmd) recordWorkspaceTransitions(statuses []machinediagnostics.WorkspaceStatus) { + current := make(map[string]struct{}, len(statuses)) + invalid := make(map[string]struct{}) + for _, status := range statuses { + if status.ID == "" { + continue + } + current[status.ID] = struct{}{} + if _, known := cmd.knownWorkspaceIDs[status.ID]; !known { + cmd.recordEvent( + machinediagnostics.Event{ + Type: machinediagnostics.EventWorkspaceDiscovered, + Level: machinediagnostics.LevelInfo, + WorkspaceID: status.ID, + Message: "Workspace state was discovered.", + }, ) } - return + if status.State == machinediagnostics.WorkspaceInvalidConfig { + invalid[status.ID] = struct{}{} + if _, alreadyReported := cmd.knownInvalidWorkspaceIDs[status.ID]; !alreadyReported { + cmd.recordEvent( + machinediagnostics.Event{ + Type: machinediagnostics.EventWorkspaceConfigErr, + Level: machinediagnostics.LevelWarn, + WorkspaceID: status.ID, + Message: "Workspace state could not be parsed.", + ErrorCode: "workspace_config_invalid", + }, + ) + } + } } - - cmd.checkAndShutdown(ctx, effectiveActivity(*latestActivity), workspace) + for id := range cmd.knownWorkspaceIDs { + if _, stillPresent := current[id]; !stillPresent { + cmd.recordEvent( + machinediagnostics.Event{ + Type: machinediagnostics.EventWorkspaceRemoved, + Level: machinediagnostics.LevelInfo, + WorkspaceID: id, + Message: "Workspace state is no longer present.", + }, + ) + } + } + cmd.knownWorkspaceIDs = current + cmd.knownInvalidWorkspaceIDs = invalid } var activityFilePath = agentconfig.ContainerActivityFile @@ -168,37 +429,47 @@ func activityHeartbeat() time.Time { return stat.ModTime() } -func (cmd *DaemonCmd) checkAndShutdown( +func (cmd *DaemonCmd) shutdownWorkspace( ctx context.Context, - latestActivity time.Time, workspace *provider2.AgentWorkspaceInfo, -) { - if cmd.effectiveShutdownAction(workspace) == config.ShutdownActionNone { - return - } - - timeout := agent.DefaultInactivityTimeout - if workspace.Agent.Timeout != "" { - parsed, err := time.ParseDuration(workspace.Agent.Timeout) - if err != nil { - log.Errorf("parse inactivity timeout, using %s: %v", timeout, err) - } else { - timeout = parsed - } - } - - deadline := latestActivity.Add(timeout) - if deadline.After(time.Now()) { - log.Infof( - "workspace %q last active %s, auto-stop in %s", - workspace.Workspace.ID, - latestActivity.Format(time.RFC3339), - time.Until(deadline).Round(time.Second), +) error { + cmd.recordEvent( + machinediagnostics.Event{ + Type: machinediagnostics.EventIdleDeadline, + Level: machinediagnostics.LevelInfo, + WorkspaceID: workspace.Workspace.ID, + Message: "Workspace inactivity deadline was reached.", + }, + ) + cmd.recordEvent( + machinediagnostics.Event{ + Type: machinediagnostics.EventShutdownStarted, + Level: machinediagnostics.LevelInfo, + WorkspaceID: workspace.Workspace.ID, + Message: "Machine shutdown action started.", + }, + ) + if err := cmd.runShutdownCommand(ctx, workspace); err != nil { + cmd.recordEvent( + machinediagnostics.Event{ + Type: machinediagnostics.EventShutdownFailed, + Level: machinediagnostics.LevelError, + WorkspaceID: workspace.Workspace.ID, + Message: "Machine shutdown action failed.", + ErrorCode: "shutdown_failed", + }, ) - return + return err } - - cmd.runShutdownCommand(ctx, workspace) + cmd.recordEvent( + machinediagnostics.Event{ + Type: machinediagnostics.EventShutdownSucceeded, + Level: machinediagnostics.LevelInfo, + WorkspaceID: workspace.Workspace.ID, + Message: "Machine shutdown command completed; provider state confirms whether the machine stopped.", + }, + ) + return nil } // effectiveShutdownAction prefers the workspace's resolved config, falling back @@ -218,11 +489,11 @@ func (cmd *DaemonCmd) effectiveShutdownAction( func (cmd *DaemonCmd) runShutdownCommand( ctx context.Context, workspace *provider2.AgentWorkspaceInfo, -) { +) error { environ, err := custom.ToEnvironWithBinaries(ctx, workspace) if err != nil { log.Errorf("build shutdown environment: %v", err) - return + return err } shutdown := strings.Join(workspace.Agent.Exec.Shutdown, " ") @@ -240,10 +511,11 @@ func (cmd *DaemonCmd) runShutdownCommand( "run shutdown command %s: %v (stdout: %s, stderr: %s)", shutdown, err, stdout.String(), stderr.String(), ) - return + return err } log.Infof("ran shutdown command (stdout: %s, stderr: %s)", stdout.String(), stderr.String()) + return nil } func (cmd *DaemonCmd) initialTouch() { @@ -301,3 +573,227 @@ func getActivity(workspaceConfig string) (*time.Time, *provider2.AgentWorkspaceI } return &activity, workspace, nil } + +// diagnosticsWorkspaceStatuses observes every discovered configuration for the +// status snapshot. It deliberately does not participate in selecting the one +// latest workspace that controls existing shutdown semantics. +type machineInactivityEvaluation struct { + statuses []machinediagnostics.WorkspaceStatus + workspace *provider2.AgentWorkspaceInfo + candidate *machinediagnostics.ShutdownCandidate + reason string +} + +type evaluatedWorkspace struct { + status machinediagnostics.WorkspaceStatus + workspace *provider2.AgentWorkspaceInfo +} + +func evaluateMachineInactivity( + configs []string, + heartbeat time.Time, + fallbackAction string, + now time.Time, +) machineInactivityEvaluation { + evaluated := evaluateWorkspaces(configs, heartbeat, fallbackAction, now) + result := machineInactivityEvaluation{statuses: workspaceStatuses(evaluated)} + if blocker := firstWorkspaceBlocker(evaluated); blocker != "" { + result.reason = blocker + return result + } + if len(evaluated) == 0 { + result.reason = "no workspace state was discovered" + return result + } + result.workspace, result.candidate = latestEligibleWorkspace(evaluated) + result.reason = "all workspaces are idle" + return result +} + +func evaluateWorkspaces( + configs []string, + heartbeat time.Time, + fallbackAction string, + now time.Time, +) []evaluatedWorkspace { + evaluated := make([]evaluatedWorkspace, 0, len(configs)) + for _, path := range configs { + evaluated = append( + evaluated, + evaluateWorkspaceInactivity(path, heartbeat, fallbackAction, now), + ) + } + return evaluated +} + +func workspaceStatuses(evaluated []evaluatedWorkspace) []machinediagnostics.WorkspaceStatus { + statuses := make([]machinediagnostics.WorkspaceStatus, 0, len(evaluated)) + for _, item := range evaluated { + statuses = append(statuses, item.status) + } + return statuses +} + +func firstWorkspaceBlocker(evaluated []evaluatedWorkspace) string { + for _, item := range evaluated { + if item.status.BlocksMachineShutdown { + return item.status.BlockerReason + } + } + return "" +} + +func latestEligibleWorkspace( + evaluated []evaluatedWorkspace, +) (*provider2.AgentWorkspaceInfo, *machinediagnostics.ShutdownCandidate) { + var selected *evaluatedWorkspace + for index := range evaluated { + item := &evaluated[index] + if selected == nil || item.status.IdleDeadlineAt.After(*selected.status.IdleDeadlineAt) || + (item.status.IdleDeadlineAt.Equal(*selected.status.IdleDeadlineAt) && item.status.ID < selected.status.ID) { + selected = item + } + } + deadline := *selected.status.IdleDeadlineAt + return selected.workspace, &machinediagnostics.ShutdownCandidate{ + WorkspaceID: selected.status.ID, + EligibleAt: &deadline, + } +} + +func evaluateWorkspaceInactivity( + path string, + heartbeat time.Time, + fallbackAction string, + now time.Time, +) evaluatedWorkspace { + workspace, result := parseWorkspace(path) + if result.status.State != "" { + return result + } + status := machinediagnostics.WorkspaceStatus{ID: workspace.Workspace.ID} + stat, err := os.Stat(path) + if err != nil { + return blockedWorkspaceResult( + status.ID, + machinediagnostics.WorkspaceNotRunning, + "Workspace state is unavailable", + workspace, + ) + } + status.ShutdownActionEnabled = shutdownActionEnabled(workspace, fallbackAction) + if !status.ShutdownActionEnabled { + return blockedWorkspaceResult( + status.ID, + machinediagnostics.WorkspaceNotConfigured, + "Auto-stop is not configured", + workspace, + ) + } + activity := maxTime(stat.ModTime(), heartbeat) + status.LastActivityAt = &activity + timeout, valid := workspaceTimeout(workspace) + if !valid { + return blockedWorkspaceResult( + status.ID, + machinediagnostics.WorkspaceInvalidConfig, + "Inactivity timeout must be a positive duration", + workspace, + ) + } + status.Timeout = timeout.String() + status.Busy = agent.HasWorkspaceBusyFile(filepath.Dir(path)) + if status.Busy { + deadline := activity.Add(busyGracePeriod).Add(timeout) + status.IdleDeadlineAt = &deadline + status.State = machinediagnostics.WorkspaceBusy + status.BlocksMachineShutdown = true + status.BlockerReason = "Workspace is busy" + return evaluatedWorkspace{status: status, workspace: workspace} + } + deadline := activity.Add(timeout) + status.IdleDeadlineAt = &deadline + if deadline.After(now) { + status.State = machinediagnostics.WorkspaceActive + status.BlocksMachineShutdown = true + status.BlockerReason = "Waiting for inactivity deadline" + } else { + status.State = machinediagnostics.WorkspaceIdleDue + } + return evaluatedWorkspace{status: status, workspace: workspace} +} + +func parseWorkspace(path string) (*provider2.AgentWorkspaceInfo, evaluatedWorkspace) { + workspace, err := agent.ParseAgentWorkspaceInfo(path) + if err != nil || workspace.Workspace == nil { + return workspace, evaluatedWorkspace{ + status: blockedWorkspaceStatus( + workspaceIDFromConfig(path), + machinediagnostics.WorkspaceInvalidConfig, + "Workspace configuration is invalid", + ), + } + } + return workspace, evaluatedWorkspace{} +} + +func blockedWorkspaceResult( + id string, + state machinediagnostics.WorkspaceEvaluationState, + reason string, + workspace *provider2.AgentWorkspaceInfo, +) evaluatedWorkspace { + return evaluatedWorkspace{ + status: blockedWorkspaceStatus(id, state, reason), + workspace: workspace, + } +} + +func shutdownActionEnabled(workspace *provider2.AgentWorkspaceInfo, fallback string) bool { + action := fallback + if workspace.LastDevContainerConfig != nil && workspace.LastDevContainerConfig.Config != nil && + workspace.LastDevContainerConfig.Config.ShutdownAction != "" { + action = workspace.LastDevContainerConfig.Config.ShutdownAction + } + return len(workspace.Agent.Exec.Shutdown) > 0 && action != config.ShutdownActionNone +} + +func maxTime(first, second time.Time) time.Time { + if second.After(first) { + return second + } + return first +} + +func workspaceTimeout(workspace *provider2.AgentWorkspaceInfo) (time.Duration, bool) { + if workspace.Agent.Timeout == "" { + return agent.DefaultInactivityTimeout, true + } + parsed, err := time.ParseDuration(workspace.Agent.Timeout) + return parsed, err == nil && parsed > 0 +} + +func blockedWorkspaceStatus( + id string, + state machinediagnostics.WorkspaceEvaluationState, + reason string, +) machinediagnostics.WorkspaceStatus { + return machinediagnostics.WorkspaceStatus{ + ID: id, + State: state, + BlocksMachineShutdown: true, + BlockerReason: reason, + } +} + +func diagnosticsWorkspaceStatuses(configs []string) []machinediagnostics.WorkspaceStatus { + return evaluateMachineInactivity(configs, activityHeartbeat(), "", time.Now()).statuses +} + +func workspaceIDFromConfig(path string) string { + parent := filepath.Dir(path) + if filepath.Base(parent) == "agent" { + parent = filepath.Dir(parent) + } + return filepath.Base(parent) +} diff --git a/cmd/internal/agent_daemon_diagnostics.go b/cmd/internal/agent_daemon_diagnostics.go new file mode 100644 index 0000000000..99927c5bd7 --- /dev/null +++ b/cmd/internal/agent_daemon_diagnostics.go @@ -0,0 +1,99 @@ +package cmdinternal + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/devsy-org/devsy/cmd/flags" + agentdaemon "github.com/devsy-org/devsy/pkg/daemon/agent" + cliflags "github.com/devsy-org/devsy/pkg/flags" + "github.com/devsy-org/devsy/pkg/flags/names" + "github.com/devsy-org/devsy/pkg/machinediagnostics" + "github.com/spf13/cobra" +) + +// DaemonDiagnosticsCmd is the machine-side, JSON-only diagnostics reader. +type DaemonDiagnosticsCmd struct { + *flags.GlobalFlags + After string + Limit int + StateRoot string + StateLayout string +} + +func NewDaemonDiagnosticsCmd(globalFlags *flags.GlobalFlags) *cobra.Command { + cmd := &DaemonDiagnosticsCmd{GlobalFlags: globalFlags} + cobraCmd := &cobra.Command{ + Use: "daemon-diagnostics", + Hidden: true, + Args: cobra.NoArgs, + RunE: func(c *cobra.Command, _ []string) error { return cmd.Run(c.Context()) }, + } + cliflags.Add( + cobraCmd, + cliflags.String(&cmd.After, "after", "", "An opaque diagnostics cursor"), + cliflags.Int( + &cmd.Limit, + "limit", + machinediagnostics.DefaultReadEvents, + "Maximum diagnostic events", + ), + cliflags.String(&cmd.StateRoot, names.StateRoot, "", "Diagnostics state-root override"), + cliflags.String( + &cmd.StateLayout, + names.StateLayout, + "", + "Diagnostics state-layout override", + ), + ) + _ = cobraCmd.Flags().MarkHidden(names.StateRoot) + _ = cobraCmd.Flags().MarkHidden(names.StateLayout) + return cobraCmd +} + +func (cmd *DaemonDiagnosticsCmd) Run( + _ context.Context, +) error { + if cmd.Limit < 0 || cmd.Limit > machinediagnostics.MaxReadEvents { + return fmt.Errorf( + "diagnostics limit must be between 0 and %d", + machinediagnostics.MaxReadEvents, + ) + } + response, err := cmd.readResponse() + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(response) +} + +func (cmd *DaemonDiagnosticsCmd) readResponse() (machinediagnostics.ReadResponse, error) { + options := machinediagnostics.ReadOptions{ + After: cmd.After, + Limit: cmd.Limit, + Interval: time.Minute, + Now: time.Now(), + } + if cmd.StateRoot == "" && cmd.StateLayout == "" { + return machinediagnostics.ReadFromLocator( + machinediagnostics.DefaultLocatorPath, + options, + ), nil + } + if cmd.StateRoot == "" || cmd.StateLayout == "" { + return machinediagnostics.ReadResponse{}, fmt.Errorf( + "daemon state root and state layout must be provided together", + ) + } + location := agentdaemon.StateLocation{ + Root: cmd.StateRoot, + Layout: agentdaemon.StateLayout(cmd.StateLayout), + } + if err := location.Validate(); err != nil { + return machinediagnostics.ReadResponse{}, err + } + return machinediagnostics.Read(machinediagnostics.DiagnosticsDir(location.Root), options), nil +} diff --git a/cmd/internal/agent_daemon_test.go b/cmd/internal/agent_daemon_test.go index 75277f0e5b..46c7de8c0a 100644 --- a/cmd/internal/agent_daemon_test.go +++ b/cmd/internal/agent_daemon_test.go @@ -7,8 +7,10 @@ import ( "testing" "time" + "github.com/devsy-org/devsy/cmd/flags" "github.com/devsy-org/devsy/pkg/agent" "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/machinediagnostics" provider2 "github.com/devsy-org/devsy/pkg/provider" "github.com/devsy-org/devsy/pkg/types" "github.com/stretchr/testify/assert" @@ -17,6 +19,20 @@ import ( const testEcho = "echo" +type recordingDiagnostics struct { + events []machinediagnostics.Event + statuses []machinediagnostics.Status +} + +func (r *recordingDiagnostics) Record(event machinediagnostics.Event) { + r.events = append(r.events, event) +} + +func (r *recordingDiagnostics) Update(status machinediagnostics.Status) { + r.statuses = append(r.statuses, status) +} +func (*recordingDiagnostics) Close() error { return nil } + func writeWorkspaceConfig(t *testing.T, dir string, shutdown types.StrArray) string { t.Helper() require.NoError(t, os.MkdirAll(dir, 0o750)) @@ -155,3 +171,287 @@ func TestEffectiveActivity(t *testing.T) { activityFilePath = touch("stale.activity", time.Now().Add(-2*time.Hour).Truncate(time.Second)) assert.Equal(t, configActivity, effectiveActivity(configActivity)) } + +func TestDaemonStateLocation(t *testing.T) { + t.Run("uses explicit canonical state", func(t *testing.T) { + cmd := &DaemonCmd{StateRoot: "/state", StateLayout: "canonical"} + location, err := cmd.stateLocation() + require.NoError(t, err) + assert.Equal(t, "/state", location.Root) + assert.Equal(t, "canonical", string(location.Layout)) + }) + + t.Run("supports legacy agent directory", func(t *testing.T) { + cmd := &DaemonCmd{GlobalFlags: &flags.GlobalFlags{}} + cmd.AgentDir = "/state/agent" + location, err := cmd.stateLocation() + require.NoError(t, err) + assert.Equal(t, "/state/agent", location.Root) + assert.Equal(t, "agent-home", string(location.Layout)) + }) + + t.Run("rejects incomplete state location", func(t *testing.T) { + _, err := (&DaemonCmd{StateRoot: "/state"}).stateLocation() + require.ErrorContains(t, err, "provided together") + }) +} + +func TestDaemonWorkspaceConfigsSupportsBothLayouts(t *testing.T) { + root := t.TempDir() + canonical := writeWorkspaceConfig( + t, + filepath.Join(root, "contexts", "default", "workspaces", "canonical", "agent"), + types.StrArray{testEcho}, + ) + agentHome := writeWorkspaceConfig( + t, + filepath.Join(root, "contexts", "default", "workspaces", "agent-home"), + types.StrArray{testEcho}, + ) + + t.Run("canonical", func(t *testing.T) { + base, configs, err := (&DaemonCmd{StateRoot: root, StateLayout: "canonical"}).workspaceConfigs() + require.NoError(t, err) + assert.Equal(t, root, base) + assert.Equal(t, []string{canonical}, configs) + }) + + t.Run("agent home", func(t *testing.T) { + base, configs, err := (&DaemonCmd{StateRoot: root, StateLayout: "agent-home"}).workspaceConfigs() + require.NoError(t, err) + assert.Equal(t, root, base) + assert.Equal(t, []string{agentHome}, configs) + }) +} + +func TestDiagnosticsWorkspaceStatuses(t *testing.T) { + root := t.TempDir() + active := writeWorkspaceConfig( + t, + filepath.Join(root, "contexts", "default", "workspaces", "active", "agent"), + types.StrArray{testEcho}, + ) + notConfigured := writeWorkspaceConfig( + t, + filepath.Join(root, "contexts", "default", "workspaces", "not-configured", "agent"), + nil, + ) + invalid := filepath.Join( + root, + "contexts", + "default", + "workspaces", + "invalid", + "agent", + provider2.WorkspaceConfigFile, + ) + require.NoError(t, os.MkdirAll(filepath.Dir(invalid), 0o750)) + require.NoError(t, os.WriteFile(invalid, []byte("invalid"), 0o600)) + + statuses := diagnosticsWorkspaceStatuses([]string{active, notConfigured, invalid}) + require.Len(t, statuses, 3) + assert.Equal(t, machinediagnostics.WorkspaceActive, statuses[0].State) + assert.Equal(t, machinediagnostics.WorkspaceNotConfigured, statuses[1].State) + assert.Equal(t, "invalid", statuses[2].ID) + assert.Equal(t, machinediagnostics.WorkspaceInvalidConfig, statuses[2].State) +} + +func TestRecordWorkspaceTransitions(t *testing.T) { + recorder := &recordingDiagnostics{} + cmd := &DaemonCmd{recorder: recorder} + cmd.recordWorkspaceTransitions([]machinediagnostics.WorkspaceStatus{{ID: "one"}, {ID: "two"}}) + cmd.recordWorkspaceTransitions([]machinediagnostics.WorkspaceStatus{{ID: "two"}, {ID: "three"}}) + require.Len(t, recorder.events, 4) + assert.Equal(t, machinediagnostics.EventWorkspaceDiscovered, recorder.events[0].Type) + assert.Equal(t, "one", recorder.events[0].WorkspaceID) + assert.Equal(t, machinediagnostics.EventWorkspaceDiscovered, recorder.events[2].Type) + assert.Equal(t, "three", recorder.events[2].WorkspaceID) + assert.Equal(t, machinediagnostics.EventWorkspaceRemoved, recorder.events[3].Type) + assert.Equal(t, "one", recorder.events[3].WorkspaceID) +} + +func TestRecordWorkspaceTransitionsReportsInvalidConfigOnce(t *testing.T) { + recorder := &recordingDiagnostics{} + cmd := &DaemonCmd{recorder: recorder} + invalid := []machinediagnostics.WorkspaceStatus{ + {ID: "broken", State: machinediagnostics.WorkspaceInvalidConfig}, + } + cmd.recordWorkspaceTransitions(invalid) + cmd.recordWorkspaceTransitions(invalid) + require.Len(t, recorder.events, 2) + assert.Equal(t, machinediagnostics.EventWorkspaceDiscovered, recorder.events[0].Type) + assert.Equal(t, machinediagnostics.EventWorkspaceConfigErr, recorder.events[1].Type) +} + +func TestUpdateDiagnosticsPreservesLastSuccessfulPatrolAfterFailure(t *testing.T) { + recorder := &recordingDiagnostics{} + cmd := &DaemonCmd{recorder: recorder, startedAt: time.Now().UTC(), Interval: "1m"} + cmd.updateDiagnostics( + diagnosticsUpdate{ + state: machinediagnostics.DaemonRunning, + health: machinediagnostics.DaemonHealthy, + }, + ) + cmd.updateDiagnostics( + diagnosticsUpdate{ + state: machinediagnostics.DaemonRunning, + health: machinediagnostics.DaemonDegraded, + diagnosticErr: &machinediagnostics.DiagnosticError{Code: "patrol_failed"}, + }, + ) + require.Len(t, recorder.statuses, 2) + require.NotNil(t, recorder.statuses[0].LastSuccessAt) + require.NotNil(t, recorder.statuses[1].LastSuccessAt) + assert.Equal(t, *recorder.statuses[0].LastSuccessAt, *recorder.statuses[1].LastSuccessAt) +} + +func TestDiagnosticsRetainsErrorHistoryWithoutInventingStartupPatrol(t *testing.T) { + recorder := &recordingDiagnostics{} + cmd := &DaemonCmd{recorder: recorder} + cmd.updateDiagnostics( + diagnosticsUpdate{ + state: machinediagnostics.DaemonStarting, + health: machinediagnostics.DaemonHealthy, + }, + ) + assert.Nil(t, recorder.statuses[0].LastPatrolAt) + cmd.updateDiagnostics( + diagnosticsUpdate{ + state: machinediagnostics.DaemonRunning, + health: machinediagnostics.DaemonDegraded, + diagnosticErr: &machinediagnostics.DiagnosticError{Code: "shutdown_failed"}, + }, + ) + cmd.updateDiagnostics( + diagnosticsUpdate{ + state: machinediagnostics.DaemonRunning, + health: machinediagnostics.DaemonHealthy, + }, + ) + assert.Equal(t, "shutdown_failed", recorder.statuses[2].LastError.Code) + assert.Equal(t, machinediagnostics.DaemonHealthy, recorder.statuses[2].Health) +} + +func TestUpdateDiagnosticsRetainsWorkspaceContextAfterShutdownFailure(t *testing.T) { + recorder := &recordingDiagnostics{} + cmd := &DaemonCmd{recorder: recorder, startedAt: time.Now().UTC()} + workspaces := []machinediagnostics.WorkspaceStatus{{ID: "ws-test"}} + candidate := &machinediagnostics.ShutdownCandidate{WorkspaceID: "ws-test"} + + cmd.updateDiagnostics(diagnosticsUpdate{ + state: machinediagnostics.DaemonRunning, + health: machinediagnostics.DaemonDegraded, + workspaces: workspaces, + candidate: candidate, + diagnosticErr: &machinediagnostics.DiagnosticError{ + Code: "shutdown_failed", + }, + }) + + require.Len(t, recorder.statuses, 1) + assert.Equal(t, workspaces, recorder.statuses[0].Workspaces) + assert.Equal(t, candidate, recorder.statuses[0].ShutdownCandidate) + assert.Equal(t, 1, recorder.statuses[0].WorkspaceCount) +} + +func TestInvalidTimeoutBlocksShutdown(t *testing.T) { + for _, timeout := range []string{"invalid", "0s", "-1m"} { + t.Run(timeout, func(t *testing.T) { + path := writeWorkspaceConfig(t, t.TempDir(), types.StrArray{testEcho}) + info, err := agent.ParseAgentWorkspaceInfo(path) + require.NoError(t, err) + info.Agent.Timeout = timeout + data, err := json.Marshal(info) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, data, 0o600)) + evaluation := evaluateMachineInactivity( + []string{path}, + time.Time{}, + config.ShutdownActionStopContainer, + time.Now().Add(24*time.Hour), + ) + assert.Nil(t, evaluation.candidate) + require.Len(t, evaluation.statuses, 1) + assert.Equal(t, machinediagnostics.WorkspaceInvalidConfig, evaluation.statuses[0].State) + assert.True(t, evaluation.statuses[0].BlocksMachineShutdown) + }) + } +} + +func TestEvaluateMachineInactivityRequiresAllWorkspacesToBeIdle(t *testing.T) { + root := t.TempDir() + first := writeWorkspaceConfig(t, filepath.Join(root, "one"), types.StrArray{testEcho}) + second := writeWorkspaceConfig(t, filepath.Join(root, "two"), types.StrArray{testEcho}) + now := time.Now().UTC().Truncate(time.Second) + old := now.Add(-2 * agent.DefaultInactivityTimeout) + require.NoError(t, os.Chtimes(first, old, old)) + require.NoError(t, os.Chtimes(second, old, old)) + + evaluation := evaluateMachineInactivity( + []string{first, second}, + time.Time{}, + config.ShutdownActionStopContainer, + now, + ) + require.NotNil(t, evaluation.workspace) + require.NotNil(t, evaluation.candidate) + assert.Equal(t, "all workspaces are idle", evaluation.reason) + + require.NoError(t, os.Chtimes(second, now, now)) + evaluation = evaluateMachineInactivity( + []string{first, second}, + time.Time{}, + config.ShutdownActionStopContainer, + now, + ) + assert.Nil(t, evaluation.workspace) + assert.Equal(t, "Waiting for inactivity deadline", evaluation.reason) + assert.Len(t, evaluation.statuses, 2) +} + +func TestEvaluateMachineInactivityUsesHeartbeatAndBusyGrace(t *testing.T) { + configPath := writeWorkspaceConfig(t, t.TempDir(), types.StrArray{testEcho}) + now := time.Now().UTC().Truncate(time.Second) + old := now.Add(-2 * agent.DefaultInactivityTimeout) + require.NoError(t, os.Chtimes(configPath, old, old)) + heartbeat := now.Add(-time.Minute) + evaluation := evaluateMachineInactivity( + []string{configPath}, + heartbeat, + config.ShutdownActionStopContainer, + now, + ) + require.Len(t, evaluation.statuses, 1) + assert.Equal(t, machinediagnostics.WorkspaceActive, evaluation.statuses[0].State) + assert.Equal(t, heartbeat, *evaluation.statuses[0].LastActivityAt) + + agent.CreateWorkspaceBusyFile(filepath.Dir(configPath)) + evaluation = evaluateMachineInactivity( + []string{configPath}, + time.Time{}, + config.ShutdownActionStopContainer, + now, + ) + assert.Equal(t, machinediagnostics.WorkspaceBusy, evaluation.statuses[0].State) + require.NotNil(t, evaluation.statuses[0].IdleDeadlineAt) + assert.WithinDuration( + t, + old.Add(busyGracePeriod).Add(agent.DefaultInactivityTimeout), + *evaluation.statuses[0].IdleDeadlineAt, + time.Second, + ) +} + +func TestEvaluateMachineInactivityBlocksDisabledWorkspace(t *testing.T) { + configPath := writeWorkspaceConfig(t, t.TempDir(), types.StrArray{testEcho}) + evaluation := evaluateMachineInactivity( + []string{configPath}, + time.Time{}, + config.ShutdownActionNone, + time.Now(), + ) + assert.Nil(t, evaluation.workspace) + require.Len(t, evaluation.statuses, 1) + assert.True(t, evaluation.statuses[0].BlocksMachineShutdown) + assert.Equal(t, "Auto-stop is not configured", evaluation.statuses[0].BlockerReason) +} diff --git a/cmd/internal/agentworkspace/logs_daemon.go b/cmd/internal/agentworkspace/logs_daemon.go index 2d757b5669..41fec4530b 100644 --- a/cmd/internal/agentworkspace/logs_daemon.go +++ b/cmd/internal/agentworkspace/logs_daemon.go @@ -2,15 +2,14 @@ package agentworkspace import ( "context" - "fmt" - "io" + "encoding/json" "os" - "path/filepath" + "time" "github.com/devsy-org/devsy/cmd/flags" - "github.com/devsy-org/devsy/pkg/agent" cliflags "github.com/devsy-org/devsy/pkg/flags" "github.com/devsy-org/devsy/pkg/flags/names" + "github.com/devsy-org/devsy/pkg/machinediagnostics" "github.com/spf13/cobra" ) @@ -40,39 +39,12 @@ func NewLogsDaemonCmd(flags *flags.GlobalFlags) *cobra.Command { } func (cmd *LogsDaemonCmd) Run(ctx context.Context) error { - // `agent workspace logs-daemon` reads agent-daemon.log, which only - // exists inside the workspace container or machine. Reject host - // invocations explicitly to surface misconfigurations early. - if agent.IsHostAgentInvocation(cmd.AgentDir) { - return fmt.Errorf( - "`devsy internal agent workspace logs-daemon` is only valid inside the workspace container or machine", - ) - } - - // get workspace - shouldExit, _, err := agent.ReadAgentWorkspaceInfo( - cmd.AgentDir, - cmd.Context, - cmd.ID, + _ = ctx + response := machinediagnostics.ReadFromLocator( + machinediagnostics.DefaultLocatorPath, + machinediagnostics.ReadOptions{ + Limit: machinediagnostics.DefaultReadEvents, Now: time.Now(), + }, ) - if err != nil { - return err - } else if shouldExit { - return nil - } - - logDir, err := agent.GetAgentDaemonLogDir(cmd.AgentDir) - if err != nil { - return err - } - - // #nosec G304 -- reads the agent's own daemon log at a derived path. - f, err := os.Open(filepath.Join(logDir, "agent-daemon.log")) - if err != nil { - return fmt.Errorf("open agent-daemon.log: %w", err) - } - defer func() { _ = f.Close() }() - - _, err = io.Copy(os.Stdout, f) - return err + return json.NewEncoder(os.Stdout).Encode(response) } diff --git a/cmd/internal/agentworkspace/up.go b/cmd/internal/agentworkspace/up.go index 204cbb0f68..24fef59902 100644 --- a/cmd/internal/agentworkspace/up.go +++ b/cmd/internal/agentworkspace/up.go @@ -767,11 +767,21 @@ func installDaemon(workspaceInfo *provider.AgentWorkspaceInfo) error { } log.Debugf("installing Devsy daemon into server") - return agentdaemon.InstallDaemon( - workspaceInfo.Agent.DataPath, - workspaceInfo.CLIOptions.DaemonInterval, - shutdownAction, - ) + location, err := agentdaemon.ResolveStateLocation(agentdaemon.ResolveStateLocationOptions{ + AgentDataPath: workspaceInfo.Agent.DataPath, + Origin: workspaceInfo.Origin, + Context: workspaceInfo.Workspace.Context, + WorkspaceID: workspaceInfo.Workspace.ID, + }) + if err != nil { + return fmt.Errorf("resolve daemon state location: %w", err) + } + return agentdaemon.InstallDaemon(agentdaemon.InstallOptions{ + StateLocation: location, + Interval: workspaceInfo.CLIOptions.DaemonInterval, + ShutdownAction: shutdownAction, + DiagnosticsReader: agentdaemon.DiagnosticsReaderIdentity(), + }) } func persistResolvedConfig( diff --git a/cmd/internal/logs_daemon.go b/cmd/internal/logs_daemon.go index d03ad3efe9..c85d4c8d4d 100644 --- a/cmd/internal/logs_daemon.go +++ b/cmd/internal/logs_daemon.go @@ -1,13 +1,17 @@ package cmdinternal import ( + "bytes" "context" + "encoding/json" "fmt" "os" + "time" "github.com/devsy-org/devsy/cmd/flags" "github.com/devsy-org/devsy/pkg/client" "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/machinediagnostics" provider2 "github.com/devsy-org/devsy/pkg/provider" "github.com/devsy-org/devsy/pkg/workspace" "github.com/spf13/cobra" @@ -74,10 +78,31 @@ func (cmd *LogsDaemonCmd) Run(ctx context.Context, args []string) error { command += fmt.Sprintf(" --agent-dir %q", agentInfo.Agent.DataPath) } - // read daemon logs - return workspaceClient.Command(ctx, client.CommandOptions{ + var stdout bytes.Buffer + if err := workspaceClient.Command(ctx, client.CommandOptions{ Command: command, - Stdout: os.Stdout, + Stdout: &stdout, Stderr: os.Stderr, - }) + }); err != nil { + return err + } + + var response machinediagnostics.ReadResponse + if err := json.Unmarshal(stdout.Bytes(), &response); err != nil { + return fmt.Errorf("decode remote daemon diagnostics: %w", err) + } + for _, event := range response.Events { + _, _ = fmt.Fprintf( + os.Stdout, + "%s %-5s %-35s %s\n", + event.Timestamp.Format(time.RFC3339), + event.Level, + event.Type, + event.Message, + ) + } + if response.Availability != machinediagnostics.AvailabilityAvailable { + _, _ = fmt.Fprintf(os.Stdout, "Daemon diagnostics: %s\n", response.Availability) + } + return nil } diff --git a/cmd/machine/diagnostics.go b/cmd/machine/diagnostics.go new file mode 100644 index 0000000000..e8a6da2e9e --- /dev/null +++ b/cmd/machine/diagnostics.go @@ -0,0 +1,390 @@ +package machine + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "strconv" + "strings" + "text/tabwriter" + "time" + + "al.essio.dev/pkg/shellescape" + "github.com/devsy-org/devsy/cmd/flags" + "github.com/devsy-org/devsy/pkg/client" + "github.com/devsy-org/devsy/pkg/config" + cliflags "github.com/devsy-org/devsy/pkg/flags" + "github.com/devsy-org/devsy/pkg/machinediagnostics" + "github.com/devsy-org/devsy/pkg/output" + "github.com/devsy-org/devsy/pkg/workspace" + "github.com/spf13/cobra" +) + +const ( + maxRemoteDiagnosticsResponse = 2 * 1024 * 1024 + unknownFreshness = "unknown" + unavailableAvailability = "unavailable" +) + +type boundedDiagnosticsBuffer struct { + bytes.Buffer + max int +} + +func (b *boundedDiagnosticsBuffer) Write(value []byte) (int, error) { + if b.Len()+len(value) > b.max { + return 0, fmt.Errorf("remote diagnostics response exceeds %d bytes", b.max) + } + return b.Buffer.Write(value) +} + +type CollectionSource struct { + Availability string `json:"availability"` + Freshness string `json:"freshness"` + ErrorCode string `json:"errorCode,omitempty"` + Message string `json:"message,omitempty"` +} + +type MachineInfo struct { + ID string `json:"id"` + Context string `json:"context"` + Provider string `json:"provider"` + State string `json:"state"` +} + +type MachineDiagnostics struct { + SchemaVersion int `json:"schemaVersion"` + Machine MachineInfo `json:"machine"` + Source CollectionSource `json:"source"` + Daemon *machinediagnostics.Status `json:"daemon,omitempty"` + Events []machinediagnostics.Event `json:"events,omitempty"` + Cursor machinediagnostics.CursorInfo `json:"cursor"` +} + +type DiagnosticsCmd struct { + *flags.GlobalFlags + After string + Limit int + NoEvents bool +} + +func NewDiagnosticsCmd(globalFlags *flags.GlobalFlags) *cobra.Command { + cmd := &DiagnosticsCmd{GlobalFlags: globalFlags} + c := &cobra.Command{ + Use: "diagnostics [name]", + Short: "Show Devsy machine diagnostic events and daemon status", + RunE: func(c *cobra.Command, args []string) error { return cmd.Run(c.Context(), args) }, + } + cliflags.Add( + c, + cliflags.String(&cmd.After, "after", "", "An opaque diagnostics cursor"), + cliflags.Int(&cmd.Limit, "limit", 20, "Maximum diagnostic events"), + cliflags.Bool(&cmd.NoEvents, "no-events", false, "Do not include diagnostic events"), + ) + return c +} + +func (cmd *DiagnosticsCmd) Run( + ctx context.Context, + args []string, +) error { + if cmd.Limit < 0 || cmd.Limit > machinediagnostics.MaxReadEvents { + return fmt.Errorf( + "diagnostics limit must be between 0 and %d", + machinediagnostics.MaxReadEvents, + ) + } + if err := machinediagnostics.ValidateCursor(cmd.After); err != nil { + return fmt.Errorf("invalid diagnostics cursor: %w", err) + } + cfg, err := config.LoadConfig(cmd.Context, cmd.Provider) + if err != nil { + return err + } + mc, err := workspace.GetMachine(cfg, args) + if err != nil { + return err + } + result, err := fetchDiagnostics(ctx, mc, diagnosticsFetchOptions{ + After: cmd.After, + Limit: cmd.Limit, + IncludeEvents: !cmd.NoEvents, + }) + if err != nil { + return err + } + return renderDiagnostics(result, cmd.ResultFormat) +} + +type diagnosticsFetchOptions struct { + After string + Limit int + IncludeEvents bool +} + +func fetchDiagnostics( + ctx context.Context, + mc client.MachineClient, + opts diagnosticsFetchOptions, +) (MachineDiagnostics, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + result := MachineDiagnostics{ + SchemaVersion: machinediagnostics.SchemaVersion, + Cursor: machinediagnostics.CursorInfo{State: machinediagnostics.CursorNone}, + } + result.Machine = MachineInfo{ + ID: mc.Machine(), + Context: mc.Context(), + Provider: mc.Provider(), + } + status, err := mc.Status(ctx, client.StatusOptions{}) + if err != nil { + if ctx.Err() != nil && ctx.Err() != context.DeadlineExceeded { + return result, ctx.Err() + } + result.Machine.State = unknownFreshness + result.Source = CollectionSource{ + Availability: unavailableAvailability, + Freshness: unknownFreshness, + ErrorCode: "machine_status_unavailable", + Message: "Devsy could not read the provider's machine state. Try again when the provider " + + "connection is available.", + } + return result, nil + } + result.Machine.State = string(status) + if !strings.EqualFold(string(status), string(client.StatusRunning)) { + if status == client.StatusStopped { + result.Source = CollectionSource{ + Availability: "machine_stopped", + Freshness: unknownFreshness, + } + } else { + result.Source = CollectionSource{ + Availability: unavailableAvailability, + Freshness: unknownFreshness, + ErrorCode: "machine_not_running", + Message: "The machine is not running.", + } + } + return result, nil + } + return fetchRemoteDiagnostics(ctx, mc, opts, result) +} + +func fetchRemoteDiagnostics( + ctx context.Context, + mc client.MachineClient, + opts diagnosticsFetchOptions, + result MachineDiagnostics, +) (MachineDiagnostics, error) { + command := shellescape.Quote( + mc.AgentPath(), + ) + " internal agent daemon-diagnostics --limit " + strconv.Itoa( + opts.Limit, + ) + if opts.After != "" { + command += " --after " + shellescape.Quote(opts.After) + } + stdout := boundedDiagnosticsBuffer{max: maxRemoteDiagnosticsResponse} + stderr := boundedDiagnosticsBuffer{max: maxRemoteDiagnosticsResponse} + err := mc.Command( + ctx, + client.CommandOptions{Command: command, Stdout: &stdout, Stderr: &stderr}, + ) + if err != nil { + result.Source = CollectionSource{ + Availability: unavailableAvailability, + Freshness: unknownFreshness, + ErrorCode: "remote_diagnostics_command_failed", + Message: "Devsy could not read remote diagnostics.", + } + return result, nil + } + var remote machinediagnostics.ReadResponse + if err := json.Unmarshal(stdout.Bytes(), &remote); err != nil { + result.Source = CollectionSource{ + Availability: unavailableAvailability, + Freshness: unknownFreshness, + ErrorCode: "remote_diagnostics_command_failed", + Message: "Remote diagnostics returned an invalid response.", + } + return result, nil + } + if remote.SchemaVersion != machinediagnostics.SchemaVersion || remote.Availability == "" || + remote.Freshness == "" { + result.Source = CollectionSource{ + Availability: unavailableAvailability, + Freshness: unknownFreshness, + ErrorCode: "invalid_diagnostics_response", + Message: "Remote diagnostics returned an unsupported or incomplete response.", + } + return result, nil + } + return applyRemoteDiagnostics(result, remote, opts.IncludeEvents), nil +} + +func applyRemoteDiagnostics( + result MachineDiagnostics, + remote machinediagnostics.ReadResponse, + includeEvents bool, +) MachineDiagnostics { + result.Source = CollectionSource{ + Availability: string(remote.Availability), + Freshness: string(remote.Freshness), + } + if remote.Error != nil { + result.Source.ErrorCode = remote.Error.Code + result.Source.Message = remote.Error.Message + } + result.Daemon = remote.Status + if includeEvents { + result.Events = remote.Events + } + result.Cursor = remote.Cursor + return result +} + +func renderDiagnostics(result MachineDiagnostics, format string) error { + mode, err := output.ResolveMode(format) + if err != nil { + return err + } + if mode == output.ModeJSON { + return json.NewEncoder(os.Stdout).Encode(result) + } + return renderDiagnosticsText(os.Stdout, result) +} + +func renderDiagnosticsText(w io.Writer, result MachineDiagnostics) error { + _, _ = fmt.Fprintf( + w, + "Machine %s\nProvider state %s\nDiagnostics %s\n", + result.Machine.ID, + result.Machine.State, + result.Source.Availability, + ) + if result.Source.Message != "" { + _, _ = fmt.Fprintf(w, "Diagnostic detail %s\n", result.Source.Message) + } + if result.Daemon != nil { //nolint:nestif // daemon details are intentionally rendered as one group. + lastPatrol := "Not yet observed" + if result.Daemon.LastPatrolAt != nil { + lastPatrol = result.Daemon.LastPatrolAt.Local().Format("2006-01-02 15:04:05") + } + _, _ = fmt.Fprintf( + w, + "Devsy daemon %s\nSnapshot freshness %s\nLast patrol %s\nWorkspaces %d\n", + result.Daemon.Health, + result.Source.Freshness, + lastPatrol, + result.Daemon.WorkspaceCount, + ) + if result.Daemon.LastError != nil { + _, _ = fmt.Fprintf( + w, + "Last daemon error %s (%s, %s)\n", + result.Daemon.LastError.Message, + result.Daemon.LastError.Code, + result.Daemon.LastError.Timestamp.Format(time.RFC3339), + ) + } + if result.Daemon.ShutdownCandidate != nil { + eligible := "now" + if result.Daemon.ShutdownCandidate.EligibleAt != nil { + eligible = result.Daemon.ShutdownCandidate.EligibleAt.Local(). + Format("2006-01-02 15:04:05") + } + _, _ = fmt.Fprintf( + w, + "Shutdown candidate %s (eligible %s)\n", + result.Daemon.ShutdownCandidate.WorkspaceID, + eligible, + ) + } + renderWorkspaceDiagnostics(w, result.Daemon.Workspaces) + } + for _, event := range result.Events { + _, _ = fmt.Fprintf( + w, + "%s %-5s %s %s\n", + event.Timestamp.Format(time.RFC3339), + event.Level, + event.Type, + event.Message, + ) + } + return nil +} + +func renderWorkspaceDiagnostics(w io.Writer, workspaces []machinediagnostics.WorkspaceStatus) { + if len(workspaces) == 0 { + return + } + _, _ = fmt.Fprintln(w, "\nWorkspace inactivity:") + tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + _, _ = fmt.Fprintln(tw, "WORKSPACE\tSTATE\tLAST ACTIVITY\tMACHINE SHUTDOWN STATUS") + for _, workspace := range workspaces { + lastActivity := "-" + if workspace.LastActivityAt != nil { + lastActivity = workspace.LastActivityAt.Local().Format("2006-01-02 15:04:05") + } + _, _ = fmt.Fprintf( + tw, + "%s\t%s\t%s\t%s\n", + workspace.ID, + workspace.State, + lastActivity, + workspaceAutoStopDetail(workspace), + ) + } + _ = tw.Flush() +} + +func workspaceAutoStopDetail( + workspace machinediagnostics.WorkspaceStatus, +) string { + if workspace.BlocksMachineShutdown && workspace.BlockerReason != "" { + return workspaceBlockerDetail(workspace) + } + if workspace.IdleDeadlineAt != nil { + return workspaceDeadlineDetail(workspace) + } + return workspaceStateDetail(workspace.State) +} + +func workspaceBlockerDetail(workspace machinediagnostics.WorkspaceStatus) string { + if workspace.State == machinediagnostics.WorkspaceActive && workspace.IdleDeadlineAt != nil { + return workspace.BlockerReason + " (" + workspace.IdleDeadlineAt.Local(). + Format("2006-01-02 15:04:05") + + ")" + } + return workspace.BlockerReason +} + +func workspaceDeadlineDetail(workspace machinediagnostics.WorkspaceStatus) string { + deadline := workspace.IdleDeadlineAt.Local().Format("2006-01-02 15:04:05") + if workspace.State == machinediagnostics.WorkspaceIdleDue { + return "Eligible now (" + deadline + ")" + } + return deadline +} + +func workspaceStateDetail(state machinediagnostics.WorkspaceEvaluationState) string { + switch state { + case machinediagnostics.WorkspaceBusy: + return "Delayed while workspace is busy" + case machinediagnostics.WorkspaceNotConfigured: + return "Auto-stop is not configured" + case machinediagnostics.WorkspaceInvalidConfig: + return "Workspace configuration is invalid" + case machinediagnostics.WorkspaceNotRunning: + return "Workspace state is unavailable" + default: + return "-" + } +} diff --git a/cmd/machine/diagnostics_test.go b/cmd/machine/diagnostics_test.go new file mode 100644 index 0000000000..935682c318 --- /dev/null +++ b/cmd/machine/diagnostics_test.go @@ -0,0 +1,170 @@ +package machine + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/client" + "github.com/devsy-org/devsy/pkg/machinediagnostics" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeMachineClient struct { + client.MachineClient + status client.Status + statusErr error + commandErr error + command string + commandCalls int + response string +} + +func (f *fakeMachineClient) Machine() string { return "machine-1" } +func (f *fakeMachineClient) Context() string { return "default" } +func (f *fakeMachineClient) Provider() string { return "fake" } +func (f *fakeMachineClient) AgentPath() string { return "/usr/local/bin/devsy" } +func (f *fakeMachineClient) Status(context.Context, client.StatusOptions) (client.Status, error) { + return f.status, f.statusErr +} + +func (f *fakeMachineClient) Command(_ context.Context, options client.CommandOptions) error { + f.commandCalls++ + f.command = options.Command + if f.commandErr != nil { + return f.commandErr + } + _, _ = options.Stdout.Write([]byte(f.response)) + return nil +} + +func TestBoundedDiagnosticsBuffer(t *testing.T) { + buffer := boundedDiagnosticsBuffer{max: 4} + _, err := buffer.Write([]byte("test")) + require.NoError(t, err) + _, err = buffer.Write([]byte("x")) + require.ErrorContains(t, err, "exceeds") + assert.Equal(t, "test", buffer.String()) +} + +func TestFetchDiagnosticsSkipsRemoteCommandForStoppedMachine(t *testing.T) { + machine := &fakeMachineClient{status: client.StatusStopped} + result, err := fetchDiagnostics( + context.Background(), + machine, + diagnosticsFetchOptions{Limit: 20, IncludeEvents: true}, + ) + require.NoError(t, err) + assert.Equal(t, "machine_stopped", result.Source.Availability) + assert.Zero(t, machine.commandCalls) +} + +func TestFetchDiagnosticsReadsRemoteResponse(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + response, err := json.Marshal(machinediagnostics.ReadResponse{ + SchemaVersion: machinediagnostics.SchemaVersion, + Availability: machinediagnostics.AvailabilityAvailable, + Freshness: machinediagnostics.FreshnessFresh, + Status: &machinediagnostics.Status{Health: machinediagnostics.DaemonHealthy}, + Events: []machinediagnostics.Event{ + { + SessionID: "session", + Sequence: 1, + Timestamp: now, + Type: machinediagnostics.EventDaemonReady, + Message: "ready", + }, + }, + Cursor: machinediagnostics.CursorInfo{ + State: machinediagnostics.CursorOK, + Next: machinediagnostics.EncodeCursor("session", 1), + }, + }) + require.NoError(t, err) + machine := &fakeMachineClient{status: client.StatusRunning, response: string(response)} + result, err := fetchDiagnostics( + context.Background(), + machine, + diagnosticsFetchOptions{Limit: 20, IncludeEvents: true}, + ) + require.NoError(t, err) + assert.Equal(t, "available", result.Source.Availability) + assert.Equal(t, machinediagnostics.DaemonHealthy, result.Daemon.Health) + assert.Len(t, result.Events, 1) + assert.Contains(t, machine.command, "daemon-diagnostics --limit 20") +} + +func TestFetchDiagnosticsRetainsAvailabilityWhenRemoteCommandFails(t *testing.T) { + machine := &fakeMachineClient{status: client.StatusRunning, commandErr: errors.New("ssh lost")} + result, err := fetchDiagnostics( + context.Background(), + machine, + diagnosticsFetchOptions{Limit: 20, IncludeEvents: true}, + ) + require.NoError(t, err) + assert.Equal(t, "unavailable", result.Source.Availability) + assert.Equal(t, "remote_diagnostics_command_failed", result.Source.ErrorCode) +} + +func TestFetchDiagnosticsRejectsIncompleteResponse(t *testing.T) { + machine := &fakeMachineClient{status: client.StatusRunning, response: `{}`} + result, err := fetchDiagnostics( + context.Background(), + machine, + diagnosticsFetchOptions{Limit: 20, IncludeEvents: true}, + ) + require.NoError(t, err) + assert.Equal(t, "unavailable", result.Source.Availability) + assert.Equal(t, "invalid_diagnostics_response", result.Source.ErrorCode) +} + +func TestFetchDiagnosticsReportsProviderFailureWithoutEndingCollection(t *testing.T) { + machine := &fakeMachineClient{statusErr: errors.New("provider connection lost")} + result, err := fetchDiagnostics( + context.Background(), + machine, + diagnosticsFetchOptions{Limit: 20, IncludeEvents: true}, + ) + require.NoError(t, err) + assert.Equal(t, "unknown", result.Machine.State) + assert.Equal(t, "machine_status_unavailable", result.Source.ErrorCode) + assert.Zero(t, machine.commandCalls) +} + +func TestRenderDiagnosticsTextIncludesWorkspaceInactivityDetails(t *testing.T) { + now := time.Date(2026, 9, 19, 1, 30, 0, 0, time.Local) + deadline := now.Add(30 * time.Minute) + var output bytes.Buffer + renderWorkspaceDiagnostics(&output, []machinediagnostics.WorkspaceStatus{ + { + ID: "api", + State: machinediagnostics.WorkspaceActive, + LastActivityAt: &now, + IdleDeadlineAt: &deadline, + }, + {ID: "docs", State: machinediagnostics.WorkspaceBusy}, + {ID: "preview", State: machinediagnostics.WorkspaceNotConfigured}, + {ID: "broken", State: machinediagnostics.WorkspaceInvalidConfig}, + }) + text := output.String() + assert.Contains(t, text, "Workspace inactivity:") + assert.Contains(t, text, "MACHINE SHUTDOWN STATUS") + assert.Contains(t, text, "api") + assert.Contains(t, text, deadline.Format("2006-01-02 15:04:05")) + assert.Contains(t, text, "Delayed while workspace is busy") + assert.Contains(t, text, "Auto-stop is not configured") + assert.Contains(t, text, "Workspace configuration is invalid") +} + +func TestWorkspaceAutoStopDetailMarksPastDeadlineEligibleNow(t *testing.T) { + deadline := time.Now().Add(-time.Minute) + detail := workspaceAutoStopDetail(machinediagnostics.WorkspaceStatus{ + State: machinediagnostics.WorkspaceIdleDue, + IdleDeadlineAt: &deadline, + }) + assert.Contains(t, detail, "Eligible now") +} diff --git a/cmd/machine/logs.go b/cmd/machine/logs.go new file mode 100644 index 0000000000..91f1c440c1 --- /dev/null +++ b/cmd/machine/logs.go @@ -0,0 +1,159 @@ +package machine + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/devsy-org/devsy/cmd/flags" + "github.com/devsy-org/devsy/pkg/client" + "github.com/devsy-org/devsy/pkg/config" + cliflags "github.com/devsy-org/devsy/pkg/flags" + "github.com/devsy-org/devsy/pkg/machinediagnostics" + "github.com/devsy-org/devsy/pkg/output" + "github.com/devsy-org/devsy/pkg/workspace" + "github.com/spf13/cobra" +) + +type LogsCmd struct { + *flags.GlobalFlags + After string + Limit int + Follow bool +} + +func NewLogsCmd(globalFlags *flags.GlobalFlags) *cobra.Command { + cmd := &LogsCmd{GlobalFlags: globalFlags} + c := &cobra.Command{ + Use: "logs [name]", + Short: "Show Devsy machine diagnostic events", + RunE: func(c *cobra.Command, args []string) error { return cmd.Run(c.Context(), args) }, + } + cliflags.Add( + c, + cliflags.String(&cmd.After, "after", "", "An opaque diagnostics cursor"), + cliflags.Int( + &cmd.Limit, + "limit", + machinediagnostics.DefaultReadEvents, + "Maximum diagnostic events", + ), + ) + c.Flags(). + BoolVarP(&cmd.Follow, "follow", "f", false, "Poll for new diagnostic events until interrupted") + return c +} + +func (cmd *LogsCmd) Run( + ctx context.Context, + args []string, +) error { + if cmd.Limit < 0 || cmd.Limit > machinediagnostics.MaxReadEvents { + return fmt.Errorf( + "diagnostics limit must be between 0 and %d", + machinediagnostics.MaxReadEvents, + ) + } + if err := machinediagnostics.ValidateCursor(cmd.After); err != nil { + return fmt.Errorf("invalid diagnostics cursor: %w", err) + } + cfg, err := config.LoadConfig(cmd.Context, cmd.Provider) + if err != nil { + return err + } + mc, err := workspace.GetMachine(cfg, args) + if err != nil { + return err + } + mode, err := output.ResolveMode(cmd.ResultFormat) + if err != nil { + return err + } + return cmd.runDiagnosticsLoop(ctx, mc, mode) +} + +func (cmd *LogsCmd) runDiagnosticsLoop( + ctx context.Context, + mc client.MachineClient, + mode string, +) error { + for { + result, err := fetchDiagnostics(ctx, mc, diagnosticsFetchOptions{ + After: cmd.After, + Limit: cmd.Limit, + IncludeEvents: true, + }) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + if err := renderLogResult(result, mode); err != nil { + return err + } + if !cmd.Follow { + return nil + } + cmd.advanceCursor(result.Cursor) + if !waitForNextLogPoll(ctx) { + return nil + } + } +} + +func renderLogResult(result MachineDiagnostics, mode string) error { + if mode == output.ModeJSON { + return json.NewEncoder(os.Stdout).Encode(result) + } + if result.Source.Availability != string(machinediagnostics.AvailabilityAvailable) { + _, _ = fmt.Fprintf( + os.Stderr, + "Diagnostics: %s %s\n", + result.Source.Availability, + result.Source.Message, + ) + } + if result.Cursor.State == machinediagnostics.CursorGap || + result.Cursor.State == machinediagnostics.CursorReset { + _, _ = fmt.Fprintf( + os.Stderr, + "Diagnostic history: %s (%s)\n", + result.Cursor.State, + result.Cursor.Reason, + ) + } + for _, event := range result.Events { + _, _ = fmt.Fprintf( + os.Stdout, + "%s %-5s %-35s %s\n", + event.Timestamp.Format(time.RFC3339), + event.Level, + event.Type, + event.Message, + ) + } + return nil +} + +func (cmd *LogsCmd) advanceCursor(cursor machinediagnostics.CursorInfo) { + if cursor.State == machinediagnostics.CursorReset { + cmd.After = "" + } + if cursor.Next != "" { + cmd.After = cursor.Next + } +} + +func waitForNextLogPoll(ctx context.Context) bool { + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} diff --git a/cmd/machine/machine.go b/cmd/machine/machine.go index 3bebdab710..ecf1ba7967 100644 --- a/cmd/machine/machine.go +++ b/cmd/machine/machine.go @@ -21,5 +21,7 @@ func NewMachineCmd(flags *flags.GlobalFlags) *cobra.Command { machineCmd.AddCommand(NewCreateCmd(flags)) machineCmd.AddCommand(NewInspectCmd(flags)) machineCmd.AddCommand(NewDescribeCmd(flags)) + machineCmd.AddCommand(NewDiagnosticsCmd(flags)) + machineCmd.AddCommand(NewLogsCmd(flags)) return machineCmd } diff --git a/cmd/workspace/delete.go b/cmd/workspace/delete.go index a72d9c3e64..a3f6aec56d 100644 --- a/cmd/workspace/delete.go +++ b/cmd/workspace/delete.go @@ -93,16 +93,16 @@ func (cmd *DeleteCmd) Run(cobraCmd *cobra.Command, args []string) error { ctx, reporter, status.Operation{Phase: status.PhaseDeletingWorkspace}, - func(context.Context) error { + func(ctx context.Context) error { var err error devsyConfig, err = cmd.loadConfig() if err != nil { return err } if len(args) <= 1 { - return cmd.deleteSingle(ctx, devsyConfig, args) + return cmd.deleteSingle(status.WithReporter(ctx, reporter), devsyConfig, args) } - return cmd.deleteMultiple(ctx, devsyConfig, args) + return cmd.deleteMultiple(status.WithReporter(ctx, reporter), devsyConfig, args) }, ) if devsyConfig == nil { diff --git a/cmd/workspace/stop.go b/cmd/workspace/stop.go index 29f0b9c6d0..424615df28 100644 --- a/cmd/workspace/stop.go +++ b/cmd/workspace/stop.go @@ -33,27 +33,7 @@ func NewStopCmd(flags *flags.GlobalFlags) *cobra.Command { Use: "stop [flags] [workspace-path|workspace-name]", Short: "Stop a workspace", RunE: func(cobraCmd *cobra.Command, args []string) error { - ctx := cobraCmd.Context() - devsyConfig, err := config.LoadConfig(cmd.Context, cmd.Provider) - if err != nil { - return err - } - - err = clientimplementation.DecodePlatformOptionsFromEnv(&cmd.Platform) - if err != nil { - return fmt.Errorf("decode platform options: %w", err) - } - - client, err := workspace2.Get(ctx, workspace2.GetOptions{ - DevsyConfig: devsyConfig, - Args: args, - Owner: cmd.Owner, - }) - if err != nil { - return err - } - - return cmd.Run(ctx, devsyConfig, client) + return cmd.runArgs(cobraCmd.Context(), args) }, ValidArgsFunction: func(rootCmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return completion.GetWorkspaceSuggestions( @@ -89,11 +69,48 @@ func (cmd *StopCmd) Run( reporter, status.Operation{Phase: status.PhaseStoppingWorkspace}, func(ctx context.Context) error { - return cmd.run(ctx, devsyConfig, client) + return cmd.run(status.WithReporter(ctx, reporter), devsyConfig, client) }, ) } +func (cmd *StopCmd) runArgs(ctx context.Context, args []string) error { + reporter, err := newWorkspaceStatusReporter( + cmd.ResultFormat, os.Stdout, cmd.Verbosity > 0 || cmd.Debug, + ) + if err != nil { + return err + } + return status.Run(status.WithReporter(ctx, reporter), reporter, + status.Operation{Phase: status.PhaseStoppingWorkspace}, func(ctx context.Context) error { + var devsyConfig *config.Config + var client client2.BaseWorkspaceClient + err := status.RunStep( + ctx, status.PhaseStoppingWorkspace, "Loading workspace", + func(ctx context.Context) error { + var err error + devsyConfig, err = config.LoadConfig(cmd.Context, cmd.Provider) + if err != nil { + return err + } + if err := clientimplementation.DecodePlatformOptionsFromEnv( + &cmd.Platform, + ); err != nil { + return fmt.Errorf("decode platform options: %w", err) + } + client, err = workspace2.Get(ctx, workspace2.GetOptions{ + DevsyConfig: devsyConfig, Args: args, Owner: cmd.Owner, + }) + return err + }, + ) + if err != nil { + return err + } + return cmd.run(ctx, devsyConfig, client) + }) +} + func (cmd *StopCmd) run( ctx context.Context, devsyConfig *config.Config, @@ -101,7 +118,12 @@ func (cmd *StopCmd) run( ) error { // lock workspace if !cmd.Platform.Enabled { - err := client.Lock(ctx) + err := status.RunStep( + ctx, + status.PhaseStoppingWorkspace, + "Waiting for workspace lock", + client.Lock, + ) if err != nil { return err } @@ -109,7 +131,17 @@ func (cmd *StopCmd) run( } // get instance status - instanceStatus, err := client.Status(ctx, client2.StatusOptions{}) + var instanceStatus client2.Status + err := status.RunStep( + ctx, + status.PhaseStoppingWorkspace, + "Checking workspace status", + func(ctx context.Context) error { + var err error + instanceStatus, err = client.Status(ctx, client2.StatusOptions{}) + return err + }, + ) if err != nil { return err } else if instanceStatus != client2.StatusRunning { @@ -117,7 +149,17 @@ func (cmd *StopCmd) run( } // stop if single machine provider - wasStopped, err := cmd.stopSingleMachine(ctx, client, devsyConfig) + var wasStopped bool + err = status.RunStep( + ctx, + status.PhaseStoppingWorkspace, + "Checking shared machine", + func(ctx context.Context) error { + var err error + wasStopped, err = cmd.stopSingleMachine(ctx, client, devsyConfig) + return err + }, + ) if err != nil { return err } else if wasStopped { @@ -126,7 +168,14 @@ func (cmd *StopCmd) run( } // stop environment - err = client.Stop(ctx, client2.StopOptions{}) + err = status.RunStep( + ctx, + status.PhaseStoppingWorkspace, + "Stopping workspace resources", + func(ctx context.Context) error { + return client.Stop(ctx, client2.StopOptions{}) + }, + ) if err != nil { return err } @@ -171,9 +220,16 @@ func (cmd *StopCmd) stopSingleMachine( } // stop the machine - err = machineClient.Stop(ctx, client2.StopOptions{}) + err = status.RunStep( + ctx, + status.PhaseStoppingWorkspace, + "Stopping machine", + func(ctx context.Context) error { + return machineClient.Stop(ctx, client2.StopOptions{}) + }, + ) if err != nil { - return false, fmt.Errorf("delete machine: %w", err) + return false, fmt.Errorf("stop machine: %w", err) } log.Debugf("stopped workspace: workspace=%s", client.Workspace()) diff --git a/desktop/e2e/fixtures/mock-devsy.cjs b/desktop/e2e/fixtures/mock-devsy.cjs index 1b8dfea549..da0084f670 100755 --- a/desktop/e2e/fixtures/mock-devsy.cjs +++ b/desktop/e2e/fixtures/mock-devsy.cjs @@ -283,6 +283,8 @@ function handleUp(args) { return } + workspaceStatus("building_image", "started", "Building workspace") + const complete = () => { workspaceStatus("resolving_config", "started") workspaceStatus("resolving_config", "succeeded") workspaceStatus("building_image", "started") @@ -292,6 +294,9 @@ function handleUp(args) { workspaceStatus("ready", "succeeded") materializeWorkspace(wsId, source, providerFlag, ideFlag, "Running") process.exit(0) + } + if (wsId === "lifecycleprobe") setTimeout(complete, 2000) + else complete() } // Handlers for `workspace task `, backing the up --detach flow above. @@ -455,14 +460,16 @@ function handleTask(args) { function handleStop(args) { const { positional } = parseArgs(args) const wsId = positional[0] - out("Stopping workspace...") - out("Workspace stopped.") - const ws = state.workspaces.find((w) => w.id === wsId) - if (ws) { - ws.status = "Stopped" - saveState(state) + workspaceStatus("stopping_workspace", "started", "Stopping workspace resources") + const complete = () => { + const latest = loadState() + const ws = latest.workspaces.find((w) => w.id === wsId) + if (ws) { ws.status = "Stopped"; saveState(latest) } + workspaceStatus("stopping_workspace", "succeeded") + process.exit(0) } - process.exit(0) + if (wsId === "lifecycleprobe") setTimeout(complete, 2000) + else complete() } function handleStart(args) { @@ -484,7 +491,8 @@ function handleDelete(args) { const { positional } = parseArgs(args) const wsId = positional[0] // *probe suffix delays deletion so lifecycle tests can observe in-flight state. - const deleteMs = /probe$/.test(wsId) ? 1500 : 0 + const deleteMs = wsId === "lifecycleprobe" ? 8000 : /probe$/.test(wsId) ? 1500 : 0 + workspaceStatus("deleting_workspace", "started", "Removing workspace resources") setTimeout(() => { out("Deleting workspace...") out("Workspace deleted.") diff --git a/desktop/e2e/workspace-progress.e2e.ts b/desktop/e2e/workspace-progress.e2e.ts new file mode 100644 index 0000000000..331680077e --- /dev/null +++ b/desktop/e2e/workspace-progress.e2e.ts @@ -0,0 +1,64 @@ +import type { ElectronApplication, Page } from "@playwright/test" +import { expect, test } from "@playwright/test" +import { launchApp, resetMockState } from "./electron-app.js" +let app: ElectronApplication +let page: Page +const id = "lifecycleprobe" +async function invoke(channel: string, args?: Record) { + return page.evaluate( + ({ channel, args }) => window.electronAPI.invoke(channel, args), + { channel, args }, + ) +} +async function finished() { + await expect + .poll( + async () => { + const snapshot = (await invoke("workspace_snapshot")) as { + jobs: Record + } + return snapshot.jobs[id]?.state + }, + { timeout: 30000 }, + ) + .toBe("succeeded") +} +test.beforeAll(async () => { + resetMockState() + ;({ app, page } = await launchApp()) + await page.click('[data-sidebar="sidebar"] a[href="#/workspaces"]') +}) +test.afterAll(async () => { + await app?.close() +}) +test("shares lifecycle progress across actions, navigation and window reload", async () => { + test.setTimeout(120000) + await invoke("workspace_up", { + source: "https://example.com/lifecycleprobe.git", + workspaceId: id, + }) + await finished() + const row = () => page.locator("tr").filter({ hasText: id }) + await expect(row()).toContainText("Running") + for (const [action, label] of [ + ["stop", "Stopping"], + ["rebuild", "Rebuilding"], + ["reset", "Resetting"], + ]) { + await invoke(`workspace_${action}`, { workspaceId: id }) + await expect(row()).toContainText(label) + await finished() + } + await invoke("workspace_up", { source: id }) + await finished() + await invoke("workspace_delete", { workspaceId: id }) + await expect(row()).toContainText("Deleting") + await page.click('[data-sidebar="sidebar"] a[href="#/machines"]') + await page.click('[data-sidebar="sidebar"] a[href="#/workspaces"]') + await expect(row()).toContainText("Deleting") + await page.reload() + await page.locator('[data-sidebar="sidebar"]').waitFor() + await expect(row()).toContainText("Deleting") + await finished() + await expect(row()).toHaveCount(0) +}) diff --git a/desktop/src/main/__tests__/ipc-up-tasks.test.ts b/desktop/src/main/__tests__/ipc-up-tasks.test.ts index 37eed13943..3900e494a1 100644 --- a/desktop/src/main/__tests__/ipc-up-tasks.test.ts +++ b/desktop/src/main/__tests__/ipc-up-tasks.test.ts @@ -1,4 +1,5 @@ // @vitest-environment node +import { WorkspaceJobs } from "../workspace-jobs.js" import { EventEmitter } from "node:events" import { beforeEach, describe, expect, it, vi } from "vitest" @@ -22,16 +23,20 @@ vi.mock("../analytics.js", () => ({ const { registerIpcHandlers } = await import("../ipc.js") -function invokeUp(workspaceId: string): Promise { +async function invokeUp(workspaceId: string): Promise { const handler = handlers.get("workspace_up") if (!handler) throw new Error("workspace_up not registered") - return handler({}, { source: workspaceId, workspaceId }) as Promise + const id = (await handler({}, { source: workspaceId, workspaceId })) as string + await new Promise((resolve) => setTimeout(resolve, 10)) + return id } -function invokeStop(workspaceId: string): Promise { +async function invokeStop(workspaceId: string): Promise { const handler = handlers.get("workspace_stop") if (!handler) throw new Error("workspace_stop not registered") - return handler({}, { workspaceId }) as Promise + const id = (await handler({}, { workspaceId })) as string + await new Promise((resolve) => setTimeout(resolve, 10)) + return id } /** A child that reports itself still alive, so cancel must await its exit. */ @@ -78,6 +83,7 @@ function setup( }, ), cancelFor: vi.fn(async () => undefined), + runRaw: vi.fn(async () => statusEnvelope("building_image")), } let stream: { onLine: (line: string, s: "stdout" | "stderr") => void @@ -93,7 +99,10 @@ function setup( }, isDestroyed: () => false, } + const jobs = new WorkspaceJobs() + jobs.setRefresh(onWorkspaceStopComplete) const deps = { + workspaceJobs: jobs, cli, state: { workspaceContext: () => "ctx", @@ -114,6 +123,7 @@ function setup( return { cli, calls, + jobs, api, sent, stream: () => stream, @@ -138,11 +148,11 @@ describe("workspace_up detached task tracking", () => { vi.clearAllMocks() }) - it("cancels the prior task before submitting a replacement", async () => { + it("stop cancels an active start before launching", async () => { const { calls } = setup() await invokeUp("ws-1") - await invokeUp("ws-1") + await invokeStop("ws-1") const cancels = calls.filter((a) => a.includes("cancel")) expect(cancels).toEqual([["workspace", "task", "cancel", "task-1"]]) @@ -158,7 +168,7 @@ describe("workspace_up detached task tracking", () => { if (!active) throw new Error("stream callback was not registered") active.onLine(`token=${secret}`, "stderr") active.onExit(0) - await Promise.resolve() + await new Promise((resolve) => setTimeout(resolve, 0)) const payload = JSON.stringify(sent) expect(payload).not.toContain(secret) @@ -178,7 +188,7 @@ describe("workspace_up detached task tracking", () => { "stderr", ) active.onExit(0) - await Promise.resolve() + await new Promise((resolve) => setTimeout(resolve, 0)) const payload = JSON.stringify(sent) expect(payload).not.toContain("git-token") @@ -216,7 +226,7 @@ describe("workspace_up detached task tracking", () => { } }) - it("serializes concurrent submissions so neither task is left orphaned", async () => { + it("rejects concurrent submissions without orphaning a task", async () => { let seq = 0 const { calls } = setup({ run: async (args) => { @@ -230,10 +240,17 @@ describe("workspace_up detached task tracking", () => { }, }) - await Promise.all([invokeUp("ws-1"), invokeUp("ws-1")]) + const submissions = await Promise.allSettled([ + invokeUp("ws-1"), + invokeUp("ws-1"), + ]) + expect( + submissions.filter((result) => result.status === "rejected"), + ).toHaveLength(1) const cancels = calls.filter((a) => a.includes("cancel")) - expect(cancels).toEqual([["workspace", "task", "cancel", "task-1"]]) + expect(cancels).toEqual([]) + expect(calls.filter((args) => args.includes("--detach"))).toHaveLength(1) }) it("keeps the task cancellable when cancellation fails", async () => { @@ -251,7 +268,7 @@ describe("workspace_up detached task tracking", () => { await invokeUp("ws-1") - await invokeUp("ws-1") + await invokeStop("ws-1") expect(calls.filter((a) => a.includes("--detach"))).toHaveLength(1) failCancel = false @@ -296,7 +313,9 @@ describe("workspace_up detached task tracking", () => { "stdout", ) - expect(sent.filter((s) => s.channel === "workspace-status")[0].payload).toMatchObject({ + expect( + sent.filter((s) => s.channel === "workspace-status")[0].payload, + ).toMatchObject({ workspaceId: "ws-1", phase: "building_image", state: "succeeded", @@ -340,6 +359,7 @@ describe("workspace_up detached task tracking", () => { "stdout", ) stream()?.onExit(0) + await new Promise((resolve) => setTimeout(resolve, 10)) await invokeUp("ws-1") expect(calls.filter((a) => a.includes("cancel"))).toEqual([]) @@ -354,6 +374,7 @@ describe("workspace_up detached task tracking", () => { "stdout", ) stream()?.onExit(1) + await new Promise((resolve) => setTimeout(resolve, 10)) await invokeUp("ws-1") expect(calls.filter((a) => a.includes("cancel"))).toEqual([]) @@ -377,13 +398,12 @@ describe("workspace_up detached task tracking", () => { await invokeUp("ws-1") stream()?.onExit(1, { code: "boom", message: "follower died" }) - expect(sent.filter((s) => s.channel === "workspace-status")[0].payload).toMatchObject({ - workspaceId: "ws-1", - phase: "failed", - state: "failed", - error: { code: "boom", message: "follower died" }, - }) - await invokeUp("ws-1") + expect( + sent + .filter((s) => s.channel === "workspace-status") + .some((s) => s.payload.state === "failed"), + ).toBe(false) + await invokeStop("ws-1") expect(calls.filter((a) => a.includes("cancel"))).toEqual([ ["workspace", "task", "cancel", "task-1"], @@ -396,7 +416,10 @@ describe("workspace_up detached task tracking", () => { await invokeStop("ws-1") stream()?.onExit(1) await vi.waitFor(() => - expect(onWorkspaceStopComplete).toHaveBeenCalledWith("ws-1"), + expect(onWorkspaceStopComplete).toHaveBeenCalledWith( + "ws-1", + expect.objectContaining({ error: expect.any(String) }), + ), ) }) }) diff --git a/desktop/src/main/__tests__/ipc-workspace-jobs.test.ts b/desktop/src/main/__tests__/ipc-workspace-jobs.test.ts index 130dd59b5e..dea3e7f3b1 100644 --- a/desktop/src/main/__tests__/ipc-workspace-jobs.test.ts +++ b/desktop/src/main/__tests__/ipc-workspace-jobs.test.ts @@ -3,125 +3,171 @@ import { EventEmitter } from "node:events" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { ProviderJobs } from "../provider-jobs.js" import { WorkspaceJobs } from "../workspace-jobs.js" - -const handlers = new Map unknown>() - +const handlers = new Map any>() vi.mock("electron", () => ({ app: { getPath: () => "/tmp", getVersion: () => "0.0.0" }, dialog: {}, ipcMain: { - handle: (channel: string, fn: (...args: unknown[]) => unknown) => { - handlers.set(channel, fn) - }, + handle: (channel: string, fn: (...args: any[]) => any) => + handlers.set(channel, fn), on: () => undefined, }, })) - vi.mock("../analytics.js", () => ({ - hashWorkspaceRef: (v: string) => v, + hashWorkspaceRef: (value: string) => value, trackEvent: () => undefined, })) - const { registerIpcHandlers } = await import("../ipc.js") - -function fakeChild() { - const child = new EventEmitter() as EventEmitter & { - exitCode: number | null - signalCode: string | null - kill: () => void - } - child.exitCode = null - child.signalCode = null - child.kill = () => undefined - return child -} - -function setup(exitCode = 0) { - const workspaceJobs = new WorkspaceJobs() - const sent: Array<{ channel: string; payload: unknown }> = [] +function setup() { + const jobs = new WorkspaceJobs() + let exit!: (code: number) => void + let line!: (line: string, stream: "stdout") => void const cli = { - run: vi.fn(async () => []), - runRaw: vi.fn(async () => ""), - runStreaming: vi.fn( - async ( - _cliArgs: string[], - _onLine: (line: string, stream: "stdout" | "stderr") => void, - onExit: (code: number, cliError?: unknown) => void, - ) => { - const child = fakeChild() - setTimeout(() => { - onExit(exitCode, exitCode === 0 ? undefined : { message: "boom" }) - }, 0) - return child - }, + run: vi.fn(async () => ({ id: "task-1" })), + runRaw: vi.fn( + async () => + '{"kind":"status","schemaVersion":1,"phase":"building_image","state":"started"}', ), - cancelFor: vi.fn(async () => undefined), + runStreaming: vi.fn(async (_args, onLine, onExit) => { + exit = onExit + line = onLine + return Object.assign(new EventEmitter(), { + exitCode: null, + signalCode: null, + kill: vi.fn(), + }) + }), + cancelFor: vi.fn(async () => {}), } - const deps = { + const send = vi.fn() + const actions = registerIpcHandlers({ cli, - state: { workspaceContext: () => "ctx", providerList: () => [] }, + state: { + workspaceContext: () => "ctx", + providerList: () => [], + workspaceList: () => [], + }, logStore: { - createLogFile: () => "/tmp/log.txt", + createLogFile: () => "/tmp/log", appendLog: () => true, - closeLog: async () => undefined, - onDrain: async () => undefined, + closeLog: async () => {}, + onDrain: async () => {}, }, - pty: { cancelFor: vi.fn(async () => undefined) }, - getMainWindow: () => ({ - webContents: { - send: (channel: string, payload: unknown) => sent.push({ channel, payload }), - }, - }), + pty: { cancelFor: vi.fn(async () => {}) }, + getMainWindow: () => ({ webContents: { send } }), providerJobs: new ProviderJobs(), - workspaceJobs, + workspaceJobs: jobs, + } as any) + return { + jobs, + cli, + send, + actions, + exit: (code: number) => exit(code), + line: (text: string) => line(text, "stdout"), } - // biome-ignore lint/suspicious/noExplicitAny: partial test doubles - registerIpcHandlers(deps as any) - return { workspaceJobs, sent } } - -function invoke(channel: string, args: Record) { - const handler = handlers.get(channel) - if (!handler) throw new Error(`${channel} not registered`) - return handler({}, args) +function invoke(channel: string, args = { workspaceId: "ws" }) { + return handlers.get(channel)!({}, args) +} +async function flush() { + for (let i = 0; i < 20; i++) await Promise.resolve() } - -describe("workspace delete job lifecycle over IPC", () => { - beforeEach(() => { - handlers.clear() - vi.clearAllMocks() - vi.useFakeTimers() +beforeEach(() => { + handlers.clear() + vi.clearAllMocks() + vi.useFakeTimers() +}) +afterEach(() => vi.useRealTimers()) +describe("workspace lifecycle IPC", () => { + it.each(["delete", "stop", "rebuild", "reset"])( + "acknowledges %s while shutdown is blocked", + async (action) => { + const ctx = setup() + let release!: () => void + ctx.cli.cancelFor.mockImplementation( + () => + new Promise((resolve) => { + release = resolve + }), + ) + const commandId = invoke(`workspace_${action}`) + expect(typeof commandId).toBe("string") + expect(ctx.jobs.get("ws")).toMatchObject({ + commandId, + state: "running", + phase: "Closing connections", + }) + await flush() + expect(ctx.cli.runStreaming).not.toHaveBeenCalled() + release() + await flush() + ctx.exit(0) + await flush() + expect(ctx.jobs.get("ws")?.state).toBe("succeeded") + }, + ) + it("records preparation failure without launching the CLI", async () => { + const ctx = setup() + ctx.cli.cancelFor.mockRejectedValue(new Error("shutdown failed")) + invoke("workspace_delete") + await flush() + expect(ctx.jobs.get("ws")).toMatchObject({ + state: "failed", + error: "shutdown failed", + }) + expect(ctx.cli.runStreaming).not.toHaveBeenCalled() + }) + it("rejects duplicate deletes", () => { + setup() + invoke("workspace_delete") + expect(() => invoke("workspace_delete")).toThrow("already in progress") + }) + it("reconciles CLI failures and keeps their error", async () => { + const ctx = setup() + const refresh = vi.fn(async () => {}) + ctx.jobs.setRefresh(refresh) + invoke("workspace_delete") + await flush() + ctx.exit(1) + await flush() + expect(ctx.jobs.get("ws")).toMatchObject({ + state: "failed", + error: "Command exited with code 1", + }) + expect(refresh).toHaveBeenCalledOnce() }) - - afterEach(() => { - vi.useRealTimers() + it("does not interpret a child phase success as completion", async () => { + const ctx = setup() + invoke("workspace_rebuild") + await flush() + ctx.line( + '{"kind":"status","schemaVersion":1,"phase":"building_image","state":"succeeded"}', + ) + expect(ctx.jobs.get("ws")?.state).toBe("running") }) - - it("shows a deleting job while the command runs, then clears it", async () => { - const { workspaceJobs, sent } = setup(0) - - // workspace_delete resolves as soon as the CLI command is launched, well - // before it exits — the job must already be visible at that point. - await invoke("workspace_delete", { workspaceId: "ws1" }) - expect(workspaceJobs.get("ws1")).toEqual({ activity: "deleting" }) - - // Deterministically flush the mock's exit callback instead of racing it - // against a fixed wall-clock wait. - await vi.runAllTimersAsync() - - expect(workspaceJobs.get("ws1")).toBeUndefined() - expect(sent.filter((entry) => entry.channel === "workspace-status").map((entry) => entry.payload)).toEqual([ - { commandId: expect.any(String), workspaceId: "ws1", phase: "deleting_workspace", state: "started" }, - { commandId: expect.any(String), workspaceId: "ws1", phase: "deleting_workspace", state: "succeeded" }, - ]) + it("checks the detached task when its follower exits cleanly", async () => { + const ctx = setup() + await handlers.get("workspace_up")!({}, { source: "ws", commandId: "up" }) + await flush() + ctx.exit(0) + await flush() + expect(ctx.cli.runRaw).toHaveBeenCalled() + expect(ctx.jobs.get("ws")?.state).toBe("running") + ctx.cli.runRaw.mockResolvedValue( + '{"kind":"result","outcome":"success","containerId":"c","remoteUser":"u","remoteWorkspaceFolder":"/w"}', + ) + await vi.advanceTimersByTimeAsync(3000) + await flush() + expect(ctx.jobs.get("ws")?.state).toBe("succeeded") }) - - it("retains the failure when the delete command fails", async () => { - const { workspaceJobs } = setup(1) - - await invoke("workspace_delete", { workspaceId: "ws1" }) - await vi.runAllTimersAsync() - - expect(workspaceJobs.get("ws1")?.error).toBe("boom") + it("uses the same registry for tray stop", async () => { + const ctx = setup() + const completion = ctx.actions.workspaceActions.stop("ws") + expect(ctx.jobs.get("ws")?.activity).toBe("stopping") + await flush() + ctx.exit(0) + await completion + expect(ctx.jobs.get("ws")?.state).toBe("succeeded") }) }) diff --git a/desktop/src/main/__tests__/machine-diagnostics-manager.test.ts b/desktop/src/main/__tests__/machine-diagnostics-manager.test.ts new file mode 100644 index 0000000000..8a3f7a0e29 --- /dev/null +++ b/desktop/src/main/__tests__/machine-diagnostics-manager.test.ts @@ -0,0 +1,54 @@ +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it, vi } from "vitest" +import type { CliRunner } from "../cli.js" +import { MachineDiagnosticsManager } from "../machine-diagnostics-manager.js" +import { MachineDiagnosticsStore } from "../machine-diagnostics-store.js" + +describe("MachineDiagnosticsManager", () => { + it("coalesces concurrent refreshes for the same machine", async () => { + const runRaw = vi.fn().mockResolvedValue(JSON.stringify({ + schemaVersion: 1, + machine: { id: "machine", context: "default", provider: "test", state: "Running" }, + source: { availability: "available", freshness: "fresh" }, + events: [], + cursor: { state: "none" }, + })) + const manager = new MachineDiagnosticsManager( + { runRaw } as unknown as CliRunner, + new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-"))), + ) + const key = { context: "default", machineId: "machine" } + const [first, second] = await Promise.all([manager.refresh(key), manager.refresh(key)]) + expect(runRaw).toHaveBeenCalledTimes(1) + expect(runRaw.mock.calls[0][0]).toEqual(expect.arrayContaining(["--context", "default"])) + expect(first).toEqual(second) + }) + + it("does not recreate a deleted cache when an older collection completes", async () => { + let resolve!: (value: string) => void + const runRaw = vi.fn(() => new Promise((done) => { resolve = done })) + const store = new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-"))) + const manager = new MachineDiagnosticsManager({ runRaw } as unknown as CliRunner, store) + const key = { context: "default", machineId: "machine" } + const pending = manager.refresh(key) + manager.delete(key) + resolve(JSON.stringify({ schemaVersion: 1, machine: { id: "machine", context: "default" }, source: { availability: "available", freshness: "fresh" }, cursor: { state: "none" } })) + await expect(pending).rejects.toThrow("superseded") + expect(store.get(key)).toBeNull() + }) + + it("retains the last good snapshot after a collection failure", async () => { + const success = JSON.stringify({ schemaVersion: 1, machine: { id: "machine", context: "default", provider: "test", state: "Running" }, source: { availability: "available", freshness: "fresh" }, events: [], cursor: { state: "none" } }) + const runRaw = vi.fn().mockResolvedValueOnce(success).mockRejectedValueOnce(new Error("network unavailable")).mockResolvedValueOnce(success) + const manager = new MachineDiagnosticsManager({ runRaw } as unknown as CliRunner, new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-")))) + const key = { context: "default", machineId: "machine" } + await manager.refresh(key) + const retained = await manager.refresh(key) + expect(retained.lastCollectionError).toBe("network unavailable") + expect(retained.response.source.availability).toBe("available") + const recovered = await manager.refresh(key) + expect(recovered.lastCollectionError).toBeUndefined() + }) +}) diff --git a/desktop/src/main/__tests__/machine-diagnostics-store.test.ts b/desktop/src/main/__tests__/machine-diagnostics-store.test.ts new file mode 100644 index 0000000000..05f1773e43 --- /dev/null +++ b/desktop/src/main/__tests__/machine-diagnostics-store.test.ts @@ -0,0 +1,54 @@ +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { MachineDiagnosticsStore } from "../machine-diagnostics-store.js" + +const response = (sequence: number) => ({ + schemaVersion: 1, + machine: { id: "machine", context: "default", provider: "test", state: "Running" }, + source: { availability: "available" as const, freshness: "fresh" as const }, + daemon: { sessionId: "session", state: "running" as const, health: "healthy" as const, startedAt: "2026-09-19T00:00:00Z", updatedAt: "2026-09-19T00:00:00Z", workspaceCount: 1 }, + events: [{ schemaVersion: 1, sessionId: "session", sequence, timestamp: `2026-09-19T00:00:0${sequence}Z`, level: "info" as const, type: "daemon.started", message: "started" }], + cursor: { next: `cursor-${sequence}`, state: "ok" as const }, +}) + +describe("MachineDiagnosticsStore", () => { + it("retains the last daemon snapshot and cursor when the remote reader fails", () => { + const store = new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-"))) + const key = { context: "default", machineId: "machine" } + const good = store.merge(key, response(1)) + const failed = store.merge(key, { ...response(2), daemon: undefined, events: [], source: { availability: "permission_denied", freshness: "unknown", message: "Access denied" }, cursor: { state: "none" } }) + expect(failed.response.daemon).toEqual(good.response.daemon) + expect(failed.cursor).toBe(good.cursor) + expect(failed.lastSuccessfulCollectionAt).toBe(good.lastSuccessfulCollectionAt) + expect(failed.lastCollectionError).toBe("Access denied") + expect(failed.response.source.freshness).toBe("stale") + expect(store.merge(key, response(2)).lastCollectionError).toBeUndefined() + }) + it("deduplicates remote events and retains them after a stop", () => { + const store = new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-"))) + const key = { context: "default", machineId: "machine" } + store.merge(key, response(1)) + const cache = store.merge(key, response(1)) + expect(cache.events).toHaveLength(1) + expect(cache.cursor).toBe("cursor-1") + expect(store.markStopped(key)?.response.source.availability).toBe("machine_stopped") + expect(store.get(key)?.events).toHaveLength(1) + }) + + it("rejects path traversal in keys", () => { + const store = new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-"))) + expect(() => store.get({ context: "../other", machineId: "machine" })).toThrow("invalid diagnostics key") + }) + + it("clears a stale cursor after a remote session reset", () => { + const store = new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-"))) + const key = { context: "default", machineId: "machine" } + store.merge(key, response(1)) + const reset = { ...response(2), cursor: { state: "reset" as const, reason: "session_changed" } } + const cache = store.merge(key, reset) + expect(cache.cursor).toBeUndefined() + expect(cache.lastCollectionError).toBeUndefined() + }) +}) diff --git a/desktop/src/main/__tests__/tray.test.ts b/desktop/src/main/__tests__/tray.test.ts index cd3cf30cc9..aed9eb07fc 100644 --- a/desktop/src/main/__tests__/tray.test.ts +++ b/desktop/src/main/__tests__/tray.test.ts @@ -10,11 +10,25 @@ vi.mock("../updater.js", () => ({ describe("buildUpdateMenuItems", () => { it("returns nothing when no update is downloaded", () => { - expect(buildUpdateMenuItems({ state: "idle", currentVersion: "1.0.0" }, () => {})).toEqual([]) - expect(buildUpdateMenuItems({ state: "checking", currentVersion: "1.0.0" }, () => {})).toEqual([]) expect( buildUpdateMenuItems( - { state: "available", currentVersion: "1.0.0", availableVersion: "1.1.0" }, + { state: "idle", currentVersion: "1.0.0" }, + () => {}, + ), + ).toEqual([]) + expect( + buildUpdateMenuItems( + { state: "checking", currentVersion: "1.0.0" }, + () => {}, + ), + ).toEqual([]) + expect( + buildUpdateMenuItems( + { + state: "available", + currentVersion: "1.0.0", + availableVersion: "1.1.0", + }, () => {}, ), ).toEqual([]) @@ -24,16 +38,36 @@ describe("buildUpdateMenuItems", () => { state: "downloading", currentVersion: "1.0.0", availableVersion: "1.1.0", - progress: { percent: 50, bytesPerSecond: 1000, transferred: 50, total: 100 }, + progress: { + percent: 50, + bytesPerSecond: 1000, + transferred: 50, + total: 100, + }, }, () => {}, ), ).toEqual([]) - expect(buildUpdateMenuItems({ state: "not-available", currentVersion: "1.0.0" }, () => {})).toEqual([]) - expect(buildUpdateMenuItems({ state: "up-to-date", currentVersion: "1.0.0" }, () => {})).toEqual([]) expect( buildUpdateMenuItems( - { state: "error", currentVersion: "1.0.0", error: "x", code: "network" }, + { state: "not-available", currentVersion: "1.0.0" }, + () => {}, + ), + ).toEqual([]) + expect( + buildUpdateMenuItems( + { state: "up-to-date", currentVersion: "1.0.0" }, + () => {}, + ), + ).toEqual([]) + expect( + buildUpdateMenuItems( + { + state: "error", + currentVersion: "1.0.0", + error: "x", + code: "network", + }, () => {}, ), ).toEqual([]) @@ -42,7 +76,11 @@ describe("buildUpdateMenuItems", () => { it("adds Update item + separator when downloaded", () => { const onInstall = vi.fn() const items = buildUpdateMenuItems( - { state: "downloaded", currentVersion: "1.0.0", availableVersion: "9.9.9" }, + { + state: "downloaded", + currentVersion: "1.0.0", + availableVersion: "9.9.9", + }, onInstall, ) expect(items).toHaveLength(2) @@ -122,7 +160,9 @@ describe("buildTrayMenuTemplate", () => { }, actions, ) - expect((empty[2].submenu as Array>)[0]).toMatchObject({ + expect( + (empty[2].submenu as Array>)[0], + ).toMatchObject({ label: "No Active Workspaces", enabled: false, }) @@ -135,8 +175,33 @@ describe("buildTrayMenuTemplate", () => { }, actions, ) - const stop = ((pending[2].submenu as Array>)[0] - .submenu as Array>)[1] + const stop = ( + (pending[2].submenu as Array>)[0] + .submenu as Array> + )[1] expect(stop).toMatchObject({ label: "Stopping…", enabled: false }) }) + it("shows shared progress and permits stopping an active start", () => { + const items = buildTrayMenuTemplate( + { + activeWorkspaces: [{ id: "ws", status: "Stopped" }], + pendingStops: new Set(), + jobs: { + ws: { + commandId: "start", + activity: "starting", + state: "running", + phase: "Building image", + }, + }, + updateStatus: { state: "idle", currentVersion: "1.0.0" }, + }, + actions, + ) + const workspace = (items[2].submenu as Array>)[0] + expect(workspace.label).toBe("ws — Starting") + expect( + (workspace.submenu as Array>)[1], + ).toMatchObject({ label: "Stop Workspace", enabled: true }) + }) }) diff --git a/desktop/src/main/__tests__/watcher.test.ts b/desktop/src/main/__tests__/watcher.test.ts index 695732a38c..79513eaff1 100644 --- a/desktop/src/main/__tests__/watcher.test.ts +++ b/desktop/src/main/__tests__/watcher.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest" +import { WorkspaceJobs } from "../workspace-jobs.js" import { Watcher } from "../watcher.js" function makeWatcher( @@ -27,7 +28,7 @@ function makeWatcher( runRaw: vi.fn(runRaw), } const providerJobs = { snapshot: vi.fn().mockReturnValue({}) } - const workspaceJobs = { snapshot: vi.fn().mockReturnValue({}) } + const workspaceJobs = new WorkspaceJobs() const watcher = new Watcher({ cli: cli as never, state: state as never, @@ -35,7 +36,7 @@ function makeWatcher( providerJobs: providerJobs as never, workspaceJobs: workspaceJobs as never, }) - return { watcher, cli, state } + return { watcher, cli, state, workspaceJobs } } describe("Watcher.refreshProviders", () => { @@ -193,3 +194,37 @@ describe("Watcher.refreshWorkspaceStatuses", () => { expect(workspaces[1].status).toBe("stopped") }) }) + +describe("workspace status ordering", () => { + it("discards an observation started before an operation", async () => { + let release!: (value: string) => void + const { watcher, state, workspaceJobs } = makeWatcher( + async () => ({}), + [{ id: "ws", status: "Running" }], + () => + new Promise((resolve) => { + release = resolve + }), + ) + const polling = watcher.refreshWorkspaceStatuses() + await Promise.resolve() + workspaceJobs.start("ws", "stopping", "stop") + release('{"state":"Running"}') + await polling + expect(state.updateWorkspaceStatus).not.toHaveBeenCalled() + }) + it("propagates targeted refresh failure without poisoning subsequent refreshes", async () => { + const { watcher, cli } = makeWatcher( + async () => ({}), + [{ id: "ws" }], + async () => { + throw new Error("offline") + }, + ) + await expect(watcher.refreshWorkspaceStatus("ws")).rejects.toThrow( + "offline", + ) + cli.runRaw.mockResolvedValue('{"state":"Stopped"}') + await expect(watcher.refreshWorkspaceStatus("ws")).resolves.toBeUndefined() + }) +}) diff --git a/desktop/src/main/__tests__/workspace-jobs.test.ts b/desktop/src/main/__tests__/workspace-jobs.test.ts index 7ce9caba7d..4546e21ae9 100644 --- a/desktop/src/main/__tests__/workspace-jobs.test.ts +++ b/desktop/src/main/__tests__/workspace-jobs.test.ts @@ -1,104 +1,101 @@ // @vitest-environment node -import { beforeEach, describe, expect, it, vi } from "vitest" +import { describe, expect, it, vi } from "vitest" import { WorkspaceJobs } from "../workspace-jobs.js" describe("WorkspaceJobs", () => { - let jobs: WorkspaceJobs - - beforeEach(() => { - jobs = new WorkspaceJobs() - }) - - it("tracks a delete until it finishes", async () => { - const generation = jobs.start("ws1") - expect(jobs.get("ws1")).toEqual({ activity: "deleting" }) - - await jobs.finish("ws1", generation) - expect(jobs.get("ws1")).toBeUndefined() - }) - - it("does not let a stale failure land on a newer job", async () => { - const first = jobs.start("ws1") - const second = jobs.start("ws1") - - await jobs.finish("ws1", first, "boom") - - expect(jobs.get("ws1")).toEqual({ activity: "deleting" }) - expect(second).not.toBe(first) - }) - - it("retains the failure so the UI can explain it", async () => { - const generation = jobs.start("ws1") - await jobs.finish("ws1", generation, "delete exited with code 1") - - expect(jobs.get("ws1")).toEqual({ - activity: "deleting", - error: "delete exited with code 1", + it("publishes action state independently of runtime status", async () => { + const jobs = new WorkspaceJobs() + const changed = vi.fn() + jobs.onChange(changed) + const generation = jobs.start("ws", "stopping", "command") + expect(jobs.get("ws")).toMatchObject({ + commandId: "command", + activity: "stopping", + state: "running", }) + jobs.progress("ws", "command", { + phase: "stopping_workspace", + step: "Waiting for lock", + state: "started", + }) + jobs.progress("ws", "command", { + phase: "stopping_workspace", + state: "succeeded", + }) + expect(jobs.get("ws")).toMatchObject({ + state: "running", + phase: "Waiting for lock", + }) + await jobs.finish("ws", generation) + expect(jobs.get("ws")?.state).toBe("succeeded") + expect(changed).toHaveBeenCalled() }) - it("does not let a later release erase a recorded failure", async () => { - const generation = jobs.start("ws1") - await jobs.finish("ws1", generation, "boom") - - await jobs.finish("ws1", generation) - - expect(jobs.get("ws1")?.error).toBe("boom") - }) - - it("ignores a failure for a workspace with no active job", async () => { - await jobs.finish("ws1", 1, "boom") - expect(jobs.get("ws1")).toBeUndefined() + it("rejects duplicates and allows delete to supersede a start", async () => { + const jobs = new WorkspaceJobs() + const first = jobs.start("ws", "starting", "first") + expect(() => jobs.start("ws", "rebuilding", "conflict")).toThrow( + "already in progress", + ) + jobs.start("ws", "deleting", "delete") + jobs.progress("ws", "first", { phase: "ready", state: "succeeded" }) + await jobs.finish("ws", first, "old failure") + expect(jobs.get("ws")).toMatchObject({ + commandId: "delete", + state: "running", + }) }) - it("refreshes the workspace list before clearing a finished job", async () => { - const order: string[] = [] - jobs.setRefresh(async () => { - order.push(`refresh(job=${jobs.get("ws1") ? "present" : "gone"})`) + it("retains a completed delete on refresh failure and retries refresh only", async () => { + const jobs = new WorkspaceJobs() + const refresh = vi + .fn() + .mockRejectedValueOnce(new Error("offline")) + .mockResolvedValue(undefined) + jobs.setRefresh(refresh) + const generation = jobs.start("ws", "deleting", "delete") + await jobs.finish("ws", generation) + expect(jobs.get("ws")).toMatchObject({ + state: "reconciling", + refreshError: "offline", }) - - const generation = jobs.start("ws1") - await jobs.finish("ws1", generation) - - expect(order).toEqual(["refresh(job=present)"]) - expect(jobs.get("ws1")).toBeUndefined() + expect(() => jobs.start("ws")).toThrow() + await jobs.retryRefresh("ws") + expect(jobs.get("ws")).toMatchObject({ + state: "succeeded", + refreshError: undefined, + }) + expect(refresh).toHaveBeenCalledTimes(2) }) - it("does not clear a newer job started while refresh was in flight", async () => { - let releaseRefresh: (() => void) | undefined + it("keeps the action visible until reconciliation finishes and ignores duplicate completion", async () => { + const jobs = new WorkspaceJobs() + let release!: () => void jobs.setRefresh( () => new Promise((resolve) => { - releaseRefresh = resolve + release = resolve }), ) - - const generation = jobs.start("ws1") - const finishing = jobs.finish("ws1", generation) - - jobs.start("ws1") - releaseRefresh?.() + const generation = jobs.start("ws") + const finishing = jobs.finish("ws", generation, "denied") + expect(jobs.get("ws")).toMatchObject({ + state: "reconciling", + error: "denied", + }) + await jobs.finish("ws", generation) + release() await finishing - - expect(jobs.get("ws1")).toEqual({ activity: "deleting" }) - }) - - it("notifies listeners on every mutation", () => { - const listener = vi.fn() - jobs.onChange(listener) - - jobs.start("ws1") - jobs.clear("ws1") - - expect(listener).toHaveBeenCalledTimes(2) + expect(jobs.get("ws")).toMatchObject({ state: "failed", error: "denied" }) + expect(() => jobs.start("ws")).not.toThrow() }) - it("does not notify when clearing an untracked workspace", () => { - const listener = vi.fn() - jobs.onChange(listener) - - jobs.clear("nonexistent") - - expect(listener).not.toHaveBeenCalled() + it("invalidates polls on acceptance and completion", async () => { + const jobs = new WorkspaceJobs() + const before = jobs.generation("ws") + const generation = jobs.start("ws") + expect(generation).not.toBe(before) + await jobs.finish("ws", generation) + expect(jobs.generation("ws")).not.toBe(generation) }) }) diff --git a/desktop/src/main/index.ts b/desktop/src/main/index.ts index b035118b0d..f516c25fab 100644 --- a/desktop/src/main/index.ts +++ b/desktop/src/main/index.ts @@ -7,6 +7,8 @@ import { CliRunner } from "./cli.js" import { DaemonManager } from "./daemon-manager.js" import { registerIpcHandlers } from "./ipc.js" import { LogStore } from "./log-store.js" +import { MachineDiagnosticsStore } from "./machine-diagnostics-store.js" +import { MachineDiagnosticsManager } from "./machine-diagnostics-manager.js" import { ProviderJobs } from "./provider-jobs.js" import { PtyManager } from "./pty.js" import { DaemonState } from "./state.js" @@ -156,6 +158,8 @@ app.whenReady().then(() => { // ~/.devsy/contexts//workspaces// subtree that `workspace delete` // unlinks. That separation closes the file-deletion race by construction. const logStore = new LogStore(join(homedir(), ".devsy", "desktop", "logs")) + const machineDiagnosticsStore = new MachineDiagnosticsStore(join(homedir(), ".devsy", "desktop", "diagnostics")) + const machineDiagnosticsManager = new MachineDiagnosticsManager(cli, machineDiagnosticsStore) try { const pruned = logStore.prune(30) if (pruned > 0) console.log(`Pruned ${pruned} old log files`) @@ -204,6 +208,8 @@ app.whenReady().then(() => { cli, state, logStore, + machineDiagnosticsStore, + machineDiagnosticsManager, pty: ptyManager, getMainWindow: () => mainWindow, providerJobs, @@ -222,13 +228,7 @@ app.whenReady().then(() => { pendingRoute = null } }, - onWorkspaceStopComplete: async (workspaceId) => { - try { - await watcher?.refreshWorkspaceStatus(workspaceId) - } finally { - watcher?.broadcastWorkspaces() - } - }, + workspaceSnapshot: () => watcher?.workspaceSnapshot(), }) // Start state watcher @@ -245,9 +245,19 @@ app.whenReady().then(() => { watcher ? watcher.refreshProviders() : Promise.resolve(), ) workspaceJobs.onChange(() => watcher?.broadcastWorkspaces()) - workspaceJobs.setRefresh(() => - watcher ? watcher.refreshWorkspaces() : Promise.resolve(), - ) + workspaceJobs.setRefresh(async (id, job) => { + if (!watcher) throw new Error("Workspace watcher unavailable") + await watcher.refreshWorkspaces() + const exists = state.workspaceList().some((workspace) => workspace.id === id) + if (job.activity === "deleting" && !job.error) { + if (exists) throw new Error("Workspace list has not caught up yet") + } else if (exists) { + await watcher.refreshWorkspaceStatus(id) + } else if (!job.error) { + throw new Error("Workspace not yet present in the list") + } + watcher.broadcastWorkspaces() + }) void watcher.start().then(runInitialProviderUpdateCheck) scheduleProviderUpdateCheck() @@ -255,6 +265,7 @@ app.whenReady().then(() => { // Set up system tray appTray = new AppTray({ state, + workspaceJobs, showDevsy, stopWorkspace: workspaceActions.stop, refreshWorkspace: (id) => diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 1c2a0294f0..16a6e2f562 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -15,6 +15,8 @@ import { hashWorkspaceRef, trackEvent } from "./analytics.js" import type { CliRunner } from "./cli.js" import { loadCatalog } from "./image-catalog.js" import type { LogStore } from "./log-store.js" +import type { MachineDiagnosticsStore } from "./machine-diagnostics-store.js" +import type { MachineDiagnosticsManager } from "./machine-diagnostics-manager.js" import type { ProviderActivity, ProviderJobs, @@ -34,6 +36,7 @@ import { } from "./updater.js" import { type ProviderEntry, parseProviderEntries } from "./watcher.js" import { normalizeWorkspaceStatus } from "./workspace-status.js" +import type { WorkspaceActivity } from "../shared/workspace-operation.js" import type { WorkspaceJobs } from "./workspace-jobs.js" const execFileAsync = promisify(execFile) @@ -97,11 +100,13 @@ interface IpcDependencies { cli: CliRunner state: DaemonState logStore: LogStore + machineDiagnosticsStore?: MachineDiagnosticsStore + machineDiagnosticsManager?: MachineDiagnosticsManager pty: PtyManager getMainWindow: () => BrowserWindow | null providerJobs: ProviderJobs workspaceJobs: WorkspaceJobs - onWorkspaceStopComplete?: (workspaceId: string) => Promise + workspaceSnapshot?: () => unknown onRendererReady?: (sender: Electron.WebContents) => void } @@ -191,6 +196,7 @@ function createLogSink( const FLUSH_MS = 64 const MAX_BATCH = 250 let buf: string[] = [] + let finished = false let timer: ReturnType | null = null function post( @@ -209,7 +215,9 @@ function createLogSink( if (!done && buf.length === 0) return const lines = buf buf = [] - getWin()?.webContents.send("command-progress", { + const win = getWin() + if (!win || win.isDestroyed?.()) return + win.webContents.send("command-progress", { commandId, lines: lines.map(redactSensitiveText), done, @@ -225,6 +233,7 @@ function createLogSink( return { line(formatted) { + if (finished) return true const safeLine = redactSensitiveText(formatted) const ok = appendLog?.(safeLine) ?? true buf.push(safeLine) @@ -233,10 +242,17 @@ function createLogSink( return ok }, async done(finalLine, extra) { + if (finished) return + finished = true const safeLine = redactSensitiveText(finalLine) - appendLog?.(safeLine) buf.push(safeLine) - await flush?.() + try { + appendLog?.(safeLine) + await flush?.() + } catch (error) { + // Log persistence must not change a command's actual outcome. + console.warn("[workspace-operation] log flush failed", error) + } post(true, { message: finalLine, ...extra }) }, } @@ -248,7 +264,7 @@ export function registerIpcHandlers(deps: IpcDependencies): { runInitialProviderUpdateCheck: () => void workspaceActions: { stop: (workspaceId: string) => Promise } } { - const { cli, state, logStore, pty, providerJobs, workspaceJobs } = deps + const { cli, state, logStore, pty, providerJobs, workspaceJobs, machineDiagnosticsStore, machineDiagnosticsManager, getMainWindow } = deps const tunnelProcesses = new Map< string, import("node:child_process").ChildProcess @@ -393,36 +409,13 @@ export function registerIpcHandlers(deps: IpcDependencies): { if (stream !== "stdout") return false const envelope = parseCliEnvelope(line) if (envelope?.kind !== "status") return false - deps.getMainWindow()?.webContents.send("workspace-status", { - commandId, - workspaceId, - ...redactOperationStatus(normalizeOperationStatus(envelope)), - }) + if (!workspaceJobs.owns(workspaceId, commandId)) return true + const status = redactOperationStatus(normalizeOperationStatus(envelope)) + workspaceJobs.progress(workspaceId, commandId, status) + deps.getMainWindow()?.webContents.send("workspace-status", { commandId, workspaceId, ...status }) return true } - function emitWorkspaceStatus( - commandId: string, - workspaceId: string, - phase: string, - state: "started" | "succeeded" | "failed", - cliError?: CLIError, - ): void { - deps.getMainWindow()?.webContents.send("workspace-status", { - commandId, - workspaceId, - phase, - state, - ...(cliError - ? { - error: { - ...redactCLIError(cliError), - }, - } - : {}), - }) - } - /** * Track a provider job for the duration of fn, so the job cannot outlive the * work it describes. Opening a job in one place and closing it in another @@ -537,6 +530,10 @@ export function registerIpcHandlers(deps: IpcDependencies): { // ── Workspaces ── ipcMain.handle("workspace_list", () => state.workspaceList()) + ipcMain.handle("workspace_snapshot", () => deps.workspaceSnapshot?.() ?? { + workspaces: state.workspaceList(), jobs: workspaceJobs.snapshot(), revision: workspaceJobs.revision, + }) + ipcMain.handle("workspace_refresh", (_event, args: { workspaceId: string }) => workspaceJobs.retryRefresh(args.workspaceId)) ipcMain.handle( "workspace_status", @@ -551,8 +548,9 @@ export function registerIpcHandlers(deps: IpcDependencies): { "15s", ] if (args.recovery) cliArgs.push("--recovery") + const generation = workspaceJobs.generation(args.workspaceId) const raw = await cli.runRaw(cliArgs) - if (!args.recovery) { + if (!args.recovery && generation === workspaceJobs.generation(args.workspaceId)) { const status = normalizeWorkspaceStatus(raw) if (status && state.updateWorkspaceStatus(args.workspaceId, status)) { // The watcher owns renderer broadcasts; this update still keeps @@ -895,9 +893,11 @@ export function registerIpcHandlers(deps: IpcDependencies): { ipcMain.handle( "machine_delete", async (_event, args: { id: string; force?: boolean }) => { - const cliArgs = ["machine", "delete", args.id] + const key = { context: state.currentContext(), machineId: args.id } + const cliArgs = ["machine", "delete", args.id, "--context", key.context] if (args.force) cliArgs.push("--force") await cli.runRaw(cliArgs) + machineDiagnosticsManager?.delete(key) }, ) @@ -906,13 +906,27 @@ export function registerIpcHandlers(deps: IpcDependencies): { }) ipcMain.handle("machine_stop", async (_event, args: { id: string }) => { - await cli.runRaw(["machine", "stop", args.id]) + const key = { context: state.currentContext(), machineId: args.id } + await cli.runRaw(["machine", "stop", args.id, "--context", key.context]) + machineDiagnosticsManager?.markStopped(key) }) ipcMain.handle("machine_status", async (_event, args: { id: string }) => { return cli.runRaw(["machine", "status", args.id, "--result-format", "json"]) }) + ipcMain.handle("machine_diagnostics_get", (_event, args: { id: string }) => { + return machineDiagnosticsManager?.getCached({ context: state.currentContext(), machineId: args.id }) ?? null + }) + + ipcMain.handle("machine_diagnostics_refresh", async (_event, args: { id: string }) => { + if (!machineDiagnosticsManager) throw new Error("machine diagnostics manager is unavailable") + const key = { context: state.currentContext(), machineId: args.id } + const merged = await machineDiagnosticsManager.refresh(key) + getMainWindow()?.webContents.send("machine-diagnostics-changed", { machineId: args.id, context: key.context, diagnostics: merged }) + return merged + }) + // ── Contexts ── ipcMain.handle("context_list", () => state.contextList()) @@ -1116,160 +1130,288 @@ export function registerIpcHandlers(deps: IpcDependencies): { const wsId = args.workspaceId ?? args.source const cmdId = args.commandId ?? crypto.randomUUID() - const logPath = logStore.createLogFile(state.workspaceContext(wsId), wsId) - const sink = createLogSink( - deps.getMainWindow, - cmdId, - (line) => logStore.appendLog(logPath, line), - () => logStore.closeLog(logPath), - ) + const activity = args.workspaceId ? "creating" : "starting" + const generation = workspaceJobs.start(wsId, activity, cmdId) + const started = performance.now() + const timing = (stage: string) => + console.debug( + `[workspace-operation] ${cmdId} ${stage} ${Math.round(performance.now() - started)}ms`, + ) + timing("accepted") + void (async () => { + const logPath = logStore.createLogFile( + state.workspaceContext(wsId), + wsId, + ) + const sink = createLogSink( + deps.getMainWindow, + cmdId, + (line) => logStore.appendLog(logPath, line), + () => logStore.closeLog(logPath), + ) - return serializePerWorkspace(wsId, async () => { - // Tear down any prior run for this workspace before starting a new - // one. - let taskId: string - try { - await cancelActiveUp(wsId) - // Submit: returns immediately with the background task's id. - const submitted = await cli.run<{ kind: string; id: string }>([ - ...cliArgs, - "--detach", - ]) - if (!submitted?.id) { - throw new Error("workspace up --detach returned no task id") + await serializePerWorkspace(wsId, async () => { + // Tear down any prior run for this workspace before starting a new + // one. + let taskId: string + try { + workspaceJobs.phase(wsId, cmdId, "Closing connections") + await cancelActiveUp(wsId) + timing("shutdown-complete") + workspaceJobs.phase(wsId, cmdId, "Launching command") + timing("cli-launch") + // Submit: returns immediately with the background task's id. + const submitted = await cli.run<{ kind: string; id: string }>([ + ...cliArgs, + "--detach", + ]) + if (!submitted?.id) { + throw new Error("workspace up --detach returned no task id") + } + taskId = submitted.id + } catch (error) { + const err = error as Error & { cliError?: CLIError } + void sink.done(formatLogLine(err.message, "ERROR"), { + level: "error", + success: false, + cliError: err.cliError ?? { + code: "up_failed", + message: err.message, + }, + }) + await workspaceJobs.finish( + wsId, + generation, + redactCLIError( + err.cliError ?? { code: "up_failed", message: err.message }, + ).message, + ) + return cmdId } - taskId = submitted.id - } catch (error) { - const err = error as Error & { cliError?: CLIError } - void sink.done(formatLogLine(err.message, "ERROR"), { - level: "error", - success: false, - cliError: err.cliError ?? { - code: "up_failed", - message: err.message, - }, - }) - return cmdId - } - activeUpTasks.set(wsId, taskId) + activeUpTasks.set(wsId, taskId) - // A newer submission may already own the entry and must stay cancellable. - const releaseTask = () => { - if (activeUpTasks.get(wsId) === taskId) { - activeUpTasks.delete(wsId) + // A newer submission may already own the entry and must stay cancellable. + const releaseTask = () => { + if (activeUpTasks.get(wsId) === taskId) { + activeUpTasks.delete(wsId) + } } - } - - let signalledDone = false - let suppressCallbacks = false - const sendWorkspaceFailureStatus = (cliError: CLIError | undefined) => { - const safeError = redactCLIError( - cliError ?? { code: "up_failed", message: "workspace up failed" }, - ) - deps.getMainWindow()?.webContents.send("workspace-status", { - commandId: cmdId, - workspaceId: wsId, - phase: "failed", - state: "failed", - error: safeError, - }) - } - let child: import("node:child_process").ChildProcess - try { - child = await cli.runStreaming( - ["workspace", "task", "logs", taskId, "--follow"], - (line, stream) => { - if (signalledDone || suppressCallbacks) return - - // Structured NDJSON envelopes only ever appear on stdout; stderr - // carries freeform zap log lines. - const envelope = - stream === "stdout" ? parseCliEnvelope(line) : undefined - - if (envelope?.kind === "status") { - const status = redactOperationStatus(normalizeOperationStatus(envelope)) - deps.getMainWindow()?.webContents.send("workspace-status", { - commandId: cmdId, - workspaceId: wsId, - ...status, - }) - return - } - - const formatted = formatLogLine(line) - - if (envelope?.kind === "result") { - signalledDone = true - releaseTask() - void sink.done(formatted, { success: true }) - return - } - - if (envelope?.kind === "error") { - signalledDone = true - releaseTask() - void sink.done(formatted, { - level: "error", - success: false, - cliError: { - code: envelope.code ?? "up_failed", - message: envelope.message, - hint: envelope.hint, - context: envelope.context, - }, - }) - return - } - if (!sink.line(formatted)) return logStore.onDrain(logPath) - }, - (code, cliError) => { - // No releaseTask: the follower dying says nothing about the - // detached worker, and would orphan a still-running task. - if (tunnelProcesses.get(wsId) === child) { - tunnelProcesses.delete(wsId) - } - if (signalledDone || suppressCallbacks) return - if (code !== 0) sendWorkspaceFailureStatus(cliError) - void sink.done( - formatLogLine( - `Exit code: ${code}`, - code === 0 ? "INFO" : "ERROR", - ), - code === 0 - ? { success: true } - : { level: "error", success: false, cliError }, - ) - }, - wsId, - ) - // Expose a method to suppress callbacks from cancelActiveUp - ;(child as unknown as { _suppressWorkspaceCallbacks?: () => void })._suppressWorkspaceCallbacks = () => { - suppressCallbacks = true + let firstProgress = true + let signalledDone = false + let suppressCallbacks = false + let child: import("node:child_process").ChildProcess + try { + child = await cli.runStreaming( + ["workspace", "task", "logs", taskId, "--follow"], + (line, stream) => { + if ( + signalledDone || + suppressCallbacks || + !workspaceJobs.owns(wsId, cmdId) + ) + return + + if (firstProgress) { + timing("first-progress") + firstProgress = false + } + // Structured NDJSON envelopes only ever appear on stdout; stderr + // carries freeform zap log lines. + const envelope = + stream === "stdout" ? parseCliEnvelope(line) : undefined + + if (envelope?.kind === "status") { + const status = redactOperationStatus( + normalizeOperationStatus(envelope), + ) + workspaceJobs.progress(wsId, cmdId, status) + deps.getMainWindow()?.webContents.send("workspace-status", { + commandId: cmdId, + workspaceId: wsId, + ...status, + }) + return + } + + const formatted = formatLogLine(line) + + if (envelope?.kind === "result") { + signalledDone = true + releaseTask() + timing("completed") + void workspaceJobs.finish(wsId, generation) + void sink.done(formatted, { success: true }) + return + } + + if (envelope?.kind === "error") { + signalledDone = true + releaseTask() + void workspaceJobs.finish( + wsId, + generation, + redactCLIError({ + code: envelope.code ?? "up_failed", + message: envelope.message, + }).message, + ) + void sink.done(formatted, { + level: "error", + success: false, + cliError: { + code: envelope.code ?? "up_failed", + message: envelope.message, + hint: envelope.hint, + context: envelope.context, + }, + }) + return + } + + if (!sink.line(formatted)) return logStore.onDrain(logPath) + }, + (code, cliError) => { + // No releaseTask: the follower dying says nothing about the + // detached worker, and would orphan a still-running task. + if (tunnelProcesses.get(wsId) === child) { + tunnelProcesses.delete(wsId) + } + if ( + signalledDone || + suppressCallbacks || + !workspaceJobs.owns(wsId, cmdId) + ) + return + // A follower exiting is not evidence that the detached task finished. + void reconcileDetachedTask( + wsId, + cmdId, + generation, + taskId, + sink, + releaseTask, + ) + }, + wsId, + ) + // Expose a method to suppress callbacks from cancelActiveUp + ;( + child as unknown as { _suppressWorkspaceCallbacks?: () => void } + )._suppressWorkspaceCallbacks = () => { + suppressCallbacks = true + } + } catch (error) { + // A failed follower does not prove that the detached task failed. + // Keep its cancellation handle and reconcile from persisted task state. + void reconcileDetachedTask( + wsId, + cmdId, + generation, + taskId, + sink, + releaseTask, + ) + return cmdId } - } catch (error) { - // The task is already submitted; keep it registered so a later - // cancel can still reach it, and close the sink so the UI isn't - // left waiting on a follower that never started. - const err = error as Error & { cliError?: CLIError } - sendWorkspaceFailureStatus(err.cliError) - void sink.done(formatLogLine(err.message, "ERROR"), { - level: "error", - success: false, - cliError: err.cliError ?? { - code: "up_follow_failed", - message: err.message, - }, - }) - return cmdId - } - tunnelProcesses.set(wsId, child) + tunnelProcesses.set(wsId, child) - return cmdId + return cmdId + }) + })().catch(async (error) => { + await workspaceJobs.finish( + wsId, + generation, + redactCLIError(cliErrorOrFallback(error, "up_failed")).message, + ) }) + return cmdId }, ) + async function reconcileDetachedTask( + wsId: string, + commandId: string, + generation: number, + taskId: string, + sink: ProgressSink, + release: () => void, + ): Promise { + while ( + workspaceJobs.owns(wsId, commandId) && + workspaceJobs.get(wsId)?.state === "running" + ) { + try { + const raw = await cli.runRaw([ + "workspace", + "task", + "get", + taskId, + "--result-format", + "json", + ]) + if (!workspaceJobs.owns(wsId, commandId)) return + for (const line of raw.split("\n")) { + const envelope = parseCliEnvelope(line) + if (envelope?.kind === "result") { + release() + await sink + .done(formatLogLine("Completed"), { success: true }) + .catch((error) => + console.warn("[workspace-operation] log flush failed", error), + ) + await workspaceJobs.finish(wsId, generation) + return + } + if (envelope?.kind === "status") + workspaceJobs.progress( + wsId, + commandId, + redactOperationStatus(normalizeOperationStatus(envelope)), + ) + } + } catch (error) { + // task get can also fail because the store/CLI is unavailable. Only a + // persisted terminal task state authorizes declaring the worker failed. + try { + const tasks = await cli.run< + Array<{ + id: string + status: string + error?: string + errorCode?: string + }> + >(["workspace", "task", "list"]) + const task = tasks.find((task) => task.id === taskId) + if ( + task?.status === "failed" && + workspaceJobs.owns(wsId, commandId) + ) { + release() + const safe = redactCLIError({ + code: task.errorCode ?? "up_failed", + message: task.error || "Workspace task failed", + }) + await sink.done(formatLogLine(safe.message, "ERROR"), { + success: false, + cliError: safe, + }) + await workspaceJobs.finish(wsId, generation, safe.message) + return + } + } catch { + /* Keep the operation and cancellation handle while unavailable. */ + } + workspaceJobs.phase( + wsId, + commandId, + "Progress unavailable · Checking task", + ) + } + await new Promise((resolve) => setTimeout(resolve, 3000)) + } + } + type WorkspaceActionSource = "renderer" | "tray" type StopWorkspaceArgs = { workspaceId: string @@ -1277,337 +1419,164 @@ export function registerIpcHandlers(deps: IpcDependencies): { commandId?: string } - async function startWorkspaceStop( + function startWorkspaceAction( args: StopWorkspaceArgs, - source: WorkspaceActionSource, - ): Promise<{ commandId: string; completion: Promise }> { - trackEvent("workspace_stop", { - workspace_ref: hashWorkspaceRef(args.workspaceId), - source, - }) - await quiesceWorkspace(args.workspaceId) + activity: WorkspaceActivity, + cliArgs: string[], + source: WorkspaceActionSource = "renderer", + ): { commandId: string; completion: Promise } { const commandId = args.commandId ?? crypto.randomUUID() - const logPath = logStore.createLogFile( - state.workspaceContext(args.workspaceId), + const generation = workspaceJobs.start( args.workspaceId, - ) - const sink = createLogSink( - deps.getMainWindow, + activity, commandId, - (line) => logStore.appendLog(logPath, line), - () => logStore.closeLog(logPath), ) - const completion = new Promise((resolve, reject) => { - const cliArgs = ["workspace", "stop", args.workspaceId] - if (args.debug) cliArgs.push("--debug") - emitWorkspaceStatus(commandId, args.workspaceId, "stopping_workspace", "started") - void cli.runStreaming( - cliArgs, - (line, stream) => { - if (forwardWorkspaceStatus(commandId, args.workspaceId, line, stream)) return - if (!sink.line(formatLogLine(line))) return logStore.onDrain(logPath) - }, - (code, cliError) => { - if (code === 0) { - emitWorkspaceStatus(commandId, args.workspaceId, "stopping_workspace", "succeeded") - } else { - emitWorkspaceStatus( - commandId, - args.workspaceId, - "stopping_workspace", - "failed", - cliError ?? { - code: "workspace_stop_failed", - message: `workspace stop exited with code ${code}`, + const started = performance.now() + const timing = (stage: string) => + console.debug( + `[workspace-operation] ${commandId} ${stage} ${Math.round(performance.now() - started)}ms`, + ) + trackEvent( + `workspace_${{ stopping: "stop", deleting: "delete", rebuilding: "rebuild", resetting: "reset", starting: "up", creating: "create" }[activity]}`, + { workspace_ref: hashWorkspaceRef(args.workspaceId), source }, + ) + timing("accepted") + const completion = (async () => { + let sink: ProgressSink | undefined + try { + workspaceJobs.phase(args.workspaceId, commandId, "Closing connections") + await quiesceWorkspace(args.workspaceId) + timing("shutdown-complete") + const logPath = logStore.createLogFile( + state.workspaceContext(args.workspaceId), + args.workspaceId, + ) + sink = createLogSink( + deps.getMainWindow, + commandId, + (line) => logStore.appendLog(logPath, line), + () => logStore.closeLog(logPath), + ) + if (args.debug) cliArgs.push("--debug") + workspaceJobs.phase(args.workspaceId, commandId, "Launching command") + timing("cli-launch") + let firstProgress = true + await new Promise((resolve, reject) => { + void cli + .runStreaming( + cliArgs, + (line, stream) => { + if (!workspaceJobs.owns(args.workspaceId, commandId)) return + if (firstProgress) { + timing("first-progress") + firstProgress = false + } + if ( + forwardWorkspaceStatus( + commandId, + args.workspaceId, + line, + stream, + ) + ) + return + if (!sink!.line(formatLogLine(line))) + return logStore.onDrain(logPath) }, + (code, cliError) => { + if (code === 0) resolve() + else + reject( + Object.assign( + new Error( + cliError?.message ?? `Command exited with code ${code}`, + ), + { cliError }, + ), + ) + }, + args.workspaceId, ) - } - void sink - .done( - formatLogLine(`Exit code: ${code}`, code === 0 ? "INFO" : "ERROR"), - { success: code === 0 }, - ) - .then(() => { - if (code === 0) resolve() - else reject(new Error(`workspace stop exited with code ${code}`)) - }) .catch(reject) - }, - args.workspaceId, - ).catch((error) => { - const cliError = cliErrorOrFallback(error, "workspace_stop_failed") - emitWorkspaceStatus( - commandId, - args.workspaceId, - "stopping_workspace", - "failed", - cliError, + }) + timing("completed") + await sink + .done(formatLogLine("Completed"), { success: true }) + .catch((error) => + console.warn("[workspace-operation] log flush failed", error), + ) + await workspaceJobs.finish(args.workspaceId, generation) + timing("reconciled") + } catch (error) { + const cliError = redactCLIError( + cliErrorOrFallback(error, "workspace_operation_failed"), ) - void sink - .done(formatLogLine(cliError.message, "ERROR"), { - level: "error", - success: false, - cliError, - }) - .catch(() => {}) - .finally(() => reject(error)) - }) - }) - void completion - .then( - () => deps.onWorkspaceStopComplete?.(args.workspaceId), - () => deps.onWorkspaceStopComplete?.(args.workspaceId), - ) - .catch((error) => { - console.warn( - `[ipc] failed to reconcile workspace ${args.workspaceId}:`, - error, + if (sink) + await sink + .done(formatLogLine(cliError.message, "ERROR"), { + success: false, + level: "error", + cliError, + }) + .catch(() => {}) + await workspaceJobs.finish( + args.workspaceId, + generation, + cliError.message, ) - }) + throw error + } + })() + // IPC acknowledges acceptance. All subsequent errors live in the shared job. + void completion.catch(() => {}) return { commandId, completion } } - ipcMain.handle("workspace_stop", async (_event, args: StopWorkspaceArgs) => { - const { commandId, completion } = await startWorkspaceStop(args, "renderer") - void completion.catch(() => {}) - return commandId - }) - + function startWorkspaceStop( + args: StopWorkspaceArgs, + source: WorkspaceActionSource, + ) { + return startWorkspaceAction( + args, + "stopping", + ["workspace", "stop", args.workspaceId], + source, + ) + } + ipcMain.handle( + "workspace_stop", + (_event, args: StopWorkspaceArgs) => + startWorkspaceStop(args, "renderer").commandId, + ) ipcMain.handle( "workspace_delete", - async (_event, args: { workspaceId: string; debug?: boolean; commandId?: string }) => { - trackEvent("workspace_delete", { - workspace_ref: hashWorkspaceRef(args.workspaceId), - }) - // Replaces the old `devsy down` command which the CLI overhaul removed: - // before invoking delete, terminate every desktop-spawned child tied to - // this workspace and wait for them to actually exit. Otherwise late - // stdout/stderr lands on a log file the CLI is about to unlink, causing - // an ENOENT crash in the main process. - await quiesceWorkspace(args.workspaceId) - const cmdId = args.commandId ?? crypto.randomUUID() - const logPath = logStore.createLogFile( - state.workspaceContext(args.workspaceId), - args.workspaceId, - ) - const sink = createLogSink( - deps.getMainWindow, - cmdId, - (line) => logStore.appendLog(logPath, line), - () => logStore.closeLog(logPath), - ) - - const cliArgs = ["workspace", "delete", args.workspaceId] - if (args.debug) cliArgs.push("--debug") - cliArgs.push("--force") - - // The card shows "Deleting" until finish() below - const jobGeneration = workspaceJobs.start(args.workspaceId) - emitWorkspaceStatus(cmdId, args.workspaceId, "deleting_workspace", "started") - - void cli.runStreaming( - cliArgs, - (line, stream) => { - if (forwardWorkspaceStatus(cmdId, args.workspaceId, line, stream)) return - if (!sink.line(formatLogLine(line))) return logStore.onDrain(logPath) - }, - (code, cliError) => { - if (code === 0) { - emitWorkspaceStatus(cmdId, args.workspaceId, "deleting_workspace", "succeeded") - } else { - emitWorkspaceStatus( - cmdId, - args.workspaceId, - "deleting_workspace", - "failed", - cliError ?? { - code: "workspace_delete_failed", - message: `workspace delete exited with code ${code}`, - }, - ) - } - void sink.done( - formatLogLine(`Exit code: ${code}`, code === 0 ? "INFO" : "ERROR"), - { success: code === 0 }, - ) - void workspaceJobs.finish( - args.workspaceId, - jobGeneration, - code === 0 - ? undefined - : cliError?.message ?? `delete exited with code ${code}`, - ) - }, + (_event, args: StopWorkspaceArgs) => + startWorkspaceAction(args, "deleting", [ + "workspace", + "delete", args.workspaceId, - ).catch((error) => { - const cliError = cliErrorOrFallback(error, "workspace_delete_failed") - emitWorkspaceStatus( - cmdId, - args.workspaceId, - "deleting_workspace", - "failed", - cliError, - ) - void sink.done(formatLogLine(cliError.message, "ERROR"), { - level: "error", - success: false, - cliError, - }) - void workspaceJobs.finish(args.workspaceId, jobGeneration, cliError.message) - }) - - return cmdId - }, + "--force", + ]).commandId, ) - ipcMain.handle( "workspace_rebuild", - async (_event, args: { workspaceId: string; debug?: boolean; commandId?: string }) => { - trackEvent("workspace_rebuild", { - workspace_ref: hashWorkspaceRef(args.workspaceId), - }) - const cmdId = args.commandId ?? crypto.randomUUID() - const logPath = logStore.createLogFile( - state.workspaceContext(args.workspaceId), - args.workspaceId, - ) - const sink = createLogSink( - deps.getMainWindow, - cmdId, - (line) => logStore.appendLog(logPath, line), - () => logStore.closeLog(logPath), - ) - - const cliArgs = ["workspace", "up", args.workspaceId, "--recreate"] - if (args.debug) cliArgs.push("--debug") - let sawFailedStatus = false - - void cli.runStreaming( - cliArgs, - (line, stream) => { - const envelope = stream === "stdout" ? parseCliEnvelope(line) : undefined - if (envelope?.kind === "status" && envelope.state === "failed") { - sawFailedStatus = true - } - if (forwardWorkspaceStatus(cmdId, args.workspaceId, line, stream)) { - return - } - if (!sink.line(formatLogLine(line))) return logStore.onDrain(logPath) - }, - (code, cliError) => { - if (code !== 0 && !sawFailedStatus) { - emitWorkspaceStatus( - cmdId, - args.workspaceId, - "rebuilding_workspace", - "failed", - cliError ?? { - code: "workspace_rebuild_failed", - message: `workspace rebuild exited with code ${code}`, - }, - ) - } - void sink.done( - formatLogLine(`Exit code: ${code}`, code === 0 ? "INFO" : "ERROR"), - code === 0 - ? { success: true } - : { level: "error", success: false, cliError }, - ) - }, + (_event, args: StopWorkspaceArgs) => + startWorkspaceAction(args, "rebuilding", [ + "workspace", + "up", args.workspaceId, - ).catch((error) => { - const cliError = cliErrorOrFallback(error, "workspace_rebuild_failed") - emitWorkspaceStatus( - cmdId, - args.workspaceId, - "rebuilding_workspace", - "failed", - cliError, - ) - void sink.done(formatLogLine(cliError.message, "ERROR"), { - level: "error", - success: false, - cliError, - }) - }) - - return cmdId - }, + "--recreate", + ]).commandId, ) - ipcMain.handle( "workspace_reset", - async (_event, args: { workspaceId: string; debug?: boolean; commandId?: string }) => { - trackEvent("workspace_reset", { - workspace_ref: hashWorkspaceRef(args.workspaceId), - }) - const cmdId = args.commandId ?? crypto.randomUUID() - const logPath = logStore.createLogFile( - state.workspaceContext(args.workspaceId), - args.workspaceId, - ) - const sink = createLogSink( - deps.getMainWindow, - cmdId, - (line) => logStore.appendLog(logPath, line), - () => logStore.closeLog(logPath), - ) - - const cliArgs = ["workspace", "up", args.workspaceId, "--reset"] - if (args.debug) cliArgs.push("--debug") - let sawFailedStatus = false - - void cli.runStreaming( - cliArgs, - (line, stream) => { - const envelope = stream === "stdout" ? parseCliEnvelope(line) : undefined - if (envelope?.kind === "status" && envelope.state === "failed") { - sawFailedStatus = true - } - if (forwardWorkspaceStatus(cmdId, args.workspaceId, line, stream)) { - return - } - if (!sink.line(formatLogLine(line))) return logStore.onDrain(logPath) - }, - (code, cliError) => { - if (code !== 0 && !sawFailedStatus) { - emitWorkspaceStatus( - cmdId, - args.workspaceId, - "resetting_workspace", - "failed", - cliError ?? { - code: "workspace_reset_failed", - message: `workspace reset exited with code ${code}`, - }, - ) - } - void sink.done( - formatLogLine(`Exit code: ${code}`, code === 0 ? "INFO" : "ERROR"), - code === 0 - ? { success: true } - : { level: "error", success: false, cliError }, - ) - }, + (_event, args: StopWorkspaceArgs) => + startWorkspaceAction(args, "resetting", [ + "workspace", + "up", args.workspaceId, - ).catch((error) => { - const cliError = cliErrorOrFallback(error, "workspace_reset_failed") - emitWorkspaceStatus( - cmdId, - args.workspaceId, - "resetting_workspace", - "failed", - cliError, - ) - void sink.done(formatLogLine(cliError.message, "ERROR"), { - level: "error", - success: false, - cliError, - }) - }) - - return cmdId - }, + "--reset", + ]).commandId, ) // ── Terminal ── diff --git a/desktop/src/main/machine-diagnostics-manager.ts b/desktop/src/main/machine-diagnostics-manager.ts new file mode 100644 index 0000000000..d9982d4414 --- /dev/null +++ b/desktop/src/main/machine-diagnostics-manager.ts @@ -0,0 +1,66 @@ +import type { MachineDiagnosticsCache, MachineDiagnosticsResponse } from "../shared/machine-diagnostics-types.js" +import type { CliRunner } from "./cli.js" +import { MachineDiagnosticsStore, type MachineDiagnosticsKey } from "./machine-diagnostics-store.js" + +function keyOf(key: MachineDiagnosticsKey): string { + return JSON.stringify([key.context, key.machineId]) +} + +export class MachineDiagnosticsManager { + private readonly inFlight = new Map>() + private readonly invalidated = new Set() + + constructor( + private readonly cli: CliRunner, + private readonly store: MachineDiagnosticsStore, + ) {} + + getCached(key: MachineDiagnosticsKey): MachineDiagnosticsCache | null { + return this.store.get(key) + } + + refresh(key: MachineDiagnosticsKey): Promise { + const id = keyOf(key) + const active = this.inFlight.get(id) + if (active) return active + this.invalidated.delete(id) + + const request = this.collect(key).finally(() => { + this.inFlight.delete(id) + this.invalidated.delete(id) + }) + this.inFlight.set(id, request) + return request + } + + markStopped(key: MachineDiagnosticsKey): MachineDiagnosticsCache | null { + if (this.inFlight.has(keyOf(key))) this.invalidated.add(keyOf(key)) + return this.store.markStopped(key) + } + + delete(key: MachineDiagnosticsKey): void { + if (this.inFlight.has(keyOf(key))) this.invalidated.add(keyOf(key)) + this.store.delete(key) + } + + private async collect(key: MachineDiagnosticsKey): Promise { + const cached = this.store.get(key) + const args = ["machine", "diagnostics", key.machineId, "--context", key.context, "--result-format", "json", "--limit", "200"] + if (cached?.cursor) args.push("--after", cached.cursor) + + try { + const response = JSON.parse(await this.cli.runRaw(args)) as MachineDiagnosticsResponse + if (this.invalidated.has(keyOf(key))) throw new Error("Diagnostics collection superseded by a machine lifecycle change.") + if (response.schemaVersion !== 1 || response.machine?.id !== key.machineId || response.machine?.context !== key.context || !response.source || !response.cursor) { + throw new Error("Remote diagnostics returned an invalid or mismatched response.") + } + return this.store.merge(key, response) + } catch (error) { + if (this.invalidated.has(keyOf(key))) throw error + const message = error instanceof Error ? error.message : "Remote diagnostics collection failed." + const retained = this.store.recordFailure(key, message) + if (retained) return retained + throw error + } + } +} diff --git a/desktop/src/main/machine-diagnostics-store.ts b/desktop/src/main/machine-diagnostics-store.ts new file mode 100644 index 0000000000..9dc17d1f71 --- /dev/null +++ b/desktop/src/main/machine-diagnostics-store.ts @@ -0,0 +1,100 @@ +import { randomUUID } from "node:crypto" +import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs" +import { basename, join } from "node:path" +import type { MachineDiagnosticEvent, MachineDiagnosticsCache, MachineDiagnosticsResponse } from "../shared/machine-diagnostics-types.js" + +export interface MachineDiagnosticsKey { + context: string + machineId: string +} + +function safe(value: string): string { + const clean = basename(value) + if (!clean || clean === "." || clean === ".." || clean !== value) throw new Error("invalid diagnostics key") + return clean +} + +function eventKey(event: MachineDiagnosticEvent): string { + return `${event.sessionId}:${event.sequence}` +} + +export class MachineDiagnosticsStore { + constructor(private readonly root: string) {} + private path(key: MachineDiagnosticsKey): string { + return join(this.root, "machines", safe(key.context), `${safe(key.machineId)}.json`) + } + + get(key: MachineDiagnosticsKey): MachineDiagnosticsCache | null { + const path = this.path(key) + try { + const cache = JSON.parse(readFileSync(path, "utf-8")) as MachineDiagnosticsCache + if (cache.response?.schemaVersion !== 1 || !Array.isArray(cache.events)) return null + return cache + } catch { + return null + } + } + merge(key: MachineDiagnosticsKey, response: MachineDiagnosticsResponse): MachineDiagnosticsCache { + if (response.source.availability !== "available") { + const retained = this.get(key) + if (retained) { + retained.response.source = { ...response.source, freshness: "stale" } + retained.response.machine = response.machine + retained.lastAttemptAt = new Date().toISOString() + retained.lastCollectionError = response.source.availability === "machine_stopped" ? undefined : (response.source.message ?? `Diagnostics are ${response.source.availability}.`) + this.write(key, retained) + return retained + } + } + const existing = this.get(key) + const events = [...(existing?.events ?? []), ...(response.events ?? [])] + const unique = new Map(events.map(event => [eventKey(event), event])) + const merged = [...unique.values()].sort((a, b) => a.timestamp.localeCompare(b.timestamp) || a.sequence - b.sequence).slice(-2000) + const { events: _events, ...responseWithoutEvents } = response + const now = new Date().toISOString() + const cache: MachineDiagnosticsCache = { + response: responseWithoutEvents, + events: merged, + cursor: response.cursor.state === "reset" ? undefined : (response.cursor.next ?? existing?.cursor), + lastAttemptAt: now, + lastSuccessfulCollectionAt: response.source.availability === "available" ? now : existing?.lastSuccessfulCollectionAt, + historyGap: Boolean(existing?.historyGap || response.cursor.state === "gap"), + } + this.write(key, cache) + return cache + } + markStopped(key: MachineDiagnosticsKey): MachineDiagnosticsCache | null { + const cache = this.get(key) + if (!cache) return null + cache.response.source = { availability: "machine_stopped", freshness: "unknown" } + cache.lastCollectionError = undefined + this.write(key, cache) + return cache + } + + recordFailure(key: MachineDiagnosticsKey, message: string): MachineDiagnosticsCache | null { + const cache = this.get(key) + if (!cache) return null + cache.lastAttemptAt = new Date().toISOString() + cache.lastCollectionError = message + cache.response.source.freshness = "stale" + this.write(key, cache) + return cache + } + + delete(key: MachineDiagnosticsKey): void { + const path = this.path(key) + if (existsSync(path)) unlinkSync(path) + } + private write(key: MachineDiagnosticsKey, cache: MachineDiagnosticsCache): void { + const path = this.path(key) + mkdirSync(join(this.root, "machines", safe(key.context)), { recursive: true, mode: 0o700 }) + const tmp = `${path}.${randomUUID()}.tmp` + try { + writeFileSync(tmp, JSON.stringify(cache), { mode: 0o600 }) + renameSync(tmp, path) + } finally { + if (existsSync(tmp)) unlinkSync(tmp) + } + } +} diff --git a/desktop/src/main/tray.ts b/desktop/src/main/tray.ts index 61e11123e9..67471dfb55 100644 --- a/desktop/src/main/tray.ts +++ b/desktop/src/main/tray.ts @@ -1,3 +1,10 @@ +import { + workspaceJobBusy, + workspaceJobInterruptible, + workspaceJobLabel, + type WorkspaceJob, +} from "../shared/workspace-operation.js" +import type { WorkspaceJobs } from "./workspace-jobs.js" import { join } from "node:path" import { app, Menu, nativeImage, nativeTheme, Tray } from "electron" import type { DaemonState, Workspace } from "./state.js" @@ -22,14 +29,12 @@ export function buildUpdateMenuItems( : version ? `Update to ${version}` : "Restart" - return [ - { label, click: onInstall }, - { type: "separator" }, - ] + return [{ label, click: onInstall }, { type: "separator" }] } export interface TrayMenuModel { activeWorkspaces: Workspace[] + jobs?: Record pendingStops: ReadonlySet updateStatus: UpdateStatus } @@ -51,19 +56,25 @@ export function buildTrayMenuTemplate( const workspaceItems: Electron.MenuItemConstructorOptions[] = active .slice(0, 10) .map((workspace) => { - const pending = model.pendingStops.has(workspace.id) + const job = model.jobs?.[workspace.id] + const pending = + model.pendingStops.has(workspace.id) || workspaceJobBusy(job) const busy = workspace.status?.trim().toLowerCase() === "busy" + const disabled = pending && !workspaceJobInterruptible(job) + const label = workspaceJobLabel(job) return { - label: `${workspace.id}${busy && !pending ? " — Busy" : ""}`, + label: `${workspace.id}${label ? ` — ${label}` : busy && !pending ? " — Busy" : ""}`, submenu: [ { label: "Open in Devsy", click: () => actions.showWorkspace(workspace.id), }, { - label: pending ? "Stopping…" : "Stop Workspace", - enabled: !pending, - click: pending + label: disabled + ? `${workspaceJobLabel(job) ?? "Stopping"}…` + : "Stop Workspace", + enabled: !disabled, + click: disabled ? undefined : () => actions.stopWorkspace(workspace.id), }, @@ -101,6 +112,7 @@ export function buildTrayMenuTemplate( } interface TrayDeps { + workspaceJobs?: WorkspaceJobs state: DaemonState showDevsy: (route?: string) => void stopWorkspace: (workspaceId: string) => Promise @@ -112,6 +124,7 @@ export class AppTray { private tray: Tray | null = null private pendingStops = new Set() private unsubscribeWorkspaceState: (() => void) | null = null + private unsubscribeWorkspaceJobs: (() => void) | null = null private unsubscribeUpdateStatus: (() => void) | null = null private readonly onThemeUpdated = (): void => { this.tray?.setImage(this.createTrayIcon()) @@ -126,6 +139,8 @@ export class AppTray { this.unsubscribeWorkspaceState = this.deps.state.onWorkspacesChange(() => this.rebuildMenu(), ) + this.unsubscribeWorkspaceJobs = + this.deps.workspaceJobs?.onChange(() => this.rebuildMenu()) ?? null this.unsubscribeUpdateStatus = onUpdateStatusChanged(() => this.rebuildMenu(), ) @@ -136,6 +151,8 @@ export class AppTray { } destroy(): void { + this.unsubscribeWorkspaceJobs?.() + this.unsubscribeWorkspaceJobs = null this.unsubscribeWorkspaceState?.() this.unsubscribeWorkspaceState = null this.unsubscribeUpdateStatus?.() @@ -158,21 +175,31 @@ export class AppTray { return icon } const variant = nativeTheme.shouldUseDarkColors ? "dark" : "light" - return nativeImage.createFromPath( - join(trayDir, `icon-tray-${variant}.png`), - ) + return nativeImage.createFromPath(join(trayDir, `icon-tray-${variant}.png`)) } private rebuildMenu(): void { if (!this.tray) return - const activeWorkspaces = this.deps.state - .workspaceList() - .filter((workspace) => isActiveWorkspaceStatus(workspace.status)) + const jobs = this.deps.workspaceJobs?.snapshot() ?? {} + const workspaces = this.deps.state.workspaceList() + const activeWorkspaces = workspaces.filter( + (workspace) => + isActiveWorkspaceStatus(workspace.status) || + workspaceJobBusy(jobs[workspace.id]), + ) + for (const [id, job] of Object.entries(jobs)) { + if ( + workspaceJobBusy(job) && + !workspaces.some((workspace) => workspace.id === id) + ) + activeWorkspaces.push({ id }) + } const template = buildTrayMenuTemplate( { activeWorkspaces, pendingStops: this.pendingStops, + jobs: this.deps.workspaceJobs?.snapshot(), updateStatus: getLastStatus(), }, { diff --git a/desktop/src/main/watcher.ts b/desktop/src/main/watcher.ts index d2779399b4..d51f9361e6 100644 --- a/desktop/src/main/watcher.ts +++ b/desktop/src/main/watcher.ts @@ -61,6 +61,7 @@ export class Watcher { private pollTimer: ReturnType | null = null private workspaceStatusTimer: ReturnType | null = null private fsWatcher: ReturnType | null = null + private workspaceRevision = 0 private polling = false private pollQueued = false // Serializes pollProviders so a manual refreshProviders() can never @@ -131,6 +132,12 @@ export class Watcher { this.pollMachines(), this.pollContexts(), ]) + for (const [id, job] of Object.entries( + this.deps.workspaceJobs.snapshot(), + )) { + if (job.state === "reconciling" && job.refreshError) + void this.deps.workspaceJobs.retryRefresh(id) + } } finally { this.polling = false if (this.pollQueued) { @@ -156,11 +163,11 @@ export class Watcher { /** Re-read the workspace list from disk now, without waiting for the next scheduled poll. */ async refreshWorkspaces(): Promise { - await this.queueWorkspacePoll() + await this.queueWorkspacePoll(true) } async refreshWorkspaceStatus(workspaceId: string): Promise { - await this.queueWorkspaceStatusPoll([workspaceId]) + await this.queueWorkspaceStatusPoll([workspaceId], true) } async refreshWorkspaceStatuses(): Promise { @@ -175,24 +182,32 @@ export class Watcher { void this.refreshWorkspaceStatuses() } - private queueWorkspaceStatusPoll(workspaceIds?: string[]): Promise { + private queueWorkspaceStatusPoll( + workspaceIds?: string[], + strict = false, + ): Promise { const run = this.workspaceStatusChain.then(() => - this.pollWorkspaceStatuses(workspaceIds), + this.pollWorkspaceStatuses(workspaceIds, strict), ) - this.workspaceStatusChain = run + this.workspaceStatusChain = run.catch(() => {}) return run } - private async pollWorkspaceStatuses(workspaceIds?: string[]): Promise { + private async pollWorkspaceStatuses( + workspaceIds?: string[], + strict = false, + ): Promise { this.workspaceStatusPolling = true let changed = false try { - const ids = workspaceIds ?? this.deps.state.workspaceList().map((ws) => ws.id) + const ids = + workspaceIds ?? this.deps.state.workspaceList().map((ws) => ws.id) const concurrency = 6 let next = 0 const worker = async (): Promise => { while (next < ids.length) { const id = ids[next++] + const generation = this.deps.workspaceJobs.generation(id) try { const raw = await this.deps.cli.runRaw([ "workspace", @@ -203,11 +218,19 @@ export class Watcher { "--timeout", "15s", ]) + if (generation !== this.deps.workspaceJobs.generation(id)) { + if (strict) + throw new Error("Workspace operation changed during refresh") + continue + } const status = normalizeWorkspaceStatus(raw) + if (!status && strict) + throw new Error("Workspace status unavailable") if (status && this.deps.state.updateWorkspaceStatus(id, status)) { changed = true } - } catch { + } catch (error) { + if (strict) throw error // Preserve the last known status and retry on the next sweep. } } @@ -225,13 +248,14 @@ export class Watcher { } } - private queueWorkspacePoll(): Promise { - const run = this.workspacePollChain.then(() => this.pollWorkspaces()) - this.workspacePollChain = run + private queueWorkspacePoll(strict = false): Promise { + const run = this.workspacePollChain.then(() => this.pollWorkspaces(strict)) + this.workspacePollChain = run.catch(() => {}) return run } - private async pollWorkspaces(): Promise { + private async pollWorkspaces(strict = false): Promise { + const generation = this.deps.workspaceJobs.epoch try { const workspaces = await this.queryWithFallback( this.deps.daemon @@ -239,6 +263,11 @@ export class Watcher { : undefined, () => this.deps.cli.run(["workspace", "list", "--skip-pro"]), ) + if (generation !== this.deps.workspaceJobs.epoch) { + if (strict) + throw new Error("Workspace operation changed during refresh") + return + } const changed = this.deps.state.updateWorkspaces(workspaces as any[]) if (changed) { this.broadcastWorkspaces() @@ -247,6 +276,7 @@ export class Watcher { // Polling is best-effort, but retain diagnostics so a broken CLI or // daemon cannot make the desktop appear empty without explanation. console.warn("[watcher] workspace poll failed", error) + if (strict) throw error } } @@ -256,10 +286,15 @@ export class Watcher { * workspace as idle between the delete finishing and the list catching up. */ broadcastWorkspaces(): void { - this.send("workspaces-changed", { + this.send("workspaces-changed", this.workspaceSnapshot()) + } + + workspaceSnapshot() { + return { + revision: ++this.workspaceRevision, workspaces: this.deps.state.workspaceList(), jobs: this.deps.workspaceJobs.snapshot(), - }) + } } /** Re-read provider state from disk now, without waiting for the next scheduled poll. */ diff --git a/desktop/src/main/workspace-jobs.ts b/desktop/src/main/workspace-jobs.ts index 141dfdd60a..522e01e5b5 100644 --- a/desktop/src/main/workspace-jobs.ts +++ b/desktop/src/main/workspace-jobs.ts @@ -1,102 +1,133 @@ -/** - * Tracks in-flight workspace delete operations in the main process, the - * same way ProviderJobs tracks provider install/init work: workspace_delete - * is fire-and-forget from the IPC handler's perspective (it returns as soon - * as the CLI command is launched, so the log-streaming UI isn't blocked), - * so nothing else records that a delete is running. Main-process ownership - * means the "Deleting" badge survives navigating away from the list or - * reloading the window while the delete is still in flight. - */ - -export interface WorkspaceJob { - activity: "deleting" - /** Set when the job ended in failure; the job is retained so the UI can show why. */ - error?: string -} +import type { OperationStatus } from "../shared/cli-error.js" +import { + type WorkspaceActivity, + type WorkspaceJob, + workspaceJobBusy, +} from "../shared/workspace-operation.js" +export type { WorkspaceJob } from "../shared/workspace-operation.js" +/** Main-owned actions are independent of the last observed runtime status. */ export class WorkspaceJobs { private jobs = new Map() - // Lets an in-flight finish() tell whether it still owns the entry. private generations = new Map() private lastGeneration = 0 private listeners = new Set<() => void>() - private refresh?: () => Promise + private refreshing = new Set() + private refresh?: (id: string, job: WorkspaceJob) => Promise + revision = 0 - /** - * Register a callback fired after every mutation, so starting or - * finishing a delete reaches the renderer without waiting for the next - * disk poll. - */ onChange(listener: () => void): () => void { this.listeners.add(listener) return () => this.listeners.delete(listener) } - private emit(): void { + this.revision++ for (const listener of this.listeners) listener() } + get epoch(): number { + return this.lastGeneration + } + generation(id: string): number { + return this.generations.get(id) ?? 0 + } + owns(id: string, commandId: string): boolean { + return this.jobs.get(id)?.commandId === commandId + } - /** - * Begin tracking a delete, clearing any previous failure. Returns a - * generation token the caller must pass to finish() so a stale exit - * callback from an earlier delete of the same workspace can't land on - * a retry that has already superseded it. - */ - start(id: string): number { - this.jobs.set(id, { activity: "deleting" }) + start( + id: string, + activity: WorkspaceActivity = "deleting", + commandId: string = crypto.randomUUID(), + ): number { + const current = this.jobs.get(id) + const canInterrupt = + (activity === "stopping" || activity === "deleting") && + (current?.activity === "starting" || current?.activity === "creating") && + current.state === "running" + if (workspaceJobBusy(current) && !current?.error && !canInterrupt) + throw new Error("A workspace operation is already in progress") const generation = ++this.lastGeneration this.generations.set(id, generation) + this.jobs.set(id, { + commandId, + activity, + state: "running", + phase: "Preparing", + }) this.emit() return generation } - - /** Stop tracking a workspace, discarding any recorded failure. */ + phase(id: string, commandId: string, phase: string): void { + const job = this.jobs.get(id) + if (!job || job.commandId !== commandId || job.state !== "running") return + this.jobs.set(id, { ...job, phase }) + this.emit() + } + progress(id: string, commandId: string, status: OperationStatus): void { + const job = this.jobs.get(id) + if (!job || job.commandId !== commandId || job.state !== "running") return + // A child finishing does not finish its parent or the command. + const phase = + status.state === "started" + ? status.step || status.phase.replaceAll("_", " ") + : job.phase + this.jobs.set(id, { ...job, status, phase }) + this.emit() + } clear(id: string): void { - this.generations.delete(id) + this.generations.set(id, ++this.lastGeneration) if (this.jobs.delete(id)) this.emit() } - - /** - * Finish the job started under `generation`. A failure is retained so the - * UI can explain it; success drops the entry once the workspace list has - * caught up, so the card doesn't flash back to its pre-delete state for - * one poll cycle. Either way, a generation mismatch means a retry has - * already superseded this job, so the call is a no-op. - */ async finish(id: string, generation: number, error?: string): Promise { - if (this.generations.get(id) !== generation) return - - if (error) { - this.jobs.set(id, { activity: "deleting", error }) - this.emit() - return - } const job = this.jobs.get(id) - if (!job) return - if (job.error) return - - await this.refresh?.() - // refresh is a CLI round-trip; a newer job may own the entry by now. - if (this.generations.get(id) !== generation) return - - this.jobs.delete(id) - this.generations.delete(id) + if (!job || this.generation(id) !== generation || job.state !== "running") + return + this.generations.set(id, ++this.lastGeneration) // invalidate pre-completion polls + this.jobs.set(id, { + ...job, + state: "reconciling", + phase: + job.activity === "deleting" ? "Refreshing list" : "Refreshing status", + error, + }) this.emit() + await this.retryRefresh(id) } - - /** - * Supplies a way to re-read the workspace list from disk, so a finished - * job isn't cleared before the list reflects the deletion. - */ - setRefresh(refresh: () => Promise): void { + async retryRefresh(id: string): Promise { + const job = this.jobs.get(id) + if ( + !job || + job.state !== "reconciling" || + this.refreshing.has(job.commandId) + ) + return + this.refreshing.add(job.commandId) + try { + await this.refresh?.(id, job) + if (this.jobs.get(id) !== job) return + this.jobs.set(id, { + ...job, + state: job.error ? "failed" : "succeeded", + phase: job.error ? "See workspace logs for details" : "", + refreshError: undefined, + }) + } catch (error) { + if (this.jobs.get(id) !== job) return + this.jobs.set(id, { + ...job, + refreshError: error instanceof Error ? error.message : String(error), + }) + } finally { + this.refreshing.delete(job.commandId) + this.emit() + } + } + setRefresh(refresh: (id: string, job: WorkspaceJob) => Promise): void { this.refresh = refresh } - get(id: string): WorkspaceJob | undefined { return this.jobs.get(id) } - - /** Snapshot for IPC, keyed by workspace id. */ snapshot(): Record { return Object.fromEntries(this.jobs) } diff --git a/desktop/src/renderer/src/lib/components/ErrorCard.svelte b/desktop/src/renderer/src/lib/components/ErrorCard.svelte index 8cb56a55ed..9c21d0bd4a 100644 --- a/desktop/src/renderer/src/lib/components/ErrorCard.svelte +++ b/desktop/src/renderer/src/lib/components/ErrorCard.svelte @@ -1,6 +1,6 @@ +
+ + {#if workspaceJobBusy(job)}{/if} + {label ?? status ?? "Checking"} + + {#if phase}{phase}{/if} + {#if job?.error}{job.error}{/if} + {#if job?.refreshError} + + {/if} +
diff --git a/desktop/src/renderer/src/lib/components/workspace/WorkspaceOperation.test.ts b/desktop/src/renderer/src/lib/components/workspace/WorkspaceOperation.test.ts new file mode 100644 index 0000000000..7086c9e37f --- /dev/null +++ b/desktop/src/renderer/src/lib/components/workspace/WorkspaceOperation.test.ts @@ -0,0 +1,58 @@ +import { cleanup, fireEvent, render, waitFor } from "@testing-library/svelte" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { mockInvoke, resetTauriMocks } from "$lib/__mocks__/tauri.js" +import { workspaceJobs } from "$lib/stores/workspaces.js" +import { toasts } from "$lib/stores/toasts.js" +import WorkspaceOperation from "./WorkspaceOperation.svelte" +vi.mock("$lib/stores/toasts.js", () => ({ + toasts: { success: vi.fn(), error: vi.fn() }, +})) +afterEach(cleanup) +beforeEach(() => { + resetTauriMocks() + workspaceJobs.set({}) + vi.clearAllMocks() +}) +describe("WorkspaceOperation", () => { + it("keeps the action visible over a stale runtime observation", () => { + workspaceJobs.set({ + ws: { + commandId: "delete", + activity: "deleting", + state: "running", + phase: "Closing connections", + }, + }) + const ui = render(WorkspaceOperation, { id: "ws", status: "Running" }) + expect(ui.getByText("Deleting")).toBeTruthy() + expect(ui.getByText("Closing connections")).toBeTruthy() + expect(ui.queryByText("Running")).toBeNull() + }) + it("retries observation only and reports an IPC rejection", async () => { + workspaceJobs.set({ + ws: { + commandId: "delete", + activity: "deleting", + state: "reconciling", + phase: "Refreshing list", + refreshError: "offline", + }, + }) + mockInvoke.mockRejectedValue(new Error("IPC unavailable")) + const ui = render(WorkspaceOperation, { id: "ws", status: "Running" }) + expect(ui.getByText("Deleted")).toBeTruthy() + await fireEvent.click(ui.getByRole("button", { name: "Retry refresh" })) + await waitFor(() => + expect(toasts.error).toHaveBeenCalledWith( + expect.stringContaining("IPC unavailable"), + ), + ) + expect(mockInvoke).toHaveBeenCalledWith("workspace_refresh", { + workspaceId: "ws", + }) + expect( + mockInvoke.mock.calls.some((call) => call[0] === "workspace_delete"), + ).toBe(false) + expect(ui.getByText("Deleted")).toBeTruthy() + }) +}) diff --git a/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.platform.test.ts b/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.platform.test.ts index 8e7307f611..c1bb653dcf 100644 --- a/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.platform.test.ts +++ b/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.platform.test.ts @@ -37,7 +37,7 @@ vi.mock("$lib/stores/providers.js", async () => { }) vi.mock("$lib/stores/workspaces.js", async () => { const { writable } = await import("svelte/store") - return { workspaces: writable<{ id: string }[]>([]) } + return { workspaces: writable<{ id: string }[]>([]), workspaceJobs: writable({}) } }) vi.mock("$lib/stores/toasts.js", () => ({ toasts: { success: vi.fn(), error: vi.fn(), info: vi.fn() }, diff --git a/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.svelte b/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.svelte index 21043f118a..76afd320f2 100644 --- a/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.svelte +++ b/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.svelte @@ -1,4 +1,6 @@