From 1ca07204d8f962c20decd4e51b8773485d42b810 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 25 Oct 2026 11:17:17 -0600 Subject: [PATCH 01/19] fix: stop idle Kubernetes workspaces and recreate terminal pods (#1698) --- cmd/internal/agentcontainer/daemon.go | 116 ++- cmd/internal/agentcontainer/deferred_hooks.go | 3 + cmd/internal/agentcontainer/lifecycle.go | 115 +++ cmd/internal/agentcontainer/lifecycle_test.go | 217 ++++++ cmd/internal/agentcontainer/post_attach.go | 3 + cmd/internal/agentcontainer/setup.go | 32 +- cmd/internal/agentcontainer/ssh_server.go | 58 +- cmd/internal/ssh_server.go | 54 +- cmd/internal/ssh_server_test.go | 25 + .../up/provider_kubernetes_inactivity.go | 422 ++++++++++ hack/ci/suites.yaml | 1 + pkg/config/paths.go | 3 + pkg/daemon/agent/activity.go | 131 ++++ pkg/daemon/agent/container_lifecycle.go | 166 ++++ pkg/daemon/agent/container_lifecycle_test.go | 158 ++++ pkg/daemon/agent/daemon.go | 48 +- pkg/daemon/agent/daemon_test.go | 1 + pkg/daemon/agent/lifecycle_monitor.go | 198 +++++ pkg/daemon/agent/lifecycle_monitor_test.go | 244 ++++++ pkg/daemon/agent/lifecycle_proc_linux.go | 30 + pkg/daemon/agent/lifecycle_proc_other.go | 5 + pkg/daemon/agent/lifecycle_protocol.go | 346 +++++++++ pkg/daemon/agent/lifecycle_protocol_test.go | 158 ++++ pkg/daemon/agent/lifecycle_unix.go | 27 + pkg/daemon/agent/lifecycle_windows.go | 14 + pkg/devcontainer/managed_lifecycle_test.go | 196 +++++ pkg/devcontainer/setup.go | 15 +- pkg/devcontainer/setup_secrets_test.go | 3 +- pkg/devcontainer/single.go | 103 ++- pkg/driver/kubernetes/client.go | 4 +- pkg/driver/kubernetes/find.go | 15 +- pkg/driver/kubernetes/pod_lifecycle.go | 365 +++++++++ pkg/driver/kubernetes/pod_lifecycle_test.go | 724 ++++++++++++++++++ pkg/driver/kubernetes/run.go | 106 +-- pkg/driver/kubernetes/wait.go | 39 +- pkg/driver/types.go | 6 + pkg/provider/provider.go | 12 +- pkg/provider/workspace.go | 15 +- providers/kubernetes/provider.yaml | 3 +- .../inactivity-timeout.mdx | 40 +- .../docs/developing-providers/agent.mdx | 6 +- .../docs/developing-providers/driver.mdx | 2 +- 42 files changed, 4006 insertions(+), 223 deletions(-) create mode 100644 cmd/internal/agentcontainer/lifecycle.go create mode 100644 cmd/internal/agentcontainer/lifecycle_test.go create mode 100644 e2e/tests/up/provider_kubernetes_inactivity.go create mode 100644 pkg/daemon/agent/activity.go create mode 100644 pkg/daemon/agent/container_lifecycle.go create mode 100644 pkg/daemon/agent/container_lifecycle_test.go create mode 100644 pkg/daemon/agent/lifecycle_monitor.go create mode 100644 pkg/daemon/agent/lifecycle_monitor_test.go create mode 100644 pkg/daemon/agent/lifecycle_proc_linux.go create mode 100644 pkg/daemon/agent/lifecycle_proc_other.go create mode 100644 pkg/daemon/agent/lifecycle_protocol.go create mode 100644 pkg/daemon/agent/lifecycle_protocol_test.go create mode 100644 pkg/daemon/agent/lifecycle_unix.go create mode 100644 pkg/daemon/agent/lifecycle_windows.go create mode 100644 pkg/devcontainer/managed_lifecycle_test.go create mode 100644 pkg/driver/kubernetes/pod_lifecycle.go create mode 100644 pkg/driver/kubernetes/pod_lifecycle_test.go diff --git a/cmd/internal/agentcontainer/daemon.go b/cmd/internal/agentcontainer/daemon.go index 41d017153c..8c4c35a355 100644 --- a/cmd/internal/agentcontainer/daemon.go +++ b/cmd/internal/agentcontainer/daemon.go @@ -77,16 +77,30 @@ func (cmd *DaemonCmd) Run(c *cobra.Command, args []string) error { ctx, stop := signal.NotifyContext(c.Context(), os.Interrupt, syscall.SIGTERM) g, ctx := errgroup.WithContext(ctx) - cmd.startDaemonTasks(ctx, g, timeoutDuration) + if err := cmd.startDaemonTasks(ctx, g, timeoutDuration); err != nil { + stop() + return err + } err = g.Wait() stop() // Restore default signal handling before exiting. - if err != nil { + os.Exit(daemonExitCode(err)) + return nil // Unreachable but needed. +} + +// daemonExitCode maps the task group result to a process exit code. Inactivity +// expiry is the normal way a managed container stops, not a failure. +func daemonExitCode(err error) int { + switch { + case err == nil: + return 0 + case errors.Is(err, agentd.ErrInactivityExpired): + log.Infof("inactivity timeout reached, daemon exiting") + return 0 + default: log.Errorf("daemon error: %v", err) - os.Exit(1) + return 1 } - os.Exit(0) - return nil // Unreachable but needed. } func (cmd *DaemonCmd) setupTimeout() (time.Duration, error) { @@ -111,10 +125,11 @@ func (cmd *DaemonCmd) startDaemonTasks( ctx context.Context, g *errgroup.Group, timeoutDuration time.Duration, -) { +) error { var tasksStarted bool - // Start process reaper. + // Start process reaper only when this process really is the init process of + // its PID namespace; otherwise it would steal other processes' children. if os.Getpid() == 1 { g.Go(func() error { agentd.RunProcessReaper(ctx) @@ -130,11 +145,14 @@ func (cmd *DaemonCmd) startDaemonTasks( }) } - // Start timeout monitor unless shutdownAction is "none". - if timeoutDuration > 0 && cmd.Config.ShutdownAction != config.ShutdownActionNone { + loop, err := cmd.newLifecycleLoop(timeoutDuration) + if err != nil { + return err + } + if loop != nil { tasksStarted = true g.Go(func() error { - return runTimeoutMonitor(ctx, timeoutDuration) + return loop.Run(ctx) }) } @@ -153,6 +171,36 @@ func (cmd *DaemonCmd) startDaemonTasks( return nil }) } + return nil +} + +// newLifecycleLoop builds the inactivity loop. A managed daemon starts unarmed +// and waits for setup; any other daemon with a timeout is armed immediately. +func (cmd *DaemonCmd) newLifecycleLoop(timeout time.Duration) (*agentd.LifecycleLoop, error) { + monitor := &agentd.LifecycleMonitor{} + loop := &agentd.LifecycleLoop{Monitor: monitor, ActivityPath: config2.ContainerActivityFile} + enabled := timeout > 0 && cmd.Config.ShutdownAction != config.ShutdownActionNone + + if !cmd.Config.Lifecycle.Managed() { + if !enabled { + return nil, nil + } + monitor.ArmNow(timeout, time.Now()) + return loop, nil + } + + owner, err := agentd.NewLifecycleOwner(config2.ContainerLifecycleDir) + if err != nil { + return nil, fmt.Errorf("publish container lifecycle owner: %w", err) + } + if err := sharedfile.EnsureMode(config2.ContainerActivityFile, 0o666); err != nil { + return nil, fmt.Errorf("ensure activity file: %w", err) + } + loop.Owner = owner + if enabled && !cmd.Config.Lifecycle.SetupArmsTimer { + monitor.ArmNow(timeout, time.Now()) + } + return loop, nil } // loadConfig loads the daemon configuration from base64-encoded JSON. @@ -205,29 +253,6 @@ func (cmd *DaemonCmd) shouldRunSsh() bool { return cmd.Config.Ssh.Workdir != "" || cmd.Config.Ssh.User != "" } -// runTimeoutMonitor monitors the activity file and signals an error if the timeout is exceeded. -func runTimeoutMonitor( - ctx context.Context, - duration time.Duration, -) error { - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return nil - case <-ticker.C: - stat, err := os.Stat(config2.ContainerActivityFile) - if err != nil { - continue - } - if !stat.ModTime().Add(duration).After(time.Now()) { - return errors.New("timeout reached, terminating daemon") - } - } - } -} - // runNetworkServer starts the network server. func runNetworkServer( ctx context.Context, @@ -260,6 +285,23 @@ func runNetworkServer( return nil } +// sshServerArgs builds the embedded SSH server command line. A managed daemon +// asks it to record connection activity so Platform and direct SSH routes keep +// the workspace alive. +func sshServerArgs(cfg *agentd.DaemonConfig) []string { + args := []string{cmdInternal, cmdAgent, cmdContainer, "ssh-server"} + if cfg.Ssh.Workdir != "" { + args = append(args, names.Flag(names.Workdir), cfg.Ssh.Workdir) + } + if cfg.Ssh.User != "" { + args = append(args, names.Flag(names.RemoteUser), cfg.Ssh.User) + } + if cfg.Lifecycle.Managed() { + args = append(args, names.Flag(names.TrackActivity)) + } + return args +} + // runSshServer starts the SSH server, sending SIGTERM on context cancellation // with a grace period before force-killing. func runSshServer(ctx context.Context, cmd *DaemonCmd) error { @@ -268,13 +310,7 @@ func runSshServer(ctx context.Context, cmd *DaemonCmd) error { return err } - args := []string{cmdInternal, cmdAgent, cmdContainer, "ssh-server"} - if cmd.Config.Ssh.Workdir != "" { - args = append(args, names.Flag(names.Workdir), cmd.Config.Ssh.Workdir) - } - if cmd.Config.Ssh.User != "" { - args = append(args, names.Flag(names.RemoteUser), cmd.Config.Ssh.User) - } + args := sshServerArgs(cmd.Config) sshCmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 sshCmd.Stdout = os.Stdout diff --git a/cmd/internal/agentcontainer/deferred_hooks.go b/cmd/internal/agentcontainer/deferred_hooks.go index 8e4c2f0952..6f254212a5 100644 --- a/cmd/internal/agentcontainer/deferred_hooks.go +++ b/cmd/internal/agentcontainer/deferred_hooks.go @@ -112,6 +112,9 @@ func (cmd *DeferredHooksCmd) Run(ctx context.Context) error { return err } + stopActivity := startWorkerActivity(ctx) + defer stopActivity() + log.Debugf("running deferred lifecycle hooks") deferred, err := setup.RunPreAttachHooks(ctx, setupInfo, setup.PreAttachOptions{ Prebuild: cmd.Prebuild, diff --git a/cmd/internal/agentcontainer/lifecycle.go b/cmd/internal/agentcontainer/lifecycle.go new file mode 100644 index 0000000000..64975cb4d4 --- /dev/null +++ b/cmd/internal/agentcontainer/lifecycle.go @@ -0,0 +1,115 @@ +//go:build !windows + +package agentcontainer + +import ( + "context" + "errors" + "fmt" + "time" + + config2 "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/devsy-org/devsy/pkg/log" + provider2 "github.com/devsy-org/devsy/pkg/provider" +) + +const ( + // ownerWaitTimeout bounds how long setup waits for the managed daemon when a + // positive timeout depends on it. + ownerWaitTimeout = 30 * time.Second + // optionalOwnerWait bounds the wait when no timeout is configured. + optionalOwnerWait = time.Second +) + +// lifecycleHandshake is setup's side of the managed daemon protocol. +type lifecycleHandshake struct { + sender lifecycleSender + timeout string + shutdownAction string +} + +type lifecycleSender interface { + Send(state agentd.LifecycleState, timeout, shutdownAction string) error +} + +func isManagedLifecycle(info *provider2.ContainerWorkspaceInfo) bool { + return info.LifecycleOwner == provider2.ContainerLifecycleOwnerManagedDaemon +} + +// beginLifecycle connects to the container's managed daemon and suspends its +// timer for the duration of setup. It returns nil when there is no handshake to +// do: not a managed container, or no timeout and no daemon to update. +func beginLifecycle( + ctx context.Context, + dir string, + info *provider2.ContainerWorkspaceInfo, + shutdownAction string, +) (*lifecycleHandshake, error) { + if !isManagedLifecycle(info) { + return nil, nil + } + wait := ownerWaitTimeout + if info.ContainerTimeout == "" { + wait = optionalOwnerWait + } + waitCtx, cancel := context.WithTimeout(ctx, wait) + defer cancel() + + client, err := agentd.ConnectLifecycle(waitCtx, dir) + if err != nil { + if info.ContainerTimeout == "" && errors.Is(err, agentd.ErrLifecycleOwnerMissing) { + log.Debugf("no container lifecycle owner and no inactivity timeout configured") + return nil, nil + } + return nil, migrationError(err) + } + + h := &lifecycleHandshake{ + sender: client, + timeout: info.ContainerTimeout, + shutdownAction: shutdownAction, + } + if err := h.send(agentd.LifecycleStateSetup); err != nil { + return nil, err + } + return h, nil +} + +func migrationError(err error) error { + return fmt.Errorf( + "this container's main process does not support the Devsy inactivity lifecycle (%w); "+ + "stop and start the workspace once (devsy workspace stop, then devsy up) to replace "+ + "the pod. The workspace volume is kept; data written to the container filesystem "+ + "outside mounted volumes is not", err) +} + +func (h *lifecycleHandshake) send(state agentd.LifecycleState) error { + if err := h.sender.Send(state, h.timeout, h.shutdownAction); err != nil { + return fmt.Errorf("update container lifecycle (%s): %w", state, err) + } + return nil +} + +// ready resets activity and arms the timer. +func (h *lifecycleHandshake) ready() error { + return h.send(agentd.LifecycleStateReady) +} + +// fail releases the suspension so a failed setup cannot leave the timer stuck. +func (h *lifecycleHandshake) fail() { + if err := h.send(agentd.LifecycleStateFailed); err != nil { + log.Errorf("release container lifecycle suspension: %v", err) + } +} + +// startWorkerActivity records activity for the lifetime of a background setup +// worker when a managed daemon owns the timer. The returned stop function ends +// the writer and leaves a fresh timestamp behind. +func startWorkerActivity(ctx context.Context) (stop func()) { + timeout, ok := agentd.ReadLifecycleTimeout(config2.ContainerLifecycleDir) + if !ok { + return func() {} + } + return agentd.StartActivityWriter(ctx, agentd.HeartbeatInterval(timeout)) +} diff --git a/cmd/internal/agentcontainer/lifecycle_test.go b/cmd/internal/agentcontainer/lifecycle_test.go new file mode 100644 index 0000000000..0c7319536c --- /dev/null +++ b/cmd/internal/agentcontainer/lifecycle_test.go @@ -0,0 +1,217 @@ +//go:build !windows + +package agentcontainer + +import ( + "context" + "errors" + "io" + "net" + "path/filepath" + "sync/atomic" + "testing" + "time" + + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + provider2 "github.com/devsy-org/devsy/pkg/provider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testRoot = "root" + +func managedInfo(timeout string) *provider2.ContainerWorkspaceInfo { + return &provider2.ContainerWorkspaceInfo{ + ContainerTimeout: timeout, + LifecycleOwner: provider2.ContainerLifecycleOwnerManagedDaemon, + } +} + +func publishOwner(t *testing.T) (*agentd.LifecycleOwner, string) { + t.Helper() + dir := filepath.Join(t.TempDir(), "lifecycle") + owner, err := agentd.NewLifecycleOwner(dir) + require.NoError(t, err) + return owner, dir +} + +func TestBeginLifecycleSuspendsThenReadyArmsSameOwner(t *testing.T) { + owner, dir := publishOwner(t) + + h, err := beginLifecycle( + context.Background(), dir, managedInfo("1m"), config.ShutdownActionStopContainer, + ) + require.NoError(t, err) + require.NotNil(t, h) + + rec, changed, err := owner.PollControl() + require.NoError(t, err) + require.True(t, changed) + assert.Equal(t, agentd.LifecycleStateSetup, rec.State) + assert.Equal(t, "1m", rec.Timeout) + assert.Equal(t, config.ShutdownActionStopContainer, rec.ShutdownAction) + + require.NoError(t, h.ready()) + rec, changed, err = owner.PollControl() + require.NoError(t, err) + require.True(t, changed) + assert.Equal(t, agentd.LifecycleStateReady, rec.State) +} + +func TestLifecycleFailureReleasesSuspension(t *testing.T) { + owner, dir := publishOwner(t) + h, err := beginLifecycle(context.Background(), dir, managedInfo("1m"), "") + require.NoError(t, err) + _, _, err = owner.PollControl() + require.NoError(t, err) + + h.fail() + + rec, changed, err := owner.PollControl() + require.NoError(t, err) + require.True(t, changed) + assert.Equal(t, agentd.LifecycleStateFailed, rec.State) +} + +func TestBeginLifecycleMissingOwnerWithTimeoutIsMigrationError(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + h, err := beginLifecycle(ctx, filepath.Join(t.TempDir(), "absent"), managedInfo("1m"), "") + + assert.Nil(t, h) + require.Error(t, err) + assert.ErrorIs(t, err, agentd.ErrLifecycleOwnerMissing) + assert.Contains(t, err.Error(), "stop and start the workspace") +} + +func TestBeginLifecycleMissingOwnerWithoutTimeoutIsSkipped(t *testing.T) { + h, err := beginLifecycle( + context.Background(), filepath.Join(t.TempDir(), "absent"), managedInfo(""), "", + ) + + assert.NoError(t, err) + assert.Nil(t, h) +} + +func TestBeginLifecycleIgnoresUnmanagedContainers(t *testing.T) { + h, err := beginLifecycle( + context.Background(), + filepath.Join(t.TempDir(), "absent"), + &provider2.ContainerWorkspaceInfo{ContainerTimeout: "1m"}, + "", + ) + + assert.NoError(t, err) + assert.Nil(t, h) +} + +func TestStartContainerDaemonNeverSpawnsSecondOwnerForManagedContainers(t *testing.T) { + cmd := &SetupContainerCmd{} + + err := cmd.startContainerDaemon(managedInfo("1m"), config.ShutdownActionStopContainer) + + assert.NoError(t, err) +} + +func TestDaemonExitCode(t *testing.T) { + assert.Equal(t, 0, daemonExitCode(nil)) + assert.Equal(t, 0, daemonExitCode(agentd.ErrInactivityExpired), "expiry is a normal shutdown") + assert.Equal(t, 0, daemonExitCode(errors.Join(agentd.ErrInactivityExpired))) + assert.Equal(t, 1, daemonExitCode(errors.New("SSH server exited abnormally"))) +} + +func TestSshServerArgs(t *testing.T) { + timeoutOnly := &agentd.DaemonConfig{ + Timeout: "1m", + Lifecycle: agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon}, + } + platform := &agentd.DaemonConfig{ + Ssh: agentd.SshConfig{Workdir: "/w", User: testRoot}, + Lifecycle: agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon}, + } + + cmd := &DaemonCmd{Config: timeoutOnly} + assert.False(t, cmd.shouldRunSsh(), "a timeout-only config must not start an SSH server") + assert.False(t, cmd.shouldRunNetworkServer()) + + assert.Equal(t, + []string{ + "internal", "agent", "container", "ssh-server", + "--workdir", "/w", "--remote-user", testRoot, "--track-activity", + }, + sshServerArgs(platform), + "managed Platform routes record connection activity", + ) + assert.NotContains( + t, + sshServerArgs(&agentd.DaemonConfig{Ssh: platform.Ssh}), + "--track-activity", + ) +} + +func TestLoadConfigCLITimeoutOverridesStoredConfig(t *testing.T) { + stored, err := agentd.EncodeDaemonConfig( + &agentd.DaemonConfig{Timeout: "5m", ShutdownAction: "none"}, + ) + require.NoError(t, err) + t.Setenv("DEVSY_WORKSPACE_DAEMON_CONFIG", stored) + cmd := &DaemonCmd{Config: &agentd.DaemonConfig{Timeout: "30s"}} + + require.NoError(t, cmd.loadConfig()) + + assert.Equal(t, "30s", cmd.Config.Timeout) + assert.Equal(t, "none", cmd.Config.ShutdownAction) +} + +func TestManagedLoopStartsUnarmedUntilSetup(t *testing.T) { + // The runtime dir is fixed in production, so only the unmanaged branch is + // observable without root; managed arming is covered by the monitor tests. + cmd := &DaemonCmd{Config: &agentd.DaemonConfig{}} + + loop, err := cmd.newLifecycleLoop(0) + assert.NoError(t, err) + assert.Nil(t, loop, "no timeout and no managed owner means no loop") + + loop, err = cmd.newLifecycleLoop(time.Minute) + require.NoError(t, err) + require.NotNil(t, loop) + assert.Nil(t, loop.Owner) +} + +type countingStarter struct{ started, stopped atomic.Int32 } + +func (c *countingStarter) start() func() { + c.started.Add(1) + return func() { c.stopped.Add(1) } +} + +func TestTrackConnectionsCountsConnectionsNotTheListener(t *testing.T) { + var starter countingStarter + inner, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + l := trackConnections(inner, starter.start) + defer func() { _ = l.Close() }() + + assert.Zero(t, starter.started.Load(), "an idle listener is not activity") + + accepted := make(chan net.Conn, 1) + go func() { + c, _ := l.Accept() + accepted <- c + }() + client, err := net.Dial("tcp", inner.Addr().String()) + require.NoError(t, err) + server := <-accepted + require.Eventually(t, func() bool { return starter.started.Load() == 1 }, + time.Second, time.Millisecond) + assert.Zero(t, starter.stopped.Load(), "activity continues while connected") + + _ = client.Close() + _, _ = io.Copy(io.Discard, server) + require.NoError(t, server.Close()) + _ = server.Close() + + assert.Equal(t, int32(1), starter.stopped.Load(), "disconnect ends the activity writer once") +} diff --git a/cmd/internal/agentcontainer/post_attach.go b/cmd/internal/agentcontainer/post_attach.go index cb35e55fea..f0d9fe5a02 100644 --- a/cmd/internal/agentcontainer/post_attach.go +++ b/cmd/internal/agentcontainer/post_attach.go @@ -55,6 +55,9 @@ func (cmd *PostAttachCmd) Run(ctx context.Context) error { return err } + stopActivity := startWorkerActivity(ctx) + defer stopActivity() + log.Debugf("running postAttachCommand hooks") // Mount values aren't in this process's env; read them back for redaction. if err := setup.RunPostAttachHooks( diff --git a/cmd/internal/agentcontainer/setup.go b/cmd/internal/agentcontainer/setup.go index a77d636dc3..23c9c0d0ca 100644 --- a/cmd/internal/agentcontainer/setup.go +++ b/cmd/internal/agentcontainer/setup.go @@ -74,6 +74,7 @@ type containerState struct { tunnelClient tunnel.TunnelClient reporter status.Reporter secretsEnv []string + lifecycle *lifecycleHandshake } // Run runs the command logic. @@ -95,6 +96,18 @@ func (cmd *SetupContainerCmd) Run(ctx context.Context) error { reporter: tunnelserver.NewTunnelStatusReporter(ctx, tunnelClient), } + if !cmd.Prebuild { + state.lifecycle, err = beginLifecycle( + ctx, + config2.ContainerLifecycleDir, + workspaceInfo, + setupInfo.MergedConfig.ShutdownAction, + ) + if err != nil { + return err + } + } + _, err = tunnelserver.ReportResult( ctx, tunnelClient, @@ -108,6 +121,9 @@ func (cmd *SetupContainerCmd) Run(ctx context.Context) error { return state.setupInfo, nil }, ) + if err != nil && state.lifecycle != nil { + state.lifecycle.fail() + } return err } @@ -307,9 +323,11 @@ func (cmd *SetupContainerCmd) setupPostAttach( return err } - shutdownAction := state.setupInfo.MergedConfig.ShutdownAction - if err := cmd.startContainerDaemon(state.workspaceInfo, shutdownAction); err != nil { - return err + if state.lifecycle == nil { + shutdownAction := state.setupInfo.MergedConfig.ShutdownAction + if err := cmd.startContainerDaemon(state.workspaceInfo, shutdownAction); err != nil { + return err + } } resolvedSetupInfo, err := compressSetupInfo(state.setupInfo) @@ -331,6 +349,10 @@ func (cmd *SetupContainerCmd) setupPostAttach( log.Errorf("failed to start postAttachCommand: %v", err) } + // Arm last so the background workers already hold the workspace active. + if state.lifecycle != nil { + return state.lifecycle.ready() + } return nil } @@ -495,6 +517,10 @@ func (cmd *SetupContainerCmd) startContainerDaemon( workspaceInfo *provider2.ContainerWorkspaceInfo, shutdownAction string, ) error { + // A managed container's main process owns the timer; never start a second one. + if isManagedLifecycle(workspaceInfo) { + return nil + } if workspaceInfo.CLIOptions.Platform.Enabled || workspaceInfo.CLIOptions.DisableDaemon || workspaceInfo.ContainerTimeout == "" { diff --git a/cmd/internal/agentcontainer/ssh_server.go b/cmd/internal/agentcontainer/ssh_server.go index 6e73f40269..602518991e 100644 --- a/cmd/internal/agentcontainer/ssh_server.go +++ b/cmd/internal/agentcontainer/ssh_server.go @@ -1,9 +1,13 @@ package agentcontainer import ( + "context" "fmt" + "net" + "sync" "github.com/devsy-org/devsy/cmd/flags" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" cliflags "github.com/devsy-org/devsy/pkg/flags" "github.com/devsy-org/devsy/pkg/flags/names" "github.com/devsy-org/devsy/pkg/log" @@ -19,6 +23,9 @@ type SSHServerCmd struct { Address string Workdir string RemoteUser string + + // TrackActivity records connection activity for the inactivity timer. + TrackActivity bool } // NewSSHServerCmd creates a new ssh command. @@ -53,6 +60,12 @@ func NewSSHServerCmd(flags *flags.GlobalFlags) *cobra.Command { "", "Directory where commands will run on the host", ), + cliflags.Bool( + &cmd.TrackActivity, + names.TrackActivity, + false, + "Record activity while connections are open", + ), ) return sshCmd } @@ -75,5 +88,48 @@ func (cmd *SSHServerCmd) Run(_ *cobra.Command, _ []string) error { return nil } - return server.ListenAndServe() + if !cmd.TrackActivity { + return server.ListenAndServe() + } + + listener, err := net.Listen("tcp", cmd.Address) + if err != nil { + return fmt.Errorf("listen on %s: %w", cmd.Address, err) + } + return server.Serve(trackConnections(listener, startConnectionActivity)) +} + +func startConnectionActivity() (stop func()) { + return agentd.StartActivityWriter(context.Background(), agentd.ActivityIntervalForLifecycle()) +} + +// trackConnections records activity from the moment a connection is accepted +// until it closes. The idle listener itself is not activity. +func trackConnections(l net.Listener, start func() (stop func())) net.Listener { + return &activityListener{Listener: l, start: start} +} + +type activityListener struct { + net.Listener + start func() (stop func()) +} + +func (l *activityListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + return &activityConn{Conn: conn, stop: l.start()}, nil +} + +type activityConn struct { + net.Conn + stop func() + once sync.Once +} + +func (c *activityConn) Close() error { + err := c.Conn.Close() + c.once.Do(c.stop) + return err } diff --git a/cmd/internal/ssh_server.go b/cmd/internal/ssh_server.go index 906476ce40..979e4b5928 100644 --- a/cmd/internal/ssh_server.go +++ b/cmd/internal/ssh_server.go @@ -11,6 +11,7 @@ import ( "github.com/devsy-org/devsy/cmd/flags" "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/flags/names" "github.com/devsy-org/devsy/pkg/log" "github.com/devsy-org/devsy/pkg/sharedfile" @@ -22,10 +23,7 @@ import ( "github.com/spf13/cobra" ) -const ( - activityHeartbeatInterval = 10 * time.Second - activityFileMode = 0o666 -) +const activityFileMode = 0o666 // shutdownTimeout bounds the grace period for in-flight SSH connections to // drain on context cancel before the listener is force-closed. @@ -62,7 +60,7 @@ func NewSSHServerCmd(globalFlags *flags.GlobalFlags) *cobra.Command { f.BoolVar(&cmd.stdio, names.Stdio, false, "Listen on stdin/stdout instead of an address") f.BoolVar(&cmd.trackActivity, names.TrackActivity, false, - "Touch the activity file every "+activityHeartbeatInterval.String()+" (only with --stdio)") + "Record activity while connected (only with --stdio)") f.StringVar(&cmd.reuseSSHAuthSock, names.ReuseSSHAuthSock, "", "If set, reuse a pre-existing SSH_AUTH_SOCK in the workspace under /tmp") _ = f.MarkHidden(names.ReuseSSHAuthSock) @@ -106,7 +104,16 @@ func (cmd *sshServerCmd) run(ctx context.Context) error { func (cmd *sshServerCmd) serveStdio(ctx context.Context, server sshserver.Server) error { if cmd.trackActivity { - go runActivityHeartbeat(ctx, config.ContainerActivityFile) + hbCtx, stopHeartbeat := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + defer close(done) + runActivityHeartbeat(hbCtx, config.ContainerActivityFile) + }() + defer func() { + stopHeartbeat() + <-done + }() } go shutdownOnCancel(ctx, server) // #nosec G118 -- see shutdownOnCancel. lis := stdio.NewStdioListener(os.Stdin, os.Stdout) @@ -204,36 +211,13 @@ func decodeBase64Bytes(encoded, label string) ([]byte, error) { return out, nil } -// runActivityHeartbeat periodically updates the activity file's mtime so -// outside watchers can detect liveness. Exits when ctx is canceled. Logs -// only on success/failure transitions so a permanently broken file does -// not spam the log. +// runActivityHeartbeat records activity immediately, periodically while ctx is +// live, and once more when ctx ends so a disconnect leaves a fresh timestamp. func runActivityHeartbeat(ctx context.Context, path string) { - if err := ensureActivityFile(path); err != nil { - log.Errorf("activity heartbeat: ensure file: %v", err) - return - } - - t := time.NewTicker(activityHeartbeatInterval) - defer t.Stop() - - var lastErr error - for { - select { - case <-ctx.Done(): - return - case now := <-t.C: - err := os.Chtimes(path, now, now) - if (err == nil) != (lastErr == nil) { - if err != nil { - log.Errorf("activity heartbeat: %v", err) - } else { - log.Infof("activity heartbeat recovered") - } - } - lastErr = err - } - } + agentd.ActivityWriter{ + Path: path, + Interval: agentd.ActivityIntervalForLifecycle(), + }.Run(ctx) } func ensureActivityFile(path string) error { diff --git a/cmd/internal/ssh_server_test.go b/cmd/internal/ssh_server_test.go index 3be14496f5..83bafaa008 100644 --- a/cmd/internal/ssh_server_test.go +++ b/cmd/internal/ssh_server_test.go @@ -244,3 +244,28 @@ func TestTouchActivityFile_UpdatesMtimeOfExistingFile(t *testing.T) { assert.WithinDuration(t, time.Now(), info.ModTime(), 2*time.Second, "touchActivityFile must advance mtime on an already-existing file") } + +func TestRunActivityHeartbeatTouchesOnConnectAndDisconnect(t *testing.T) { + path := filepath.Join(t.TempDir(), "activity") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + runActivityHeartbeat(ctx, path) + close(done) + }() + + require.Eventually(t, func() bool { + info, err := os.Stat(path) + return err == nil && time.Since(info.ModTime()) < 2*time.Second + }, 2*time.Second, time.Millisecond, "connect must record activity before the first interval") + + stale := time.Now().Add(-time.Hour) + require.NoError(t, os.Chtimes(path, stale, stale)) + cancel() + <-done + + info, err := os.Stat(path) + require.NoError(t, err) + assert.WithinDuration(t, time.Now(), info.ModTime(), 2*time.Second, + "disconnect must leave a fresh timestamp so idle time starts then") +} diff --git a/e2e/tests/up/provider_kubernetes_inactivity.go b/e2e/tests/up/provider_kubernetes_inactivity.go new file mode 100644 index 0000000000..22bdcf56bd --- /dev/null +++ b/e2e/tests/up/provider_kubernetes_inactivity.go @@ -0,0 +1,422 @@ +package up + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" +) + +const ( + inactivityLabel = "up-provider-kubernetes-inactivity" + inactivityImage = "ghcr.io/devsy-org/test-images/go:1" + inactivityPollEvery = 3 * time.Second + sentinelName = "idle-sentinel" +) + +// idleEnv is one isolated namespace, provider and workspace. +type idleEnv struct { + f *framework.Framework + ns string + dir string + wsID string +} + +func newIdleEnv(ctx context.Context, initialDir string, providerOpts ...string) *idleEnv { + e := &idleEnv{ + f: framework.NewDefaultFramework(initialDir + "/bin"), + ns: fmt.Sprintf("devsy-idle-%d", time.Now().UnixNano()%1_000_000_000), + } + e.dir = framework.CopyTestdata("tests/up/testdata/kubernetes", initialDir) + + opts := []string{ + "-o", + "KUBERNETES_NAMESPACE=" + e.ns, + "-o", + "CREATE_NAMESPACE=true", + "-o", + "POD_TIMEOUT=5m", + } + for _, o := range providerOpts { + opts = append(opts, "-o", o) + } + _ = e.f.DevsyProviderDelete(ctx, "kubernetes") + framework.ExpectNoError(e.f.DevsyProviderAdd(ctx, append([]string{"kubernetes"}, opts...)...)) + + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + c, cancel := framework.DetachedCleanupContext(cleanupCtx, 3*time.Minute) + defer cancel() + _ = e.f.DevsyWorkspaceDelete(c, e.dir) + _ = e.f.DevsyProviderDelete(c, "kubernetes") + _ = exec.CommandContext(c, "kubectl", "delete", "namespace", e.ns, // #nosec G204 -- test-owned namespace + "--ignore-not-found", "--wait=false"). + Run() + }) + return e +} + +func (e *idleEnv) writeDevcontainer(content string) { + dir := filepath.Join(e.dir, ".devcontainer") + framework.ExpectNoError(os.MkdirAll(dir, 0o750)) + framework.ExpectNoError( + os.WriteFile(filepath.Join(dir, "devcontainer.json"), []byte(content), 0o600), + ) +} + +func (e *idleEnv) up(ctx context.Context) error { + return e.f.DevsyUp(ctx, e.dir) +} + +func (e *idleEnv) workspaceID(ctx context.Context) string { + if e.wsID == "" { + ws, err := e.f.FindWorkspace(ctx, e.dir) + framework.ExpectNoError(err) + e.wsID = ws.ID + } + return e.wsID +} + +func (e *idleEnv) kubectlJSON(ctx context.Context, into any, args ...string) error { + full := append(append([]string{}, args...), "-n", e.ns, "-o", "json") + cmd := exec.CommandContext(ctx, "kubectl", full...) // #nosec G204 -- test-owned kubectl args + out, err := cmd.Output() + if err != nil { + return err + } + return json.Unmarshal(out, into) +} + +func (e *idleEnv) pod(ctx context.Context) (*corev1.Pod, error) { + list := &corev1.PodList{} + if err := e.kubectlJSON(ctx, list, "get", "pods", "-l", "devsy.sh/created=true"); err != nil { + return nil, err + } + if len(list.Items) != 1 { + return nil, fmt.Errorf("want exactly 1 workspace pod, got %d", len(list.Items)) + } + return &list.Items[0], nil +} + +func (e *idleEnv) pvcUID(ctx context.Context) string { + list := &corev1.PersistentVolumeClaimList{} + framework.ExpectNoError(e.kubectlJSON(ctx, list, "get", "pvc")) + gomega.Expect(list.Items).To(gomega.HaveLen(1)) + return string(list.Items[0].UID) +} + +func (e *idleEnv) ssh(ctx context.Context, command string) string { + out, err := e.f.DevsySSH(ctx, e.dir, command) + framework.ExpectNoError(err) + return strings.TrimSpace(out) +} + +// holdSSH keeps one tracked SSH session open until the returned stop is called. +func (e *idleEnv) holdSSH(ctx context.Context) (stop func()) { + cmd := exec.CommandContext( + ctx, + filepath.Join(e.f.DevsyBinDir, e.f.DevsyBinName), // #nosec G204 -- test binary + cmdWorkspace, + cmdSSH, + e.dir, + flagCommand, + "sleep 3600", + ) + framework.ExpectNoError(cmd.Start()) + stop = func() { + if cmd.Process != nil { + _ = cmd.Process.Kill() + } + _ = cmd.Wait() + } + ginkgo.DeferCleanup(stop) + return stop +} + +func podTerminal(pod *corev1.Pod) bool { + return pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed +} + +func devsyRestarts(pod *corev1.Pod) int32 { + var n int32 + for _, c := range pod.Status.ContainerStatuses { + n += c.RestartCount + } + return n +} + +// waitStopped observes the pod through the Kubernetes API, never through +// workspace ssh, which would itself keep the workspace active. +func (e *idleEnv) waitStopped(ctx context.Context, uid string, within time.Duration) *corev1.Pod { + var last *corev1.Pod + gomega.Eventually(func() bool { + pod, err := e.pod(ctx) + if err != nil { + return false + } + last = pod + return string(pod.UID) == uid && podTerminal(pod) + }, within, inactivityPollEvery).Should(gomega.BeTrue(), "workspace pod should reach a terminal phase") + + gomega.Expect(last.Status.Phase).ToNot(gomega.Equal(corev1.PodRunning)) + gomega.Expect(devsyRestarts(last)).To(gomega.BeZero(), "the stopped pod must not restart") + for _, c := range last.Status.ContainerStatuses { + gomega.Expect(c.State.Terminated). + ToNot(gomega.BeNil(), "container %s should be terminated", c.Name) + } + return last +} + +// expectRunningFor asserts the same pod instance keeps running for the whole window. +func (e *idleEnv) expectRunningFor(ctx context.Context, uid string, window time.Duration) { + gomega.Consistently(func() string { + pod, err := e.pod(ctx) + if err != nil || string(pod.UID) != uid { + return "missing" + } + return string(pod.Status.Phase) + }, window, inactivityPollEvery).Should(gomega.Equal(string(corev1.PodRunning))) +} + +func devcontainerWith(extra string) string { + return fmt.Sprintf(`{"name":"idle","image":%q%s}`, inactivityImage, extra) +} + +const sidecarTemplate = "spec:\n containers:\n - name: sidecar\n image: " + inactivityImage + + "\n command: [sleep, infinity]\n" + +const nativeSidecarTemplate = "spec:\n initContainers:\n - name: mesh\n image: " + inactivityImage + + "\n command: [sleep, infinity]\n restartPolicy: Always\n" + +var _ = ginkgo.Describe( + "kubernetes workspace inactivity timeout", + ginkgo.Label(inactivityLabel), + ginkgo.Serial, + func() { + var initialDir string + + ginkgo.BeforeEach(func() { + var err error + initialDir, err = os.Getwd() + framework.ExpectNoError(err) + }) + + ginkgo.It( + "stops an idle workspace and resumes it with the volume intact", + func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir, "INACTIVITY_TIMEOUT=1m") + framework.ExpectNoError(e.up(ctx)) + + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + gomega.Expect(pod.Spec.Containers).To(gomega.HaveLen(1)) + pvc := e.pvcUID(ctx) + workspaceUID := e.workspaceUID(ctx) + sentinel := "/workspaces/" + e.workspaceID(ctx) + "/" + sentinelName + + for cycle := 1; cycle <= 2; cycle++ { + ginkgo.By(fmt.Sprintf("cycle %d: writing a sentinel and going idle", cycle)) + e.ssh(ctx, fmt.Sprintf("echo cycle-%d > %s", cycle, sentinel)) + podUID := string(pod.UID) + + e.waitStopped(ctx, podUID, 4*time.Minute) + gomega.Expect(e.pvcUID(ctx)).To(gomega.Equal(pvc)) + + ginkgo.By(fmt.Sprintf("cycle %d: resuming with a plain up", cycle)) + framework.ExpectNoError(e.up(ctx)) + pod, err = e.pod(ctx) + framework.ExpectNoError(err) + gomega.Expect(string(pod.UID)). + ToNot(gomega.Equal(podUID), "a new pod replaced the stopped one") + gomega.Expect(pod.Status.Phase).To(gomega.Equal(corev1.PodRunning)) + gomega.Expect(e.pvcUID(ctx)).To(gomega.Equal(pvc)) + gomega.Expect(e.workspaceUID(ctx)).To(gomega.Equal(workspaceUID)) + gomega.Expect(e.ssh(ctx, "cat "+sentinel)). + To(gomega.Equal(fmt.Sprintf("cycle-%d", cycle))) + } + }, + ginkgo.SpecTimeout(framework.TimeoutVeryLong()), + ) + + ginkgo.It("stays running while a tracked SSH session is open and stops after it ends", + func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir, "INACTIVITY_TIMEOUT=30s") + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + + stop := e.holdSSH(ctx) + e.expectRunningFor(ctx, string(pod.UID), 75*time.Second) + + stop() + e.waitStopped(ctx, string(pod.UID), 3*time.Minute) + }, ginkgo.SpecTimeout(framework.TimeoutLong())) + + ginkgo.It("does not stop when shutdownAction is none", func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir, "INACTIVITY_TIMEOUT=30s") + e.writeDevcontainer(devcontainerWith(`,"shutdownAction":"none"`)) + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + + e.expectRunningFor(ctx, string(pod.UID), 90*time.Second) + }, ginkgo.SpecTimeout(framework.TimeoutLong())) + + ginkgo.It("does not stop when the timeout is unset", func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir) + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + + e.expectRunningFor(ctx, string(pod.UID), 75*time.Second) + }, ginkgo.SpecTimeout(framework.TimeoutLong())) + + ginkgo.It( + "keeps the workspace alive during a long postAttachCommand", + func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir, "INACTIVITY_TIMEOUT=30s") + e.writeDevcontainer(devcontainerWith(`,"postAttachCommand":"sleep 80"`)) + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + + e.expectRunningFor(ctx, string(pod.UID), 60*time.Second) + e.waitStopped(ctx, string(pod.UID), 4*time.Minute) + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + + ginkgo.It( + "recreates a terminal pod with unchanged options on the next up", + func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir) + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + pvc := e.pvcUID(ctx) + + out, err := exec.CommandContext(ctx, "kubectl", "exec", "-n", e.ns, pod.Name, // #nosec G204 -- test pod + "-c", "devsy", "--", "sh", "-c", "kill -TERM 1"). + CombinedOutput() + framework.ExpectNoError(err, string(out)) + e.waitStopped(ctx, string(pod.UID), 2*time.Minute) + + framework.ExpectNoError(e.up(ctx)) + fresh, err := e.pod(ctx) + framework.ExpectNoError(err) + gomega.Expect(fresh.UID).ToNot(gomega.Equal(pod.UID)) + gomega.Expect(e.pvcUID(ctx)).To(gomega.Equal(pvc)) + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + + ginkgo.It( + "stops the whole pod when a native sidecar is present", + func(ctx context.Context) { + e := newIdleEnv( + ctx, + initialDir, + "INACTIVITY_TIMEOUT=30s", + "POD_MANIFEST_TEMPLATE="+nativeSidecarTemplate, + ) + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + + stopped := e.waitStopped(ctx, string(pod.UID), 4*time.Minute) + for _, c := range stopped.Status.InitContainerStatuses { + gomega.Expect(c.State.Terminated). + ToNot(gomega.BeNil(), "sidecar %s should end with the pod", c.Name) + } + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + + ginkgo.It( + "rejects extra regular containers only when a timeout is enabled", + func(ctx context.Context) { + e := newIdleEnv( + ctx, + initialDir, + "INACTIVITY_TIMEOUT=30s", + "POD_MANIFEST_TEMPLATE="+sidecarTemplate, + ) + + _, stderr, err := e.f.ExecCommandCapture(ctx, + []string{cmdWorkspace, "up", e.dir, "--ide", "none"}) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(stderr).To(gomega.ContainSubstring("sidecar")) + pvcs := &corev1.PersistentVolumeClaimList{} + framework.ExpectNoError(e.kubectlJSON(ctx, pvcs, "get", "pvc")) + gomega.Expect(pvcs.Items).To(gomega.BeEmpty(), "nothing is created for a rejected pod") + + framework.ExpectNoError(e.f.DevsyWorkspaceDelete(ctx, e.dir)) + framework.ExpectNoError(os.Chdir(initialDir)) + e = newIdleEnv( + ctx, + initialDir, + "INACTIVITY_TIMEOUT=0", + "POD_MANIFEST_TEMPLATE="+sidecarTemplate, + ) + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + gomega.Expect(pod.Spec.Containers).To(gomega.HaveLen(2)) + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + + ginkgo.It( + "stops and resumes under a restricted non-root security context", + func(ctx context.Context) { + framework.ExpectNoError(labelNamespaceRestricted(ctx)) + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + _ = exec.CommandContext(cleanupCtx, "kubectl", "delete", "namespace", restrictedNamespace, + "--ignore-not-found", "--wait=false"). + Run() + }) + e := newIdleEnv(ctx, initialDir, + "INACTIVITY_TIMEOUT=30s", + "STRICT_SECURITY=true", + "AGENT_SECURITY_CONTEXT="+restrictedSecurityContextYAML, + "POD_MANIFEST_TEMPLATE="+restrictedPodManifestTemplate, + "AGENT_INSTALL_PATH=/tmp/devsy", + ) + e.ns = restrictedNamespace + framework.ExpectNoError(e.f.DevsyProviderUse( + ctx, + "kubernetes", + "-o", + "KUBERNETES_NAMESPACE="+restrictedNamespace, + "-o", + "CREATE_NAMESPACE=false", + )) + + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + e.waitStopped(ctx, string(pod.UID), 4*time.Minute) + + framework.ExpectNoError(e.up(ctx)) + fresh, err := e.pod(ctx) + framework.ExpectNoError(err) + gomega.Expect(fresh.UID).ToNot(gomega.Equal(pod.UID)) + }, + ginkgo.SpecTimeout(framework.TimeoutVeryLong()), + ) + }, +) + +func (e *idleEnv) workspaceUID(ctx context.Context) string { + ws, err := e.f.FindWorkspace(ctx, e.dir) + framework.ExpectNoError(err) + return ws.UID +} diff --git a/hack/ci/suites.yaml b/hack/ci/suites.yaml index 83b3856940..ff9ab7e673 100644 --- a/hack/ci/suites.yaml +++ b/hack/ci/suites.yaml @@ -217,6 +217,7 @@ e2e: - { group: integration, label: up-private-token, requires_secret: true } - { group: integration, label: up-provider-kubernetes, free_disk_space: true, install_kind: true } - { group: integration, label: up-provider-kubernetes-restricted-scc, free_disk_space: true, install_kind: true } + - { group: integration, label: up-provider-kubernetes-inactivity, free_disk_space: true, install_kind: true, test_timeout: 60m, job_timeout_minutes: 75 } - { group: integration, label: up-provider-docker, install_kind: true } - { group: integration, label: snapshot, host_setup: [insecure-registry] } - group: integration diff --git a/pkg/config/paths.go b/pkg/config/paths.go index 40074189dc..d334676d27 100644 --- a/pkg/config/paths.go +++ b/pkg/config/paths.go @@ -45,6 +45,9 @@ const ( // ContainerActivityFile is touched by SSH/fleet servers to record container liveness. ContainerActivityFile = "/tmp/" + BinaryName + ".activity" + // ContainerLifecycleDir holds the managed container daemon's owner and control records. + ContainerLifecycleDir = "/tmp/" + BinaryName + "-container-lifecycle" + // WorkspaceBusyFile is the per-workspace lock file written under the workspace folder. WorkspaceBusyFile = "workspace.lock" ) diff --git a/pkg/daemon/agent/activity.go b/pkg/daemon/agent/activity.go new file mode 100644 index 0000000000..7b7d79ff6d --- /dev/null +++ b/pkg/daemon/agent/activity.go @@ -0,0 +1,131 @@ +package agent + +import ( + "context" + "errors" + "os" + "sync" + "time" + + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/log" + "github.com/devsy-org/devsy/pkg/sharedfile" +) + +const ( + // DefaultActivityInterval is the longest gap between activity touches. + DefaultActivityInterval = 10 * time.Second + + activityFileMode = 0o666 + minTickInterval = 10 * time.Millisecond +) + +// HeartbeatInterval is how often activity is recorded for a given timeout: at +// most the default cadence, and at most a third of short timeouts. +func HeartbeatInterval(timeout time.Duration) time.Duration { + if timeout <= 0 { + return DefaultActivityInterval + } + return min(DefaultActivityInterval, max(timeout/3, minTickInterval)) +} + +// TouchActivity records activity at now, creating the shared file if needed. +func TouchActivity(path string, now time.Time) error { + if err := sharedfile.EnsureMode(path, activityFileMode); err != nil { + return err + } + err := os.Chtimes(path, now, now) + if errors.Is(err, os.ErrPermission) { + // Setting an explicit time needs ownership; any writer may bump mtime. + return touchByWrite(path) + } + return err +} + +// touchByWrite updates the modification time through a write, which the +// 0666 activity file permits for every user. +func touchByWrite(path string) error { + f, err := os.OpenFile(path, os.O_WRONLY, 0) // #nosec G304 -- fixed activity file path + if err != nil { + return err + } + _, writeErr := f.WriteAt([]byte{0}, 0) + return errors.Join(writeErr, f.Close()) +} + +// ActivityWriter records activity immediately, periodically, and once more when +// its context ends. Zero-value hooks use the real clock and file. +type ActivityWriter struct { + Path string + Interval time.Duration + Touch func(path string, now time.Time) error + Now func() time.Time + // Tick returns a tick channel and its stop function. + Tick func(d time.Duration) (<-chan time.Time, func()) +} + +// Run blocks until ctx is done. +func (w ActivityWriter) Run(ctx context.Context) { + w = w.withDefaults() + + var lastErr error + record := func() { + err := w.Touch(w.Path, w.Now()) + if (err == nil) != (lastErr == nil) { + if err != nil { + log.Errorf("activity writer: %v", err) + } else { + log.Infof("activity writer recovered") + } + } + lastErr = err + } + + ticks, stop := w.Tick(w.Interval) + defer stop() + record() + for { + select { + case <-ctx.Done(): + record() + return + case <-ticks: + record() + } + } +} + +func (w ActivityWriter) withDefaults() ActivityWriter { + if w.Touch == nil { + w.Touch = TouchActivity + } + if w.Now == nil { + w.Now = time.Now + } + if w.Tick == nil { + w.Tick = realTick + } + if w.Interval <= 0 { + w.Interval = DefaultActivityInterval + } + return w +} + +func realTick(d time.Duration) (<-chan time.Time, func()) { + t := time.NewTicker(d) + return t.C, t.Stop +} + +// StartActivityWriter runs an ActivityWriter on the shared activity file until +// the returned stop function is called; stop waits for the final touch. +func StartActivityWriter(ctx context.Context, interval time.Duration) (stop func()) { + wctx, cancel := context.WithCancel(ctx) + var wg sync.WaitGroup + wg.Go(func() { + ActivityWriter{Path: config.ContainerActivityFile, Interval: interval}.Run(wctx) + }) + return func() { + cancel() + wg.Wait() + } +} diff --git a/pkg/daemon/agent/container_lifecycle.go b/pkg/daemon/agent/container_lifecycle.go new file mode 100644 index 0000000000..033326d5a8 --- /dev/null +++ b/pkg/daemon/agent/container_lifecycle.go @@ -0,0 +1,166 @@ +package agent + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/devsy-org/devsy/pkg/devcontainer/config" +) + +const ( + canonicalTimeoutField = "agent.containerInactivityTimeout" + legacyTimeoutField = "agent.kubernetes.inactivityTimeout" + + // LifecycleOwnerManagedDaemon marks a container whose main process is the + // Devsy daemon and owns inactivity shutdown. + LifecycleOwnerManagedDaemon = "managedDaemon" +) + +// ErrInactivityExpired is returned by the lifecycle loop when the idle timeout +// elapsed. It is the normal way a managed container asks to stop. +var ErrInactivityExpired = errors.New("inactivity timeout reached") + +// LifecycleConfig tells the daemon who owns inactivity shutdown. +type LifecycleConfig struct { + // Owner is LifecycleOwnerManagedDaemon when the daemon is the container's main + // process and owns the timer. + Owner string `json:"owner,omitempty"` + // SetupArmsTimer means the daemon starts unarmed and container setup arms it. + SetupArmsTimer bool `json:"setupArmsTimer,omitempty"` +} + +// Managed reports whether the daemon owns the inactivity timer. +func (l LifecycleConfig) Managed() bool { + return l.Owner == LifecycleOwnerManagedDaemon +} + +// LifecycleOptions are the explicit lifecycle inputs for building a daemon config. +type LifecycleOptions struct { + // Managed marks the daemon as the container's main process and timer owner. + Managed bool + // Timeout is the effective inactivity timeout; zero disables it. + Timeout time.Duration +} + +// TimeoutInputs are the configured values that decide the effective timeout. +type TimeoutInputs struct { + // Canonical is agent.containerInactivityTimeout. + Canonical string + // Legacy is agent.kubernetes.inactivityTimeout, used only when Canonical is empty. + Legacy string + ShutdownAction string + DisableDaemon bool +} + +// ResolveInactivityTimeout returns the single effective inactivity timeout for a +// managed container. Zero means disabled. +func ResolveInactivityTimeout(in TimeoutInputs) (time.Duration, error) { + timeout, err := configuredTimeout(in) + if err != nil { + return 0, err + } + if timeout == 0 || in.DisableDaemon || in.ShutdownAction == config.ShutdownActionNone { + return 0, nil + } + if in.ShutdownAction == config.ShutdownActionStopCompose { + return 0, fmt.Errorf( + "inactivity timeout %s is set but shutdownAction %q is not supported by the kubernetes driver", + timeout, + config.ShutdownActionStopCompose, + ) + } + return timeout, nil +} + +// configuredTimeout merges the canonical field and its legacy alias. +func configuredTimeout(in TimeoutInputs) (time.Duration, error) { + canonical, err := parseTimeoutField(canonicalTimeoutField, in.Canonical) + if err != nil { + return 0, err + } + legacy, err := parseTimeoutField(legacyTimeoutField, in.Legacy) + if err != nil { + return 0, err + } + + hasCanonical := strings.TrimSpace(in.Canonical) != "" + if !hasCanonical { + return legacy, nil + } + if strings.TrimSpace(in.Legacy) != "" && canonical != legacy { + return 0, fmt.Errorf( + "%s (%q) conflicts with deprecated alias %s (%q); set only %s", + canonicalTimeoutField, + in.Canonical, + legacyTimeoutField, + in.Legacy, + canonicalTimeoutField, + ) + } + return canonical, nil +} + +func parseTimeoutField(field, value string) (time.Duration, error) { + value = strings.TrimSpace(value) + if value == "" { + return 0, nil + } + d, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("parse %s %q: %w", field, value, err) + } + if d < 0 { + return 0, fmt.Errorf("%s %q must not be negative", field, value) + } + return d, nil +} + +// FormatTimeout renders a timeout for DaemonConfig/ContainerWorkspaceInfo; zero is "". +func FormatTimeout(d time.Duration) string { + if d <= 0 { + return "" + } + return d.String() +} + +// DecodeDaemonConfig parses the base64 JSON produced by GetEncodedWorkspaceDaemonConfig. +func DecodeDaemonConfig(encoded string) (*DaemonConfig, error) { + raw, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil, fmt.Errorf("decode daemon config: %w", err) + } + cfg := &DaemonConfig{} + if err := json.Unmarshal(raw, cfg); err != nil { + return nil, fmt.Errorf("unmarshal daemon config: %w", err) + } + return cfg, nil +} + +// EncodeDaemonConfig is the inverse of DecodeDaemonConfig. +func EncodeDaemonConfig(cfg *DaemonConfig) (string, error) { + raw, err := json.Marshal(cfg) + if err != nil { + return "", fmt.Errorf("marshal daemon config: %w", err) + } + return base64.StdEncoding.EncodeToString(raw), nil +} + +// RefreshLifecycle rewrites the generated lifecycle policy of a stored encoded +// daemon config, leaving platform and SSH settings untouched. An empty stored +// config yields a minimal timeout-only config. +func RefreshLifecycle(encoded string, timeout time.Duration) (string, error) { + cfg := &DaemonConfig{} + if encoded != "" { + var err error + if cfg, err = DecodeDaemonConfig(encoded); err != nil { + return "", err + } + } + cfg.Timeout = FormatTimeout(timeout) + cfg.Lifecycle = LifecycleConfig{Owner: LifecycleOwnerManagedDaemon, SetupArmsTimer: true} + return EncodeDaemonConfig(cfg) +} diff --git a/pkg/daemon/agent/container_lifecycle_test.go b/pkg/daemon/agent/container_lifecycle_test.go new file mode 100644 index 0000000000..f4aecb921d --- /dev/null +++ b/pkg/daemon/agent/container_lifecycle_test.go @@ -0,0 +1,158 @@ +package agent + +import ( + "testing" + "time" + + "github.com/devsy-org/api/pkg/devsy" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + provider2 "github.com/devsy-org/devsy/pkg/provider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testWorkdir = "/workspaces/app" + +func TestResolveInactivityTimeout(t *testing.T) { + tests := []struct { + name string + in TimeoutInputs + want time.Duration + }{ + {name: "empty disables", in: TimeoutInputs{}}, + {name: "zero disables", in: TimeoutInputs{Canonical: "0"}}, + {name: "zero duration disables", in: TimeoutInputs{Canonical: "0s"}}, + {name: "positive", in: TimeoutInputs{Canonical: "10m"}, want: 10 * time.Minute}, + {name: "legacy alone", in: TimeoutInputs{Legacy: "5m"}, want: 5 * time.Minute}, + { + name: "equal aliases", + in: TimeoutInputs{Canonical: "1m", Legacy: "60s"}, + want: time.Minute, + }, + { + name: "shutdown none disables", + in: TimeoutInputs{Canonical: "1m", ShutdownAction: config.ShutdownActionNone}, + }, + {name: "disabled daemon disables", in: TimeoutInputs{Canonical: "1m", DisableDaemon: true}}, + { + name: "stopContainer enabled", + in: TimeoutInputs{ + Canonical: "1m", + ShutdownAction: config.ShutdownActionStopContainer, + }, + want: time.Minute, + }, + { + name: "stopCompose without timeout is fine", + in: TimeoutInputs{ShutdownAction: config.ShutdownActionStopCompose}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ResolveInactivityTimeout(tt.in) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestResolveInactivityTimeoutRejectsBadConfiguration(t *testing.T) { + tests := []struct { + name string + in TimeoutInputs + wantErr string + }{ + {"malformed", TimeoutInputs{Canonical: "soon"}, "agent.containerInactivityTimeout"}, + {"malformed legacy", TimeoutInputs{Legacy: "soon"}, "agent.kubernetes.inactivityTimeout"}, + {"negative", TimeoutInputs{Canonical: "-1m"}, "must not be negative"}, + { + "conflicting aliases", + TimeoutInputs{Canonical: "1m", Legacy: "2m"}, + "conflicts with deprecated alias", + }, + { + "stopCompose rejected", + TimeoutInputs{Canonical: "1m", ShutdownAction: config.ShutdownActionStopCompose}, + "not supported by the kubernetes driver", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := ResolveInactivityTimeout(tt.in) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestBuildWorkspaceDaemonConfig_ManagedWithoutPlatform(t *testing.T) { + merged := &config.MergedDevContainerConfig{} + merged.ShutdownAction = config.ShutdownActionStopContainer + merged.RemoteUser = "vscode" + + cfg, err := BuildWorkspaceDaemonConfig( + devsy.PlatformOptions{}, + &provider2.Workspace{}, + &config.SubstitutionContext{ContainerWorkspaceFolder: testWorkdir}, + merged, + LifecycleOptions{Managed: true, Timeout: time.Minute}, + ) + require.NoError(t, err) + + assert.Equal(t, "1m0s", cfg.Timeout) + assert.Equal(t, config.ShutdownActionStopContainer, cfg.ShutdownAction) + assert.True(t, cfg.Lifecycle.Managed()) + assert.True(t, cfg.Lifecycle.SetupArmsTimer) + assert.Empty(t, cfg.Ssh, "a timeout-only config must not request the embedded SSH server") + assert.Empty(t, cfg.Platform.AccessKey) +} + +func TestBuildWorkspaceDaemonConfig_PlatformKeepsNetworking(t *testing.T) { + merged := &config.MergedDevContainerConfig{} + merged.ShutdownAction = config.ShutdownActionNone + + cfg, err := BuildWorkspaceDaemonConfig( + devsy.PlatformOptions{AccessKey: "key", PlatformHost: "p", WorkspaceHost: "w"}, + &provider2.Workspace{}, + &config.SubstitutionContext{ContainerWorkspaceFolder: testWorkdir}, + merged, + LifecycleOptions{}, + ) + require.NoError(t, err) + + assert.Equal(t, "key", cfg.Platform.AccessKey) + assert.Equal(t, SshConfig{Workdir: testWorkdir, User: "root"}, cfg.Ssh) + assert.Equal(t, config.ShutdownActionNone, cfg.ShutdownAction) + assert.False(t, cfg.Lifecycle.Managed()) +} + +func TestRefreshLifecycleKeepsPlatformAndReplacesPolicy(t *testing.T) { + stored, err := EncodeDaemonConfig(&DaemonConfig{ + Platform: devsy.PlatformOptions{AccessKey: "key"}, + Ssh: SshConfig{User: "root"}, + Timeout: "5m", + ShutdownAction: config.ShutdownActionStopContainer, + }) + require.NoError(t, err) + + refreshed, err := RefreshLifecycle(stored, 2*time.Minute) + require.NoError(t, err) + cfg, err := DecodeDaemonConfig(refreshed) + require.NoError(t, err) + + assert.Equal(t, "2m0s", cfg.Timeout) + assert.True(t, cfg.Lifecycle.Managed()) + assert.Equal(t, "key", cfg.Platform.AccessKey) + assert.Equal(t, "root", cfg.Ssh.User) + assert.Equal(t, config.ShutdownActionStopContainer, cfg.ShutdownAction) +} + +func TestRefreshLifecycleFromNothingIsTimeoutOnly(t *testing.T) { + refreshed, err := RefreshLifecycle("", 0) + require.NoError(t, err) + cfg, err := DecodeDaemonConfig(refreshed) + require.NoError(t, err) + + assert.Empty(t, cfg.Timeout) + assert.True(t, cfg.Lifecycle.Managed()) + assert.Empty(t, cfg.Ssh) +} diff --git a/pkg/daemon/agent/daemon.go b/pkg/daemon/agent/daemon.go index c8f16cc6e8..99c41aba9a 100644 --- a/pkg/daemon/agent/daemon.go +++ b/pkg/daemon/agent/daemon.go @@ -1,8 +1,6 @@ package agent import ( - "encoding/base64" - "encoding/json" "fmt" "os" "os/exec" @@ -32,6 +30,7 @@ type DaemonConfig struct { Ssh SshConfig `json:"ssh"` Timeout string `json:"timeout"` ShutdownAction string `json:"shutdownAction,omitempty"` + Lifecycle LifecycleConfig `json:"lifecycle,omitzero"` } func BuildWorkspaceDaemonConfig( @@ -39,6 +38,7 @@ func BuildWorkspaceDaemonConfig( workspaceConfig *provider2.Workspace, substitutionContext *config.SubstitutionContext, mergedConfig *config.MergedDevContainerConfig, + lifecycle LifecycleOptions, ) (*DaemonConfig, error) { var workdir string if workspaceConfig.Source.GitSubPath != "" { @@ -55,25 +55,28 @@ func BuildWorkspaceDaemonConfig( workdir = substitutionContext.ContainerWorkspaceFolder } - // Get remote user; default to "root" if empty. - user := mergedConfig.RemoteUser - if user == "" { - user = "root" + daemonConfig := &DaemonConfig{ + Platform: platformOptions, + Timeout: FormatTimeout(lifecycle.Timeout), + ShutdownAction: mergedConfig.ShutdownAction, + } + if lifecycle.Managed { + daemonConfig.Lifecycle = LifecycleConfig{ + Owner: LifecycleOwnerManagedDaemon, + SetupArmsTimer: true, + } } - // build info isn't required in the workspace and can be omitted - platformOptions.Build = nil - - shutdownAction := mergedConfig.ShutdownAction - - daemonConfig := &DaemonConfig{ - Platform: platformOptions, - Ssh: SshConfig{ - Workdir: workdir, - User: user, - }, - ShutdownAction: shutdownAction, + // A timeout-only config must not request the embedded SSH or network server. + if platformOptions.AccessKey != "" { + user := mergedConfig.RemoteUser + if user == "" { + user = "root" + } + daemonConfig.Ssh = SshConfig{Workdir: workdir, User: user} } + // build info isn't required in the workspace and can be omitted + daemonConfig.Platform.Build = nil return daemonConfig, nil } @@ -83,23 +86,20 @@ func GetEncodedWorkspaceDaemonConfig( workspaceConfig *provider2.Workspace, substitutionContext *config.SubstitutionContext, mergedConfig *config.MergedDevContainerConfig, + lifecycle LifecycleOptions, ) (string, error) { daemonConfig, err := BuildWorkspaceDaemonConfig( platformOptions, workspaceConfig, substitutionContext, mergedConfig, + lifecycle, ) if err != nil { return "", err } - data, err := json.Marshal(daemonConfig) - if err != nil { - return "", err - } - encoded := base64.StdEncoding.EncodeToString(data) - return encoded, nil + return EncodeDaemonConfig(daemonConfig) } const systemdDir = "/etc/systemd/system" diff --git a/pkg/daemon/agent/daemon_test.go b/pkg/daemon/agent/daemon_test.go index a19c3b10f5..d205f3b424 100644 --- a/pkg/daemon/agent/daemon_test.go +++ b/pkg/daemon/agent/daemon_test.go @@ -155,6 +155,7 @@ func TestBuildWorkspaceDaemonConfig_ShutdownAction(t *testing.T) { &provider2.Workspace{}, &config.SubstitutionContext{}, merged, + LifecycleOptions{}, ) if err != nil { t.Fatalf("unexpected error: %v", err) diff --git a/pkg/daemon/agent/lifecycle_monitor.go b/pkg/daemon/agent/lifecycle_monitor.go new file mode 100644 index 0000000000..38eeaa49bd --- /dev/null +++ b/pkg/daemon/agent/lifecycle_monitor.go @@ -0,0 +1,198 @@ +package agent + +import ( + "context" + "errors" + "fmt" + "os" + "sync" + "time" + + pkgconfig "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/log" +) + +const ( + idleCheckInterval = 500 * time.Millisecond + maxCheckInterval = time.Second + // unreadableLogAfter is how many consecutive failed checks precede a warning. + unreadableLogAfter = 3 +) + +// Decision is the outcome of one inactivity check. +type Decision struct { + Expired bool + // ActivityUnreadable means the activity file could not be read, so the check + // failed safe and the timer is not effective. + ActivityUnreadable bool + Failures int +} + +// LifecycleMonitor decides when the container has been idle for too long. It +// holds no clock or file handles; callers pass observations in. +type LifecycleMonitor struct { + mu sync.Mutex + timeout time.Duration + armed bool + suspended bool + armedAt time.Time + failures int +} + +// Apply updates the timer from a control record. It reports whether the activity +// timestamp should be reset to now. +func (m *LifecycleMonitor) Apply(rec ControlRecord, now time.Time) (bool, error) { + var timeout time.Duration + if rec.Timeout != "" { + var err error + if timeout, err = time.ParseDuration(rec.Timeout); err != nil || timeout < 0 { + return false, fmt.Errorf("invalid lifecycle timeout %q", rec.Timeout) + } + } + enabled := timeout > 0 && rec.ShutdownAction != config.ShutdownActionNone + + m.mu.Lock() + defer m.mu.Unlock() + m.timeout = timeout + switch rec.State { + case LifecycleStateSetup: + m.suspended = true + return false, nil + case LifecycleStateReady, LifecycleStateFailed: + m.suspended = false + m.armed = enabled + m.armedAt = now + m.failures = 0 + return enabled, nil + default: + return false, fmt.Errorf("unknown lifecycle state %q", rec.State) + } +} + +// ArmNow arms the timer without a handshake, for daemons that own no setup. +func (m *LifecycleMonitor) ArmNow(timeout time.Duration, now time.Time) { + m.mu.Lock() + defer m.mu.Unlock() + m.timeout, m.armed, m.suspended, m.armedAt = timeout, timeout > 0, false, now +} + +// Evaluate checks the idle time. A missing activity file counts as no activity +// since arming; any other read error fails safe. +func (m *LifecycleMonitor) Evaluate(now, modTime time.Time, statErr error) Decision { + m.mu.Lock() + defer m.mu.Unlock() + if !m.armed || m.suspended { + return Decision{} + } + if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { + m.failures++ + return Decision{ActivityUnreadable: true, Failures: m.failures} + } + m.failures = 0 + last := m.armedAt + if statErr == nil && modTime.After(last) { + last = modTime + } + return Decision{Expired: now.Sub(last) >= m.timeout} +} + +// CheckInterval bounds how long the loop sleeps between checks. +func (m *LifecycleMonitor) CheckInterval() time.Duration { + m.mu.Lock() + defer m.mu.Unlock() + if !m.armed { + return idleCheckInterval + } + return min(maxCheckInterval, max(m.timeout/4, minTickInterval)) +} + +// LifecycleLoop is the production adapter around LifecycleMonitor. +type LifecycleLoop struct { + // Owner is nil for daemons armed directly with ArmNow. + Owner *LifecycleOwner + Monitor *LifecycleMonitor + ActivityPath string +} + +// Run returns ErrInactivityExpired when the container went idle, nil when ctx +// ended first. +func (l *LifecycleLoop) Run(ctx context.Context) error { + var controlErr, unreadable bool + timer := time.NewTimer(0) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-timer.C: + } + now := time.Now() + controlErr = l.applyControl(now, controlErr) + + fi, err := os.Stat(l.ActivityPath) + var mod time.Time + if err == nil { + mod = fi.ModTime() + } + d := l.Monitor.Evaluate(now, mod, err) + unreadable = logUnreadable(d, err, unreadable) + if d.Expired { + log.Infof("container idle for the configured inactivity timeout, shutting down") + return ErrInactivityExpired + } + timer.Reset(l.Monitor.CheckInterval()) + } +} + +func (l *LifecycleLoop) applyControl(now time.Time, hadErr bool) bool { + if l.Owner == nil { + return false + } + rec, changed, err := l.Owner.PollControl() + if err != nil { + if !hadErr { + log.Errorf("read container lifecycle control record: %v", err) + } + return true + } + if !changed { + return false + } + reset, err := l.Monitor.Apply(rec, now) + if err != nil { + log.Errorf("apply container lifecycle control record: %v", err) + return true + } + if reset { + if err := TouchActivity(l.ActivityPath, now); err != nil { + log.Errorf("reset activity timestamp: %v", err) + } + } + return false +} + +func logUnreadable(d Decision, err error, was bool) bool { + switch { + case d.ActivityUnreadable && d.Failures >= unreadableLogAfter && !was: + log.Errorf( + "cannot read activity file; inactivity timeout is not effective until it is readable: %v", + err, + ) + return true + case !d.ActivityUnreadable && was: + log.Infof("activity file readable again; inactivity timeout is effective") + return false + } + return was +} + +// ActivityIntervalForLifecycle derives the activity cadence from the published +// policy, falling back to the default. +func ActivityIntervalForLifecycle() time.Duration { + timeout, ok := ReadLifecycleTimeout(pkgconfig.ContainerLifecycleDir) + if !ok { + return DefaultActivityInterval + } + return HeartbeatInterval(timeout) +} diff --git a/pkg/daemon/agent/lifecycle_monitor_test.go b/pkg/daemon/agent/lifecycle_monitor_test.go new file mode 100644 index 0000000000..a0f3850850 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_monitor_test.go @@ -0,0 +1,244 @@ +package agent + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var t0 = time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + +func control(state LifecycleState, timeout, action string) ControlRecord { + return ControlRecord{ + Version: LifecycleProtocolVersion, State: state, Timeout: timeout, ShutdownAction: action, + } +} + +func TestMonitorStartsUnarmed(t *testing.T) { + m := &LifecycleMonitor{} + + d := m.Evaluate(t0.Add(24*time.Hour), t0, nil) + + assert.False(t, d.Expired) +} + +func TestMonitorSuspendedDuringSetupThenArmed(t *testing.T) { + m := &LifecycleMonitor{} + + reset, err := m.Apply(control(LifecycleStateSetup, "1m", ""), t0) + require.NoError(t, err) + assert.False(t, reset) + assert.False(t, m.Evaluate(t0.Add(time.Hour), t0, nil).Expired, "suspended by setup") + + ready := t0.Add(10 * time.Minute) + reset, err = m.Apply(control(LifecycleStateReady, "1m", ""), ready) + require.NoError(t, err) + assert.True(t, reset, "arming resets the activity timestamp") + + assert.False(t, m.Evaluate(ready.Add(59*time.Second), t0, nil).Expired, + "old activity cannot expire a freshly armed timer") + assert.True(t, m.Evaluate(ready.Add(time.Minute), t0, nil).Expired) +} + +func TestMonitorActivityKeepsWorkspaceAlive(t *testing.T) { + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) + require.NoError(t, err) + + recent := t0.Add(50 * time.Second) + + assert.False(t, m.Evaluate(t0.Add(100*time.Second), recent, nil).Expired) + assert.True(t, m.Evaluate(t0.Add(111*time.Second), recent, nil).Expired) +} + +func TestMonitorFailedSetupReleasesSuspensionWithFreshInterval(t *testing.T) { + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateSetup, "1m", ""), t0) + require.NoError(t, err) + + failedAt := t0.Add(30 * time.Minute) + _, err = m.Apply(control(LifecycleStateFailed, "1m", ""), failedAt) + require.NoError(t, err) + + assert.False(t, m.Evaluate(failedAt.Add(30*time.Second), t0, nil).Expired) + assert.True(t, m.Evaluate(failedAt.Add(time.Minute), t0, nil).Expired) +} + +func TestMonitorRepeatedSetupSuspendsAgainAndUpdatesPolicy(t *testing.T) { + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) + require.NoError(t, err) + + _, err = m.Apply(control(LifecycleStateSetup, "5m", ""), t0.Add(time.Hour)) + require.NoError(t, err) + assert.False(t, m.Evaluate(t0.Add(2*time.Hour), t0, nil).Expired) + + rearmed := t0.Add(3 * time.Hour) + _, err = m.Apply(control(LifecycleStateReady, "5m", ""), rearmed) + require.NoError(t, err) + assert.False(t, m.Evaluate(rearmed.Add(4*time.Minute), rearmed, nil).Expired) + assert.True(t, m.Evaluate(rearmed.Add(5*time.Minute), rearmed, nil).Expired) +} + +func TestMonitorDisabledPolicyDisarms(t *testing.T) { + for name, rec := range map[string]ControlRecord{ + "empty timeout": control(LifecycleStateReady, "", ""), + "shutdown none": control(LifecycleStateReady, "1m", "none"), + } { + t.Run(name, func(t *testing.T) { + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) + require.NoError(t, err) + + reset, err := m.Apply(rec, t0) + + require.NoError(t, err) + assert.False(t, reset) + assert.False(t, m.Evaluate(t0.Add(24*time.Hour), t0, nil).Expired) + }) + } +} + +func TestMonitorRejectsInvalidControlRecords(t *testing.T) { + m := &LifecycleMonitor{} + + _, err := m.Apply(control(LifecycleStateReady, "later", ""), t0) + assert.Error(t, err) + _, err = m.Apply(control("bogus", "1m", ""), t0) + assert.Error(t, err) +} + +func TestMonitorUnreadableActivityFailsSafe(t *testing.T) { + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) + require.NoError(t, err) + denied := &fs.PathError{Op: "stat", Path: "activity", Err: fs.ErrPermission} + + var d Decision + for range 5 { + d = m.Evaluate(t0.Add(time.Hour), time.Time{}, denied) + } + + assert.False(t, d.Expired, "an active workspace must not stop because activity is unreadable") + assert.True(t, d.ActivityUnreadable) + assert.Equal(t, 5, d.Failures) +} + +func TestMonitorMissingActivityFileCountsFromArming(t *testing.T) { + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) + require.NoError(t, err) + + d := m.Evaluate(t0.Add(time.Minute), time.Time{}, os.ErrNotExist) + + assert.True(t, d.Expired) + assert.False(t, d.ActivityUnreadable) +} + +func TestIntervalsScaleWithShortTimeouts(t *testing.T) { + assert.Equal(t, 10*time.Second, HeartbeatInterval(0)) + assert.Equal(t, 10*time.Second, HeartbeatInterval(time.Minute), "minute scale keeps 10s") + assert.Equal(t, 10*time.Second, HeartbeatInterval(30*time.Second)) + assert.Equal(t, 2*time.Second, HeartbeatInterval(6*time.Second)) + assert.LessOrEqual(t, HeartbeatInterval(900*time.Millisecond), 300*time.Millisecond) + + m := &LifecycleMonitor{} + assert.Equal(t, idleCheckInterval, m.CheckInterval(), "unarmed polls for control records") + _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) + require.NoError(t, err) + assert.Equal(t, time.Second, m.CheckInterval()) + _, err = m.Apply(control(LifecycleStateReady, "2s", ""), t0) + require.NoError(t, err) + assert.Equal(t, 500*time.Millisecond, m.CheckInterval()) +} + +func TestLifecycleLoopStopsIdleManagedDaemon(t *testing.T) { + owner, dir := newTestOwner(t) + activity := filepath.Join(t.TempDir(), "activity") + loop := &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}, ActivityPath: activity} + require.NoError(t, connectNow(t, dir).Send(LifecycleStateReady, "300ms", "")) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + err := loop.Run(ctx) + + assert.True(t, errors.Is(err, ErrInactivityExpired), "got %v", err) +} + +func TestLifecycleLoopStaysUnarmedWithoutHandshake(t *testing.T) { + owner, _ := newTestOwner(t) + loop := &LifecycleLoop{ + Owner: owner, + Monitor: &LifecycleMonitor{}, + ActivityPath: filepath.Join(t.TempDir(), "activity"), + } + ctx, cancel := context.WithTimeout(context.Background(), 800*time.Millisecond) + defer cancel() + + err := loop.Run(ctx) + + assert.NoError(t, err) +} + +func TestLifecycleLoopRespectsFreshActivity(t *testing.T) { + owner, dir := newTestOwner(t) + activity := filepath.Join(t.TempDir(), "activity") + loop := &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}, ActivityPath: activity} + require.NoError(t, connectNow(t, dir).Send(LifecycleStateReady, "400ms", "")) + + ctx, cancel := context.WithTimeout(context.Background(), 1200*time.Millisecond) + defer cancel() + go ActivityWriter{Path: activity, Interval: 100 * time.Millisecond}.Run(ctx) + + assert.NoError(t, loop.Run(ctx), "continuous activity must keep the workspace running") +} + +func TestActivityWriterTouchesImmediatelyPeriodicallyAndOnExit(t *testing.T) { + ticks := make(chan time.Time) + var mu sync.Mutex + var touched []time.Time + count := func() int { mu.Lock(); defer mu.Unlock(); return len(touched) } + w := ActivityWriter{ + Path: "activity", + Interval: time.Second, + Touch: func(_ string, now time.Time) error { + mu.Lock() + defer mu.Unlock() + touched = append(touched, now) + return nil + }, + Now: func() time.Time { return t0 }, + Tick: func(time.Duration) (<-chan time.Time, func()) { return ticks, func() {} }, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { w.Run(ctx); close(done) }() + + require.Eventually(t, func() bool { return count() == 1 }, time.Second, time.Millisecond) + ticks <- time.Time{} + require.Eventually(t, func() bool { return count() == 2 }, time.Second, time.Millisecond) + cancel() + <-done + + assert.Equal(t, 3, count(), "start, one tick, and a final touch at exit") +} + +func TestTouchByWriteAdvancesModTimeForNonOwners(t *testing.T) { + path := filepath.Join(t.TempDir(), "activity") + require.NoError(t, TouchActivity(path, t0)) + require.NoError(t, os.Chtimes(path, t0, t0)) + + require.NoError(t, touchByWrite(path)) + + info, err := os.Stat(path) + require.NoError(t, err) + assert.WithinDuration(t, time.Now(), info.ModTime(), 2*time.Second) +} diff --git a/pkg/daemon/agent/lifecycle_proc_linux.go b/pkg/daemon/agent/lifecycle_proc_linux.go new file mode 100644 index 0000000000..8e9f187f14 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_proc_linux.go @@ -0,0 +1,30 @@ +//go:build linux + +package agent + +import ( + "os" + "strconv" + "strings" +) + +// procStartTicks returns field 22 of /proc//stat, which distinguishes a +// process from a later one that reuses its PID. +func procStartTicks(pid int) string { + raw, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat") // #nosec G304 + if err != nil { + return "" + } + stat := string(raw) + // The command name may contain spaces; fields resume after the last ')'. + end := strings.LastIndexByte(stat, ')') + if end < 0 { + return "" + } + fields := strings.Fields(stat[end+1:]) + // fields[0] is state (field 3), so start time (field 22) is fields[19]. + if len(fields) < 20 { + return "" + } + return fields[19] +} diff --git a/pkg/daemon/agent/lifecycle_proc_other.go b/pkg/daemon/agent/lifecycle_proc_other.go new file mode 100644 index 0000000000..0026fadae4 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_proc_other.go @@ -0,0 +1,5 @@ +//go:build !linux + +package agent + +func procStartTicks(int) string { return "" } diff --git a/pkg/daemon/agent/lifecycle_protocol.go b/pkg/daemon/agent/lifecycle_protocol.go new file mode 100644 index 0000000000..e3bb28da7c --- /dev/null +++ b/pkg/daemon/agent/lifecycle_protocol.go @@ -0,0 +1,346 @@ +package agent + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" +) + +const ( + // LifecycleProtocolVersion is bumped when owner or control records change shape. + LifecycleProtocolVersion = 1 + + ownerFileName = "owner.json" + controlFileName = "control.json" + maxRecordBytes = 4096 + connectPollEvery = 100 * time.Millisecond + runtimeDirMode = 0o755 + recordFileMode = 0o644 +) + +var ( + // ErrLifecycleOwnerMissing means no live managed daemon published an owner record. + ErrLifecycleOwnerMissing = errors.New("container lifecycle owner not found") + // ErrLifecycleOwnerIncompatible means the owner speaks another protocol version. + ErrLifecycleOwnerIncompatible = errors.New("container lifecycle owner is incompatible") +) + +// LifecycleState is what setup tells the daemon about the timer. +type LifecycleState string + +const ( + // LifecycleStateSetup suspends the timer while synchronous setup runs. + LifecycleStateSetup LifecycleState = "setup" + // LifecycleStateReady resets activity and arms the timer. + LifecycleStateReady LifecycleState = "ready" + // LifecycleStateFailed releases the suspension and starts a fresh idle interval. + LifecycleStateFailed LifecycleState = "failed" +) + +// OwnerRecord is published by the daemon process that owns the timer. +type OwnerRecord struct { + Version int `json:"version"` + Generation string `json:"generation"` + PID int `json:"pid"` + StartTicks string `json:"startTicks,omitempty"` +} + +// ControlRecord is written by setup. It never carries credentials. +type ControlRecord struct { + Version int `json:"version"` + Generation string `json:"generation"` + Timeout string `json:"timeout"` + ShutdownAction string `json:"shutdownAction"` + State LifecycleState `json:"state"` +} + +// trustedOwner reports whether files owned by uid may be believed by a process +// running as euid. +func trustedOwner(uid, euid int) bool { + return uid == euid || uid == 0 +} + +func checkDir(dir string, euid int) error { + fi, err := os.Lstat(dir) + if err != nil { + return err + } + if !fi.IsDir() { + return fmt.Errorf("lifecycle runtime path %s is not a directory", dir) + } + if uid, ok := fileOwner(fi); ok && !trustedOwner(uid, euid) { + return fmt.Errorf("lifecycle runtime dir %s is owned by uid %d", dir, uid) + } + if fi.Mode().Perm()&0o022 != 0 { + return fmt.Errorf("lifecycle runtime dir %s is writable by others", dir) + } + return nil +} + +func writeRecord(dir, name string, v any, euid int) error { + if err := checkDir(dir, euid); err != nil { + return err + } + raw, err := json.Marshal(v) + if err != nil { + return err + } + suffix := make([]byte, 6) + if _, err := rand.Read(suffix); err != nil { + return err + } + tmp := filepath.Join(dir, "."+name+"."+hex.EncodeToString(suffix)+".tmp") + f, err := openNoFollow(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, recordFileMode) + if err != nil { + return fmt.Errorf("create lifecycle record: %w", err) + } + if _, err := f.Write(raw); err != nil { + _ = f.Close() + _ = os.Remove(tmp) + return fmt.Errorf("write lifecycle record: %w", err) + } + if err := f.Close(); err != nil { + _ = os.Remove(tmp) + return err + } + if err := os.Rename(tmp, filepath.Join(dir, name)); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("publish lifecycle record: %w", err) + } + return nil +} + +// recordReader loads records from a runtime dir. When strict, the dir and each +// file must be owned by a trusted uid; lenient reads only reject symlinks and +// non-regular files. +type recordReader struct { + dir string + euid int + strict bool +} + +func (r recordReader) read(name string, out any) error { + if r.strict { + if err := checkDir(r.dir, r.euid); err != nil { + return err + } + } + f, err := openNoFollow(filepath.Join(r.dir, name), os.O_RDONLY, 0) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + if err := r.checkFile(f, name); err != nil { + return err + } + raw, err := io.ReadAll(io.LimitReader(f, maxRecordBytes)) + if err != nil { + return err + } + return json.Unmarshal(raw, out) +} + +func (r recordReader) checkFile(f *os.File, name string) error { + fi, err := f.Stat() + if err != nil { + return err + } + if !fi.Mode().IsRegular() || fi.Size() > maxRecordBytes { + return fmt.Errorf("lifecycle record %s is not a small regular file", name) + } + if uid, ok := fileOwner(fi); r.strict && ok && !trustedOwner(uid, r.euid) { + return fmt.Errorf("lifecycle record %s is owned by uid %d", name, uid) + } + return nil +} + +// LifecycleOwner is the daemon-side handle of the lifecycle protocol. +type LifecycleOwner struct { + dir string + euid int + generation string + last ControlRecord +} + +// NewLifecycleOwner prepares a private runtime dir, discards records from any +// earlier process, and publishes a fresh owner record. +func NewLifecycleOwner(dir string) (*LifecycleOwner, error) { + euid := os.Geteuid() + if err := prepareRuntimeDir(dir, euid); err != nil { + return nil, err + } + gen := make([]byte, 16) + if _, err := rand.Read(gen); err != nil { + return nil, err + } + o := &LifecycleOwner{dir: dir, euid: euid, generation: hex.EncodeToString(gen)} + pid := os.Getpid() + err := writeRecord(dir, ownerFileName, OwnerRecord{ + Version: LifecycleProtocolVersion, + Generation: o.generation, + PID: pid, + StartTicks: procStartTicks(pid), + }, euid) + if err != nil { + return nil, err + } + return o, nil +} + +func prepareRuntimeDir(dir string, euid int) error { + if err := os.MkdirAll(filepath.Dir(dir), runtimeDirMode); err != nil { + return err + } + if err := os.Mkdir(dir, runtimeDirMode); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + if err := requireOwnedDir(dir, euid); err != nil { + return err + } + if err := os.Chmod(dir, runtimeDirMode); err != nil { + return err + } + if err := checkDir(dir, euid); err != nil { + return err + } + return discardStaleRecords(dir) +} + +func requireOwnedDir(dir string, euid int) error { + fi, err := os.Lstat(dir) + if err != nil { + return err + } + if uid, ok := fileOwner(fi); ok && uid != euid { + return fmt.Errorf("lifecycle runtime dir %s is owned by uid %d, not %d", dir, uid, euid) + } + return nil +} + +// discardStaleRecords removes records from an earlier process or mount so they +// cannot arm this one. +func discardStaleRecords(dir string) error { + for _, name := range []string{ownerFileName, controlFileName} { + if err := os.Remove( + filepath.Join(dir, name), + ); err != nil && + !errors.Is(err, os.ErrNotExist) { + return err + } + } + return nil +} + +// Generation identifies this daemon process. +func (o *LifecycleOwner) Generation() string { return o.generation } + +// PollControl returns the control record when it is valid for this generation +// and differs from the last one returned. +func (o *LifecycleOwner) PollControl() (ControlRecord, bool, error) { + var rec ControlRecord + err := recordReader{dir: o.dir, euid: o.euid, strict: true}.read(controlFileName, &rec) + switch { + case errors.Is(err, os.ErrNotExist): + return ControlRecord{}, false, nil + case err != nil: + return ControlRecord{}, false, err + case rec.Version != LifecycleProtocolVersion || rec.Generation != o.generation: + return ControlRecord{}, false, nil + case rec == o.last: + return ControlRecord{}, false, nil + } + o.last = rec + return rec, true, nil +} + +// LifecycleClient is the setup-side handle of the lifecycle protocol. +type LifecycleClient struct { + dir string + euid int + owner OwnerRecord +} + +// ConnectLifecycle waits, bounded by ctx, for a live owner record in dir. +func ConnectLifecycle(ctx context.Context, dir string) (*LifecycleClient, error) { + euid := os.Geteuid() + var lastErr error + for { + owner, err := liveOwner(dir, euid) + switch { + case owner != nil: + return &LifecycleClient{dir: dir, euid: euid, owner: *owner}, nil + case errors.Is(err, ErrLifecycleOwnerIncompatible): + return nil, err + case err != nil: + lastErr = err + } + select { + case <-ctx.Done(): + if lastErr != nil { + return nil, fmt.Errorf("%w: %w", ErrLifecycleOwnerMissing, lastErr) + } + return nil, ErrLifecycleOwnerMissing + case <-time.After(connectPollEvery): + } + } +} + +// liveOwner returns the published owner when its process is still the one that +// wrote the record. A missing or stale record is (nil, nil). +func liveOwner(dir string, euid int) (*OwnerRecord, error) { + var owner OwnerRecord + err := recordReader{dir: dir, euid: euid, strict: true}.read(ownerFileName, &owner) + switch { + case errors.Is(err, os.ErrNotExist): + return nil, nil + case err != nil: + return nil, err + case owner.Version != LifecycleProtocolVersion: + return nil, fmt.Errorf("%w: protocol version %d, want %d", + ErrLifecycleOwnerIncompatible, owner.Version, LifecycleProtocolVersion) + case !ownerAlive(owner): + return nil, nil + } + return &owner, nil +} + +func ownerAlive(o OwnerRecord) bool { + if o.PID <= 0 || !processExists(o.PID) { + return false + } + ticks := procStartTicks(o.PID) + return o.StartTicks == "" || ticks == "" || ticks == o.StartTicks +} + +// Send publishes the timer state for the connected owner. +func (c *LifecycleClient) Send(state LifecycleState, timeout, shutdownAction string) error { + return writeRecord(c.dir, controlFileName, ControlRecord{ + Version: LifecycleProtocolVersion, + Generation: c.owner.Generation, + Timeout: timeout, + ShutdownAction: shutdownAction, + State: state, + }, c.euid) +} + +// ReadLifecycleTimeout leniently reads the timeout from the current control +// record, for deriving activity cadence. It reports false when there is none. +func ReadLifecycleTimeout(dir string) (time.Duration, bool) { + var rec ControlRecord + err := recordReader{dir: dir}.read(controlFileName, &rec) + if err != nil || rec.Version != LifecycleProtocolVersion { + return 0, false + } + d, err := time.ParseDuration(rec.Timeout) + if err != nil || d <= 0 { + return 0, rec.Timeout == "" + } + return d, true +} diff --git a/pkg/daemon/agent/lifecycle_protocol_test.go b/pkg/daemon/agent/lifecycle_protocol_test.go new file mode 100644 index 0000000000..534e49e7f8 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_protocol_test.go @@ -0,0 +1,158 @@ +package agent + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestOwner(t *testing.T) (*LifecycleOwner, string) { + t.Helper() + dir := filepath.Join(t.TempDir(), "lifecycle") + owner, err := NewLifecycleOwner(dir) + require.NoError(t, err) + return owner, dir +} + +func connectNow(t *testing.T, dir string) *LifecycleClient { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client, err := ConnectLifecycle(ctx, dir) + require.NoError(t, err) + return client +} + +func TestSetupControlRecordReachesOwner(t *testing.T) { + owner, dir := newTestOwner(t) + client := connectNow(t, dir) + + require.NoError(t, client.Send(LifecycleStateSetup, "1m", "stopContainer")) + + rec, changed, err := owner.PollControl() + require.NoError(t, err) + require.True(t, changed) + assert.Equal(t, LifecycleStateSetup, rec.State) + assert.Equal(t, "1m", rec.Timeout) + assert.Equal(t, owner.Generation(), rec.Generation) + + _, changed, err = owner.PollControl() + require.NoError(t, err) + assert.False(t, changed, "an unchanged record is not reapplied") +} + +func TestControlRecordCarriesNoExtraFields(t *testing.T) { + _, dir := newTestOwner(t) + require.NoError(t, connectNow(t, dir).Send(LifecycleStateReady, "1m", "none")) + + raw, err := os.ReadFile(filepath.Join(dir, controlFileName)) //nolint:gosec // test path + require.NoError(t, err) + var fields map[string]any + require.NoError(t, json.Unmarshal(raw, &fields)) + + keys := make([]string, 0, len(fields)) + for k := range fields { + keys = append(keys, k) + } + assert.ElementsMatch( + t, + []string{"version", "generation", "timeout", "shutdownAction", "state"}, + keys, + ) +} + +func TestStaleGenerationCannotArmNewOwner(t *testing.T) { + first, dir := newTestOwner(t) + oldClient := connectNow(t, dir) + require.NoError(t, oldClient.Send(LifecycleStateReady, "1m", "")) + require.NotEmpty(t, first.Generation()) + + // A restarted daemon on the same directory discards the old record. + second, err := NewLifecycleOwner(dir) + require.NoError(t, err) + require.NotEqual(t, first.Generation(), second.Generation()) + _, changed, err := second.PollControl() + require.NoError(t, err) + assert.False(t, changed) + + // Even a stale record that reappears is ignored by generation. + require.NoError(t, oldClient.Send(LifecycleStateReady, "1m", "")) + _, changed, err = second.PollControl() + require.NoError(t, err) + assert.False(t, changed) +} + +func TestConnectLifecycleWithoutOwnerFailsBounded(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + + _, err := ConnectLifecycle(ctx, filepath.Join(t.TempDir(), "absent")) + + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) +} + +func TestConnectLifecycleRejectsIncompatibleOwner(t *testing.T) { + _, dir := newTestOwner(t) + require.NoError(t, writeRecord(dir, ownerFileName, OwnerRecord{ + Version: LifecycleProtocolVersion + 1, Generation: "g", PID: os.Getpid(), + }, os.Geteuid())) + + _, err := ConnectLifecycle(context.Background(), dir) + + assert.ErrorIs(t, err, ErrLifecycleOwnerIncompatible) +} + +func TestConnectLifecycleIgnoresOwnerOfDeadProcess(t *testing.T) { + _, dir := newTestOwner(t) + require.NoError(t, writeRecord(dir, ownerFileName, OwnerRecord{ + Version: LifecycleProtocolVersion, Generation: "g", PID: 0x7ffffffe, + }, os.Geteuid())) + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + + _, err := ConnectLifecycle(ctx, dir) + + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) +} + +func TestOwnerRejectsSymlinkedControlRecord(t *testing.T) { + owner, dir := newTestOwner(t) + target := filepath.Join(t.TempDir(), "target.json") + require.NoError(t, os.WriteFile(target, []byte(`{}`), 0o600)) + require.NoError(t, os.Symlink(target, filepath.Join(dir, controlFileName))) + + _, changed, err := owner.PollControl() + + assert.Error(t, err) + assert.False(t, changed) +} + +func TestNewLifecycleOwnerRejectsSymlinkedDir(t *testing.T) { + base := t.TempDir() + real := filepath.Join(base, "real") + require.NoError(t, os.Mkdir(real, 0o755)) //nolint:gosec // test dir + link := filepath.Join(base, "link") + require.NoError(t, os.Symlink(real, link)) + + _, err := NewLifecycleOwner(link) + + assert.Error(t, err) +} + +func TestReadLifecycleTimeoutFollowsControlRecord(t *testing.T) { + _, dir := newTestOwner(t) + _, ok := ReadLifecycleTimeout(dir) + assert.False(t, ok, "no control record yet") + + require.NoError(t, connectNow(t, dir).Send(LifecycleStateSetup, "45s", "")) + + timeout, ok := ReadLifecycleTimeout(dir) + assert.True(t, ok) + assert.Equal(t, 45*time.Second, timeout) +} diff --git a/pkg/daemon/agent/lifecycle_unix.go b/pkg/daemon/agent/lifecycle_unix.go new file mode 100644 index 0000000000..0ab165f623 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_unix.go @@ -0,0 +1,27 @@ +//go:build !windows + +package agent + +import ( + "os" + "syscall" +) + +func fileOwner(fi os.FileInfo) (int, bool) { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, false + } + return int(st.Uid), true +} + +// openNoFollow refuses a trailing symlink and never blocks on a FIFO. +func openNoFollow(path string, flag int, mode os.FileMode) (*os.File, error) { + //nolint:gosec // callers pass fixed lifecycle record paths + return os.OpenFile(path, flag|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, mode) +} + +func processExists(pid int) bool { + err := syscall.Kill(pid, 0) + return err == nil || err == syscall.EPERM +} diff --git a/pkg/daemon/agent/lifecycle_windows.go b/pkg/daemon/agent/lifecycle_windows.go new file mode 100644 index 0000000000..07bf1d0342 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_windows.go @@ -0,0 +1,14 @@ +//go:build windows + +package agent + +import "os" + +func fileOwner(os.FileInfo) (int, bool) { return 0, false } + +func openNoFollow(path string, flag int, mode os.FileMode) (*os.File, error) { + //nolint:gosec // callers pass fixed lifecycle record paths + return os.OpenFile(path, flag, mode) +} + +func processExists(int) bool { return false } diff --git a/pkg/devcontainer/managed_lifecycle_test.go b/pkg/devcontainer/managed_lifecycle_test.go new file mode 100644 index 0000000000..8958670d55 --- /dev/null +++ b/pkg/devcontainer/managed_lifecycle_test.go @@ -0,0 +1,196 @@ +package devcontainer + +import ( + "context" + "testing" + "time" + + "github.com/devsy-org/api/pkg/devsy" + pkgagent "github.com/devsy-org/devsy/pkg/agent" + pkgconfig "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testEntrypoint = "/app" + +type managedMockDriver struct{ *mockDriver } + +func (*managedMockDriver) ManagedLifecycle() bool { return true } + +func managedRunner(timeout string) (*runner, *managedMockDriver) { + d := &managedMockDriver{mockDriver: &mockDriver{}} + r := newTestRunner(d) + r.workspaceConfig.Workspace = &provider.Workspace{} + r.workspaceConfig.Agent.ContainerTimeout = timeout + return r, d +} + +func resolveParamsFor(platform devsy.PlatformOptions) *resolveParams { + return &resolveParams{ + substitutionContext: &config.SubstitutionContext{ + ContainerWorkspaceFolder: "/workspaces/app", + }, + options: UpOptions{CLIOptions: provider.CLIOptions{Platform: platform}}, + } +} + +func mergedWithEnv() *config.MergedDevContainerConfig { + merged := &config.MergedDevContainerConfig{} + merged.ContainerEnv = map[string]string{} + merged.ShutdownAction = config.ShutdownActionStopContainer + return merged +} + +func injectedConfig(t *testing.T, merged *config.MergedDevContainerConfig) *agentd.DaemonConfig { + t.Helper() + encoded, ok := merged.ContainerEnv[pkgconfig.EnvWorkspaceDaemonConfig] + require.True(t, ok, "daemon config was not injected") + cfg, err := agentd.DecodeDaemonConfig(encoded) + require.NoError(t, err) + return cfg +} + +func TestInjectDaemonEntrypointManagedWithoutPlatform(t *testing.T) { + r, _ := managedRunner("1m") + merged := mergedWithEnv() + + require.NoError(t, r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged)) + + cfg := injectedConfig(t, merged) + assert.Equal(t, "1m0s", cfg.Timeout) + assert.Equal(t, config.ShutdownActionStopContainer, cfg.ShutdownAction) + assert.True(t, cfg.Lifecycle.Managed()) + assert.Empty(t, cfg.Ssh, "no embedded SSH for a timeout-only config") + assert.Empty(t, cfg.Platform.AccessKey) +} + +func TestInjectDaemonEntrypointManagedWithPlatformKeepsNetworking(t *testing.T) { + r, _ := managedRunner("1m") + merged := mergedWithEnv() + platform := devsy.PlatformOptions{AccessKey: "key", PlatformHost: "p", WorkspaceHost: "w"} + + require.NoError(t, r.injectDaemonEntrypoint(resolveParamsFor(platform), merged)) + + cfg := injectedConfig(t, merged) + assert.Equal(t, "key", cfg.Platform.AccessKey) + assert.Equal(t, "root", cfg.Ssh.User) + assert.Equal(t, "1m0s", cfg.Timeout) +} + +func TestInjectDaemonEntrypointUnmanagedWithoutPlatformIsUntouched(t *testing.T) { + r := newTestRunner(&mockDriver{}) + r.workspaceConfig.Workspace = &provider.Workspace{} + merged := mergedWithEnv() + + require.NoError(t, r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged)) + + assert.Empty(t, merged.ContainerEnv) +} + +func TestInjectDaemonEntrypointResolvesLegacyAlias(t *testing.T) { + r, _ := managedRunner("") + r.workspaceConfig.Agent.Kubernetes.InactivityTimeout = "5m" + merged := mergedWithEnv() + + require.NoError(t, r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged)) + + assert.Equal(t, "5m0s", injectedConfig(t, merged).Timeout) +} + +func TestInjectDaemonEntrypointPropagatesConfigErrors(t *testing.T) { + r, _ := managedRunner("1m") + r.workspaceConfig.Agent.Kubernetes.InactivityTimeout = "2m" + + err := r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), mergedWithEnv()) + + assert.ErrorContains(t, err, "conflicts") +} + +func TestInjectDaemonEntrypointShutdownNoneDisablesTimeout(t *testing.T) { + r, _ := managedRunner("1m") + merged := mergedWithEnv() + merged.ShutdownAction = config.ShutdownActionNone + + require.NoError(t, r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged)) + + cfg := injectedConfig(t, merged) + assert.Empty(t, cfg.Timeout) + assert.Equal(t, config.ShutdownActionNone, cfg.ShutdownAction) +} + +func TestWorkspaceInfoCarriesEffectiveTimeoutAndOwner(t *testing.T) { + r, _ := managedRunner("") + r.workspaceConfig.Agent.Kubernetes.InactivityTimeout = "90s" + + compressed, err := r.compressWorkspaceConfig(mergedWithEnv()) + require.NoError(t, err) + info, _, err := pkgagent.DecodeContainerWorkspaceInfo(compressed) + require.NoError(t, err) + + assert.Equal(t, "1m30s", info.ContainerTimeout) + assert.Equal(t, provider.ContainerLifecycleOwnerManagedDaemon, info.LifecycleOwner) +} + +func TestManagedTimeoutConflict(t *testing.T) { + no, yes := false, true + withEntrypoint := &config.BuildInfo{ImageDetails: &config.ImageDetails{ + Config: config.ImageDetailsConfig{Entrypoint: []string{testEntrypoint}}, + }} + empty := &config.BuildInfo{ImageDetails: &config.ImageDetails{}} + + tests := []struct { + name string + override *bool + build *config.BuildInfo + wantErr string + }{ + {name: "default override", build: withEntrypoint}, + {name: "explicit override", override: &yes, build: withEntrypoint}, + { + name: "user entrypoint bypasses daemon", + override: &no, + build: withEntrypoint, + wantErr: "overrideCommand", + }, + {name: "no user entrypoint to bypass", override: &no, build: empty}, + { + name: "dockerless nested container", + build: &config.BuildInfo{Dockerless: &config.BuildInfoDockerless{}}, + wantErr: "dockerless", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + merged := &config.MergedDevContainerConfig{} + merged.OverrideCommand = tt.override + + err := managedTimeoutConflict(time.Minute, merged, tt.build) + + if tt.wantErr == "" { + assert.NoError(t, err) + return + } + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestFindRunningContainerOrFailRejectsTerminalObjectsForManagedDrivers(t *testing.T) { + r, d := managedRunner("1m") + d.findResult = &config.ContainerDetails{ + State: config.ContainerDetailsState{Status: config.ContainerStatusExited}, + } + + _, err := r.findRunningContainerOrFail(context.Background(), "start") + + assert.ErrorContains(t, err, "not running") + + plain := newTestRunner(&mockDriver{findResult: d.findResult}) + details, err := plain.findRunningContainerOrFail(context.Background(), "start") + require.NoError(t, err) + assert.NotNil(t, details, "other drivers keep their existing behavior") +} diff --git a/pkg/devcontainer/setup.go b/pkg/devcontainer/setup.go index f457cb682a..811e0d82b5 100644 --- a/pkg/devcontainer/setup.go +++ b/pkg/devcontainer/setup.go @@ -18,6 +18,7 @@ import ( "github.com/devsy-org/devsy/pkg/agent/tunnelserver" "github.com/devsy-org/devsy/pkg/compress" pkgconfig "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/devcontainer/crane" "github.com/devsy-org/devsy/pkg/devcontainer/sshtunnel" @@ -316,7 +317,7 @@ func (r *runner) prepareSetupInfo(params *setupContainerParams) (*setupInfo, err return nil, err } - workspaceConfigCompressed, err := r.compressWorkspaceConfig() + workspaceConfigCompressed, err := r.compressWorkspaceConfig(result.MergedConfig) if err != nil { return nil, err } @@ -375,7 +376,9 @@ func (r *runner) compressResult(result *config.Result) (string, error) { return compressed, nil } -func (r *runner) compressWorkspaceConfig() (string, error) { +func (r *runner) compressWorkspaceConfig( + mergedConfig *config.MergedDevContainerConfig, +) (string, error) { cliOptions := r.workspaceConfig.CLIOptions cliOptions.SecretsEnv = nil cliOptions.SecretsMount = nil @@ -396,6 +399,14 @@ func (r *runner) compressWorkspaceConfig() (string, error) { Agent: r.workspaceConfig.Agent, ContentFolder: r.workspaceConfig.ContentFolder, } + if r.managedLifecycle() { + timeout, err := r.effectiveContainerTimeout(mergedConfig.ShutdownAction) + if err != nil { + return "", err + } + workspaceConfig.ContainerTimeout = agentd.FormatTimeout(timeout) + workspaceConfig.LifecycleOwner = provider2.ContainerLifecycleOwnerManagedDaemon + } workspaceConfig.PullFromInsideContainer = resolvePullFromInsideContainer( r.workspaceConfig.CLIOptions, r.workspaceConfig.Workspace.Source.GitRepository, diff --git a/pkg/devcontainer/setup_secrets_test.go b/pkg/devcontainer/setup_secrets_test.go index db61ba5fb8..2fe2ea879a 100644 --- a/pkg/devcontainer/setup_secrets_test.go +++ b/pkg/devcontainer/setup_secrets_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/devsy-org/devsy/pkg/compress" + "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/provider" ) @@ -31,7 +32,7 @@ func TestCompressWorkspaceConfig_StripsSecretValues(t *testing.T) { }, } - compressed, err := r.compressWorkspaceConfig() + compressed, err := r.compressWorkspaceConfig(&config.MergedDevContainerConfig{}) if err != nil { t.Fatal(err) } diff --git a/pkg/devcontainer/single.go b/pkg/devcontainer/single.go index 4c256eae2d..7a7939556b 100644 --- a/pkg/devcontainer/single.go +++ b/pkg/devcontainer/single.go @@ -7,7 +7,9 @@ import ( "maps" "os" "strings" + "time" + "github.com/devsy-org/api/pkg/devsy" "github.com/devsy-org/devsy/pkg/agent/delivery" "github.com/devsy-org/devsy/pkg/clierr" "github.com/devsy-org/devsy/pkg/command" @@ -390,7 +392,13 @@ func (r *runner) resolveNewContainer( return nil, err } - r.injectDaemonEntrypoint(p, mergedConfig) + if err := r.injectDaemonEntrypoint(p, mergedConfig); err != nil { + return nil, err + } + + if err := r.validateManagedTimeoutTopology(mergedConfig, buildInfo); err != nil { + return nil, err + } r.attemptPreStartDelivery(ctx, mergedConfig, p, buildInfo) @@ -675,27 +683,97 @@ func (r *runner) deleteForRecreate(ctx context.Context) error { } // injectDaemonEntrypoint adds the workspace daemon config to the container -// environment when platform configuration is provided. +// environment for Platform workspaces and for drivers whose main process is the +// managed lifecycle daemon. func (r *runner) injectDaemonEntrypoint( p *resolveParams, mergedConfig *config.MergedDevContainerConfig, -) { - if p.options.Platform.AccessKey == "" { - return +) error { + managed := r.managedLifecycle() + platform := p.options.Platform + if platform.AccessKey == "" { + platform = devsy.PlatformOptions{} + } + if !managed && platform.AccessKey == "" { + return nil + } + + var timeout time.Duration + if managed { + var err error + timeout, err = r.effectiveContainerTimeout(mergedConfig.ShutdownAction) + if err != nil { + return err + } } - log.Debugf("Platform config detected, injecting Devsy daemon entrypoint.") data, err := agent.GetEncodedWorkspaceDaemonConfig( - p.options.Platform, + platform, r.workspaceConfig.Workspace, p.substitutionContext, mergedConfig, + agent.LifecycleOptions{Managed: managed, Timeout: timeout}, ) if err != nil { - log.Errorf("Failed to marshal daemon config: %v", err) - return + return fmt.Errorf("build daemon config: %w", err) } mergedConfig.ContainerEnv[pkgconfig.EnvWorkspaceDaemonConfig] = data + return nil +} + +// validateManagedTimeoutTopology rejects configurations in which the managed +// daemon would not be the process whose exit stops the container. +func (r *runner) validateManagedTimeoutTopology( + mergedConfig *config.MergedDevContainerConfig, + buildInfo *config.BuildInfo, +) error { + if !r.managedLifecycle() { + return nil + } + timeout, err := r.effectiveContainerTimeout(mergedConfig.ShutdownAction) + if err != nil || timeout == 0 { + return err + } + return managedTimeoutConflict(timeout, mergedConfig, buildInfo) +} + +func managedTimeoutConflict( + timeout time.Duration, + mergedConfig *config.MergedDevContainerConfig, + buildInfo *config.BuildInfo, +) error { + if buildInfo.Dockerless != nil { + return fmt.Errorf( + "inactivity timeout %s is not supported with dockerless builds because the nested "+ + "workspace container does not end the pod when it exits; unset the inactivity "+ + "timeout or disable dockerless", timeout) + } + overridden := mergedConfig.OverrideCommand == nil || *mergedConfig.OverrideCommand + if overridden || buildInfo.ImageDetails == nil { + return nil + } + if len(buildInfo.ImageDetails.Config.Entrypoint)+len(buildInfo.ImageDetails.Config.Cmd) == 0 { + return nil + } + return fmt.Errorf( + "inactivity timeout %s requires the Devsy daemon to be the container's main process, "+ + "but overrideCommand is false and the image entrypoint would run instead; set "+ + "overrideCommand to true or unset the inactivity timeout", timeout) +} + +func (r *runner) managedLifecycle() bool { + d, ok := r.driver.(driver.ManagedLifecycleDriver) + return ok && d.ManagedLifecycle() +} + +func (r *runner) effectiveContainerTimeout(shutdownAction string) (time.Duration, error) { + agentConfig := r.workspaceConfig.Agent + return agent.ResolveInactivityTimeout(agent.TimeoutInputs{ + Canonical: agentConfig.ContainerTimeout, + Legacy: agentConfig.Kubernetes.InactivityTimeout, + ShutdownAction: shutdownAction, + DisableDaemon: r.workspaceConfig.CLIOptions.DisableDaemon, + }) } // findRunningContainerOrFail re-fetches container details from the driver and @@ -713,6 +791,13 @@ func (r *runner) findRunningContainerOrFail( if details == nil { return nil, fmt.Errorf("dev container %s not found after %s", r.id, operation) } + // A terminal object must not satisfy startup when the daemon owns the lifecycle. + if r.managedLifecycle() && details.State.Status != config.ContainerStatusRunning { + return nil, fmt.Errorf( + "dev container %s is %s after %s, not running", + r.id, details.State.Status, operation, + ) + } return details, nil } diff --git a/pkg/driver/kubernetes/client.go b/pkg/driver/kubernetes/client.go index b631520dc8..7f1a3932e2 100644 --- a/pkg/driver/kubernetes/client.go +++ b/pkg/driver/kubernetes/client.go @@ -17,7 +17,7 @@ import ( ) type Client struct { - client *kubernetes.Clientset + client kubernetes.Interface config *rest.Config } @@ -63,7 +63,7 @@ func NewClient(kubeConfig, kubeContext string) (*Client, string, error) { }, namespace, nil } -func (c *Client) Client() *kubernetes.Clientset { +func (c *Client) Client() kubernetes.Interface { return c.client } diff --git a/pkg/driver/kubernetes/find.go b/pkg/driver/kubernetes/find.go index 3e579c169f..c6046c641e 100644 --- a/pkg/driver/kubernetes/find.go +++ b/pkg/driver/kubernetes/find.go @@ -2,10 +2,10 @@ package kubernetes import ( "context" + "fmt" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/log" - corev1 "k8s.io/api/core/v1" ) func (k *KubernetesDriver) FindDevContainer( @@ -27,17 +27,12 @@ func (k *KubernetesDriver) FindDevContainer( // check pod pod, err := k.getPod(ctx, pvc.Name) if err != nil { - log.Debugf("Error finding pod: %v", err) - log.Warn( - "If the pod does not come up automatically it is stuck in an error state. " + - "Recreate the workspace to recover from this", - ) - pod = nil + return nil, fmt.Errorf("find pod for workspace %q: %w", workspaceId, err) } // determine status status := config.ContainerStatusExited - if pod != nil && isPodRunning(pod) { + if pod != nil && classifyPod(pod) == podRunning { status = config.ContainerStatusRunning } @@ -59,7 +54,3 @@ func (k *KubernetesDriver) FindDevContainer( }, }, nil } - -func isPodRunning(pod *corev1.Pod) bool { - return pod.Status.Phase == corev1.PodRunning -} diff --git a/pkg/driver/kubernetes/pod_lifecycle.go b/pkg/driver/kubernetes/pod_lifecycle.go new file mode 100644 index 0000000000..6a0eb4f9b9 --- /dev/null +++ b/pkg/driver/kubernetes/pod_lifecycle.go @@ -0,0 +1,365 @@ +package kubernetes + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "strings" + "time" + + pkgconfig "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/driver" + "github.com/devsy-org/devsy/pkg/log" + provider2 "github.com/devsy-org/devsy/pkg/provider" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" +) + +const maxReplaceAttempts = 3 + +var ( + podPollInterval = time.Second + podDeleteTimeout = 2 * time.Minute +) + +// errPodChanged means the pod observed before a delete is no longer the pod +// that exists, so the caller must observe again. +var errPodChanged = errors.New("pod changed since it was observed") + +type podState int + +const ( + podPending podState = iota + podRunning + podTerminating + podTerminal + podUnknown +) + +func classifyPod(pod *corev1.Pod) podState { + switch { + case pod.DeletionTimestamp != nil: + return podTerminating + case pod.Status.Phase == corev1.PodFailed, pod.Status.Phase == corev1.PodSucceeded: + return podTerminal + case pod.Status.Phase == corev1.PodPending: + return podPending + case pod.Status.Phase == corev1.PodRunning: + if devsyContainerTerminated(pod) { + return podTerminal + } + return podRunning + default: + return podUnknown + } +} + +func devsyContainerTerminated(pod *corev1.Pod) bool { + for i := range pod.Status.ContainerStatuses { + c := &pod.Status.ContainerStatuses[i] + if c.Name == DevContainerName { + return IsTerminated(c) + } + } + return false +} + +// verifyOwnedPod refuses to touch a pod that Devsy did not create for this +// workspace. Pods that predate the workspace UID label are matched by the +// workspace volume claim. +func verifyOwnedPod(pod *corev1.Pod, id, workspaceUID string) error { + if pod.Name != id || pod.Labels[DevsyCreatedLabel] != "true" { + return fmt.Errorf("pod %q exists but was not created by Devsy; refusing to replace it", id) + } + if uid := pod.Labels[DevsyWorkspaceUIDLabel]; uid != "" { + if workspaceUID != "" && uid != workspaceUID { + return fmt.Errorf( + "pod %q belongs to workspace %q, not %q; refusing to replace it", + id, uid, workspaceUID, + ) + } + return nil + } + if !usesWorkspaceVolume(pod, id) { + return fmt.Errorf("pod %q does not use the workspace volume; refusing to replace it", id) + } + return nil +} + +func usesWorkspaceVolume(pod *corev1.Pod, claim string) bool { + for _, v := range pod.Spec.Volumes { + if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName == claim { + return true + } + } + return false +} + +// reconcileExistingPod makes sure no stale pod blocks creating the workspace +// pod. It reports true when a healthy pod with unchanged options is reused. +func (k *KubernetesDriver) reconcileExistingPod( + ctx context.Context, + id string, + options *driver.RunOptions, +) (bool, error) { + for range maxReplaceAttempts { + pod, err := k.getPod(ctx, id) + if err != nil { + return false, fmt.Errorf("get pod: %s: %w", id, err) + } + if pod == nil { + return false, nil + } + if err := verifyOwnedPod(pod, id, options.UID); err != nil { + return false, err + } + + reused, err := k.reconcileObservedPod(ctx, id, pod) + if errors.Is(err, errPodChanged) { + continue + } + if err != nil || reused { + return reused, err + } + } + return false, fmt.Errorf("pod %q kept changing while it was being replaced", id) +} + +// reconcileObservedPod handles one observation. It returns (false, nil) after +// removing the pod, so the caller observes again and then creates. +func (k *KubernetesDriver) reconcileObservedPod( + ctx context.Context, + id string, + pod *corev1.Pod, +) (bool, error) { + switch classifyPod(pod) { + case podTerminating: + log.Debugf("Pod %q is terminating, waiting for it to be deleted", id) + return false, k.waitPodGone(ctx, id, pod.UID) + case podTerminal: + log.Infof( + "Replacing stopped pod %q (phase %s), workspace volume is kept", + id, + pod.Status.Phase, + ) + return false, k.deleteObservedPod(ctx, id, pod.UID) + case podUnknown: + return false, fmt.Errorf( + "pod %q is in phase %q; its node may be unreachable, not replacing it", + id, pod.Status.Phase, + ) + } + + if !optionsEqual(storedProviderOptions(pod), k.options) { + log.Debug("Provider options changed") + return false, k.deleteObservedPod(ctx, id, pod.UID) + } + log.Debugf("Pod %q already exists and nothing changed, skipping update", id) + if classifyPod(pod) == podPending { + if _, err := k.waitPodRunning(ctx, id); err != nil { + return false, err + } + } + return true, nil +} + +func storedProviderOptions(pod *corev1.Pod) *provider2.ProviderKubernetesDriverConfig { + existing := &provider2.ProviderKubernetesDriverConfig{} + if raw := pod.GetAnnotations()[DevsyLastAppliedAnnotation]; raw != "" { + if err := json.Unmarshal([]byte(raw), existing); err != nil { + log.Errorf("Error unmarshalling existing provider options, continuing...: %s", err) + } + } + return existing +} + +// deleteObservedPod deletes exactly the observed pod instance and waits until +// the API server confirms it is gone. Finalizers are never removed. +func (k *KubernetesDriver) deleteObservedPod(ctx context.Context, id string, uid types.UID) error { + err := k.client.Client().CoreV1().Pods(k.namespace).Delete(ctx, id, metav1.DeleteOptions{ + GracePeriodSeconds: new(int64(10)), + Preconditions: &metav1.Preconditions{UID: &uid}, + }) + switch { + case err == nil: + case kerrors.IsNotFound(err): + return nil + case kerrors.IsConflict(err): + return errPodChanged + default: + return fmt.Errorf("delete pod %q: %w", id, err) + } + return k.waitPodGone(ctx, id, uid) +} + +// waitPodGone polls until the pod instance uid is absent. Only NotFound counts +// as absent; any other API error is returned. +func (k *KubernetesDriver) waitPodGone(ctx context.Context, id string, uid types.UID) error { + err := wait.PollUntilContextTimeout( + ctx, + podPollInterval, + podDeleteTimeout, + true, + func(ctx context.Context) (bool, error) { + pod, err := k.client.Client(). + CoreV1(). + Pods(k.namespace). + Get(ctx, id, metav1.GetOptions{}) + switch { + case kerrors.IsNotFound(err): + return true, nil + case err != nil: + return false, fmt.Errorf("check pod %q deletion: %w", id, err) + } + return uid != "" && pod.UID != uid, nil + }, + ) + if err != nil && ctx.Err() == nil && wait.Interrupted(err) { + return fmt.Errorf("timeout waiting for pod %q to be deleted: %w", id, err) + } + return err +} + +// ManagedLifecycle reports that the workspace pod's main process is the Devsy +// daemon, which owns inactivity shutdown. +func (k *KubernetesDriver) ManagedLifecycle() bool { return true } + +var _ driver.ManagedLifecycleDriver = (*KubernetesDriver)(nil) + +// effectiveTimeout resolves the current provider timeout for a stored shutdown action. +func (k *KubernetesDriver) effectiveTimeout(shutdownAction string) (time.Duration, error) { + var canonical string + if k.agentConfig != nil { + canonical = k.agentConfig.ContainerTimeout + } + return agentd.ResolveInactivityTimeout(agentd.TimeoutInputs{ + Canonical: canonical, + Legacy: k.options.InactivityTimeout, + ShutdownAction: shutdownAction, + }) +} + +// refreshStoredLifecycle regenerates the lifecycle policy in run options that +// were persisted with the volume, so a replacement pod uses current provider +// values rather than those from when the workspace was created. +func (k *KubernetesDriver) refreshStoredLifecycle( + options *driver.RunOptions, +) (*driver.RunOptions, error) { + if options == nil { + return nil, nil + } + stored := options.Env[pkgconfig.EnvWorkspaceDaemonConfig] + action := "" + if stored != "" { + cfg, err := agentd.DecodeDaemonConfig(stored) + if err != nil { + return nil, err + } + action = cfg.ShutdownAction + } + timeout, err := k.effectiveTimeout(action) + if err != nil { + return nil, err + } + refreshed, err := agentd.RefreshLifecycle(stored, timeout) + if err != nil { + return nil, err + } + out := *options + out.Env = make(map[string]string, len(options.Env)+1) + maps.Copy(out.Env, options.Env) + out.Env[pkgconfig.EnvWorkspaceDaemonConfig] = refreshed + return &out, nil +} + +// lifecycleTimeout returns the positive timeout the generated daemon config +// asks the managed daemon to enforce, or zero. +func lifecycleTimeout(options *driver.RunOptions) (time.Duration, error) { + raw := options.Env[pkgconfig.EnvWorkspaceDaemonConfig] + if raw == "" { + return 0, nil + } + cfg, err := agentd.DecodeDaemonConfig(raw) + if err != nil { + return 0, err + } + if !cfg.Lifecycle.Managed() || cfg.Timeout == "" || + cfg.ShutdownAction == config.ShutdownActionNone { + return 0, nil + } + return time.ParseDuration(cfg.Timeout) +} + +// validatePodTopology rejects pods in which another regular container could +// keep the pod Running after the Devsy daemon exits. Native sidecars +// (restartable init containers) end with the main container and are allowed. +func validatePodTopology(pod *corev1.Pod, timeout time.Duration) error { + if timeout <= 0 { + return nil + } + hasDevsy := false + for i := range pod.Spec.Containers { + hasDevsy = hasDevsy || pod.Spec.Containers[i].Name == DevContainerName + } + if !hasDevsy { + return fmt.Errorf( + "inactivity timeout %s needs a %q container in the pod", + timeout, + DevContainerName, + ) + } + return validateNoExtraContainers(pod, timeout) +} + +// validateNoExtraContainers is the part of the topology check that can run on +// the pod template, before the devsy container is added to it. +func validateNoExtraContainers(pod *corev1.Pod, timeout time.Duration) error { + if timeout <= 0 { + return nil + } + var extra []string + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name != DevContainerName { + extra = append(extra, pod.Spec.Containers[i].Name) + } + } + if len(extra) > 0 { + return fmt.Errorf( + "inactivity timeout %s cannot stop a pod with extra regular containers (%s) because "+ + "the pod keeps running while they do; move them to native sidecars "+ + "(init containers with restartPolicy Always) or unset the inactivity timeout", + timeout, strings.Join(extra, ", "), + ) + } + return nil +} + +// rejectAdmittedPod removes a freshly created pod whose admitted topology +// violates the stop contract. The workspace volume is kept. +func (k *KubernetesDriver) rejectAdmittedPod( + ctx context.Context, + created *corev1.Pod, + timeout time.Duration, +) error { + topologyErr := validatePodTopology(created, timeout) + if topologyErr == nil { + return nil + } + cleanup := k.deleteObservedPod(ctx, created.Name, created.UID) + if cleanup != nil { + return fmt.Errorf( + "%w (admission changed the pod; removing it also failed: %w)", topologyErr, cleanup, + ) + } + return fmt.Errorf( + "admission changed the pod and the inactivity timeout cannot be enforced: %w", + topologyErr, + ) +} diff --git a/pkg/driver/kubernetes/pod_lifecycle_test.go b/pkg/driver/kubernetes/pod_lifecycle_test.go new file mode 100644 index 0000000000..0e0c5179ec --- /dev/null +++ b/pkg/driver/kubernetes/pod_lifecycle_test.go @@ -0,0 +1,724 @@ +package kubernetes + +import ( + "context" + "encoding/json" + "errors" + "os" + "sync/atomic" + "testing" + "time" + + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/driver" + provider2 "github.com/devsy-org/devsy/pkg/provider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" +) + +const ( + testNamespace = "devsy-test" + testPodID = "devsy-ws" + testWorkspace = "ws-uid" +) + +var podsResource = schema.GroupResource{Resource: "pods"} + +type fakeCluster struct { + cs *fake.Clientset + k *KubernetesDriver +} + +func newFakeCluster(t *testing.T, objects ...runtime.Object) *fakeCluster { + t.Helper() + oldPoll, oldTimeout := podPollInterval, podDeleteTimeout + podPollInterval, podDeleteTimeout = 5*time.Millisecond, 2*time.Second + t.Cleanup(func() { podPollInterval, podDeleteTimeout = oldPoll, oldTimeout }) + + cs := fake.NewClientset(objects...) + return &fakeCluster{ + cs: cs, + k: &KubernetesDriver{ + client: &Client{client: cs}, + namespace: testNamespace, + options: &provider2.ProviderKubernetesDriverConfig{ + PodTimeout: "2s", + DiskSize: "1Gi", + }, + agentConfig: &provider2.ProviderAgentConfig{}, + }, + } +} + +func (c *fakeCluster) opts() *driver.RunOptions { + return &driver.RunOptions{UID: testWorkspace} +} + +type podSpec struct { + uid string + phase corev1.PodPhase + options *provider2.ProviderKubernetesDriverConfig + deleting bool + foreign bool + devsy *corev1.ContainerStatus +} + +func (c *fakeCluster) pod(t *testing.T, s podSpec) *corev1.Pod { + t.Helper() + if s.options == nil { + s.options = c.k.options + } + raw, err := json.Marshal(s.options) + require.NoError(t, err) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: testPodID, + Namespace: testNamespace, + UID: types.UID(s.uid), + Labels: map[string]string{DevsyWorkspaceUIDLabel: testWorkspace}, + Annotations: map[string]string{DevsyLastAppliedAnnotation: string(raw)}, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: DevContainerName}}}, + Status: corev1.PodStatus{Phase: s.phase}, + } + if !s.foreign { + pod.Labels[DevsyCreatedLabel] = "true" + } + if s.deleting { + now := metav1.Now() + pod.DeletionTimestamp = &now + pod.Finalizers = []string{"test/hold"} + } + if s.devsy != nil { + s.devsy.Name = DevContainerName + pod.Status.ContainerStatuses = []corev1.ContainerStatus{*s.devsy} + } + return pod +} + +func (c *fakeCluster) putPod(t *testing.T, pod *corev1.Pod) { + t.Helper() + pods := c.cs.CoreV1().Pods(testNamespace) + _, err := pods.Get(context.Background(), pod.Name, metav1.GetOptions{}) + if kerrors.IsNotFound(err) { + _, err = pods.Create(context.Background(), pod, metav1.CreateOptions{}) + } else { + _, err = pods.Update(context.Background(), pod, metav1.UpdateOptions{}) + } + require.NoError(t, err) +} + +func (c *fakeCluster) podExists(t *testing.T) bool { + t.Helper() + _, err := c.cs.CoreV1(). + Pods(testNamespace). + Get(context.Background(), testPodID, metav1.GetOptions{}) + if kerrors.IsNotFound(err) { + return false + } + require.NoError(t, err) + return true +} + +func (c *fakeCluster) deleteCalls() int { + n := 0 + for _, a := range c.cs.Actions() { + if a.Matches("delete", "pods") { + n++ + } + } + return n +} + +func (c *fakeCluster) reconcile(ctx context.Context) (bool, error) { + return c.k.reconcileExistingPod(ctx, testPodID, c.opts()) +} + +func TestReconcileNoPodAllowsCreate(t *testing.T) { + c := newFakeCluster(t) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.False(t, reused) +} + +func TestReconcileReusesRunningPodWithSameOptions(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{ + uid: "u1", + phase: corev1.PodRunning, + devsy: &corev1.ContainerStatus{ + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }, + })) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.True(t, reused) + assert.Zero(t, c.deleteCalls()) +} + +func TestReconcileReplacesTerminalPodEvenWithSameOptions(t *testing.T) { + for _, phase := range []corev1.PodPhase{corev1.PodFailed, corev1.PodSucceeded} { + t.Run(string(phase), func(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: phase})) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.False(t, reused, "a terminal pod must not be reused") + assert.False( + t, + c.podExists(t), + "the terminal pod is gone before the new one is created", + ) + }) + } +} + +func TestReconcileReplacesPodWhoseDevsyContainerExited(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{ + uid: "u1", phase: corev1.PodRunning, + devsy: &corev1.ContainerStatus{State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}, + }}, + })) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.False(t, reused) + assert.False(t, c.podExists(t)) +} + +func TestReconcileReplacesPodWhenOptionsChanged(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{ + uid: "u1", phase: corev1.PodRunning, + options: &provider2.ProviderKubernetesDriverConfig{PodTimeout: "2s", DiskSize: "9Gi"}, + })) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.False(t, reused) + assert.False(t, c.podExists(t)) +} + +func TestReconcilePendingPodWaitsForReadiness(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending})) + var ready atomic.Bool + go func() { + time.Sleep(40 * time.Millisecond) + running := c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning}) + running.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: DevContainerName, Ready: true, + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }} + c.putPod(t, running) + ready.Store(true) + }() + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.True(t, reused) + assert.True(t, ready.Load(), "success must wait for the pod, not just its object") +} + +func TestReconcileTerminatingPodWaitsForDeletionThenAllowsCreate(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning, deleting: true})) + go func() { + time.Sleep(40 * time.Millisecond) + _ = c.cs.CoreV1().Pods(testNamespace).Delete( + context.Background(), testPodID, metav1.DeleteOptions{}, + ) + }() + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.False(t, reused) + assert.False(t, c.podExists(t)) + assert.Zero(t, c.deleteCalls()-1, "a terminating pod is waited on, not deleted again") +} + +func TestReconcileForbiddenLookupIsNotAbsence(t *testing.T) { + c := newFakeCluster(t) + c.cs.PrependReactor("get", "pods", func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, kerrors.NewForbidden(podsResource, testPodID, errors.New("rbac")) + }) + + reused, err := c.reconcile(context.Background()) + + require.Error(t, err) + assert.True(t, kerrors.IsForbidden(err), "forbidden must surface") + assert.Contains(t, err.Error(), "forbidden") + assert.False(t, reused) +} + +func TestReconcileDeleteForbiddenSurfaces(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodFailed})) + c.cs.PrependReactor("delete", "pods", func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, kerrors.NewForbidden(podsResource, testPodID, errors.New("rbac")) + }) + + _, err := c.reconcile(context.Background()) + + require.Error(t, err) + assert.Contains(t, err.Error(), "forbidden") + assert.True(t, c.podExists(t), "nothing is assumed deleted") +} + +func TestWaitPodGoneDoesNotTreatLookupErrorsAsDeleted(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodFailed})) + c.cs.PrependReactor("get", "pods", func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("connection refused") + }) + + err := c.k.waitPodGone(context.Background(), testPodID, "u1") + + require.Error(t, err) + assert.Contains(t, err.Error(), "connection refused") +} + +func TestReconcileUIDConflictReobservesReplacementPod(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "old", phase: corev1.PodFailed})) + var conflicted atomic.Bool + c.cs.PrependReactor( + "delete", + "pods", + func(a clienttesting.Action) (bool, runtime.Object, error) { + if conflicted.Swap(true) { + return false, nil, nil + } + // Someone replaced the pod between our observation and our delete. + replacement := c.pod(t, podSpec{ + uid: "new", + phase: corev1.PodRunning, + devsy: &corev1.ContainerStatus{ + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }, + }) + require.NoError(t, c.cs.Tracker().Update( + corev1.SchemeGroupVersion.WithResource("pods"), replacement, testNamespace, + )) + return true, nil, kerrors.NewConflict(podsResource, testPodID, errors.New("uid")) + }, + ) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.True(t, reused, "the healthy replacement is kept, not deleted by stale knowledge") + assert.True(t, c.podExists(t)) +} + +func TestReconcileDeleteCarriesObservedUID(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "observed", phase: corev1.PodFailed})) + var got *types.UID + c.cs.PrependReactor( + "delete", + "pods", + func(a clienttesting.Action) (bool, runtime.Object, error) { + if del, ok := a.(clienttesting.DeleteActionImpl); ok && + del.DeleteOptions.Preconditions != nil { + got = del.DeleteOptions.Preconditions.UID + } + return false, nil, nil + }, + ) + + _, err := c.reconcile(context.Background()) + + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, types.UID("observed"), *got) +} + +func TestReconcileCancellationWhileWaitingForDeletion(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning, deleting: true})) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err := c.reconcile(ctx) + + require.Error(t, err) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.True(t, c.podExists(t)) +} + +func TestReconcileRefusesPodsNotOwnedByDevsy(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodFailed, foreign: true})) + + _, err := c.reconcile(context.Background()) + + require.Error(t, err) + assert.Contains(t, err.Error(), "not created by Devsy") + assert.True(t, c.podExists(t)) +} + +func TestReconcileRefusesPodOfAnotherWorkspace(t *testing.T) { + c := newFakeCluster(t) + pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodFailed}) + pod.Labels[DevsyWorkspaceUIDLabel] = "someone-else" + c.putPod(t, pod) + + _, err := c.reconcile(context.Background()) + + require.Error(t, err) + assert.True(t, c.podExists(t)) +} + +func TestReconcileAdoptsOldPodThroughWorkspaceVolume(t *testing.T) { + c := newFakeCluster(t) + pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodFailed}) + delete(pod.Labels, DevsyWorkspaceUIDLabel) + pod.Spec.Volumes = []corev1.Volume{{ + Name: DevContainerName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: testPodID}, + }, + }} + c.putPod(t, pod) + + _, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.False(t, c.podExists(t)) +} + +func TestReplacementPreservesWorkspaceVolume(t *testing.T) { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: testPodID, Namespace: testNamespace, UID: "pvc-uid"}, + } + c := newFakeCluster(t, pvc) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodSucceeded})) + + _, err := c.reconcile(context.Background()) + require.NoError(t, err) + + got, err := c.cs.CoreV1().PersistentVolumeClaims(testNamespace). + Get(context.Background(), testPodID, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, types.UID("pvc-uid"), got.UID) + for _, a := range c.cs.Actions() { + assert.False(t, a.Matches("delete", "persistentvolumeclaims"), "PVC must never be deleted") + } +} + +func TestFindDevContainerReportsLookupErrors(t *testing.T) { + c := newFakeCluster(t, workspacePVC(t)) + c.cs.PrependReactor("get", "pods", func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, kerrors.NewForbidden(podsResource, testPodID, errors.New("rbac")) + }) + + details, err := c.k.FindDevContainer(context.Background(), "ws") + + require.Error(t, err) + assert.Nil(t, details) +} + +func TestFindDevContainerTreatsTerminalPodAsStopped(t *testing.T) { + c := newFakeCluster(t, workspacePVC(t)) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodSucceeded})) + + details, err := c.k.FindDevContainer(context.Background(), "ws") + + require.NoError(t, err) + assert.Equal(t, config.ContainerStatusExited, details.State.Status) +} + +func workspacePVC(t *testing.T) *corev1.PersistentVolumeClaim { + t.Helper() + info, err := json.Marshal( + &DevContainerInfo{WorkspaceID: "ws", Options: &driver.RunOptions{UID: testWorkspace}}, + ) + require.NoError(t, err) + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: testPodID, Namespace: testNamespace, + Annotations: map[string]string{DevsyInfoAnnotation: string(info)}, + }} +} + +func daemonOptions(t *testing.T, cfg agentd.DaemonConfig) *driver.RunOptions { + t.Helper() + encoded, err := agentd.EncodeDaemonConfig(&cfg) + require.NoError(t, err) + return &driver.RunOptions{Env: map[string]string{"DEVSY_WORKSPACE_DAEMON_CONFIG": encoded}} +} + +func TestTopologyValidation(t *testing.T) { + regular := func(names ...string) *corev1.Pod { + pod := &corev1.Pod{} + for _, n := range names { + pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{Name: n}) + } + return pod + } + always := corev1.ContainerRestartPolicyAlways + + t.Run("single devsy container", func(t *testing.T) { + assert.NoError(t, validatePodTopology(regular(DevContainerName), time.Minute)) + }) + t.Run("native sidecar and init container allowed", func(t *testing.T) { + pod := regular(DevContainerName) + pod.Spec.InitContainers = []corev1.Container{ + {Name: "mesh", RestartPolicy: &always}, + {Name: "setup"}, + } + assert.NoError(t, validatePodTopology(pod, time.Minute)) + }) + t.Run("extra regular containers rejected by name", func(t *testing.T) { + err := validatePodTopology(regular(DevContainerName, "istio-proxy", "logger"), time.Minute) + require.Error(t, err) + assert.Contains(t, err.Error(), "istio-proxy") + assert.Contains(t, err.Error(), "logger") + }) + t.Run("extra containers fine when timeout disabled", func(t *testing.T) { + assert.NoError(t, validatePodTopology(regular(DevContainerName, "istio-proxy"), 0)) + }) + t.Run("missing devsy container rejected", func(t *testing.T) { + assert.Error(t, validatePodTopology(regular("other"), time.Minute)) + }) +} + +func TestLifecycleTimeoutFromGeneratedConfig(t *testing.T) { + managed := agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon} + + got, err := lifecycleTimeout( + daemonOptions(t, agentd.DaemonConfig{Timeout: "2m", Lifecycle: managed}), + ) + require.NoError(t, err) + assert.Equal(t, 2*time.Minute, got) + + got, err = lifecycleTimeout(daemonOptions(t, agentd.DaemonConfig{ + Timeout: "2m", Lifecycle: managed, ShutdownAction: config.ShutdownActionNone, + })) + require.NoError(t, err) + assert.Zero(t, got, "shutdownAction none disables the timeout") + + got, err = lifecycleTimeout(&driver.RunOptions{}) + require.NoError(t, err) + assert.Zero(t, got) +} + +func TestRefreshStoredLifecycleUsesCurrentProviderValues(t *testing.T) { + c := newFakeCluster(t) + c.k.agentConfig.ContainerTimeout = "7m" + stored := daemonOptions(t, agentd.DaemonConfig{ + Timeout: "1m", + ShutdownAction: config.ShutdownActionStopContainer, + Ssh: agentd.SshConfig{User: "root"}, + }) + + refreshed, err := c.k.refreshStoredLifecycle(stored) + require.NoError(t, err) + + cfg, err := agentd.DecodeDaemonConfig(refreshed.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"]) + require.NoError(t, err) + assert.Equal(t, "7m0s", cfg.Timeout) + assert.True(t, cfg.Lifecycle.Managed()) + assert.Equal(t, "root", cfg.Ssh.User) + oldCfg, err := agentd.DecodeDaemonConfig(stored.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"]) + require.NoError(t, err) + assert.Equal(t, "1m", oldCfg.Timeout, "stored options are not mutated") +} + +func TestRefreshStoredLifecycleLegacyAliasAndConflict(t *testing.T) { + c := newFakeCluster(t) + c.k.options.InactivityTimeout = "3m" + + refreshed, err := c.k.refreshStoredLifecycle(&driver.RunOptions{}) + require.NoError(t, err) + cfg, err := agentd.DecodeDaemonConfig(refreshed.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"]) + require.NoError(t, err) + assert.Equal(t, "3m0s", cfg.Timeout) + + c.k.agentConfig.ContainerTimeout = "4m" + _, err = c.k.refreshStoredLifecycle(&driver.RunOptions{}) + assert.ErrorContains(t, err, "conflicts") +} + +func TestAdmittedPodWithInjectedContainerIsRemovedAndVolumeKept(t *testing.T) { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: testPodID, Namespace: testNamespace}, + } + c := newFakeCluster(t, pvc) + admitted := c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending}) + admitted.Spec.Containers = append(admitted.Spec.Containers, corev1.Container{Name: "injected"}) + c.putPod(t, admitted) + + err := c.k.rejectAdmittedPod(context.Background(), admitted, time.Minute) + + require.Error(t, err) + assert.Contains(t, err.Error(), "injected") + assert.False(t, c.podExists(t)) + _, getErr := c.cs.CoreV1().PersistentVolumeClaims(testNamespace). + Get(context.Background(), testPodID, metav1.GetOptions{}) + assert.NoError(t, getErr) +} + +func TestAdmittedPodAcceptedWhenTopologyHolds(t *testing.T) { + c := newFakeCluster(t) + pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending}) + c.putPod(t, pod) + + require.NoError(t, c.k.rejectAdmittedPod(context.Background(), pod, time.Minute)) + assert.True(t, c.podExists(t)) +} + +func TestExplicitStopWaitsForPodToBeGone(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning})) + + require.NoError(t, c.k.StopDevContainer(context.Background(), "ws")) + + assert.False(t, c.podExists(t)) +} + +func runOptionsForWorkspace(timeout string) *driver.RunOptions { + cfg, _ := agentd.EncodeDaemonConfig(&agentd.DaemonConfig{ + Timeout: timeout, + ShutdownAction: config.ShutdownActionStopContainer, + Lifecycle: agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon}, + }) + return &driver.RunOptions{ + UID: testWorkspace, + Image: "example/image:1", + Entrypoint: "/bin/sh", + Env: map[string]string{"DEVSY_WORKSPACE_DAEMON_CONFIG": cfg}, + WorkspaceMount: &config.Mount{Type: "bind", Target: "/workspaces/app"}, + } +} + +func readyOnCreate(c *fakeCluster, uid types.UID) { + c.cs.PrependReactor( + "create", + "pods", + func(a clienttesting.Action) (bool, runtime.Object, error) { + pod := a.(clienttesting.CreateAction).GetObject().(*corev1.Pod) + pod.UID = uid + pod.Status.Phase = corev1.PodRunning + pod.Status.ContainerStatuses = nil + for _, container := range pod.Spec.Containers { + pod.Status.ContainerStatuses = append( + pod.Status.ContainerStatuses, + corev1.ContainerStatus{ + Name: container.Name, Ready: true, + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }, + ) + } + return false, nil, nil + }, + ) +} + +func TestRunContainerRecreatesTerminalPodAndKeepsVolume(t *testing.T) { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: testPodID, Namespace: testNamespace, UID: "pvc-uid"}, + } + c := newFakeCluster(t, pvc) + c.putPod(t, c.pod(t, podSpec{uid: "stopped", phase: corev1.PodSucceeded})) + readyOnCreate(c, "fresh") + + err := c.k.runContainer(context.Background(), testPodID, runOptionsForWorkspace("1m"), false) + + require.NoError(t, err) + pod, err := c.cs.CoreV1(). + Pods(testNamespace). + Get(context.Background(), testPodID, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, types.UID("fresh"), pod.UID, "a new pod replaced the terminal one") + assert.Equal(t, corev1.RestartPolicyNever, pod.Spec.RestartPolicy) + gotPVC, err := c.cs.CoreV1().PersistentVolumeClaims(testNamespace). + Get(context.Background(), testPodID, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, types.UID("pvc-uid"), gotPVC.UID) + for _, a := range c.cs.Actions() { + assert.False(t, a.Matches("delete", "persistentvolumeclaims")) + } +} + +func TestRunContainerRejectsExtraContainersBeforeTouchingTheCluster(t *testing.T) { + c := newFakeCluster(t) + template := t.TempDir() + "/pod.yaml" + require.NoError( + t, + writeFile(template, "spec:\n containers:\n - name: istio-proxy\n image: proxy\n"), + ) + c.k.options.PodManifestTemplate = template + + err := c.k.runContainer(context.Background(), testPodID, runOptionsForWorkspace("1m"), false) + + require.Error(t, err) + assert.Contains(t, err.Error(), "istio-proxy") + for _, a := range c.cs.Actions() { + assert.False(t, a.Matches("create", "pods") || a.Matches("delete", "pods") || + a.Matches("create", "secrets"), "nothing is created or deleted: %v", a) + } +} + +func TestRunContainerAllowsExtraContainersWhenTimeoutDisabled(t *testing.T) { + c := newFakeCluster(t) + template := t.TempDir() + "/pod.yaml" + require.NoError( + t, + writeFile(template, "spec:\n containers:\n - name: istio-proxy\n image: proxy\n"), + ) + c.k.options.PodManifestTemplate = template + readyOnCreate(c, "fresh") + + err := c.k.runContainer(context.Background(), testPodID, runOptionsForWorkspace(""), false) + + assert.NoError(t, err) +} + +func TestRunContainerRemovesPodWhoseAdmittedTopologyBreaksTheContract(t *testing.T) { + c := newFakeCluster(t) + c.cs.PrependReactor( + "create", + "pods", + func(a clienttesting.Action) (bool, runtime.Object, error) { + pod := a.(clienttesting.CreateAction).GetObject().(*corev1.Pod) + pod.UID = "admitted" + pod.Spec.Containers = append( + pod.Spec.Containers, + corev1.Container{Name: "webhook-injected"}, + ) + return false, nil, nil + }, + ) + + err := c.k.runContainer(context.Background(), testPodID, runOptionsForWorkspace("1m"), false) + + require.Error(t, err) + assert.Contains(t, err.Error(), "webhook-injected") + assert.False(t, c.podExists(t)) +} + +func writeFile(path, content string) error { + return os.WriteFile(path, []byte(content), 0o600) +} diff --git a/pkg/driver/kubernetes/run.go b/pkg/driver/kubernetes/run.go index b1fcbdb188..90d16e3e7d 100644 --- a/pkg/driver/kubernetes/run.go +++ b/pkg/driver/kubernetes/run.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strconv" "strings" + "time" pkgconfig "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/devcontainer/config" @@ -48,6 +49,9 @@ func (k *KubernetesDriver) RunDevContainer( log.Debugf("Running devcontainer for workspace %q", workspaceId) workspaceId = getID(workspaceId) + if err := k.preflightTemplate(options); err != nil { + return err + } if err := k.ensureNamespace(ctx); err != nil { return err } @@ -60,6 +64,23 @@ func (k *KubernetesDriver) RunDevContainer( return k.runContainer(ctx, workspaceId, options, initialize) } +// preflightTemplate rejects an unusable pod template before any cluster object, +// including the workspace volume, is created. +func (k *KubernetesDriver) preflightTemplate(options *driver.RunOptions) error { + if options == nil { + return nil + } + pod, err := loadPodTemplate(k.options.PodManifestTemplate) + if err != nil { + return err + } + timeout, err := lifecycleTimeout(options) + if err != nil { + return fmt.Errorf("read lifecycle policy: %w", err) + } + return validateNoExtraContainers(pod, timeout) +} + func (k *KubernetesDriver) ensureNamespace(ctx context.Context) error { if k.namespace == "" || k.options.CreateNamespace != pkgconfig.BoolTrue { return nil @@ -93,7 +114,10 @@ func (k *KubernetesDriver) ensureDevContainerPvc( } if options == nil && containerInfo != nil { - options = containerInfo.Options + var err error + if options, err = k.refreshStoredLifecycle(containerInfo.Options); err != nil { + return false, nil, err + } } return false, options, nil @@ -114,7 +138,15 @@ func (k *KubernetesDriver) runContainer( return err } - skip, err := k.reconcileExistingPod(ctx, id) + timeout, err := lifecycleTimeout(options) + if err != nil { + return fmt.Errorf("read lifecycle policy: %w", err) + } + if err := validatePodTopology(pod, timeout); err != nil { + return err + } + + skip, err := k.reconcileExistingPod(ctx, id, options) if err != nil { return err } @@ -122,7 +154,7 @@ func (k *KubernetesDriver) runContainer( return nil } - return k.runPod(ctx, id, pod) + return k.runPod(ctx, id, pod, timeout) } func (k *KubernetesDriver) buildPod( @@ -140,6 +172,14 @@ func (k *KubernetesDriver) buildPod( if err != nil { return nil, err } + // Reject before any cluster object is created for this run. + timeout, err := lifecycleTimeout(options) + if err != nil { + return nil, fmt.Errorf("read lifecycle policy: %w", err) + } + if err := validateNoExtraContainers(pod, timeout); err != nil { + return nil, err + } initContainers, err := k.getInitContainers(options, pod, initialize) if err != nil { @@ -493,41 +533,12 @@ func (k *KubernetesDriver) finalizePodSpec(pod *corev1.Pod, id string, pullSecre pod.Spec.RestartPolicy = corev1.RestartPolicyNever } -func (k *KubernetesDriver) reconcileExistingPod(ctx context.Context, id string) (bool, error) { - existingPod, err := k.getPod(ctx, id) - if err != nil { - return false, fmt.Errorf("get pod: %s: %w", id, err) - } - if existingPod == nil { - return false, nil - } - - existingOptions := &provider2.ProviderKubernetesDriverConfig{} - if raw := existingPod.GetAnnotations()[DevsyLastAppliedAnnotation]; raw != "" { - if err = json.Unmarshal([]byte(raw), existingOptions); err != nil { - log.Errorf("Error unmarshalling existing provider options, continuing...: %s", err) - } - } - - // Nothing changed, can safely return - if optionsEqual(existingOptions, k.options) { - log.Debugf( - "Pod %q already exists and nothing changed, skipping update", - existingPod.Name, - ) - return true, nil - } - - // Stop the current pod - log.Debug("Provider options changed") - if err := k.waitPodDeleted(ctx, id); err != nil { - return false, fmt.Errorf("stop devcontainer: %s: %w", id, err) - } - - return false, nil -} - -func (k *KubernetesDriver) runPod(ctx context.Context, id string, pod *corev1.Pod) error { +func (k *KubernetesDriver) runPod( + ctx context.Context, + id string, + pod *corev1.Pod, + timeout time.Duration, +) error { var err error // set configuration before creating the pod @@ -552,10 +563,16 @@ func (k *KubernetesDriver) runPod(ctx context.Context, id string, pod *corev1.Po // create the pod log.Debugf("Create Pod %q", id) - _, err = k.client.Client().CoreV1().Pods(k.namespace).Create(ctx, pod, metav1.CreateOptions{}) + created, err := k.client.Client(). + CoreV1(). + Pods(k.namespace). + Create(ctx, pod, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("create pod: %w", err) } + if err := k.rejectAdmittedPod(ctx, created, timeout); err != nil { + return err + } // wait for pod running log.Debugf("Waiting for DevContainer Pod %q to come up", id) @@ -711,12 +728,11 @@ func (k *KubernetesDriver) StartDevContainer(ctx context.Context, workspaceId st return fmt.Errorf("persistent volume %q not found", workspaceId) } - return k.runContainer( - ctx, - workspaceId, - containerInfo.Options, - false, - ) + options, err := k.refreshStoredLifecycle(containerInfo.Options) + if err != nil { + return err + } + return k.runContainer(ctx, workspaceId, options, false) } func getID(workspaceID string) string { diff --git a/pkg/driver/kubernetes/wait.go b/pkg/driver/kubernetes/wait.go index eae366c09b..e624dae93c 100644 --- a/pkg/driver/kubernetes/wait.go +++ b/pkg/driver/kubernetes/wait.go @@ -2,13 +2,11 @@ package kubernetes import ( "context" - "errors" "fmt" "strings" "time" "github.com/devsy-org/devsy/pkg/driver/kubernetes/throttledlogger" - "github.com/devsy-org/devsy/pkg/log" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,7 +25,7 @@ func (k *KubernetesDriver) waitPodRunning(ctx context.Context, id string) (*core var pod *corev1.Pod err = wait.PollUntilContextTimeout( ctx, - time.Second, + podPollInterval, timeoutDuration, true, func(ctx context.Context) (bool, error) { @@ -36,7 +34,7 @@ func (k *KubernetesDriver) waitPodRunning(ctx context.Context, id string) (*core if err != nil { return false, err } else if pod == nil { - return true, nil + return false, fmt.Errorf("pod %q disappeared while waiting for it to start", id) } return k.evaluatePodStatus(ctx, pod, &podStatusParams{ @@ -225,17 +223,6 @@ func (k *KubernetesDriver) checkContainerStatus( c *corev1.ContainerStatus, throttledLogger *throttledlogger.ThrottledLogger, ) (bool, error) { - // delete succeeded pods - if IsTerminated(c) && Succeeded(c) { - // delete pod that is succeeded - log.Debugf("Delete Pod %q because it is succeeded", id) - if err := k.waitPodDeleted(ctx, id); err != nil { - return false, err - } - - return false, nil - } - if IsWaiting(c) { return handleContainerWaiting(id, c, throttledLogger) } @@ -335,6 +322,8 @@ func restartableInitContainer(p *corev1.ContainerRestartPolicy) bool { return p != nil && *p == corev1.ContainerRestartPolicyAlways } +// waitPodDeleted deletes the pod for an explicit stop and waits until the API +// server reports it gone. func (k *KubernetesDriver) waitPodDeleted(ctx context.Context, id string) error { err := k.client.Client().CoreV1().Pods(k.namespace).Delete(ctx, id, metav1.DeleteOptions{ GracePeriodSeconds: &[]int64{10}[0], @@ -347,23 +336,5 @@ func (k *KubernetesDriver) waitPodDeleted(ctx context.Context, id string) error return fmt.Errorf("delete pod: %w", err) } - err = wait.PollUntilContextTimeout( - ctx, - time.Second, - time.Minute*2, - true, - func(ctx context.Context) (bool, error) { - _, err := k.client.Client().CoreV1().Pods(k.namespace).Get(ctx, id, metav1.GetOptions{}) - if err != nil { - return true, nil - } - - return false, nil - }, - ) - if err != nil { - return errors.New("timeout waiting for pod to be deleted") - } - - return nil + return k.waitPodGone(ctx, id, "") } diff --git a/pkg/driver/types.go b/pkg/driver/types.go index 590fe9588c..31398252c4 100644 --- a/pkg/driver/types.go +++ b/pkg/driver/types.go @@ -54,6 +54,12 @@ type WorkspaceChowner interface { RequiresWorkspaceChown() bool } +// ManagedLifecycleDriver is a capability interface for drivers whose container +// main process is the Devsy daemon, which then owns inactivity shutdown. +type ManagedLifecycleDriver interface { + ManagedLifecycle() bool +} + // ReprovisioningDriver is a capability interface for drivers that can reprovision // an existing devcontainer in place. Reprovisioning re-runs the container, so it // embeds RunOptionsDriver. diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 2bfd3c446c..7c9855b227 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -92,12 +92,12 @@ type ProviderAgentConfig struct { // DownloadURL is the base url where to download the agent from DownloadURL string `json:"downloadURL,omitempty"` - // Timeout is the timeout in minutes to wait until the agent tries - // to turn of the server. + // Timeout is the Go duration (e.g. 10m) of inactivity after which the agent + // tries to turn off the server. Timeout string `json:"inactivityTimeout,omitempty"` - // ContainerTimeout is the timeout in minutes to wait until the agent tries - // to delete the container. + // ContainerTimeout is the Go duration (e.g. 10m) of inactivity after which the + // container is stopped. Empty or zero disables it. ContainerTimeout string `json:"containerInactivityTimeout,omitempty"` // InjectGitCredentials signals Devsy if git credentials should get synced into @@ -440,7 +440,9 @@ type ProviderKubernetesDriverConfig struct { ClusterRole string `json:"clusterRole,omitempty"` ServiceAccount string `json:"serviceAccount,omitempty"` - Architecture string `json:"architecture,omitempty"` + Architecture string `json:"architecture,omitempty"` + // InactivityTimeout is a deprecated alias of Agent.ContainerTimeout, used only + // when the canonical field is empty. InactivityTimeout string `json:"inactivityTimeout,omitempty"` StorageClass string `json:"storageClass,omitempty"` diff --git a/pkg/provider/workspace.go b/pkg/provider/workspace.go index ea8edc910a..c5cf1a4b95 100644 --- a/pkg/provider/workspace.go +++ b/pkg/provider/workspace.go @@ -154,6 +154,13 @@ type WorkspaceSource struct { Snapshot string `json:"snapshot,omitempty"` } +// ContainerLifecycleOwner identifies who owns inactivity shutdown in a container. +type ContainerLifecycleOwner string + +// ContainerLifecycleOwnerManagedDaemon means the container's main process is the +// Devsy daemon, which owns the inactivity timer. +const ContainerLifecycleOwnerManagedDaemon ContainerLifecycleOwner = "managedDaemon" + type ContainerWorkspaceInfo struct { // IDE holds the ide config options IDE WorkspaceIDEConfig `json:"ide"` @@ -164,10 +171,14 @@ type ContainerWorkspaceInfo struct { // Dockerless holds custom dockerless configuration Dockerless ProviderDockerlessOptions `json:"dockerless"` - // ContainerTimeout is the timeout in minutes to wait until the agent tries - // to delete the container. + // ContainerTimeout is the effective Go duration of inactivity after which + // the container is stopped. Empty disables it. ContainerTimeout string `json:"containerInactivityTimeout,omitempty"` + // LifecycleOwner names the process that owns inactivity shutdown inside the + // container. Empty means the legacy detached daemon. + LifecycleOwner ContainerLifecycleOwner `json:"lifecycleOwner,omitempty"` + // Source is a WorkspaceSource to be used inside the container Source WorkspaceSource `json:"source"` diff --git a/providers/kubernetes/provider.yaml b/providers/kubernetes/provider.yaml index 30b2397d7c..a4873121fe 100644 --- a/providers/kubernetes/provider.yaml +++ b/providers/kubernetes/provider.yaml @@ -61,7 +61,7 @@ options: description: If defined, Devsy will use the given service account for the dev container. global: true INACTIVITY_TIMEOUT: - description: "If defined, will automatically stop the pod after the inactivity period. Examples: 10m, 1h" + description: "If defined, the workspace pod stops after this much inactivity (no tracked SSH session or setup work). Empty or 0 disables it. The finished pod may stay visible until the next up, which replaces it and keeps the volume. Examples: 10m, 1h" POD_TIMEOUT: description: "Determines how long the provider waits for the workspace pod to come up. Examples: 10m, 1h" default: 10m @@ -135,7 +135,6 @@ agent: clusterRole: ${CLUSTER_ROLE} serviceAccount: ${SERVICE_ACCOUNT} architecture: ${ARCHITECTURE} - inactivityTimeout: ${INACTIVITY_TIMEOUT} storageClass: ${STORAGE_CLASS} pvcAccessMode: ${PVC_ACCESS_MODE} pvcAnnotations: ${PVC_ANNOTATIONS} diff --git a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx index 318644e0ac..3c39bbc0be 100644 --- a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx @@ -9,11 +9,13 @@ Providers can stop idle workspaces to save resources. The workspace keeps its da Providers that support it expose an `INACTIVITY_TIMEOUT` option. Built-in providers with this option are Apple, Docker, Kubernetes, Microsandbox, and Podman. Values use Go duration format, for example `10m` or `1h`. See [Set provider options](../managing-providers/manage-providers.mdx#set-provider-options). -If the option is unset, a machine provider's own default applies. Devsy uses 20 minutes when the provider sets no default. If `INACTIVITY_TIMEOUT` is unset, Docker, Kubernetes, Podman, Apple, and Microsandbox containers never time out. +If the option is unset, a machine provider's own default applies. Devsy uses 20 minutes when the provider sets no default. If `INACTIVITY_TIMEOUT` is unset or `0`, Docker, Kubernetes, Podman, Apple, and Microsandbox containers never time out. ## How it works -**Non-machine providers** such as Docker, Podman, Kubernetes, Apple, and Microsandbox start a daemon in the container that watches for activity. When the timeout passes with no activity, that daemon exits. For Microsandbox, Devsy also passes the timeout to the microsandbox CLI as `--idle-timeout`. +**Kubernetes** runs the Devsy daemon as the main process of the workspace pod. The daemon starts without a timer. The `devsy workspace up` setup suspends the timer while it runs, then arms it, so a slow setup never counts as idle time. When the timeout passes with no activity, the daemon stops its children, logs that the inactivity timeout was reached, and exits normally. The pod ends, and the workspace volume is kept. See [Kubernetes](#kubernetes). + +**Other non-machine providers** such as Docker, Podman, Apple, and Microsandbox start a detached daemon in the container that watches for activity. Its exit does not by itself stop the container. For Microsandbox, Devsy also passes the timeout to the microsandbox CLI as `--idle-timeout`. The Apple provider's option is documented to delete the container instead of stopping it. Files outside mounted volumes are lost when a container is deleted. @@ -23,6 +25,38 @@ The Apple provider's option is documented to delete the container instead of sto See the [agent guide](../developing-providers/agent.mdx#machine-providers) for how providers implement this, and [Machine diagnostics](../managing-machines/machine-diagnostics.mdx) for debugging. +## What counts as activity + +A workspace is active while any of these hold: + +- A tracked `devsy workspace ssh` session is connected. Activity is recorded when the session connects, periodically while it stays open, and when it ends, so the idle period starts at disconnect. +- Setup work is running: the synchronous `devsy workspace up` setup, deferred lifecycle hooks, and `postAttachCommand`. + +CPU use and processes you started yourself do not count. A command such as `nohup make &` in a closed session does not keep the workspace alive. + +The daemon samples activity about once a second at most, so a workspace stops up to a few seconds after the timeout passes. For timeouts of a minute or more, sessions record activity every 10 seconds. Shorter timeouts use a proportionally shorter interval. If the daemon cannot read the activity file, it logs an error and does not stop the workspace. + +## Kubernetes + +Set `INACTIVITY_TIMEOUT` on the Kubernetes provider, for example `1h`. The deprecated `inactivityTimeout` field under `agent.kubernetes` in a custom provider is an alias used only when `containerInactivityTimeout` is empty. If both are set, they must match. + +The timeout is off when it is empty or `0`, when `shutdownAction` is `none`, or when `devsy workspace up` runs with `--disable-daemon`. A negative or unparsable value is an error. `stopCompose` with a positive timeout is rejected on Kubernetes. + +When the workspace stops: + +- The pod object can stay visible in the `Succeeded` or `Failed` phase until the next `devsy workspace up` or an explicit `devsy workspace stop`. The next `up` deletes that finished pod and creates a new one that mounts the same persistent volume claim. No recreate flag is needed. +- Files on the workspace volume persist. Files written to the container filesystem outside mounted volumes are lost, because the new pod starts from the image. +- `devsy workspace stop` still deletes the pod through the Kubernetes API from your machine. The pod gets no Kubernetes credentials of its own. + +Supported pod layout: + +- One regular `devsy` container, any number of normal init containers, and native sidecars (init containers with `restartPolicy: Always`). Native sidecars end with the pod. +- With a positive timeout, extra regular containers in `spec.containers`, from a pod manifest template or injected by an admission webhook, are rejected. The pod would keep running while they run. Devsy names the containers in the error. Without a timeout, or with `shutdownAction: none`, extra containers are allowed. +- With a positive timeout, the Devsy daemon must be the container's main process. A `devcontainer.json` with `overrideCommand: false` and an image entrypoint or command is rejected. +- Dockerless builds do not support a positive timeout. + +Existing pods created before this behavior have a main process that cannot enforce the timeout. `devsy workspace up` on such a pod fails with an error that says to stop and start the workspace once. Run `devsy workspace stop`, then `devsy workspace up`. The volume is kept; replacing the agent binary does not update a daemon that is already running. + ## Shutdown action Set `shutdownAction` in `devcontainer.json`; it is not a provider option. @@ -30,4 +64,4 @@ Set `shutdownAction` in `devcontainer.json`; it is not a provider option. - `none` disables the container timeout monitor. For machines, it also prevents `agent.exec.shutdown` from running. - Any other value enables the container timeout monitor. For machines, `agent.exec.shutdown` runs when the timeout passes, if the command is set. -When `shutdownAction` is unset, the default is `stopCompose` for Compose workspaces and `stopContainer` otherwise. The inactivity timeout path only checks whether the value is `none`; the page does not describe per-value container effects. +When `shutdownAction` is unset, the default is `stopCompose` for Compose workspaces and `stopContainer` otherwise. Apart from `none`, and `stopCompose` being rejected on Kubernetes, the value does not change what the container timeout does. diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx index 4cece38afc..a5e261b486 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx @@ -33,7 +33,7 @@ Option values can be used in this section. - **local**: set to `true` when Devsy runs directly on this machine instead of reaching a remote one. The built-in providers do this. - **driver**: how the container is deployed. One of `docker` (default), `kubernetes`, `apple`, `microsandbox`, `custom` or `external`. See [Drivers](./driver.mdx). - **inactivityTimeout**: how long to wait before stopping an idle machine. For machine providers. -- **containerInactivityTimeout**: how long to wait before stopping an idle container. For non-machine providers. +- **containerInactivityTimeout**: how long the container may be idle before it stops. A Go duration such as `10m`; empty or `0` disables it. For non-machine providers. - **injectGitCredentials**, **injectDockerCredentials**: whether to copy those credentials into the environment. - **exec.shutdown**: the command that stops the machine once it is idle. Option values are available in it. - **binaries**: helper binaries for `exec.shutdown`. Same format as [provider binaries](./binaries.mdx). @@ -44,7 +44,9 @@ Devsy tracks whether anyone is connected and stops the environment after the tim ### Non-machine providers -The agent ends the container's main process, which stops the container. Set the timeout with `containerInactivityTimeout`. +Set the timeout with `containerInactivityTimeout`. It is a Go duration such as `10m`; empty or `0` disables it. + +On the `kubernetes` driver, the Devsy daemon is the pod's main process. It exits normally when the workspace has been idle for the timeout, which ends the pod while the volume is kept. Other container drivers start a detached daemon whose exit does not by itself stop the container. See [Auto-Inactivity Timeout](../developing-in-workspaces/inactivity-timeout.mdx#kubernetes) for activity rules and supported pod layouts. ### Machine providers diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx index 8d2ff1a22c..ef1e91dae9 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx @@ -89,7 +89,7 @@ Example Kubernetes provider that uses local kubectl to run a workspace in the cu name: simple-kubernetes version: v0.0.1 agent: - containerInactivityTimeout: 10m # Pod will automatically kill itself after timeout + containerInactivityTimeout: 10m # Pod stops itself after being idle for the timeout path: ${DEVSY} driver: kubernetes kubernetes: From 835a4a7855b8d6ce74585d1290c2aff8ea2caba6 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 25 Oct 2026 12:17:26 -0600 Subject: [PATCH 02/19] fix: satisfy lint in kubernetes inactivity e2e (#1698) --- e2e/tests/up/provider_kubernetes_inactivity.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/e2e/tests/up/provider_kubernetes_inactivity.go b/e2e/tests/up/provider_kubernetes_inactivity.go index 22bdcf56bd..4902a323dc 100644 --- a/e2e/tests/up/provider_kubernetes_inactivity.go +++ b/e2e/tests/up/provider_kubernetes_inactivity.go @@ -17,6 +17,7 @@ import ( ) const ( + flagIDEOption = "--ide" inactivityLabel = "up-provider-kubernetes-inactivity" inactivityImage = "ghcr.io/devsy-org/test-images/go:1" inactivityPollEvery = 3 * time.Second @@ -351,12 +352,13 @@ var _ = ginkgo.Describe( ) _, stderr, err := e.f.ExecCommandCapture(ctx, - []string{cmdWorkspace, "up", e.dir, "--ide", "none"}) + []string{cmdWorkspace, "up", e.dir, flagIDEOption, kubernetesIgnoreIDE}) gomega.Expect(err).To(gomega.HaveOccurred()) gomega.Expect(stderr).To(gomega.ContainSubstring("sidecar")) pvcs := &corev1.PersistentVolumeClaimList{} framework.ExpectNoError(e.kubectlJSON(ctx, pvcs, "get", "pvc")) - gomega.Expect(pvcs.Items).To(gomega.BeEmpty(), "nothing is created for a rejected pod") + gomega.Expect(pvcs.Items). + To(gomega.BeEmpty(), "nothing is created for a rejected pod") framework.ExpectNoError(e.f.DevsyWorkspaceDelete(ctx, e.dir)) framework.ExpectNoError(os.Chdir(initialDir)) From 5d5ef32d973779d918ff63277617e19379deadd3 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 25 Oct 2026 12:18:01 -0600 Subject: [PATCH 03/19] test: avoid shadowing builtin real in lifecycle test (#1698) --- pkg/daemon/agent/lifecycle_protocol_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/daemon/agent/lifecycle_protocol_test.go b/pkg/daemon/agent/lifecycle_protocol_test.go index 534e49e7f8..e1cba6ab11 100644 --- a/pkg/daemon/agent/lifecycle_protocol_test.go +++ b/pkg/daemon/agent/lifecycle_protocol_test.go @@ -135,10 +135,10 @@ func TestOwnerRejectsSymlinkedControlRecord(t *testing.T) { func TestNewLifecycleOwnerRejectsSymlinkedDir(t *testing.T) { base := t.TempDir() - real := filepath.Join(base, "real") - require.NoError(t, os.Mkdir(real, 0o755)) //nolint:gosec // test dir + target := filepath.Join(base, "target") + require.NoError(t, os.Mkdir(target, 0o755)) //nolint:gosec // test dir link := filepath.Join(base, "link") - require.NoError(t, os.Symlink(real, link)) + require.NoError(t, os.Symlink(target, link)) _, err := NewLifecycleOwner(link) From 71d152b4b70f4d9707a410c8f29cc28c846be92c Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 26 Oct 2026 00:18:36 -0600 Subject: [PATCH 04/19] test: cover waitPodDeleted absent and timeout paths (#1698) --- pkg/driver/kubernetes/pod_lifecycle_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/driver/kubernetes/pod_lifecycle_test.go b/pkg/driver/kubernetes/pod_lifecycle_test.go index 0e0c5179ec..5fcdd02b18 100644 --- a/pkg/driver/kubernetes/pod_lifecycle_test.go +++ b/pkg/driver/kubernetes/pod_lifecycle_test.go @@ -598,6 +598,27 @@ func TestExplicitStopWaitsForPodToBeGone(t *testing.T) { assert.False(t, c.podExists(t)) } +func TestWaitPodDeletedIgnoresAlreadyAbsentPod(t *testing.T) { + c := newFakeCluster(t) + + require.NoError(t, c.k.waitPodDeleted(context.Background(), testPodID)) +} + +func TestWaitPodDeletedTimesOutWhenPodStaysTerminating(t *testing.T) { + c := newFakeCluster(t) + podDeleteTimeout = 50 * time.Millisecond + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning, deleting: true})) + c.cs.PrependReactor("delete", "pods", func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + + err := c.k.waitPodDeleted(context.Background(), testPodID) + + require.Error(t, err) + assert.ErrorContains(t, err, "timeout waiting for pod") + assert.True(t, c.podExists(t)) +} + func runOptionsForWorkspace(timeout string) *driver.RunOptions { cfg, _ := agentd.EncodeDaemonConfig(&agentd.DaemonConfig{ Timeout: timeout, From c7a46bbf377148ad8da0b35439de3fa3c4e07a58 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 26 Oct 2026 01:38:24 -0600 Subject: [PATCH 05/19] fix: make Kubernetes inactivity lifecycle fail safe (#1698) --- cmd/internal/agentcontainer/lifecycle_test.go | 41 ++-- pkg/daemon/agent/activity.go | 124 ++++++++++-- pkg/daemon/agent/lifecycle_monitor.go | 56 +++++- pkg/daemon/agent/lifecycle_monitor_test.go | 184 +++++++++++++++++- pkg/daemon/agent/lifecycle_protocol.go | 126 ++++++++++-- pkg/daemon/agent/lifecycle_protocol_test.go | 135 ++++++++++--- .../inactivity-timeout.mdx | 10 +- 7 files changed, 594 insertions(+), 82 deletions(-) diff --git a/cmd/internal/agentcontainer/lifecycle_test.go b/cmd/internal/agentcontainer/lifecycle_test.go index 0c7319536c..b55ea4ca33 100644 --- a/cmd/internal/agentcontainer/lifecycle_test.go +++ b/cmd/internal/agentcontainer/lifecycle_test.go @@ -36,42 +36,49 @@ func publishOwner(t *testing.T) (*agentd.LifecycleOwner, string) { return owner, dir } +func runLifecycleOwner(t *testing.T, owner *agentd.LifecycleOwner, dir string) *agentd.LifecycleMonitor { + t.Helper() + monitor := &agentd.LifecycleMonitor{} + loop := &agentd.LifecycleLoop{ + Owner: owner, Monitor: monitor, ActivityPath: filepath.Join(dir, "activity"), + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- loop.Run(ctx) }() + t.Cleanup(func() { cancel(); assert.NoError(t, <-done) }) + return monitor +} + func TestBeginLifecycleSuspendsThenReadyArmsSameOwner(t *testing.T) { owner, dir := publishOwner(t) + monitor := runLifecycleOwner(t, owner, dir) + monitor.ArmNow(time.Minute, time.Now()) h, err := beginLifecycle( context.Background(), dir, managedInfo("1m"), config.ShutdownActionStopContainer, ) require.NoError(t, err) require.NotNil(t, h) - - rec, changed, err := owner.PollControl() - require.NoError(t, err) - require.True(t, changed) - assert.Equal(t, agentd.LifecycleStateSetup, rec.State) - assert.Equal(t, "1m", rec.Timeout) - assert.Equal(t, config.ShutdownActionStopContainer, rec.ShutdownAction) + assert.False(t, monitor.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil).Expired, + "setup is suspended before beginLifecycle returns") require.NoError(t, h.ready()) - rec, changed, err = owner.PollControl() - require.NoError(t, err) - require.True(t, changed) - assert.Equal(t, agentd.LifecycleStateReady, rec.State) + + assert.True(t, monitor.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil).Expired, + "ready has re-armed the timer before returning") } func TestLifecycleFailureReleasesSuspension(t *testing.T) { owner, dir := publishOwner(t) + monitor := runLifecycleOwner(t, owner, dir) h, err := beginLifecycle(context.Background(), dir, managedInfo("1m"), "") require.NoError(t, err) - _, _, err = owner.PollControl() - require.NoError(t, err) + assert.False(t, monitor.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil).Expired) h.fail() - rec, changed, err := owner.PollControl() - require.NoError(t, err) - require.True(t, changed) - assert.Equal(t, agentd.LifecycleStateFailed, rec.State) + assert.True(t, monitor.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil).Expired, + "failed setup releases the suspension") } func TestBeginLifecycleMissingOwnerWithTimeoutIsMigrationError(t *testing.T) { diff --git a/pkg/daemon/agent/activity.go b/pkg/daemon/agent/activity.go index 7b7d79ff6d..76a9661b96 100644 --- a/pkg/daemon/agent/activity.go +++ b/pkg/daemon/agent/activity.go @@ -4,7 +4,11 @@ import ( "context" "errors" "os" + "path/filepath" + "strconv" + "strings" "sync" + "sync/atomic" "time" "github.com/devsy-org/devsy/pkg/config" @@ -13,6 +17,10 @@ import ( ) const ( + // failureMarkerInfix separates the activity file path from the pid of a writer + // that currently cannot record activity. + failureMarkerInfix = ".failing." + // DefaultActivityInterval is the longest gap between activity touches. DefaultActivityInterval = 10 * time.Second @@ -20,6 +28,8 @@ const ( minTickInterval = 10 * time.Millisecond ) +var writerSequence atomic.Uint64 + // HeartbeatInterval is how often activity is recorded for a given timeout: at // most the default cadence, and at most a third of short timeouts. func HeartbeatInterval(timeout time.Duration) time.Duration { @@ -62,6 +72,9 @@ type ActivityWriter struct { Now func() time.Time // Tick returns a tick channel and its stop function. Tick func(d time.Duration) (<-chan time.Time, func()) + // Report publishes whether this writer currently fails to record activity. + // It defaults to a failure marker next to the activity file. + Report func(path string, failed bool) error } // Run blocks until ctx is done. @@ -69,32 +82,44 @@ func (w ActivityWriter) Run(ctx context.Context) { w = w.withDefaults() var lastErr error - record := func() { - err := w.Touch(w.Path, w.Now()) - if (err == nil) != (lastErr == nil) { - if err != nil { - log.Errorf("activity writer: %v", err) - } else { - log.Infof("activity writer recovered") - } - } - lastErr = err - } + // A writer that exits has no recording left to vouch for or against. + defer func() { _ = w.Report(w.Path, false) }() ticks, stop := w.Tick(w.Interval) defer stop() - record() + lastErr = w.record(lastErr) for { select { case <-ctx.Done(): - record() + w.record(lastErr) return case <-ticks: - record() + lastErr = w.record(lastErr) } } } +// record touches the activity file and publishes the resulting health. It +// returns the touch error so the caller can detect transitions. +func (w ActivityWriter) record(lastErr error) error { + err := w.Touch(w.Path, w.Now()) + if (err == nil) != (lastErr == nil) { + if err != nil { + log.Errorf("activity writer: %v", err) + } else { + log.Infof("activity writer recovered") + } + } + // Failures are re-reported each tick so a marker that could not be written + // once is retried. + if err != nil || lastErr != nil { + if reportErr := w.Report(w.Path, err != nil); reportErr != nil { + log.Errorf("activity writer: publish recording health: %v", reportErr) + } + } + return err +} + func (w ActivityWriter) withDefaults() ActivityWriter { if w.Touch == nil { w.Touch = TouchActivity @@ -108,6 +133,9 @@ func (w ActivityWriter) withDefaults() ActivityWriter { if w.Interval <= 0 { w.Interval = DefaultActivityInterval } + if w.Report == nil { + w.Report = recordingFailureReporter(w.Path) + } return w } @@ -129,3 +157,71 @@ func StartActivityWriter(ctx context.Context, interval time.Duration) (stop func wg.Wait() } } + +// failureMarkerPath prefixes markers with the writer's PID so the owner can +// discard reports left by exited writers. +func failureMarkerPath(activityPath string, pid int) string { + return activityPath + failureMarkerInfix + strconv.Itoa(pid) +} + +// recordingFailureReporter isolates each writer's health, including concurrent +// connections in one process. +func recordingFailureReporter(activityPath string) func(string, bool) error { + marker := failureMarkerPath(activityPath, os.Getpid()) + "." + + strconv.FormatUint(writerSequence.Add(1), 10) + return func(_ string, failed bool) error { + if !failed { + if err := os.Remove(marker); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil + } + f, err := openNoFollow(marker, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, activityFileMode) + if err != nil { + return err + } + _, writeErr := f.WriteString(procStartTicks(os.Getpid())) + return errors.Join(writeErr, f.Close()) + } +} + +// ActivityRecordingFailing reports whether any live writer has published a +// failure marker for activityPath. Markers of exited writers are removed. +func ActivityRecordingFailing(activityPath string) bool { + dir, base := filepath.Split(activityPath) + entries, err := os.ReadDir(filepath.Clean(dir)) + if err != nil { + return true + } + prefix := base + failureMarkerInfix + failing := false + for _, entry := range entries { + writer, ok := strings.CutPrefix(entry.Name(), prefix) + if !ok { + continue + } + marker := filepath.Join(dir, entry.Name()) + pidText, _, _ := strings.Cut(writer, ".") + if markerWriterAlive(marker, pidText) { + failing = true + } else { + _ = os.Remove(marker) + } + } + return failing +} + +func markerWriterAlive(marker, pidText string) bool { + pid, err := strconv.Atoi(pidText) + if err != nil || pid <= 0 { + return false + } + if !processExists(pid) { + return false + } + ticks, err := sharedfile.ReadFile(marker) + if err != nil { + return true + } + return sameProcess(pid, string(ticks)) +} diff --git a/pkg/daemon/agent/lifecycle_monitor.go b/pkg/daemon/agent/lifecycle_monitor.go index 38eeaa49bd..33b3c52169 100644 --- a/pkg/daemon/agent/lifecycle_monitor.go +++ b/pkg/daemon/agent/lifecycle_monitor.go @@ -27,6 +27,9 @@ type Decision struct { // failed safe and the timer is not effective. ActivityUnreadable bool Failures int + // RecordingFailed means a writer cannot record activity, so an old timestamp + // proves nothing and expiry is withheld until recording recovers. + RecordingFailed bool } // LifecycleMonitor decides when the container has been idle for too long. It @@ -38,6 +41,8 @@ type LifecycleMonitor struct { suspended bool armedAt time.Time failures int + // recordingFailed is set while any activity writer reports it cannot record. + recordingFailed bool } // Apply updates the timer from a control record. It reports whether the activity @@ -77,6 +82,14 @@ func (m *LifecycleMonitor) ArmNow(timeout time.Duration, now time.Time) { m.timeout, m.armed, m.suspended, m.armedAt = timeout, timeout > 0, false, now } +// SetRecordingFailed records whether any activity writer currently fails to +// record activity. +func (m *LifecycleMonitor) SetRecordingFailed(failed bool) { + m.mu.Lock() + defer m.mu.Unlock() + m.recordingFailed = failed +} + // Evaluate checks the idle time. A missing activity file counts as no activity // since arming; any other read error fails safe. func (m *LifecycleMonitor) Evaluate(now, modTime time.Time, statErr error) Decision { @@ -85,6 +98,9 @@ func (m *LifecycleMonitor) Evaluate(now, modTime time.Time, statErr error) Decis if !m.armed || m.suspended { return Decision{} } + if m.recordingFailed { + return Decision{RecordingFailed: true} + } if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { m.failures++ return Decision{ActivityUnreadable: true, Failures: m.failures} @@ -118,7 +134,7 @@ type LifecycleLoop struct { // Run returns ErrInactivityExpired when the container went idle, nil when ctx // ended first. func (l *LifecycleLoop) Run(ctx context.Context) error { - var controlErr, unreadable bool + var controlErr, unreadable, recordingFailed bool timer := time.NewTimer(0) defer timer.Stop() for { @@ -129,6 +145,7 @@ func (l *LifecycleLoop) Run(ctx context.Context) error { } now := time.Now() controlErr = l.applyControl(now, controlErr) + l.Monitor.SetRecordingFailed(ActivityRecordingFailing(l.ActivityPath)) fi, err := os.Stat(l.ActivityPath) var mod time.Time @@ -137,6 +154,7 @@ func (l *LifecycleLoop) Run(ctx context.Context) error { } d := l.Monitor.Evaluate(now, mod, err) unreadable = logUnreadable(d, err, unreadable) + recordingFailed = logRecordingFailed(d, recordingFailed) if d.Expired { log.Infof("container idle for the configured inactivity timeout, shutting down") return ErrInactivityExpired @@ -159,17 +177,39 @@ func (l *LifecycleLoop) applyControl(now time.Time, hadErr bool) bool { if !changed { return false } - reset, err := l.Monitor.Apply(rec, now) - if err != nil { - log.Errorf("apply container lifecycle control record: %v", err) - return true - } - if reset { + return l.applyRecord(rec, now) +} + +func (l *LifecycleLoop) applyRecord(rec ControlRecord, now time.Time) bool { + reset, applyErr := l.Monitor.Apply(rec, now) + if applyErr == nil && reset { if err := TouchActivity(l.ActivityPath, now); err != nil { log.Errorf("reset activity timestamp: %v", err) } } - return false + // Acknowledge only after the state change took effect: setup waits for the + // ack before starting work the timer must not interrupt. + ackErr := l.Owner.Acknowledge(rec, applyErr) + if applyErr != nil { + log.Errorf("apply container lifecycle control record: %v", applyErr) + } + if ackErr != nil { + log.Errorf("acknowledge container lifecycle control record: %v", ackErr) + } + return applyErr != nil || ackErr != nil +} + +func logRecordingFailed(d Decision, was bool) bool { + switch { + case d.RecordingFailed && !was: + log.Errorf("an activity writer cannot record activity; " + + "inactivity shutdown is withheld until recording recovers") + return true + case !d.RecordingFailed && was: + log.Infof("activity recording recovered; inactivity timeout is effective") + return false + } + return was } func logUnreadable(d Decision, err error, was bool) bool { diff --git a/pkg/daemon/agent/lifecycle_monitor_test.go b/pkg/daemon/agent/lifecycle_monitor_test.go index a0f3850850..7ebeae2607 100644 --- a/pkg/daemon/agent/lifecycle_monitor_test.go +++ b/pkg/daemon/agent/lifecycle_monitor_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sync" + "sync/atomic" "testing" "time" @@ -164,7 +165,7 @@ func TestLifecycleLoopStopsIdleManagedDaemon(t *testing.T) { owner, dir := newTestOwner(t) activity := filepath.Join(t.TempDir(), "activity") loop := &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}, ActivityPath: activity} - require.NoError(t, connectNow(t, dir).Send(LifecycleStateReady, "300ms", "")) + mustPublish(t, connectNow(t, dir), control(LifecycleStateReady, "300ms", "")) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -192,7 +193,7 @@ func TestLifecycleLoopRespectsFreshActivity(t *testing.T) { owner, dir := newTestOwner(t) activity := filepath.Join(t.TempDir(), "activity") loop := &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}, ActivityPath: activity} - require.NoError(t, connectNow(t, dir).Send(LifecycleStateReady, "400ms", "")) + mustPublish(t, connectNow(t, dir), control(LifecycleStateReady, "400ms", "")) ctx, cancel := context.WithTimeout(context.Background(), 1200*time.Millisecond) defer cancel() @@ -201,6 +202,185 @@ func TestLifecycleLoopRespectsFreshActivity(t *testing.T) { assert.NoError(t, loop.Run(ctx), "continuous activity must keep the workspace running") } +// idleForAnHour evaluates the monitor an hour from now against an ancient +// activity timestamp. +func idleForAnHour(m *LifecycleMonitor) Decision { + return m.Evaluate(time.Now().Add(time.Hour), t0, nil) +} + +func runLoop(t *testing.T, loop *LifecycleLoop) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- loop.Run(ctx) }() + t.Cleanup(func() { cancel(); <-done }) +} + +func TestSetupSuspendsTheTimerBeforeSendReturns(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + client := connectNow(t, dir) + mustPublish(t, client, control(LifecycleStateReady, "1m", "")) + runLoop(t, &LifecycleLoop{ + Owner: owner, Monitor: monitor, ActivityPath: filepath.Join(t.TempDir(), "activity"), + }) + require.Eventually(t, func() bool { return idleForAnHour(monitor).Expired }, + 5*time.Second, 10*time.Millisecond, "the armed timer expires on an old timestamp") + + require.NoError(t, client.Send(LifecycleStateSetup, "1m", "")) + + assert.False(t, idleForAnHour(monitor).Expired, + "setup must be protected as soon as Send returns") +} + +func TestRepeatedReadySetupCyclesEachTakeEffect(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + client := connectNow(t, dir) + runLoop(t, &LifecycleLoop{ + Owner: owner, Monitor: monitor, ActivityPath: filepath.Join(t.TempDir(), "activity"), + }) + + for range 2 { + require.NoError(t, client.Send(LifecycleStateReady, "1m", "")) + require.True(t, idleForAnHour(monitor).Expired, "ready arms the timer") + require.NoError(t, client.Send(LifecycleStateSetup, "1m", "")) + require.False(t, idleForAnHour(monitor).Expired, "setup suspends it again") + } +} + +func TestMonitorWithholdsExpiryWhileRecordingFails(t *testing.T) { + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) + require.NoError(t, err) + + m.SetRecordingFailed(true) + d := m.Evaluate(t0.Add(time.Hour), t0, nil) + assert.False(t, d.Expired) + assert.True(t, d.RecordingFailed) + + m.SetRecordingFailed(false) + assert.True(t, m.Evaluate(t0.Add(time.Hour), t0, nil).Expired) +} + +func TestLoopDoesNotExpireWhileAWriterCannotRecordActivity(t *testing.T) { + owner, dir := newTestOwner(t) + activity := filepath.Join(t.TempDir(), "activity") + loop := &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}, ActivityPath: activity} + mustPublish(t, connectNow(t, dir), control(LifecycleStateReady, "300ms", "")) + + ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond) + defer cancel() + writerCtx, stopWriter := context.WithCancel(context.Background()) + writerDone := make(chan struct{}) + go func() { + defer close(writerDone) + ActivityWriter{ + Path: activity, + Interval: 50 * time.Millisecond, + Touch: func(string, time.Time) error { return os.ErrPermission }, + }.Run(writerCtx) + }() + defer func() { stopWriter(); <-writerDone }() + + assert.NoError(t, loop.Run(ctx), + "an active session whose heartbeat cannot be written must survive") +} + +func TestLoopExpiresAgainOnceTheWriterRecovers(t *testing.T) { + owner, dir := newTestOwner(t) + activity := filepath.Join(t.TempDir(), "activity") + loop := &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}, ActivityPath: activity} + mustPublish(t, connectNow(t, dir), control(LifecycleStateReady, "300ms", "")) + var failing atomic.Bool + failing.Store(true) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + writerCtx, stopWriter := context.WithCancel(ctx) + writerDone := make(chan struct{}) + go func() { + defer close(writerDone) + ActivityWriter{ + Path: activity, + Interval: 50 * time.Millisecond, + Touch: func(path string, now time.Time) error { + if failing.Load() { + return os.ErrPermission + } + return TouchActivity(path, now) + }, + }.Run(writerCtx) + }() + time.AfterFunc(600*time.Millisecond, func() { failing.Store(false); stopWriter() }) + + err := loop.Run(ctx) + <-writerDone + + assert.ErrorIs(t, err, ErrInactivityExpired) +} + +func TestFailureMarkersOfExitedWritersAreIgnoredAndRemoved(t *testing.T) { + activity := filepath.Join(t.TempDir(), "activity") + marker := failureMarkerPath(activity, 0x7ffffffe) + require.NoError(t, os.WriteFile(marker, nil, 0o600)) + + assert.False(t, ActivityRecordingFailing(activity)) + + _, err := os.Stat(marker) + assert.ErrorIs(t, err, fs.ErrNotExist) +} + +func TestWriterFailureMarkerFollowsRecordingHealth(t *testing.T) { + activity := filepath.Join(t.TempDir(), "activity") + ticks := make(chan time.Time) + var failing atomic.Bool + failing.Store(true) + w := ActivityWriter{ + Path: activity, + Interval: time.Second, + Touch: func(string, time.Time) error { + if failing.Load() { + return os.ErrPermission + } + return nil + }, + Tick: func(time.Duration) (<-chan time.Time, func()) { return ticks, func() {} }, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { w.Run(ctx); close(done) }() + + require.Eventually(t, func() bool { return ActivityRecordingFailing(activity) }, + time.Second, time.Millisecond, "a failing writer publishes a marker") + failing.Store(false) + ticks <- time.Time{} + require.Eventually(t, func() bool { return !ActivityRecordingFailing(activity) }, + time.Second, time.Millisecond, "recovery clears the marker") + + failing.Store(true) + ticks <- time.Time{} + require.Eventually(t, func() bool { return ActivityRecordingFailing(activity) }, + time.Second, time.Millisecond) + cancel() + <-done + assert.False(t, ActivityRecordingFailing(activity), "an exited writer leaves no marker") +} + +func TestOneWriterCannotClearAnotherWritersFailure(t *testing.T) { + activity := filepath.Join(t.TempDir(), "activity") + first := recordingFailureReporter(activity) + second := recordingFailureReporter(activity) + require.NoError(t, first(activity, true)) + require.NoError(t, second(activity, true)) + require.NoError(t, second(activity, false)) + + assert.True(t, ActivityRecordingFailing(activity), + "another writer recovering or disconnecting must not release a failing writer") + + require.NoError(t, first(activity, false)) + assert.False(t, ActivityRecordingFailing(activity)) +} + func TestActivityWriterTouchesImmediatelyPeriodicallyAndOnExit(t *testing.T) { ticks := make(chan time.Time) var mu sync.Mutex diff --git a/pkg/daemon/agent/lifecycle_protocol.go b/pkg/daemon/agent/lifecycle_protocol.go index e3bb28da7c..f707244ee9 100644 --- a/pkg/daemon/agent/lifecycle_protocol.go +++ b/pkg/daemon/agent/lifecycle_protocol.go @@ -15,10 +15,14 @@ import ( const ( // LifecycleProtocolVersion is bumped when owner or control records change shape. - LifecycleProtocolVersion = 1 + LifecycleProtocolVersion = 2 - ownerFileName = "owner.json" - controlFileName = "control.json" + ownerFileName = "owner.json" + controlFileName = "control.json" + ackFileName = "ack.json" + // ackWaitTimeout bounds how long setup waits for the daemon to apply a record. + ackWaitTimeout = 10 * time.Second + ackPollEvery = 20 * time.Millisecond maxRecordBytes = 4096 connectPollEvery = 100 * time.Millisecond runtimeDirMode = 0o755 @@ -28,6 +32,10 @@ const ( var ( // ErrLifecycleOwnerMissing means no live managed daemon published an owner record. ErrLifecycleOwnerMissing = errors.New("container lifecycle owner not found") + // ErrLifecycleNotAcknowledged means the owner did not confirm applying a record in time. + ErrLifecycleNotAcknowledged = errors.New( + "container lifecycle owner did not acknowledge the update", + ) // ErrLifecycleOwnerIncompatible means the owner speaks another protocol version. ErrLifecycleOwnerIncompatible = errors.New("container lifecycle owner is incompatible") ) @@ -56,9 +64,24 @@ type OwnerRecord struct { type ControlRecord struct { Version int `json:"version"` Generation string `json:"generation"` + ClientID string `json:"clientID"` Timeout string `json:"timeout"` ShutdownAction string `json:"shutdownAction"` State LifecycleState `json:"state"` + // Sequence is unique per Send, so an identical repeated state is never + // mistaken for one the owner already applied. + Sequence uint64 `json:"sequence"` +} + +// AckRecord is published by the owner once it has applied the control record with +// the same Sequence. Setup must not start protected work before it sees the ack. +type AckRecord struct { + Version int `json:"version"` + Generation string `json:"generation"` + ClientID string `json:"clientID"` + Sequence uint64 `json:"sequence"` + // Error is set when the owner rejected the record. + Error string `json:"error,omitempty"` } // trustedOwner reports whether files owned by uid may be believed by a process @@ -227,7 +250,7 @@ func requireOwnedDir(dir string, euid int) error { // discardStaleRecords removes records from an earlier process or mount so they // cannot arm this one. func discardStaleRecords(dir string) error { - for _, name := range []string{ownerFileName, controlFileName} { + for _, name := range []string{ownerFileName, controlFileName, ackFileName} { if err := os.Remove( filepath.Join(dir, name), ); err != nil && @@ -260,11 +283,29 @@ func (o *LifecycleOwner) PollControl() (ControlRecord, bool, error) { return rec, true, nil } +// Acknowledge tells the connected client that rec has been applied, or rejected +// when applyErr is set. +func (o *LifecycleOwner) Acknowledge(rec ControlRecord, applyErr error) error { + ack := AckRecord{ + Version: LifecycleProtocolVersion, + Generation: o.generation, + Sequence: rec.Sequence, + ClientID: rec.ClientID, + } + if applyErr != nil { + ack.Error = applyErr.Error() + } + return writeRecord(o.dir, ackFileName, ack, o.euid) +} + // LifecycleClient is the setup-side handle of the lifecycle protocol. type LifecycleClient struct { - dir string - euid int - owner OwnerRecord + dir string + euid int + clientID string + owner OwnerRecord + sequence uint64 + ackTimeout time.Duration } // ConnectLifecycle waits, bounded by ctx, for a live owner record in dir. @@ -275,7 +316,13 @@ func ConnectLifecycle(ctx context.Context, dir string) (*LifecycleClient, error) owner, err := liveOwner(dir, euid) switch { case owner != nil: - return &LifecycleClient{dir: dir, euid: euid, owner: *owner}, nil + return &LifecycleClient{ + dir: dir, + euid: euid, + owner: *owner, + clientID: rand.Text(), + ackTimeout: ackWaitTimeout, + }, nil case errors.Is(err, ErrLifecycleOwnerIncompatible): return nil, err case err != nil: @@ -312,22 +359,75 @@ func liveOwner(dir string, euid int) (*OwnerRecord, error) { } func ownerAlive(o OwnerRecord) bool { - if o.PID <= 0 || !processExists(o.PID) { + return o.PID > 0 && sameProcess(o.PID, o.StartTicks) +} + +// sameProcess reports whether pid is alive and, when both sides know it, still +// the process that started at startTicks. +func sameProcess(pid int, startTicks string) bool { + if !processExists(pid) { return false } - ticks := procStartTicks(o.PID) - return o.StartTicks == "" || ticks == "" || ticks == o.StartTicks + ticks := procStartTicks(pid) + return startTicks == "" || ticks == "" || ticks == startTicks } -// Send publishes the timer state for the connected owner. +// Send publishes the timer state for the connected owner and waits until the +// owner has applied it, so callers can rely on the new state taking effect. func (c *LifecycleClient) Send(state LifecycleState, timeout, shutdownAction string) error { - return writeRecord(c.dir, controlFileName, ControlRecord{ + sequence, err := c.publish(state, timeout, shutdownAction) + if err != nil { + return err + } + return c.awaitAck(sequence) +} + +func (c *LifecycleClient) publish( + state LifecycleState, + timeout, shutdownAction string, +) (uint64, error) { + c.sequence++ + err := writeRecord(c.dir, controlFileName, ControlRecord{ Version: LifecycleProtocolVersion, Generation: c.owner.Generation, + ClientID: c.clientID, Timeout: timeout, ShutdownAction: shutdownAction, State: state, + Sequence: c.sequence, }, c.euid) + return c.sequence, err +} + +func (c *LifecycleClient) readAck(sequence uint64) (AckRecord, bool) { + var ack AckRecord + err := recordReader{dir: c.dir, euid: c.euid, strict: true}.read(ackFileName, &ack) + ok := err == nil && ack.Version == LifecycleProtocolVersion && + ack.Generation == c.owner.Generation && ack.ClientID == c.clientID && + ack.Sequence == sequence + return ack, ok +} + +func (c *LifecycleClient) awaitAck(sequence uint64) error { + deadline := time.Now().Add(c.ackTimeout) + for { + if ack, ok := c.readAck(sequence); ok { + if ack.Error != "" { + return fmt.Errorf("container lifecycle owner rejected the update: %s", ack.Error) + } + return nil + } + if !ownerAlive(c.owner) { + return fmt.Errorf( + "%w: the owner exited before applying the update", + ErrLifecycleOwnerMissing, + ) + } + if !time.Now().Before(deadline) { + return ErrLifecycleNotAcknowledged + } + time.Sleep(ackPollEvery) + } } // ReadLifecycleTimeout leniently reads the timeout from the current control diff --git a/pkg/daemon/agent/lifecycle_protocol_test.go b/pkg/daemon/agent/lifecycle_protocol_test.go index e1cba6ab11..b76e769933 100644 --- a/pkg/daemon/agent/lifecycle_protocol_test.go +++ b/pkg/daemon/agent/lifecycle_protocol_test.go @@ -2,7 +2,7 @@ package agent import ( "context" - "encoding/json" + "errors" "os" "path/filepath" "testing" @@ -29,11 +29,20 @@ func connectNow(t *testing.T, dir string) *LifecycleClient { return client } +// mustPublish writes a control record without waiting for the owner, for tests +// that drive the owner by hand. +func mustPublish(t *testing.T, c *LifecycleClient, rec ControlRecord) uint64 { + t.Helper() + sequence, err := c.publish(rec.State, rec.Timeout, rec.ShutdownAction) + require.NoError(t, err) + return sequence +} + func TestSetupControlRecordReachesOwner(t *testing.T) { owner, dir := newTestOwner(t) client := connectNow(t, dir) - require.NoError(t, client.Send(LifecycleStateSetup, "1m", "stopContainer")) + mustPublish(t, client, control(LifecycleStateSetup, "1m", "stopContainer")) rec, changed, err := owner.PollControl() require.NoError(t, err) @@ -47,30 +56,10 @@ func TestSetupControlRecordReachesOwner(t *testing.T) { assert.False(t, changed, "an unchanged record is not reapplied") } -func TestControlRecordCarriesNoExtraFields(t *testing.T) { - _, dir := newTestOwner(t) - require.NoError(t, connectNow(t, dir).Send(LifecycleStateReady, "1m", "none")) - - raw, err := os.ReadFile(filepath.Join(dir, controlFileName)) //nolint:gosec // test path - require.NoError(t, err) - var fields map[string]any - require.NoError(t, json.Unmarshal(raw, &fields)) - - keys := make([]string, 0, len(fields)) - for k := range fields { - keys = append(keys, k) - } - assert.ElementsMatch( - t, - []string{"version", "generation", "timeout", "shutdownAction", "state"}, - keys, - ) -} - func TestStaleGenerationCannotArmNewOwner(t *testing.T) { first, dir := newTestOwner(t) oldClient := connectNow(t, dir) - require.NoError(t, oldClient.Send(LifecycleStateReady, "1m", "")) + mustPublish(t, oldClient, control(LifecycleStateReady, "1m", "")) require.NotEmpty(t, first.Generation()) // A restarted daemon on the same directory discards the old record. @@ -82,7 +71,7 @@ func TestStaleGenerationCannotArmNewOwner(t *testing.T) { assert.False(t, changed) // Even a stale record that reappears is ignored by generation. - require.NoError(t, oldClient.Send(LifecycleStateReady, "1m", "")) + mustPublish(t, oldClient, control(LifecycleStateReady, "1m", "")) _, changed, err = second.PollControl() require.NoError(t, err) assert.False(t, changed) @@ -150,9 +139,105 @@ func TestReadLifecycleTimeoutFollowsControlRecord(t *testing.T) { _, ok := ReadLifecycleTimeout(dir) assert.False(t, ok, "no control record yet") - require.NoError(t, connectNow(t, dir).Send(LifecycleStateSetup, "45s", "")) + mustPublish(t, connectNow(t, dir), control(LifecycleStateSetup, "45s", "")) timeout, ok := ReadLifecycleTimeout(dir) assert.True(t, ok) assert.Equal(t, 45*time.Second, timeout) } + +func TestSendReturnsOnlyAfterOwnerAppliesTheRecord(t *testing.T) { + owner, dir := newTestOwner(t) + client := connectNow(t, dir) + sent := make(chan error, 1) + go func() { sent <- client.Send(LifecycleStateSetup, "1m", "") }() + + select { + case err := <-sent: + t.Fatalf("Send returned before the owner applied the record: %v", err) + case <-time.After(150 * time.Millisecond): + } + + rec := pollNow(t, owner) + assert.Equal(t, LifecycleStateSetup, rec.State) + require.NoError(t, owner.Acknowledge(rec, nil)) + + select { + case err := <-sent: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("Send did not return after the owner acknowledged the record") + } +} + +func pollNow(t *testing.T, owner *LifecycleOwner) ControlRecord { + t.Helper() + rec, changed, err := owner.PollControl() + require.NoError(t, err) + require.True(t, changed) + return rec +} + +func TestIdenticalRepeatedRecordsAreEachDeliveredToTheOwner(t *testing.T) { + owner, dir := newTestOwner(t) + client := connectNow(t, dir) + + mustPublish(t, client, control(LifecycleStateReady, "1m", "")) + first := pollNow(t, owner) + mustPublish(t, client, control(LifecycleStateReady, "1m", "")) + second := pollNow(t, owner) + + assert.Greater(t, second.Sequence, first.Sequence) +} + +func TestSecondClientRecordIsNeverTakenForTheFirstClientsRecord(t *testing.T) { + owner, dir := newTestOwner(t) + + mustPublish(t, connectNow(t, dir), control(LifecycleStateReady, "1m", "")) + first := pollNow(t, owner) + mustPublish(t, connectNow(t, dir), control(LifecycleStateReady, "1m", "")) + second := pollNow(t, owner) + + assert.NotEqual(t, first.ClientID, second.ClientID) +} + +func TestSendDoesNotAcceptAnotherClientsAcknowledgement(t *testing.T) { + owner, dir := newTestOwner(t) + first := connectNow(t, dir) + mustPublish(t, first, control(LifecycleStateReady, "1m", "")) + require.NoError(t, owner.Acknowledge(pollNow(t, owner), nil)) + + second := connectNow(t, dir) + second.ackTimeout = 100 * time.Millisecond + + assert.ErrorIs(t, second.Send(LifecycleStateSetup, "1m", ""), + ErrLifecycleNotAcknowledged) +} + +func TestSendFailsWhenOwnerNeverAcknowledges(t *testing.T) { + _, dir := newTestOwner(t) + client := connectNow(t, dir) + client.ackTimeout = 100 * time.Millisecond + + err := client.Send(LifecycleStateSetup, "1m", "") + + assert.ErrorIs(t, err, ErrLifecycleNotAcknowledged) +} + +func TestSendSurfacesOwnerRejection(t *testing.T) { + owner, dir := newTestOwner(t) + client := connectNow(t, dir) + sent := make(chan error, 1) + go func() { sent <- client.Send(LifecycleStateSetup, "soon", "") }() + + require.Eventually(t, func() bool { + rec, changed, err := owner.PollControl() + if err != nil || !changed { + return false + } + require.NoError(t, owner.Acknowledge(rec, errors.New("invalid lifecycle timeout"))) + return true + }, 5*time.Second, 10*time.Millisecond) + + assert.ErrorContains(t, <-sent, "invalid lifecycle timeout") +} diff --git a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx index 3c39bbc0be..7ce6f323c1 100644 --- a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx @@ -34,7 +34,7 @@ A workspace is active while any of these hold: CPU use and processes you started yourself do not count. A command such as `nohup make &` in a closed session does not keep the workspace alive. -The daemon samples activity about once a second at most, so a workspace stops up to a few seconds after the timeout passes. For timeouts of a minute or more, sessions record activity every 10 seconds. Shorter timeouts use a proportionally shorter interval. If the daemon cannot read the activity file, it logs an error and does not stop the workspace. +The daemon samples activity about once a second at most, so a workspace stops up to a few seconds after the timeout passes. For timeouts of a minute or more, sessions record activity every 10 seconds. Shorter timeouts use a proportionally shorter interval. If the daemon cannot read the activity file, it logs an error and does not stop the workspace. The same holds when a session or hook cannot write the activity file: the daemon logs an error and does not stop the workspace until recording recovers. ## Kubernetes @@ -48,14 +48,18 @@ When the workspace stops: - Files on the workspace volume persist. Files written to the container filesystem outside mounted volumes are lost, because the new pod starts from the image. - `devsy workspace stop` still deletes the pod through the Kubernetes API from your machine. The pod gets no Kubernetes credentials of its own. -Supported pod layout: +Pod layout rules Devsy enforces: - One regular `devsy` container, any number of normal init containers, and native sidecars (init containers with `restartPolicy: Always`). Native sidecars end with the pod. - With a positive timeout, extra regular containers in `spec.containers`, from a pod manifest template or injected by an admission webhook, are rejected. The pod would keep running while they run. Devsy names the containers in the error. Without a timeout, or with `shutdownAction: none`, extra containers are allowed. - With a positive timeout, the Devsy daemon must be the container's main process. A `devcontainer.json` with `overrideCommand: false` and an image entrypoint or command is rejected. - Dockerless builds do not support a positive timeout. -Existing pods created before this behavior have a main process that cannot enforce the timeout. `devsy workspace up` on such a pod fails with an error that says to stop and start the workspace once. Run `devsy workspace stop`, then `devsy workspace up`. The volume is kept; replacing the agent binary does not update a daemon that is already running. + +The layout rules have unit coverage against a fake Kubernetes API. A kind cluster running Kubernetes v1.36.4 on Linux arm64 has also verified a single regular `devsy` container: two idle-stop/resume cycles preserved workspace volume data, and an open tracked SSH session prevented shutdown until disconnect. Native sidecars, additional init containers, restricted non-root pod security, Devsy Platform routes, and upgrades from an earlier Devsy version have not been verified on a real cluster. Test your layout on a non-production workspace before relying on the timeout. + + +Pods created by an earlier Devsy version have a main process that cannot enforce the timeout. Devsy detects this and `devsy workspace up` on such a pod fails with an error that says to stop and start the workspace once. Run `devsy workspace stop`, then `devsy workspace up`. Replacing the agent binary does not update a daemon that is already running. ## Shutdown action From f48183a2d9e812ea033b2cc238cb80a15584f23e Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 26 Oct 2026 04:02:03 -0600 Subject: [PATCH 06/19] fix: bind inactivity protection to live daemon connections --- cmd/internal/agentcontainer/daemon.go | 1 + cmd/internal/agentcontainer/deferred_hooks.go | 5 +- cmd/internal/agentcontainer/lifecycle.go | 67 +- cmd/internal/agentcontainer/lifecycle_test.go | 93 ++- cmd/internal/agentcontainer/post_attach.go | 9 +- cmd/internal/agentcontainer/setup.go | 8 +- cmd/internal/agentcontainer/ssh_server.go | 21 +- cmd/internal/ssh_server.go | 35 +- cmd/internal/ssh_server_test.go | 41 -- go.mod | 2 +- pkg/daemon/agent/activity.go | 229 +++---- pkg/daemon/agent/lifecycle_monitor.go | 249 ++++---- pkg/daemon/agent/lifecycle_monitor_test.go | 359 ++--------- pkg/daemon/agent/lifecycle_peer_linux_test.go | 112 ++++ pkg/daemon/agent/lifecycle_protocol.go | 437 +++++++------ pkg/daemon/agent/lifecycle_protocol_test.go | 573 +++++++++++++----- pkg/daemon/agent/lifecycle_server.go | 232 +++++++ .../inactivity-timeout.mdx | 10 +- .../docs/developing-providers/agent.mdx | 5 +- 19 files changed, 1462 insertions(+), 1026 deletions(-) create mode 100644 pkg/daemon/agent/lifecycle_peer_linux_test.go create mode 100644 pkg/daemon/agent/lifecycle_server.go diff --git a/cmd/internal/agentcontainer/daemon.go b/cmd/internal/agentcontainer/daemon.go index 8c4c35a355..25679a8b66 100644 --- a/cmd/internal/agentcontainer/daemon.go +++ b/cmd/internal/agentcontainer/daemon.go @@ -194,6 +194,7 @@ func (cmd *DaemonCmd) newLifecycleLoop(timeout time.Duration) (*agentd.Lifecycle return nil, fmt.Errorf("publish container lifecycle owner: %w", err) } if err := sharedfile.EnsureMode(config2.ContainerActivityFile, 0o666); err != nil { + _ = owner.Close() return nil, fmt.Errorf("ensure activity file: %w", err) } loop.Owner = owner diff --git a/cmd/internal/agentcontainer/deferred_hooks.go b/cmd/internal/agentcontainer/deferred_hooks.go index 6f254212a5..77d8796b47 100644 --- a/cmd/internal/agentcontainer/deferred_hooks.go +++ b/cmd/internal/agentcontainer/deferred_hooks.go @@ -112,7 +112,10 @@ func (cmd *DeferredHooksCmd) Run(ctx context.Context) error { return err } - stopActivity := startWorkerActivity(ctx) + ctx, stopActivity, err := startWorkerActivity(ctx) + if err != nil { + return fmt.Errorf("register deferred-hook activity: %w", err) + } defer stopActivity() log.Debugf("running deferred lifecycle hooks") diff --git a/cmd/internal/agentcontainer/lifecycle.go b/cmd/internal/agentcontainer/lifecycle.go index 64975cb4d4..b2969ef2fa 100644 --- a/cmd/internal/agentcontainer/lifecycle.go +++ b/cmd/internal/agentcontainer/lifecycle.go @@ -8,38 +8,31 @@ import ( "fmt" "time" - config2 "github.com/devsy-org/devsy/pkg/config" agentd "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/log" provider2 "github.com/devsy-org/devsy/pkg/provider" ) const ( - // ownerWaitTimeout bounds how long setup waits for the managed daemon when a - // positive timeout depends on it. + // ownerWaitTimeout bounds managed-owner startup, even with no timer. ownerWaitTimeout = 30 * time.Second - // optionalOwnerWait bounds the wait when no timeout is configured. - optionalOwnerWait = time.Second ) // lifecycleHandshake is setup's side of the managed daemon protocol. type lifecycleHandshake struct { - sender lifecycleSender + client *agentd.LifecycleClient + ctx context.Context + cancel context.CancelCauseFunc timeout string shutdownAction string } -type lifecycleSender interface { - Send(state agentd.LifecycleState, timeout, shutdownAction string) error -} - func isManagedLifecycle(info *provider2.ContainerWorkspaceInfo) bool { return info.LifecycleOwner == provider2.ContainerLifecycleOwnerManagedDaemon } -// beginLifecycle connects to the container's managed daemon and suspends its -// timer for the duration of setup. It returns nil when there is no handshake to -// do: not a managed container, or no timeout and no daemon to update. +// beginLifecycle requires an acknowledged managed owner before setup can run. +// Unmanaged containers do not participate in this protocol. func beginLifecycle( ctx context.Context, dir string, @@ -49,26 +42,18 @@ func beginLifecycle( if !isManagedLifecycle(info) { return nil, nil } - wait := ownerWaitTimeout - if info.ContainerTimeout == "" { - wait = optionalOwnerWait - } - waitCtx, cancel := context.WithTimeout(ctx, wait) + waitCtx, cancel := context.WithTimeout(ctx, ownerWaitTimeout) defer cancel() client, err := agentd.ConnectLifecycle(waitCtx, dir) if err != nil { - if info.ContainerTimeout == "" && errors.Is(err, agentd.ErrLifecycleOwnerMissing) { - log.Debugf("no container lifecycle owner and no inactivity timeout configured") - return nil, nil - } return nil, migrationError(err) } + workCtx, stop := client.ProtectedContext(ctx) h := &lifecycleHandshake{ - sender: client, - timeout: info.ContainerTimeout, - shutdownAction: shutdownAction, + client: client, ctx: workCtx, cancel: stop, + timeout: info.ContainerTimeout, shutdownAction: shutdownAction, } if err := h.send(agentd.LifecycleStateSetup); err != nil { return nil, err @@ -85,12 +70,30 @@ func migrationError(err error) error { } func (h *lifecycleHandshake) send(state agentd.LifecycleState) error { - if err := h.sender.Send(state, h.timeout, h.shutdownAction); err != nil { + if err := h.client.Send(state, h.timeout, h.shutdownAction); err != nil { + h.cancel(err) + _ = h.client.Close() return fmt.Errorf("update container lifecycle (%s): %w", state, err) } return nil } +func (h *lifecycleHandshake) close() { + h.cancel(context.Canceled) + _ = h.client.Close() +} + +func (h *lifecycleHandshake) finish(err error) error { + if err == nil { + return nil + } + h.fail() + if cause := context.Cause(h.ctx); errors.Is(cause, agentd.ErrLifecycleOwnerMissing) { + return migrationError(cause) + } + return err +} + // ready resets activity and arms the timer. func (h *lifecycleHandshake) ready() error { return h.send(agentd.LifecycleStateReady) @@ -103,13 +106,7 @@ func (h *lifecycleHandshake) fail() { } } -// startWorkerActivity records activity for the lifetime of a background setup -// worker when a managed daemon owns the timer. The returned stop function ends -// the writer and leaves a fresh timestamp behind. -func startWorkerActivity(ctx context.Context) (stop func()) { - timeout, ok := agentd.ReadLifecycleTimeout(config2.ContainerLifecycleDir) - if !ok { - return func() {} - } - return agentd.StartActivityWriter(ctx, agentd.HeartbeatInterval(timeout)) +// startWorkerActivity completes managed registration before starting hooks. +func startWorkerActivity(ctx context.Context) (context.Context, func(), error) { + return agentd.StartActivityWriter(ctx, agentd.DefaultActivityInterval) } diff --git a/cmd/internal/agentcontainer/lifecycle_test.go b/cmd/internal/agentcontainer/lifecycle_test.go index b55ea4ca33..50773a1e0b 100644 --- a/cmd/internal/agentcontainer/lifecycle_test.go +++ b/cmd/internal/agentcontainer/lifecycle_test.go @@ -7,6 +7,7 @@ import ( "errors" "io" "net" + "os" "path/filepath" "sync/atomic" "testing" @@ -30,13 +31,20 @@ func managedInfo(timeout string) *provider2.ContainerWorkspaceInfo { func publishOwner(t *testing.T) (*agentd.LifecycleOwner, string) { t.Helper() - dir := filepath.Join(t.TempDir(), "lifecycle") + dir, err := os.MkdirTemp("/tmp", "devsy-lifecycle-cmd-") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) owner, err := agentd.NewLifecycleOwner(dir) require.NoError(t, err) + t.Cleanup(func() { _ = owner.Close() }) return owner, dir } -func runLifecycleOwner(t *testing.T, owner *agentd.LifecycleOwner, dir string) *agentd.LifecycleMonitor { +func runLifecycleOwner( + t *testing.T, + owner *agentd.LifecycleOwner, + dir string, +) *agentd.LifecycleMonitor { t.Helper() monitor := &agentd.LifecycleMonitor{} loop := &agentd.LifecycleLoop{ @@ -45,7 +53,12 @@ func runLifecycleOwner(t *testing.T, owner *agentd.LifecycleOwner, dir string) * ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- loop.Run(ctx) }() - t.Cleanup(func() { cancel(); assert.NoError(t, <-done) }) + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + assert.ErrorIs(t, err, agentd.ErrLifecycleOwnerMissing) + } + }) return monitor } @@ -59,6 +72,7 @@ func TestBeginLifecycleSuspendsThenReadyArmsSameOwner(t *testing.T) { ) require.NoError(t, err) require.NotNil(t, h) + defer h.close() assert.False(t, monitor.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil).Expired, "setup is suspended before beginLifecycle returns") @@ -73,6 +87,7 @@ func TestLifecycleFailureReleasesSuspension(t *testing.T) { monitor := runLifecycleOwner(t, owner, dir) h, err := beginLifecycle(context.Background(), dir, managedInfo("1m"), "") require.NoError(t, err) + defer h.close() assert.False(t, monitor.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil).Expired) h.fail() @@ -93,12 +108,12 @@ func TestBeginLifecycleMissingOwnerWithTimeoutIsMigrationError(t *testing.T) { assert.Contains(t, err.Error(), "stop and start the workspace") } -func TestBeginLifecycleMissingOwnerWithoutTimeoutIsSkipped(t *testing.T) { - h, err := beginLifecycle( - context.Background(), filepath.Join(t.TempDir(), "absent"), managedInfo(""), "", - ) +func TestBeginLifecycleRequiresManagedOwnerEvenWithoutTimeout(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + h, err := beginLifecycle(ctx, filepath.Join(t.TempDir(), "absent"), managedInfo(""), "") - assert.NoError(t, err) + assert.ErrorIs(t, err, agentd.ErrLifecycleOwnerMissing) assert.Nil(t, h) } @@ -189,9 +204,10 @@ func TestManagedLoopStartsUnarmedUntilSetup(t *testing.T) { type countingStarter struct{ started, stopped atomic.Int32 } -func (c *countingStarter) start() func() { +func (c *countingStarter) start() (context.Context, func(), error) { c.started.Add(1) - return func() { c.stopped.Add(1) } + ctx, cancel := context.WithCancel(context.Background()) + return ctx, func() { cancel(); c.stopped.Add(1) }, nil } func TestTrackConnectionsCountsConnectionsNotTheListener(t *testing.T) { @@ -222,3 +238,60 @@ func TestTrackConnectionsCountsConnectionsNotTheListener(t *testing.T) { assert.Equal(t, int32(1), starter.stopped.Load(), "disconnect ends the activity writer once") } + +func TestSetupConnectionLossCancelsProtectedWork(t *testing.T) { + owner, dir := publishOwner(t) + monitor := runLifecycleOwner(t, owner, dir) + h, err := beginLifecycle(context.Background(), dir, managedInfo("1m"), "") + require.NoError(t, err) + defer h.close() + + require.NoError(t, owner.Close()) + + select { + case <-h.ctx.Done(): + assert.ErrorIs(t, context.Cause(h.ctx), agentd.ErrLifecycleOwnerMissing) + case <-time.After(time.Second): + t.Fatal("setup continued after its protection connection was lost") + } + require.Eventually(t, func() bool { + return monitor.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil).Expired + }, time.Second, time.Millisecond) +} + +func TestBeginLifecycleRequiresAcknowledgementBeforeReturningSetup(t *testing.T) { + _, dir := publishOwner(t) + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + h, err := beginLifecycle(ctx, dir, managedInfo("1m"), "") + + assert.ErrorIs(t, err, agentd.ErrLifecycleOwnerMissing) + assert.Nil(t, h) +} + +func TestTrackingLossClosesTheSSHConnection(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + inner, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + listener := trackConnections(inner, func() (context.Context, func(), error) { + return ctx, cancel, nil + }) + defer func() { _ = listener.Close() }() + accepted := make(chan net.Conn, 1) + go func() { conn, _ := listener.Accept(); accepted <- conn }() + client, err := net.Dial("tcp", inner.Addr().String()) + require.NoError(t, err) + defer func() { _ = client.Close() }() + server := <-accepted + require.NotNil(t, server) + defer func() { _ = server.Close() }() + + cancel() + + require.NoError(t, client.SetReadDeadline(time.Now().Add(time.Second))) + var buf [1]byte + _, err = client.Read(buf[:]) + assert.ErrorIs(t, err, io.EOF, "SSH must not remain open without lifecycle tracking") +} diff --git a/cmd/internal/agentcontainer/post_attach.go b/cmd/internal/agentcontainer/post_attach.go index f0d9fe5a02..4bc5c02484 100644 --- a/cmd/internal/agentcontainer/post_attach.go +++ b/cmd/internal/agentcontainer/post_attach.go @@ -5,6 +5,7 @@ package agentcontainer import ( "context" "encoding/json" + "fmt" "github.com/devsy-org/devsy/cmd/flags" "github.com/devsy-org/devsy/pkg/compress" @@ -55,7 +56,10 @@ func (cmd *PostAttachCmd) Run(ctx context.Context) error { return err } - stopActivity := startWorkerActivity(ctx) + ctx, stopActivity, err := startWorkerActivity(ctx) + if err != nil { + return fmt.Errorf("register post-attach activity: %w", err) + } defer stopActivity() log.Debugf("running postAttachCommand hooks") @@ -66,6 +70,9 @@ func (cmd *PostAttachCmd) Run(ctx context.Context) error { secretsEnvFromEnvironment(), setup.MountSecretsForRedaction(), ); err != nil { + if ctx.Err() != nil { + return context.Cause(ctx) + } log.Errorf("postAttachCommand failed: %v", err) } diff --git a/cmd/internal/agentcontainer/setup.go b/cmd/internal/agentcontainer/setup.go index 23c9c0d0ca..46503da60d 100644 --- a/cmd/internal/agentcontainer/setup.go +++ b/cmd/internal/agentcontainer/setup.go @@ -107,6 +107,10 @@ func (cmd *SetupContainerCmd) Run(ctx context.Context) error { return err } } + if state.lifecycle != nil { + ctx = state.lifecycle.ctx + defer state.lifecycle.close() + } _, err = tunnelserver.ReportResult( ctx, @@ -121,8 +125,8 @@ func (cmd *SetupContainerCmd) Run(ctx context.Context) error { return state.setupInfo, nil }, ) - if err != nil && state.lifecycle != nil { - state.lifecycle.fail() + if state.lifecycle != nil { + return state.lifecycle.finish(err) } return err } diff --git a/cmd/internal/agentcontainer/ssh_server.go b/cmd/internal/agentcontainer/ssh_server.go index 602518991e..f4a34accbe 100644 --- a/cmd/internal/agentcontainer/ssh_server.go +++ b/cmd/internal/agentcontainer/ssh_server.go @@ -7,6 +7,7 @@ import ( "sync" "github.com/devsy-org/devsy/cmd/flags" + pkgconfig "github.com/devsy-org/devsy/pkg/config" agentd "github.com/devsy-org/devsy/pkg/daemon/agent" cliflags "github.com/devsy-org/devsy/pkg/flags" "github.com/devsy-org/devsy/pkg/flags/names" @@ -99,19 +100,22 @@ func (cmd *SSHServerCmd) Run(_ *cobra.Command, _ []string) error { return server.Serve(trackConnections(listener, startConnectionActivity)) } -func startConnectionActivity() (stop func()) { - return agentd.StartActivityWriter(context.Background(), agentd.ActivityIntervalForLifecycle()) +func startConnectionActivity() (context.Context, func(), error) { + return (agentd.ActivityWriter{ + Path: pkgconfig.ContainerActivityFile, LifecycleDir: pkgconfig.ContainerLifecycleDir, + RequireOwner: true, + }).Start(context.Background()) } // trackConnections records activity from the moment a connection is accepted // until it closes. The idle listener itself is not activity. -func trackConnections(l net.Listener, start func() (stop func())) net.Listener { +func trackConnections(l net.Listener, start func() (context.Context, func(), error)) net.Listener { return &activityListener{Listener: l, start: start} } type activityListener struct { net.Listener - start func() (stop func()) + start func() (context.Context, func(), error) } func (l *activityListener) Accept() (net.Conn, error) { @@ -119,7 +123,14 @@ func (l *activityListener) Accept() (net.Conn, error) { if err != nil { return nil, err } - return &activityConn{Conn: conn, stop: l.start()}, nil + ctx, stop, err := l.start() + if err != nil { + _ = conn.Close() + return nil, fmt.Errorf("register SSH connection activity: %w", err) + } + tracked := &activityConn{Conn: conn, stop: stop} + go func() { <-ctx.Done(); _ = tracked.Close() }() + return tracked, nil } type activityConn struct { diff --git a/cmd/internal/ssh_server.go b/cmd/internal/ssh_server.go index 979e4b5928..478bfb6e23 100644 --- a/cmd/internal/ssh_server.go +++ b/cmd/internal/ssh_server.go @@ -10,7 +10,6 @@ import ( "time" "github.com/devsy-org/devsy/cmd/flags" - "github.com/devsy-org/devsy/pkg/config" agentd "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/flags/names" "github.com/devsy-org/devsy/pkg/log" @@ -104,21 +103,21 @@ func (cmd *sshServerCmd) run(ctx context.Context) error { func (cmd *sshServerCmd) serveStdio(ctx context.Context, server sshserver.Server) error { if cmd.trackActivity { - hbCtx, stopHeartbeat := context.WithCancel(ctx) - done := make(chan struct{}) - go func() { - defer close(done) - runActivityHeartbeat(hbCtx, config.ContainerActivityFile) - }() - defer func() { - stopHeartbeat() - <-done - }() + workCtx, stop, err := agentd.StartActivityWriter(ctx, agentd.DefaultActivityInterval) + if err != nil { + return fmt.Errorf("register SSH activity: %w", err) + } + ctx = workCtx + defer stop() } go shutdownOnCancel(ctx, server) // #nosec G118 -- see shutdownOnCancel. lis := stdio.NewStdioListener(os.Stdin, os.Stdout) log.Debug("serving ssh on stdio") - return ignoreServerClosed(server.Serve(lis)) + err := ignoreServerClosed(server.Serve(lis)) + if cause := context.Cause(ctx); cause != nil && !errors.Is(cause, context.Canceled) { + return cause + } + return err } func (cmd *sshServerCmd) serveListener(ctx context.Context, server sshserver.Server) error { @@ -211,21 +210,11 @@ func decodeBase64Bytes(encoded, label string) ([]byte, error) { return out, nil } -// runActivityHeartbeat records activity immediately, periodically while ctx is -// live, and once more when ctx ends so a disconnect leaves a fresh timestamp. -func runActivityHeartbeat(ctx context.Context, path string) { - agentd.ActivityWriter{ - Path: path, - Interval: agentd.ActivityIntervalForLifecycle(), - }.Run(ctx) -} - func ensureActivityFile(path string) error { return sharedfile.EnsureMode(path, activityFileMode) } -// touchActivityFile is for callers reporting activity on discrete events -// rather than runActivityHeartbeat's fixed interval (e.g. fleet-server). +// touchActivityFile records discrete legacy activity (for example fleet-server). func touchActivityFile(path string) { if err := ensureActivityFile(path); err != nil { log.Errorf("touch activity file: ensure file: %v", err) diff --git a/cmd/internal/ssh_server_test.go b/cmd/internal/ssh_server_test.go index 83bafaa008..3ee3904572 100644 --- a/cmd/internal/ssh_server_test.go +++ b/cmd/internal/ssh_server_test.go @@ -174,22 +174,6 @@ func TestIgnoreServerClosed(t *testing.T) { } } -func TestRunActivityHeartbeatExitsOnContextCancel(t *testing.T) { - path := filepath.Join(t.TempDir(), "activity") - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - runActivityHeartbeat(ctx, path) - close(done) - }() - cancel() - select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatal("heartbeat did not exit within 2s of context cancel") - } -} - func TestEnsureActivityFile_CreatesWorldWritableFile(t *testing.T) { path := filepath.Join(t.TempDir(), "devsy.activity") @@ -244,28 +228,3 @@ func TestTouchActivityFile_UpdatesMtimeOfExistingFile(t *testing.T) { assert.WithinDuration(t, time.Now(), info.ModTime(), 2*time.Second, "touchActivityFile must advance mtime on an already-existing file") } - -func TestRunActivityHeartbeatTouchesOnConnectAndDisconnect(t *testing.T) { - path := filepath.Join(t.TempDir(), "activity") - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - runActivityHeartbeat(ctx, path) - close(done) - }() - - require.Eventually(t, func() bool { - info, err := os.Stat(path) - return err == nil && time.Since(info.ModTime()) < 2*time.Second - }, 2*time.Second, time.Millisecond, "connect must record activity before the first interval") - - stale := time.Now().Add(-time.Hour) - require.NoError(t, os.Chtimes(path, stale, stale)) - cancel() - <-done - - info, err := os.Stat(path) - require.NoError(t, err) - assert.WithinDuration(t, time.Now(), info.ModTime(), 2*time.Second, - "disconnect must leave a fresh timestamp so idle time starts then") -} diff --git a/go.mod b/go.mod index a0ff5c8e72..12302e123d 100644 --- a/go.mod +++ b/go.mod @@ -64,6 +64,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.12.1 github.com/tailscale/hujson v0.0.0-20260727124030-b80ff77dac4f + github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc github.com/tidwall/gjson v1.19.0 github.com/tonistiigi/fsutil v0.0.0-20260819142231-83cac42c1c52 github.com/u-root/u-root v0.16.0 @@ -456,7 +457,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/tailscale/certstore v0.1.1-0.20260409135935-3638fb84b77d // indirect github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 // indirect - github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc // indirect github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 // indirect github.com/tailscale/wireguard-go v0.0.0-20260730222847-4affce44577c // indirect github.com/theupdateframework/go-tuf v0.7.0 // indirect diff --git a/pkg/daemon/agent/activity.go b/pkg/daemon/agent/activity.go index 76a9661b96..9aae639f48 100644 --- a/pkg/daemon/agent/activity.go +++ b/pkg/daemon/agent/activity.go @@ -4,11 +4,7 @@ import ( "context" "errors" "os" - "path/filepath" - "strconv" - "strings" "sync" - "sync/atomic" "time" "github.com/devsy-org/devsy/pkg/config" @@ -17,43 +13,22 @@ import ( ) const ( - // failureMarkerInfix separates the activity file path from the pid of a writer - // that currently cannot record activity. - failureMarkerInfix = ".failing." - - // DefaultActivityInterval is the longest gap between activity touches. DefaultActivityInterval = 10 * time.Second - - activityFileMode = 0o666 - minTickInterval = 10 * time.Millisecond + activityFileMode = 0o666 + minTickInterval = 10 * time.Millisecond ) -var writerSequence atomic.Uint64 - -// HeartbeatInterval is how often activity is recorded for a given timeout: at -// most the default cadence, and at most a third of short timeouts. -func HeartbeatInterval(timeout time.Duration) time.Duration { - if timeout <= 0 { - return DefaultActivityInterval - } - return min(DefaultActivityInterval, max(timeout/3, minTickInterval)) -} - -// TouchActivity records activity at now, creating the shared file if needed. func TouchActivity(path string, now time.Time) error { if err := sharedfile.EnsureMode(path, activityFileMode); err != nil { return err } err := os.Chtimes(path, now, now) if errors.Is(err, os.ErrPermission) { - // Setting an explicit time needs ownership; any writer may bump mtime. return touchByWrite(path) } return err } -// touchByWrite updates the modification time through a write, which the -// 0666 activity file permits for every user. func touchByWrite(path string) error { f, err := os.OpenFile(path, os.O_WRONLY, 0) // #nosec G304 -- fixed activity file path if err != nil { @@ -63,35 +38,99 @@ func touchByWrite(path string) error { return errors.Join(writeErr, f.Close()) } -// ActivityWriter records activity immediately, periodically, and once more when -// its context ends. Zero-value hooks use the real clock and file. +// ActivityWriter retains file heartbeats for non-managed daemons. Managed work +// is protected by its acknowledged connection, including when all writes fail. type ActivityWriter struct { - Path string - Interval time.Duration - Touch func(path string, now time.Time) error - Now func() time.Time - // Tick returns a tick channel and its stop function. - Tick func(d time.Duration) (<-chan time.Time, func()) - // Report publishes whether this writer currently fails to record activity. - // It defaults to a failure marker next to the activity file. - Report func(path string, failed bool) error + Path string + LifecycleDir string + RequireOwner bool + Interval time.Duration + Touch func(string, time.Time) error + Now func() time.Time + Tick func(time.Duration) (<-chan time.Time, func()) +} + +// Start returns only after registering with any live managed owner. The caller +// must use workCtx and stop its work if that context is cancelled. +func (w ActivityWriter) Start(ctx context.Context) (context.Context, func(), error) { + client, err := w.connect(ctx) + if err != nil { + return nil, nil, err + } + if client != nil { + return startRegisteredActivity(ctx, client) + } + w = w.withDefaults() + workCtx, cancel := context.WithCancel(ctx) + lastErr := w.record(nil) + done := make(chan struct{}) + go func() { defer close(done); w.run(workCtx, lastErr) }() + var once sync.Once + return workCtx, func() { once.Do(func() { cancel(); <-done }) }, nil } -// Run blocks until ctx is done. +func startRegisteredActivity( + ctx context.Context, + client *LifecycleClient, +) (context.Context, func(), error) { + workCtx, cancel := client.ProtectedContext(ctx) + if err := client.Send(LifecycleStateActivity, "", ""); err != nil { + cancel(err) + _ = client.Close() + return nil, nil, err + } + // The live connection itself is the activity. No periodic file write or + // health publication is needed, even when storage is completely unavailable. + var once sync.Once + return workCtx, func() { + once.Do(func() { + cancel(context.Canceled) + if err := client.Send(LifecycleStateRelease, "", ""); err != nil { + log.Debugf("activity release failed; closing its protection connection: %v", err) + } + _ = client.Close() + }) + }, nil +} + +// Run records legacy file activity without claiming managed protection. Managed +// callers use Start so registration completes before their work begins. func (w ActivityWriter) Run(ctx context.Context) { w = w.withDefaults() + w.run(ctx, w.record(nil)) +} - var lastErr error - // A writer that exits has no recording left to vouch for or against. - defer func() { _ = w.Report(w.Path, false) }() +func (w ActivityWriter) connect(ctx context.Context) (*LifecycleClient, error) { + if w.LifecycleDir == "" { + if w.RequireOwner { + return nil, ErrLifecycleOwnerMissing + } + return nil, nil + } + owner, err := liveOwner(w.LifecycleDir, os.Geteuid()) + if err != nil { + return nil, err + } + if owner != nil { + return connectOwner(ctx, w.LifecycleDir, *owner) + } + if w.RequireOwner { + return nil, ErrLifecycleOwnerMissing + } + // A discovery gap in an existing managed runtime is not a legacy workspace. + if _, err := os.Lstat(w.LifecycleDir); !errors.Is(err, os.ErrNotExist) { + return nil, ErrLifecycleOwnerMissing + } + return nil, nil +} +func (w ActivityWriter) run(ctx context.Context, lastErr error) { ticks, stop := w.Tick(w.Interval) defer stop() - lastErr = w.record(lastErr) for { select { case <-ctx.Done(): - w.record(lastErr) + _ = w.record(lastErr) return case <-ticks: lastErr = w.record(lastErr) @@ -99,8 +138,6 @@ func (w ActivityWriter) Run(ctx context.Context) { } } -// record touches the activity file and publishes the resulting health. It -// returns the touch error so the caller can detect transitions. func (w ActivityWriter) record(lastErr error) error { err := w.Touch(w.Path, w.Now()) if (err == nil) != (lastErr == nil) { @@ -110,13 +147,6 @@ func (w ActivityWriter) record(lastErr error) error { log.Infof("activity writer recovered") } } - // Failures are re-reported each tick so a marker that could not be written - // once is retried. - if err != nil || lastErr != nil { - if reportErr := w.Report(w.Path, err != nil); reportErr != nil { - log.Errorf("activity writer: publish recording health: %v", reportErr) - } - } return err } @@ -133,9 +163,6 @@ func (w ActivityWriter) withDefaults() ActivityWriter { if w.Interval <= 0 { w.Interval = DefaultActivityInterval } - if w.Report == nil { - w.Report = recordingFailureReporter(w.Path) - } return w } @@ -144,84 +171,20 @@ func realTick(d time.Duration) (<-chan time.Time, func()) { return t.C, t.Stop } -// StartActivityWriter runs an ActivityWriter on the shared activity file until -// the returned stop function is called; stop waits for the final touch. -func StartActivityWriter(ctx context.Context, interval time.Duration) (stop func()) { - wctx, cancel := context.WithCancel(ctx) - var wg sync.WaitGroup - wg.Go(func() { - ActivityWriter{Path: config.ContainerActivityFile, Interval: interval}.Run(wctx) - }) - return func() { - cancel() - wg.Wait() - } -} - -// failureMarkerPath prefixes markers with the writer's PID so the owner can -// discard reports left by exited writers. -func failureMarkerPath(activityPath string, pid int) string { - return activityPath + failureMarkerInfix + strconv.Itoa(pid) -} - -// recordingFailureReporter isolates each writer's health, including concurrent -// connections in one process. -func recordingFailureReporter(activityPath string) func(string, bool) error { - marker := failureMarkerPath(activityPath, os.Getpid()) + "." + - strconv.FormatUint(writerSequence.Add(1), 10) - return func(_ string, failed bool) error { - if !failed { - if err := os.Remove(marker); err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - return nil - } - f, err := openNoFollow(marker, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, activityFileMode) +func StartActivityWriter( + ctx context.Context, + interval time.Duration, +) (context.Context, func(), error) { + required := false + if encoded := os.Getenv(config.EnvWorkspaceDaemonConfig); encoded != "" { + daemon, err := DecodeDaemonConfig(encoded) if err != nil { - return err + return nil, nil, err } - _, writeErr := f.WriteString(procStartTicks(os.Getpid())) - return errors.Join(writeErr, f.Close()) - } -} - -// ActivityRecordingFailing reports whether any live writer has published a -// failure marker for activityPath. Markers of exited writers are removed. -func ActivityRecordingFailing(activityPath string) bool { - dir, base := filepath.Split(activityPath) - entries, err := os.ReadDir(filepath.Clean(dir)) - if err != nil { - return true - } - prefix := base + failureMarkerInfix - failing := false - for _, entry := range entries { - writer, ok := strings.CutPrefix(entry.Name(), prefix) - if !ok { - continue - } - marker := filepath.Join(dir, entry.Name()) - pidText, _, _ := strings.Cut(writer, ".") - if markerWriterAlive(marker, pidText) { - failing = true - } else { - _ = os.Remove(marker) - } - } - return failing -} - -func markerWriterAlive(marker, pidText string) bool { - pid, err := strconv.Atoi(pidText) - if err != nil || pid <= 0 { - return false - } - if !processExists(pid) { - return false - } - ticks, err := sharedfile.ReadFile(marker) - if err != nil { - return true + required = daemon.Lifecycle.Managed() } - return sameProcess(pid, string(ticks)) + return (ActivityWriter{ + Path: config.ContainerActivityFile, LifecycleDir: config.ContainerLifecycleDir, + RequireOwner: required, Interval: interval, + }).Start(ctx) } diff --git a/pkg/daemon/agent/lifecycle_monitor.go b/pkg/daemon/agent/lifecycle_monitor.go index 33b3c52169..e5faa58a16 100644 --- a/pkg/daemon/agent/lifecycle_monitor.go +++ b/pkg/daemon/agent/lifecycle_monitor.go @@ -8,98 +8,130 @@ import ( "sync" "time" - pkgconfig "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/log" ) const ( - idleCheckInterval = 500 * time.Millisecond - maxCheckInterval = time.Second - // unreadableLogAfter is how many consecutive failed checks precede a warning. + idleCheckInterval = 500 * time.Millisecond + maxCheckInterval = time.Second unreadableLogAfter = 3 ) // Decision is the outcome of one inactivity check. type Decision struct { - Expired bool - // ActivityUnreadable means the activity file could not be read, so the check - // failed safe and the timer is not effective. + Expired bool ActivityUnreadable bool Failures int - // RecordingFailed means a writer cannot record activity, so an old timestamp - // proves nothing and expiry is withheld until recording recovers. - RecordingFailed bool } -// LifecycleMonitor decides when the container has been idle for too long. It -// holds no clock or file handles; callers pass observations in. +// LifecycleMonitor serializes protection changes with the decision to stop. type LifecycleMonitor struct { mu sync.Mutex timeout time.Duration armed bool - suspended bool armedAt time.Time failures int - // recordingFailed is set while any activity writer reports it cannot record. - recordingFailed bool + protected map[string]struct{} + stopping bool } -// Apply updates the timer from a control record. It reports whether the activity -// timestamp should be reset to now. +// Apply changes one client's protection. Acknowledgements are sent only after +// this method returns; no filesystem write is needed to retain protection. func (m *LifecycleMonitor) Apply(rec ControlRecord, now time.Time) (bool, error) { - var timeout time.Duration - if rec.Timeout != "" { - var err error - if timeout, err = time.ParseDuration(rec.Timeout); err != nil || timeout < 0 { - return false, fmt.Errorf("invalid lifecycle timeout %q", rec.Timeout) - } - } - enabled := timeout > 0 && rec.ShutdownAction != config.ShutdownActionNone - m.mu.Lock() defer m.mu.Unlock() + if m.stopping { + return false, ErrLifecycleOwnerMissing + } + if rec.State == LifecycleStateActivity { + m.protect(rec.ClientID) + return false, nil + } + if rec.State == LifecycleStateRelease { + m.release(rec.ClientID, now) + return false, nil + } + timeout, err := parseLifecyclePolicy(rec) + if err != nil { + return false, err + } m.timeout = timeout - switch rec.State { - case LifecycleStateSetup: - m.suspended = true + m.armed = timeout > 0 && rec.ShutdownAction != config.ShutdownActionNone + if rec.State == LifecycleStateSetup { + m.protect(rec.ClientID) return false, nil - case LifecycleStateReady, LifecycleStateFailed: - m.suspended = false - m.armed = enabled - m.armedAt = now - m.failures = 0 - return enabled, nil + } + delete(m.protected, rec.ClientID) + m.armedAt = now + m.failures = 0 + return m.armed, nil +} + +func parseLifecyclePolicy(rec ControlRecord) (time.Duration, error) { + switch rec.State { + case LifecycleStateSetup, LifecycleStateReady, LifecycleStateFailed: default: - return false, fmt.Errorf("unknown lifecycle state %q", rec.State) + return 0, fmt.Errorf("unknown lifecycle state %q", rec.State) } + if rec.Timeout == "" { + return 0, nil + } + timeout, err := time.ParseDuration(rec.Timeout) + if err != nil || timeout < 0 { + return 0, fmt.Errorf("invalid lifecycle timeout %q", rec.Timeout) + } + return timeout, nil } -// ArmNow arms the timer without a handshake, for daemons that own no setup. -func (m *LifecycleMonitor) ArmNow(timeout time.Duration, now time.Time) { +// Release ends only this client's protection and starts a fresh idle interval. +func (m *LifecycleMonitor) Release(id string, now time.Time) { m.mu.Lock() defer m.mu.Unlock() - m.timeout, m.armed, m.suspended, m.armedAt = timeout, timeout > 0, false, now + m.release(id, now) } -// SetRecordingFailed records whether any activity writer currently fails to -// record activity. -func (m *LifecycleMonitor) SetRecordingFailed(failed bool) { +// ArmNow arms daemons that have no managed setup handshake. +func (m *LifecycleMonitor) ArmNow(timeout time.Duration, now time.Time) { m.mu.Lock() defer m.mu.Unlock() - m.recordingFailed = failed + m.timeout, m.armed, m.armedAt = timeout, timeout > 0, now } -// Evaluate checks the idle time. A missing activity file counts as no activity -// since arming; any other read error fails safe. +// Evaluate observes the timer without committing the daemon to shutdown. func (m *LifecycleMonitor) Evaluate(now, modTime time.Time, statErr error) Decision { m.mu.Lock() defer m.mu.Unlock() - if !m.armed || m.suspended { - return Decision{} + return m.evaluate(now, modTime, statErr) +} + +// CheckInterval bounds the delay before observing idle time or process death. +func (m *LifecycleMonitor) CheckInterval() time.Duration { + m.mu.Lock() + defer m.mu.Unlock() + if !m.armed { + return idleCheckInterval + } + return min(maxCheckInterval, max(m.timeout/4, minTickInterval)) +} + +func (m *LifecycleMonitor) protect(id string) { + if m.protected == nil { + m.protected = make(map[string]struct{}) + } + m.protected[id] = struct{}{} +} + +func (m *LifecycleMonitor) release(id string, now time.Time) { + if _, ok := m.protected[id]; ok { + delete(m.protected, id) + m.armedAt = now } - if m.recordingFailed { - return Decision{RecordingFailed: true} +} + +func (m *LifecycleMonitor) evaluate(now, modTime time.Time, statErr error) Decision { + if !m.armed || len(m.protected) > 0 { + return Decision{} } if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { m.failures++ @@ -113,48 +145,47 @@ func (m *LifecycleMonitor) Evaluate(now, modTime time.Time, statErr error) Decis return Decision{Expired: now.Sub(last) >= m.timeout} } -// CheckInterval bounds how long the loop sleeps between checks. -func (m *LifecycleMonitor) CheckInterval() time.Duration { +func (m *LifecycleMonitor) commitExpiry(now, modTime time.Time, statErr error) Decision { m.mu.Lock() defer m.mu.Unlock() - if !m.armed { - return idleCheckInterval - } - return min(maxCheckInterval, max(m.timeout/4, minTickInterval)) + d := m.evaluate(now, modTime, statErr) + // A racing registration must be rejected once shutdown has won this lock. + m.stopping = m.stopping || d.Expired + return d } -// LifecycleLoop is the production adapter around LifecycleMonitor. +// LifecycleLoop uses live registrations for managed containers and the legacy +// activity file only for daemons without a managed owner. type LifecycleLoop struct { - // Owner is nil for daemons armed directly with ArmNow. Owner *LifecycleOwner Monitor *LifecycleMonitor ActivityPath string } -// Run returns ErrInactivityExpired when the container went idle, nil when ctx -// ended first. +func lifecycleContextError(ctx context.Context) error { + cause := context.Cause(ctx) + if errors.Is(cause, context.Canceled) || errors.Is(cause, context.DeadlineExceeded) { + return nil + } + return cause +} + func (l *LifecycleLoop) Run(ctx context.Context) error { - var controlErr, unreadable, recordingFailed bool + ctx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + stopOwner := l.startOwner(ctx, cancel) + defer stopOwner() + var unreadable bool timer := time.NewTimer(0) defer timer.Stop() for { select { case <-ctx.Done(): - return nil + return lifecycleContextError(ctx) case <-timer.C: } - now := time.Now() - controlErr = l.applyControl(now, controlErr) - l.Monitor.SetRecordingFailed(ActivityRecordingFailing(l.ActivityPath)) - - fi, err := os.Stat(l.ActivityPath) - var mod time.Time - if err == nil { - mod = fi.ModTime() - } - d := l.Monitor.Evaluate(now, mod, err) + d, err := l.observe(time.Now()) unreadable = logUnreadable(d, err, unreadable) - recordingFailed = logRecordingFailed(d, recordingFailed) if d.Expired { log.Infof("container idle for the configured inactivity timeout, shutting down") return ErrInactivityExpired @@ -163,53 +194,35 @@ func (l *LifecycleLoop) Run(ctx context.Context) error { } } -func (l *LifecycleLoop) applyControl(now time.Time, hadErr bool) bool { +func (l *LifecycleLoop) startOwner(ctx context.Context, cancel context.CancelCauseFunc) func() { if l.Owner == nil { - return false - } - rec, changed, err := l.Owner.PollControl() - if err != nil { - if !hadErr { - log.Errorf("read container lifecycle control record: %v", err) + return func() {} + } + stopped := make(chan struct{}) + go func() { + defer close(stopped) + err := l.Owner.Serve(ctx, l.Monitor) + if ctx.Err() == nil { + if err == nil { + err = ErrLifecycleOwnerMissing + } + cancel(err) } - return true - } - if !changed { - return false - } - return l.applyRecord(rec, now) + }() + return func() { cancel(nil); _ = l.Owner.Close(); <-stopped } } -func (l *LifecycleLoop) applyRecord(rec ControlRecord, now time.Time) bool { - reset, applyErr := l.Monitor.Apply(rec, now) - if applyErr == nil && reset { - if err := TouchActivity(l.ActivityPath, now); err != nil { - log.Errorf("reset activity timestamp: %v", err) - } - } - // Acknowledge only after the state change took effect: setup waits for the - // ack before starting work the timer must not interrupt. - ackErr := l.Owner.Acknowledge(rec, applyErr) - if applyErr != nil { - log.Errorf("apply container lifecycle control record: %v", applyErr) +func (l *LifecycleLoop) observe(now time.Time) (Decision, error) { + if l.Owner != nil { + l.Owner.pruneDeadClients(l.Monitor, now) + return l.Monitor.commitExpiry(now, time.Time{}, nil), nil } - if ackErr != nil { - log.Errorf("acknowledge container lifecycle control record: %v", ackErr) + fi, err := os.Stat(l.ActivityPath) + var mod time.Time + if err == nil { + mod = fi.ModTime() } - return applyErr != nil || ackErr != nil -} - -func logRecordingFailed(d Decision, was bool) bool { - switch { - case d.RecordingFailed && !was: - log.Errorf("an activity writer cannot record activity; " + - "inactivity shutdown is withheld until recording recovers") - return true - case !d.RecordingFailed && was: - log.Infof("activity recording recovered; inactivity timeout is effective") - return false - } - return was + return l.Monitor.commitExpiry(now, mod, err), err } func logUnreadable(d Decision, err error, was bool) bool { @@ -226,13 +239,3 @@ func logUnreadable(d Decision, err error, was bool) bool { } return was } - -// ActivityIntervalForLifecycle derives the activity cadence from the published -// policy, falling back to the default. -func ActivityIntervalForLifecycle() time.Duration { - timeout, ok := ReadLifecycleTimeout(pkgconfig.ContainerLifecycleDir) - if !ok { - return DefaultActivityInterval - } - return HeartbeatInterval(timeout) -} diff --git a/pkg/daemon/agent/lifecycle_monitor_test.go b/pkg/daemon/agent/lifecycle_monitor_test.go index 7ebeae2607..3351d1e4cd 100644 --- a/pkg/daemon/agent/lifecycle_monitor_test.go +++ b/pkg/daemon/agent/lifecycle_monitor_test.go @@ -2,12 +2,10 @@ package agent import ( "context" - "errors" "io/fs" "os" "path/filepath" "sync" - "sync/atomic" "testing" "time" @@ -19,43 +17,36 @@ var t0 = time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) func control(state LifecycleState, timeout, action string) ControlRecord { return ControlRecord{ - Version: LifecycleProtocolVersion, State: state, Timeout: timeout, ShutdownAction: action, + Version: LifecycleProtocolVersion, + ClientID: "client", + State: state, + Timeout: timeout, + ShutdownAction: action, } } func TestMonitorStartsUnarmed(t *testing.T) { - m := &LifecycleMonitor{} - - d := m.Evaluate(t0.Add(24*time.Hour), t0, nil) - - assert.False(t, d.Expired) + assert.False(t, (&LifecycleMonitor{}).Evaluate(t0.Add(24*time.Hour), t0, nil).Expired) } func TestMonitorSuspendedDuringSetupThenArmed(t *testing.T) { m := &LifecycleMonitor{} - reset, err := m.Apply(control(LifecycleStateSetup, "1m", ""), t0) require.NoError(t, err) assert.False(t, reset) - assert.False(t, m.Evaluate(t0.Add(time.Hour), t0, nil).Expired, "suspended by setup") - + assert.False(t, m.Evaluate(t0.Add(time.Hour), t0, nil).Expired) ready := t0.Add(10 * time.Minute) - reset, err = m.Apply(control(LifecycleStateReady, "1m", ""), ready) + _, err = m.Apply(control(LifecycleStateReady, "1m", ""), ready) require.NoError(t, err) - assert.True(t, reset, "arming resets the activity timestamp") - - assert.False(t, m.Evaluate(ready.Add(59*time.Second), t0, nil).Expired, - "old activity cannot expire a freshly armed timer") + assert.False(t, m.Evaluate(ready.Add(59*time.Second), t0, nil).Expired) assert.True(t, m.Evaluate(ready.Add(time.Minute), t0, nil).Expired) } -func TestMonitorActivityKeepsWorkspaceAlive(t *testing.T) { +func TestMonitorLegacyActivityKeepsWorkspaceAlive(t *testing.T) { m := &LifecycleMonitor{} _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) require.NoError(t, err) - recent := t0.Add(50 * time.Second) - assert.False(t, m.Evaluate(t0.Add(100*time.Second), recent, nil).Expired) assert.True(t, m.Evaluate(t0.Add(111*time.Second), recent, nil).Expired) } @@ -64,321 +55,83 @@ func TestMonitorFailedSetupReleasesSuspensionWithFreshInterval(t *testing.T) { m := &LifecycleMonitor{} _, err := m.Apply(control(LifecycleStateSetup, "1m", ""), t0) require.NoError(t, err) - failedAt := t0.Add(30 * time.Minute) _, err = m.Apply(control(LifecycleStateFailed, "1m", ""), failedAt) require.NoError(t, err) - assert.False(t, m.Evaluate(failedAt.Add(30*time.Second), t0, nil).Expired) assert.True(t, m.Evaluate(failedAt.Add(time.Minute), t0, nil).Expired) } -func TestMonitorRepeatedSetupSuspendsAgainAndUpdatesPolicy(t *testing.T) { +func TestMonitorReleaseIsIdempotentAndStartsIdleAtDisconnect(t *testing.T) { m := &LifecycleMonitor{} - _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) - require.NoError(t, err) - - _, err = m.Apply(control(LifecycleStateSetup, "5m", ""), t0.Add(time.Hour)) - require.NoError(t, err) - assert.False(t, m.Evaluate(t0.Add(2*time.Hour), t0, nil).Expired) - - rearmed := t0.Add(3 * time.Hour) - _, err = m.Apply(control(LifecycleStateReady, "5m", ""), rearmed) + _, err := m.Apply(control(LifecycleStateSetup, "1m", ""), t0) require.NoError(t, err) - assert.False(t, m.Evaluate(rearmed.Add(4*time.Minute), rearmed, nil).Expired) - assert.True(t, m.Evaluate(rearmed.Add(5*time.Minute), rearmed, nil).Expired) + m.Release("client", t0.Add(time.Hour)) + m.Release("client", t0.Add(time.Hour+30*time.Second)) + assert.False(t, m.Evaluate(t0.Add(time.Hour+59*time.Second), t0, nil).Expired) + assert.True(t, m.Evaluate(t0.Add(time.Hour+time.Minute), t0, nil).Expired) } func TestMonitorDisabledPolicyDisarms(t *testing.T) { - for name, rec := range map[string]ControlRecord{ - "empty timeout": control(LifecycleStateReady, "", ""), - "shutdown none": control(LifecycleStateReady, "1m", "none"), + for _, rec := range []ControlRecord{ + control(LifecycleStateReady, "", ""), control(LifecycleStateReady, "1m", "none"), } { - t.Run(name, func(t *testing.T) { - m := &LifecycleMonitor{} - _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) - require.NoError(t, err) - - reset, err := m.Apply(rec, t0) - - require.NoError(t, err) - assert.False(t, reset) - assert.False(t, m.Evaluate(t0.Add(24*time.Hour), t0, nil).Expired) - }) + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) + require.NoError(t, err) + _, err = m.Apply(rec, t0) + require.NoError(t, err) + assert.False(t, m.Evaluate(t0.Add(24*time.Hour), t0, nil).Expired) } } -func TestMonitorRejectsInvalidControlRecords(t *testing.T) { - m := &LifecycleMonitor{} - - _, err := m.Apply(control(LifecycleStateReady, "later", ""), t0) - assert.Error(t, err) - _, err = m.Apply(control("bogus", "1m", ""), t0) - assert.Error(t, err) -} - -func TestMonitorUnreadableActivityFailsSafe(t *testing.T) { +func TestMonitorInvalidPolicyDoesNotReplaceCurrentDeadline(t *testing.T) { m := &LifecycleMonitor{} _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) require.NoError(t, err) - denied := &fs.PathError{Op: "stat", Path: "activity", Err: fs.ErrPermission} - - var d Decision - for range 5 { - d = m.Evaluate(t0.Add(time.Hour), time.Time{}, denied) + for _, rec := range []ControlRecord{control(LifecycleStateReady, "later", ""), control("bogus", "1h", "")} { + _, err := m.Apply(rec, t0.Add(30*time.Second)) + require.Error(t, err) } + assert.True(t, m.Evaluate(t0.Add(time.Minute), t0, nil).Expired) +} - assert.False(t, d.Expired, "an active workspace must not stop because activity is unreadable") +func TestMonitorUnreadableLegacyActivityFailsSafe(t *testing.T) { + m := &LifecycleMonitor{} + m.ArmNow(time.Minute, t0) + denied := &fs.PathError{Op: "stat", Path: "activity", Err: fs.ErrPermission} + d := m.Evaluate(t0.Add(time.Hour), time.Time{}, denied) + assert.False(t, d.Expired) assert.True(t, d.ActivityUnreadable) - assert.Equal(t, 5, d.Failures) } -func TestMonitorMissingActivityFileCountsFromArming(t *testing.T) { +func TestMonitorRejectsNewWorkOnceShutdownIsCommitted(t *testing.T) { m := &LifecycleMonitor{} - _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) - require.NoError(t, err) - - d := m.Evaluate(t0.Add(time.Minute), time.Time{}, os.ErrNotExist) - - assert.True(t, d.Expired) - assert.False(t, d.ActivityUnreadable) + m.ArmNow(time.Minute, t0) + require.True(t, m.commitExpiry(t0.Add(time.Minute), t0, nil).Expired) + _, err := m.Apply(control(LifecycleStateSetup, "1m", ""), t0.Add(time.Minute)) + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) } -func TestIntervalsScaleWithShortTimeouts(t *testing.T) { - assert.Equal(t, 10*time.Second, HeartbeatInterval(0)) - assert.Equal(t, 10*time.Second, HeartbeatInterval(time.Minute), "minute scale keeps 10s") - assert.Equal(t, 10*time.Second, HeartbeatInterval(30*time.Second)) - assert.Equal(t, 2*time.Second, HeartbeatInterval(6*time.Second)) - assert.LessOrEqual(t, HeartbeatInterval(900*time.Millisecond), 300*time.Millisecond) - +func TestMonitorRegistrationPreventsCommittingShutdown(t *testing.T) { m := &LifecycleMonitor{} - assert.Equal(t, idleCheckInterval, m.CheckInterval(), "unarmed polls for control records") - _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) + m.ArmNow(time.Minute, t0) + _, err := m.Apply(control(LifecycleStateActivity, "", ""), t0) require.NoError(t, err) - assert.Equal(t, time.Second, m.CheckInterval()) - _, err = m.Apply(control(LifecycleStateReady, "2s", ""), t0) - require.NoError(t, err) - assert.Equal(t, 500*time.Millisecond, m.CheckInterval()) + assert.False(t, m.commitExpiry(t0.Add(time.Hour), t0, nil).Expired) + m.Release("client", t0.Add(time.Hour)) + assert.True(t, m.commitExpiry(t0.Add(time.Hour+time.Minute), t0, nil).Expired) } func TestLifecycleLoopStopsIdleManagedDaemon(t *testing.T) { owner, dir := newTestOwner(t) - activity := filepath.Join(t.TempDir(), "activity") - loop := &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}, ActivityPath: activity} - mustPublish(t, connectNow(t, dir), control(LifecycleStateReady, "300ms", "")) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - err := loop.Run(ctx) - - assert.True(t, errors.Is(err, ErrInactivityExpired), "got %v", err) -} - -func TestLifecycleLoopStaysUnarmedWithoutHandshake(t *testing.T) { - owner, _ := newTestOwner(t) - loop := &LifecycleLoop{ - Owner: owner, - Monitor: &LifecycleMonitor{}, - ActivityPath: filepath.Join(t.TempDir(), "activity"), - } - ctx, cancel := context.WithTimeout(context.Background(), 800*time.Millisecond) - defer cancel() - - err := loop.Run(ctx) - - assert.NoError(t, err) -} - -func TestLifecycleLoopRespectsFreshActivity(t *testing.T) { - owner, dir := newTestOwner(t) - activity := filepath.Join(t.TempDir(), "activity") - loop := &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}, ActivityPath: activity} - mustPublish(t, connectNow(t, dir), control(LifecycleStateReady, "400ms", "")) - - ctx, cancel := context.WithTimeout(context.Background(), 1200*time.Millisecond) - defer cancel() - go ActivityWriter{Path: activity, Interval: 100 * time.Millisecond}.Run(ctx) - - assert.NoError(t, loop.Run(ctx), "continuous activity must keep the workspace running") -} - -// idleForAnHour evaluates the monitor an hour from now against an ancient -// activity timestamp. -func idleForAnHour(m *LifecycleMonitor) Decision { - return m.Evaluate(time.Now().Add(time.Hour), t0, nil) -} - -func runLoop(t *testing.T, loop *LifecycleLoop) { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - go func() { done <- loop.Run(ctx) }() - t.Cleanup(func() { cancel(); <-done }) -} - -func TestSetupSuspendsTheTimerBeforeSendReturns(t *testing.T) { - owner, dir := newTestOwner(t) - monitor := &LifecycleMonitor{} + go func() { done <- (&LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}}).Run(ctx) }() client := connectNow(t, dir) - mustPublish(t, client, control(LifecycleStateReady, "1m", "")) - runLoop(t, &LifecycleLoop{ - Owner: owner, Monitor: monitor, ActivityPath: filepath.Join(t.TempDir(), "activity"), - }) - require.Eventually(t, func() bool { return idleForAnHour(monitor).Expired }, - 5*time.Second, 10*time.Millisecond, "the armed timer expires on an old timestamp") - - require.NoError(t, client.Send(LifecycleStateSetup, "1m", "")) - - assert.False(t, idleForAnHour(monitor).Expired, - "setup must be protected as soon as Send returns") -} - -func TestRepeatedReadySetupCyclesEachTakeEffect(t *testing.T) { - owner, dir := newTestOwner(t) - monitor := &LifecycleMonitor{} - client := connectNow(t, dir) - runLoop(t, &LifecycleLoop{ - Owner: owner, Monitor: monitor, ActivityPath: filepath.Join(t.TempDir(), "activity"), - }) - - for range 2 { - require.NoError(t, client.Send(LifecycleStateReady, "1m", "")) - require.True(t, idleForAnHour(monitor).Expired, "ready arms the timer") - require.NoError(t, client.Send(LifecycleStateSetup, "1m", "")) - require.False(t, idleForAnHour(monitor).Expired, "setup suspends it again") - } -} - -func TestMonitorWithholdsExpiryWhileRecordingFails(t *testing.T) { - m := &LifecycleMonitor{} - _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) - require.NoError(t, err) - - m.SetRecordingFailed(true) - d := m.Evaluate(t0.Add(time.Hour), t0, nil) - assert.False(t, d.Expired) - assert.True(t, d.RecordingFailed) - - m.SetRecordingFailed(false) - assert.True(t, m.Evaluate(t0.Add(time.Hour), t0, nil).Expired) -} - -func TestLoopDoesNotExpireWhileAWriterCannotRecordActivity(t *testing.T) { - owner, dir := newTestOwner(t) - activity := filepath.Join(t.TempDir(), "activity") - loop := &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}, ActivityPath: activity} - mustPublish(t, connectNow(t, dir), control(LifecycleStateReady, "300ms", "")) - - ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond) - defer cancel() - writerCtx, stopWriter := context.WithCancel(context.Background()) - writerDone := make(chan struct{}) - go func() { - defer close(writerDone) - ActivityWriter{ - Path: activity, - Interval: 50 * time.Millisecond, - Touch: func(string, time.Time) error { return os.ErrPermission }, - }.Run(writerCtx) - }() - defer func() { stopWriter(); <-writerDone }() - - assert.NoError(t, loop.Run(ctx), - "an active session whose heartbeat cannot be written must survive") -} - -func TestLoopExpiresAgainOnceTheWriterRecovers(t *testing.T) { - owner, dir := newTestOwner(t) - activity := filepath.Join(t.TempDir(), "activity") - loop := &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}, ActivityPath: activity} - mustPublish(t, connectNow(t, dir), control(LifecycleStateReady, "300ms", "")) - var failing atomic.Bool - failing.Store(true) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - writerCtx, stopWriter := context.WithCancel(ctx) - writerDone := make(chan struct{}) - go func() { - defer close(writerDone) - ActivityWriter{ - Path: activity, - Interval: 50 * time.Millisecond, - Touch: func(path string, now time.Time) error { - if failing.Load() { - return os.ErrPermission - } - return TouchActivity(path, now) - }, - }.Run(writerCtx) - }() - time.AfterFunc(600*time.Millisecond, func() { failing.Store(false); stopWriter() }) - - err := loop.Run(ctx) - <-writerDone - - assert.ErrorIs(t, err, ErrInactivityExpired) -} - -func TestFailureMarkersOfExitedWritersAreIgnoredAndRemoved(t *testing.T) { - activity := filepath.Join(t.TempDir(), "activity") - marker := failureMarkerPath(activity, 0x7ffffffe) - require.NoError(t, os.WriteFile(marker, nil, 0o600)) - - assert.False(t, ActivityRecordingFailing(activity)) - - _, err := os.Stat(marker) - assert.ErrorIs(t, err, fs.ErrNotExist) -} - -func TestWriterFailureMarkerFollowsRecordingHealth(t *testing.T) { - activity := filepath.Join(t.TempDir(), "activity") - ticks := make(chan time.Time) - var failing atomic.Bool - failing.Store(true) - w := ActivityWriter{ - Path: activity, - Interval: time.Second, - Touch: func(string, time.Time) error { - if failing.Load() { - return os.ErrPermission - } - return nil - }, - Tick: func(time.Duration) (<-chan time.Time, func()) { return ticks, func() {} }, - } - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { w.Run(ctx); close(done) }() - - require.Eventually(t, func() bool { return ActivityRecordingFailing(activity) }, - time.Second, time.Millisecond, "a failing writer publishes a marker") - failing.Store(false) - ticks <- time.Time{} - require.Eventually(t, func() bool { return !ActivityRecordingFailing(activity) }, - time.Second, time.Millisecond, "recovery clears the marker") - - failing.Store(true) - ticks <- time.Time{} - require.Eventually(t, func() bool { return ActivityRecordingFailing(activity) }, - time.Second, time.Millisecond) - cancel() - <-done - assert.False(t, ActivityRecordingFailing(activity), "an exited writer leaves no marker") -} - -func TestOneWriterCannotClearAnotherWritersFailure(t *testing.T) { - activity := filepath.Join(t.TempDir(), "activity") - first := recordingFailureReporter(activity) - second := recordingFailureReporter(activity) - require.NoError(t, first(activity, true)) - require.NoError(t, second(activity, true)) - require.NoError(t, second(activity, false)) - - assert.True(t, ActivityRecordingFailing(activity), - "another writer recovering or disconnecting must not release a failing writer") - - require.NoError(t, first(activity, false)) - assert.False(t, ActivityRecordingFailing(activity)) + require.NoError(t, client.Send(LifecycleStateReady, "300ms", "")) + assert.ErrorIs(t, <-done, ErrInactivityExpired) } func TestActivityWriterTouchesImmediatelyPeriodicallyAndOnExit(t *testing.T) { @@ -387,8 +140,7 @@ func TestActivityWriterTouchesImmediatelyPeriodicallyAndOnExit(t *testing.T) { var touched []time.Time count := func() int { mu.Lock(); defer mu.Unlock(); return len(touched) } w := ActivityWriter{ - Path: "activity", - Interval: time.Second, + Path: "activity", Interval: time.Second, Touch: func(_ string, now time.Time) error { mu.Lock() defer mu.Unlock() @@ -401,23 +153,20 @@ func TestActivityWriterTouchesImmediatelyPeriodicallyAndOnExit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { w.Run(ctx); close(done) }() - + defer func() { cancel(); <-done }() require.Eventually(t, func() bool { return count() == 1 }, time.Second, time.Millisecond) ticks <- time.Time{} require.Eventually(t, func() bool { return count() == 2 }, time.Second, time.Millisecond) cancel() <-done - - assert.Equal(t, 3, count(), "start, one tick, and a final touch at exit") + assert.Equal(t, 3, count()) } func TestTouchByWriteAdvancesModTimeForNonOwners(t *testing.T) { path := filepath.Join(t.TempDir(), "activity") require.NoError(t, TouchActivity(path, t0)) require.NoError(t, os.Chtimes(path, t0, t0)) - require.NoError(t, touchByWrite(path)) - info, err := os.Stat(path) require.NoError(t, err) assert.WithinDuration(t, time.Now(), info.ModTime(), 2*time.Second) diff --git a/pkg/daemon/agent/lifecycle_peer_linux_test.go b/pkg/daemon/agent/lifecycle_peer_linux_test.go new file mode 100644 index 0000000000..28e045b242 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_peer_linux_test.go @@ -0,0 +1,112 @@ +//go:build linux + +package agent + +import ( + "bufio" + "context" + "encoding/json" + "os" + "os/exec" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type lifecyclePeerIdentity struct { + UID int `json:"uid"` + GID int `json:"gid"` + Groups []int `json:"groups"` + PID int `json:"pid"` +} + +func TestNonrootLifecyclePeerChild(t *testing.T) { + dir := os.Getenv("DEVSY_NONROOT_LIFECYCLE_CHILD") + if dir == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client, err := ConnectLifecycle(ctx, dir) + require.NoError(t, err) + defer func() { _ = client.Close() }() + require.NoError(t, client.Send(LifecycleStateActivity, "", "")) + groups, err := os.Getgroups() + require.NoError(t, err) + require.NoError(t, json.NewEncoder(os.Stdout).Encode(lifecyclePeerIdentity{ + UID: os.Getuid(), GID: os.Getgid(), Groups: groups, PID: os.Getpid(), + })) + _, err = bufio.NewReader(os.Stdin).ReadString('\n') + require.NoError(t, err) + require.Error(t, client.Send(LifecycleStateReady, "0", "none")) + _, err = os.Stdout.WriteString("rejected\n") + require.NoError(t, err) +} + +func readPeerLine(t *testing.T, scanner *bufio.Scanner) string { + t.Helper() + line := make(chan string, 1) + go func() { scanner.Scan(); line <- scanner.Text() }() + select { + case got := <-line: + return got + case <-time.After(5 * time.Second): + t.Fatal("nonroot peer did not report its state") + return "" + } +} + +func TestRootOwnerAllowsNonrootActivityButRejectsControl(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires a root owner to exercise the actual cross-UID boundary") + } + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + binary, err := os.Executable() + require.NoError(t, err) + cmd := exec.Command( + binary, + "-test.run=^TestNonrootLifecyclePeerChild$", + ) // #nosec G204 -- current test binary + cmd.Env = append(os.Environ(), "DEVSY_NONROOT_LIFECYCLE_CHILD="+dir) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: 65534, Gid: 65534, Groups: []uint32{65534}}, + Pdeathsig: syscall.SIGKILL, + } + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + stdin, err := cmd.StdinPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + scanner := bufio.NewScanner(stdout) + var identity lifecyclePeerIdentity + require.NoError(t, json.Unmarshal([]byte(readPeerLine(t, scanner)), &identity)) + assert.Equal(t, 65534, identity.UID) + assert.Equal(t, 65534, identity.GID) + assert.Equal(t, []int{65534}, identity.Groups) + assert.Equal(t, cmd.Process.Pid, identity.PID) + assert.False( + t, + idleForAnHour(monitor).Expired, + "actual nonroot process holds activity under a root owner", + ) + + _, err = stdin.Write([]byte("control\n")) + require.NoError(t, err) + require.Equal(t, "rejected", readPeerLine(t, scanner)) + require.NoError(t, cmd.Wait()) + + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + time.Second, + time.Millisecond, + "nonowner control cannot disable the timer or leave a permanent hold", + ) +} diff --git a/pkg/daemon/agent/lifecycle_protocol.go b/pkg/daemon/agent/lifecycle_protocol.go index f707244ee9..af9d922b14 100644 --- a/pkg/daemon/agent/lifecycle_protocol.go +++ b/pkg/daemon/agent/lifecycle_protocol.go @@ -1,6 +1,7 @@ package agent import ( + "bufio" "context" "crypto/rand" "encoding/hex" @@ -8,51 +9,51 @@ import ( "errors" "fmt" "io" + "net" "os" "path/filepath" + "strconv" + "sync" "time" + + "github.com/tailscale/peercred" ) const ( - // LifecycleProtocolVersion is bumped when owner or control records change shape. - LifecycleProtocolVersion = 2 - - ownerFileName = "owner.json" - controlFileName = "control.json" - ackFileName = "ack.json" - // ackWaitTimeout bounds how long setup waits for the daemon to apply a record. - ackWaitTimeout = 10 * time.Second - ackPollEvery = 20 * time.Millisecond - maxRecordBytes = 4096 - connectPollEvery = 100 * time.Millisecond - runtimeDirMode = 0o755 - recordFileMode = 0o644 + LifecycleProtocolVersion = 3 + ownerFileName = "owner.json" + socketFileName = "owner.sock" + ackWaitTimeout = 10 * time.Second + ackPollEvery = 100 * time.Millisecond + maxRecordBytes = 4096 + connectPollEvery = 100 * time.Millisecond + runtimeDirMode = 0o755 + recordFileMode = 0o644 ) var ( - // ErrLifecycleOwnerMissing means no live managed daemon published an owner record. - ErrLifecycleOwnerMissing = errors.New("container lifecycle owner not found") - // ErrLifecycleNotAcknowledged means the owner did not confirm applying a record in time. + ErrLifecycleOwnerMissing = errors.New( + "container lifecycle owner unavailable; stop and start the workspace to replace its pod", + ) ErrLifecycleNotAcknowledged = errors.New( "container lifecycle owner did not acknowledge the update", ) - // ErrLifecycleOwnerIncompatible means the owner speaks another protocol version. - ErrLifecycleOwnerIncompatible = errors.New("container lifecycle owner is incompatible") + ErrLifecycleOwnerIncompatible = errors.New( + "container lifecycle owner is incompatible; stop and start the workspace to replace its pod", + ) ) -// LifecycleState is what setup tells the daemon about the timer. type LifecycleState string const ( - // LifecycleStateSetup suspends the timer while synchronous setup runs. - LifecycleStateSetup LifecycleState = "setup" - // LifecycleStateReady resets activity and arms the timer. - LifecycleStateReady LifecycleState = "ready" - // LifecycleStateFailed releases the suspension and starts a fresh idle interval. - LifecycleStateFailed LifecycleState = "failed" + LifecycleStateSetup LifecycleState = "setup" + LifecycleStateReady LifecycleState = "ready" + LifecycleStateFailed LifecycleState = "failed" + LifecycleStateActivity LifecycleState = "activity" + LifecycleStateRelease LifecycleState = "release" ) -// OwnerRecord is published by the daemon process that owns the timer. +// OwnerRecord identifies the daemon instance, not a lease on work. type OwnerRecord struct { Version int `json:"version"` Generation string `json:"generation"` @@ -60,35 +61,29 @@ type OwnerRecord struct { StartTicks string `json:"startTicks,omitempty"` } -// ControlRecord is written by setup. It never carries credentials. +// ControlRecord belongs to one connection and operation. PID/start identity +// prevents inherited descriptors or PID reuse from orphaning setup protection. type ControlRecord struct { Version int `json:"version"` Generation string `json:"generation"` ClientID string `json:"clientID"` + Sequence uint64 `json:"sequence"` + PID int `json:"pid"` + StartTicks string `json:"startTicks,omitempty"` Timeout string `json:"timeout"` ShutdownAction string `json:"shutdownAction"` State LifecycleState `json:"state"` - // Sequence is unique per Send, so an identical repeated state is never - // mistaken for one the owner already applied. - Sequence uint64 `json:"sequence"` } -// AckRecord is published by the owner once it has applied the control record with -// the same Sequence. Setup must not start protected work before it sees the ack. type AckRecord struct { Version int `json:"version"` Generation string `json:"generation"` ClientID string `json:"clientID"` Sequence uint64 `json:"sequence"` - // Error is set when the owner rejected the record. - Error string `json:"error,omitempty"` + Error string `json:"error,omitempty"` } -// trustedOwner reports whether files owned by uid may be believed by a process -// running as euid. -func trustedOwner(uid, euid int) bool { - return uid == euid || uid == 0 -} +func trustedOwner(uid, euid int) bool { return uid == euid || uid == 0 } func checkDir(dir string, euid int) error { fi, err := os.Lstat(dir) @@ -124,25 +119,20 @@ func writeRecord(dir, name string, v any, euid int) error { if err != nil { return fmt.Errorf("create lifecycle record: %w", err) } + defer func() { _ = os.Remove(tmp) }() if _, err := f.Write(raw); err != nil { _ = f.Close() - _ = os.Remove(tmp) return fmt.Errorf("write lifecycle record: %w", err) } if err := f.Close(); err != nil { - _ = os.Remove(tmp) return err } if err := os.Rename(tmp, filepath.Join(dir, name)); err != nil { - _ = os.Remove(tmp) return fmt.Errorf("publish lifecycle record: %w", err) } return nil } -// recordReader loads records from a runtime dir. When strict, the dir and each -// file must be owned by a trusted uid; lenient reads only reject symlinks and -// non-regular files. type recordReader struct { dir string euid int @@ -184,40 +174,39 @@ func (r recordReader) checkFile(f *os.File, name string) error { return nil } -// LifecycleOwner is the daemon-side handle of the lifecycle protocol. -type LifecycleOwner struct { - dir string - euid int - generation string - last ControlRecord -} - -// NewLifecycleOwner prepares a private runtime dir, discards records from any -// earlier process, and publishes a fresh owner record. +// NewLifecycleOwner binds before publishing discovery. All subsequent safety +// decisions and acknowledgements use the connection, never writable files. func NewLifecycleOwner(dir string) (*LifecycleOwner, error) { euid := os.Geteuid() if err := prepareRuntimeDir(dir, euid); err != nil { return nil, err } - gen := make([]byte, 16) - if _, err := rand.Read(gen); err != nil { + listener, err := net.Listen("unix", filepath.Join(dir, socketFileName)) + if err != nil { + return nil, err + } + // #nosec G302 -- container-local activity accepts workspace users; controls require peer ownership. + if err := os.Chmod(filepath.Join(dir, socketFileName), 0o666); err != nil { + _ = listener.Close() return nil, err } - o := &LifecycleOwner{dir: dir, euid: euid, generation: hex.EncodeToString(gen)} - pid := os.Getpid() - err := writeRecord(dir, ownerFileName, OwnerRecord{ - Version: LifecycleProtocolVersion, - Generation: o.generation, - PID: pid, - StartTicks: procStartTicks(pid), + o := &LifecycleOwner{ + euid: euid, generation: rand.Text(), listener: listener, + clients: make(map[net.Conn]*lifecycleSession), + } + o.writeAck = writeSocketAck + err = writeRecord(dir, ownerFileName, OwnerRecord{ + Version: LifecycleProtocolVersion, Generation: o.generation, + PID: os.Getpid(), StartTicks: procStartTicks(os.Getpid()), }, euid) if err != nil { + _ = o.Close() return nil, err } return o, nil } -func prepareRuntimeDir(dir string, euid int) error { +func createRuntimeDir(dir string, euid int) error { if err := os.MkdirAll(filepath.Dir(dir), runtimeDirMode); err != nil { return err } @@ -230,27 +219,14 @@ func prepareRuntimeDir(dir string, euid int) error { if err := os.Chmod(dir, runtimeDirMode); err != nil { return err } - if err := checkDir(dir, euid); err != nil { - return err - } - return discardStaleRecords(dir) + return checkDir(dir, euid) } -func requireOwnedDir(dir string, euid int) error { - fi, err := os.Lstat(dir) - if err != nil { +func prepareRuntimeDir(dir string, euid int) error { + if err := createRuntimeDir(dir, euid); err != nil { return err } - if uid, ok := fileOwner(fi); ok && uid != euid { - return fmt.Errorf("lifecycle runtime dir %s is owned by uid %d, not %d", dir, uid, euid) - } - return nil -} - -// discardStaleRecords removes records from an earlier process or mount so they -// cannot arm this one. -func discardStaleRecords(dir string) error { - for _, name := range []string{ownerFileName, controlFileName, ackFileName} { + for _, name := range []string{ownerFileName, socketFileName} { if err := os.Remove( filepath.Join(dir, name), ); err != nil && @@ -261,86 +237,82 @@ func discardStaleRecords(dir string) error { return nil } -// Generation identifies this daemon process. -func (o *LifecycleOwner) Generation() string { return o.generation } - -// PollControl returns the control record when it is valid for this generation -// and differs from the last one returned. -func (o *LifecycleOwner) PollControl() (ControlRecord, bool, error) { - var rec ControlRecord - err := recordReader{dir: o.dir, euid: o.euid, strict: true}.read(controlFileName, &rec) - switch { - case errors.Is(err, os.ErrNotExist): - return ControlRecord{}, false, nil - case err != nil: - return ControlRecord{}, false, err - case rec.Version != LifecycleProtocolVersion || rec.Generation != o.generation: - return ControlRecord{}, false, nil - case rec == o.last: - return ControlRecord{}, false, nil - } - o.last = rec - return rec, true, nil -} - -// Acknowledge tells the connected client that rec has been applied, or rejected -// when applyErr is set. -func (o *LifecycleOwner) Acknowledge(rec ControlRecord, applyErr error) error { - ack := AckRecord{ - Version: LifecycleProtocolVersion, - Generation: o.generation, - Sequence: rec.Sequence, - ClientID: rec.ClientID, +func requireOwnedDir(dir string, euid int) error { + fi, err := os.Lstat(dir) + if err != nil { + return err } - if applyErr != nil { - ack.Error = applyErr.Error() + if !fi.IsDir() { + return fmt.Errorf("lifecycle runtime path %s is not a directory", dir) + } + if uid, ok := fileOwner(fi); ok && uid != euid { + return fmt.Errorf("lifecycle runtime dir %s is owned by uid %d, not %d", dir, uid, euid) } - return writeRecord(o.dir, ackFileName, ack, o.euid) + return nil } -// LifecycleClient is the setup-side handle of the lifecycle protocol. +func (o *LifecycleOwner) Generation() string { return o.generation } + +// LifecycleClient never reconnects silently: losing protection cancels the work. +// A fresh operation must explicitly connect to the current owner instance. type LifecycleClient struct { - dir string - euid int - clientID string + conn net.Conn owner OwnerRecord + clientID string + mu sync.Mutex sequence uint64 ackTimeout time.Duration + acks chan AckRecord + done chan struct{} + closeOnce sync.Once } -// ConnectLifecycle waits, bounded by ctx, for a live owner record in dir. func ConnectLifecycle(ctx context.Context, dir string) (*LifecycleClient, error) { - euid := os.Geteuid() - var lastErr error for { - owner, err := liveOwner(dir, euid) - switch { - case owner != nil: - return &LifecycleClient{ - dir: dir, - euid: euid, - owner: *owner, - clientID: rand.Text(), - ackTimeout: ackWaitTimeout, - }, nil - case errors.Is(err, ErrLifecycleOwnerIncompatible): + owner, err := liveOwner(dir, os.Geteuid()) + if err != nil { return nil, err - case err != nil: - lastErr = err + } + if owner != nil { + return connectOwner(ctx, dir, *owner) } select { case <-ctx.Done(): - if lastErr != nil { - return nil, fmt.Errorf("%w: %w", ErrLifecycleOwnerMissing, lastErr) - } return nil, ErrLifecycleOwnerMissing case <-time.After(connectPollEvery): } } } -// liveOwner returns the published owner when its process is still the one that -// wrote the record. A missing or stale record is (nil, nil). +func connectOwner(ctx context.Context, dir string, owner OwnerRecord) (*LifecycleClient, error) { + ctx, cancel := context.WithTimeout(ctx, ackWaitTimeout) + defer cancel() + conn, err := (&net.Dialer{}).DialContext(ctx, "unix", filepath.Join(dir, socketFileName)) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrLifecycleOwnerMissing, err) + } + creds, err := peercred.Get(conn) + if err != nil { + _ = conn.Close() + return nil, err + } + uid, ok := creds.UserID() + pid, hasPID := creds.PID() + if !ok || (uid != "0" && uid != strconv.Itoa(os.Geteuid())) || (hasPID && pid != owner.PID) { + _ = conn.Close() + return nil, fmt.Errorf( + "%w: socket owner differs from discovery record", + ErrLifecycleOwnerIncompatible, + ) + } + c := &LifecycleClient{ + conn: conn, owner: owner, clientID: rand.Text(), ackTimeout: ackWaitTimeout, + acks: make(chan AckRecord, 8), done: make(chan struct{}), + } + go c.readAcks() + return c, nil +} + func liveOwner(dir string, euid int) (*OwnerRecord, error) { var owner OwnerRecord err := recordReader{dir: dir, euid: euid, strict: true}.read(ownerFileName, &owner) @@ -348,22 +320,22 @@ func liveOwner(dir string, euid int) (*OwnerRecord, error) { case errors.Is(err, os.ErrNotExist): return nil, nil case err != nil: - return nil, err + return nil, fmt.Errorf("%w: %w", ErrLifecycleOwnerMissing, err) case owner.Version != LifecycleProtocolVersion: - return nil, fmt.Errorf("%w: protocol version %d, want %d", - ErrLifecycleOwnerIncompatible, owner.Version, LifecycleProtocolVersion) + return nil, fmt.Errorf( + "%w: protocol version %d, want %d", + ErrLifecycleOwnerIncompatible, + owner.Version, + LifecycleProtocolVersion, + ) case !ownerAlive(owner): - return nil, nil + return nil, ErrLifecycleOwnerMissing } return &owner, nil } -func ownerAlive(o OwnerRecord) bool { - return o.PID > 0 && sameProcess(o.PID, o.StartTicks) -} +func ownerAlive(o OwnerRecord) bool { return o.PID > 0 && sameProcess(o.PID, o.StartTicks) } -// sameProcess reports whether pid is alive and, when both sides know it, still -// the process that started at startTicks. func sameProcess(pid int, startTicks string) bool { if !processExists(pid) { return false @@ -372,75 +344,136 @@ func sameProcess(pid int, startTicks string) bool { return startTicks == "" || ticks == "" || ticks == startTicks } -// Send publishes the timer state for the connected owner and waits until the -// owner has applied it, so callers can rely on the new state taking effect. -func (c *LifecycleClient) Send(state LifecycleState, timeout, shutdownAction string) error { - sequence, err := c.publish(state, timeout, shutdownAction) - if err != nil { - return err +func (c *LifecycleClient) readAcks() { + defer func() { _ = c.Close() }() + scanner := bufio.NewScanner(c.conn) + scanner.Buffer(make([]byte, 512), maxRecordBytes) + for scanner.Scan() { + var ack AckRecord + if json.Unmarshal(scanner.Bytes(), &ack) != nil || + !c.validAck(ack) { + return + } + select { + case c.acks <- ack: + case <-c.done: + return + default: + return + } } - return c.awaitAck(sequence) } -func (c *LifecycleClient) publish( +func (c *LifecycleClient) Done() <-chan struct{} { return c.done } + +func (c *LifecycleClient) Close() error { + var err error + c.closeOnce.Do(func() { err = c.conn.Close(); close(c.done) }) + return err +} + +// ProtectedContext aborts work on socket loss instead of leaving it untracked. +func (c *LifecycleClient) ProtectedContext( + parent context.Context, +) (context.Context, context.CancelCauseFunc) { + ctx, cancel := context.WithCancelCause(parent) + go func() { + select { + case <-parent.Done(): + cancel(context.Cause(parent)) + _ = c.Close() + case <-c.done: + cancel(ErrLifecycleOwnerMissing) + case <-ctx.Done(): + } + }() + return ctx, cancel +} + +// Send retries the same operation while the connection survives. The owner's +// cached result makes acknowledgement recovery independent of applying state. +func (c *LifecycleClient) Send( state LifecycleState, timeout, shutdownAction string, -) (uint64, error) { +) (sendErr error) { + c.mu.Lock() + defer c.mu.Unlock() + defer func() { + if sendErr != nil { + _ = c.Close() + } + }() c.sequence++ - err := writeRecord(c.dir, controlFileName, ControlRecord{ - Version: LifecycleProtocolVersion, - Generation: c.owner.Generation, - ClientID: c.clientID, - Timeout: timeout, - ShutdownAction: shutdownAction, - State: state, - Sequence: c.sequence, - }, c.euid) - return c.sequence, err -} - -func (c *LifecycleClient) readAck(sequence uint64) (AckRecord, bool) { - var ack AckRecord - err := recordReader{dir: c.dir, euid: c.euid, strict: true}.read(ackFileName, &ack) - ok := err == nil && ack.Version == LifecycleProtocolVersion && - ack.Generation == c.owner.Generation && ack.ClientID == c.clientID && - ack.Sequence == sequence - return ack, ok + rec := ControlRecord{ + Version: LifecycleProtocolVersion, Generation: c.owner.Generation, + ClientID: c.clientID, Sequence: c.sequence, + PID: os.Getpid(), StartTicks: procStartTicks(os.Getpid()), + State: state, Timeout: timeout, ShutdownAction: shutdownAction, + } + return c.sendRecord(rec) } -func (c *LifecycleClient) awaitAck(sequence uint64) error { +func (c *LifecycleClient) sendRecord(rec ControlRecord) error { + raw, err := json.Marshal(rec) + if err != nil { + return err + } + if len(raw) >= maxRecordBytes { + return errors.New("lifecycle message exceeds size limit") + } + raw = append(raw, '\n') deadline := time.Now().Add(c.ackTimeout) for { - if ack, ok := c.readAck(sequence); ok { - if ack.Error != "" { - return fmt.Errorf("container lifecycle owner rejected the update: %s", ack.Error) - } - return nil + ack, err := c.exchange(raw, rec.Sequence, deadline) + if err != nil { + return err } - if !ownerAlive(c.owner) { - return fmt.Errorf( - "%w: the owner exited before applying the update", - ErrLifecycleOwnerMissing, - ) + if ack == nil { + continue } - if !time.Now().Before(deadline) { - return ErrLifecycleNotAcknowledged + if ack.Error != "" { + return fmt.Errorf("container lifecycle owner rejected the update: %s", ack.Error) } - time.Sleep(ackPollEvery) + return nil } } -// ReadLifecycleTimeout leniently reads the timeout from the current control -// record, for deriving activity cadence. It reports false when there is none. -func ReadLifecycleTimeout(dir string) (time.Duration, bool) { - var rec ControlRecord - err := recordReader{dir: dir}.read(controlFileName, &rec) - if err != nil || rec.Version != LifecycleProtocolVersion { - return 0, false +func (c *LifecycleClient) exchange( + raw []byte, + sequence uint64, + deadline time.Time, +) (*AckRecord, error) { + if err := c.conn.SetWriteDeadline(deadline); err != nil { + return nil, fmt.Errorf("%w: %w", ErrLifecycleOwnerMissing, err) } - d, err := time.ParseDuration(rec.Timeout) - if err != nil || d <= 0 { - return 0, rec.Timeout == "" + if _, err := c.conn.Write(raw); err != nil { + return nil, fmt.Errorf("%w: %w", ErrLifecycleOwnerMissing, err) } - return d, true + return c.awaitAck(sequence, deadline) +} + +func (c *LifecycleClient) awaitAck(sequence uint64, deadline time.Time) (*AckRecord, error) { + timer := time.NewTimer(min(ackPollEvery, max(time.Until(deadline), 0))) + defer timer.Stop() + for { + select { + case ack := <-c.acks: + if ack.Sequence == sequence { + return &ack, nil + } + case <-c.done: + return nil, ErrLifecycleOwnerMissing + case <-timer.C: + if !time.Now().Before(deadline) { + return nil, ErrLifecycleNotAcknowledged + } + return nil, nil + } + } +} + +func (c *LifecycleClient) validAck(ack AckRecord) bool { + return ack.Version == LifecycleProtocolVersion && + ack.Generation == c.owner.Generation && + ack.ClientID == c.clientID } diff --git a/pkg/daemon/agent/lifecycle_protocol_test.go b/pkg/daemon/agent/lifecycle_protocol_test.go index b76e769933..b749121def 100644 --- a/pkg/daemon/agent/lifecycle_protocol_test.go +++ b/pkg/daemon/agent/lifecycle_protocol_test.go @@ -1,10 +1,16 @@ package agent import ( + "bufio" + "bytes" "context" - "errors" + "encoding/json" + "net" "os" + "os/exec" "path/filepath" + "runtime" + "sync/atomic" "testing" "time" @@ -14,9 +20,16 @@ import ( func newTestOwner(t *testing.T) (*LifecycleOwner, string) { t.Helper() - dir := filepath.Join(t.TempDir(), "lifecycle") + if runtime.GOOS == "windows" { + t.Skip("managed Kubernetes lifecycle uses Unix peer identity") + } + // Darwin's Unix socket path limit is shorter than testing.TempDir's names. + dir, err := os.MkdirTemp("/tmp", "devsy-lifecycle-") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) owner, err := NewLifecycleOwner(dir) require.NoError(t, err) + t.Cleanup(func() { _ = owner.Close() }) return owner, dir } @@ -26,218 +39,502 @@ func connectNow(t *testing.T, dir string) *LifecycleClient { defer cancel() client, err := ConnectLifecycle(ctx, dir) require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) return client } -// mustPublish writes a control record without waiting for the owner, for tests -// that drive the owner by hand. -func mustPublish(t *testing.T, c *LifecycleClient, rec ControlRecord) uint64 { +func runLoop(t *testing.T, loop *LifecycleLoop) { t.Helper() - sequence, err := c.publish(rec.State, rec.Timeout, rec.ShutdownAction) - require.NoError(t, err) - return sequence + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- loop.Run(ctx) }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) + } + }) } -func TestSetupControlRecordReachesOwner(t *testing.T) { +func idleForAnHour(m *LifecycleMonitor) Decision { + return m.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil) +} + +func TestSetupIsProtectedBeforeSendReturnsAndReadyResetsIt(t *testing.T) { owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) client := connectNow(t, dir) + for range 2 { + require.NoError(t, client.Send(LifecycleStateSetup, "1m", "")) + assert.False(t, idleForAnHour(monitor).Expired) + require.NoError(t, client.Send(LifecycleStateReady, "1m", "")) + assert.True(t, idleForAnHour(monitor).Expired) + } +} - mustPublish(t, client, control(LifecycleStateSetup, "1m", "stopContainer")) +func TestAcknowledgementFailureRetriesWithoutReapplyingReady(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + firstApplied := make(chan time.Time, 1) + var writes atomic.Int32 + owner.writeAck = func(conn net.Conn, ack AckRecord) error { + if writes.Add(1) == 1 { + monitor.mu.Lock() + firstApplied <- monitor.armedAt + monitor.mu.Unlock() + return os.ErrPermission + } + return writeSocketAck(conn, ack) + } + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + client := connectNow(t, dir) - rec, changed, err := owner.PollControl() - require.NoError(t, err) - require.True(t, changed) - assert.Equal(t, LifecycleStateSetup, rec.State) - assert.Equal(t, "1m", rec.Timeout) - assert.Equal(t, owner.Generation(), rec.Generation) + require.NoError(t, client.Send(LifecycleStateReady, "1m", "")) - _, changed, err = owner.PollControl() - require.NoError(t, err) - assert.False(t, changed, "an unchanged record is not reapplied") + appliedAt := <-firstApplied + assert.True(t, monitor.Evaluate(appliedAt.Add(time.Minute), time.Time{}, nil).Expired, + "retrying the ACK must not shift the original idle deadline") } -func TestStaleGenerationCannotArmNewOwner(t *testing.T) { - first, dir := newTestOwner(t) - oldClient := connectNow(t, dir) - mustPublish(t, oldClient, control(LifecycleStateReady, "1m", "")) - require.NotEmpty(t, first.Generation()) +func TestUnacknowledgedSetupCanBeClosedWithoutOrphaningProtection(t *testing.T) { + owner, dir := newTestOwner(t) + owner.writeAck = func(net.Conn, AckRecord) error { return os.ErrPermission } + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + client := connectNow(t, dir) + client.ackTimeout = 250 * time.Millisecond - // A restarted daemon on the same directory discards the old record. - second, err := NewLifecycleOwner(dir) + require.ErrorIs(t, client.Send(LifecycleStateSetup, "1m", ""), ErrLifecycleNotAcknowledged) + require.NoError(t, client.Close()) + + require.Eventually(t, func() bool { return idleForAnHour(monitor).Expired }, + time.Second, time.Millisecond, "failed handshake cleanup must release setup") +} + +func TestLiveSlowSetupHasNoLeaseThatExpiresIt(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + var loopErr error + go func() { defer close(done); loopErr = (&LifecycleLoop{Owner: owner, Monitor: monitor}).Run(ctx) }() + defer func() { cancel(); <-done }() + client := connectNow(t, dir) + require.NoError(t, client.Send(LifecycleStateSetup, "150ms", "")) + + assert.Never(t, func() bool { return monitor.Evaluate(time.Now(), time.Time{}, nil).Expired }, + 500*time.Millisecond, 10*time.Millisecond) + require.NoError(t, client.Close()) + select { + case <-done: + assert.ErrorIs(t, loopErr, ErrInactivityExpired) + case <-time.After(5 * time.Second): + t.Fatal("disconnected setup remained suspended") + } +} + +func TestOneSetupCannotReleaseAnotherSetupsProtection(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + first, second := connectNow(t, dir), connectNow(t, dir) + require.NoError(t, first.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, second.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, first.Send(LifecycleStateReady, "1m", "")) + assert.False(t, idleForAnHour(monitor).Expired) + require.NoError(t, second.Send(LifecycleStateFailed, "1m", "")) + assert.True(t, idleForAnHour(monitor).Expired) +} + +func TestStorageFailureCannotHideARegisteredLiveWriter(t *testing.T) { + owner, dir := newTestOwner(t) + if os.Geteuid() == 0 { + t.Skip("read-only directory permissions require an unprivileged writer") + } + // Stat remains readable with an old timestamp, but touching the malformed + // activity file and creating ANY health/publication inode both fail. + activity := filepath.Join(dir, "activity") + require.NoError(t, os.Mkdir(activity, 0o700)) + old := time.Now().Add(-time.Hour) + require.NoError(t, os.Chtimes(activity, old, old)) + // #nosec G302 -- readable but unwritable storage is the regression scenario. + require.NoError(t, os.Chmod(dir, 0o555)) + t.Cleanup(func() { + // #nosec G302 -- restore owner-write access so temporary state can be removed. + _ = os.Chmod(dir, 0o755) + }) + require.Error(t, TouchActivity(activity, time.Now())) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor, ActivityPath: activity}) + client := connectNow(t, dir) + require.NoError(t, client.Send(LifecycleStateReady, "1m", "")) + require.Error( + t, + writeRecord( + dir, + "unwritable-health.json", + control(LifecycleStateReady, "1m", ""), + os.Geteuid(), + ), + ) + + workCtx, stop, err := (ActivityWriter{Path: activity, LifecycleDir: dir, Interval: 10 * time.Millisecond}).Start( + context.Background(), + ) require.NoError(t, err) - require.NotEqual(t, first.Generation(), second.Generation()) - _, changed, err := second.PollControl() + defer stop() + assert.NoError(t, workCtx.Err()) + assert.False(t, monitor.Evaluate(time.Now().Add(time.Hour), old, nil).Expired, + "neither failed file write can remove in-memory live registration") + + stop() + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + time.Second, + time.Millisecond, + ) +} + +func TestReleaseAcknowledgementRecoveryDoesNotResetIdleTwice(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + var dropped atomic.Bool + firstRelease := make(chan time.Time, 1) + owner.writeAck = func(conn net.Conn, ack AckRecord) error { + if ack.Sequence == 2 && dropped.CompareAndSwap(false, true) { + monitor.mu.Lock() + firstRelease <- monitor.armedAt + monitor.mu.Unlock() + return os.ErrPermission + } + return writeSocketAck(conn, ack) + } + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + require.NoError(t, connectNow(t, dir).Send(LifecycleStateReady, "1m", "")) + _, stop, err := (ActivityWriter{Path: filepath.Join(dir, "activity"), LifecycleDir: dir}).Start( + context.Background(), + ) require.NoError(t, err) - assert.False(t, changed) + defer stop() + + stop() + + appliedAt := <-firstRelease + assert.True(t, monitor.Evaluate(appliedAt.Add(time.Minute), time.Time{}, nil).Expired, + "release must retry after publication failure without another idle reset") +} - // Even a stale record that reappears is ignored by generation. - mustPublish(t, oldClient, control(LifecycleStateReady, "1m", "")) - _, changed, err = second.PollControl() +func TestOwnerLossCancelsProtectedWriterAndRejectsFurtherOperations(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + workCtx, stop, err := (ActivityWriter{Path: filepath.Join(dir, "activity"), LifecycleDir: dir}).Start( + context.Background(), + ) require.NoError(t, err) - assert.False(t, changed) + defer stop() + + require.NoError(t, owner.Close()) + + select { + case <-workCtx.Done(): + assert.ErrorIs(t, context.Cause(workCtx), ErrLifecycleOwnerMissing) + case <-time.After(time.Second): + t.Fatal("work continued after losing its lifecycle owner") + } + _, _, err = (ActivityWriter{Path: filepath.Join(dir, "activity"), LifecycleDir: dir}).Start( + context.Background(), + ) + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) } -func TestConnectLifecycleWithoutOwnerFailsBounded(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) +func TestLifecycleChildProcess(t *testing.T) { + dir := os.Getenv("DEVSY_LIFECYCLE_TEST_CHILD") + if dir == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + client, err := ConnectLifecycle(ctx, dir) + if err != nil { + t.Fatal(err) + } + defer func() { _ = client.Close() }() + if err := client.Send(LifecycleStateSetup, "1m", ""); err != nil { + t.Fatal(err) + } + if _, err := os.Stdout.WriteString("registered\n"); err != nil { + t.Fatal(err) + } + <-client.Done() +} - _, err := ConnectLifecycle(ctx, filepath.Join(t.TempDir(), "absent")) +func TestKilledSetupProcessReleasesProtectionAndCanReconnect(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + binary, err := os.Executable() + require.NoError(t, err) + cmd := exec.Command( + binary, + "-test.run=^TestLifecycleChildProcess$", + ) // #nosec G204 -- current test binary + cmd.Env = append(os.Environ(), "DEVSY_LIFECYCLE_TEST_CHILD="+dir) + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + registered := make(chan string, 1) + go func() { scanner := bufio.NewScanner(stdout); scanner.Scan(); registered <- scanner.Text() }() + select { + case line := <-registered: + require.Equal(t, "registered", line) + case <-time.After(5 * time.Second): + t.Fatal("child did not establish acknowledged setup") + } + assert.False(t, idleForAnHour(monitor).Expired) + + require.NoError(t, cmd.Process.Kill()) + require.Error(t, cmd.Wait()) + + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + time.Second, + time.Millisecond, + ) + reconnected := connectNow(t, dir) + require.NoError(t, reconnected.Send(LifecycleStateSetup, "1m", "")) + assert.False(t, idleForAnHour(monitor).Expired) + require.NoError(t, reconnected.Send(LifecycleStateReady, "1m", "")) + assert.True(t, idleForAnHour(monitor).Expired) +} - assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) +func TestOwnerRestartInvalidatesExistingConnectionAndNewOwnerStartsUnarmed(t *testing.T) { + first, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: first, Monitor: monitor}) + old := connectNow(t, dir) + require.NoError(t, old.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, first.Close()) + second, err := NewLifecycleOwner(dir) + require.NoError(t, err) + t.Cleanup(func() { _ = second.Close() }) + fresh := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: second, Monitor: fresh}) + + assert.ErrorIs(t, old.Send(LifecycleStateReady, "1m", ""), ErrLifecycleOwnerMissing) + assert.False(t, idleForAnHour(fresh).Expired) + require.NoError(t, connectNow(t, dir).Send(LifecycleStateReady, "1m", "")) + assert.True(t, idleForAnHour(fresh).Expired) } func TestConnectLifecycleRejectsIncompatibleOwner(t *testing.T) { _, dir := newTestOwner(t) require.NoError(t, writeRecord(dir, ownerFileName, OwnerRecord{ - Version: LifecycleProtocolVersion + 1, Generation: "g", PID: os.Getpid(), + Version: LifecycleProtocolVersion - 1, Generation: "old", PID: os.Getpid(), }, os.Geteuid())) - _, err := ConnectLifecycle(context.Background(), dir) - assert.ErrorIs(t, err, ErrLifecycleOwnerIncompatible) } -func TestConnectLifecycleIgnoresOwnerOfDeadProcess(t *testing.T) { - _, dir := newTestOwner(t) - require.NoError(t, writeRecord(dir, ownerFileName, OwnerRecord{ - Version: LifecycleProtocolVersion, Generation: "g", PID: 0x7ffffffe, - }, os.Geteuid())) - ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) +func TestConnectLifecycleMissingOwnerIsBounded(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() + _, err := ConnectLifecycle(ctx, filepath.Join(t.TempDir(), "absent")) + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) +} - _, err := ConnectLifecycle(ctx, dir) +func TestClientSeesRejectedPolicyWithoutArming(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + err := connectNow(t, dir).Send(LifecycleStateSetup, "soon", "") + assert.ErrorContains(t, err, "invalid lifecycle timeout") + assert.False(t, idleForAnHour(monitor).Expired) +} + +func TestMissingManagedDiscoveryNeverFallsBackToUntrackedWork(t *testing.T) { + owner, dir := newTestOwner(t) + require.NoError(t, owner.Close()) + require.NoError(t, os.Remove(filepath.Join(dir, ownerFileName))) + + ctx, stop, err := (ActivityWriter{ + Path: filepath.Join(dir, "activity"), LifecycleDir: dir, + }).Start(context.Background()) + if stop != nil { + defer stop() + } assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) + assert.Nil(t, ctx) } -func TestOwnerRejectsSymlinkedControlRecord(t *testing.T) { - owner, dir := newTestOwner(t) - target := filepath.Join(t.TempDir(), "target.json") - require.NoError(t, os.WriteFile(target, []byte(`{}`), 0o600)) - require.NoError(t, os.Symlink(target, filepath.Join(dir, controlFileName))) +func TestDeadOwnerNeverFallsBackToLegacyActivity(t *testing.T) { + _, dir := newTestOwner(t) + require.NoError(t, writeRecord(dir, ownerFileName, OwnerRecord{ + Version: LifecycleProtocolVersion, Generation: "dead", PID: 0x7ffffffe, + }, os.Geteuid())) - _, changed, err := owner.PollControl() + _, stop, err := (ActivityWriter{ + Path: filepath.Join(dir, "activity"), LifecycleDir: dir, + }).Start(context.Background()) + if stop != nil { + defer stop() + } - assert.Error(t, err) - assert.False(t, changed) + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) } -func TestNewLifecycleOwnerRejectsSymlinkedDir(t *testing.T) { - base := t.TempDir() - target := filepath.Join(base, "target") - require.NoError(t, os.Mkdir(target, 0o755)) //nolint:gosec // test dir - link := filepath.Join(base, "link") +func TestNewOwnerRejectsSymlinkWithoutChangingTargetPermissions(t *testing.T) { + _, dir := newTestOwner(t) + target := filepath.Join(dir, "private") + require.NoError(t, os.Mkdir(target, 0o700)) + link := filepath.Join(dir, "link") require.NoError(t, os.Symlink(target, link)) _, err := NewLifecycleOwner(link) - assert.Error(t, err) + require.Error(t, err) + info, err := os.Stat(target) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o700), info.Mode().Perm()) } -func TestReadLifecycleTimeoutFollowsControlRecord(t *testing.T) { +func TestConnectRejectsSymlinkedDiscovery(t *testing.T) { _, dir := newTestOwner(t) - _, ok := ReadLifecycleTimeout(dir) - assert.False(t, ok, "no control record yet") + target := filepath.Join(dir, "untrusted") + require.NoError(t, os.WriteFile(target, []byte(`{}`), 0o600)) + require.NoError(t, os.Remove(filepath.Join(dir, ownerFileName))) + require.NoError(t, os.Symlink(target, filepath.Join(dir, ownerFileName))) - mustPublish(t, connectNow(t, dir), control(LifecycleStateSetup, "45s", "")) + _, err := ConnectLifecycle(context.Background(), dir) - timeout, ok := ReadLifecycleTimeout(dir) - assert.True(t, ok) - assert.Equal(t, 45*time.Second, timeout) + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) } -func TestSendReturnsOnlyAfterOwnerAppliesTheRecord(t *testing.T) { - owner, dir := newTestOwner(t) - client := connectNow(t, dir) - sent := make(chan error, 1) - go func() { sent <- client.Send(LifecycleStateSetup, "1m", "") }() - - select { - case err := <-sent: - t.Fatalf("Send returned before the owner applied the record: %v", err) - case <-time.After(150 * time.Millisecond): +func TestManagedWorkCannotFallBackDuringInitialOwnerStartup(t *testing.T) { + _, dir := newTestOwner(t) + activity := filepath.Join(dir, "activity") + _, stop, err := (ActivityWriter{ + Path: activity, LifecycleDir: filepath.Join(dir, "not-started"), RequireOwner: true, + }).Start(context.Background()) + if stop != nil { + defer stop() } - rec := pollNow(t, owner) - assert.Equal(t, LifecycleStateSetup, rec.State) - require.NoError(t, owner.Acknowledge(rec, nil)) - - select { - case err := <-sent: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("Send did not return after the owner acknowledged the record") - } + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) + _, err = os.Stat(activity) + assert.ErrorIs(t, err, os.ErrNotExist, "no untracked activity loop should be started") } -func pollNow(t *testing.T, owner *LifecycleOwner) ControlRecord { +func rawLifecycleConnection(t *testing.T, dir string) net.Conn { t.Helper() - rec, changed, err := owner.PollControl() + conn, err := net.Dial("unix", filepath.Join(dir, socketFileName)) require.NoError(t, err) - require.True(t, changed) - return rec + t.Cleanup(func() { _ = conn.Close() }) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(time.Second))) + return conn } -func TestIdenticalRepeatedRecordsAreEachDeliveredToTheOwner(t *testing.T) { - owner, dir := newTestOwner(t) - client := connectNow(t, dir) - - mustPublish(t, client, control(LifecycleStateReady, "1m", "")) - first := pollNow(t, owner) - mustPublish(t, client, control(LifecycleStateReady, "1m", "")) - second := pollNow(t, owner) - - assert.Greater(t, second.Sequence, first.Sequence) +func requireRejectedConnection(t *testing.T, conn net.Conn) { + t.Helper() + var buf [1]byte + n, err := conn.Read(buf[:]) + require.Error(t, err) + assert.Zero(t, n) + if networkErr, ok := err.(net.Error); ok { + assert.False( + t, + networkErr.Timeout(), + "the owner must reject, not leave an unbounded pending request", + ) + } } -func TestSecondClientRecordIsNeverTakenForTheFirstClientsRecord(t *testing.T) { +func TestSocketRejectsForgedProcessIdentityWithoutDisarming(t *testing.T) { owner, dir := newTestOwner(t) - - mustPublish(t, connectNow(t, dir), control(LifecycleStateReady, "1m", "")) - first := pollNow(t, owner) - mustPublish(t, connectNow(t, dir), control(LifecycleStateReady, "1m", "")) - second := pollNow(t, owner) - - assert.NotEqual(t, first.ClientID, second.ClientID) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + conn := rawLifecycleConnection(t, dir) + require.NoError(t, json.NewEncoder(conn).Encode(ControlRecord{ + Version: LifecycleProtocolVersion, Generation: owner.Generation(), + ClientID: "forged", Sequence: 1, PID: os.Getpid() + 1, + State: LifecycleStateReady, Timeout: "0", + })) + + requireRejectedConnection(t, conn) + assert.True(t, idleForAnHour(monitor).Expired, "a forged peer cannot disable the current timer") + require.NoError(t, connectNow(t, dir).Send(LifecycleStateSetup, "1m", "")) + assert.False(t, idleForAnHour(monitor).Expired, "the authenticated owner must remain usable") } -func TestSendDoesNotAcceptAnotherClientsAcknowledgement(t *testing.T) { +func TestSocketMessageBoundRejectsUnterminatedOversizedFrame(t *testing.T) { owner, dir := newTestOwner(t) - first := connectNow(t, dir) - mustPublish(t, first, control(LifecycleStateReady, "1m", "")) - require.NoError(t, owner.Acknowledge(pollNow(t, owner), nil)) - - second := connectNow(t, dir) - second.ackTimeout = 100 * time.Millisecond - - assert.ErrorIs(t, second.Send(LifecycleStateSetup, "1m", ""), - ErrLifecycleNotAcknowledged) -} - -func TestSendFailsWhenOwnerNeverAcknowledges(t *testing.T) { - _, dir := newTestOwner(t) - client := connectNow(t, dir) - client.ackTimeout = 100 * time.Millisecond - - err := client.Send(LifecycleStateSetup, "1m", "") + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + conn := rawLifecycleConnection(t, dir) + _, err := conn.Write(bytes.Repeat([]byte{'x'}, maxRecordBytes+1)) + require.NoError(t, err) - assert.ErrorIs(t, err, ErrLifecycleNotAcknowledged) + requireRejectedConnection(t, conn) + assert.True(t, idleForAnHour(monitor).Expired) + require.NoError(t, connectNow(t, dir).Send(LifecycleStateSetup, "1m", "")) + assert.False(t, idleForAnHour(monitor).Expired) } -func TestSendSurfacesOwnerRejection(t *testing.T) { +func TestLiveClientBoundRefusesUnprotectedWorkAndRecoversCapacity(t *testing.T) { owner, dir := newTestOwner(t) - client := connectNow(t, dir) - sent := make(chan error, 1) - go func() { sent <- client.Send(LifecycleStateSetup, "soon", "") }() + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + var first *LifecycleClient + for range maxLifecycleClients { + client := connectNow(t, dir) + if first == nil { + first = client + } + require.NoError(t, client.Send(LifecycleStateActivity, "", "")) + } + writer := ActivityWriter{LifecycleDir: dir, RequireOwner: true} + ctx, stop, err := writer.Start(context.Background()) + if stop != nil { + defer stop() + } + assert.Error(t, err, "capacity exhaustion must not silently fall back to legacy tracking") + assert.Nil(t, ctx) + require.NoError(t, first.Close()) + + var regainedStop func() require.Eventually(t, func() bool { - rec, changed, err := owner.PollControl() - if err != nil || !changed { + _, release, err := writer.Start(context.Background()) + if err != nil { return false } - require.NoError(t, owner.Acknowledge(rec, errors.New("invalid lifecycle timeout"))) + regainedStop = release return true - }, 5*time.Second, 10*time.Millisecond) + }, 3*time.Second, 10*time.Millisecond) + defer regainedStop() + assert.False(t, idleForAnHour(monitor).Expired) +} - assert.ErrorContains(t, <-sent, "invalid lifecycle timeout") +func TestClientVerifiesSocketPeerAgainstDiscoveryIdentity(t *testing.T) { + owner, dir := newTestOwner(t) + if os.Getppid() <= 0 { + t.Skip("a distinct live parent is needed for the forged discovery identity") + } + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}}) + require.NoError(t, writeRecord(dir, ownerFileName, OwnerRecord{ + Version: LifecycleProtocolVersion, Generation: owner.Generation(), PID: os.Getppid(), + }, os.Geteuid())) + + _, err := ConnectLifecycle(context.Background(), dir) + + assert.ErrorIs(t, err, ErrLifecycleOwnerIncompatible) } diff --git a/pkg/daemon/agent/lifecycle_server.go b/pkg/daemon/agent/lifecycle_server.go new file mode 100644 index 0000000000..e4fb5202af --- /dev/null +++ b/pkg/daemon/agent/lifecycle_server.go @@ -0,0 +1,232 @@ +package agent + +import ( + "bufio" + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "net" + "strconv" + "sync" + "time" + + "github.com/devsy-org/devsy/pkg/log" + "github.com/tailscale/peercred" +) + +const maxLifecycleClients = 128 + +type lifecycleSession struct { + conn net.Conn + pid int + startTicks string + uid string + clientID string + protectionID string + applied ControlRecord + result AckRecord +} + +// LifecycleOwner keeps live work and applied operations in memory. Discovery +// and policy files are never evidence that a registered writer is healthy. +type LifecycleOwner struct { + euid int + generation string + listener net.Listener + mu sync.Mutex + closed bool + clients map[net.Conn]*lifecycleSession + writeAck func(net.Conn, AckRecord) error +} + +func (o *LifecycleOwner) Close() error { + o.mu.Lock() + if o.closed { + o.mu.Unlock() + return nil + } + o.closed = true + conns := make([]net.Conn, 0, len(o.clients)) + for conn := range o.clients { + conns = append(conns, conn) + } + o.mu.Unlock() + err := o.listener.Close() + for _, conn := range conns { + _ = conn.Close() + } + if errors.Is(err, net.ErrClosed) { + return nil + } + return err +} + +func (o *LifecycleOwner) Serve(ctx context.Context, monitor *LifecycleMonitor) error { + stop := context.AfterFunc(ctx, func() { _ = o.Close() }) + defer stop() + var workers sync.WaitGroup + defer func() { _ = o.Close(); workers.Wait() }() + for { + conn, err := o.listener.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) || ctx.Err() != nil { + return nil + } + return fmt.Errorf("accept lifecycle client: %w", err) + } + session, err := o.addClient(conn) + if err != nil { + _ = conn.Close() + continue + } + workers.Go(func() { o.serveClient(session, monitor) }) + } +} + +func (o *LifecycleOwner) addClient(conn net.Conn) (*lifecycleSession, error) { + creds, err := peercred.Get(conn) + if err != nil { + return nil, err + } + pid, hasPID := creds.PID() + uid, hasUID := creds.UserID() + if !hasPID || !hasUID { + return nil, errors.New("lifecycle peer identity is unavailable") + } + s := &lifecycleSession{ + conn: conn, pid: pid, uid: uid, + startTicks: procStartTicks(pid), protectionID: rand.Text(), + } + o.mu.Lock() + defer o.mu.Unlock() + if o.closed || len(o.clients) >= maxLifecycleClients { + return nil, errors.New("lifecycle owner connection limit reached") + } + o.clients[conn] = s + return s, nil +} + +func (o *LifecycleOwner) removeClient(s *lifecycleSession, monitor *LifecycleMonitor) { + _ = s.conn.Close() + o.mu.Lock() + delete(o.clients, s.conn) + id := s.protectionID + o.mu.Unlock() + monitor.Release(id, time.Now()) +} + +func (o *LifecycleOwner) pruneDeadClients(monitor *LifecycleMonitor, now time.Time) { + o.mu.Lock() + dead := make([]*lifecycleSession, 0) + for _, s := range o.clients { + if !sameProcess(s.pid, s.startTicks) { + dead = append(dead, s) + } + } + for _, s := range dead { + delete(o.clients, s.conn) + monitor.Release(s.protectionID, now) + } + o.mu.Unlock() + for _, s := range dead { + _ = s.conn.Close() + } +} + +func (s *lifecycleSession) validateIdentity(rec ControlRecord) error { + if rec.ClientID == "" || rec.Sequence == 0 || rec.PID != s.pid || + (rec.StartTicks != "" && s.startTicks != "" && rec.StartTicks != s.startTicks) { + return errors.New("lifecycle client identity does not match its connection") + } + return nil +} + +func (o *LifecycleOwner) authorize(s *lifecycleSession, rec ControlRecord) error { + if rec.State == LifecycleStateActivity || rec.State == LifecycleStateRelease { + return nil + } + if s.uid == "0" || s.uid == strconv.Itoa(o.euid) { + return nil + } + return errors.New("lifecycle policy update requires the daemon's user or root") +} + +func (o *LifecycleOwner) validate(s *lifecycleSession, rec ControlRecord) error { + if rec.Version != LifecycleProtocolVersion || rec.Generation != o.generation { + return ErrLifecycleOwnerIncompatible + } + if err := s.validateIdentity(rec); err != nil { + return err + } + if err := o.authorize(s, rec); err != nil { + return err + } + o.mu.Lock() + defer o.mu.Unlock() + if s.clientID != "" && s.clientID != rec.ClientID { + return errors.New("lifecycle connection changed client identity") + } + s.clientID = rec.ClientID + return nil +} + +func (o *LifecycleOwner) serveClient(s *lifecycleSession, monitor *LifecycleMonitor) { + defer o.removeClient(s, monitor) + // An unauthenticated idle socket is not protected work and must not consume + // a connection slot indefinitely. Registered live work has no time lease. + _ = s.conn.SetReadDeadline(time.Now().Add(ackWaitTimeout)) + scanner := bufio.NewScanner(s.conn) + scanner.Buffer(make([]byte, 512), maxRecordBytes) + // Applied results are connection-owned and retained independently of ACKs. + for scanner.Scan() { + var rec ControlRecord + if json.Unmarshal(scanner.Bytes(), &rec) != nil { + return + } + if err := o.validate(s, rec); err != nil { + return + } + _ = s.conn.SetReadDeadline(time.Time{}) + result, err := o.applyOperation(s, rec, monitor) + if err != nil { + return + } + if err := o.writeAck(s.conn, result); err != nil { + log.Debugf("lifecycle ACK delivery failed; retaining applied result for retry: %v", err) + } + } +} + +func (o *LifecycleOwner) applyOperation( + s *lifecycleSession, + rec ControlRecord, + monitor *LifecycleMonitor, +) (AckRecord, error) { + if rec == s.applied { + return s.result, nil + } + if rec.Sequence != s.applied.Sequence+1 { + return AckRecord{}, errors.New("lifecycle sequence is not consecutive") + } + state := rec + state.ClientID = s.protectionID + _, applyErr := monitor.Apply(state, time.Now()) + s.applied = rec + s.result = AckRecord{ + Version: LifecycleProtocolVersion, Generation: o.generation, + ClientID: rec.ClientID, Sequence: rec.Sequence, + } + if applyErr != nil { + s.result.Error = applyErr.Error() + } + return s.result, nil +} + +func writeSocketAck(conn net.Conn, ack AckRecord) error { + if err := conn.SetWriteDeadline(time.Now().Add(ackPollEvery)); err != nil { + return err + } + return json.NewEncoder(conn).Encode(ack) +} diff --git a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx index 7ce6f323c1..d2fffc7fa3 100644 --- a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx @@ -13,7 +13,7 @@ If the option is unset, a machine provider's own default applies. Devsy uses 20 ## How it works -**Kubernetes** runs the Devsy daemon as the main process of the workspace pod. The daemon starts without a timer. The `devsy workspace up` setup suspends the timer while it runs, then arms it, so a slow setup never counts as idle time. When the timeout passes with no activity, the daemon stops its children, logs that the inactivity timeout was reached, and exits normally. The pod ends, and the workspace volume is kept. See [Kubernetes](#kubernetes). +**Kubernetes** runs the Devsy daemon as the main process of the workspace pod. Setup registers a persistent local connection and waits for the daemon to acknowledge suspension before starting protected work. A live setup has no expiring lease. Finishing or losing that setup connection releases its suspension and starts a fresh idle interval. When the timeout passes with no activity, the daemon stops its children, logs that the inactivity timeout was reached, and exits normally. The pod ends, and the workspace volume is kept. See [Kubernetes](#kubernetes). **Other non-machine providers** such as Docker, Podman, Apple, and Microsandbox start a detached daemon in the container that watches for activity. Its exit does not by itself stop the container. For Microsandbox, Devsy also passes the timeout to the microsandbox CLI as `--idle-timeout`. @@ -34,7 +34,7 @@ A workspace is active while any of these hold: CPU use and processes you started yourself do not count. A command such as `nohup make &` in a closed session does not keep the workspace alive. -The daemon samples activity about once a second at most, so a workspace stops up to a few seconds after the timeout passes. For timeouts of a minute or more, sessions record activity every 10 seconds. Shorter timeouts use a proportionally shorter interval. If the daemon cannot read the activity file, it logs an error and does not stop the workspace. The same holds when a session or hook cannot write the activity file: the daemon logs an error and does not stop the workspace until recording recovers. +The daemon checks idle time about once a second at most, so a workspace stops up to a few seconds after the timeout passes. On Kubernetes, SSH sessions and hooks register with the managed daemon before doing work and remain active for their connection's lifetime. Registration and shutdown decisions stay in memory: failed activity-file writes or unavailable disk space cannot hide registered live work. Losing the daemon connection aborts the protected work rather than continuing without activity tracking. Other container daemons still use legacy activity-file timestamps; unreadable activity fails safe. ## Kubernetes @@ -55,11 +55,11 @@ Pod layout rules Devsy enforces: - With a positive timeout, the Devsy daemon must be the container's main process. A `devcontainer.json` with `overrideCommand: false` and an image entrypoint or command is rejected. - Dockerless builds do not support a positive timeout. - -The layout rules have unit coverage against a fake Kubernetes API. A kind cluster running Kubernetes v1.36.4 on Linux arm64 has also verified a single regular `devsy` container: two idle-stop/resume cycles preserved workspace volume data, and an open tracked SSH session prevented shutdown until disconnect. Native sidecars, additional init containers, restricted non-root pod security, Devsy Platform routes, and upgrades from an earlier Devsy version have not been verified on a real cluster. Test your layout on a non-production workspace before relying on the timeout. + +The enforced restrictions above do not establish end-to-end support for every pod layout. Restricted non-root security, native sidecars/additional init containers, admission-webhook layouts, Devsy Platform routes, and upgrades from an earlier Devsy version require separate cluster validation. Test your layout on a non-production workspace before relying on the timeout. -Pods created by an earlier Devsy version have a main process that cannot enforce the timeout. Devsy detects this and `devsy workspace up` on such a pod fails with an error that says to stop and start the workspace once. Run `devsy workspace stop`, then `devsy workspace up`. Replacing the agent binary does not update a daemon that is already running. +Existing pods whose running daemon uses an older lifecycle protocol must be replaced once. `devsy workspace up` reports the incompatible owner and asks you to stop and start the workspace. Run `devsy workspace stop`, then `devsy workspace up`. The persistent volume is retained; replacing the agent binary alone does not upgrade the daemon process that already owns the pod. ## Shutdown action diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx index a5e261b486..0064d13659 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx @@ -46,7 +46,10 @@ Devsy tracks whether anyone is connected and stops the environment after the tim Set the timeout with `containerInactivityTimeout`. It is a Go duration such as `10m`; empty or `0` disables it. -On the `kubernetes` driver, the Devsy daemon is the pod's main process. It exits normally when the workspace has been idle for the timeout, which ends the pod while the volume is kept. Other container drivers start a detached daemon whose exit does not by itself stop the container. See [Auto-Inactivity Timeout](../developing-in-workspaces/inactivity-timeout.mdx#kubernetes) for activity rules and supported pod layouts. +On the `kubernetes` driver, the Devsy daemon is the pod's main process. SSH, hooks and setup obtain acknowledged protection over a persistent Unix socket before doing work. Setup suspension belongs to that connection, so a dead setup process cannot leave the timer suspended indefinitely; loss of the owner connection aborts the client work. The daemon exits normally when the workspace has been idle for the timeout, ending the pod while keeping its volume. Other container drivers start a detached daemon whose exit does not by itself stop the container. See [Auto-Inactivity Timeout](../developing-in-workspaces/inactivity-timeout.mdx#kubernetes) for activity rules and pod-layout restrictions. + +The managed activity socket is container-local, not host remote-access IPC. Its `0666` transport permissions let non-root workspace processes register activity under a root daemon; the owner checks kernel peer UID/PID and process-start identity, and only root or the daemon's own UID can change setup or timeout policy. Any local container user can keep a live activity registration, so this is not an authorization boundary against untrusted code already running in that container. Do not expose the socket outside the container. Host remote-access owner IPC retains its separate `0600` policy, and workspace authorization remains context plus workspace UID at the existing tunnel boundary. + ### Machine providers From e9162377c1534c294e2312578e146e89eee248ab Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 26 Oct 2026 04:26:28 -0600 Subject: [PATCH 07/19] test: cover nonroot lifecycle client capacity release --- pkg/daemon/agent/lifecycle_peer_linux_test.go | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/pkg/daemon/agent/lifecycle_peer_linux_test.go b/pkg/daemon/agent/lifecycle_peer_linux_test.go index 28e045b242..78fc486f5c 100644 --- a/pkg/daemon/agent/lifecycle_peer_linux_test.go +++ b/pkg/daemon/agent/lifecycle_peer_linux_test.go @@ -97,10 +97,12 @@ func TestRootOwnerAllowsNonrootActivityButRejectsControl(t *testing.T) { "actual nonroot process holds activity under a root owner", ) - _, err = stdin.Write([]byte("control\n")) - require.NoError(t, err) - require.Equal(t, "rejected", readPeerLine(t, scanner)) - require.NoError(t, cmd.Wait()) + assertNonrootControlReleasesCapacity(t, dir, func() { + _, err = stdin.Write([]byte("control\n")) + require.NoError(t, err) + require.Equal(t, "rejected", readPeerLine(t, scanner)) + require.NoError(t, cmd.Wait()) + }) require.Eventually( t, @@ -110,3 +112,24 @@ func TestRootOwnerAllowsNonrootActivityButRejectsControl(t *testing.T) { "nonowner control cannot disable the timer or leave a permanent hold", ) } + +func assertNonrootControlReleasesCapacity(t *testing.T, dir string, reject func()) { + t.Helper() + for range maxLifecycleClients - 1 { + client := connectNow(t, dir) + require.NoError(t, client.Send(LifecycleStateActivity, "", "")) + defer func() { _ = client.Close() }() + } + writer := ActivityWriter{LifecycleDir: dir, RequireOwner: true} + _, _, err := writer.Start(context.Background()) + require.Error(t, err, "a full owner must refuse untracked work") + reject() + require.Eventually(t, func() bool { + _, stop, err := writer.Start(context.Background()) + if err != nil { + return false + } + stop() + return true + }, time.Second, time.Millisecond, "nonroot control rejection must return client capacity") +} From 6e5545dede84c602f33ce09aa654500099e7161d Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 26 Oct 2026 07:59:26 -0600 Subject: [PATCH 08/19] fix: bound lifecycle activity without starving managed control --- cmd/internal/agentcontainer/daemon.go | 48 +--- cmd/internal/agentcontainer/lifecycle_test.go | 205 ++++++++++++++++ cmd/internal/agentcontainer/ssh_server.go | 47 ++-- cmd/internal/ssh_activity_linux_test.go | 65 +++++ pkg/daemon/agent/activity.go | 11 +- pkg/daemon/agent/container_config.go | 40 +++ .../agent/lifecycle_admission_linux_test.go | 231 ++++++++++++++++++ pkg/daemon/agent/lifecycle_deadline_test.go | 145 +++++++++++ .../agent/lifecycle_identity_linux_test.go | 176 +++++++++++++ pkg/daemon/agent/lifecycle_peer_linux_test.go | 2 +- pkg/daemon/agent/lifecycle_proc_linux.go | 17 ++ pkg/daemon/agent/lifecycle_proc_other.go | 6 + pkg/daemon/agent/lifecycle_protocol.go | 14 +- pkg/daemon/agent/lifecycle_protocol_test.go | 35 ++- pkg/daemon/agent/lifecycle_server.go | 146 +++++++++-- .../inactivity-timeout.mdx | 5 + .../docs/developing-providers/agent.mdx | 6 +- 17 files changed, 1107 insertions(+), 92 deletions(-) create mode 100644 cmd/internal/ssh_activity_linux_test.go create mode 100644 pkg/daemon/agent/container_config.go create mode 100644 pkg/daemon/agent/lifecycle_admission_linux_test.go create mode 100644 pkg/daemon/agent/lifecycle_deadline_test.go create mode 100644 pkg/daemon/agent/lifecycle_identity_linux_test.go diff --git a/cmd/internal/agentcontainer/daemon.go b/cmd/internal/agentcontainer/daemon.go index 25679a8b66..6f975bd0e6 100644 --- a/cmd/internal/agentcontainer/daemon.go +++ b/cmd/internal/agentcontainer/daemon.go @@ -2,14 +2,11 @@ package agentcontainer import ( "context" - "encoding/base64" - "encoding/json" "errors" "fmt" "os" "os/exec" "os/signal" - "strings" "syscall" "time" @@ -26,10 +23,7 @@ import ( "golang.org/x/sync/errgroup" ) -const ( - RootDir = config2.ContainerDataDir - DaemonConfigPath = "/var/run/secrets/" + config2.BinaryName + "/daemon_config" -) +const RootDir = config2.ContainerDataDir type DaemonCmd struct { Config *agentd.DaemonConfig @@ -207,38 +201,20 @@ func (cmd *DaemonCmd) newLifecycleLoop(timeout time.Duration) (*agentd.Lifecycle // loadConfig loads the daemon configuration from base64-encoded JSON. // If a CLI-provided timeout exists, it will override the timeout in the config. func (cmd *DaemonCmd) loadConfig() error { - // check local file - encodedCfg := "" - configBytes, err := os.ReadFile(DaemonConfigPath) + cfg, err := agentd.LoadContainerDaemonConfig() if err != nil { - if errors.Is(err, os.ErrNotExist) { - // check environment variable - encodedCfg = os.Getenv(config2.EnvWorkspaceDaemonConfig) - } else { - return fmt.Errorf("get daemon config file %s: %w", DaemonConfigPath, err) - } - } else { - encodedCfg = string(configBytes) + return err } - - if strings.TrimSpace(encodedCfg) != "" { - decoded, err := base64.StdEncoding.DecodeString(encodedCfg) - if err != nil { - return fmt.Errorf("error decoding daemon config: %w", err) - } - var cfg agentd.DaemonConfig - if err = json.Unmarshal(decoded, &cfg); err != nil { - return fmt.Errorf("error unmarshalling daemon config: %w", err) - } - if cmd.Config.Timeout != "" { - cfg.Timeout = cmd.Config.Timeout - } - if cmd.Config.ShutdownAction != "" { - cfg.ShutdownAction = cmd.Config.ShutdownAction - } - cmd.Config = &cfg + if cfg == nil { + return nil } - + if cmd.Config.Timeout != "" { + cfg.Timeout = cmd.Config.Timeout + } + if cmd.Config.ShutdownAction != "" { + cfg.ShutdownAction = cmd.Config.ShutdownAction + } + cmd.Config = cfg return nil } diff --git a/cmd/internal/agentcontainer/lifecycle_test.go b/cmd/internal/agentcontainer/lifecycle_test.go index 50773a1e0b..4b90efd7cd 100644 --- a/cmd/internal/agentcontainer/lifecycle_test.go +++ b/cmd/internal/agentcontainer/lifecycle_test.go @@ -4,10 +4,13 @@ package agentcontainer import ( "context" + "crypto/ed25519" + "crypto/rand" "errors" "io" "net" "os" + "os/exec" "path/filepath" "sync/atomic" "testing" @@ -16,8 +19,11 @@ import ( agentd "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/devcontainer/config" provider2 "github.com/devsy-org/devsy/pkg/provider" + devsyssh "github.com/devsy-org/ssh" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + gossh "golang.org/x/crypto/ssh" ) const testRoot = "root" @@ -295,3 +301,202 @@ func TestTrackingLossClosesTheSSHConnection(t *testing.T) { _, err = client.Read(buf[:]) assert.ErrorIs(t, err, io.EOF, "SSH must not remain open without lifecycle tracking") } + +func TestSSHServerRejectsOnlyUnregisteredConnectionAndKeepsServing(t *testing.T) { + var attempts atomic.Int32 + server, listener, done := newTrackedTestSSHServer(t, func() (context.Context, func(), error) { + if attempts.Add(1) == 2 { + return nil, nil, errors.New("activity capacity unavailable") + } + ctx, cancel := context.WithCancel(context.Background()) + return ctx, cancel, nil + }) + address := listener.Addr().String() + key := server.HostSigners[0].PublicKey() + existing, err := dialTrackedTestSSH(address, key) + require.NoError(t, err) + defer func() { _ = existing.Close() }() + + denied, err := dialTrackedTestSSH(address, key) + if denied != nil { + _ = denied.Close() + } + require.Error(t, err, "a connection without registered activity must not reach SSH") + assertTrackedSSHCommand(t, existing) + later, err := dialTrackedTestSSH(address, key) + require.NoError(t, err, "one rejected admission must not terminate SSH serving") + defer func() { _ = later.Close() }() + assertTrackedSSHCommand(t, later) + + require.NoError(t, listener.Close()) + select { + case err := <-done: + assert.ErrorIs(t, err, net.ErrClosed) + case <-time.After(time.Second): + t.Fatal("SSH serving did not terminate after the underlying listener closed") + } +} + +func newTrackedTestSSHServer( + t *testing.T, + start func() (context.Context, func(), error), +) (*devsyssh.Server, net.Listener, <-chan error) { + t.Helper() + _, privateKey, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + signer, err := gossh.NewSignerFromKey(privateKey) + require.NoError(t, err) + server := &devsyssh.Server{Handler: func(session devsyssh.Session) { + _, _ = io.WriteString(session, "registered\n") + }} + server.AddHostKey(signer) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = server.Close(); _ = listener.Close() }) + done := make(chan error, 1) + go func() { done <- server.Serve(trackConnections(listener, start)) }() + return server, listener, done +} + +func dialTrackedTestSSH(address string, key gossh.PublicKey) (*gossh.Client, error) { + conn, err := net.DialTimeout("tcp", address, time.Second) + if err != nil { + return nil, err + } + if err := conn.SetDeadline(time.Now().Add(time.Second)); err != nil { + _ = conn.Close() + return nil, err + } + client, channels, requests, err := gossh.NewClientConn(conn, address, &gossh.ClientConfig{ + User: "workspace", HostKeyCallback: gossh.FixedHostKey(key), + }) + if err != nil { + _ = conn.Close() + return nil, err + } + _ = conn.SetDeadline(time.Time{}) + return gossh.NewClient(client, channels, requests), nil +} + +func assertTrackedSSHCommand(t *testing.T, client *gossh.Client) { + t.Helper() + session, err := client.NewSession() + require.NoError(t, err) + defer func() { _ = session.Close() }() + out, err := session.Output("check") + require.NoError(t, err) + assert.Equal(t, "registered\n", string(out)) +} + +func TestEmbeddedSSHActivitySaturationPreservesControlAndExistingSessions(t *testing.T) { + owner, dir := publishOwner(t) + runLifecycleOwner(t, owner, dir) + writer := agentd.ActivityWriter{LifecycleDir: dir, RequireOwner: true} + admissions := make(chan error, 1) + server, listener, done := newTrackedTestSSHServer(t, func() (context.Context, func(), error) { + ctx, stop, err := writer.Start(context.Background()) + admissions <- err + return ctx, stop, err + }) + address, key := listener.Addr().String(), server.HostSigners[0].PublicKey() + existing, err := dialTrackedTestSSH(address, key) + require.NoError(t, err) + defer func() { _ = existing.Close() }() + require.NoError(t, nextSSHAdmission(t, admissions)) + raw := saturateEmbeddedSSHActivity(t, address, admissions) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + control, err := agentd.ConnectLifecycle(ctx, dir) + require.NoError( + t, + err, + "owner-UID network activity must leave actual control admission available", + ) + defer func() { _ = control.Close() }() + require.NoError(t, control.Send(agentd.LifecycleStateSetup, "1m", "")) + require.NoError(t, control.Send(agentd.LifecycleStateReady, "1m", "")) + assertTrackedSSHCommand(t, existing) + require.NoError(t, raw[0].Close()) + var later *gossh.Client + require.Eventually(t, func() bool { + client, err := dialTrackedTestSSH(address, key) + admission := nextSSHAdmission(t, admissions) + if err != nil { + return false + } + assert.NoError(t, admission) + later = client + return true + }, 3*time.Second, 10*time.Millisecond) + defer func() { _ = later.Close() }() + assertTrackedSSHCommand(t, later) + require.NoError(t, listener.Close()) + select { + case err := <-done: + assert.ErrorIs(t, err, net.ErrClosed) + case <-time.After(time.Second): + t.Fatal("a real listener close must remain terminal") + } +} + +func nextSSHAdmission(t *testing.T, admissions <-chan error) error { + t.Helper() + select { + case err := <-admissions: + return err + case <-time.After(time.Second): + t.Fatal("SSH stopped accepting instead of reporting activity admission") + return nil + } +} + +func saturateEmbeddedSSHActivity(t *testing.T, address string, admissions <-chan error) []net.Conn { + t.Helper() + var accepted []net.Conn + t.Cleanup(func() { + for _, conn := range accepted { + _ = conn.Close() + } + }) + for range 129 { + conn, err := net.DialTimeout("tcp", address, time.Second) + require.NoError(t, err) + if nextSSHAdmission(t, admissions) == nil { + accepted = append(accepted, conn) + continue + } + defer func() { _ = conn.Close() }() + require.NoError(t, conn.SetReadDeadline(time.Now().Add(time.Second))) + var buf [1]byte + _, err = conn.Read(buf[:]) + require.ErrorIs(t, err, io.EOF, "unregistered TCP must not reach the SSH handshake") + return accepted + } + t.Fatal("raw TCP activity exceeded the hard lifecycle resource bound") + return nil +} + +func TestManagedSSHServerCancellationRemainsTerminal(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + // #nosec G204 -- re-execute the current test binary with fixed test selection. + cmd := exec.CommandContext(ctx, binary, "-test.run=^TestManagedSSHServerContextChild$") + cmd.Env = append(os.Environ(), "DEVSY_MANAGED_SSH_CONTEXT_CHILD=1") + output, err := cmd.CombinedOutput() + require.NoError(t, err, "managed listener did not terminate on cancellation: %s", output) +} + +func TestManagedSSHServerContextChild(t *testing.T) { + if os.Getenv("DEVSY_MANAGED_SSH_CONTEXT_CHILD") == "" { + return + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + command := &cobra.Command{} + command.SetContext(ctx) + cmd := &SSHServerCmd{Address: "127.0.0.1:0", TrackActivity: true} + assert.ErrorIs(t, cmd.Run(command, nil), context.Canceled) +} diff --git a/cmd/internal/agentcontainer/ssh_server.go b/cmd/internal/agentcontainer/ssh_server.go index f4a34accbe..2a8f875f0f 100644 --- a/cmd/internal/agentcontainer/ssh_server.go +++ b/cmd/internal/agentcontainer/ssh_server.go @@ -72,7 +72,7 @@ func NewSSHServerCmd(flags *flags.GlobalFlags) *cobra.Command { } // Run runs the command logic. -func (cmd *SSHServerCmd) Run(_ *cobra.Command, _ []string) error { +func (cmd *SSHServerCmd) Run(c *cobra.Command, _ []string) error { server, err := sshserver.NewContainerServer(cmd.Address, cmd.Workdir) if err != nil { return err @@ -93,18 +93,28 @@ func (cmd *SSHServerCmd) Run(_ *cobra.Command, _ []string) error { return server.ListenAndServe() } + ctx := c.Context() listener, err := net.Listen("tcp", cmd.Address) if err != nil { return fmt.Errorf("listen on %s: %w", cmd.Address, err) } - return server.Serve(trackConnections(listener, startConnectionActivity)) + stopClosing := context.AfterFunc(ctx, func() { _ = listener.Close() }) + defer stopClosing() + defer func() { _ = listener.Close() }() + err = server.Serve(trackConnections(listener, func() (context.Context, func(), error) { + return startConnectionActivity(ctx) + })) + if cause := context.Cause(ctx); cause != nil { + return cause + } + return err } -func startConnectionActivity() (context.Context, func(), error) { +func startConnectionActivity(ctx context.Context) (context.Context, func(), error) { return (agentd.ActivityWriter{ Path: pkgconfig.ContainerActivityFile, LifecycleDir: pkgconfig.ContainerLifecycleDir, RequireOwner: true, - }).Start(context.Background()) + }).Start(ctx) } // trackConnections records activity from the moment a connection is accepted @@ -119,18 +129,25 @@ type activityListener struct { } func (l *activityListener) Accept() (net.Conn, error) { - conn, err := l.Listener.Accept() - if err != nil { - return nil, err - } - ctx, stop, err := l.start() - if err != nil { - _ = conn.Close() - return nil, fmt.Errorf("register SSH connection activity: %w", err) + reportedFailure := false + for { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + ctx, stop, err := l.start() + if err != nil { + _ = conn.Close() + if !reportedFailure { + log.Debugf("reject SSH connection without activity registration: %v", err) + reportedFailure = true + } + continue + } + tracked := &activityConn{Conn: conn, stop: stop} + go func() { <-ctx.Done(); _ = tracked.Close() }() + return tracked, nil } - tracked := &activityConn{Conn: conn, stop: stop} - go func() { <-ctx.Done(); _ = tracked.Close() }() - return tracked, nil } type activityConn struct { diff --git a/cmd/internal/ssh_activity_linux_test.go b/cmd/internal/ssh_activity_linux_test.go new file mode 100644 index 0000000000..0a963af3fb --- /dev/null +++ b/cmd/internal/ssh_activity_linux_test.go @@ -0,0 +1,65 @@ +//go:build linux + +package cmdinternal + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTrackedStdioUsesMountedManagedConfigWithoutEnvironmentOrDiscovery(t *testing.T) { + if os.Getenv("DEVSY_MOUNTED_SECRET_TEST") == "" { + t.Skip("requires the isolated root container with a private secret mount") + } + require.Zero(t, os.Geteuid()) + t.Setenv(config.EnvWorkspaceDaemonConfig, "") + encoded, err := agentd.EncodeDaemonConfig(&agentd.DaemonConfig{ + Timeout: "1m", Lifecycle: agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon}, + }) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(agentd.ContainerDaemonConfigPath), 0o755)) + // #nosec G306 -- mirror the readable, root-owned Kubernetes projected secret. + require.NoError(t, os.WriteFile(agentd.ContainerDaemonConfigPath, []byte(encoded), 0o644)) + binary, err := os.Executable() + require.NoError(t, err) + // #nosec G204 -- re-execute the current test binary with fixed test selection. + cmd := exec.Command(binary, "-test.run=^TestMountedManagedConfigStdioChild$") + cmd.Env = append(os.Environ(), "DEVSY_MOUNTED_SECRET_CHILD=1") + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: 65534, Gid: 65534, Groups: []uint32{65534}}, + Pdeathsig: syscall.SIGKILL, + } + output, err := cmd.CombinedOutput() + t.Logf("nonroot stdio caller: %s", output) + require.NoError(t, err) +} + +func TestMountedManagedConfigStdioChild(t *testing.T) { + if os.Getenv("DEVSY_MOUNTED_SECRET_CHILD") == "" { + return + } + require.Equal(t, 65534, os.Geteuid()) + require.Empty(t, os.Getenv(config.EnvWorkspaceDaemonConfig)) + _, err := os.Lstat(config.ContainerLifecycleDir) + require.ErrorIs(t, err, os.ErrNotExist) + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + cmd := &sshServerCmd{stdio: true, trackActivity: true, address: "127.0.0.1:0", workdir: "/tmp"} + err = cmd.run(ctx) + assert.ErrorIs( + t, + err, + agentd.ErrLifecycleOwnerMissing, + "mounted managed configuration must refuse untracked stdio before SSH work, even without discovery", + ) +} diff --git a/pkg/daemon/agent/activity.go b/pkg/daemon/agent/activity.go index 9aae639f48..78fcfdeafd 100644 --- a/pkg/daemon/agent/activity.go +++ b/pkg/daemon/agent/activity.go @@ -175,14 +175,11 @@ func StartActivityWriter( ctx context.Context, interval time.Duration, ) (context.Context, func(), error) { - required := false - if encoded := os.Getenv(config.EnvWorkspaceDaemonConfig); encoded != "" { - daemon, err := DecodeDaemonConfig(encoded) - if err != nil { - return nil, nil, err - } - required = daemon.Lifecycle.Managed() + daemon, err := LoadContainerDaemonConfig() + if err != nil { + return nil, nil, err } + required := daemon != nil && daemon.Lifecycle.Managed() return (ActivityWriter{ Path: config.ContainerActivityFile, LifecycleDir: config.ContainerLifecycleDir, RequireOwner: required, Interval: interval, diff --git a/pkg/daemon/agent/container_config.go b/pkg/daemon/agent/container_config.go new file mode 100644 index 0000000000..158de82a23 --- /dev/null +++ b/pkg/daemon/agent/container_config.go @@ -0,0 +1,40 @@ +package agent + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/devsy-org/devsy/pkg/config" +) + +const ContainerDaemonConfigPath = "/var/run/secrets/" + config.BinaryName + "/daemon_config" + +func LoadContainerDaemonConfig() (*DaemonConfig, error) { + encoded, err := readContainerDaemonConfig(ContainerDaemonConfigPath) + if err != nil { + return nil, err + } + if strings.TrimSpace(encoded) == "" { + return nil, nil + } + return DecodeDaemonConfig(encoded) +} + +func readContainerDaemonConfig(path string) (string, error) { + // Projected Kubernetes secrets use symlinks and take precedence over the + // process environment, matching the daemon's existing config source. + raw, err := os.ReadFile(path) // #nosec G304 -- fixed secret path or isolated test fixture + if errors.Is(err, os.ErrNotExist) { + return os.Getenv(config.EnvWorkspaceDaemonConfig), nil + } + if err != nil { + return "", fmt.Errorf("read container daemon config: %w", err) + } + encoded := string(raw) + if strings.TrimSpace(encoded) == "" { + return "", errors.New("mounted container daemon config is empty") + } + return encoded, nil +} diff --git a/pkg/daemon/agent/lifecycle_admission_linux_test.go b/pkg/daemon/agent/lifecycle_admission_linux_test.go new file mode 100644 index 0000000000..8d990ec206 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_admission_linux_test.go @@ -0,0 +1,231 @@ +//go:build linux + +package agent + +import ( + "bufio" + "context" + "encoding/json" + "io" + "net" + "os" + "os/exec" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type capacityPeerReport struct { + lifecyclePeerIdentity + Live int `json:"live"` +} + +type capacityPeer struct { + cmd *exec.Cmd + input io.WriteCloser + scanner *bufio.Scanner + report capacityPeerReport +} + +func startCapacityPeer(t *testing.T, dir string, uid uint32, mode string) *capacityPeer { + t.Helper() + binary, err := os.Executable() + require.NoError(t, err) + // #nosec G204 -- re-execute the current test binary with fixed test selection. + cmd := exec.Command(binary, "-test.run=^TestLifecycleCapacityPeerChild$") + cmd.Env = append(os.Environ(), "DEVSY_CAPACITY_DIR="+dir, "DEVSY_CAPACITY_MODE="+mode) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: uid, Gid: uid, Groups: []uint32{uid}}, + Pdeathsig: syscall.SIGKILL, + } + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + input, err := cmd.StdinPipe() + require.NoError(t, err) + cmd.Stderr = os.Stderr + require.NoError(t, cmd.Start()) + peer := &capacityPeer{cmd: cmd, input: input, scanner: bufio.NewScanner(stdout)} + t.Cleanup(func() { _ = peer.input.Close(); _ = cmd.Process.Kill(); _ = cmd.Wait() }) + require.NoError(t, json.Unmarshal([]byte(readPeerLine(t, peer.scanner)), &peer.report)) + require.Equal(t, int(uid), peer.report.UID) + require.Equal(t, int(uid), peer.report.GID) + require.Equal(t, []int{int(uid)}, peer.report.Groups) + require.Equal(t, cmd.Process.Pid, peer.report.PID) + return peer +} + +func (p *capacityPeer) close(t *testing.T) { + t.Helper() + _, err := io.WriteString(p.input, "exit\n") + require.NoError(t, err) + require.NoError(t, p.cmd.Wait()) +} + +func TestLowerTrustPendingClientsPreservePIDUIDAndOwnerFairness(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires real cross-UID clients under a root owner") + } + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + pending := startCapacityPeer(t, dir, 65534, "pending") + require.Equal(t, maxLifecyclePIDClients, pending.report.Live) + allowed := startCapacityPeer(t, dir, 65534, "activity") + require.Equal(t, maxLifecyclePIDClients, allowed.report.Live) + denied := startCapacityPeer(t, dir, 65534, "activity") + require.Zero(t, denied.report.Live, "UID exhaustion must not begin unregistered work") + otherUID := startCapacityPeer(t, dir, 65533, "activity") + require.Equal(t, maxLifecyclePIDClients, otherUID.report.Live) + control := connectNow(t, dir) + require.NoError(t, control.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, control.Send(LifecycleStateReady, "1m", "")) + for _, peer := range []*capacityPeer{pending, allowed, denied, otherUID} { + peer.close(t) + } + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + time.Second, + time.Millisecond, + ) +} + +func TestRootAdmissionSurvivesLowerTrustReleaseOnlySaturationAndDeath(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires real cross-UID clients under a root owner") + } + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + var peers []*capacityPeer + for i := range maxOtherLifecycleClients / maxLifecyclePIDClients { + peer := startCapacityPeer(t, dir, uint32(65000+i), "release") + require.Equal(t, maxLifecyclePIDClients, peer.report.Live) + peers = append(peers, peer) + } + denied := startCapacityPeer(t, dir, 64999, "activity") + require.Zero(t, denied.report.Live, "lower-trust saturation must not begin unregistered work") + control := connectNow(t, dir) + require.NoError(t, control.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, control.Send(LifecycleStateReady, "1m", "")) + assert.True(t, idleForAnHour(monitor).Expired, "release-only sockets do not retain work") + require.NoError(t, peers[0].cmd.Process.Kill()) + require.Error(t, peers[0].cmd.Wait()) + owner.pruneDeadClients(monitor, time.Now()) + replacement := startCapacityPeer(t, dir, 64999, "activity") + require.Equal(t, maxLifecyclePIDClients, replacement.report.Live) + for _, peer := range append(peers[1:], denied, replacement) { + peer.close(t) + } + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + time.Second, + time.Millisecond, + ) +} + +func TestLifecycleCapacityPeerChild(t *testing.T) { + mode := os.Getenv("DEVSY_CAPACITY_MODE") + if mode == "" { + return + } + dir := os.Getenv("DEVSY_CAPACITY_DIR") + var clients []*LifecycleClient + var pending []net.Conn + if mode == "pending" { + pending = openPendingCapacityConnections(t, dir) + } else { + clients = openRegisteredCapacityConnections(dir, mode) + } + defer func() { + for _, conn := range pending { + _ = conn.Close() + } + for _, client := range clients { + _ = client.Close() + } + }() + groups, err := os.Getgroups() + require.NoError(t, err) + report := capacityPeerReport{lifecyclePeerIdentity: lifecyclePeerIdentity{ + UID: os.Getuid(), GID: os.Getgid(), Groups: groups, PID: os.Getpid(), + }, Live: len(clients)} + if mode == "pending" { + report.Live = countOpenPendingConnections(pending) + } + require.NoError(t, json.NewEncoder(os.Stdout).Encode(report)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if mode == "release" { + go chatterReleaseClients(ctx, clients) + } + _, err = bufio.NewReader(os.Stdin).ReadString('\n') + require.NoError(t, err) +} + +func openPendingCapacityConnections(t *testing.T, dir string) []net.Conn { + t.Helper() + var conns []net.Conn + for range maxLifecycleClients { + conn, err := net.DialTimeout("unix", dir+"/"+socketFileName, time.Second) + require.NoError(t, err) + conns = append(conns, conn) + } + return conns +} + +func countOpenPendingConnections(conns []net.Conn) int { + live := 0 + for _, conn := range conns { + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Millisecond)) + var buf [1]byte + _, err := conn.Read(buf[:]) + if timeout, ok := err.(net.Error); ok && timeout.Timeout() { + live++ + } + } + return live +} + +func openRegisteredCapacityConnections(dir, mode string) []*LifecycleClient { + var clients []*LifecycleClient + state := LifecycleStateActivity + if mode == "release" { + state = LifecycleStateRelease + } + for range maxLifecycleClients { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + client, err := ConnectLifecycle(ctx, dir) + cancel() + if err != nil { + break + } + if err := client.Send(state, "", ""); err != nil { + _ = client.Close() + break + } + clients = append(clients, client) + } + return clients +} + +func chatterReleaseClients(ctx context.Context, clients []*LifecycleClient) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + for _, client := range clients { + _ = client.Send(LifecycleStateRelease, "", "") + } + } + } +} diff --git a/pkg/daemon/agent/lifecycle_deadline_test.go b/pkg/daemon/agent/lifecycle_deadline_test.go new file mode 100644 index 0000000000..aa91e5631a --- /dev/null +++ b/pkg/daemon/agent/lifecycle_deadline_test.go @@ -0,0 +1,145 @@ +package agent + +import ( + "encoding/json" + "io" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type lifecycleDeadlineProbe struct { + opened time.Time + closed <-chan time.Time +} + +func TestUnprotectedChatterAndExactRetriesCannotExtendConnectionLifetime(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + fresh := newLifecycleDeadlineProbe(t, owner, dir, "fresh") + retry := newLifecycleDeadlineProbe(t, owner, dir, "retry") + pending := newLifecycleDeadlineProbe(t, owner, dir, "pending") + protected := newLifecycleDeadlineProbe(t, owner, dir, "invalid-ready") + slow := newLifecycleDeadlineProbe(t, owner, dir, "slowloris") + for _, probe := range []*lifecycleDeadlineProbe{fresh, retry, pending, slow} { + select { + case closed := <-probe.closed: + assert.WithinDuration(t, probe.opened.Add(ackWaitTimeout), closed, 2*time.Second) + case <-time.After(ackWaitTimeout + 2*time.Second): + t.Fatal("unprotected connection outlived its original admission deadline") + } + } + select { + case <-protected.closed: + t.Fatal("a failed policy update removed an earlier valid setup hold") + default: + } + assert.False(t, idleForAnHour(monitor).Expired, "live protected work has no connection lease") +} + +func newLifecycleDeadlineProbe( + t *testing.T, + owner *LifecycleOwner, + dir, mode string, +) *lifecycleDeadlineProbe { + t.Helper() + opened := time.Now() + conn, err := net.DialTimeout("unix", filepath.Join(dir, socketFileName), time.Second) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + record := ControlRecord{ + Version: LifecycleProtocolVersion, Generation: owner.generation, + ClientID: mode, Sequence: 1, PID: os.Getpid(), StartTicks: procStartTicks(os.Getpid()), + State: LifecycleStateRelease, + } + if mode != "pending" && mode != "slowloris" { + initializeDeadlineProbe(t, conn, &record, mode) + } + closed := make(chan time.Time, 1) + done := make(chan struct{}) + go func() { _, _ = io.Copy(io.Discard, conn); closed <- time.Now(); close(done) }() + if mode == "fresh" || mode == "retry" { + go chatterLifecycleWire(conn, record, mode == "retry", done) + } + if mode == "slowloris" { + go chatterIncompleteLifecycleFrame(conn, done) + } + return &lifecycleDeadlineProbe{opened: opened, closed: closed} +} + +func initializeDeadlineProbe(t *testing.T, conn net.Conn, record *ControlRecord, mode string) { + t.Helper() + if mode == "invalid-ready" { + record.State, record.Timeout = LifecycleStateSetup, "1m" + } + require.NoError(t, json.NewEncoder(conn).Encode(record)) + var ack AckRecord + require.NoError(t, json.NewDecoder(conn).Decode(&ack)) + require.Empty(t, ack.Error) + if mode != "invalid-ready" { + return + } + record.State, record.Timeout, record.Sequence = LifecycleStateReady, "invalid", 2 + require.NoError(t, json.NewEncoder(conn).Encode(record)) + require.NoError(t, json.NewDecoder(conn).Decode(&ack)) + require.NotEmpty(t, ack.Error, "the policy transition must actually be rejected") +} + +func chatterLifecycleWire(conn net.Conn, record ControlRecord, retry bool, done <-chan struct{}) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + encoder := json.NewEncoder(conn) + for { + select { + case <-done: + return + case <-ticker.C: + if !retry { + record.Sequence++ + } + if err := encoder.Encode(record); err != nil { + return + } + } + } +} + +func TestMalformedLifecycleFrameDoesNotStopTheOwner(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + conn, err := net.DialTimeout("unix", filepath.Join(dir, socketFileName), time.Second) + require.NoError(t, err) + defer func() { _ = conn.Close() }() + _, err = io.WriteString(conn, "{\n") + require.NoError(t, err) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(time.Second))) + var buf [1]byte + _, err = conn.Read(buf[:]) + assert.ErrorIs(t, err, io.EOF) + client := connectNow(t, dir) + require.NoError(t, client.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, client.Send(LifecycleStateReady, "1m", "")) +} + +func chatterIncompleteLifecycleFrame(conn net.Conn, done <-chan struct{}) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-done: + return + case <-ticker.C: + if _, err := io.WriteString(conn, " "); err != nil { + return + } + } + } +} diff --git a/pkg/daemon/agent/lifecycle_identity_linux_test.go b/pkg/daemon/agent/lifecycle_identity_linux_test.go new file mode 100644 index 0000000000..081ffda16d --- /dev/null +++ b/pkg/daemon/agent/lifecycle_identity_linux_test.go @@ -0,0 +1,176 @@ +//go:build linux + +package agent + +import ( + "bufio" + "context" + "encoding/json" + "net" + "os" + "os/exec" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type inheritedPeerReport struct { + PID int `json:"pid"` + HolderPID int `json:"holderPid"` +} + +func TestInheritedSocketCannotRetainActivityAfterOriginalProcessDies(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires an actual unprivileged peer and inherited descriptor") + } + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + binary, err := os.Executable() + require.NoError(t, err) + // #nosec G204 -- re-execute the current test binary with fixed test selection. + cmd := exec.Command(binary, "-test.run=^TestInheritedSocketParentChild$") + cmd.Env = append(os.Environ(), "DEVSY_INHERITED_SOCKET_DIR="+dir) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: 65534, Gid: 65534, Groups: []uint32{65534}}, + Pdeathsig: syscall.SIGKILL, + } + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + input, err := cmd.StdinPipe() + require.NoError(t, err) + defer func() { _ = input.Close() }() + cmd.Stderr = os.Stderr + require.NoError(t, cmd.Start()) + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + var report inheritedPeerReport + require.NoError(t, json.Unmarshal([]byte(readPeerLine(t, bufio.NewScanner(stdout))), &report)) + require.Equal(t, cmd.Process.Pid, report.PID) + t.Cleanup(func() { + _ = syscall.Kill(report.HolderPID, syscall.SIGKILL) + _, _ = syscall.Wait4(report.HolderPID, nil, 0, nil) + }) + assert.False(t, idleForAnHour(monitor).Expired) + require.NoError(t, cmd.Process.Kill()) + require.Error(t, cmd.Wait()) + require.True( + t, + processExists(report.HolderPID), + "the inherited socket holder must remain alive", + ) + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + 2*time.Second, + time.Millisecond, + ) + require.True( + t, + processExists(report.HolderPID), + "EOF from holder exit must not explain release", + ) + assertReclaimedUIDBudget(t, dir) +} + +func assertReclaimedUIDBudget(t *testing.T, dir string) { + t.Helper() + first := startCapacityPeer(t, dir, 65534, "activity") + second := startCapacityPeer(t, dir, 65534, "activity") + third := startCapacityPeer(t, dir, 65534, "activity") + require.Equal(t, maxLifecyclePIDClients, first.report.Live) + require.Equal(t, maxLifecyclePIDClients, second.report.Live) + require.Zero(t, third.report.Live, "prune followed by EOF must not decrement UID usage twice") + first.close(t) + second.close(t) + third.close(t) +} + +func TestInheritedSocketParentChild(t *testing.T) { + dir := os.Getenv("DEVSY_INHERITED_SOCKET_DIR") + if dir == "" { + return + } + client, err := ConnectLifecycle(context.Background(), dir) + require.NoError(t, err) + defer func() { _ = client.Close() }() + require.NoError(t, client.Send(LifecycleStateActivity, "", "")) + file, err := client.conn.(*net.UnixConn).File() + require.NoError(t, err) + defer func() { _ = file.Close() }() + binary, err := os.Executable() + require.NoError(t, err) + // #nosec G204 -- re-execute the current test binary with fixed test selection. + holder := exec.Command(binary, "-test.run=^TestInheritedSocketHolderChild$") + holder.Env = append(os.Environ(), "DEVSY_INHERITED_SOCKET_HOLDER=1") + holder.ExtraFiles = []*os.File{file} + stdout, err := holder.StdoutPipe() + require.NoError(t, err) + holder.Stderr = os.Stderr + require.NoError(t, holder.Start()) + defer func() { _ = holder.Process.Kill(); _ = holder.Wait() }() + require.Equal(t, "holding", readPeerLine(t, bufio.NewScanner(stdout))) + require.NoError( + t, + json.NewEncoder(os.Stdout). + Encode(inheritedPeerReport{PID: os.Getpid(), HolderPID: holder.Process.Pid}), + ) + _, err = bufio.NewReader(os.Stdin).ReadString('\n') + require.NoError(t, err) +} + +func TestInheritedSocketHolderChild(t *testing.T) { + if os.Getenv("DEVSY_INHERITED_SOCKET_HOLDER") == "" { + return + } + file := os.NewFile(3, "inherited-lifecycle-socket") + require.NotNil(t, file) + defer func() { _ = file.Close() }() + info, err := file.Stat() + require.NoError(t, err) + require.Equal(t, os.ModeSocket, info.Mode().Type()) + _, err = os.Stdout.WriteString("holding\n") + require.NoError(t, err) + time.Sleep(time.Minute) +} + +func TestProcessStartIdentityRejectsReusedPID(t *testing.T) { + ticks, err := processStartIdentity(os.Getpid()) + require.NoError(t, err) + require.NotEmpty(t, ticks) + assert.True(t, sameProcess(os.Getpid(), ticks)) + assert.False( + t, + sameProcess(os.Getpid(), ""), + "a missing start identity must not reduce to PID liveness", + ) + assert.False( + t, + sameProcess(os.Getpid(), ticks+"0"), + "a live PID does not validate another process generation", + ) +} + +func TestLinuxLifecycleFailsClosedWithoutReadableProcIdentity(t *testing.T) { + if os.Getenv("DEVSY_MASKED_PROC_TEST") == "" { + t.Skip("run in the isolated container with /proc masked") + } + require.Empty(t, procStartTicks(os.Getpid())) + assert.False(t, sameProcess(os.Getpid(), "previous-process-generation")) + dir, err := os.MkdirTemp("/tmp", "ds-proc-") + require.NoError(t, err) + defer func() { _ = os.RemoveAll(dir) }() + owner, err := NewLifecycleOwner(dir) + if owner != nil { + _ = owner.Close() + } + assert.ErrorIs( + t, + err, + ErrLifecycleOwnerMissing, + "unverifiable process identity must not start an owner", + ) +} diff --git a/pkg/daemon/agent/lifecycle_peer_linux_test.go b/pkg/daemon/agent/lifecycle_peer_linux_test.go index 78fc486f5c..372b048665 100644 --- a/pkg/daemon/agent/lifecycle_peer_linux_test.go +++ b/pkg/daemon/agent/lifecycle_peer_linux_test.go @@ -117,7 +117,7 @@ func assertNonrootControlReleasesCapacity(t *testing.T, dir string, reject func( t.Helper() for range maxLifecycleClients - 1 { client := connectNow(t, dir) - require.NoError(t, client.Send(LifecycleStateActivity, "", "")) + require.NoError(t, client.Send(LifecycleStateSetup, "1m", "")) defer func() { _ = client.Close() }() } writer := ActivityWriter{LifecycleDir: dir, RequireOwner: true} diff --git a/pkg/daemon/agent/lifecycle_proc_linux.go b/pkg/daemon/agent/lifecycle_proc_linux.go index 8e9f187f14..32fb28c068 100644 --- a/pkg/daemon/agent/lifecycle_proc_linux.go +++ b/pkg/daemon/agent/lifecycle_proc_linux.go @@ -3,6 +3,7 @@ package agent import ( + "fmt" "os" "strconv" "strings" @@ -28,3 +29,19 @@ func procStartTicks(pid int) string { } return fields[19] } + +func processStartIdentity(pid int) (string, error) { + ticks := procStartTicks(pid) + if ticks == "" { + return "", fmt.Errorf( + "%w: cannot read process start identity from /proc/%d/stat", + ErrLifecycleOwnerMissing, + pid, + ) + } + return ticks, nil +} + +func sameProcess(pid int, startTicks string) bool { + return startTicks != "" && processExists(pid) && procStartTicks(pid) == startTicks +} diff --git a/pkg/daemon/agent/lifecycle_proc_other.go b/pkg/daemon/agent/lifecycle_proc_other.go index 0026fadae4..60e42344e5 100644 --- a/pkg/daemon/agent/lifecycle_proc_other.go +++ b/pkg/daemon/agent/lifecycle_proc_other.go @@ -3,3 +3,9 @@ package agent func procStartTicks(int) string { return "" } + +func processStartIdentity(int) (string, error) { return "", nil } + +func sameProcess(pid int, startTicks string) bool { + return processExists(pid) && (startTicks == "" || procStartTicks(pid) == startTicks) +} diff --git a/pkg/daemon/agent/lifecycle_protocol.go b/pkg/daemon/agent/lifecycle_protocol.go index af9d922b14..f0705900fb 100644 --- a/pkg/daemon/agent/lifecycle_protocol.go +++ b/pkg/daemon/agent/lifecycle_protocol.go @@ -178,6 +178,10 @@ func (r recordReader) checkFile(f *os.File, name string) error { // decisions and acknowledgements use the connection, never writable files. func NewLifecycleOwner(dir string) (*LifecycleOwner, error) { euid := os.Geteuid() + ticks, err := processStartIdentity(os.Getpid()) + if err != nil { + return nil, err + } if err := prepareRuntimeDir(dir, euid); err != nil { return nil, err } @@ -197,7 +201,7 @@ func NewLifecycleOwner(dir string) (*LifecycleOwner, error) { o.writeAck = writeSocketAck err = writeRecord(dir, ownerFileName, OwnerRecord{ Version: LifecycleProtocolVersion, Generation: o.generation, - PID: os.Getpid(), StartTicks: procStartTicks(os.Getpid()), + PID: os.Getpid(), StartTicks: ticks, }, euid) if err != nil { _ = o.Close() @@ -336,14 +340,6 @@ func liveOwner(dir string, euid int) (*OwnerRecord, error) { func ownerAlive(o OwnerRecord) bool { return o.PID > 0 && sameProcess(o.PID, o.StartTicks) } -func sameProcess(pid int, startTicks string) bool { - if !processExists(pid) { - return false - } - ticks := procStartTicks(pid) - return startTicks == "" || ticks == "" || ticks == startTicks -} - func (c *LifecycleClient) readAcks() { defer func() { _ = c.Close() }() scanner := bufio.NewScanner(c.conn) diff --git a/pkg/daemon/agent/lifecycle_protocol_test.go b/pkg/daemon/agent/lifecycle_protocol_test.go index b749121def..f6c18e9e73 100644 --- a/pkg/daemon/agent/lifecycle_protocol_test.go +++ b/pkg/daemon/agent/lifecycle_protocol_test.go @@ -499,7 +499,7 @@ func TestLiveClientBoundRefusesUnprotectedWorkAndRecoversCapacity(t *testing.T) if first == nil { first = client } - require.NoError(t, client.Send(LifecycleStateActivity, "", "")) + require.NoError(t, client.Send(LifecycleStateSetup, "1m", "")) } writer := ActivityWriter{LifecycleDir: dir, RequireOwner: true} ctx, stop, err := writer.Start(context.Background()) @@ -532,9 +532,42 @@ func TestClientVerifiesSocketPeerAgainstDiscoveryIdentity(t *testing.T) { runLoop(t, &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}}) require.NoError(t, writeRecord(dir, ownerFileName, OwnerRecord{ Version: LifecycleProtocolVersion, Generation: owner.Generation(), PID: os.Getppid(), + StartTicks: procStartTicks(os.Getppid()), }, os.Geteuid())) _, err := ConnectLifecycle(context.Background(), dir) assert.ErrorIs(t, err, ErrLifecycleOwnerIncompatible) } + +func TestOwnerActivityCannotConsumeReservedControlAdmission(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + writer := ActivityWriter{LifecycleDir: dir, RequireOwner: true} + var stops []func() + for range maxOtherLifecycleClients { + _, stop, err := writer.Start(context.Background()) + require.NoError(t, err) + stops = append(stops, stop) + t.Cleanup(stop) + } + _, stop, err := writer.Start(context.Background()) + if stop != nil { + stop() + } + require.Error(t, err, "owner-UID activity must not consume the control reserve") + control := connectNow(t, dir) + require.NoError(t, control.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, control.Send(LifecycleStateReady, "1m", "")) + for _, stop := range stops { + stop() + } + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + time.Second, + time.Millisecond, + ) +} diff --git a/pkg/daemon/agent/lifecycle_server.go b/pkg/daemon/agent/lifecycle_server.go index e4fb5202af..b5bf0e56c0 100644 --- a/pkg/daemon/agent/lifecycle_server.go +++ b/pkg/daemon/agent/lifecycle_server.go @@ -16,13 +16,25 @@ import ( "github.com/tailscale/peercred" ) -const maxLifecycleClients = 128 +const ( + maxLifecycleClients = 128 + reservedLifecycleClients = 32 + maxOtherLifecycleClients = maxLifecycleClients - reservedLifecycleClients + maxLifecycleUIDClients = 32 + maxLifecyclePIDClients = 16 +) + +var errLifecycleClientLimit = errors.New("lifecycle owner connection limit reached") type lifecycleSession struct { conn net.Conn pid int startTicks string uid string + privileged bool + sharedSlot bool + protected bool + readDeadline time.Time clientID string protectionID string applied ControlRecord @@ -32,13 +44,16 @@ type lifecycleSession struct { // LifecycleOwner keeps live work and applied operations in memory. Discovery // and policy files are never evidence that a registered writer is healthy. type LifecycleOwner struct { - euid int - generation string - listener net.Listener - mu sync.Mutex - closed bool - clients map[net.Conn]*lifecycleSession - writeAck func(net.Conn, AckRecord) error + euid int + generation string + listener net.Listener + mu sync.Mutex + closed bool + clients map[net.Conn]*lifecycleSession + otherClients int + clientsByUID map[string]int + clientsByPID map[int]int + writeAck func(net.Conn, AckRecord) error } func (o *LifecycleOwner) Close() error { @@ -92,26 +107,33 @@ func (o *LifecycleOwner) addClient(conn net.Conn) (*lifecycleSession, error) { } pid, hasPID := creds.PID() uid, hasUID := creds.UserID() - if !hasPID || !hasUID { + if !hasPID || !hasUID || pid <= 0 { return nil, errors.New("lifecycle peer identity is unavailable") } - s := &lifecycleSession{ - conn: conn, pid: pid, uid: uid, - startTicks: procStartTicks(pid), protectionID: rand.Text(), - } + privileged := uid == "0" || uid == strconv.Itoa(o.euid) o.mu.Lock() defer o.mu.Unlock() - if o.closed || len(o.clients) >= maxLifecycleClients { - return nil, errors.New("lifecycle owner connection limit reached") + if o.closed || !o.canAdmit(uid, pid, privileged) { + return nil, errLifecycleClientLimit + } + ticks, err := processStartIdentity(pid) + if err != nil { + return nil, err + } + s := &lifecycleSession{ + conn: conn, pid: pid, uid: uid, privileged: privileged, sharedSlot: !privileged, + startTicks: ticks, protectionID: rand.Text(), + readDeadline: time.Now().Add(ackWaitTimeout), } o.clients[conn] = s + o.countClient(s) return s, nil } func (o *LifecycleOwner) removeClient(s *lifecycleSession, monitor *LifecycleMonitor) { _ = s.conn.Close() o.mu.Lock() - delete(o.clients, s.conn) + o.removeClientLocked(s) id := s.protectionID o.mu.Unlock() monitor.Release(id, time.Now()) @@ -126,7 +148,7 @@ func (o *LifecycleOwner) pruneDeadClients(monitor *LifecycleMonitor, now time.Ti } } for _, s := range dead { - delete(o.clients, s.conn) + o.removeClientLocked(s) monitor.Release(s.protectionID, now) } o.mu.Unlock() @@ -147,7 +169,7 @@ func (o *LifecycleOwner) authorize(s *lifecycleSession, rec ControlRecord) error if rec.State == LifecycleStateActivity || rec.State == LifecycleStateRelease { return nil } - if s.uid == "0" || s.uid == strconv.Itoa(o.euid) { + if s.privileged { return nil } return errors.New("lifecycle policy update requires the daemon's user or root") @@ -163,20 +185,27 @@ func (o *LifecycleOwner) validate(s *lifecycleSession, rec ControlRecord) error if err := o.authorize(s, rec); err != nil { return err } + return o.validateSession(s, rec) +} + +func (o *LifecycleOwner) validateSession(s *lifecycleSession, rec ControlRecord) error { o.mu.Lock() defer o.mu.Unlock() + if o.closed || o.clients[s.conn] != s { + return ErrLifecycleOwnerMissing + } if s.clientID != "" && s.clientID != rec.ClientID { return errors.New("lifecycle connection changed client identity") } s.clientID = rec.ClientID - return nil + return o.classifyActivity(s, rec.State) } func (o *LifecycleOwner) serveClient(s *lifecycleSession, monitor *LifecycleMonitor) { defer o.removeClient(s, monitor) // An unauthenticated idle socket is not protected work and must not consume // a connection slot indefinitely. Registered live work has no time lease. - _ = s.conn.SetReadDeadline(time.Now().Add(ackWaitTimeout)) + _ = s.conn.SetReadDeadline(s.readDeadline) scanner := bufio.NewScanner(s.conn) scanner.Buffer(make([]byte, 512), maxRecordBytes) // Applied results are connection-owned and retained independently of ACKs. @@ -188,7 +217,6 @@ func (o *LifecycleOwner) serveClient(s *lifecycleSession, monitor *LifecycleMoni if err := o.validate(s, rec); err != nil { return } - _ = s.conn.SetReadDeadline(time.Time{}) result, err := o.applyOperation(s, rec, monitor) if err != nil { return @@ -212,7 +240,11 @@ func (o *LifecycleOwner) applyOperation( } state := rec state.ClientID = s.protectionID - _, applyErr := monitor.Apply(state, time.Now()) + now := time.Now() + _, applyErr := monitor.Apply(state, now) + if applyErr == nil { + s.transitionProtection(rec.State, now) + } s.applied = rec s.result = AckRecord{ Version: LifecycleProtocolVersion, Generation: o.generation, @@ -230,3 +262,73 @@ func writeSocketAck(conn net.Conn, ack AckRecord) error { } return json.NewEncoder(conn).Encode(ack) } + +func (o *LifecycleOwner) canAdmit(uid string, pid int, privileged bool) bool { + if len(o.clients) >= maxLifecycleClients { + return false + } + return privileged || (o.otherClients < maxOtherLifecycleClients && + o.clientsByUID[uid] < maxLifecycleUIDClients && + o.clientsByPID[pid] < maxLifecyclePIDClients) +} + +func (o *LifecycleOwner) countClient(s *lifecycleSession) { + if s.sharedSlot { + o.otherClients++ + } + if s.privileged { + return + } + if o.clientsByUID == nil { + o.clientsByUID = make(map[string]int) + o.clientsByPID = make(map[int]int) + } + o.clientsByUID[s.uid]++ + o.clientsByPID[s.pid]++ +} + +func (o *LifecycleOwner) removeClientLocked(s *lifecycleSession) { + if _, present := o.clients[s.conn]; !present { + return + } + delete(o.clients, s.conn) + if s.sharedSlot { + o.otherClients-- + } + if s.privileged { + return + } + o.clientsByUID[s.uid]-- + o.clientsByPID[s.pid]-- + if o.clientsByUID[s.uid] == 0 { + delete(o.clientsByUID, s.uid) + } + if o.clientsByPID[s.pid] == 0 { + delete(o.clientsByPID, s.pid) + } +} + +func (s *lifecycleSession) transitionProtection(state LifecycleState, now time.Time) { + protected := state == LifecycleStateSetup || state == LifecycleStateActivity + if protected == s.protected { + return + } + s.protected = protected + s.readDeadline = time.Time{} + if !protected { + s.readDeadline = now.Add(ackWaitTimeout) + } + _ = s.conn.SetReadDeadline(s.readDeadline) +} + +func (o *LifecycleOwner) classifyActivity(s *lifecycleSession, state LifecycleState) error { + if s.sharedSlot || (state != LifecycleStateActivity && state != LifecycleStateRelease) { + return nil + } + if o.otherClients >= maxOtherLifecycleClients { + return errLifecycleClientLimit + } + s.sharedSlot = true + o.otherClients++ + return nil +} diff --git a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx index d2fffc7fa3..2dd7c389eb 100644 --- a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx @@ -36,6 +36,11 @@ CPU use and processes you started yourself do not count. A command such as `nohu The daemon checks idle time about once a second at most, so a workspace stops up to a few seconds after the timeout passes. On Kubernetes, SSH sessions and hooks register with the managed daemon before doing work and remain active for their connection's lifetime. Registration and shutdown decisions stay in memory: failed activity-file writes or unavailable disk space cannot hide registered live work. Losing the daemon connection aborts the protected work rather than continuing without activity tracking. Other container daemons still use legacy activity-file timestamps; unreadable activity fails safe. +Managed clients also read the daemon configuration mounted at `/var/run/secrets/devsy/daemon_config`; they do not rely only on the process environment or runtime directory. Missing discovery in a known managed workspace is an error, not a reason to fall back to file heartbeats. Linux process-start identity must remain readable through `/proc`; otherwise registration fails or the affected protected connection is cancelled. + +All Unix users in the same container intentionally retain the ability to inhibit idle stop through a live activity connection, as they could through the earlier shared heartbeat. This is not inter-user isolation inside a container. Only root or the daemon's own UID can change setup and timeout policy. + + ## Kubernetes Set `INACTIVITY_TIMEOUT` on the Kubernetes provider, for example `1h`. The deprecated `inactivityTimeout` field under `agent.kubernetes` in a custom provider is an alias used only when `containerInactivityTimeout` is empty. If both are set, they must match. diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx index 0064d13659..293302dd66 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx @@ -48,7 +48,11 @@ Set the timeout with `containerInactivityTimeout`. It is a Go duration such as ` On the `kubernetes` driver, the Devsy daemon is the pod's main process. SSH, hooks and setup obtain acknowledged protection over a persistent Unix socket before doing work. Setup suspension belongs to that connection, so a dead setup process cannot leave the timer suspended indefinitely; loss of the owner connection aborts the client work. The daemon exits normally when the workspace has been idle for the timeout, ending the pod while keeping its volume. Other container drivers start a detached daemon whose exit does not by itself stop the container. See [Auto-Inactivity Timeout](../developing-in-workspaces/inactivity-timeout.mdx#kubernetes) for activity rules and pod-layout restrictions. -The managed activity socket is container-local, not host remote-access IPC. Its `0666` transport permissions let non-root workspace processes register activity under a root daemon; the owner checks kernel peer UID/PID and process-start identity, and only root or the daemon's own UID can change setup or timeout policy. Any local container user can keep a live activity registration, so this is not an authorization boundary against untrusted code already running in that container. Do not expose the socket outside the container. Host remote-access owner IPC retains its separate `0600` policy, and workspace authorization remains context plus workspace UID at the existing tunnel boundary. +The managed activity socket is container-local, not host remote-access IPC. Its `0666` permissions intentionally preserve the [existing cross-user heartbeat authority](https://github.com/devsy-org/devsy/blob/fe22f9a1225d6c1547001755e94c95bb2cdbc8db/cmd/internal/agentcontainer/daemon.go): any same-container UID may inhibit inactivity through its own live activity connection. This grants no general policy/control authority. The owner validates kernel peer UID/PID and process-start identity; only root or the daemon's UID can change setup or timeout policy. These checks do not isolate processes sharing a UID or add authentication to the existing SSH bridge. Do not expose the lifecycle socket outside its container. Host remote-access owner IPC retains its separate `0600` policy and context-plus-workspace-UID authorization. + +Admission is bounded to 128 connections. Non-owner pending and registered connections share a 96-slot activity budget, with at most 32 per UID and 16 per PID. Owner-UID activity and release requests also enter that shared budget, so an embedded SSH carrier cannot consume the remaining setup/control reserve. Owner/root pending admission is provisional, not proof of a control role; deliberate abuse by that already-privileged UID is outside this boundary. Unprotected connections have a nonrenewable 10-second deadline, restarted only when protected work ends; release-only chatter and exact ACK retries cannot renew it. Live acknowledged setup and activity have no time lease. A denied activity registration closes only that incoming SSH connection, not the listener or existing sessions. + +Managed-mode detection uses the same mounted daemon configuration as the daemon itself, ahead of the process environment. An unreadable or malformed configuration aborts registration; absent runtime discovery cannot select legacy mode when managed configuration is present. Linux registration and liveness checks fail closed when the process-start identity cannot be established. ### Machine providers From 94e3322356b102cd288ca6b24804d6c4af0dc9b9 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 26 Oct 2026 08:18:59 -0600 Subject: [PATCH 09/19] fix: release lifecycle protection for zombie peers --- .../agent/lifecycle_identity_linux_test.go | 23 ++++++++++++++++++- pkg/daemon/agent/lifecycle_proc_linux.go | 9 +++++--- .../docs/developing-providers/agent.mdx | 2 ++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/pkg/daemon/agent/lifecycle_identity_linux_test.go b/pkg/daemon/agent/lifecycle_identity_linux_test.go index 081ffda16d..0860307b87 100644 --- a/pkg/daemon/agent/lifecycle_identity_linux_test.go +++ b/pkg/daemon/agent/lifecycle_identity_linux_test.go @@ -4,11 +4,13 @@ package agent import ( "bufio" + "bytes" "context" "encoding/json" "net" "os" "os/exec" + "strconv" "syscall" "testing" "time" @@ -56,7 +58,13 @@ func TestInheritedSocketCannotRetainActivityAfterOriginalProcessDies(t *testing. }) assert.False(t, idleForAnHour(monitor).Expired) require.NoError(t, cmd.Process.Kill()) - require.Error(t, cmd.Wait()) + require.Eventually( + t, + func() bool { return isUnreapedZombie(report.PID) }, + time.Second, + time.Millisecond, + "the opener must be dead but still owned by this living parent", + ) require.True( t, processExists(report.HolderPID), @@ -74,6 +82,12 @@ func TestInheritedSocketCannotRetainActivityAfterOriginalProcessDies(t *testing. "EOF from holder exit must not explain release", ) assertReclaimedUIDBudget(t, dir) + require.True( + t, + isUnreapedZombie(report.PID), + "release and capacity recovery must precede reaping", + ) + require.Error(t, cmd.Wait()) } func assertReclaimedUIDBudget(t *testing.T, dir string) { @@ -174,3 +188,10 @@ func TestLinuxLifecycleFailsClosedWithoutReadableProcIdentity(t *testing.T) { "unverifiable process identity must not start an owner", ) } + +func isUnreapedZombie(pid int) bool { + raw, err := os.ReadFile( + "/proc/" + strconv.Itoa(pid) + "/status", + ) // #nosec G304 -- actual test child PID + return err == nil && bytes.Contains(raw, []byte("\nState:\tZ")) +} diff --git a/pkg/daemon/agent/lifecycle_proc_linux.go b/pkg/daemon/agent/lifecycle_proc_linux.go index 32fb28c068..0beee53290 100644 --- a/pkg/daemon/agent/lifecycle_proc_linux.go +++ b/pkg/daemon/agent/lifecycle_proc_linux.go @@ -9,8 +9,7 @@ import ( "strings" ) -// procStartTicks returns field 22 of /proc//stat, which distinguishes a -// process from a later one that reuses its PID. +// procStartTicks returns the start identity of a live process from /proc. func procStartTicks(pid int) string { raw, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat") // #nosec G304 if err != nil { @@ -27,6 +26,10 @@ func procStartTicks(pid int) string { if len(fields) < 20 { return "" } + // Zombies retain both PID and start time until their living parent reaps them. + if fields[0] == "Z" || fields[0] == "X" || fields[0] == "x" { + return "" + } return fields[19] } @@ -34,7 +37,7 @@ func processStartIdentity(pid int) (string, error) { ticks := procStartTicks(pid) if ticks == "" { return "", fmt.Errorf( - "%w: cannot read process start identity from /proc/%d/stat", + "%w: cannot establish live process identity from /proc/%d/stat", ErrLifecycleOwnerMissing, pid, ) diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx index 293302dd66..aecb7a57b0 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx @@ -52,6 +52,8 @@ The managed activity socket is container-local, not host remote-access IPC. Its Admission is bounded to 128 connections. Non-owner pending and registered connections share a 96-slot activity budget, with at most 32 per UID and 16 per PID. Owner-UID activity and release requests also enter that shared budget, so an embedded SSH carrier cannot consume the remaining setup/control reserve. Owner/root pending admission is provisional, not proof of a control role; deliberate abuse by that already-privileged UID is outside this boundary. Unprotected connections have a nonrenewable 10-second deadline, restarted only when protected work ends; release-only chatter and exact ACK retries cannot renew it. Live acknowledged setup and activity have no time lease. A denied activity registration closes only that incoming SSH connection, not the listener or existing sessions. +An exited peer, including an unreaped zombie, loses its registration even if another process inherited its socket. Linux liveness uses process state and start time from the same `/proc` snapshot; it does not rely only on the PID remaining present. + Managed-mode detection uses the same mounted daemon configuration as the daemon itself, ahead of the process environment. An unreadable or malformed configuration aborts registration; absent runtime discovery cannot select legacy mode when managed configuration is present. Linux registration and liveness checks fail closed when the process-start identity cannot be established. From be9ac0022c86ba0ac4ee3a815d42b5268cb7696a Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 26 Oct 2026 09:29:20 -0600 Subject: [PATCH 10/19] style: satisfy lifecycle lint across supported builds --- cmd/internal/ssh_activity_linux_test.go | 1 + .../agent/lifecycle_admission_linux_test.go | 7 ++-- pkg/daemon/agent/lifecycle_deadline_test.go | 6 ++- .../agent/lifecycle_identity_linux_test.go | 12 +++++- pkg/daemon/agent/lifecycle_protocol.go | 40 +++++++++---------- pkg/daemon/agent/lifecycle_server.go | 20 +++++++--- 6 files changed, 54 insertions(+), 32 deletions(-) diff --git a/cmd/internal/ssh_activity_linux_test.go b/cmd/internal/ssh_activity_linux_test.go index 0a963af3fb..3aa3856bfc 100644 --- a/cmd/internal/ssh_activity_linux_test.go +++ b/cmd/internal/ssh_activity_linux_test.go @@ -27,6 +27,7 @@ func TestTrackedStdioUsesMountedManagedConfigWithoutEnvironmentOrDiscovery(t *te Timeout: "1m", Lifecycle: agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon}, }) require.NoError(t, err) + // #nosec G301 -- mirror a Kubernetes secret directory traversable by the nonroot child. require.NoError(t, os.MkdirAll(filepath.Dir(agentd.ContainerDaemonConfigPath), 0o755)) // #nosec G306 -- mirror the readable, root-owned Kubernetes projected secret. require.NoError(t, os.WriteFile(agentd.ContainerDaemonConfigPath, []byte(encoded), 0o644)) diff --git a/pkg/daemon/agent/lifecycle_admission_linux_test.go b/pkg/daemon/agent/lifecycle_admission_linux_test.go index 8d990ec206..fcf5d9059c 100644 --- a/pkg/daemon/agent/lifecycle_admission_linux_test.go +++ b/pkg/daemon/agent/lifecycle_admission_linux_test.go @@ -72,7 +72,7 @@ func TestLowerTrustPendingClientsPreservePIDUIDAndOwnerFairness(t *testing.T) { monitor := &LifecycleMonitor{} monitor.ArmNow(time.Minute, time.Now()) runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) - pending := startCapacityPeer(t, dir, 65534, "pending") + pending := startCapacityPeer(t, dir, 65534, lifecyclePendingMode) require.Equal(t, maxLifecyclePIDClients, pending.report.Live) allowed := startCapacityPeer(t, dir, 65534, "activity") require.Equal(t, maxLifecyclePIDClients, allowed.report.Live) @@ -138,7 +138,7 @@ func TestLifecycleCapacityPeerChild(t *testing.T) { dir := os.Getenv("DEVSY_CAPACITY_DIR") var clients []*LifecycleClient var pending []net.Conn - if mode == "pending" { + if mode == lifecyclePendingMode { pending = openPendingCapacityConnections(t, dir) } else { clients = openRegisteredCapacityConnections(dir, mode) @@ -156,7 +156,7 @@ func TestLifecycleCapacityPeerChild(t *testing.T) { report := capacityPeerReport{lifecyclePeerIdentity: lifecyclePeerIdentity{ UID: os.Getuid(), GID: os.Getgid(), Groups: groups, PID: os.Getpid(), }, Live: len(clients)} - if mode == "pending" { + if mode == lifecyclePendingMode { report.Live = countOpenPendingConnections(pending) } require.NoError(t, json.NewEncoder(os.Stdout).Encode(report)) @@ -173,6 +173,7 @@ func openPendingCapacityConnections(t *testing.T, dir string) []net.Conn { t.Helper() var conns []net.Conn for range maxLifecycleClients { + // #nosec G704 -- Unix IPC in this test's private fixture, not a network destination. conn, err := net.DialTimeout("unix", dir+"/"+socketFileName, time.Second) require.NoError(t, err) conns = append(conns, conn) diff --git a/pkg/daemon/agent/lifecycle_deadline_test.go b/pkg/daemon/agent/lifecycle_deadline_test.go index aa91e5631a..6d883dffae 100644 --- a/pkg/daemon/agent/lifecycle_deadline_test.go +++ b/pkg/daemon/agent/lifecycle_deadline_test.go @@ -13,6 +13,8 @@ import ( "github.com/stretchr/testify/require" ) +const lifecyclePendingMode = "pending" + type lifecycleDeadlineProbe struct { opened time.Time closed <-chan time.Time @@ -25,7 +27,7 @@ func TestUnprotectedChatterAndExactRetriesCannotExtendConnectionLifetime(t *test runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) fresh := newLifecycleDeadlineProbe(t, owner, dir, "fresh") retry := newLifecycleDeadlineProbe(t, owner, dir, "retry") - pending := newLifecycleDeadlineProbe(t, owner, dir, "pending") + pending := newLifecycleDeadlineProbe(t, owner, dir, lifecyclePendingMode) protected := newLifecycleDeadlineProbe(t, owner, dir, "invalid-ready") slow := newLifecycleDeadlineProbe(t, owner, dir, "slowloris") for _, probe := range []*lifecycleDeadlineProbe{fresh, retry, pending, slow} { @@ -59,7 +61,7 @@ func newLifecycleDeadlineProbe( ClientID: mode, Sequence: 1, PID: os.Getpid(), StartTicks: procStartTicks(os.Getpid()), State: LifecycleStateRelease, } - if mode != "pending" && mode != "slowloris" { + if mode != lifecyclePendingMode && mode != "slowloris" { initializeDeadlineProbe(t, conn, &record, mode) } closed := make(chan time.Time, 1) diff --git a/pkg/daemon/agent/lifecycle_identity_linux_test.go b/pkg/daemon/agent/lifecycle_identity_linux_test.go index 0860307b87..c6bf28a86c 100644 --- a/pkg/daemon/agent/lifecycle_identity_linux_test.go +++ b/pkg/daemon/agent/lifecycle_identity_linux_test.go @@ -65,6 +65,17 @@ func TestInheritedSocketCannotRetainActivityAfterOriginalProcessDies(t *testing. time.Millisecond, "the opener must be dead but still owned by this living parent", ) + assertZombieReleaseBeforeReap(t, monitor, dir, report) + require.Error(t, cmd.Wait()) +} + +func assertZombieReleaseBeforeReap( + t *testing.T, + monitor *LifecycleMonitor, + dir string, + report inheritedPeerReport, +) { + t.Helper() require.True( t, processExists(report.HolderPID), @@ -87,7 +98,6 @@ func TestInheritedSocketCannotRetainActivityAfterOriginalProcessDies(t *testing. isUnreapedZombie(report.PID), "release and capacity recovery must precede reaping", ) - require.Error(t, cmd.Wait()) } func assertReclaimedUIDBudget(t *testing.T, dir string) { diff --git a/pkg/daemon/agent/lifecycle_protocol.go b/pkg/daemon/agent/lifecycle_protocol.go index f0705900fb..ba21190e37 100644 --- a/pkg/daemon/agent/lifecycle_protocol.go +++ b/pkg/daemon/agent/lifecycle_protocol.go @@ -340,26 +340,6 @@ func liveOwner(dir string, euid int) (*OwnerRecord, error) { func ownerAlive(o OwnerRecord) bool { return o.PID > 0 && sameProcess(o.PID, o.StartTicks) } -func (c *LifecycleClient) readAcks() { - defer func() { _ = c.Close() }() - scanner := bufio.NewScanner(c.conn) - scanner.Buffer(make([]byte, 512), maxRecordBytes) - for scanner.Scan() { - var ack AckRecord - if json.Unmarshal(scanner.Bytes(), &ack) != nil || - !c.validAck(ack) { - return - } - select { - case c.acks <- ack: - case <-c.done: - return - default: - return - } - } -} - func (c *LifecycleClient) Done() <-chan struct{} { return c.done } func (c *LifecycleClient) Close() error { @@ -409,6 +389,26 @@ func (c *LifecycleClient) Send( return c.sendRecord(rec) } +func (c *LifecycleClient) readAcks() { + defer func() { _ = c.Close() }() + scanner := bufio.NewScanner(c.conn) + scanner.Buffer(make([]byte, 512), maxRecordBytes) + for scanner.Scan() { + var ack AckRecord + if json.Unmarshal(scanner.Bytes(), &ack) != nil || + !c.validAck(ack) { + return + } + select { + case c.acks <- ack: + case <-c.done: + return + default: + return + } + } +} + func (c *LifecycleClient) sendRecord(rec ControlRecord) error { raw, err := json.Marshal(rec) if err != nil { diff --git a/pkg/daemon/agent/lifecycle_server.go b/pkg/daemon/agent/lifecycle_server.go index b5bf0e56c0..2d50f91a6c 100644 --- a/pkg/daemon/agent/lifecycle_server.go +++ b/pkg/daemon/agent/lifecycle_server.go @@ -101,15 +101,10 @@ func (o *LifecycleOwner) Serve(ctx context.Context, monitor *LifecycleMonitor) e } func (o *LifecycleOwner) addClient(conn net.Conn) (*lifecycleSession, error) { - creds, err := peercred.Get(conn) + pid, uid, err := readLifecyclePeer(conn) if err != nil { return nil, err } - pid, hasPID := creds.PID() - uid, hasUID := creds.UserID() - if !hasPID || !hasUID || pid <= 0 { - return nil, errors.New("lifecycle peer identity is unavailable") - } privileged := uid == "0" || uid == strconv.Itoa(o.euid) o.mu.Lock() defer o.mu.Unlock() @@ -130,6 +125,19 @@ func (o *LifecycleOwner) addClient(conn net.Conn) (*lifecycleSession, error) { return s, nil } +func readLifecyclePeer(conn net.Conn) (int, string, error) { + creds, err := peercred.Get(conn) + if err != nil { + return 0, "", err + } + pid, hasPID := creds.PID() + uid, hasUID := creds.UserID() + if !hasPID || !hasUID || pid <= 0 { + return 0, "", errors.New("lifecycle peer identity is unavailable") + } + return pid, uid, nil +} + func (o *LifecycleOwner) removeClient(s *lifecycleSession, monitor *LifecycleMonitor) { _ = s.conn.Close() o.mu.Lock() From 1b5e307966b59ca2c059003463b6e59808df034e Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 26 Oct 2026 23:17:48 -0600 Subject: [PATCH 11/19] fix(kubernetes): preserve lifecycle intent across recreate and resume --- cmd/internal/agentcontainer/lifecycle.go | 6 +- cmd/internal/agentcontainer/lifecycle_test.go | 1 - pkg/daemon/agent/container_lifecycle.go | 41 +- pkg/daemon/agent/container_lifecycle_test.go | 9 +- pkg/devcontainer/config/container_details.go | 3 + .../kubernetes_recreate_preflight_test.go | 296 ++++++++++ pkg/devcontainer/managed_lifecycle_test.go | 273 ++++++++- pkg/devcontainer/setup.go | 25 +- pkg/devcontainer/setup_secrets_test.go | 2 +- pkg/devcontainer/single.go | 159 +++--- pkg/driver/kubernetes/driver.go | 21 +- pkg/driver/kubernetes/find.go | 15 +- pkg/driver/kubernetes/pod_lifecycle.go | 30 +- pkg/driver/kubernetes/pod_lifecycle_test.go | 354 +++++++++++- pkg/driver/kubernetes/pod_topology.go | 165 ++++++ pkg/driver/kubernetes/pvc.go | 78 +++ pkg/driver/kubernetes/pvc_metadata_test.go | 536 ++++++++++++++++++ pkg/driver/kubernetes/pvc_test.go | 96 ++-- pkg/driver/kubernetes/run.go | 73 +-- pkg/driver/types.go | 8 +- .../inactivity-timeout.mdx | 10 + 21 files changed, 1993 insertions(+), 208 deletions(-) create mode 100644 pkg/devcontainer/kubernetes_recreate_preflight_test.go create mode 100644 pkg/driver/kubernetes/pod_topology.go create mode 100644 pkg/driver/kubernetes/pvc_metadata_test.go diff --git a/cmd/internal/agentcontainer/lifecycle.go b/cmd/internal/agentcontainer/lifecycle.go index b2969ef2fa..87d0b1630a 100644 --- a/cmd/internal/agentcontainer/lifecycle.go +++ b/cmd/internal/agentcontainer/lifecycle.go @@ -64,9 +64,9 @@ func beginLifecycle( func migrationError(err error) error { return fmt.Errorf( "this container's main process does not support the Devsy inactivity lifecycle (%w); "+ - "stop and start the workspace once (devsy workspace stop, then devsy up) to replace "+ - "the pod. The workspace volume is kept; data written to the container filesystem "+ - "outside mounted volumes is not", err) + "stop and start the workspace once (devsy workspace stop, then devsy workspace up) "+ + "to replace the pod. The workspace volume is kept; data written to the "+ + "container filesystem outside mounted volumes is not", err) } func (h *lifecycleHandshake) send(state agentd.LifecycleState) error { diff --git a/cmd/internal/agentcontainer/lifecycle_test.go b/cmd/internal/agentcontainer/lifecycle_test.go index 4b90efd7cd..a7d4685938 100644 --- a/cmd/internal/agentcontainer/lifecycle_test.go +++ b/cmd/internal/agentcontainer/lifecycle_test.go @@ -111,7 +111,6 @@ func TestBeginLifecycleMissingOwnerWithTimeoutIsMigrationError(t *testing.T) { assert.Nil(t, h) require.Error(t, err) assert.ErrorIs(t, err, agentd.ErrLifecycleOwnerMissing) - assert.Contains(t, err.Error(), "stop and start the workspace") } func TestBeginLifecycleRequiresManagedOwnerEvenWithoutTimeout(t *testing.T) { diff --git a/pkg/daemon/agent/container_lifecycle.go b/pkg/daemon/agent/container_lifecycle.go index 033326d5a8..f5a1c40f2f 100644 --- a/pkg/daemon/agent/container_lifecycle.go +++ b/pkg/daemon/agent/container_lifecycle.go @@ -46,6 +46,33 @@ type LifecycleOptions struct { Timeout time.Duration } +// ResolveManagedLifecycle selects ownership for a managed-capable driver's topology. +// Dockerless owns the pod process; its nested workspace cannot stop that pod. +func ResolveManagedLifecycle( + timeout time.Duration, + dockerless, imageEntrypoint bool, +) (LifecycleOptions, error) { + if dockerless { + if timeout > 0 { + return LifecycleOptions{}, fmt.Errorf( + "inactivity timeout %s is not supported with dockerless builds because the nested "+ + "workspace container does not end the pod when it exits; unset the inactivity "+ + "timeout or disable dockerless", timeout) + } + return LifecycleOptions{}, nil + } + if imageEntrypoint { + if timeout > 0 { + return LifecycleOptions{}, fmt.Errorf( + "inactivity timeout %s requires the Devsy daemon to be the container's main "+ + "process, but overrideCommand is false and the image entrypoint would run "+ + "instead; set overrideCommand to true or unset the inactivity timeout", timeout) + } + return LifecycleOptions{}, nil + } + return LifecycleOptions{Managed: true, Timeout: timeout}, nil +} + // TimeoutInputs are the configured values that decide the effective timeout. type TimeoutInputs struct { // Canonical is agent.containerInactivityTimeout. @@ -152,7 +179,11 @@ func EncodeDaemonConfig(cfg *DaemonConfig) (string, error) { // RefreshLifecycle rewrites the generated lifecycle policy of a stored encoded // daemon config, leaving platform and SSH settings untouched. An empty stored // config yields a minimal timeout-only config. -func RefreshLifecycle(encoded string, timeout time.Duration) (string, error) { +func RefreshLifecycle( + encoded string, + lifecycle LifecycleOptions, + shutdownAction string, +) (string, error) { cfg := &DaemonConfig{} if encoded != "" { var err error @@ -160,7 +191,11 @@ func RefreshLifecycle(encoded string, timeout time.Duration) (string, error) { return "", err } } - cfg.Timeout = FormatTimeout(timeout) - cfg.Lifecycle = LifecycleConfig{Owner: LifecycleOwnerManagedDaemon, SetupArmsTimer: true} + cfg.Timeout = FormatTimeout(lifecycle.Timeout) + cfg.ShutdownAction = shutdownAction + cfg.Lifecycle = LifecycleConfig{} + if lifecycle.Managed { + cfg.Lifecycle = LifecycleConfig{Owner: LifecycleOwnerManagedDaemon, SetupArmsTimer: true} + } return EncodeDaemonConfig(cfg) } diff --git a/pkg/daemon/agent/container_lifecycle_test.go b/pkg/daemon/agent/container_lifecycle_test.go index f4aecb921d..aeaefacc39 100644 --- a/pkg/daemon/agent/container_lifecycle_test.go +++ b/pkg/daemon/agent/container_lifecycle_test.go @@ -134,7 +134,10 @@ func TestRefreshLifecycleKeepsPlatformAndReplacesPolicy(t *testing.T) { }) require.NoError(t, err) - refreshed, err := RefreshLifecycle(stored, 2*time.Minute) + refreshed, err := RefreshLifecycle( + stored, LifecycleOptions{Managed: true, Timeout: 2 * time.Minute}, + config.ShutdownActionStopContainer, + ) require.NoError(t, err) cfg, err := DecodeDaemonConfig(refreshed) require.NoError(t, err) @@ -146,8 +149,8 @@ func TestRefreshLifecycleKeepsPlatformAndReplacesPolicy(t *testing.T) { assert.Equal(t, config.ShutdownActionStopContainer, cfg.ShutdownAction) } -func TestRefreshLifecycleFromNothingIsTimeoutOnly(t *testing.T) { - refreshed, err := RefreshLifecycle("", 0) +func TestRefreshLifecycleKeepsZeroTimeoutManaged(t *testing.T) { + refreshed, err := RefreshLifecycle("", LifecycleOptions{Managed: true}, "") require.NoError(t, err) cfg, err := DecodeDaemonConfig(refreshed) require.NoError(t, err) diff --git a/pkg/devcontainer/config/container_details.go b/pkg/devcontainer/config/container_details.go index 38aa970a3b..72f25a83ef 100644 --- a/pkg/devcontainer/config/container_details.go +++ b/pkg/devcontainer/config/container_details.go @@ -36,6 +36,9 @@ type ContainerDetails struct { ImageID string `json:"Image,omitempty"` ImageManifestDescriptor *ocispec.Descriptor `json:"ImageManifestDescriptor,omitempty"` Created string `json:"Created,omitempty"` + Dockerless bool `json:"Dockerless,omitempty"` + ImageEntrypoint bool `json:"ImageEntrypoint,omitempty"` + LifecycleTopologyError string `json:"LifecycleTopologyError,omitempty"` State ContainerDetailsState `json:"State"` Config ContainerDetailsConfig `json:"Config"` Mounts []ContainerMount `json:"Mounts,omitempty"` diff --git a/pkg/devcontainer/kubernetes_recreate_preflight_test.go b/pkg/devcontainer/kubernetes_recreate_preflight_test.go new file mode 100644 index 0000000000..ee4d5fc5a4 --- /dev/null +++ b/pkg/devcontainer/kubernetes_recreate_preflight_test.go @@ -0,0 +1,296 @@ +package devcontainer + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/devcontainer/metadata" + kubernetesdriver "github.com/devsy-org/devsy/pkg/driver/kubernetes" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/status" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubescheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +const ( + kubeRecreateWorkspace = "policy-workspace" + kubeRecreateNamespace = "policy-test" + kubeRecreateUID = "policy-owner" + kubeRecreateContext = "fixture" + kubeRecreateArchitecture = "amd64" + regularSidecarTemplate = "spec:\n containers:\n - name: helper\n image: example/helper:1\n" +) + +type kubeTemplatePolicyCase struct { + name string + template string + timeout string + metadataNone bool + disableDaemon bool + reject bool +} + +func TestKubernetesRecreateTemplatePreflightPreservesExistingPod(t *testing.T) { + for _, tc := range []kubeTemplatePolicyCase{ + {name: "positive regular sidecar rejected", template: regularSidecarTemplate, timeout: "1m", reject: true}, + {name: "zero regular sidecar allowed", template: regularSidecarTemplate}, + { + name: "resolved image shutdown none allows sidecar", + template: regularSidecarTemplate, timeout: "1m", metadataNone: true, + }, + {name: "current disable daemon allows sidecar", template: regularSidecarTemplate, timeout: "1m", disableDaemon: true}, + { + name: "positive normal init allowed", timeout: "1m", + template: "spec:\n initContainers:\n - name: prepare\n image: example/init:1\n", + }, + { + name: "positive native sidecar allowed", timeout: "1m", + template: "spec:\n initContainers:\n - name: helper\n" + + " image: example/helper:1\n restartPolicy: Always\n", + }, + } { + t.Run(tc.name, func(t *testing.T) { + r, params, api := newKubernetesTemplateRecreateFixture(t, tc) + before, _, fixtureErr := api.snapshot() + require.NoError(t, fixtureErr) + + _, _, err := r.buildNewContainerConfig(t.Context(), params) + + after, writes, fixtureErr := api.snapshot() + require.NoError(t, fixtureErr) + if tc.reject { + assert.Error(t, err) + assert.Equal( + t, + before, + after, + "unsupported template must not stop the existing pod", + ) + assert.Empty(t, writes, "preflight must not change API state") + return + } + require.NoError(t, err) + assert.Nil( + t, + after, + "supported explicit recreation reaches the real Kubernetes stop boundary", + ) + assert.Equal(t, []string{http.MethodDelete + " " + api.podPath}, writes) + }) + } +} + +func newKubernetesTemplateRecreateFixture( + t *testing.T, + tc kubeTemplatePolicyCase, +) (*runner, *resolveParams, *kubeRecreateAPI) { + t.Helper() + api := newKubeRecreateAPI() + server := httptest.NewServer(api) + t.Cleanup(server.Close) + info := newKubeRecreateWorkspaceInfo(t, server.URL, tc) + d, err := kubernetesdriver.NewKubernetesDriver(info) + require.NoError(t, err) + r := newTestRunner(d) + r.id, r.workspaceConfig, r.reporter = kubeRecreateWorkspace, info, status.Nop() + imageMetadata := &config.ImageMetadataConfig{} + if tc.metadataNone { + imageConfig := &config.DevContainerConfig{} + imageConfig.ShutdownAction = config.ShutdownActionNone + config.AddConfigToImageMetadata(imageConfig, imageMetadata) + } + labels, err := json.Marshal(imageMetadata.Raw) + require.NoError(t, err) + image := &config.ImageDetails{ID: "fixture-image", Config: config.ImageDetailsConfig{ + Labels: map[string]string{metadata.ImageMetadataLabel: string(labels)}, + }} + r.imageBackend = &separateImages{ + details: image, + buildInfo: &config.BuildInfo{ + ImageName: "fixture-image", + ImageDetails: image, + ImageMetadata: imageMetadata, + }, + } + params := recreateResolveParams() + params.options.Recreate = true + params.parsedConfig.Config.Image = "example/workspace:1" + params.parsedConfig.Raw = config.CloneDevContainerConfig(params.parsedConfig.Config) + params.substitutionContext = &config.SubstitutionContext{ + ContainerWorkspaceFolder: managedWorkspaceFolder, + } + return r, params, api +} + +func newKubeRecreateWorkspaceInfo( + t *testing.T, + serverURL string, + tc kubeTemplatePolicyCase, +) *provider.AgentWorkspaceInfo { + t.Helper() + dir := t.TempDir() + kubeconfig := filepath.Join(dir, "kubeconfig") + data, err := clientcmd.Write(clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{ + kubeRecreateContext: {Server: serverURL}, + }, + AuthInfos: map[string]*clientcmdapi.AuthInfo{kubeRecreateContext: {}}, + Contexts: map[string]*clientcmdapi.Context{kubeRecreateContext: { + Cluster: kubeRecreateContext, AuthInfo: kubeRecreateContext, + Namespace: kubeRecreateNamespace, + }}, + CurrentContext: kubeRecreateContext, + }) + require.NoError(t, err) + require.NoError(t, os.WriteFile(kubeconfig, data, 0o600)) + template := filepath.Join(dir, "pod.yaml") + require.NoError(t, os.WriteFile(template, []byte(tc.template), 0o600)) + return &provider.AgentWorkspaceInfo{ + Workspace: &provider.Workspace{ID: kubeRecreateWorkspace, UID: kubeRecreateUID}, + Agent: provider.ProviderAgentConfig{ + Driver: provider.KubernetesDriver, + ContainerTimeout: tc.timeout, + Kubernetes: provider.ProviderKubernetesDriverConfig{ + KubernetesConfig: kubeconfig, + KubernetesNamespace: kubeRecreateNamespace, + PodManifestTemplate: template, + Architecture: kubeRecreateArchitecture, + }, + }, + CLIOptions: provider.CLIOptions{DisableDaemon: tc.disableDaemon}, + } +} + +type kubeRecreateAPI struct { + mu sync.Mutex + podPath string + pod *corev1.Pod + writes []string + err error +} + +func newKubeRecreateAPI() *kubeRecreateAPI { + name := "devsy-" + kubeRecreateWorkspace + return &kubeRecreateAPI{ + podPath: "/api/v1/namespaces/" + kubeRecreateNamespace + "/pods/" + name, + pod: &corev1.Pod{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: kubeRecreateNamespace, + UID: "existing-pod", + ResourceVersion: "7", + Labels: map[string]string{ + kubernetesdriver.DevsyCreatedLabel: stringTrue, + kubernetesdriver.DevsyWorkspaceUIDLabel: kubeRecreateUID, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: kubernetesdriver.DevContainerName, Image: "example/old:1"}, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + } +} + +func (s *kubeRecreateAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + if r.Method != http.MethodGet && r.Method != http.MethodHead { + s.writes = append(s.writes, r.Method+" "+r.URL.Path) + } + if r.URL.Path != s.podPath || s.pod == nil { + s.writeStatus(w, http.StatusNotFound, metav1.StatusReasonNotFound) + return + } + s.servePod(w, r) +} + +func (s *kubeRecreateAPI) servePod(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + s.writeObject(w, http.StatusOK, s.pod) + case http.MethodDelete: + if !s.deleteMatches(w, r) { + return + } + s.pod = nil + s.writeObject(w, http.StatusOK, &metav1.Status{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Status", + }, + Status: metav1.StatusSuccess, + }) + default: + s.writeStatus(w, http.StatusMethodNotAllowed, metav1.StatusReasonMethodNotAllowed) + } +} + +func (s *kubeRecreateAPI) deleteMatches(w http.ResponseWriter, r *http.Request) bool { + body, err := io.ReadAll(r.Body) + if err != nil { + s.err = err + s.writeStatus(w, http.StatusBadRequest, metav1.StatusReasonBadRequest) + return false + } + options := &metav1.DeleteOptions{} + if _, _, err := kubescheme.Codecs.UniversalDeserializer(). + Decode(body, nil, options); err != nil { + s.err = err + s.writeStatus(w, http.StatusBadRequest, metav1.StatusReasonBadRequest) + return false + } + if preconditions := options.Preconditions; preconditions != nil { + if preconditions.UID != nil && *preconditions.UID != s.pod.UID { + s.writeStatus(w, http.StatusConflict, metav1.StatusReasonConflict) + return false + } + if preconditions.ResourceVersion != nil && + *preconditions.ResourceVersion != s.pod.ResourceVersion { + s.writeStatus(w, http.StatusConflict, metav1.StatusReasonConflict) + return false + } + } + return true +} + +func (s *kubeRecreateAPI) writeStatus( + w http.ResponseWriter, + code int32, + reason metav1.StatusReason, +) { + s.writeObject(w, code, &metav1.Status{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Status"}, + Status: metav1.StatusFailure, Reason: reason, Code: code, + }) +} + +func (s *kubeRecreateAPI) writeObject(w http.ResponseWriter, code int32, object runtime.Object) { + w.Header().Set("Content-Type", runtime.ContentTypeJSON) + w.WriteHeader(int(code)) + if err := json.NewEncoder(w).Encode(object); err != nil { + s.err = err + } +} + +func (s *kubeRecreateAPI) snapshot() (*corev1.Pod, []string, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.pod.DeepCopy(), append([]string(nil), s.writes...), s.err +} diff --git a/pkg/devcontainer/managed_lifecycle_test.go b/pkg/devcontainer/managed_lifecycle_test.go index 8958670d55..5e0624d435 100644 --- a/pkg/devcontainer/managed_lifecycle_test.go +++ b/pkg/devcontainer/managed_lifecycle_test.go @@ -11,16 +11,28 @@ import ( agentd "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/status" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -const testEntrypoint = "/app" +const ( + managedWorkspaceHost = "workspace" + managedWorkspaceFolder = "/workspaces/app" + testEntrypoint = "/app" +) -type managedMockDriver struct{ *mockDriver } +type managedMockDriver struct { + *mockDriver + shutdownAction string +} func (*managedMockDriver) ManagedLifecycle() bool { return true } +func (d *managedMockDriver) SetLifecycleShutdownAction(action string) { + d.shutdownAction = action +} + func managedRunner(timeout string) (*runner, *managedMockDriver) { d := &managedMockDriver{mockDriver: &mockDriver{}} r := newTestRunner(d) @@ -32,7 +44,7 @@ func managedRunner(timeout string) (*runner, *managedMockDriver) { func resolveParamsFor(platform devsy.PlatformOptions) *resolveParams { return &resolveParams{ substitutionContext: &config.SubstitutionContext{ - ContainerWorkspaceFolder: "/workspaces/app", + ContainerWorkspaceFolder: managedWorkspaceFolder, }, options: UpOptions{CLIOptions: provider.CLIOptions{Platform: platform}}, } @@ -58,7 +70,10 @@ func TestInjectDaemonEntrypointManagedWithoutPlatform(t *testing.T) { r, _ := managedRunner("1m") merged := mergedWithEnv() - require.NoError(t, r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged)) + require.NoError( + t, + r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged, false, false), + ) cfg := injectedConfig(t, merged) assert.Equal(t, "1m0s", cfg.Timeout) @@ -73,7 +88,7 @@ func TestInjectDaemonEntrypointManagedWithPlatformKeepsNetworking(t *testing.T) merged := mergedWithEnv() platform := devsy.PlatformOptions{AccessKey: "key", PlatformHost: "p", WorkspaceHost: "w"} - require.NoError(t, r.injectDaemonEntrypoint(resolveParamsFor(platform), merged)) + require.NoError(t, r.injectDaemonEntrypoint(resolveParamsFor(platform), merged, false, false)) cfg := injectedConfig(t, merged) assert.Equal(t, "key", cfg.Platform.AccessKey) @@ -86,7 +101,10 @@ func TestInjectDaemonEntrypointUnmanagedWithoutPlatformIsUntouched(t *testing.T) r.workspaceConfig.Workspace = &provider.Workspace{} merged := mergedWithEnv() - require.NoError(t, r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged)) + require.NoError( + t, + r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged, false, false), + ) assert.Empty(t, merged.ContainerEnv) } @@ -96,7 +114,10 @@ func TestInjectDaemonEntrypointResolvesLegacyAlias(t *testing.T) { r.workspaceConfig.Agent.Kubernetes.InactivityTimeout = "5m" merged := mergedWithEnv() - require.NoError(t, r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged)) + require.NoError( + t, + r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged, false, false), + ) assert.Equal(t, "5m0s", injectedConfig(t, merged).Timeout) } @@ -105,7 +126,12 @@ func TestInjectDaemonEntrypointPropagatesConfigErrors(t *testing.T) { r, _ := managedRunner("1m") r.workspaceConfig.Agent.Kubernetes.InactivityTimeout = "2m" - err := r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), mergedWithEnv()) + err := r.injectDaemonEntrypoint( + resolveParamsFor(devsy.PlatformOptions{}), + mergedWithEnv(), + false, + false, + ) assert.ErrorContains(t, err, "conflicts") } @@ -115,7 +141,10 @@ func TestInjectDaemonEntrypointShutdownNoneDisablesTimeout(t *testing.T) { merged := mergedWithEnv() merged.ShutdownAction = config.ShutdownActionNone - require.NoError(t, r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged)) + require.NoError( + t, + r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged, false, false), + ) cfg := injectedConfig(t, merged) assert.Empty(t, cfg.Timeout) @@ -126,7 +155,7 @@ func TestWorkspaceInfoCarriesEffectiveTimeoutAndOwner(t *testing.T) { r, _ := managedRunner("") r.workspaceConfig.Agent.Kubernetes.InactivityTimeout = "90s" - compressed, err := r.compressWorkspaceConfig(mergedWithEnv()) + compressed, err := r.compressWorkspaceConfig(mergedWithEnv(), false, false) require.NoError(t, err) info, _, err := pkgagent.DecodeContainerWorkspaceInfo(compressed) require.NoError(t, err) @@ -194,3 +223,227 @@ func TestFindRunningContainerOrFailRejectsTerminalObjectsForManagedDrivers(t *te require.NoError(t, err) assert.NotNil(t, details, "other drivers keep their existing behavior") } + +func TestDockerlessBootstrapDoesNotRequireManagedOwner(t *testing.T) { + for _, platform := range []devsy.PlatformOptions{ + {}, + {AccessKey: "key", PlatformHost: "platform", WorkspaceHost: managedWorkspaceHost}, + } { + name := "without platform" + if platform.AccessKey != "" { + name = "with platform" + } + t.Run(name, func(t *testing.T) { + r, _ := managedRunner("") + merged := mergedWithEnv() + params := resolveParamsFor(platform) + require.NoError(t, r.injectDaemonEntrypoint(params, merged, true, false)) + + info, err := r.prepareSetupInfo(&setupContainerParams{ + rawConfig: &config.DevContainerConfig{}, + containerDetails: &config.ContainerDetails{ + Dockerless: true, + State: config.ContainerDetailsState{Status: config.ContainerStatusRunning}, + }, + mergedConfig: merged, + substitutionContext: params.substitutionContext, + }) + require.NoError(t, err) + workspace, _, err := pkgagent.DecodeContainerWorkspaceInfo( + info.workspaceConfigCompressed, + ) + require.NoError(t, err) + assert.Empty(t, workspace.LifecycleOwner) + assert.Empty(t, workspace.ContainerTimeout) + + daemon := injectedConfig(t, merged) + assert.Equal(t, merged.ShutdownAction, daemon.ShutdownAction) + assert.False(t, daemon.Lifecycle.Managed()) + assert.False(t, daemon.Lifecycle.SetupArmsTimer) + assert.Empty(t, daemon.Timeout) + if platform.AccessKey == "" { + assert.Empty(t, daemon.Platform.AccessKey) + assert.Empty(t, daemon.Ssh) + return + } + assert.Equal(t, platform, daemon.Platform) + assert.Equal( + t, + agentd.SshConfig{User: containerRootUser, Workdir: managedWorkspaceFolder}, + daemon.Ssh, + ) + }) + } +} + +func TestDockerlessProducerRejectsEnabledTimeout(t *testing.T) { + for _, legacy := range []bool{false, true} { + name := "canonical" + if legacy { + name = "legacy" + } + t.Run(name, func(t *testing.T) { + r, _ := managedRunner("1m") + if legacy { + r.workspaceConfig.Agent.ContainerTimeout = "" + r.workspaceConfig.Agent.Kubernetes.InactivityTimeout = "1m" + } + merged := mergedWithEnv() + err := r.injectDaemonEntrypoint( + resolveParamsFor(devsy.PlatformOptions{}), + merged, + true, + false, + ) + require.ErrorContains(t, err, "dockerless") + assert.NotContains(t, merged.ContainerEnv, pkgconfig.EnvWorkspaceDaemonConfig) + + _, err = r.compressWorkspaceConfig(merged, true, false) + assert.ErrorContains(t, err, "dockerless") + }) + } +} + +func TestDockerlessProducerHonorsDisabledTimeout(t *testing.T) { + for _, disableDaemon := range []bool{false, true} { + name := "shutdown none" + if disableDaemon { + name = "disabled daemon" + } + t.Run(name, func(t *testing.T) { + r, _ := managedRunner("1m") + merged := mergedWithEnv() + if disableDaemon { + r.workspaceConfig.CLIOptions.DisableDaemon = true + } else { + merged.ShutdownAction = config.ShutdownActionNone + } + require.NoError( + t, + r.injectDaemonEntrypoint( + resolveParamsFor(devsy.PlatformOptions{}), + merged, + true, + false, + ), + ) + compressed, err := r.compressWorkspaceConfig(merged, true, false) + require.NoError(t, err) + workspace, _, err := pkgagent.DecodeContainerWorkspaceInfo(compressed) + require.NoError(t, err) + assert.Empty(t, workspace.LifecycleOwner) + assert.Empty(t, workspace.ContainerTimeout) + daemon := injectedConfig(t, merged) + assert.Equal(t, merged.ShutdownAction, daemon.ShutdownAction) + assert.False(t, daemon.Lifecycle.Managed()) + assert.Empty(t, daemon.Timeout) + }) + } +} + +func TestZeroTimeoutKubernetesStillRequiresManagedOwner(t *testing.T) { + r, _ := managedRunner("") + merged := mergedWithEnv() + require.NoError( + t, + r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged, false, false), + ) + assert.True(t, injectedConfig(t, merged).Lifecycle.Managed()) + + compressed, err := r.compressWorkspaceConfig(merged, false, false) + require.NoError(t, err) + workspace, _, err := pkgagent.DecodeContainerWorkspaceInfo(compressed) + require.NoError(t, err) + assert.Equal(t, provider.ContainerLifecycleOwnerManagedDaemon, workspace.LifecycleOwner) + assert.Empty(t, workspace.ContainerTimeout) +} + +func TestZeroTimeoutOwnershipFollowsActualImageMain(t *testing.T) { + no := false + for _, tt := range []struct { + name string + override *bool + image config.ImageDetailsConfig + managed bool + }{ + { + name: "default override retains the daemon", + image: config.ImageDetailsConfig{Entrypoint: []string{testEntrypoint}}, + managed: true, + }, + {name: "empty image main retains the daemon", override: &no, managed: true}, + { + name: "external entrypoint owns the container", override: &no, + image: config.ImageDetailsConfig{Entrypoint: []string{testEntrypoint}}, + }, + { + name: "external command owns the container", override: &no, + image: config.ImageDetailsConfig{Cmd: []string{testEntrypoint}}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + r, _ := managedRunner("") + merged := mergedWithEnv() + merged.OverrideCommand = tt.override + image := &config.ImageDetails{Config: tt.image} + external := usesImageEntrypoint(merged, image) + require.NoError(t, r.injectDaemonEntrypoint( + resolveParamsFor(devsy.PlatformOptions{}), merged, false, external, + )) + assert.Equal(t, tt.managed, injectedConfig(t, merged).Lifecycle.Managed()) + + _, args := GetContainerEntrypointAndArgs(merged, image) + if external { + want := append([]string{}, tt.image.Entrypoint...) + want = append(want, tt.image.Cmd...) + assert.Equal(t, want, args[3:]) + } else { + assert.Empty(t, args[3:]) + } + compressed, err := r.compressWorkspaceConfig(merged, false, external) + require.NoError(t, err) + info, _, err := pkgagent.DecodeContainerWorkspaceInfo(compressed) + require.NoError(t, err) + if tt.managed { + assert.Equal(t, provider.ContainerLifecycleOwnerManagedDaemon, info.LifecycleOwner) + } else { + assert.Empty(t, info.LifecycleOwner) + } + }) + } +} + +func TestUnsupportedManagedRecreatePreservesExistingContainer(t *testing.T) { + const ( + workspaceUID = "workspace-uid" + inspectedImageID = "image" + builtImageName = "built-image" + baseImageName = "alpine" + ) + r, d := managedRunner("1m") + r.workspaceConfig.Workspace = &provider.Workspace{ID: r.id, UID: workspaceUID} + image := &config.ImageDetails{ID: inspectedImageID, Config: config.ImageDetailsConfig{ + Entrypoint: []string{testEntrypoint}, + }} + r.imageBackend = &separateImages{ + details: image, + buildInfo: &config.BuildInfo{ + ImageName: builtImageName, ImageDetails: image, + ImageMetadata: &config.ImageMetadataConfig{}, + }, + } + r.reporter = status.Nop() + no := false + params := recreateResolveParams() + params.options.Recreate = true + params.parsedConfig.Config.Image = baseImageName + params.parsedConfig.Config.OverrideCommand = &no + params.parsedConfig.Raw = config.CloneDevContainerConfig(params.parsedConfig.Config) + params.substitutionContext = &config.SubstitutionContext{} + + _, _, err := r.buildNewContainerConfig(context.Background(), params) + + require.Error(t, err) + assert.False(t, d.stopCalled) + assert.False(t, d.deleteCalled) +} diff --git a/pkg/devcontainer/setup.go b/pkg/devcontainer/setup.go index 811e0d82b5..9e5e3c8721 100644 --- a/pkg/devcontainer/setup.go +++ b/pkg/devcontainer/setup.go @@ -310,6 +310,12 @@ func (r *runner) legacyInject(ctx context.Context, timeout time.Duration) error } func (r *runner) prepareSetupInfo(params *setupContainerParams) (*setupInfo, error) { + if params.containerDetails.LifecycleTopologyError != "" { + return nil, fmt.Errorf( + "workspace lifecycle topology: %s", + params.containerDetails.LifecycleTopologyError, + ) + } result := r.buildResult(params) compressed, err := r.compressResult(result) @@ -317,7 +323,11 @@ func (r *runner) prepareSetupInfo(params *setupContainerParams) (*setupInfo, err return nil, err } - workspaceConfigCompressed, err := r.compressWorkspaceConfig(result.MergedConfig) + workspaceConfigCompressed, err := r.compressWorkspaceConfig( + result.MergedConfig, + params.containerDetails.Dockerless, + params.containerDetails.ImageEntrypoint, + ) if err != nil { return nil, err } @@ -378,6 +388,7 @@ func (r *runner) compressResult(result *config.Result) (string, error) { func (r *runner) compressWorkspaceConfig( mergedConfig *config.MergedDevContainerConfig, + dockerless, imageEntrypoint bool, ) (string, error) { cliOptions := r.workspaceConfig.CLIOptions cliOptions.SecretsEnv = nil @@ -399,12 +410,14 @@ func (r *runner) compressWorkspaceConfig( Agent: r.workspaceConfig.Agent, ContentFolder: r.workspaceConfig.ContentFolder, } + lifecycle, err := r.lifecycleOptions(mergedConfig, dockerless, imageEntrypoint) + if err != nil { + return "", err + } if r.managedLifecycle() { - timeout, err := r.effectiveContainerTimeout(mergedConfig.ShutdownAction) - if err != nil { - return "", err - } - workspaceConfig.ContainerTimeout = agentd.FormatTimeout(timeout) + workspaceConfig.ContainerTimeout = agentd.FormatTimeout(lifecycle.Timeout) + } + if lifecycle.Managed { workspaceConfig.LifecycleOwner = provider2.ContainerLifecycleOwnerManagedDaemon } workspaceConfig.PullFromInsideContainer = resolvePullFromInsideContainer( diff --git a/pkg/devcontainer/setup_secrets_test.go b/pkg/devcontainer/setup_secrets_test.go index 2fe2ea879a..7696922b59 100644 --- a/pkg/devcontainer/setup_secrets_test.go +++ b/pkg/devcontainer/setup_secrets_test.go @@ -32,7 +32,7 @@ func TestCompressWorkspaceConfig_StripsSecretValues(t *testing.T) { }, } - compressed, err := r.compressWorkspaceConfig(&config.MergedDevContainerConfig{}) + compressed, err := r.compressWorkspaceConfig(&config.MergedDevContainerConfig{}, false, false) if err != nil { t.Fatal(err) } diff --git a/pkg/devcontainer/single.go b/pkg/devcontainer/single.go index 7a7939556b..282f65951e 100644 --- a/pkg/devcontainer/single.go +++ b/pkg/devcontainer/single.go @@ -265,7 +265,17 @@ func (r *runner) resolveExistingContainer( r.recovering = true } - containerDetails, err := r.ensureRunning(ctx, containerDetails) + var mergedConfig *config.MergedDevContainerConfig + var err error + if d, ok := r.driver.(driver.ManagedLifecycleDriver); ok && d.ManagedLifecycle() { + mergedConfig, err = r.mergeExistingContainerConfig(ctx, containerDetails, p) + if err != nil { + return nil, err + } + d.SetLifecycleShutdownAction(mergedConfig.ShutdownAction) + } + + containerDetails, err = r.ensureRunning(ctx, containerDetails) if err != nil { return nil, err } @@ -275,9 +285,11 @@ func (r *runner) resolveExistingContainer( p.substitutionContext.ContainerWorkspaceFolder = containerDetails.Config.WorkingDir } - mergedConfig, err := r.mergeExistingContainerConfig(ctx, containerDetails, p) - if err != nil { - return nil, err + if mergedConfig == nil { + mergedConfig, err = r.mergeExistingContainerConfig(ctx, containerDetails, p) + if err != nil { + return nil, err + } } containerDetails, err = r.reprovisionIfNeeded(ctx, containerDetails) @@ -392,11 +404,11 @@ func (r *runner) resolveNewContainer( return nil, err } - if err := r.injectDaemonEntrypoint(p, mergedConfig); err != nil { - return nil, err - } - - if err := r.validateManagedTimeoutTopology(mergedConfig, buildInfo); err != nil { + imageEntrypoint := buildInfo.Dockerless == nil && + usesImageEntrypoint(mergedConfig, buildInfo.ImageDetails) + if err := r.injectDaemonEntrypoint( + p, mergedConfig, buildInfo.Dockerless != nil, imageEntrypoint, + ); err != nil { return nil, err } @@ -415,6 +427,9 @@ func (r *runner) resolveNewContainer( if err != nil { return nil, err } + // New run options take precedence over metadata from an existing workspace volume. + containerDetails.Dockerless = buildInfo.Dockerless != nil + containerDetails.ImageEntrypoint = imageEntrypoint if w := r.lingerWarning(ctx); w != "" { hostWarnings = append(hostWarnings, w) @@ -490,7 +505,6 @@ func (r *runner) buildNewContainerConfig( return nil, nil, err } - _, validatesImageRun := r.driver.(driver.ImageRunValidator) mergedConfig, err := config.MergeConfiguration( activeConfig.Config, buildInfo.ImageMetadata.Config, @@ -504,10 +518,8 @@ func (r *runner) buildNewContainerConfig( return nil, nil, err } config.MergeExtraRemoteEnvConfig(mergedConfig, overlay) - if validatesImageRun { - if err := r.validateImageRun(p, activeConfig.Config, mergedConfig, buildInfo); err != nil { - return nil, nil, err - } + if err := r.validateImageRun(p, activeConfig.Config, mergedConfig, buildInfo); err != nil { + return nil, nil, err } if p.options.Recreate { if err := r.deleteForRecreate(ctx); err != nil { @@ -552,6 +564,16 @@ func (r *runner) validateImageRun( mergedConfig *config.MergedDevContainerConfig, buildInfo *config.BuildInfo, ) error { + if d, ok := r.driver.(driver.ManagedLifecycleDriver); ok && d.ManagedLifecycle() { + d.SetLifecycleShutdownAction(mergedConfig.ShutdownAction) + } + if err := r.validateManagedTimeoutTopology(mergedConfig, buildInfo); err != nil { + return err + } + validator, validates := r.driver.(driver.ImageRunValidator) + if !validates { + return nil + } var options *driver.RunOptions var err error if buildInfo.Dockerless != nil { @@ -572,7 +594,7 @@ func (r *runner) validateImageRun( if err != nil { return fmt.Errorf("validate image run options: %w", err) } - return r.driver.(driver.ImageRunValidator).ValidateRunImageDevContainer( + return validator.ValidateRunImageDevContainer( &driver.RunImageDevContainerParams{ WorkspaceID: r.id, Options: options, ParsedConfig: parsedConfig, }, @@ -688,31 +710,26 @@ func (r *runner) deleteForRecreate(ctx context.Context) error { func (r *runner) injectDaemonEntrypoint( p *resolveParams, mergedConfig *config.MergedDevContainerConfig, + dockerless, imageEntrypoint bool, ) error { - managed := r.managedLifecycle() + lifecycle, err := r.lifecycleOptions(mergedConfig, dockerless, imageEntrypoint) + if err != nil { + return err + } platform := p.options.Platform if platform.AccessKey == "" { platform = devsy.PlatformOptions{} } - if !managed && platform.AccessKey == "" { + if !r.managedLifecycle() && platform.AccessKey == "" { return nil } - var timeout time.Duration - if managed { - var err error - timeout, err = r.effectiveContainerTimeout(mergedConfig.ShutdownAction) - if err != nil { - return err - } - } - data, err := agent.GetEncodedWorkspaceDaemonConfig( platform, r.workspaceConfig.Workspace, p.substitutionContext, mergedConfig, - agent.LifecycleOptions{Managed: managed, Timeout: timeout}, + lifecycle, ) if err != nil { return fmt.Errorf("build daemon config: %w", err) @@ -742,23 +759,12 @@ func managedTimeoutConflict( mergedConfig *config.MergedDevContainerConfig, buildInfo *config.BuildInfo, ) error { - if buildInfo.Dockerless != nil { - return fmt.Errorf( - "inactivity timeout %s is not supported with dockerless builds because the nested "+ - "workspace container does not end the pod when it exits; unset the inactivity "+ - "timeout or disable dockerless", timeout) - } - overridden := mergedConfig.OverrideCommand == nil || *mergedConfig.OverrideCommand - if overridden || buildInfo.ImageDetails == nil { - return nil - } - if len(buildInfo.ImageDetails.Config.Entrypoint)+len(buildInfo.ImageDetails.Config.Cmd) == 0 { - return nil - } - return fmt.Errorf( - "inactivity timeout %s requires the Devsy daemon to be the container's main process, "+ - "but overrideCommand is false and the image entrypoint would run instead; set "+ - "overrideCommand to true or unset the inactivity timeout", timeout) + _, err := agent.ResolveManagedLifecycle( + timeout, + buildInfo.Dockerless != nil, + usesImageEntrypoint(mergedConfig, buildInfo.ImageDetails), + ) + return err } func (r *runner) managedLifecycle() bool { @@ -766,6 +772,20 @@ func (r *runner) managedLifecycle() bool { return ok && d.ManagedLifecycle() } +func (r *runner) lifecycleOptions( + mergedConfig *config.MergedDevContainerConfig, + dockerless, imageEntrypoint bool, +) (agent.LifecycleOptions, error) { + if !r.managedLifecycle() { + return agent.LifecycleOptions{}, nil + } + timeout, err := r.effectiveContainerTimeout(mergedConfig.ShutdownAction) + if err != nil { + return agent.LifecycleOptions{}, err + } + return agent.ResolveManagedLifecycle(timeout, dockerless, imageEntrypoint) +} + func (r *runner) effectiveContainerTimeout(shutdownAction string) (time.Duration, error) { agentConfig := r.workspaceConfig.Agent return agent.ResolveInactivityTimeout(agent.TimeoutInputs{ @@ -1125,25 +1145,26 @@ func (r *runner) getRunOptions( } return &driver.RunOptions{ - UID: r.workspaceUID(), - Image: buildInfo.ImageName, - ImageBuilt: buildInfo.BuiltLocally, - User: user, - RemoteUser: effectiveRemoteUser(mergedConfig, user), - Entrypoint: entrypoint, - Cmd: cmd, - Env: mergedConfig.ContainerEnv, - CapAdd: mergedConfig.CapAdd, - Labels: labels, - Privileged: mergedConfig.Privileged, - Init: mergedConfig.Init, - WorkspaceMount: workspaceMountPtr, - SecurityOpt: mergedConfig.SecurityOpt, - Mounts: mounts, - Userns: substitutionContext.Userns, - UidMap: substitutionContext.UidMap, - GidMap: substitutionContext.GidMap, - Platform: r.workspaceConfig.CLIOptions.RunPlatform, + UID: r.workspaceUID(), + Image: buildInfo.ImageName, + ImageBuilt: buildInfo.BuiltLocally, + User: user, + RemoteUser: effectiveRemoteUser(mergedConfig, user), + ImageEntrypoint: usesImageEntrypoint(mergedConfig, buildInfo.ImageDetails), + Entrypoint: entrypoint, + Cmd: cmd, + Env: mergedConfig.ContainerEnv, + CapAdd: mergedConfig.CapAdd, + Labels: labels, + Privileged: mergedConfig.Privileged, + Init: mergedConfig.Init, + WorkspaceMount: workspaceMountPtr, + SecurityOpt: mergedConfig.SecurityOpt, + Mounts: mounts, + Userns: substitutionContext.Userns, + UidMap: substitutionContext.UidMap, + GidMap: substitutionContext.GidMap, + Platform: r.workspaceConfig.CLIOptions.RunPlatform, }, nil } @@ -1279,6 +1300,15 @@ func GetStartScript(mergedConfig *config.MergedDevContainerConfig) string { return joinShellStatements(statements...) } +func usesImageEntrypoint( + mergedConfig *config.MergedDevContainerConfig, + imageDetails *config.ImageDetails, +) bool { + return imageDetails != nil && mergedConfig.OverrideCommand != nil && + !*mergedConfig.OverrideCommand && + len(imageDetails.Config.Entrypoint)+len(imageDetails.Config.Cmd) > 0 +} + func GetContainerEntrypointAndArgs( mergedConfig *config.MergedDevContainerConfig, imageDetails *config.ImageDetails, @@ -1288,8 +1318,7 @@ func GetContainerEntrypointAndArgs( GetStartScript(mergedConfig), "-", } // `wait $!` allows for the `trap` to run (synchronous `sleep` would not). - if imageDetails != nil && mergedConfig.OverrideCommand != nil && - !*mergedConfig.OverrideCommand { + if usesImageEntrypoint(mergedConfig, imageDetails) { cmd = append(cmd, imageDetails.Config.Entrypoint...) cmd = append(cmd, imageDetails.Config.Cmd...) } diff --git a/pkg/driver/kubernetes/driver.go b/pkg/driver/kubernetes/driver.go index 22d56e6f02..3cbb0e2bb0 100644 --- a/pkg/driver/kubernetes/driver.go +++ b/pkg/driver/kubernetes/driver.go @@ -39,10 +39,11 @@ func NewKubernetesDriver( log.Debugf("Use Kubernetes Namespace %q", namespace) return &KubernetesDriver{ - client: client, - namespace: namespace, - options: &options, - agentConfig: &workspaceInfo.Agent, + client: client, + namespace: namespace, + options: &options, + agentConfig: &workspaceInfo.Agent, + disableDaemon: workspaceInfo.CLIOptions.DisableDaemon, }, nil } @@ -74,14 +75,22 @@ type KubernetesDriver struct { client *Client - options *provider2.ProviderKubernetesDriverConfig - agentConfig *provider2.ProviderAgentConfig + options *provider2.ProviderKubernetesDriverConfig + agentConfig *provider2.ProviderAgentConfig + disableDaemon bool + shutdownAction string + hasShutdownAction bool } func (k *KubernetesDriver) CanReprovision() bool { return true } +func (k *KubernetesDriver) SetLifecycleShutdownAction(shutdownAction string) { + k.shutdownAction = shutdownAction + k.hasShutdownAction = true +} + func (k *KubernetesDriver) SupportsMountType(mountType string) bool { switch mountType { case driver.MountTypeBind, driver.MountTypeVolume, driver.MountTypeTmpfs: diff --git a/pkg/driver/kubernetes/find.go b/pkg/driver/kubernetes/find.go index c6046c641e..61bec4c505 100644 --- a/pkg/driver/kubernetes/find.go +++ b/pkg/driver/kubernetes/find.go @@ -23,6 +23,9 @@ func (k *KubernetesDriver) FindDevContainer( } else if pvc == nil { return nil, nil } + if containerInfo.Options == nil { + return nil, fmt.Errorf("workspace volume %q has no saved run options", workspaceId) + } // check pod pod, err := k.getPod(ctx, pvc.Name) @@ -41,10 +44,18 @@ func (k *KubernetesDriver) FindDevContainer( if pod != nil { startedAt = pod.CreationTimestamp.String() } + dockerless, imageEntrypoint, topologyErr := observedPodTopology(pod, containerInfo.Options) + topologyError := "" + if topologyErr != nil { + topologyError = topologyErr.Error() + } return &config.ContainerDetails{ - ID: pvc.Name, - Created: pvc.CreationTimestamp.String(), + ID: pvc.Name, + Created: pvc.CreationTimestamp.String(), + Dockerless: dockerless, + ImageEntrypoint: imageEntrypoint, + LifecycleTopologyError: topologyError, State: config.ContainerDetailsState{ Status: status, StartedAt: startedAt, diff --git a/pkg/driver/kubernetes/pod_lifecycle.go b/pkg/driver/kubernetes/pod_lifecycle.go index 6a0eb4f9b9..7f2b17dc0a 100644 --- a/pkg/driver/kubernetes/pod_lifecycle.go +++ b/pkg/driver/kubernetes/pod_lifecycle.go @@ -121,7 +121,7 @@ func (k *KubernetesDriver) reconcileExistingPod( return false, err } - reused, err := k.reconcileObservedPod(ctx, id, pod) + reused, err := k.reconcileDesiredPod(ctx, pod, options) if errors.Is(err, errPodChanged) { continue } @@ -170,6 +170,17 @@ func (k *KubernetesDriver) reconcileObservedPod( return true, nil } +func (k *KubernetesDriver) reconcileDesiredPod( + ctx context.Context, + pod *corev1.Pod, + options *driver.RunOptions, +) (bool, error) { + if classifyPod(pod) == podRunning && podLaunchDiffers(pod, options) { + return false, k.deleteObservedPod(ctx, pod.Name, pod.UID) + } + return k.reconcileObservedPod(ctx, pod.Name, pod) +} + func storedProviderOptions(pod *corev1.Pod) *provider2.ProviderKubernetesDriverConfig { existing := &provider2.ProviderKubernetesDriverConfig{} if raw := pod.GetAnnotations()[DevsyLastAppliedAnnotation]; raw != "" { @@ -231,7 +242,10 @@ func (k *KubernetesDriver) waitPodGone(ctx context.Context, id string, uid types // daemon, which owns inactivity shutdown. func (k *KubernetesDriver) ManagedLifecycle() bool { return true } -var _ driver.ManagedLifecycleDriver = (*KubernetesDriver)(nil) +var ( + _ driver.ManagedLifecycleDriver = (*KubernetesDriver)(nil) + _ driver.ImageRunValidator = (*KubernetesDriver)(nil) +) // effectiveTimeout resolves the current provider timeout for a stored shutdown action. func (k *KubernetesDriver) effectiveTimeout(shutdownAction string) (time.Duration, error) { @@ -243,6 +257,7 @@ func (k *KubernetesDriver) effectiveTimeout(shutdownAction string) (time.Duratio Canonical: canonical, Legacy: k.options.InactivityTimeout, ShutdownAction: shutdownAction, + DisableDaemon: k.disableDaemon, }) } @@ -264,11 +279,20 @@ func (k *KubernetesDriver) refreshStoredLifecycle( } action = cfg.ShutdownAction } + if k.hasShutdownAction { + action = k.shutdownAction + } timeout, err := k.effectiveTimeout(action) if err != nil { return nil, err } - refreshed, err := agentd.RefreshLifecycle(stored, timeout) + lifecycle, err := agentd.ResolveManagedLifecycle( + timeout, options.Dockerless, options.ImageEntrypoint, + ) + if err != nil { + return nil, err + } + refreshed, err := agentd.RefreshLifecycle(stored, lifecycle, action) if err != nil { return nil, err } diff --git a/pkg/driver/kubernetes/pod_lifecycle_test.go b/pkg/driver/kubernetes/pod_lifecycle_test.go index 5fcdd02b18..597d7bd8f5 100644 --- a/pkg/driver/kubernetes/pod_lifecycle_test.go +++ b/pkg/driver/kubernetes/pod_lifecycle_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + pkgconfig "github.com/devsy-org/devsy/pkg/config" agentd "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/driver" @@ -26,12 +27,14 @@ import ( ) const ( - testNamespace = "devsy-test" - testPodID = "devsy-ws" - testWorkspace = "ws-uid" + testNamespace = "devsy-test" + testPodID = "devsy-ws" + testWorkspace = "ws-uid" + podResourceName = "pods" + preservedCustomerMetadata = "keep" ) -var podsResource = schema.GroupResource{Resource: "pods"} +var podsResource = schema.GroupResource{Resource: podResourceName} type fakeCluster struct { cs *fake.Clientset @@ -91,7 +94,7 @@ func (c *fakeCluster) pod(t *testing.T, s podSpec) *corev1.Pod { Status: corev1.PodStatus{Phase: s.phase}, } if !s.foreign { - pod.Labels[DevsyCreatedLabel] = "true" + pod.Labels[DevsyCreatedLabel] = pkgconfig.BoolTrue } if s.deleting { now := metav1.Now() @@ -132,7 +135,7 @@ func (c *fakeCluster) podExists(t *testing.T) bool { func (c *fakeCluster) deleteCalls() int { n := 0 for _, a := range c.cs.Actions() { - if a.Matches("delete", "pods") { + if a.Matches("delete", podResourceName) { n++ } } @@ -260,9 +263,13 @@ func TestReconcileTerminatingPodWaitsForDeletionThenAllowsCreate(t *testing.T) { func TestReconcileForbiddenLookupIsNotAbsence(t *testing.T) { c := newFakeCluster(t) - c.cs.PrependReactor("get", "pods", func(clienttesting.Action) (bool, runtime.Object, error) { - return true, nil, kerrors.NewForbidden(podsResource, testPodID, errors.New("rbac")) - }) + c.cs.PrependReactor( + "get", + podResourceName, + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, kerrors.NewForbidden(podsResource, testPodID, errors.New("rbac")) + }, + ) reused, err := c.reconcile(context.Background()) @@ -275,9 +282,13 @@ func TestReconcileForbiddenLookupIsNotAbsence(t *testing.T) { func TestReconcileDeleteForbiddenSurfaces(t *testing.T) { c := newFakeCluster(t) c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodFailed})) - c.cs.PrependReactor("delete", "pods", func(clienttesting.Action) (bool, runtime.Object, error) { - return true, nil, kerrors.NewForbidden(podsResource, testPodID, errors.New("rbac")) - }) + c.cs.PrependReactor( + "delete", + podResourceName, + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, kerrors.NewForbidden(podsResource, testPodID, errors.New("rbac")) + }, + ) _, err := c.reconcile(context.Background()) @@ -289,9 +300,13 @@ func TestReconcileDeleteForbiddenSurfaces(t *testing.T) { func TestWaitPodGoneDoesNotTreatLookupErrorsAsDeleted(t *testing.T) { c := newFakeCluster(t) c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodFailed})) - c.cs.PrependReactor("get", "pods", func(clienttesting.Action) (bool, runtime.Object, error) { - return true, nil, errors.New("connection refused") - }) + c.cs.PrependReactor( + "get", + podResourceName, + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("connection refused") + }, + ) err := c.k.waitPodGone(context.Background(), testPodID, "u1") @@ -305,7 +320,7 @@ func TestReconcileUIDConflictReobservesReplacementPod(t *testing.T) { var conflicted atomic.Bool c.cs.PrependReactor( "delete", - "pods", + podResourceName, func(a clienttesting.Action) (bool, runtime.Object, error) { if conflicted.Swap(true) { return false, nil, nil @@ -319,7 +334,7 @@ func TestReconcileUIDConflictReobservesReplacementPod(t *testing.T) { }, }) require.NoError(t, c.cs.Tracker().Update( - corev1.SchemeGroupVersion.WithResource("pods"), replacement, testNamespace, + corev1.SchemeGroupVersion.WithResource(podResourceName), replacement, testNamespace, )) return true, nil, kerrors.NewConflict(podsResource, testPodID, errors.New("uid")) }, @@ -338,7 +353,7 @@ func TestReconcileDeleteCarriesObservedUID(t *testing.T) { var got *types.UID c.cs.PrependReactor( "delete", - "pods", + podResourceName, func(a clienttesting.Action) (bool, runtime.Object, error) { if del, ok := a.(clienttesting.DeleteActionImpl); ok && del.DeleteOptions.Preconditions != nil { @@ -430,9 +445,13 @@ func TestReplacementPreservesWorkspaceVolume(t *testing.T) { func TestFindDevContainerReportsLookupErrors(t *testing.T) { c := newFakeCluster(t, workspacePVC(t)) - c.cs.PrependReactor("get", "pods", func(clienttesting.Action) (bool, runtime.Object, error) { - return true, nil, kerrors.NewForbidden(podsResource, testPodID, errors.New("rbac")) - }) + c.cs.PrependReactor( + "get", + podResourceName, + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, kerrors.NewForbidden(podsResource, testPodID, errors.New("rbac")) + }, + ) details, err := c.k.FindDevContainer(context.Background(), "ws") @@ -561,6 +580,96 @@ func TestRefreshStoredLifecycleLegacyAliasAndConflict(t *testing.T) { assert.ErrorContains(t, err, "conflicts") } +func TestStoredLifecycleOwnershipFollowsDockerlessTopology(t *testing.T) { + for _, dockerless := range []bool{false, true} { + name := "ordinary image keeps managed ownership" + if dockerless { + name = "dockerless clears stale managed ownership" + } + t.Run(name, func(t *testing.T) { + c := newFakeCluster(t) + cfg := agentd.DaemonConfig{ + Timeout: "1m", + ShutdownAction: config.ShutdownActionStopContainer, + Lifecycle: agentd.LifecycleConfig{ + Owner: agentd.LifecycleOwnerManagedDaemon, SetupArmsTimer: true, + }, + Ssh: agentd.SshConfig{User: "vscode", Workdir: "/workspaces/app"}, + } + cfg.Platform.AccessKey = "key" + cfg.Platform.PlatformHost = "platform" + cfg.Platform.WorkspaceHost = "workspace" + stored := daemonOptions(t, cfg) + stored.Dockerless = dockerless + original := stored.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"] + + refreshed, err := c.k.refreshStoredLifecycle(stored) + require.NoError(t, err) + got, err := agentd.DecodeDaemonConfig(refreshed.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"]) + require.NoError(t, err) + assert.Equal(t, !dockerless, got.Lifecycle.Managed()) + assert.Equal(t, !dockerless, got.Lifecycle.SetupArmsTimer) + assert.Empty(t, got.Timeout) + assert.Equal(t, cfg.Platform, got.Platform) + assert.Equal(t, cfg.Ssh, got.Ssh) + assert.Equal(t, cfg.ShutdownAction, got.ShutdownAction) + assert.Equal(t, original, stored.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"]) + }) + } +} + +func TestDockerlessResumeRejectsEnabledTimeoutBeforeMutation(t *testing.T) { + for _, legacy := range []bool{false, true} { + name := "canonical" + if legacy { + name = "legacy" + } + t.Run(name, func(t *testing.T) { + pvc := workspacePVC(t) + info, err := json.Marshal(&DevContainerInfo{ + WorkspaceID: "ws", + Options: &driver.RunOptions{UID: testWorkspace, Dockerless: true}, + }) + require.NoError(t, err) + pvc.Annotations[DevsyInfoAnnotation] = string(info) + c := newFakeCluster(t, pvc) + if legacy { + c.k.options.InactivityTimeout = "1m" + } else { + c.k.agentConfig.ContainerTimeout = "1m" + } + + err = c.k.RunDevContainer(context.Background(), "ws", nil) + require.ErrorContains(t, err, "dockerless") + for _, action := range c.cs.Actions() { + assert.Equal( + t, + "get", + action.GetVerb(), + "invalid resume must not mutate cluster state", + ) + } + }) + } +} + +func TestDockerlessStoredLifecycleHonorsShutdownNone(t *testing.T) { + c := newFakeCluster(t) + c.k.agentConfig.ContainerTimeout = "1m" + stored := daemonOptions(t, agentd.DaemonConfig{ + ShutdownAction: config.ShutdownActionNone, + }) + stored.Dockerless = true + + refreshed, err := c.k.refreshStoredLifecycle(stored) + require.NoError(t, err) + cfg, err := agentd.DecodeDaemonConfig(refreshed.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"]) + require.NoError(t, err) + assert.False(t, cfg.Lifecycle.Managed()) + assert.Empty(t, cfg.Timeout) + assert.Equal(t, config.ShutdownActionNone, cfg.ShutdownAction) +} + func TestAdmittedPodWithInjectedContainerIsRemovedAndVolumeKept(t *testing.T) { pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: testPodID, Namespace: testNamespace}, @@ -608,9 +717,13 @@ func TestWaitPodDeletedTimesOutWhenPodStaysTerminating(t *testing.T) { c := newFakeCluster(t) podDeleteTimeout = 50 * time.Millisecond c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning, deleting: true})) - c.cs.PrependReactor("delete", "pods", func(clienttesting.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) + c.cs.PrependReactor( + "delete", + podResourceName, + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, nil + }, + ) err := c.k.waitPodDeleted(context.Background(), testPodID) @@ -635,12 +748,14 @@ func runOptionsForWorkspace(timeout string) *driver.RunOptions { } func readyOnCreate(c *fakeCluster, uid types.UID) { + nextUID := uid c.cs.PrependReactor( "create", - "pods", + podResourceName, func(a clienttesting.Action) (bool, runtime.Object, error) { pod := a.(clienttesting.CreateAction).GetObject().(*corev1.Pod) - pod.UID = uid + pod.UID = nextUID + nextUID += "-next" pod.Status.Phase = corev1.PodRunning pod.Status.ContainerStatuses = nil for _, container := range pod.Spec.Containers { @@ -697,8 +812,12 @@ func TestRunContainerRejectsExtraContainersBeforeTouchingTheCluster(t *testing.T require.Error(t, err) assert.Contains(t, err.Error(), "istio-proxy") for _, a := range c.cs.Actions() { - assert.False(t, a.Matches("create", "pods") || a.Matches("delete", "pods") || - a.Matches("create", "secrets"), "nothing is created or deleted: %v", a) + assert.False(t, + a.Matches("create", podResourceName) || + a.Matches("delete", podResourceName) || + a.Matches("create", "secrets"), + "nothing is created or deleted: %v", a, + ) } } @@ -721,7 +840,7 @@ func TestRunContainerRemovesPodWhoseAdmittedTopologyBreaksTheContract(t *testing c := newFakeCluster(t) c.cs.PrependReactor( "create", - "pods", + podResourceName, func(a clienttesting.Action) (bool, runtime.Object, error) { pod := a.(clienttesting.CreateAction).GetObject().(*corev1.Pod) pod.UID = "admitted" @@ -740,6 +859,183 @@ func TestRunContainerRemovesPodWhoseAdmittedTopologyBreaksTheContract(t *testing assert.False(t, c.podExists(t)) } +func TestDockerlessSuppressorsSurviveRepeatAndResume(t *testing.T) { + for _, disableDaemon := range []bool{false, true} { + name := "shutdown none" + if disableDaemon { + name = "current disable-daemon flag" + } + t.Run(name, func(t *testing.T) { + c := newFakeCluster(t) + c.k.agentConfig.ContainerTimeout = "1m" + c.k.disableDaemon = disableDaemon + options := topologyRunOptions(t, true) + action := config.ShutdownActionStopContainer + if !disableDaemon { + action = config.ShutdownActionNone + } + options.Env = daemonOptions(t, agentd.DaemonConfig{ + ShutdownAction: action, + }).Env + readyOnCreate(c, "initial") + ctx := context.Background() + + require.NoError(t, c.k.RunDevContainer(ctx, "ws", options)) + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + assertRunningTopology(t, c, options) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + require.NoError(t, c.k.StartDevContainer(ctx, "ws")) + assertRunningTopology(t, c, options) + + c.k.disableDaemon = false + c.k.SetLifecycleShutdownAction(config.ShutdownActionStopContainer) + c.cs.ClearActions() + err := c.k.RunDevContainer(ctx, "ws", nil) + require.ErrorContains(t, err, "dockerless") + assertOnlyReads(t, c) + }) + } +} + +func TestDockerlessRejectsNewPositivePolicyBeforeResumeMutation(t *testing.T) { + c := newFakeCluster(t) + options := topologyRunOptions(t, true) + readyOnCreate(c, "initial") + ctx := context.Background() + require.NoError(t, c.k.RunDevContainer(ctx, "ws", options)) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + c.k.agentConfig.ContainerTimeout = "1m" + c.cs.ClearActions() + + err := c.k.StartDevContainer(ctx, "ws") + + require.ErrorContains(t, err, "dockerless") + assertOnlyReads(t, c) +} + +func TestExternalImageMainStaysUnmanagedAcrossResume(t *testing.T) { + c := newFakeCluster(t) + options := runOptionsForWorkspace("") + options.ImageEntrypoint = true + options.Cmd = []string{"-c", `exec "$@"`, "-", "/app"} + options.Env = daemonOptions(t, agentd.DaemonConfig{ + ShutdownAction: config.ShutdownActionStopContainer, + }).Env + ctx := context.Background() + readyOnCreate(c, "external") + require.NoError(t, c.k.RunDevContainer(ctx, "ws", options)) + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + assertRunningTopology(t, c, options) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + require.NoError(t, c.k.StartDevContainer(ctx, "ws")) + assertRunningTopology(t, c, options) + c.k.agentConfig.ContainerTimeout = "1m" + c.cs.ClearActions() + + err := c.k.RunDevContainer(ctx, "ws", nil) + + require.ErrorContains(t, err, "overrideCommand") + assertOnlyReads(t, c) +} + +func TestLegacyDockerlessResumeDoesNotRequireGenerationMetadata(t *testing.T) { + options := topologyRunOptions(t, true) + options.Env = map[string]string{"DOCKERLESS": pkgconfig.BoolTrue} + pvc := workspacePVC(t) + pvc.Annotations["customer.example/annotation"] = preservedCustomerMetadata + raw, err := json.Marshal(&DevContainerInfo{WorkspaceID: "ws", Options: options}) + require.NoError(t, err) + pvc.Annotations[DevsyInfoAnnotation] = string(raw) + c := newFakeCluster(t, pvc) + c.putPod(t, c.pod(t, podSpec{uid: "legacy", phase: corev1.PodSucceeded})) + readyOnCreate(c, "resumed") + + require.NoError(t, c.k.RunDevContainer(context.Background(), "ws", nil)) + + assertRunningTopology(t, c, options) + kept, err := c.cs.CoreV1().PersistentVolumeClaims(testNamespace). + Get(context.Background(), testPodID, metav1.GetOptions{}) + require.NoError(t, err) + assertPreservedVolume(t, pvc, kept) +} + +func TestCurrentShutdownActionOverridesStoredNoneOnResume(t *testing.T) { + c := newFakeCluster(t) + c.k.agentConfig.ContainerTimeout = "1m" + options := runOptionsForWorkspace("") + options.Env = daemonOptions(t, agentd.DaemonConfig{ + ShutdownAction: config.ShutdownActionNone, + Lifecycle: agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon}, + }).Env + ctx := context.Background() + readyOnCreate(c, "disabled") + require.NoError(t, c.k.RunDevContainer(ctx, "ws", options)) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + c.k.SetLifecycleShutdownAction(config.ShutdownActionStopContainer) + + require.NoError(t, c.k.StartDevContainer(ctx, "ws")) + + secret, err := c.cs.CoreV1().Secrets(testNamespace). + Get(ctx, getDaemonSecretName(testPodID), metav1.GetOptions{}) + require.NoError(t, err) + cfg, err := agentd.DecodeDaemonConfig(string(secret.Data[DaemonConfigKey])) + require.NoError(t, err) + timeout, err := lifecycleTimeout(daemonOptions(t, *cfg)) + require.NoError(t, err) + assert.Equal(t, time.Minute, timeout) + assert.Equal(t, config.ShutdownActionStopContainer, cfg.ShutdownAction) +} + +func topologyRunOptions(t *testing.T, dockerless bool) *driver.RunOptions { + t.Helper() + options := runOptionsForWorkspace("") + options.Dockerless = dockerless + if dockerless { + options.Image = "example/dockerless:1" + options.Entrypoint = "/.dockerless/dockerless" + options.Cmd = []string{"start", "--wait"} + options.Env = daemonOptions(t, agentd.DaemonConfig{ + ShutdownAction: config.ShutdownActionStopContainer, + }).Env + } + return options +} + +func assertRunningTopology(t *testing.T, c *fakeCluster, options *driver.RunOptions) { + t.Helper() + ctx := context.Background() + details, err := c.k.FindDevContainer(ctx, "ws") + require.NoError(t, err) + assert.Equal(t, options.Dockerless, details.Dockerless) + assert.Equal(t, options.ImageEntrypoint, details.ImageEntrypoint) + pod, err := c.cs.CoreV1().Pods(testNamespace).Get(ctx, testPodID, metav1.GetOptions{}) + require.NoError(t, err) + var main *corev1.Container + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == DevContainerName { + main = &pod.Spec.Containers[i] + } + } + require.NotNil(t, main) + assert.Equal(t, options.Image, main.Image) + assert.Equal(t, []string{options.Entrypoint}, main.Command) + assert.Equal(t, options.Cmd, main.Args) + secret, err := c.cs.CoreV1().Secrets(testNamespace). + Get(ctx, getDaemonSecretName(testPodID), metav1.GetOptions{}) + require.NoError(t, err) + cfg, err := agentd.DecodeDaemonConfig(string(secret.Data[DaemonConfigKey])) + require.NoError(t, err) + assert.Equal(t, !options.Dockerless && !options.ImageEntrypoint, cfg.Lifecycle.Managed()) + assert.Empty(t, cfg.Timeout) +} + +func assertOnlyReads(t *testing.T, c *fakeCluster) { + t.Helper() + for _, action := range c.cs.Actions() { + assert.Equal(t, "get", action.GetVerb(), "policy rejection must not mutate cluster state") + } +} + func writeFile(path, content string) error { return os.WriteFile(path, []byte(content), 0o600) } diff --git a/pkg/driver/kubernetes/pod_topology.go b/pkg/driver/kubernetes/pod_topology.go new file mode 100644 index 0000000000..f43714f11e --- /dev/null +++ b/pkg/driver/kubernetes/pod_topology.go @@ -0,0 +1,165 @@ +package kubernetes + +import ( + "context" + "fmt" + "reflect" + "slices" + "strconv" + + "github.com/devsy-org/devsy/pkg/driver" + corev1 "k8s.io/api/core/v1" +) + +type podTopology struct { + dockerless bool + imageMain bool +} + +func decodePodTopology(pod *corev1.Pod) (podTopology, bool, error) { + rawDockerless, hasDockerless := pod.Annotations[DevsyDockerlessAnnotation] + rawImageMain, hasImageMain := pod.Annotations[DevsyImageEntrypointAnnotation] + if !hasDockerless && !hasImageMain { + return podTopology{}, false, nil + } + dockerless, err := strconv.ParseBool(rawDockerless) + if err != nil { + return podTopology{}, false, fmt.Errorf( + "pod %q has invalid Dockerless topology metadata", + pod.Name, + ) + } + imageMain, err := strconv.ParseBool(rawImageMain) + if err != nil { + return podTopology{}, false, fmt.Errorf( + "pod %q has invalid image-main topology metadata", + pod.Name, + ) + } + return podTopology{dockerless: dockerless, imageMain: imageMain}, true, nil +} + +func podLaunchDiffers(pod *corev1.Pod, options *driver.RunOptions) bool { + topology, known, err := decodePodTopology(pod) + if err == nil && known && + (topology.dockerless != options.Dockerless || topology.imageMain != options.ImageEntrypoint) { + return true + } + return podCommandDiffers(pod, options) +} + +func podCommandDiffers(pod *corev1.Pod, options *driver.RunOptions) bool { + var command []string + if options.Entrypoint != "" { + command = []string{options.Entrypoint} + } + for _, container := range pod.Spec.Containers { + if container.Name == DevContainerName { + return container.Image != options.Image || + !slices.Equal( + container.Command, + command, + ) || !slices.Equal(container.Args, options.Cmd) + } + } + return true +} + +func observedPodTopology(pod *corev1.Pod, options *driver.RunOptions) (bool, bool, error) { + if pod == nil { + return options.Dockerless, options.ImageEntrypoint, validateImageMainRecipe(options) + } + if err := verifyOwnedPod(pod, pod.Name, options.UID); err != nil { + return false, false, err + } + topology, known, err := decodePodTopology(pod) + if err != nil { + return false, false, err + } + if known { + return topology.dockerless, topology.imageMain, nil + } + if podCommandDiffers(pod, options) { + return false, false, fmt.Errorf( + "legacy pod %q does not match the saved launch recipe; run devsy workspace up --recreate", + pod.Name, + ) + } + return options.Dockerless, options.ImageEntrypoint, validateImageMainRecipe(options) +} + +func validateImageMainRecipe(options *driver.RunOptions) error { + // The legacy producer framed the script with three shell arguments, then + // appended the image main. Missing facts on that specific recipe are ambiguous. + if !options.Dockerless && !options.ImageEntrypoint && options.Entrypoint == "/bin/sh" && + len(options.Cmd) > 3 && options.Cmd[0] == "-c" && options.Cmd[2] == "-" { + return fmt.Errorf( + "saved shell recipe carries an external command without its image-main fact; run devsy workspace up --recreate", + ) + } + return nil +} + +func (k *KubernetesDriver) verifyDesiredPod( + ctx context.Context, + id string, + options *driver.RunOptions, +) error { + pvc, info, err := k.getDevContainerPvc(ctx, id) + if err != nil { + return err + } + if pvc == nil { + return fmt.Errorf("workspace volume %q disappeared during startup", id) + } + if err := verifyVolumeOwner(pvc, info, options.UID); err != nil { + return err + } + if !reflect.DeepEqual(info.Options, options) { + return fmt.Errorf( + "workspace %q desired run configuration changed during startup; retry the operation", + id, + ) + } + pod, err := k.getPod(ctx, id) + if err != nil { + return err + } + if pod == nil { + return errPodChanged + } + return verifyDesiredTopology(pod, options) +} + +func verifyDesiredTopology(pod *corev1.Pod, options *driver.RunOptions) error { + dockerless, imageMain, err := observedPodTopology(pod, options) + if err != nil { + return err + } + if dockerless != options.Dockerless || imageMain != options.ImageEntrypoint || + podCommandDiffers(pod, options) { + return fmt.Errorf( + "pod %q no longer matches the desired launch topology; run devsy workspace up --recreate", + pod.Name, + ) + } + return nil +} + +func (k *KubernetesDriver) verifyExistingPodOwner( + ctx context.Context, + id, workspaceUID string, +) error { + pod, err := k.getPod(ctx, id) + if err != nil { + return err + } + if pod == nil { + return nil + } + if err := verifyOwnedPod(pod, id, workspaceUID); err != nil { + return err + } + _, _, err = decodePodTopology(pod) + return err +} diff --git a/pkg/driver/kubernetes/pvc.go b/pkg/driver/kubernetes/pvc.go index 43c11718fb..50e04a74c7 100644 --- a/pkg/driver/kubernetes/pvc.go +++ b/pkg/driver/kubernetes/pvc.go @@ -5,11 +5,13 @@ import ( "encoding/json" "fmt" "maps" + "reflect" pkgconfig "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/driver" "github.com/devsy-org/devsy/pkg/log" corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -124,3 +126,79 @@ func (k *KubernetesDriver) getDevContainerInformation( return string(containerInfo), nil } + +func (k *KubernetesDriver) resolveRunOptions( + saved *DevContainerInfo, + options *driver.RunOptions, +) (*driver.RunOptions, error) { + if options == nil && saved != nil { + options = saved.Options + } + if options == nil { + return nil, fmt.Errorf("no options provided and no saved run configuration available") + } + resolved, err := k.refreshStoredLifecycle(options) + if err != nil { + return nil, err + } + if err := validateImageMainRecipe(resolved); err != nil { + return nil, err + } + if err := k.preflightTemplate(resolved); err != nil { + return nil, err + } + return resolved, nil +} + +// A single resource-version-guarded update publishes intent before pod/config changes. +func (k *KubernetesDriver) persistRunOptions( + ctx context.Context, + pvc *corev1.PersistentVolumeClaim, + saved *DevContainerInfo, + options *driver.RunOptions, +) error { + if err := verifyVolumeOwner(pvc, saved, options.UID); err != nil { + return err + } + if reflect.DeepEqual(saved.Options, options) { + return nil + } + next := *saved + next.Options = options + raw, err := json.Marshal(&next) + if err != nil { + return err + } + updated := pvc.DeepCopy() + updated.Annotations[DevsyInfoAnnotation] = string(raw) + _, err = k.client.Client().CoreV1().PersistentVolumeClaims(k.namespace). + Update(ctx, updated, metav1.UpdateOptions{}) + if kerrors.IsForbidden(err) { + return fmt.Errorf( + "persist workspace run configuration: Kubernetes permission update on "+ + "persistentvolumeclaims in namespace %q is required for changed run configuration: %w", + k.namespace, err, + ) + } + if err != nil { + return fmt.Errorf( + "persist workspace run configuration; no pod/config changes were made: %w", + err, + ) + } + return nil +} + +func verifyVolumeOwner( + pvc *corev1.PersistentVolumeClaim, + info *DevContainerInfo, + workspaceUID string, +) error { + if uid := pvc.Labels[DevsyWorkspaceUIDLabel]; uid != "" && uid != workspaceUID { + return fmt.Errorf("workspace volume %q belongs to a different workspace", pvc.Name) + } + if info.Options != nil && info.Options.UID != "" && info.Options.UID != workspaceUID { + return fmt.Errorf("workspace volume %q has run options for a different workspace", pvc.Name) + } + return nil +} diff --git a/pkg/driver/kubernetes/pvc_metadata_test.go b/pkg/driver/kubernetes/pvc_metadata_test.go new file mode 100644 index 0000000000..8868a3d591 --- /dev/null +++ b/pkg/driver/kubernetes/pvc_metadata_test.go @@ -0,0 +1,536 @@ +package kubernetes + +import ( + "context" + "encoding/json" + "errors" + "testing" + + pkgconfig "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/driver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + clienttesting "k8s.io/client-go/testing" +) + +const ( + parentMountRecreateOperation = "recreate" + parentMountReuseOperation = "reuse" +) + +var pvcResource = schema.GroupVersionResource{Version: "v1", Resource: "persistentvolumeclaims"} + +func canonicalTopologyOptions(t *testing.T, dockerless bool) *driver.RunOptions { + t.Helper() + options := topologyRunOptions(t, dockerless) + if !dockerless { + options.Env = daemonOptions(t, agentd.DaemonConfig{ + ShutdownAction: config.ShutdownActionStopContainer, + Lifecycle: agentd.LifecycleConfig{ + Owner: agentd.LifecycleOwnerManagedDaemon, SetupArmsTimer: true, + }, + }).Env + } + return options +} + +func savedRunVolume(t *testing.T, options *driver.RunOptions) *corev1.PersistentVolumeClaim { + t.Helper() + pvc := workspacePVC(t) + pvc.UID, pvc.ResourceVersion = "workspace-volume", "4" + pvc.Labels = map[string]string{ + DevsyWorkspaceUIDLabel: testWorkspace, + "customer.example/label": preservedCustomerMetadata, + } + pvc.Annotations["customer.example/annotation"] = preservedCustomerMetadata + pvc.Spec.VolumeName = "customer-volume" + pvc.Finalizers = []string{"kubernetes.io/pvc-protection"} + raw, err := json.Marshal(&DevContainerInfo{WorkspaceID: "ws", Options: options}) + require.NoError(t, err) + pvc.Annotations[DevsyInfoAnnotation] = string(raw) + return pvc +} + +func assertPreservedVolume(t *testing.T, original, current *corev1.PersistentVolumeClaim) { + t.Helper() + assert.Equal(t, original.UID, current.UID) + assert.Equal(t, original.Labels, current.Labels) + assert.Equal(t, original.Spec, current.Spec) + assert.Equal(t, original.Finalizers, current.Finalizers) + assert.Equal( + t, + original.Annotations["customer.example/annotation"], + current.Annotations["customer.example/annotation"], + ) +} + +func TestRecreatedTopologyControlsRepeatAndResume(t *testing.T) { + for _, dockerless := range []bool{false, true} { + name := "ordinary to dockerless" + if !dockerless { + name = "dockerless to ordinary" + } + t.Run(name, func(t *testing.T) { + initial := canonicalTopologyOptions(t, !dockerless) + volume := savedRunVolume(t, initial) + c := newFakeCluster(t, volume) + ctx := context.Background() + readyOnCreate(c, "initial") + require.NoError(t, c.k.RunDevContainer(ctx, "ws", initial)) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + next := canonicalTopologyOptions(t, dockerless) + require.NoError(t, c.k.RunDevContainer(ctx, "ws", next)) + assertRunningTopology(t, c, next) + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + assertRunningTopology(t, c, next) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + require.NoError(t, c.k.StartDevContainer(ctx, "ws")) + assertRunningTopology(t, c, next) + current, info, err := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, err) + assertPreservedVolume(t, volume, current) + assert.Equal(t, next, info.Options) + }) + } +} + +func TestUnchangedRunOptionsDoNotRequireVolumeUpdate(t *testing.T) { + options := canonicalTopologyOptions(t, false) + c := newFakeCluster(t, savedRunVolume(t, options)) + c.cs.PrependReactor( + "update", + "persistentvolumeclaims", + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, kerrors.NewForbidden( + pvcResource.GroupResource(), + testPodID, + errors.New("update denied"), + ) + }, + ) + readyOnCreate(c, "stable") + ctx := context.Background() + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + first, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + reused, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + assert.Equal(t, first.UID, reused.UID) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + require.NoError(t, c.k.StartDevContainer(ctx, "ws")) + resumed, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + assert.NotEqual(t, first.UID, resumed.UID) + assertRunningTopology(t, c, options) +} + +func TestVolumePublicationErrorsLeavePreviousRecordAndNoNewPod(t *testing.T) { + for _, cause := range []error{ + kerrors.NewForbidden(pvcResource.GroupResource(), testPodID, errors.New("update denied")), + kerrors.NewConflict(pvcResource.GroupResource(), testPodID, errors.New("concurrent owner")), + } { + t.Run(cause.Error(), func(t *testing.T) { + old := savedRunVolume(t, canonicalTopologyOptions(t, false)) + c := newFakeCluster(t, old) + c.cs.PrependReactor( + "update", + "persistentvolumeclaims", + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, cause + }, + ) + err := c.k.RunDevContainer( + context.Background(), + "ws", + canonicalTopologyOptions(t, true), + ) + require.ErrorIs(t, err, cause) + if kerrors.IsForbidden(cause) { + assert.ErrorContains(t, err, "permission update") + } + current, _, getErr := c.k.getDevContainerPvc(context.Background(), testPodID) + require.NoError(t, getErr) + assert.Equal(t, old, current) + for _, action := range c.cs.Actions() { + if action.GetVerb() != "get" { + assert.True( + t, + action.Matches("update", "persistentvolumeclaims"), + "no pod/config changes after failed publication: %v", + action, + ) + } + } + }) + } +} + +func TestFailedPodCreationRetainsNewDesiredOptionsForRetry(t *testing.T) { + old := savedRunVolume(t, canonicalTopologyOptions(t, false)) + c := newFakeCluster(t, old) + next := canonicalTopologyOptions(t, true) + readyOnCreate(c, "retried") + reject := true + c.cs.PrependReactor( + "create", + podResourceName, + func(clienttesting.Action) (bool, runtime.Object, error) { + if reject { + reject = false + return true, nil, errors.New("pod admission unavailable") + } + return false, nil, nil + }, + ) + ctx := context.Background() + require.ErrorContains(t, c.k.RunDevContainer(ctx, "ws", next), "pod admission unavailable") + current, info, err := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, err) + assertPreservedVolume(t, old, current) + assert.Equal(t, next, info.Options) + assert.False(t, c.podExists(t)) + c.cs.ClearActions() + + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + + assertRunningTopology(t, c, next) + for _, action := range c.cs.Actions() { + assert.False( + t, + action.Matches("update", "persistentvolumeclaims"), + "retry must use the published desired state", + ) + } +} + +func TestStaleVolumeSnapshotCannotOverwriteReplacedVolume(t *testing.T) { + old := savedRunVolume(t, canonicalTopologyOptions(t, false)) + c := newFakeCluster(t, old) + stale, info, err := c.k.getDevContainerPvc(context.Background(), testPodID) + require.NoError(t, err) + replacement := old.DeepCopy() + replacement.UID, replacement.ResourceVersion = "replacement-volume", "5" + require.NoError(t, c.cs.Tracker().Update(pvcResource, replacement, testNamespace)) + c.cs.PrependReactor( + "update", + "persistentvolumeclaims", + func(a clienttesting.Action) (bool, runtime.Object, error) { + requested := a.(clienttesting.UpdateAction).GetObject().(*corev1.PersistentVolumeClaim) + if requested.UID != replacement.UID || + requested.ResourceVersion != replacement.ResourceVersion { + return true, nil, kerrors.NewConflict( + pvcResource.GroupResource(), + testPodID, + errors.New("volume changed"), + ) + } + return false, nil, nil + }, + ) + + err = c.k.persistRunOptions( + context.Background(), + stale, + info, + canonicalTopologyOptions(t, true), + ) + + require.True(t, kerrors.IsConflict(err)) + current, _, err := c.k.getDevContainerPvc(context.Background(), testPodID) + require.NoError(t, err) + assert.Equal(t, replacement, current) +} + +func TestForeignWorkspaceVolumeIsNotUpdated(t *testing.T) { + old := savedRunVolume(t, canonicalTopologyOptions(t, false)) + old.Labels[DevsyWorkspaceUIDLabel] = "another-workspace" + c := newFakeCluster(t, old) + + err := c.k.RunDevContainer(context.Background(), "ws", canonicalTopologyOptions(t, true)) + + require.ErrorContains(t, err, "different workspace") + assertOnlyReads(t, c) +} + +func TestLivePodTopologyOverridesPendingDesiredRecord(t *testing.T) { + old := canonicalTopologyOptions(t, false) + c := newFakeCluster(t, savedRunVolume(t, old)) + ctx := context.Background() + readyOnCreate(c, "running") + require.NoError(t, c.k.RunDevContainer(ctx, "ws", old)) + volume, info, err := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, err) + next := canonicalTopologyOptions(t, true) + require.NoError(t, c.k.persistRunOptions(ctx, volume, info, next)) + + details, err := c.k.FindDevContainer(ctx, "ws") + + require.NoError(t, err) + assert.False(t, details.Dockerless, "the live pod is still the ordinary generation") + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + assertRunningTopology(t, c, next) +} + +func TestForeignPodPreventsVolumeAndConfigMutation(t *testing.T) { + c := newFakeCluster(t, savedRunVolume(t, canonicalTopologyOptions(t, false))) + c.putPod(t, c.pod(t, podSpec{uid: "foreign", phase: corev1.PodRunning, foreign: true})) + c.cs.ClearActions() + + err := c.k.RunDevContainer(context.Background(), "ws", canonicalTopologyOptions(t, true)) + + require.Error(t, err) + assertOnlyReads(t, c) +} + +func TestLegacyRunningDockerlessPodIsReusedWithoutNewAnnotations(t *testing.T) { + options := topologyRunOptions(t, true) + options.Env = map[string]string{"DOCKERLESS": pkgconfig.BoolTrue} + c := newFakeCluster(t, savedRunVolume(t, options)) + pod := c.pod(t, podSpec{uid: "legacy-running", phase: corev1.PodRunning}) + pod.Spec.Containers[0].Image = options.Image + pod.Spec.Containers[0].Command = []string{options.Entrypoint} + pod.Spec.Containers[0].Args = options.Cmd + c.putPod(t, pod) + ctx := context.Background() + + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + + current, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + assert.Equal(t, pod.UID, current.UID) + assertRunningTopology(t, c, options) +} + +func TestMalformedLiveTopologyFailsClosedWithoutDeletingWork(t *testing.T) { + options := canonicalTopologyOptions(t, false) + c := newFakeCluster(t, savedRunVolume(t, options)) + readyOnCreate(c, "protected") + ctx := context.Background() + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + pod, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + pod.Annotations[DevsyDockerlessAnnotation] = "not-a-boolean" + require.NoError(t, c.cs.Tracker().Update( + schema.GroupVersionResource{Version: "v1", Resource: podResourceName}, pod, testNamespace, + )) + + err = c.k.RunDevContainer(ctx, "ws", nil) + + require.Error(t, err) + current, getErr := c.k.getPod(ctx, testPodID) + require.NoError(t, getErr) + assert.Equal(t, pod.UID, current.UID) +} + +func TestPositiveUnsupportedTopologyDoesNotPublishDesiredState(t *testing.T) { + for _, dockerless := range []bool{false, true} { + name := "external image main" + if dockerless { + name = "dockerless" + } + t.Run(name, func(t *testing.T) { + volume := savedRunVolume(t, canonicalTopologyOptions(t, false)) + c := newFakeCluster(t, volume) + c.k.agentConfig.ContainerTimeout = "1m" + next := canonicalTopologyOptions(t, dockerless) + if !dockerless { + next.ImageEntrypoint = true + next.Cmd = []string{"-c", `exec "$@"`, "-", "/app"} + } + + err := c.k.RunDevContainer(context.Background(), "ws", next) + + require.Error(t, err) + assertOnlyReads(t, c) + current, _, getErr := c.k.getDevContainerPvc(context.Background(), testPodID) + require.NoError(t, getErr) + assert.Equal(t, volume, current) + }) + } +} + +func TestConcurrentUserMetadataConflictIsNotOverwritten(t *testing.T) { + volume := savedRunVolume(t, canonicalTopologyOptions(t, false)) + c := newFakeCluster(t, volume) + concurrent := volume.DeepCopy() + concurrent.ResourceVersion = "5" + concurrent.Labels["customer.example/new"] = "concurrent" + c.cs.PrependReactor( + "update", + "persistentvolumeclaims", + func(clienttesting.Action) (bool, runtime.Object, error) { + err := c.cs.Tracker().Update(pvcResource, concurrent, testNamespace) + if err != nil { + return true, nil, err + } + return true, nil, kerrors.NewConflict( + pvcResource.GroupResource(), + testPodID, + errors.New("concurrent metadata"), + ) + }, + ) + + err := c.k.RunDevContainer(context.Background(), "ws", canonicalTopologyOptions(t, true)) + + require.True(t, kerrors.IsConflict(err)) + current, _, err := c.k.getDevContainerPvc(context.Background(), testPodID) + require.NoError(t, err) + assert.Equal(t, concurrent, current) + assert.False(t, c.podExists(t)) +} + +func TestParentWorkspaceVolumeMountLifecycle(t *testing.T) { + for _, operation := range []string{"initial", parentMountReuseOperation, parentMountRecreateOperation, "resume"} { + t.Run(operation, func(t *testing.T) { + options := canonicalTopologyOptions(t, false) + c, volume := newParentMountLifecycleCluster(t, operation, options) + ctx := context.Background() + var err error + switch operation { + case "resume": + err = c.k.StartDevContainer(ctx, "ws") + case parentMountReuseOperation: + err = c.k.RunDevContainer(ctx, "ws", nil) + default: + err = c.k.RunDevContainer(ctx, "ws", options) + } + require.NoError(t, err) + assertParentMountedWorkspace(t, c, options.Image) + pod, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + if operation == parentMountReuseOperation { + assert.Equal(t, types.UID("existing"), pod.UID) + } else { + assert.Equal(t, types.UID("replacement"), pod.UID) + } + current, _, err := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, err) + if volume != nil { + assertPreservedVolume(t, volume, current) + } + }) + } +} + +func newParentMountLifecycleCluster( + t *testing.T, + operation string, + options *driver.RunOptions, +) (*fakeCluster, *corev1.PersistentVolumeClaim) { + t.Helper() + var volume *corev1.PersistentVolumeClaim + var c *fakeCluster + if operation == "initial" { + c = newFakeCluster(t) + } else { + volume = savedRunVolume(t, options) + c = newFakeCluster(t, volume) + } + c.k.options.WorkspaceVolumeMount = "/workspaces" + if operation == parentMountReuseOperation || operation == parentMountRecreateOperation { + pod := c.pod(t, podSpec{uid: "existing", phase: corev1.PodRunning}) + pod.Spec.Containers[0].Image = options.Image + pod.Spec.Containers[0].Command = []string{options.Entrypoint} + pod.Spec.Containers[0].Args = options.Cmd + pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ + {Name: DevContainerName, MountPath: "/workspaces", SubPath: "devsy/0"}, + } + c.putPod(t, pod) + } + readyOnCreate(c, "replacement") + if operation == parentMountRecreateOperation { + require.NoError(t, c.k.StopDevContainer(context.Background(), "ws")) + options.Image = "example/recreated:2" + } + return c, volume +} + +func assertParentMountedWorkspace(t *testing.T, c *fakeCluster, image string) { + t.Helper() + ctx := context.Background() + pod, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + var mounted *corev1.VolumeMount + for i := range pod.Spec.Containers { + container := &pod.Spec.Containers[i] + if container.Name != DevContainerName { + continue + } + assert.Equal(t, image, container.Image) + for j := range container.VolumeMounts { + if container.VolumeMounts[j].Name == DevContainerName { + mounted = &container.VolumeMounts[j] + } + } + } + require.NotNil(t, mounted) + assert.Equal(t, "/workspaces", mounted.MountPath) + assert.Equal(t, "devsy/0", mounted.SubPath) + _, info, err := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, err) + assert.Equal(t, "/workspaces/app", info.Options.WorkspaceMount.Target) + assert.Equal(t, image, info.Options.Image) +} + +func TestMalformedLiveTopologyRejectsProviderAndCommandDriftBeforeMutation(t *testing.T) { + for _, drift := range []string{"provider options", "command", "partial topology"} { + t.Run(drift, func(t *testing.T) { + options := canonicalTopologyOptions(t, false) + c := newFakeCluster(t, savedRunVolume(t, options)) + readyOnCreate(c, "protected") + ctx := context.Background() + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + pod, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + pod.Annotations[DevsyDockerlessAnnotation] = "not-a-boolean" + if drift == "partial topology" { + pod.Annotations[DevsyDockerlessAnnotation] = "false" + delete(pod.Annotations, DevsyImageEntrypointAnnotation) + } + require.NoError(t, c.cs.Tracker().Update( + schema.GroupVersionResource{ + Version: "v1", + Resource: podResourceName, + }, pod, testNamespace, + ), + ) + beforeVolume, _, err := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, err) + beforeSecret, err := c.cs.CoreV1().Secrets(testNamespace). + Get(ctx, getDaemonSecretName(testPodID), metav1.GetOptions{}) + require.NoError(t, err) + next := canonicalTopologyOptions(t, false) + if drift == "command" { + next.Cmd = []string{"-c", "exec sleep 3600"} + } else { + c.k.options.DiskSize = "2Gi" + } + c.cs.ClearActions() + + err = c.k.RunDevContainer(ctx, "ws", next) + + require.Error(t, err) + assertOnlyReads(t, c) + currentPod, getErr := c.k.getPod(ctx, testPodID) + require.NoError(t, getErr) + assert.Equal(t, pod, currentPod) + currentVolume, _, getErr := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, getErr) + assert.Equal(t, beforeVolume, currentVolume) + currentSecret, getErr := c.cs.CoreV1().Secrets(testNamespace). + Get(ctx, getDaemonSecretName(testPodID), metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Equal(t, beforeSecret, currentSecret) + }) + } +} diff --git a/pkg/driver/kubernetes/pvc_test.go b/pkg/driver/kubernetes/pvc_test.go index fbd1d20ab1..a1792bc8b4 100644 --- a/pkg/driver/kubernetes/pvc_test.go +++ b/pkg/driver/kubernetes/pvc_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/http/httptest" + "strconv" "strings" "sync" "testing" @@ -18,6 +19,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -100,6 +102,10 @@ func (m *mockPvcServer) dispatch( m.handlePost(w, r, encoder, decoder) return true } + if r.Method == http.MethodPut && len(parts) == 6 { + m.handlePut(w, r, encoder, decoder) + return true + } return false } @@ -150,6 +156,8 @@ func (m *mockPvcServer) handlePost( w.WriteHeader(http.StatusBadRequest) return } + pvc.UID = types.UID("pvc-" + pvc.Name) + pvc.ResourceVersion = "1" m.mu.Lock() m.pvcs[pvc.Name] = pvc m.mu.Unlock() @@ -158,6 +166,51 @@ func (m *mockPvcServer) handlePost( _ = encoder.Encode(pvc, w) } +func (m *mockPvcServer) handlePut( + w http.ResponseWriter, + r *http.Request, + encoder runtime.Encoder, + decoder runtime.Decoder, +) { + body, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + obj, _, err := decoder.Decode(body, nil, nil) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + pvc, ok := obj.(*corev1.PersistentVolumeClaim) + if !ok { + w.WriteHeader(http.StatusBadRequest) + return + } + m.mu.Lock() + defer m.mu.Unlock() + current := m.pvcs[pvc.Name] + w.Header().Set("Content-Type", runtime.ContentTypeJSON) + if current == nil || current.UID != pvc.UID || current.ResourceVersion != pvc.ResourceVersion { + w.WriteHeader(http.StatusConflict) + _ = encoder.Encode(&metav1.Status{ + TypeMeta: metav1.TypeMeta{Kind: "Status", APIVersion: "v1"}, + Status: metav1.StatusFailure, Reason: metav1.StatusReasonConflict, + Code: http.StatusConflict, + }, w) + return + } + version, err := strconv.Atoi(current.ResourceVersion) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + pvc.ResourceVersion = strconv.Itoa(version + 1) + m.pvcs[pvc.Name] = pvc + w.WriteHeader(http.StatusOK) + _ = encoder.Encode(pvc, w) +} + func newMockPvcDriver( t *testing.T, initialPvcs map[string]*corev1.PersistentVolumeClaim, @@ -167,7 +220,10 @@ func newMockPvcDriver( pvcs: make(map[string]*corev1.PersistentVolumeClaim), } for k, v := range initialPvcs { - mock.pvcs[k] = v.DeepCopy() + copy := v.DeepCopy() + copy.UID = types.UID("pvc-" + copy.Name) + copy.ResourceVersion = "1" + mock.pvcs[k] = copy } srv := httptest.NewServer(mock) t.Cleanup(srv.Close) @@ -219,9 +275,6 @@ func testEnsureMissingPvcWithoutOptions(t *testing.T) { if opts != nil { t.Errorf("opts = %v, want nil", opts) } - if !strings.Contains(err.Error(), "no options provided") { - t.Errorf("unexpected error message: %v", err) - } } func testEnsureMissingPvcWithOptions(t *testing.T) { @@ -230,16 +283,13 @@ func testEnsureMissingPvcWithOptions(t *testing.T) { UID: testWorkspaceID, Image: "golang:1.24", } - initialize, opts, err := k.ensureDevContainerPvc(context.Background(), testWorkspaceID, options) + initialize, _, err := k.ensureDevContainerPvc(context.Background(), testWorkspaceID, options) if err != nil { t.Fatalf("ensureDevContainerPvc: %v", err) } if !initialize { t.Errorf("initialize = false, want true") } - if opts != options { - t.Errorf("opts = %v, want %v", opts, options) - } mock.mu.Lock() created, exists := mock.pvcs[testWorkspaceID] mock.mu.Unlock() @@ -273,18 +323,6 @@ func runExistingPvcTest( return opts, err } -func testEnsureExistingPvcWithOptions(t *testing.T) { - savedOptions := &driver.RunOptions{UID: testWorkspaceID, Image: testSavedImage} - callerOptions := &driver.RunOptions{UID: testWorkspaceID, Image: "caller-image"} - opts, err := runExistingPvcTest(t, savedOptions, callerOptions) - if err != nil { - t.Fatalf("ensureDevContainerPvc: %v", err) - } - if opts != callerOptions { - t.Errorf("opts = %v, want caller options %v", opts, callerOptions) - } -} - func testEnsureExistingPvcWithoutCallerOptions(t *testing.T) { savedOptions := &driver.RunOptions{UID: testWorkspaceID, Image: testSavedImage} opts, err := runExistingPvcTest(t, savedOptions, nil) @@ -296,29 +334,11 @@ func testEnsureExistingPvcWithoutCallerOptions(t *testing.T) { } } -func testEnsureExistingPvcWithNilSavedOptions(t *testing.T) { - opts, err := runExistingPvcTest(t, nil, nil) - if err != nil { - t.Fatalf("ensureDevContainerPvc: %v", err) - } - if opts != nil { - t.Errorf("opts = %v, want nil", opts) - } -} - func TestEnsureDevContainerPvc(t *testing.T) { t.Run("missing PVC without options errors", testEnsureMissingPvcWithoutOptions) t.Run("missing PVC with options creates PVC and initializes", testEnsureMissingPvcWithOptions) - t.Run( - "existing PVC with options retains options and does not initialize", - testEnsureExistingPvcWithOptions, - ) t.Run( "existing PVC without caller options reuses saved options", testEnsureExistingPvcWithoutCallerOptions, ) - t.Run( - "existing PVC with nil saved options preserves nil options", - testEnsureExistingPvcWithNilSavedOptions, - ) } diff --git a/pkg/driver/kubernetes/run.go b/pkg/driver/kubernetes/run.go index 90d16e3e7d..29b19e977a 100644 --- a/pkg/driver/kubernetes/run.go +++ b/pkg/driver/kubernetes/run.go @@ -25,11 +25,13 @@ const ( DevContainerName = pkgconfig.BinaryName InitContainerName = pkgconfig.BinaryName + "-init" - DevsyCreatedLabel = pkgconfig.K8sCreatedLabel - DevsyWorkspaceLabel = pkgconfig.K8sWorkspaceLabel - DevsyWorkspaceUIDLabel = pkgconfig.K8sWorkspaceUIDLabel - DevsyInfoAnnotation = pkgconfig.K8sInfoAnnotation - DevsyLastAppliedAnnotation = pkgconfig.K8sLastAppliedAnnotation + DevsyCreatedLabel = pkgconfig.K8sCreatedLabel + DevsyWorkspaceLabel = pkgconfig.K8sWorkspaceLabel + DevsyWorkspaceUIDLabel = pkgconfig.K8sWorkspaceUIDLabel + DevsyInfoAnnotation = pkgconfig.K8sInfoAnnotation + DevsyLastAppliedAnnotation = pkgconfig.K8sLastAppliedAnnotation + DevsyDockerlessAnnotation = DevsyInfoAnnotation + "-dockerless" + DevsyImageEntrypointAnnotation = DevsyInfoAnnotation + "-image-entrypoint" ) var ExtraDevsyLabels = map[string]string{ @@ -49,19 +51,22 @@ func (k *KubernetesDriver) RunDevContainer( log.Debugf("Running devcontainer for workspace %q", workspaceId) workspaceId = getID(workspaceId) - if err := k.preflightTemplate(options); err != nil { - return err - } - if err := k.ensureNamespace(ctx); err != nil { + initialize, options, err := k.ensureDevContainerPvc(ctx, workspaceId, options) + if err != nil { return err } - initialize, options, err := k.ensureDevContainerPvc(ctx, workspaceId, options) - if err != nil { + if err := k.runContainer(ctx, workspaceId, options, initialize); err != nil { return err } + return k.verifyDesiredPod(ctx, workspaceId, options) +} - return k.runContainer(ctx, workspaceId, options, initialize) +func (k *KubernetesDriver) ValidateRunImageDevContainer( + params *driver.RunImageDevContainerParams, +) error { + _, err := k.resolveRunOptions(nil, params.Options) + return err } // preflightTemplate rejects an unusable pod template before any cluster object, @@ -98,12 +103,16 @@ func (k *KubernetesDriver) ensureDevContainerPvc( if err != nil { return false, nil, err } + options, err = k.resolveRunOptions(containerInfo, options) + if err != nil { + return false, nil, err + } + if err := k.verifyExistingPodOwner(ctx, workspaceId, options.UID); err != nil { + return false, nil, err + } if pvc == nil { - if options == nil { - return false, nil, fmt.Errorf( - "no options provided and no persistent volume claim found for workspace %q", - workspaceId, - ) + if err := k.ensureNamespace(ctx); err != nil { + return false, nil, err } if err := k.createPersistentVolumeClaim(ctx, workspaceId, options); err != nil { @@ -113,11 +122,8 @@ func (k *KubernetesDriver) ensureDevContainerPvc( return true, options, nil } - if options == nil && containerInfo != nil { - var err error - if options, err = k.refreshStoredLifecycle(containerInfo.Options); err != nil { - return false, nil, err - } + if err := k.persistRunOptions(ctx, pvc, containerInfo, options); err != nil { + return false, nil, err } return false, options, nil @@ -220,6 +226,11 @@ func (k *KubernetesDriver) buildPod( }); err != nil { return nil, err } + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[DevsyDockerlessAnnotation] = strconv.FormatBool(options.Dockerless) + pod.Annotations[DevsyImageEntrypointAnnotation] = strconv.FormatBool(options.ImageEntrypoint) return pod, nil } @@ -301,7 +312,9 @@ func (k *KubernetesDriver) resolveWorkspaceMount( mount.Target, ) default: - mount.Target = k.options.WorkspaceVolumeMount + podMount := *mount + podMount.Target = k.options.WorkspaceVolumeMount + mount = &podMount log.Debugf("Using workspace volume mount: %s", k.options.WorkspaceVolumeMount) } @@ -720,19 +733,7 @@ func (k *KubernetesDriver) StartDevContainer(ctx context.Context, workspaceId st log.Debugf("Starting devcontainer for workspace %q", workspaceId) defer log.Debugf("Done starting devcontainer for workspace %q", workspaceId) - workspaceId = getID(workspaceId) - _, containerInfo, err := k.getDevContainerPvc(ctx, workspaceId) - if err != nil { - return err - } else if containerInfo == nil { - return fmt.Errorf("persistent volume %q not found", workspaceId) - } - - options, err := k.refreshStoredLifecycle(containerInfo.Options) - if err != nil { - return err - } - return k.runContainer(ctx, workspaceId, options, false) + return k.RunDevContainer(ctx, workspaceId, nil) } func getID(workspaceID string) string { diff --git a/pkg/driver/types.go b/pkg/driver/types.go index 31398252c4..b961acc283 100644 --- a/pkg/driver/types.go +++ b/pkg/driver/types.go @@ -54,10 +54,11 @@ type WorkspaceChowner interface { RequiresWorkspaceChown() bool } -// ManagedLifecycleDriver is a capability interface for drivers whose container -// main process is the Devsy daemon, which then owns inactivity shutdown. +// ManagedLifecycleDriver identifies drivers that can run the container's main +// process as the Devsy daemon. Actual ownership also depends on the run topology. type ManagedLifecycleDriver interface { ManagedLifecycle() bool + SetLifecycleShutdownAction(shutdownAction string) } // ReprovisioningDriver is a capability interface for drivers that can reprovision @@ -184,6 +185,9 @@ type RunOptions struct { // Dockerless builds the developer filesystem inside the runner after creation. Dockerless bool `json:"dockerless,omitempty"` + // ImageEntrypoint is set from inspected image metadata, never reconstructed from argv. + ImageEntrypoint bool `json:"imageEntrypoint,omitempty"` + // AllowRecreate authorizes replacing an existing VM and discarding its root disk. AllowRecreate bool `json:"allowRecreate,omitempty"` diff --git a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx index 2dd7c389eb..02a00174cf 100644 --- a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx @@ -47,12 +47,18 @@ Set `INACTIVITY_TIMEOUT` on the Kubernetes provider, for example `1h`. The depre The timeout is off when it is empty or `0`, when `shutdownAction` is `none`, or when `devsy workspace up` runs with `--disable-daemon`. A negative or unparsable value is an error. `stopCompose` with a positive timeout is rejected on Kubernetes. +`--disable-daemon` applies only to that invocation; it is not saved as a permanent timeout exemption. A later invocation without the flag uses the current provider timeout and shutdown action again. + +The Kubernetes credentials Devsy uses on your machine need `update` permission on `persistentvolumeclaims` in the workspace namespace when the saved run configuration changes, including a recreation that changes the workspace image or Dockerless topology. Devsy atomically updates only its run-information annotation before changing daemon configuration or creating the replacement pod. Unchanged run options do not cause an update. Devsy does not change your RBAC roles automatically. A denied or conflicting update prevents those subsequent changes and retains the volume and its previous record. An explicit `--recreate` can already have stopped the old pod through the existing recreation flow; it does not promise that the old process remains running after an error. + When the workspace stops: - The pod object can stay visible in the `Succeeded` or `Failed` phase until the next `devsy workspace up` or an explicit `devsy workspace stop`. The next `up` deletes that finished pod and creates a new one that mounts the same persistent volume claim. No recreate flag is needed. - Files on the workspace volume persist. Files written to the container filesystem outside mounted volumes are lost, because the new pod starts from the image. - `devsy workspace stop` still deletes the pod through the Kubernetes API from your machine. The pod gets no Kubernetes credentials of its own. +When `WORKSPACE_VOLUME_MOUNT` selects a parent of the workspace folder, the pod mounts that parent while the saved logical workspace folder remains unchanged. Reuse and resume keep the same workspace subdirectory on the volume. + Pod layout rules Devsy enforces: - One regular `devsy` container, any number of normal init containers, and native sidecars (init containers with `restartPolicy: Always`). Native sidecars end with the pod. @@ -60,6 +66,10 @@ Pod layout rules Devsy enforces: - With a positive timeout, the Devsy daemon must be the container's main process. A `devcontainer.json` with `overrideCommand: false` and an image entrypoint or command is rejected. - Dockerless builds do not support a positive timeout. +The configured pod manifest template is checked against the merged shutdown action and the current invocation's `--disable-daemon` before an explicit `--recreate` stops the existing pod. Admission-webhook changes can only be checked after pod creation. + +At zero effective timeout, Dockerless and an external image main (`overrideCommand: false` with an image entrypoint or command) do not advertise a managed lifecycle owner. The ordinary default entrypoint still uses the managed daemon at zero, so setup and connection tracking retain the same owner contract. + The enforced restrictions above do not establish end-to-end support for every pod layout. Restricted non-root security, native sidecars/additional init containers, admission-webhook layouts, Devsy Platform routes, and upgrades from an earlier Devsy version require separate cluster validation. Test your layout on a non-production workspace before relying on the timeout. From d753560c1ab7ef418994f67a95154899b51eb9cc Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 27 Oct 2026 12:34:17 -0600 Subject: [PATCH 12/19] test(kubernetes): avoid shadowing builtin copy in pvc test --- pkg/driver/kubernetes/pvc_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/driver/kubernetes/pvc_test.go b/pkg/driver/kubernetes/pvc_test.go index a1792bc8b4..1ab9851bd0 100644 --- a/pkg/driver/kubernetes/pvc_test.go +++ b/pkg/driver/kubernetes/pvc_test.go @@ -220,10 +220,10 @@ func newMockPvcDriver( pvcs: make(map[string]*corev1.PersistentVolumeClaim), } for k, v := range initialPvcs { - copy := v.DeepCopy() - copy.UID = types.UID("pvc-" + copy.Name) - copy.ResourceVersion = "1" - mock.pvcs[k] = copy + pvc := v.DeepCopy() + pvc.UID = types.UID("pvc-" + pvc.Name) + pvc.ResourceVersion = "1" + mock.pvcs[k] = pvc } srv := httptest.NewServer(mock) t.Cleanup(srv.Close) From 06159cc1f46d3daaf4a971a48a9affc86305d710 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 27 Oct 2026 14:00:35 -0600 Subject: [PATCH 13/19] fix(kubernetes): keep volumes without saved options findable and clarify vanished-pod error --- pkg/driver/kubernetes/find.go | 24 ++++++++++++------- pkg/driver/kubernetes/pod_lifecycle_test.go | 26 +++++++++++++++++++++ pkg/driver/kubernetes/pod_topology.go | 6 ++++- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/pkg/driver/kubernetes/find.go b/pkg/driver/kubernetes/find.go index 61bec4c505..0f63328126 100644 --- a/pkg/driver/kubernetes/find.go +++ b/pkg/driver/kubernetes/find.go @@ -23,9 +23,6 @@ func (k *KubernetesDriver) FindDevContainer( } else if pvc == nil { return nil, nil } - if containerInfo.Options == nil { - return nil, fmt.Errorf("workspace volume %q has no saved run options", workspaceId) - } // check pod pod, err := k.getPod(ctx, pvc.Name) @@ -44,10 +41,21 @@ func (k *KubernetesDriver) FindDevContainer( if pod != nil { startedAt = pod.CreationTimestamp.String() } - dockerless, imageEntrypoint, topologyErr := observedPodTopology(pod, containerInfo.Options) - topologyError := "" - if topologyErr != nil { - topologyError = topologyErr.Error() + // A volume without saved options still has to be found so stop and delete can clean it up. + var ( + dockerless, imageEntrypoint bool + topologyError string + labels = map[string]string{} + ) + if containerInfo.Options == nil { + topologyError = fmt.Sprintf("workspace volume %q has no saved run options", workspaceId) + } else { + var topologyErr error + dockerless, imageEntrypoint, topologyErr = observedPodTopology(pod, containerInfo.Options) + if topologyErr != nil { + topologyError = topologyErr.Error() + } + labels = config.ListToObject(containerInfo.Options.Labels) } return &config.ContainerDetails{ @@ -61,7 +69,7 @@ func (k *KubernetesDriver) FindDevContainer( StartedAt: startedAt, }, Config: config.ContainerDetailsConfig{ - Labels: config.ListToObject(containerInfo.Options.Labels), + Labels: labels, }, }, nil } diff --git a/pkg/driver/kubernetes/pod_lifecycle_test.go b/pkg/driver/kubernetes/pod_lifecycle_test.go index 597d7bd8f5..7ce37c4d43 100644 --- a/pkg/driver/kubernetes/pod_lifecycle_test.go +++ b/pkg/driver/kubernetes/pod_lifecycle_test.go @@ -469,6 +469,32 @@ func TestFindDevContainerTreatsTerminalPodAsStopped(t *testing.T) { assert.Equal(t, config.ContainerStatusExited, details.State.Status) } +func TestFindDevContainerKeepsVolumeWithoutSavedOptionsFindable(t *testing.T) { + pvc := workspacePVC(t) + raw, err := json.Marshal(&DevContainerInfo{WorkspaceID: "ws"}) + require.NoError(t, err) + pvc.Annotations[DevsyInfoAnnotation] = string(raw) + c := newFakeCluster(t, pvc) + + details, err := c.k.FindDevContainer(context.Background(), "ws") + + require.NoError(t, err, "stop and delete must still find a volume with no saved options") + require.NotNil(t, details) + assert.Equal(t, config.ContainerStatusExited, details.State.Status) + assert.Contains(t, details.LifecycleTopologyError, "no saved run options") +} + +func TestVerifyDesiredPodReportsMissingPodWithoutInternalSentinel(t *testing.T) { + options := canonicalTopologyOptions(t, false) + c := newFakeCluster(t, savedRunVolume(t, options)) + + err := c.k.verifyDesiredPod(context.Background(), testPodID, options) + + require.Error(t, err) + assert.NotErrorIs(t, err, errPodChanged) + assert.Contains(t, err.Error(), "retry the operation") +} + func workspacePVC(t *testing.T) *corev1.PersistentVolumeClaim { t.Helper() info, err := json.Marshal( diff --git a/pkg/driver/kubernetes/pod_topology.go b/pkg/driver/kubernetes/pod_topology.go index f43714f11e..6584941416 100644 --- a/pkg/driver/kubernetes/pod_topology.go +++ b/pkg/driver/kubernetes/pod_topology.go @@ -126,7 +126,11 @@ func (k *KubernetesDriver) verifyDesiredPod( return err } if pod == nil { - return errPodChanged + return fmt.Errorf( + "pod for workspace %q is gone right after it started; it may have been deleted "+ + "or already stopped for inactivity; retry the operation", + id, + ) } return verifyDesiredTopology(pod, options) } From acee274f093f31954f7be8534abf406b6e781f92 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 27 Oct 2026 15:08:38 -0600 Subject: [PATCH 14/19] fix(kubernetes): confirm hook registration before arming and validate reused pods --- cmd/internal/agentcontainer/lifecycle.go | 101 +++++++++++++++++- cmd/internal/agentcontainer/lifecycle_test.go | 46 ++++++++ cmd/internal/agentcontainer/setup.go | 32 +++++- pkg/driver/kubernetes/pod_lifecycle.go | 14 ++- pkg/driver/kubernetes/pod_lifecycle_test.go | 33 ++++++ 5 files changed, 219 insertions(+), 7 deletions(-) diff --git a/cmd/internal/agentcontainer/lifecycle.go b/cmd/internal/agentcontainer/lifecycle.go index 87d0b1630a..37fae30aff 100644 --- a/cmd/internal/agentcontainer/lifecycle.go +++ b/cmd/internal/agentcontainer/lifecycle.go @@ -6,6 +6,10 @@ import ( "context" "errors" "fmt" + "os" + "os/exec" + "path/filepath" + "strings" "time" agentd "github.com/devsy-org/devsy/pkg/daemon/agent" @@ -16,6 +20,15 @@ import ( const ( // ownerWaitTimeout bounds managed-owner startup, even with no timer. ownerWaitTimeout = 30 * time.Second + + // envWorkerReadyFile names the file a hook worker creates once it has registered. + envWorkerReadyFile = "DEVSY_WORKER_READY_FILE" + // workerReadyTimeout bounds how long setup waits for a spawned worker to register. + workerReadyTimeout = 10 * time.Second + workerReadyPoll = 50 * time.Millisecond + workerReadyDirMode = 0o700 + + workerReadyDirPrefix = "devsy-worker-ready-" ) // lifecycleHandshake is setup's side of the managed daemon protocol. @@ -108,5 +121,91 @@ func (h *lifecycleHandshake) fail() { // startWorkerActivity completes managed registration before starting hooks. func startWorkerActivity(ctx context.Context) (context.Context, func(), error) { - return agentd.StartActivityWriter(ctx, agentd.DefaultActivityInterval) + ctx, stop, err := agentd.StartActivityWriter(ctx, agentd.DefaultActivityInterval) + if err == nil { + signalWorkerReady() + } + return ctx, stop, err +} + +// workerReadiness lets setup keep its own protection until the hook workers it +// spawned have registered theirs. Methods are no-ops on a nil receiver, which +// is how unmanaged containers skip the handshake. +type workerReadiness struct { + dir string + spawned []string +} + +func newWorkerReadiness() (*workerReadiness, error) { + dir, err := os.MkdirTemp("", workerReadyDirPrefix) + if err != nil { + return nil, fmt.Errorf("create worker readiness dir: %w", err) + } + if err := os.Chmod(dir, workerReadyDirMode); err != nil { + _ = os.RemoveAll(dir) + return nil, fmt.Errorf("secure worker readiness dir: %w", err) + } + return &workerReadiness{dir: dir}, nil +} + +// prepare tells a worker about to be spawned where to report registration. Call +// it only when the worker is really started, so wait never expects a skipped one. +func (w *workerReadiness) prepare(name string, cmd *exec.Cmd) { + if w == nil { + return + } + if cmd.Env == nil { + cmd.Env = os.Environ() + } + cmd.Env = append(cmd.Env, envWorkerReadyFile+"="+filepath.Join(w.dir, name)) + w.spawned = append(w.spawned, name) +} + +// wait blocks until every prepared worker has registered or the bound elapses. +func (w *workerReadiness) wait(ctx context.Context) error { + if w == nil { + return nil + } + ctx, cancel := context.WithTimeout(ctx, workerReadyTimeout) + defer cancel() + ticker := time.NewTicker(workerReadyPoll) + defer ticker.Stop() + for _, name := range w.spawned { + for { + if _, err := os.Stat(filepath.Join(w.dir, name)); err == nil { + break + } + select { + case <-ctx.Done(): + return fmt.Errorf("hook worker %s did not register: %w", name, context.Cause(ctx)) + case <-ticker.C: + } + } + } + return nil +} + +func (w *workerReadiness) close() { + if w != nil { + _ = os.RemoveAll(w.dir) + } +} + +// signalWorkerReady reports registration to the setup that spawned this worker. +func signalWorkerReady() { + path := os.Getenv(envWorkerReadyFile) + if path == "" { + return + } + // Hooks inherit this process's env and have no use for the path. + _ = os.Unsetenv(envWorkerReadyFile) + if !filepath.IsAbs(path) || filepath.Clean(path) != path || + !strings.HasPrefix(filepath.Base(filepath.Dir(path)), workerReadyDirPrefix) { + log.Debugf("ignoring unexpected worker readiness path %q", path) + return + } + // #nosec G703 -- validated above to sit directly inside setup's private readiness dir. + if err := os.WriteFile(path, nil, 0o600); err != nil { + log.Debugf("report worker registration: %v", err) + } } diff --git a/cmd/internal/agentcontainer/lifecycle_test.go b/cmd/internal/agentcontainer/lifecycle_test.go index a7d4685938..2c79b0f41c 100644 --- a/cmd/internal/agentcontainer/lifecycle_test.go +++ b/cmd/internal/agentcontainer/lifecycle_test.go @@ -12,6 +12,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "sync/atomic" "testing" "time" @@ -499,3 +500,48 @@ func TestManagedSSHServerContextChild(t *testing.T) { cmd := &SSHServerCmd{Address: "127.0.0.1:0", TrackActivity: true} assert.ErrorIs(t, cmd.Run(command, nil), context.Canceled) } + +func TestWorkerReadinessWaitsForSpawnedWorkerRegistration(t *testing.T) { + workers, err := newWorkerReadiness() + require.NoError(t, err) + defer workers.close() + cmd := &exec.Cmd{} + workers.prepare("post-attach", cmd) + var path string + for _, kv := range cmd.Env { + if v, ok := strings.CutPrefix(kv, envWorkerReadyFile+"="); ok { + path = v + } + } + require.NotEmpty(t, path, "the worker must be told where to report") + + shortCtx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + require.ErrorContains(t, workers.wait(shortCtx), "post-attach did not register") + + t.Setenv(envWorkerReadyFile, path) + signalWorkerReady() + + require.NoError(t, workers.wait(context.Background())) + assert.Empty(t, os.Getenv(envWorkerReadyFile), "hooks must not inherit the marker path") +} + +func TestWorkerReadinessIsNoopForUnmanagedContainers(t *testing.T) { + var workers *workerReadiness + cmd := &exec.Cmd{} + + workers.prepare("deferred-hooks", cmd) + + assert.Empty(t, cmd.Env) + require.NoError(t, workers.wait(context.Background())) + workers.close() +} + +func TestWorkerReadinessOnlyWaitsForWorkersThatWereSpawned(t *testing.T) { + workers, err := newWorkerReadiness() + require.NoError(t, err) + defer workers.close() + + require.NoError(t, workers.wait(context.Background()), + "a worker that was already running registers on its own and is not awaited") +} diff --git a/cmd/internal/agentcontainer/setup.go b/cmd/internal/agentcontainer/setup.go index 46503da60d..927fb9db37 100644 --- a/cmd/internal/agentcontainer/setup.go +++ b/cmd/internal/agentcontainer/setup.go @@ -339,22 +339,34 @@ func (cmd *SetupContainerCmd) setupPostAttach( return fmt.Errorf("re-serialize setup info: %w", err) } + var workers *workerReadiness + if state.lifecycle != nil { + workers, err = newWorkerReadiness() + if err != nil { + return err + } + defer workers.close() + } + if !deferred.Empty() { err = cmd.startDeferredHooks( resolvedSetupInfo, cmd.DotfilesRepo, cmd.DotfilesScript, - state.secretsEnv, + state.secretsEnv, workers, ) if err != nil { log.Errorf("failed to start deferred lifecycle hooks: %v", err) } } - if err := cmd.startPostAttachHooks(state); err != nil { + if err := cmd.startPostAttachHooks(state, workers); err != nil { log.Errorf("failed to start postAttachCommand: %v", err) } - // Arm last so the background workers already hold the workspace active. + // Arm last, once the spawned workers hold the workspace active themselves. if state.lifecycle != nil { + if err := workers.wait(state.lifecycle.ctx); err != nil { + log.Errorf("hook protection not confirmed before arming: %v", err) + } return state.lifecycle.ready() } return nil @@ -372,16 +384,22 @@ func compressSetupInfo(setupInfo *config.Result) (string, error) { func (cmd *SetupContainerCmd) startDeferredHooks( setupInfo, dotfilesRepo, dotfilesScript string, secretsEnv []string, + workers *workerReadiness, ) error { return command.StartBackgroundOnce("devsy.deferred-hooks", func() (*exec.Cmd, error) { log.Debugf("starting deferred lifecycle hooks as background process") - return buildDeferredHooksCmd( + execCmd, err := buildDeferredHooksCmd( setupInfo, cmd.Prebuild, dotfilesRepo, dotfilesScript, secretsEnv, ) + if err != nil { + return nil, err + } + workers.prepare("deferred-hooks", execCmd) + return execCmd, nil }) } @@ -558,7 +576,10 @@ func (cmd *SetupContainerCmd) startContainerDaemon( }) } -func (cmd *SetupContainerCmd) startPostAttachHooks(state *containerState) error { +func (cmd *SetupContainerCmd) startPostAttachHooks( + state *containerState, + workers *workerReadiness, +) error { if len(state.setupInfo.MergedConfig.PostAttachCommands) == 0 { return nil } @@ -579,6 +600,7 @@ func (cmd *SetupContainerCmd) startPostAttachHooks(state *containerState) error Args: append([]string{binaryPath}, args...), } execCmd.Env = secretsEnvOverride(state.secretsEnv) + workers.prepare("post-attach", execCmd) return execCmd, nil }) } diff --git a/pkg/driver/kubernetes/pod_lifecycle.go b/pkg/driver/kubernetes/pod_lifecycle.go index 7f2b17dc0a..2f493d39ed 100644 --- a/pkg/driver/kubernetes/pod_lifecycle.go +++ b/pkg/driver/kubernetes/pod_lifecycle.go @@ -178,7 +178,19 @@ func (k *KubernetesDriver) reconcileDesiredPod( if classifyPod(pod) == podRunning && podLaunchDiffers(pod, options) { return false, k.deleteObservedPod(ctx, pod.Name, pod.UID) } - return k.reconcileObservedPod(ctx, pod.Name, pod) + reused, err := k.reconcileObservedPod(ctx, pod.Name, pod) + if err != nil || !reused { + return reused, err + } + // A pod admitted while the timeout was off can hold sidecars that outlive the daemon. + timeout, err := lifecycleTimeout(options) + if err != nil { + return false, fmt.Errorf("read lifecycle policy: %w", err) + } + if err := validatePodTopology(pod, timeout); err != nil { + return false, err + } + return true, nil } func storedProviderOptions(pod *corev1.Pod) *provider2.ProviderKubernetesDriverConfig { diff --git a/pkg/driver/kubernetes/pod_lifecycle_test.go b/pkg/driver/kubernetes/pod_lifecycle_test.go index 7ce37c4d43..a374bb37e5 100644 --- a/pkg/driver/kubernetes/pod_lifecycle_test.go +++ b/pkg/driver/kubernetes/pod_lifecycle_test.go @@ -484,6 +484,39 @@ func TestFindDevContainerKeepsVolumeWithoutSavedOptionsFindable(t *testing.T) { assert.Contains(t, details.LifecycleTopologyError, "no saved run options") } +func TestReusedRunningPodWithExtraContainerIsRejectedWhenTimeoutEnabled(t *testing.T) { + managed := agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon} + c := newFakeCluster(t) + pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning}) + pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{Name: "istio-proxy"}) + c.putPod(t, pod) + + reused, err := c.k.reconcileDesiredPod( + context.Background(), pod, + daemonOptions(t, agentd.DaemonConfig{Timeout: "1m", Lifecycle: managed}), + ) + + require.ErrorContains(t, err, "istio-proxy") + assert.False(t, reused) + assert.True(t, c.podExists(t), "a rejected reuse must not delete the pod") +} + +func TestReusedRunningPodWithExtraContainerIsKeptWhenTimeoutDisabled(t *testing.T) { + managed := agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon} + c := newFakeCluster(t) + pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning}) + pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{Name: "istio-proxy"}) + c.putPod(t, pod) + + reused, err := c.k.reconcileDesiredPod( + context.Background(), pod, + daemonOptions(t, agentd.DaemonConfig{Lifecycle: managed}), + ) + + require.NoError(t, err) + assert.True(t, reused) +} + func TestVerifyDesiredPodReportsMissingPodWithoutInternalSentinel(t *testing.T) { options := canonicalTopologyOptions(t, false) c := newFakeCluster(t, savedRunVolume(t, options)) From 217cd9539fc297b0f32f3e914aabedc5407b0354 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 27 Oct 2026 16:03:41 -0600 Subject: [PATCH 15/19] fix(kubernetes): await hook workers only after a successful launch --- cmd/internal/agentcontainer/lifecycle.go | 29 +++++--- cmd/internal/agentcontainer/lifecycle_test.go | 53 +++++++++++---- cmd/internal/agentcontainer/setup.go | 66 ++++++++++--------- 3 files changed, 96 insertions(+), 52 deletions(-) diff --git a/cmd/internal/agentcontainer/lifecycle.go b/cmd/internal/agentcontainer/lifecycle.go index 37fae30aff..738a23e9a6 100644 --- a/cmd/internal/agentcontainer/lifecycle.go +++ b/cmd/internal/agentcontainer/lifecycle.go @@ -148,17 +148,26 @@ func newWorkerReadiness() (*workerReadiness, error) { return &workerReadiness{dir: dir}, nil } -// prepare tells a worker about to be spawned where to report registration. Call -// it only when the worker is really started, so wait never expects a skipped one. -func (w *workerReadiness) prepare(name string, cmd *exec.Cmd) { - if w == nil { - return - } - if cmd.Env == nil { - cmd.Env = os.Environ() +// launch runs start, which receives a function that tells the worker about to +// be spawned where to report registration. The worker is awaited only when +// start succeeds and actually built a command, so a failed or skipped launch +// cannot stall setup for a marker that will never arrive. +func (w *workerReadiness) launch(name string, start func(prepare func(*exec.Cmd)) error) error { + built := false + err := start(func(cmd *exec.Cmd) { + built = true + if w == nil { + return + } + if cmd.Env == nil { + cmd.Env = os.Environ() + } + cmd.Env = append(cmd.Env, envWorkerReadyFile+"="+filepath.Join(w.dir, name)) + }) + if err == nil && built && w != nil { + w.spawned = append(w.spawned, name) } - cmd.Env = append(cmd.Env, envWorkerReadyFile+"="+filepath.Join(w.dir, name)) - w.spawned = append(w.spawned, name) + return err } // wait blocks until every prepared worker has registered or the bound elapses. diff --git a/cmd/internal/agentcontainer/lifecycle_test.go b/cmd/internal/agentcontainer/lifecycle_test.go index 2c79b0f41c..f7ed8d99bc 100644 --- a/cmd/internal/agentcontainer/lifecycle_test.go +++ b/cmd/internal/agentcontainer/lifecycle_test.go @@ -501,19 +501,27 @@ func TestManagedSSHServerContextChild(t *testing.T) { assert.ErrorIs(t, cmd.Run(command, nil), context.Canceled) } +func readyPath(t *testing.T, cmd *exec.Cmd) string { + t.Helper() + for _, kv := range cmd.Env { + if v, ok := strings.CutPrefix(kv, envWorkerReadyFile+"="); ok { + return v + } + } + t.Fatal("the worker must be told where to report") + return "" +} + func TestWorkerReadinessWaitsForSpawnedWorkerRegistration(t *testing.T) { workers, err := newWorkerReadiness() require.NoError(t, err) defer workers.close() cmd := &exec.Cmd{} - workers.prepare("post-attach", cmd) - var path string - for _, kv := range cmd.Env { - if v, ok := strings.CutPrefix(kv, envWorkerReadyFile+"="); ok { - path = v - } - } - require.NotEmpty(t, path, "the worker must be told where to report") + require.NoError(t, workers.launch("post-attach", func(prepare func(*exec.Cmd)) error { + prepare(cmd) + return nil + })) + path := readyPath(t, cmd) shortCtx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) defer cancel() @@ -530,18 +538,41 @@ func TestWorkerReadinessIsNoopForUnmanagedContainers(t *testing.T) { var workers *workerReadiness cmd := &exec.Cmd{} - workers.prepare("deferred-hooks", cmd) + err := workers.launch("deferred-hooks", func(prepare func(*exec.Cmd)) error { + prepare(cmd) + return nil + }) + require.NoError(t, err) assert.Empty(t, cmd.Env) require.NoError(t, workers.wait(context.Background())) workers.close() } -func TestWorkerReadinessOnlyWaitsForWorkersThatWereSpawned(t *testing.T) { +func TestWorkerReadinessDoesNotAwaitFailedOrSkippedLaunches(t *testing.T) { workers, err := newWorkerReadiness() require.NoError(t, err) defer workers.close() + launchErr := errors.New("start process") + + err = workers.launch("deferred-hooks", func(prepare func(*exec.Cmd)) error { + prepare(&exec.Cmd{}) + return launchErr + }) + require.ErrorIs(t, err, launchErr) + require.NoError(t, workers.launch("post-attach", func(func(*exec.Cmd)) error { + return nil + }), "a worker that was already running registers on its own") require.NoError(t, workers.wait(context.Background()), - "a worker that was already running registers on its own and is not awaited") + "neither a failed nor a skipped launch may be awaited") +} + +func TestWorkerReadinessRejectsMarkerOutsideReadinessDir(t *testing.T) { + outside := filepath.Join(t.TempDir(), "ready") + t.Setenv(envWorkerReadyFile, outside) + + signalWorkerReady() + + assert.NoFileExists(t, outside) } diff --git a/cmd/internal/agentcontainer/setup.go b/cmd/internal/agentcontainer/setup.go index 927fb9db37..8138e442fd 100644 --- a/cmd/internal/agentcontainer/setup.go +++ b/cmd/internal/agentcontainer/setup.go @@ -386,20 +386,22 @@ func (cmd *SetupContainerCmd) startDeferredHooks( setupInfo, dotfilesRepo, dotfilesScript string, secretsEnv []string, workers *workerReadiness, ) error { - return command.StartBackgroundOnce("devsy.deferred-hooks", func() (*exec.Cmd, error) { - log.Debugf("starting deferred lifecycle hooks as background process") - execCmd, err := buildDeferredHooksCmd( - setupInfo, - cmd.Prebuild, - dotfilesRepo, - dotfilesScript, - secretsEnv, - ) - if err != nil { - return nil, err - } - workers.prepare("deferred-hooks", execCmd) - return execCmd, nil + return workers.launch("deferred-hooks", func(prepare func(*exec.Cmd)) error { + return command.StartBackgroundOnce("devsy.deferred-hooks", func() (*exec.Cmd, error) { + log.Debugf("starting deferred lifecycle hooks as background process") + execCmd, err := buildDeferredHooksCmd( + setupInfo, + cmd.Prebuild, + dotfilesRepo, + dotfilesScript, + secretsEnv, + ) + if err != nil { + return nil, err + } + prepare(execCmd) + return execCmd, nil + }) }) } @@ -584,24 +586,26 @@ func (cmd *SetupContainerCmd) startPostAttachHooks( return nil } - return command.StartBackground("devsy.post-attach", func() (*exec.Cmd, error) { - log.Debugf("starting postAttachCommand as background process") - binaryPath, err := os.Executable() - if err != nil { - return nil, err - } + return workers.launch("post-attach", func(prepare func(*exec.Cmd)) error { + return command.StartBackground("devsy.post-attach", func() (*exec.Cmd, error) { + log.Debugf("starting postAttachCommand as background process") + binaryPath, err := os.Executable() + if err != nil { + return nil, err + } - args := []string{ - cmdInternal, cmdAgent, cmdContainer, "post-attach", - names.Flag(names.SetupInfo), cmd.SetupInfo, - } - execCmd := &exec.Cmd{ - Path: binaryPath, - Args: append([]string{binaryPath}, args...), - } - execCmd.Env = secretsEnvOverride(state.secretsEnv) - workers.prepare("post-attach", execCmd) - return execCmd, nil + args := []string{ + cmdInternal, cmdAgent, cmdContainer, "post-attach", + names.Flag(names.SetupInfo), cmd.SetupInfo, + } + execCmd := &exec.Cmd{ + Path: binaryPath, + Args: append([]string{binaryPath}, args...), + } + execCmd.Env = secretsEnvOverride(state.secretsEnv) + prepare(execCmd) + return execCmd, nil + }) }) } From f44155ea6f9066c2d0feae13fb52e5ffbfd18cef Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 27 Oct 2026 17:51:53 -0600 Subject: [PATCH 16/19] fix(kubernetes): replace stale pending pods, compare options semantically, reject rewritten commands --- pkg/daemon/agent/container_lifecycle_test.go | 14 ++++- pkg/driver/kubernetes/pod_lifecycle.go | 32 +++++++++- pkg/driver/kubernetes/pod_lifecycle_test.go | 61 +++++++++++++++++++- pkg/driver/kubernetes/pod_topology.go | 16 ++++- pkg/driver/kubernetes/pvc.go | 3 +- pkg/driver/kubernetes/run.go | 2 +- 6 files changed, 116 insertions(+), 12 deletions(-) diff --git a/pkg/daemon/agent/container_lifecycle_test.go b/pkg/daemon/agent/container_lifecycle_test.go index aeaefacc39..bdeaf7e3b1 100644 --- a/pkg/daemon/agent/container_lifecycle_test.go +++ b/pkg/daemon/agent/container_lifecycle_test.go @@ -57,13 +57,23 @@ func TestResolveInactivityTimeout(t *testing.T) { } func TestResolveInactivityTimeoutRejectsBadConfiguration(t *testing.T) { + const malformed = "soon" tests := []struct { name string in TimeoutInputs wantErr string }{ - {"malformed", TimeoutInputs{Canonical: "soon"}, "agent.containerInactivityTimeout"}, - {"malformed legacy", TimeoutInputs{Legacy: "soon"}, "agent.kubernetes.inactivityTimeout"}, + {"malformed", TimeoutInputs{Canonical: malformed}, "agent.containerInactivityTimeout"}, + { + "malformed legacy", + TimeoutInputs{Legacy: malformed}, + "agent.kubernetes.inactivityTimeout", + }, + { + "malformed legacy beside canonical", + TimeoutInputs{Canonical: "10m", Legacy: malformed}, + "agent.kubernetes.inactivityTimeout", + }, {"negative", TimeoutInputs{Canonical: "-1m"}, "must not be negative"}, { "conflicting aliases", diff --git a/pkg/driver/kubernetes/pod_lifecycle.go b/pkg/driver/kubernetes/pod_lifecycle.go index 2f493d39ed..2965109c21 100644 --- a/pkg/driver/kubernetes/pod_lifecycle.go +++ b/pkg/driver/kubernetes/pod_lifecycle.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "maps" + "slices" "strings" "time" @@ -175,7 +176,9 @@ func (k *KubernetesDriver) reconcileDesiredPod( pod *corev1.Pod, options *driver.RunOptions, ) (bool, error) { - if classifyPod(pod) == podRunning && podLaunchDiffers(pod, options) { + // A pending pod has not started its containers, so replacing it loses nothing. + if state := classifyPod(pod); (state == podRunning || state == podPending) && + podLaunchDiffers(pod, options) { return false, k.deleteObservedPod(ctx, pod.Name, pod.UID) } reused, err := k.reconcileObservedPod(ctx, pod.Name, pod) @@ -377,14 +380,39 @@ func validateNoExtraContainers(pod *corev1.Pod, timeout time.Duration) error { return nil } +// launchCommandChanged reports whether admission rewrote what the devsy +// container executes, which could displace the managed daemon. +func launchCommandChanged(requested, admitted *corev1.Pod) bool { + want, got := devsyContainer(requested), devsyContainer(admitted) + if want == nil || got == nil { + return false + } + return !slices.Equal(want.Command, got.Command) || !slices.Equal(want.Args, got.Args) +} + +func devsyContainer(pod *corev1.Pod) *corev1.Container { + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == DevContainerName { + return &pod.Spec.Containers[i] + } + } + return nil +} + // rejectAdmittedPod removes a freshly created pod whose admitted topology // violates the stop contract. The workspace volume is kept. func (k *KubernetesDriver) rejectAdmittedPod( ctx context.Context, - created *corev1.Pod, + created, requested *corev1.Pod, timeout time.Duration, ) error { topologyErr := validatePodTopology(created, timeout) + if topologyErr == nil && timeout > 0 && launchCommandChanged(requested, created) { + topologyErr = fmt.Errorf( + "the %q container no longer starts the Devsy daemon as its main process", + DevContainerName, + ) + } if topologyErr == nil { return nil } diff --git a/pkg/driver/kubernetes/pod_lifecycle_test.go b/pkg/driver/kubernetes/pod_lifecycle_test.go index a374bb37e5..519838efa8 100644 --- a/pkg/driver/kubernetes/pod_lifecycle_test.go +++ b/pkg/driver/kubernetes/pod_lifecycle_test.go @@ -734,11 +734,12 @@ func TestAdmittedPodWithInjectedContainerIsRemovedAndVolumeKept(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: testPodID, Namespace: testNamespace}, } c := newFakeCluster(t, pvc) - admitted := c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending}) + requested := c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending}) + admitted := requested.DeepCopy() admitted.Spec.Containers = append(admitted.Spec.Containers, corev1.Container{Name: "injected"}) c.putPod(t, admitted) - err := c.k.rejectAdmittedPod(context.Background(), admitted, time.Minute) + err := c.k.rejectAdmittedPod(context.Background(), admitted, requested, time.Minute) require.Error(t, err) assert.Contains(t, err.Error(), "injected") @@ -753,10 +754,64 @@ func TestAdmittedPodAcceptedWhenTopologyHolds(t *testing.T) { pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending}) c.putPod(t, pod) - require.NoError(t, c.k.rejectAdmittedPod(context.Background(), pod, time.Minute)) + err := c.k.rejectAdmittedPod(context.Background(), pod, pod.DeepCopy(), time.Minute) + + require.NoError(t, err) assert.True(t, c.podExists(t)) } +func TestAdmittedPodWithRewrittenCommandIsRemovedWhenTimeoutEnabled(t *testing.T) { + requested := newFakeCluster(t).pod(t, podSpec{uid: "u1", phase: corev1.PodPending}) + requested.Spec.Containers[0].Command = []string{"/bin/sh"} + admitted := requested.DeepCopy() + admitted.Spec.Containers[0].Command = []string{"/webhook/entrypoint"} + + t.Run("enabled", func(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, admitted) + + err := c.k.rejectAdmittedPod(context.Background(), admitted, requested, time.Minute) + + require.ErrorContains(t, err, "Devsy daemon as its main process") + assert.False(t, c.podExists(t)) + }) + t.Run("disabled", func(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, admitted) + + require.NoError(t, c.k.rejectAdmittedPod(context.Background(), admitted, requested, 0)) + assert.True(t, c.podExists(t), "no stop contract is at stake without a timeout") + }) +} + +func TestPendingPodWithDifferentLaunchRecipeIsReplaced(t *testing.T) { + c := newFakeCluster(t) + pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending}) + pod.Spec.Containers[0].Image = "registry.example/old:1" + c.putPod(t, pod) + options := &driver.RunOptions{Image: "registry.example/new:2"} + + reused, err := c.k.reconcileDesiredPod(context.Background(), pod, options) + + require.NoError(t, err) + assert.False(t, reused, "the caller creates the requested pod") + assert.False(t, c.podExists(t), "a stale pending pod must not be waited on") +} + +func TestSameRunOptionsTreatsNilAndEmptyCollectionsAsEqual(t *testing.T) { + const image = "registry.example/app:1" + nilForm := &driver.RunOptions{Image: image} + emptyForm := &driver.RunOptions{ + Image: image, Cmd: []string{}, Env: map[string]string{}, Labels: []string{}, + } + + assert.True(t, sameRunOptions(nilForm, emptyForm)) + assert.False(t, sameRunOptions(nilForm, &driver.RunOptions{Image: "other"})) + assert.False(t, sameRunOptions(nilForm, &driver.RunOptions{Image: image, Cmd: []string{"x"}})) + assert.True(t, sameRunOptions(nil, nil)) + assert.False(t, sameRunOptions(nilForm, nil)) +} + func TestExplicitStopWaitsForPodToBeGone(t *testing.T) { c := newFakeCluster(t) c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning})) diff --git a/pkg/driver/kubernetes/pod_topology.go b/pkg/driver/kubernetes/pod_topology.go index 6584941416..e69f244d1f 100644 --- a/pkg/driver/kubernetes/pod_topology.go +++ b/pkg/driver/kubernetes/pod_topology.go @@ -1,9 +1,10 @@ package kubernetes import ( + "bytes" "context" + "encoding/json" "fmt" - "reflect" "slices" "strconv" @@ -100,6 +101,17 @@ func validateImageMainRecipe(options *driver.RunOptions) error { return nil } +// sameRunOptions compares the persisted JSON form, in which a nil and an empty +// collection are the same value. +func sameRunOptions(a, b *driver.RunOptions) bool { + if a == nil || b == nil { + return a == b + } + left, leftErr := json.Marshal(a) + right, rightErr := json.Marshal(b) + return leftErr == nil && rightErr == nil && bytes.Equal(left, right) +} + func (k *KubernetesDriver) verifyDesiredPod( ctx context.Context, id string, @@ -115,7 +127,7 @@ func (k *KubernetesDriver) verifyDesiredPod( if err := verifyVolumeOwner(pvc, info, options.UID); err != nil { return err } - if !reflect.DeepEqual(info.Options, options) { + if !sameRunOptions(info.Options, options) { return fmt.Errorf( "workspace %q desired run configuration changed during startup; retry the operation", id, diff --git a/pkg/driver/kubernetes/pvc.go b/pkg/driver/kubernetes/pvc.go index 50e04a74c7..b1122285a2 100644 --- a/pkg/driver/kubernetes/pvc.go +++ b/pkg/driver/kubernetes/pvc.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "maps" - "reflect" pkgconfig "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/driver" @@ -160,7 +159,7 @@ func (k *KubernetesDriver) persistRunOptions( if err := verifyVolumeOwner(pvc, saved, options.UID); err != nil { return err } - if reflect.DeepEqual(saved.Options, options) { + if sameRunOptions(saved.Options, options) { return nil } next := *saved diff --git a/pkg/driver/kubernetes/run.go b/pkg/driver/kubernetes/run.go index 29b19e977a..6833d17f8c 100644 --- a/pkg/driver/kubernetes/run.go +++ b/pkg/driver/kubernetes/run.go @@ -583,7 +583,7 @@ func (k *KubernetesDriver) runPod( if err != nil { return fmt.Errorf("create pod: %w", err) } - if err := k.rejectAdmittedPod(ctx, created, timeout); err != nil { + if err := k.rejectAdmittedPod(ctx, created, pod, timeout); err != nil { return err } From b15e12afa71825222099dcf5bfc1edb9268b48b1 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 27 Oct 2026 19:12:08 -0600 Subject: [PATCH 17/19] fix(daemon): drop stale duplicate acknowledgements instead of closing the connection --- pkg/daemon/agent/lifecycle_protocol.go | 19 +++++++++-- pkg/daemon/agent/lifecycle_protocol_test.go | 37 +++++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/pkg/daemon/agent/lifecycle_protocol.go b/pkg/daemon/agent/lifecycle_protocol.go index ba21190e37..b897f762cb 100644 --- a/pkg/daemon/agent/lifecycle_protocol.go +++ b/pkg/daemon/agent/lifecycle_protocol.go @@ -399,12 +399,27 @@ func (c *LifecycleClient) readAcks() { !c.validAck(ack) { return } + if !c.deliverAck(ack) { + return + } + } +} + +// deliverAck queues ack for the sender. Every resend makes the owner repeat the +// same ACK and nothing drains them between sends, so a full queue holds stale +// duplicates: drop the oldest instead of closing a live protected connection. +func (c *LifecycleClient) deliverAck(ack AckRecord) bool { + for { select { case c.acks <- ack: + return true case <-c.done: - return + return false default: - return + select { + case <-c.acks: + default: + } } } } diff --git a/pkg/daemon/agent/lifecycle_protocol_test.go b/pkg/daemon/agent/lifecycle_protocol_test.go index f6c18e9e73..8781a2936f 100644 --- a/pkg/daemon/agent/lifecycle_protocol_test.go +++ b/pkg/daemon/agent/lifecycle_protocol_test.go @@ -571,3 +571,40 @@ func TestOwnerActivityCannotConsumeReservedControlAdmission(t *testing.T) { time.Millisecond, ) } + +func TestDuplicateAcknowledgementsDoNotCloseALiveConnection(t *testing.T) { + owner, client := net.Pipe() + c := &LifecycleClient{ + conn: client, + owner: OwnerRecord{Generation: "generation"}, + clientID: "client", + acks: make(chan AckRecord, 8), + done: make(chan struct{}), + } + go c.readAcks() + defer func() { _ = c.Close() }() + defer func() { _ = owner.Close() }() + + raw, err := json.Marshal(AckRecord{ + Version: LifecycleProtocolVersion, Generation: "generation", + ClientID: "client", Sequence: 1, + }) + require.NoError(t, err) + raw = append(raw, '\n') + for range 4 * cap(c.acks) { + _, err := owner.Write(raw) + require.NoError(t, err) + } + + select { + case <-c.done: + t.Fatal("stale duplicate acknowledgements must not close a protected connection") + case <-time.After(200 * time.Millisecond): + } + select { + case ack := <-c.acks: + assert.Equal(t, uint64(1), ack.Sequence, "a matching acknowledgement is still delivered") + default: + t.Fatal("expected a buffered acknowledgement") + } +} From 79af40f1c9dde79ce9243761d0d2a0e881cebf88 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 27 Oct 2026 19:13:05 -0600 Subject: [PATCH 18/19] test(daemon): use distinct constants in the duplicate acknowledgement test --- pkg/daemon/agent/lifecycle_protocol_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/daemon/agent/lifecycle_protocol_test.go b/pkg/daemon/agent/lifecycle_protocol_test.go index 8781a2936f..200934194d 100644 --- a/pkg/daemon/agent/lifecycle_protocol_test.go +++ b/pkg/daemon/agent/lifecycle_protocol_test.go @@ -573,11 +573,15 @@ func TestOwnerActivityCannotConsumeReservedControlAdmission(t *testing.T) { } func TestDuplicateAcknowledgementsDoNotCloseALiveConnection(t *testing.T) { + const ( + generation = "ack-generation" + clientID = "ack-client" + ) owner, client := net.Pipe() c := &LifecycleClient{ conn: client, - owner: OwnerRecord{Generation: "generation"}, - clientID: "client", + owner: OwnerRecord{Generation: generation}, + clientID: clientID, acks: make(chan AckRecord, 8), done: make(chan struct{}), } @@ -586,8 +590,8 @@ func TestDuplicateAcknowledgementsDoNotCloseALiveConnection(t *testing.T) { defer func() { _ = owner.Close() }() raw, err := json.Marshal(AckRecord{ - Version: LifecycleProtocolVersion, Generation: "generation", - ClientID: "client", Sequence: 1, + Version: LifecycleProtocolVersion, Generation: generation, + ClientID: clientID, Sequence: 1, }) require.NoError(t, err) raw = append(raw, '\n') From 16167562d926d853cfe10231760ef88ed73affdd Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 27 Oct 2026 20:32:50 -0600 Subject: [PATCH 19/19] fix(ide): hold managed lifecycle activity for connected Fleet sessions --- cmd/internal/fleet_server.go | 91 +++++++++++++++++++++++-------- cmd/internal/fleet_server_test.go | 54 ++++++++++++++++++ 2 files changed, 123 insertions(+), 22 deletions(-) create mode 100644 cmd/internal/fleet_server_test.go diff --git a/cmd/internal/fleet_server.go b/cmd/internal/fleet_server.go index 3370f74f95..a82718c723 100644 --- a/cmd/internal/fleet_server.go +++ b/cmd/internal/fleet_server.go @@ -1,6 +1,7 @@ package cmdinternal import ( + "context" "os" "path/filepath" "regexp" @@ -8,9 +9,10 @@ import ( "time" "github.com/devsy-org/devsy/cmd/flags" - "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" cliflags "github.com/devsy-org/devsy/pkg/flags" "github.com/devsy-org/devsy/pkg/flags/names" + "github.com/devsy-org/devsy/pkg/log" "github.com/spf13/cobra" ) @@ -45,34 +47,60 @@ func NewFleetServerCmd(flags *flags.GlobalFlags) *cobra.Command { return fleetCmd } -// Run runs the command logic. +var ( + fleetFirstConnection = regexp.MustCompile(`.*Received authorization request.*`) + fleetConnStatus = regexp.MustCompile(`.*Notify.*`) +) + +// fleetActivity is the activity writer held for the life of one Fleet session. +type fleetActivity struct { + ctx context.Context + stop func() +} + +func (a *fleetActivity) held() bool { return a.stop != nil } + +// lost reports that the lifecycle owner dropped the registration. +func (a *fleetActivity) lost() bool { return a.held() && a.ctx.Err() != nil } + +func (a *fleetActivity) acquire(parent context.Context) error { + ctx, stop, err := agentd.StartActivityWriter(parent, agentd.DefaultActivityInterval) + if err != nil { + return err + } + a.ctx, a.stop = ctx, stop + return nil +} + +func (a *fleetActivity) release() { + if a.stop != nil { + a.stop() + a.ctx, a.stop = nil, nil + } +} + +// Run runs the command logic. While a Fleet client is connected it holds an +// activity writer, which registers with a managed lifecycle owner or keeps the +// legacy activity file fresh, and releases it when the session ends. func (c *FleetServerCmd) Run(cmd *cobra.Command, _ []string) error { logFile := filepath.Join(os.Getenv("HOME"), ".cache/JetBrains/Fleet/log/fleet.log") - firstConnection := regexp.MustCompile(`.*Received authorization request.*`) - connStatus := regexp.MustCompile(`.*Notify.*`) + + var activity fleetActivity + defer activity.release() for { select { case <-time.After(time.Second * 10): - - log, err := os.ReadFile(logFile) - if err != nil { - continue - } - - // check if we had at least one fleet client connection, before - // this point, we do not check for connected/disconnected strings - initialized := firstConnection.FindStringSubmatch(string(log)) - if len(initialized) == 0 { - continue + if activity.lost() { + activity.release() } - - connString := connStatus.FindAllStringSubmatch(string(log), -1) - - // if ouf last occurrence of notify if "Notify ID connected" - // we have an active session, so let's keep alive - if strings.Contains(connString[len(connString)-1][0], "is connected") { - touchActivityFile(config.ContainerActivityFile) + switch active := fleetSessionActive(logFile); { + case active && !activity.held(): + if err := activity.acquire(cmd.Context()); err != nil { + log.Errorf("register fleet session activity: %v", err) + } + case !active: + activity.release() } case <-cmd.Context().Done(): // context is done - either canceled or time is up for timeout @@ -80,3 +108,22 @@ func (c *FleetServerCmd) Run(cmd *cobra.Command, _ []string) error { } } } + +// fleetSessionActive reports whether the Fleet log shows a connected client. +// Before the first client connects, the connected/disconnected lines are ignored. +func fleetSessionActive(logFile string) bool { + // #nosec G304 G703 -- fixed Fleet log path under $HOME, not caller-controlled. + raw, err := os.ReadFile(logFile) + if err != nil { + return false + } + if len(fleetFirstConnection.FindStringSubmatch(string(raw))) == 0 { + return false + } + // if our last occurrence of notify is "Notify ID connected" we have an active session + statuses := fleetConnStatus.FindAllStringSubmatch(string(raw), -1) + if len(statuses) == 0 { + return false + } + return strings.Contains(statuses[len(statuses)-1][0], "is connected") +} diff --git a/cmd/internal/fleet_server_test.go b/cmd/internal/fleet_server_test.go new file mode 100644 index 0000000000..b4dedf0408 --- /dev/null +++ b/cmd/internal/fleet_server_test.go @@ -0,0 +1,54 @@ +package cmdinternal + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + fleetAuthorized = "12:00 Received authorization request from client\n" + fleetConnected = "12:01 Notify abc is connected\n" + fleetDisconnected = "12:02 Notify abc is disconnected\n" +) + +func writeFleetLog(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "fleet.log") + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + return path +} + +func TestFleetSessionActive(t *testing.T) { + tests := []struct { + name string + log string + want bool + }{ + {name: "connected after authorization", log: fleetAuthorized + fleetConnected, want: true}, + { + name: "disconnected last", + log: fleetAuthorized + fleetConnected + fleetDisconnected, + }, + { + name: "reconnected after a disconnect", + log: fleetAuthorized + fleetConnected + fleetDisconnected + fleetConnected, + want: true, + }, + {name: "connected line before any authorization is ignored", log: fleetConnected}, + {name: "authorized but no status line yet", log: fleetAuthorized}, + {name: "empty log", log: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, fleetSessionActive(writeFleetLog(t, tt.log))) + }) + } +} + +func TestFleetSessionActiveMissingLogIsInactive(t *testing.T) { + assert.False(t, fleetSessionActive(filepath.Join(t.TempDir(), "absent.log"))) +}