diff --git a/cmd/internal/agentcontainer/daemon.go b/cmd/internal/agentcontainer/daemon.go index 41d017153c..6f975bd0e6 100644 --- a/cmd/internal/agentcontainer/daemon.go +++ b/cmd/internal/agentcontainer/daemon.go @@ -2,14 +2,11 @@ package agentcontainer import ( "context" - "encoding/base64" - "encoding/json" "errors" "fmt" "os" "os/exec" "os/signal" - "strings" "syscall" "time" @@ -26,10 +23,7 @@ import ( "golang.org/x/sync/errgroup" ) -const ( - RootDir = config2.ContainerDataDir - DaemonConfigPath = "/var/run/secrets/" + config2.BinaryName + "/daemon_config" -) +const RootDir = config2.ContainerDataDir type DaemonCmd struct { Config *agentd.DaemonConfig @@ -77,16 +71,30 @@ func (cmd *DaemonCmd) Run(c *cobra.Command, args []string) error { ctx, stop := signal.NotifyContext(c.Context(), os.Interrupt, syscall.SIGTERM) g, ctx := errgroup.WithContext(ctx) - cmd.startDaemonTasks(ctx, g, timeoutDuration) + if err := cmd.startDaemonTasks(ctx, g, timeoutDuration); err != nil { + stop() + return err + } err = g.Wait() stop() // Restore default signal handling before exiting. - if err != nil { + os.Exit(daemonExitCode(err)) + return nil // Unreachable but needed. +} + +// daemonExitCode maps the task group result to a process exit code. Inactivity +// expiry is the normal way a managed container stops, not a failure. +func daemonExitCode(err error) int { + switch { + case err == nil: + return 0 + case errors.Is(err, agentd.ErrInactivityExpired): + log.Infof("inactivity timeout reached, daemon exiting") + return 0 + default: log.Errorf("daemon error: %v", err) - os.Exit(1) + return 1 } - os.Exit(0) - return nil // Unreachable but needed. } func (cmd *DaemonCmd) setupTimeout() (time.Duration, error) { @@ -111,10 +119,11 @@ func (cmd *DaemonCmd) startDaemonTasks( ctx context.Context, g *errgroup.Group, timeoutDuration time.Duration, -) { +) error { var tasksStarted bool - // Start process reaper. + // Start process reaper only when this process really is the init process of + // its PID namespace; otherwise it would steal other processes' children. if os.Getpid() == 1 { g.Go(func() error { agentd.RunProcessReaper(ctx) @@ -130,11 +139,14 @@ func (cmd *DaemonCmd) startDaemonTasks( }) } - // Start timeout monitor unless shutdownAction is "none". - if timeoutDuration > 0 && cmd.Config.ShutdownAction != config.ShutdownActionNone { + loop, err := cmd.newLifecycleLoop(timeoutDuration) + if err != nil { + return err + } + if loop != nil { tasksStarted = true g.Go(func() error { - return runTimeoutMonitor(ctx, timeoutDuration) + return loop.Run(ctx) }) } @@ -153,43 +165,56 @@ func (cmd *DaemonCmd) startDaemonTasks( return nil }) } + return nil +} + +// newLifecycleLoop builds the inactivity loop. A managed daemon starts unarmed +// and waits for setup; any other daemon with a timeout is armed immediately. +func (cmd *DaemonCmd) newLifecycleLoop(timeout time.Duration) (*agentd.LifecycleLoop, error) { + monitor := &agentd.LifecycleMonitor{} + loop := &agentd.LifecycleLoop{Monitor: monitor, ActivityPath: config2.ContainerActivityFile} + enabled := timeout > 0 && cmd.Config.ShutdownAction != config.ShutdownActionNone + + if !cmd.Config.Lifecycle.Managed() { + if !enabled { + return nil, nil + } + monitor.ArmNow(timeout, time.Now()) + return loop, nil + } + + owner, err := agentd.NewLifecycleOwner(config2.ContainerLifecycleDir) + if err != nil { + return nil, fmt.Errorf("publish container lifecycle owner: %w", err) + } + if err := sharedfile.EnsureMode(config2.ContainerActivityFile, 0o666); err != nil { + _ = owner.Close() + return nil, fmt.Errorf("ensure activity file: %w", err) + } + loop.Owner = owner + if enabled && !cmd.Config.Lifecycle.SetupArmsTimer { + monitor.ArmNow(timeout, time.Now()) + } + return loop, nil } // loadConfig loads the daemon configuration from base64-encoded JSON. // If a CLI-provided timeout exists, it will override the timeout in the config. func (cmd *DaemonCmd) loadConfig() error { - // check local file - encodedCfg := "" - configBytes, err := os.ReadFile(DaemonConfigPath) + cfg, err := agentd.LoadContainerDaemonConfig() if err != nil { - if errors.Is(err, os.ErrNotExist) { - // check environment variable - encodedCfg = os.Getenv(config2.EnvWorkspaceDaemonConfig) - } else { - return fmt.Errorf("get daemon config file %s: %w", DaemonConfigPath, err) - } - } else { - encodedCfg = string(configBytes) + return err } - - if strings.TrimSpace(encodedCfg) != "" { - decoded, err := base64.StdEncoding.DecodeString(encodedCfg) - if err != nil { - return fmt.Errorf("error decoding daemon config: %w", err) - } - var cfg agentd.DaemonConfig - if err = json.Unmarshal(decoded, &cfg); err != nil { - return fmt.Errorf("error unmarshalling daemon config: %w", err) - } - if cmd.Config.Timeout != "" { - cfg.Timeout = cmd.Config.Timeout - } - if cmd.Config.ShutdownAction != "" { - cfg.ShutdownAction = cmd.Config.ShutdownAction - } - cmd.Config = &cfg + if cfg == nil { + return nil } - + if cmd.Config.Timeout != "" { + cfg.Timeout = cmd.Config.Timeout + } + if cmd.Config.ShutdownAction != "" { + cfg.ShutdownAction = cmd.Config.ShutdownAction + } + cmd.Config = cfg return nil } @@ -205,29 +230,6 @@ func (cmd *DaemonCmd) shouldRunSsh() bool { return cmd.Config.Ssh.Workdir != "" || cmd.Config.Ssh.User != "" } -// runTimeoutMonitor monitors the activity file and signals an error if the timeout is exceeded. -func runTimeoutMonitor( - ctx context.Context, - duration time.Duration, -) error { - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return nil - case <-ticker.C: - stat, err := os.Stat(config2.ContainerActivityFile) - if err != nil { - continue - } - if !stat.ModTime().Add(duration).After(time.Now()) { - return errors.New("timeout reached, terminating daemon") - } - } - } -} - // runNetworkServer starts the network server. func runNetworkServer( ctx context.Context, @@ -260,6 +262,23 @@ func runNetworkServer( return nil } +// sshServerArgs builds the embedded SSH server command line. A managed daemon +// asks it to record connection activity so Platform and direct SSH routes keep +// the workspace alive. +func sshServerArgs(cfg *agentd.DaemonConfig) []string { + args := []string{cmdInternal, cmdAgent, cmdContainer, "ssh-server"} + if cfg.Ssh.Workdir != "" { + args = append(args, names.Flag(names.Workdir), cfg.Ssh.Workdir) + } + if cfg.Ssh.User != "" { + args = append(args, names.Flag(names.RemoteUser), cfg.Ssh.User) + } + if cfg.Lifecycle.Managed() { + args = append(args, names.Flag(names.TrackActivity)) + } + return args +} + // runSshServer starts the SSH server, sending SIGTERM on context cancellation // with a grace period before force-killing. func runSshServer(ctx context.Context, cmd *DaemonCmd) error { @@ -268,13 +287,7 @@ func runSshServer(ctx context.Context, cmd *DaemonCmd) error { return err } - args := []string{cmdInternal, cmdAgent, cmdContainer, "ssh-server"} - if cmd.Config.Ssh.Workdir != "" { - args = append(args, names.Flag(names.Workdir), cmd.Config.Ssh.Workdir) - } - if cmd.Config.Ssh.User != "" { - args = append(args, names.Flag(names.RemoteUser), cmd.Config.Ssh.User) - } + args := sshServerArgs(cmd.Config) sshCmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 sshCmd.Stdout = os.Stdout diff --git a/cmd/internal/agentcontainer/deferred_hooks.go b/cmd/internal/agentcontainer/deferred_hooks.go index 8e4c2f0952..77d8796b47 100644 --- a/cmd/internal/agentcontainer/deferred_hooks.go +++ b/cmd/internal/agentcontainer/deferred_hooks.go @@ -112,6 +112,12 @@ func (cmd *DeferredHooksCmd) Run(ctx context.Context) error { return err } + ctx, stopActivity, err := startWorkerActivity(ctx) + if err != nil { + return fmt.Errorf("register deferred-hook activity: %w", err) + } + defer stopActivity() + log.Debugf("running deferred lifecycle hooks") deferred, err := setup.RunPreAttachHooks(ctx, setupInfo, setup.PreAttachOptions{ Prebuild: cmd.Prebuild, diff --git a/cmd/internal/agentcontainer/lifecycle.go b/cmd/internal/agentcontainer/lifecycle.go new file mode 100644 index 0000000000..738a23e9a6 --- /dev/null +++ b/cmd/internal/agentcontainer/lifecycle.go @@ -0,0 +1,220 @@ +//go:build !windows + +package agentcontainer + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/devsy-org/devsy/pkg/log" + provider2 "github.com/devsy-org/devsy/pkg/provider" +) + +const ( + // ownerWaitTimeout bounds managed-owner startup, even with no timer. + ownerWaitTimeout = 30 * time.Second + + // envWorkerReadyFile names the file a hook worker creates once it has registered. + envWorkerReadyFile = "DEVSY_WORKER_READY_FILE" + // workerReadyTimeout bounds how long setup waits for a spawned worker to register. + workerReadyTimeout = 10 * time.Second + workerReadyPoll = 50 * time.Millisecond + workerReadyDirMode = 0o700 + + workerReadyDirPrefix = "devsy-worker-ready-" +) + +// lifecycleHandshake is setup's side of the managed daemon protocol. +type lifecycleHandshake struct { + client *agentd.LifecycleClient + ctx context.Context + cancel context.CancelCauseFunc + timeout string + shutdownAction string +} + +func isManagedLifecycle(info *provider2.ContainerWorkspaceInfo) bool { + return info.LifecycleOwner == provider2.ContainerLifecycleOwnerManagedDaemon +} + +// beginLifecycle requires an acknowledged managed owner before setup can run. +// Unmanaged containers do not participate in this protocol. +func beginLifecycle( + ctx context.Context, + dir string, + info *provider2.ContainerWorkspaceInfo, + shutdownAction string, +) (*lifecycleHandshake, error) { + if !isManagedLifecycle(info) { + return nil, nil + } + waitCtx, cancel := context.WithTimeout(ctx, ownerWaitTimeout) + defer cancel() + + client, err := agentd.ConnectLifecycle(waitCtx, dir) + if err != nil { + return nil, migrationError(err) + } + + workCtx, stop := client.ProtectedContext(ctx) + h := &lifecycleHandshake{ + client: client, ctx: workCtx, cancel: stop, + timeout: info.ContainerTimeout, shutdownAction: shutdownAction, + } + if err := h.send(agentd.LifecycleStateSetup); err != nil { + return nil, err + } + return h, nil +} + +func migrationError(err error) error { + return fmt.Errorf( + "this container's main process does not support the Devsy inactivity lifecycle (%w); "+ + "stop and start the workspace once (devsy workspace stop, then devsy workspace up) "+ + "to replace the pod. The workspace volume is kept; data written to the "+ + "container filesystem outside mounted volumes is not", err) +} + +func (h *lifecycleHandshake) send(state agentd.LifecycleState) error { + if err := h.client.Send(state, h.timeout, h.shutdownAction); err != nil { + h.cancel(err) + _ = h.client.Close() + return fmt.Errorf("update container lifecycle (%s): %w", state, err) + } + return nil +} + +func (h *lifecycleHandshake) close() { + h.cancel(context.Canceled) + _ = h.client.Close() +} + +func (h *lifecycleHandshake) finish(err error) error { + if err == nil { + return nil + } + h.fail() + if cause := context.Cause(h.ctx); errors.Is(cause, agentd.ErrLifecycleOwnerMissing) { + return migrationError(cause) + } + return err +} + +// ready resets activity and arms the timer. +func (h *lifecycleHandshake) ready() error { + return h.send(agentd.LifecycleStateReady) +} + +// fail releases the suspension so a failed setup cannot leave the timer stuck. +func (h *lifecycleHandshake) fail() { + if err := h.send(agentd.LifecycleStateFailed); err != nil { + log.Errorf("release container lifecycle suspension: %v", err) + } +} + +// startWorkerActivity completes managed registration before starting hooks. +func startWorkerActivity(ctx context.Context) (context.Context, func(), error) { + ctx, stop, err := agentd.StartActivityWriter(ctx, agentd.DefaultActivityInterval) + if err == nil { + signalWorkerReady() + } + return ctx, stop, err +} + +// workerReadiness lets setup keep its own protection until the hook workers it +// spawned have registered theirs. Methods are no-ops on a nil receiver, which +// is how unmanaged containers skip the handshake. +type workerReadiness struct { + dir string + spawned []string +} + +func newWorkerReadiness() (*workerReadiness, error) { + dir, err := os.MkdirTemp("", workerReadyDirPrefix) + if err != nil { + return nil, fmt.Errorf("create worker readiness dir: %w", err) + } + if err := os.Chmod(dir, workerReadyDirMode); err != nil { + _ = os.RemoveAll(dir) + return nil, fmt.Errorf("secure worker readiness dir: %w", err) + } + return &workerReadiness{dir: dir}, nil +} + +// launch runs start, which receives a function that tells the worker about to +// be spawned where to report registration. The worker is awaited only when +// start succeeds and actually built a command, so a failed or skipped launch +// cannot stall setup for a marker that will never arrive. +func (w *workerReadiness) launch(name string, start func(prepare func(*exec.Cmd)) error) error { + built := false + err := start(func(cmd *exec.Cmd) { + built = true + if w == nil { + return + } + if cmd.Env == nil { + cmd.Env = os.Environ() + } + cmd.Env = append(cmd.Env, envWorkerReadyFile+"="+filepath.Join(w.dir, name)) + }) + if err == nil && built && w != nil { + w.spawned = append(w.spawned, name) + } + return err +} + +// wait blocks until every prepared worker has registered or the bound elapses. +func (w *workerReadiness) wait(ctx context.Context) error { + if w == nil { + return nil + } + ctx, cancel := context.WithTimeout(ctx, workerReadyTimeout) + defer cancel() + ticker := time.NewTicker(workerReadyPoll) + defer ticker.Stop() + for _, name := range w.spawned { + for { + if _, err := os.Stat(filepath.Join(w.dir, name)); err == nil { + break + } + select { + case <-ctx.Done(): + return fmt.Errorf("hook worker %s did not register: %w", name, context.Cause(ctx)) + case <-ticker.C: + } + } + } + return nil +} + +func (w *workerReadiness) close() { + if w != nil { + _ = os.RemoveAll(w.dir) + } +} + +// signalWorkerReady reports registration to the setup that spawned this worker. +func signalWorkerReady() { + path := os.Getenv(envWorkerReadyFile) + if path == "" { + return + } + // Hooks inherit this process's env and have no use for the path. + _ = os.Unsetenv(envWorkerReadyFile) + if !filepath.IsAbs(path) || filepath.Clean(path) != path || + !strings.HasPrefix(filepath.Base(filepath.Dir(path)), workerReadyDirPrefix) { + log.Debugf("ignoring unexpected worker readiness path %q", path) + return + } + // #nosec G703 -- validated above to sit directly inside setup's private readiness dir. + if err := os.WriteFile(path, nil, 0o600); err != nil { + log.Debugf("report worker registration: %v", err) + } +} diff --git a/cmd/internal/agentcontainer/lifecycle_test.go b/cmd/internal/agentcontainer/lifecycle_test.go new file mode 100644 index 0000000000..f7ed8d99bc --- /dev/null +++ b/cmd/internal/agentcontainer/lifecycle_test.go @@ -0,0 +1,578 @@ +//go:build !windows + +package agentcontainer + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "errors" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + provider2 "github.com/devsy-org/devsy/pkg/provider" + devsyssh "github.com/devsy-org/ssh" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gossh "golang.org/x/crypto/ssh" +) + +const testRoot = "root" + +func managedInfo(timeout string) *provider2.ContainerWorkspaceInfo { + return &provider2.ContainerWorkspaceInfo{ + ContainerTimeout: timeout, + LifecycleOwner: provider2.ContainerLifecycleOwnerManagedDaemon, + } +} + +func publishOwner(t *testing.T) (*agentd.LifecycleOwner, string) { + t.Helper() + dir, err := os.MkdirTemp("/tmp", "devsy-lifecycle-cmd-") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + owner, err := agentd.NewLifecycleOwner(dir) + require.NoError(t, err) + t.Cleanup(func() { _ = owner.Close() }) + return owner, dir +} + +func runLifecycleOwner( + t *testing.T, + owner *agentd.LifecycleOwner, + dir string, +) *agentd.LifecycleMonitor { + t.Helper() + monitor := &agentd.LifecycleMonitor{} + loop := &agentd.LifecycleLoop{ + Owner: owner, Monitor: monitor, ActivityPath: filepath.Join(dir, "activity"), + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- loop.Run(ctx) }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + assert.ErrorIs(t, err, agentd.ErrLifecycleOwnerMissing) + } + }) + return monitor +} + +func TestBeginLifecycleSuspendsThenReadyArmsSameOwner(t *testing.T) { + owner, dir := publishOwner(t) + monitor := runLifecycleOwner(t, owner, dir) + monitor.ArmNow(time.Minute, time.Now()) + + h, err := beginLifecycle( + context.Background(), dir, managedInfo("1m"), config.ShutdownActionStopContainer, + ) + require.NoError(t, err) + require.NotNil(t, h) + defer h.close() + assert.False(t, monitor.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil).Expired, + "setup is suspended before beginLifecycle returns") + + require.NoError(t, h.ready()) + + assert.True(t, monitor.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil).Expired, + "ready has re-armed the timer before returning") +} + +func TestLifecycleFailureReleasesSuspension(t *testing.T) { + owner, dir := publishOwner(t) + monitor := runLifecycleOwner(t, owner, dir) + h, err := beginLifecycle(context.Background(), dir, managedInfo("1m"), "") + require.NoError(t, err) + defer h.close() + assert.False(t, monitor.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil).Expired) + + h.fail() + + assert.True(t, monitor.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil).Expired, + "failed setup releases the suspension") +} + +func TestBeginLifecycleMissingOwnerWithTimeoutIsMigrationError(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + h, err := beginLifecycle(ctx, filepath.Join(t.TempDir(), "absent"), managedInfo("1m"), "") + + assert.Nil(t, h) + require.Error(t, err) + assert.ErrorIs(t, err, agentd.ErrLifecycleOwnerMissing) +} + +func TestBeginLifecycleRequiresManagedOwnerEvenWithoutTimeout(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + h, err := beginLifecycle(ctx, filepath.Join(t.TempDir(), "absent"), managedInfo(""), "") + + assert.ErrorIs(t, err, agentd.ErrLifecycleOwnerMissing) + assert.Nil(t, h) +} + +func TestBeginLifecycleIgnoresUnmanagedContainers(t *testing.T) { + h, err := beginLifecycle( + context.Background(), + filepath.Join(t.TempDir(), "absent"), + &provider2.ContainerWorkspaceInfo{ContainerTimeout: "1m"}, + "", + ) + + assert.NoError(t, err) + assert.Nil(t, h) +} + +func TestStartContainerDaemonNeverSpawnsSecondOwnerForManagedContainers(t *testing.T) { + cmd := &SetupContainerCmd{} + + err := cmd.startContainerDaemon(managedInfo("1m"), config.ShutdownActionStopContainer) + + assert.NoError(t, err) +} + +func TestDaemonExitCode(t *testing.T) { + assert.Equal(t, 0, daemonExitCode(nil)) + assert.Equal(t, 0, daemonExitCode(agentd.ErrInactivityExpired), "expiry is a normal shutdown") + assert.Equal(t, 0, daemonExitCode(errors.Join(agentd.ErrInactivityExpired))) + assert.Equal(t, 1, daemonExitCode(errors.New("SSH server exited abnormally"))) +} + +func TestSshServerArgs(t *testing.T) { + timeoutOnly := &agentd.DaemonConfig{ + Timeout: "1m", + Lifecycle: agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon}, + } + platform := &agentd.DaemonConfig{ + Ssh: agentd.SshConfig{Workdir: "/w", User: testRoot}, + Lifecycle: agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon}, + } + + cmd := &DaemonCmd{Config: timeoutOnly} + assert.False(t, cmd.shouldRunSsh(), "a timeout-only config must not start an SSH server") + assert.False(t, cmd.shouldRunNetworkServer()) + + assert.Equal(t, + []string{ + "internal", "agent", "container", "ssh-server", + "--workdir", "/w", "--remote-user", testRoot, "--track-activity", + }, + sshServerArgs(platform), + "managed Platform routes record connection activity", + ) + assert.NotContains( + t, + sshServerArgs(&agentd.DaemonConfig{Ssh: platform.Ssh}), + "--track-activity", + ) +} + +func TestLoadConfigCLITimeoutOverridesStoredConfig(t *testing.T) { + stored, err := agentd.EncodeDaemonConfig( + &agentd.DaemonConfig{Timeout: "5m", ShutdownAction: "none"}, + ) + require.NoError(t, err) + t.Setenv("DEVSY_WORKSPACE_DAEMON_CONFIG", stored) + cmd := &DaemonCmd{Config: &agentd.DaemonConfig{Timeout: "30s"}} + + require.NoError(t, cmd.loadConfig()) + + assert.Equal(t, "30s", cmd.Config.Timeout) + assert.Equal(t, "none", cmd.Config.ShutdownAction) +} + +func TestManagedLoopStartsUnarmedUntilSetup(t *testing.T) { + // The runtime dir is fixed in production, so only the unmanaged branch is + // observable without root; managed arming is covered by the monitor tests. + cmd := &DaemonCmd{Config: &agentd.DaemonConfig{}} + + loop, err := cmd.newLifecycleLoop(0) + assert.NoError(t, err) + assert.Nil(t, loop, "no timeout and no managed owner means no loop") + + loop, err = cmd.newLifecycleLoop(time.Minute) + require.NoError(t, err) + require.NotNil(t, loop) + assert.Nil(t, loop.Owner) +} + +type countingStarter struct{ started, stopped atomic.Int32 } + +func (c *countingStarter) start() (context.Context, func(), error) { + c.started.Add(1) + ctx, cancel := context.WithCancel(context.Background()) + return ctx, func() { cancel(); c.stopped.Add(1) }, nil +} + +func TestTrackConnectionsCountsConnectionsNotTheListener(t *testing.T) { + var starter countingStarter + inner, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + l := trackConnections(inner, starter.start) + defer func() { _ = l.Close() }() + + assert.Zero(t, starter.started.Load(), "an idle listener is not activity") + + accepted := make(chan net.Conn, 1) + go func() { + c, _ := l.Accept() + accepted <- c + }() + client, err := net.Dial("tcp", inner.Addr().String()) + require.NoError(t, err) + server := <-accepted + require.Eventually(t, func() bool { return starter.started.Load() == 1 }, + time.Second, time.Millisecond) + assert.Zero(t, starter.stopped.Load(), "activity continues while connected") + + _ = client.Close() + _, _ = io.Copy(io.Discard, server) + require.NoError(t, server.Close()) + _ = server.Close() + + assert.Equal(t, int32(1), starter.stopped.Load(), "disconnect ends the activity writer once") +} + +func TestSetupConnectionLossCancelsProtectedWork(t *testing.T) { + owner, dir := publishOwner(t) + monitor := runLifecycleOwner(t, owner, dir) + h, err := beginLifecycle(context.Background(), dir, managedInfo("1m"), "") + require.NoError(t, err) + defer h.close() + + require.NoError(t, owner.Close()) + + select { + case <-h.ctx.Done(): + assert.ErrorIs(t, context.Cause(h.ctx), agentd.ErrLifecycleOwnerMissing) + case <-time.After(time.Second): + t.Fatal("setup continued after its protection connection was lost") + } + require.Eventually(t, func() bool { + return monitor.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil).Expired + }, time.Second, time.Millisecond) +} + +func TestBeginLifecycleRequiresAcknowledgementBeforeReturningSetup(t *testing.T) { + _, dir := publishOwner(t) + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + h, err := beginLifecycle(ctx, dir, managedInfo("1m"), "") + + assert.ErrorIs(t, err, agentd.ErrLifecycleOwnerMissing) + assert.Nil(t, h) +} + +func TestTrackingLossClosesTheSSHConnection(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + inner, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + listener := trackConnections(inner, func() (context.Context, func(), error) { + return ctx, cancel, nil + }) + defer func() { _ = listener.Close() }() + accepted := make(chan net.Conn, 1) + go func() { conn, _ := listener.Accept(); accepted <- conn }() + client, err := net.Dial("tcp", inner.Addr().String()) + require.NoError(t, err) + defer func() { _ = client.Close() }() + server := <-accepted + require.NotNil(t, server) + defer func() { _ = server.Close() }() + + cancel() + + require.NoError(t, client.SetReadDeadline(time.Now().Add(time.Second))) + var buf [1]byte + _, err = client.Read(buf[:]) + assert.ErrorIs(t, err, io.EOF, "SSH must not remain open without lifecycle tracking") +} + +func TestSSHServerRejectsOnlyUnregisteredConnectionAndKeepsServing(t *testing.T) { + var attempts atomic.Int32 + server, listener, done := newTrackedTestSSHServer(t, func() (context.Context, func(), error) { + if attempts.Add(1) == 2 { + return nil, nil, errors.New("activity capacity unavailable") + } + ctx, cancel := context.WithCancel(context.Background()) + return ctx, cancel, nil + }) + address := listener.Addr().String() + key := server.HostSigners[0].PublicKey() + existing, err := dialTrackedTestSSH(address, key) + require.NoError(t, err) + defer func() { _ = existing.Close() }() + + denied, err := dialTrackedTestSSH(address, key) + if denied != nil { + _ = denied.Close() + } + require.Error(t, err, "a connection without registered activity must not reach SSH") + assertTrackedSSHCommand(t, existing) + later, err := dialTrackedTestSSH(address, key) + require.NoError(t, err, "one rejected admission must not terminate SSH serving") + defer func() { _ = later.Close() }() + assertTrackedSSHCommand(t, later) + + require.NoError(t, listener.Close()) + select { + case err := <-done: + assert.ErrorIs(t, err, net.ErrClosed) + case <-time.After(time.Second): + t.Fatal("SSH serving did not terminate after the underlying listener closed") + } +} + +func newTrackedTestSSHServer( + t *testing.T, + start func() (context.Context, func(), error), +) (*devsyssh.Server, net.Listener, <-chan error) { + t.Helper() + _, privateKey, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + signer, err := gossh.NewSignerFromKey(privateKey) + require.NoError(t, err) + server := &devsyssh.Server{Handler: func(session devsyssh.Session) { + _, _ = io.WriteString(session, "registered\n") + }} + server.AddHostKey(signer) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = server.Close(); _ = listener.Close() }) + done := make(chan error, 1) + go func() { done <- server.Serve(trackConnections(listener, start)) }() + return server, listener, done +} + +func dialTrackedTestSSH(address string, key gossh.PublicKey) (*gossh.Client, error) { + conn, err := net.DialTimeout("tcp", address, time.Second) + if err != nil { + return nil, err + } + if err := conn.SetDeadline(time.Now().Add(time.Second)); err != nil { + _ = conn.Close() + return nil, err + } + client, channels, requests, err := gossh.NewClientConn(conn, address, &gossh.ClientConfig{ + User: "workspace", HostKeyCallback: gossh.FixedHostKey(key), + }) + if err != nil { + _ = conn.Close() + return nil, err + } + _ = conn.SetDeadline(time.Time{}) + return gossh.NewClient(client, channels, requests), nil +} + +func assertTrackedSSHCommand(t *testing.T, client *gossh.Client) { + t.Helper() + session, err := client.NewSession() + require.NoError(t, err) + defer func() { _ = session.Close() }() + out, err := session.Output("check") + require.NoError(t, err) + assert.Equal(t, "registered\n", string(out)) +} + +func TestEmbeddedSSHActivitySaturationPreservesControlAndExistingSessions(t *testing.T) { + owner, dir := publishOwner(t) + runLifecycleOwner(t, owner, dir) + writer := agentd.ActivityWriter{LifecycleDir: dir, RequireOwner: true} + admissions := make(chan error, 1) + server, listener, done := newTrackedTestSSHServer(t, func() (context.Context, func(), error) { + ctx, stop, err := writer.Start(context.Background()) + admissions <- err + return ctx, stop, err + }) + address, key := listener.Addr().String(), server.HostSigners[0].PublicKey() + existing, err := dialTrackedTestSSH(address, key) + require.NoError(t, err) + defer func() { _ = existing.Close() }() + require.NoError(t, nextSSHAdmission(t, admissions)) + raw := saturateEmbeddedSSHActivity(t, address, admissions) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + control, err := agentd.ConnectLifecycle(ctx, dir) + require.NoError( + t, + err, + "owner-UID network activity must leave actual control admission available", + ) + defer func() { _ = control.Close() }() + require.NoError(t, control.Send(agentd.LifecycleStateSetup, "1m", "")) + require.NoError(t, control.Send(agentd.LifecycleStateReady, "1m", "")) + assertTrackedSSHCommand(t, existing) + require.NoError(t, raw[0].Close()) + var later *gossh.Client + require.Eventually(t, func() bool { + client, err := dialTrackedTestSSH(address, key) + admission := nextSSHAdmission(t, admissions) + if err != nil { + return false + } + assert.NoError(t, admission) + later = client + return true + }, 3*time.Second, 10*time.Millisecond) + defer func() { _ = later.Close() }() + assertTrackedSSHCommand(t, later) + require.NoError(t, listener.Close()) + select { + case err := <-done: + assert.ErrorIs(t, err, net.ErrClosed) + case <-time.After(time.Second): + t.Fatal("a real listener close must remain terminal") + } +} + +func nextSSHAdmission(t *testing.T, admissions <-chan error) error { + t.Helper() + select { + case err := <-admissions: + return err + case <-time.After(time.Second): + t.Fatal("SSH stopped accepting instead of reporting activity admission") + return nil + } +} + +func saturateEmbeddedSSHActivity(t *testing.T, address string, admissions <-chan error) []net.Conn { + t.Helper() + var accepted []net.Conn + t.Cleanup(func() { + for _, conn := range accepted { + _ = conn.Close() + } + }) + for range 129 { + conn, err := net.DialTimeout("tcp", address, time.Second) + require.NoError(t, err) + if nextSSHAdmission(t, admissions) == nil { + accepted = append(accepted, conn) + continue + } + defer func() { _ = conn.Close() }() + require.NoError(t, conn.SetReadDeadline(time.Now().Add(time.Second))) + var buf [1]byte + _, err = conn.Read(buf[:]) + require.ErrorIs(t, err, io.EOF, "unregistered TCP must not reach the SSH handshake") + return accepted + } + t.Fatal("raw TCP activity exceeded the hard lifecycle resource bound") + return nil +} + +func TestManagedSSHServerCancellationRemainsTerminal(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + // #nosec G204 -- re-execute the current test binary with fixed test selection. + cmd := exec.CommandContext(ctx, binary, "-test.run=^TestManagedSSHServerContextChild$") + cmd.Env = append(os.Environ(), "DEVSY_MANAGED_SSH_CONTEXT_CHILD=1") + output, err := cmd.CombinedOutput() + require.NoError(t, err, "managed listener did not terminate on cancellation: %s", output) +} + +func TestManagedSSHServerContextChild(t *testing.T) { + if os.Getenv("DEVSY_MANAGED_SSH_CONTEXT_CHILD") == "" { + return + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + command := &cobra.Command{} + command.SetContext(ctx) + cmd := &SSHServerCmd{Address: "127.0.0.1:0", TrackActivity: true} + assert.ErrorIs(t, cmd.Run(command, nil), context.Canceled) +} + +func readyPath(t *testing.T, cmd *exec.Cmd) string { + t.Helper() + for _, kv := range cmd.Env { + if v, ok := strings.CutPrefix(kv, envWorkerReadyFile+"="); ok { + return v + } + } + t.Fatal("the worker must be told where to report") + return "" +} + +func TestWorkerReadinessWaitsForSpawnedWorkerRegistration(t *testing.T) { + workers, err := newWorkerReadiness() + require.NoError(t, err) + defer workers.close() + cmd := &exec.Cmd{} + require.NoError(t, workers.launch("post-attach", func(prepare func(*exec.Cmd)) error { + prepare(cmd) + return nil + })) + path := readyPath(t, cmd) + + shortCtx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + require.ErrorContains(t, workers.wait(shortCtx), "post-attach did not register") + + t.Setenv(envWorkerReadyFile, path) + signalWorkerReady() + + require.NoError(t, workers.wait(context.Background())) + assert.Empty(t, os.Getenv(envWorkerReadyFile), "hooks must not inherit the marker path") +} + +func TestWorkerReadinessIsNoopForUnmanagedContainers(t *testing.T) { + var workers *workerReadiness + cmd := &exec.Cmd{} + + err := workers.launch("deferred-hooks", func(prepare func(*exec.Cmd)) error { + prepare(cmd) + return nil + }) + + require.NoError(t, err) + assert.Empty(t, cmd.Env) + require.NoError(t, workers.wait(context.Background())) + workers.close() +} + +func TestWorkerReadinessDoesNotAwaitFailedOrSkippedLaunches(t *testing.T) { + workers, err := newWorkerReadiness() + require.NoError(t, err) + defer workers.close() + launchErr := errors.New("start process") + + err = workers.launch("deferred-hooks", func(prepare func(*exec.Cmd)) error { + prepare(&exec.Cmd{}) + return launchErr + }) + require.ErrorIs(t, err, launchErr) + require.NoError(t, workers.launch("post-attach", func(func(*exec.Cmd)) error { + return nil + }), "a worker that was already running registers on its own") + + require.NoError(t, workers.wait(context.Background()), + "neither a failed nor a skipped launch may be awaited") +} + +func TestWorkerReadinessRejectsMarkerOutsideReadinessDir(t *testing.T) { + outside := filepath.Join(t.TempDir(), "ready") + t.Setenv(envWorkerReadyFile, outside) + + signalWorkerReady() + + assert.NoFileExists(t, outside) +} diff --git a/cmd/internal/agentcontainer/post_attach.go b/cmd/internal/agentcontainer/post_attach.go index cb35e55fea..4bc5c02484 100644 --- a/cmd/internal/agentcontainer/post_attach.go +++ b/cmd/internal/agentcontainer/post_attach.go @@ -5,6 +5,7 @@ package agentcontainer import ( "context" "encoding/json" + "fmt" "github.com/devsy-org/devsy/cmd/flags" "github.com/devsy-org/devsy/pkg/compress" @@ -55,6 +56,12 @@ func (cmd *PostAttachCmd) Run(ctx context.Context) error { return err } + ctx, stopActivity, err := startWorkerActivity(ctx) + if err != nil { + return fmt.Errorf("register post-attach activity: %w", err) + } + defer stopActivity() + log.Debugf("running postAttachCommand hooks") // Mount values aren't in this process's env; read them back for redaction. if err := setup.RunPostAttachHooks( @@ -63,6 +70,9 @@ func (cmd *PostAttachCmd) Run(ctx context.Context) error { secretsEnvFromEnvironment(), setup.MountSecretsForRedaction(), ); err != nil { + if ctx.Err() != nil { + return context.Cause(ctx) + } log.Errorf("postAttachCommand failed: %v", err) } diff --git a/cmd/internal/agentcontainer/setup.go b/cmd/internal/agentcontainer/setup.go index a77d636dc3..8138e442fd 100644 --- a/cmd/internal/agentcontainer/setup.go +++ b/cmd/internal/agentcontainer/setup.go @@ -74,6 +74,7 @@ type containerState struct { tunnelClient tunnel.TunnelClient reporter status.Reporter secretsEnv []string + lifecycle *lifecycleHandshake } // Run runs the command logic. @@ -95,6 +96,22 @@ func (cmd *SetupContainerCmd) Run(ctx context.Context) error { reporter: tunnelserver.NewTunnelStatusReporter(ctx, tunnelClient), } + if !cmd.Prebuild { + state.lifecycle, err = beginLifecycle( + ctx, + config2.ContainerLifecycleDir, + workspaceInfo, + setupInfo.MergedConfig.ShutdownAction, + ) + if err != nil { + return err + } + } + if state.lifecycle != nil { + ctx = state.lifecycle.ctx + defer state.lifecycle.close() + } + _, err = tunnelserver.ReportResult( ctx, tunnelClient, @@ -108,6 +125,9 @@ func (cmd *SetupContainerCmd) Run(ctx context.Context) error { return state.setupInfo, nil }, ) + if state.lifecycle != nil { + return state.lifecycle.finish(err) + } return err } @@ -307,9 +327,11 @@ func (cmd *SetupContainerCmd) setupPostAttach( return err } - shutdownAction := state.setupInfo.MergedConfig.ShutdownAction - if err := cmd.startContainerDaemon(state.workspaceInfo, shutdownAction); err != nil { - return err + if state.lifecycle == nil { + shutdownAction := state.setupInfo.MergedConfig.ShutdownAction + if err := cmd.startContainerDaemon(state.workspaceInfo, shutdownAction); err != nil { + return err + } } resolvedSetupInfo, err := compressSetupInfo(state.setupInfo) @@ -317,20 +339,36 @@ func (cmd *SetupContainerCmd) setupPostAttach( return fmt.Errorf("re-serialize setup info: %w", err) } + var workers *workerReadiness + if state.lifecycle != nil { + workers, err = newWorkerReadiness() + if err != nil { + return err + } + defer workers.close() + } + if !deferred.Empty() { err = cmd.startDeferredHooks( resolvedSetupInfo, cmd.DotfilesRepo, cmd.DotfilesScript, - state.secretsEnv, + state.secretsEnv, workers, ) if err != nil { log.Errorf("failed to start deferred lifecycle hooks: %v", err) } } - if err := cmd.startPostAttachHooks(state); err != nil { + if err := cmd.startPostAttachHooks(state, workers); err != nil { log.Errorf("failed to start postAttachCommand: %v", err) } + // Arm last, once the spawned workers hold the workspace active themselves. + if state.lifecycle != nil { + if err := workers.wait(state.lifecycle.ctx); err != nil { + log.Errorf("hook protection not confirmed before arming: %v", err) + } + return state.lifecycle.ready() + } return nil } @@ -346,16 +384,24 @@ func compressSetupInfo(setupInfo *config.Result) (string, error) { func (cmd *SetupContainerCmd) startDeferredHooks( setupInfo, dotfilesRepo, dotfilesScript string, secretsEnv []string, + workers *workerReadiness, ) error { - return command.StartBackgroundOnce("devsy.deferred-hooks", func() (*exec.Cmd, error) { - log.Debugf("starting deferred lifecycle hooks as background process") - return buildDeferredHooksCmd( - setupInfo, - cmd.Prebuild, - dotfilesRepo, - dotfilesScript, - secretsEnv, - ) + return workers.launch("deferred-hooks", func(prepare func(*exec.Cmd)) error { + return command.StartBackgroundOnce("devsy.deferred-hooks", func() (*exec.Cmd, error) { + log.Debugf("starting deferred lifecycle hooks as background process") + execCmd, err := buildDeferredHooksCmd( + setupInfo, + cmd.Prebuild, + dotfilesRepo, + dotfilesScript, + secretsEnv, + ) + if err != nil { + return nil, err + } + prepare(execCmd) + return execCmd, nil + }) }) } @@ -495,6 +541,10 @@ func (cmd *SetupContainerCmd) startContainerDaemon( workspaceInfo *provider2.ContainerWorkspaceInfo, shutdownAction string, ) error { + // A managed container's main process owns the timer; never start a second one. + if isManagedLifecycle(workspaceInfo) { + return nil + } if workspaceInfo.CLIOptions.Platform.Enabled || workspaceInfo.CLIOptions.DisableDaemon || workspaceInfo.ContainerTimeout == "" { @@ -528,28 +578,34 @@ func (cmd *SetupContainerCmd) startContainerDaemon( }) } -func (cmd *SetupContainerCmd) startPostAttachHooks(state *containerState) error { +func (cmd *SetupContainerCmd) startPostAttachHooks( + state *containerState, + workers *workerReadiness, +) error { if len(state.setupInfo.MergedConfig.PostAttachCommands) == 0 { return nil } - return command.StartBackground("devsy.post-attach", func() (*exec.Cmd, error) { - log.Debugf("starting postAttachCommand as background process") - binaryPath, err := os.Executable() - if err != nil { - return nil, err - } + return workers.launch("post-attach", func(prepare func(*exec.Cmd)) error { + return command.StartBackground("devsy.post-attach", func() (*exec.Cmd, error) { + log.Debugf("starting postAttachCommand as background process") + binaryPath, err := os.Executable() + if err != nil { + return nil, err + } - args := []string{ - cmdInternal, cmdAgent, cmdContainer, "post-attach", - names.Flag(names.SetupInfo), cmd.SetupInfo, - } - execCmd := &exec.Cmd{ - Path: binaryPath, - Args: append([]string{binaryPath}, args...), - } - execCmd.Env = secretsEnvOverride(state.secretsEnv) - return execCmd, nil + args := []string{ + cmdInternal, cmdAgent, cmdContainer, "post-attach", + names.Flag(names.SetupInfo), cmd.SetupInfo, + } + execCmd := &exec.Cmd{ + Path: binaryPath, + Args: append([]string{binaryPath}, args...), + } + execCmd.Env = secretsEnvOverride(state.secretsEnv) + prepare(execCmd) + return execCmd, nil + }) }) } diff --git a/cmd/internal/agentcontainer/ssh_server.go b/cmd/internal/agentcontainer/ssh_server.go index 6e73f40269..2a8f875f0f 100644 --- a/cmd/internal/agentcontainer/ssh_server.go +++ b/cmd/internal/agentcontainer/ssh_server.go @@ -1,9 +1,14 @@ package agentcontainer import ( + "context" "fmt" + "net" + "sync" "github.com/devsy-org/devsy/cmd/flags" + pkgconfig "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" cliflags "github.com/devsy-org/devsy/pkg/flags" "github.com/devsy-org/devsy/pkg/flags/names" "github.com/devsy-org/devsy/pkg/log" @@ -19,6 +24,9 @@ type SSHServerCmd struct { Address string Workdir string RemoteUser string + + // TrackActivity records connection activity for the inactivity timer. + TrackActivity bool } // NewSSHServerCmd creates a new ssh command. @@ -53,12 +61,18 @@ func NewSSHServerCmd(flags *flags.GlobalFlags) *cobra.Command { "", "Directory where commands will run on the host", ), + cliflags.Bool( + &cmd.TrackActivity, + names.TrackActivity, + false, + "Record activity while connections are open", + ), ) return sshCmd } // Run runs the command logic. -func (cmd *SSHServerCmd) Run(_ *cobra.Command, _ []string) error { +func (cmd *SSHServerCmd) Run(c *cobra.Command, _ []string) error { server, err := sshserver.NewContainerServer(cmd.Address, cmd.Workdir) if err != nil { return err @@ -75,5 +89,75 @@ func (cmd *SSHServerCmd) Run(_ *cobra.Command, _ []string) error { return nil } - return server.ListenAndServe() + if !cmd.TrackActivity { + return server.ListenAndServe() + } + + ctx := c.Context() + listener, err := net.Listen("tcp", cmd.Address) + if err != nil { + return fmt.Errorf("listen on %s: %w", cmd.Address, err) + } + stopClosing := context.AfterFunc(ctx, func() { _ = listener.Close() }) + defer stopClosing() + defer func() { _ = listener.Close() }() + err = server.Serve(trackConnections(listener, func() (context.Context, func(), error) { + return startConnectionActivity(ctx) + })) + if cause := context.Cause(ctx); cause != nil { + return cause + } + return err +} + +func startConnectionActivity(ctx context.Context) (context.Context, func(), error) { + return (agentd.ActivityWriter{ + Path: pkgconfig.ContainerActivityFile, LifecycleDir: pkgconfig.ContainerLifecycleDir, + RequireOwner: true, + }).Start(ctx) +} + +// trackConnections records activity from the moment a connection is accepted +// until it closes. The idle listener itself is not activity. +func trackConnections(l net.Listener, start func() (context.Context, func(), error)) net.Listener { + return &activityListener{Listener: l, start: start} +} + +type activityListener struct { + net.Listener + start func() (context.Context, func(), error) +} + +func (l *activityListener) Accept() (net.Conn, error) { + reportedFailure := false + for { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + ctx, stop, err := l.start() + if err != nil { + _ = conn.Close() + if !reportedFailure { + log.Debugf("reject SSH connection without activity registration: %v", err) + reportedFailure = true + } + continue + } + tracked := &activityConn{Conn: conn, stop: stop} + go func() { <-ctx.Done(); _ = tracked.Close() }() + return tracked, nil + } +} + +type activityConn struct { + net.Conn + stop func() + once sync.Once +} + +func (c *activityConn) Close() error { + err := c.Conn.Close() + c.once.Do(c.stop) + return err } diff --git a/cmd/internal/fleet_server.go b/cmd/internal/fleet_server.go index 3370f74f95..a82718c723 100644 --- a/cmd/internal/fleet_server.go +++ b/cmd/internal/fleet_server.go @@ -1,6 +1,7 @@ package cmdinternal import ( + "context" "os" "path/filepath" "regexp" @@ -8,9 +9,10 @@ import ( "time" "github.com/devsy-org/devsy/cmd/flags" - "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" cliflags "github.com/devsy-org/devsy/pkg/flags" "github.com/devsy-org/devsy/pkg/flags/names" + "github.com/devsy-org/devsy/pkg/log" "github.com/spf13/cobra" ) @@ -45,34 +47,60 @@ func NewFleetServerCmd(flags *flags.GlobalFlags) *cobra.Command { return fleetCmd } -// Run runs the command logic. +var ( + fleetFirstConnection = regexp.MustCompile(`.*Received authorization request.*`) + fleetConnStatus = regexp.MustCompile(`.*Notify.*`) +) + +// fleetActivity is the activity writer held for the life of one Fleet session. +type fleetActivity struct { + ctx context.Context + stop func() +} + +func (a *fleetActivity) held() bool { return a.stop != nil } + +// lost reports that the lifecycle owner dropped the registration. +func (a *fleetActivity) lost() bool { return a.held() && a.ctx.Err() != nil } + +func (a *fleetActivity) acquire(parent context.Context) error { + ctx, stop, err := agentd.StartActivityWriter(parent, agentd.DefaultActivityInterval) + if err != nil { + return err + } + a.ctx, a.stop = ctx, stop + return nil +} + +func (a *fleetActivity) release() { + if a.stop != nil { + a.stop() + a.ctx, a.stop = nil, nil + } +} + +// Run runs the command logic. While a Fleet client is connected it holds an +// activity writer, which registers with a managed lifecycle owner or keeps the +// legacy activity file fresh, and releases it when the session ends. func (c *FleetServerCmd) Run(cmd *cobra.Command, _ []string) error { logFile := filepath.Join(os.Getenv("HOME"), ".cache/JetBrains/Fleet/log/fleet.log") - firstConnection := regexp.MustCompile(`.*Received authorization request.*`) - connStatus := regexp.MustCompile(`.*Notify.*`) + + var activity fleetActivity + defer activity.release() for { select { case <-time.After(time.Second * 10): - - log, err := os.ReadFile(logFile) - if err != nil { - continue - } - - // check if we had at least one fleet client connection, before - // this point, we do not check for connected/disconnected strings - initialized := firstConnection.FindStringSubmatch(string(log)) - if len(initialized) == 0 { - continue + if activity.lost() { + activity.release() } - - connString := connStatus.FindAllStringSubmatch(string(log), -1) - - // if ouf last occurrence of notify if "Notify ID connected" - // we have an active session, so let's keep alive - if strings.Contains(connString[len(connString)-1][0], "is connected") { - touchActivityFile(config.ContainerActivityFile) + switch active := fleetSessionActive(logFile); { + case active && !activity.held(): + if err := activity.acquire(cmd.Context()); err != nil { + log.Errorf("register fleet session activity: %v", err) + } + case !active: + activity.release() } case <-cmd.Context().Done(): // context is done - either canceled or time is up for timeout @@ -80,3 +108,22 @@ func (c *FleetServerCmd) Run(cmd *cobra.Command, _ []string) error { } } } + +// fleetSessionActive reports whether the Fleet log shows a connected client. +// Before the first client connects, the connected/disconnected lines are ignored. +func fleetSessionActive(logFile string) bool { + // #nosec G304 G703 -- fixed Fleet log path under $HOME, not caller-controlled. + raw, err := os.ReadFile(logFile) + if err != nil { + return false + } + if len(fleetFirstConnection.FindStringSubmatch(string(raw))) == 0 { + return false + } + // if our last occurrence of notify is "Notify ID connected" we have an active session + statuses := fleetConnStatus.FindAllStringSubmatch(string(raw), -1) + if len(statuses) == 0 { + return false + } + return strings.Contains(statuses[len(statuses)-1][0], "is connected") +} diff --git a/cmd/internal/fleet_server_test.go b/cmd/internal/fleet_server_test.go new file mode 100644 index 0000000000..b4dedf0408 --- /dev/null +++ b/cmd/internal/fleet_server_test.go @@ -0,0 +1,54 @@ +package cmdinternal + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + fleetAuthorized = "12:00 Received authorization request from client\n" + fleetConnected = "12:01 Notify abc is connected\n" + fleetDisconnected = "12:02 Notify abc is disconnected\n" +) + +func writeFleetLog(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "fleet.log") + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + return path +} + +func TestFleetSessionActive(t *testing.T) { + tests := []struct { + name string + log string + want bool + }{ + {name: "connected after authorization", log: fleetAuthorized + fleetConnected, want: true}, + { + name: "disconnected last", + log: fleetAuthorized + fleetConnected + fleetDisconnected, + }, + { + name: "reconnected after a disconnect", + log: fleetAuthorized + fleetConnected + fleetDisconnected + fleetConnected, + want: true, + }, + {name: "connected line before any authorization is ignored", log: fleetConnected}, + {name: "authorized but no status line yet", log: fleetAuthorized}, + {name: "empty log", log: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, fleetSessionActive(writeFleetLog(t, tt.log))) + }) + } +} + +func TestFleetSessionActiveMissingLogIsInactive(t *testing.T) { + assert.False(t, fleetSessionActive(filepath.Join(t.TempDir(), "absent.log"))) +} diff --git a/cmd/internal/ssh_activity_linux_test.go b/cmd/internal/ssh_activity_linux_test.go new file mode 100644 index 0000000000..3aa3856bfc --- /dev/null +++ b/cmd/internal/ssh_activity_linux_test.go @@ -0,0 +1,66 @@ +//go:build linux + +package cmdinternal + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTrackedStdioUsesMountedManagedConfigWithoutEnvironmentOrDiscovery(t *testing.T) { + if os.Getenv("DEVSY_MOUNTED_SECRET_TEST") == "" { + t.Skip("requires the isolated root container with a private secret mount") + } + require.Zero(t, os.Geteuid()) + t.Setenv(config.EnvWorkspaceDaemonConfig, "") + encoded, err := agentd.EncodeDaemonConfig(&agentd.DaemonConfig{ + Timeout: "1m", Lifecycle: agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon}, + }) + require.NoError(t, err) + // #nosec G301 -- mirror a Kubernetes secret directory traversable by the nonroot child. + require.NoError(t, os.MkdirAll(filepath.Dir(agentd.ContainerDaemonConfigPath), 0o755)) + // #nosec G306 -- mirror the readable, root-owned Kubernetes projected secret. + require.NoError(t, os.WriteFile(agentd.ContainerDaemonConfigPath, []byte(encoded), 0o644)) + binary, err := os.Executable() + require.NoError(t, err) + // #nosec G204 -- re-execute the current test binary with fixed test selection. + cmd := exec.Command(binary, "-test.run=^TestMountedManagedConfigStdioChild$") + cmd.Env = append(os.Environ(), "DEVSY_MOUNTED_SECRET_CHILD=1") + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: 65534, Gid: 65534, Groups: []uint32{65534}}, + Pdeathsig: syscall.SIGKILL, + } + output, err := cmd.CombinedOutput() + t.Logf("nonroot stdio caller: %s", output) + require.NoError(t, err) +} + +func TestMountedManagedConfigStdioChild(t *testing.T) { + if os.Getenv("DEVSY_MOUNTED_SECRET_CHILD") == "" { + return + } + require.Equal(t, 65534, os.Geteuid()) + require.Empty(t, os.Getenv(config.EnvWorkspaceDaemonConfig)) + _, err := os.Lstat(config.ContainerLifecycleDir) + require.ErrorIs(t, err, os.ErrNotExist) + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + cmd := &sshServerCmd{stdio: true, trackActivity: true, address: "127.0.0.1:0", workdir: "/tmp"} + err = cmd.run(ctx) + assert.ErrorIs( + t, + err, + agentd.ErrLifecycleOwnerMissing, + "mounted managed configuration must refuse untracked stdio before SSH work, even without discovery", + ) +} diff --git a/cmd/internal/ssh_server.go b/cmd/internal/ssh_server.go index 906476ce40..478bfb6e23 100644 --- a/cmd/internal/ssh_server.go +++ b/cmd/internal/ssh_server.go @@ -10,7 +10,7 @@ import ( "time" "github.com/devsy-org/devsy/cmd/flags" - "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/flags/names" "github.com/devsy-org/devsy/pkg/log" "github.com/devsy-org/devsy/pkg/sharedfile" @@ -22,10 +22,7 @@ import ( "github.com/spf13/cobra" ) -const ( - activityHeartbeatInterval = 10 * time.Second - activityFileMode = 0o666 -) +const activityFileMode = 0o666 // shutdownTimeout bounds the grace period for in-flight SSH connections to // drain on context cancel before the listener is force-closed. @@ -62,7 +59,7 @@ func NewSSHServerCmd(globalFlags *flags.GlobalFlags) *cobra.Command { f.BoolVar(&cmd.stdio, names.Stdio, false, "Listen on stdin/stdout instead of an address") f.BoolVar(&cmd.trackActivity, names.TrackActivity, false, - "Touch the activity file every "+activityHeartbeatInterval.String()+" (only with --stdio)") + "Record activity while connected (only with --stdio)") f.StringVar(&cmd.reuseSSHAuthSock, names.ReuseSSHAuthSock, "", "If set, reuse a pre-existing SSH_AUTH_SOCK in the workspace under /tmp") _ = f.MarkHidden(names.ReuseSSHAuthSock) @@ -106,12 +103,21 @@ func (cmd *sshServerCmd) run(ctx context.Context) error { func (cmd *sshServerCmd) serveStdio(ctx context.Context, server sshserver.Server) error { if cmd.trackActivity { - go runActivityHeartbeat(ctx, config.ContainerActivityFile) + workCtx, stop, err := agentd.StartActivityWriter(ctx, agentd.DefaultActivityInterval) + if err != nil { + return fmt.Errorf("register SSH activity: %w", err) + } + ctx = workCtx + defer stop() } go shutdownOnCancel(ctx, server) // #nosec G118 -- see shutdownOnCancel. lis := stdio.NewStdioListener(os.Stdin, os.Stdout) log.Debug("serving ssh on stdio") - return ignoreServerClosed(server.Serve(lis)) + err := ignoreServerClosed(server.Serve(lis)) + if cause := context.Cause(ctx); cause != nil && !errors.Is(cause, context.Canceled) { + return cause + } + return err } func (cmd *sshServerCmd) serveListener(ctx context.Context, server sshserver.Server) error { @@ -204,44 +210,11 @@ func decodeBase64Bytes(encoded, label string) ([]byte, error) { return out, nil } -// runActivityHeartbeat periodically updates the activity file's mtime so -// outside watchers can detect liveness. Exits when ctx is canceled. Logs -// only on success/failure transitions so a permanently broken file does -// not spam the log. -func runActivityHeartbeat(ctx context.Context, path string) { - if err := ensureActivityFile(path); err != nil { - log.Errorf("activity heartbeat: ensure file: %v", err) - return - } - - t := time.NewTicker(activityHeartbeatInterval) - defer t.Stop() - - var lastErr error - for { - select { - case <-ctx.Done(): - return - case now := <-t.C: - err := os.Chtimes(path, now, now) - if (err == nil) != (lastErr == nil) { - if err != nil { - log.Errorf("activity heartbeat: %v", err) - } else { - log.Infof("activity heartbeat recovered") - } - } - lastErr = err - } - } -} - func ensureActivityFile(path string) error { return sharedfile.EnsureMode(path, activityFileMode) } -// touchActivityFile is for callers reporting activity on discrete events -// rather than runActivityHeartbeat's fixed interval (e.g. fleet-server). +// touchActivityFile records discrete legacy activity (for example fleet-server). func touchActivityFile(path string) { if err := ensureActivityFile(path); err != nil { log.Errorf("touch activity file: ensure file: %v", err) diff --git a/cmd/internal/ssh_server_test.go b/cmd/internal/ssh_server_test.go index 3be14496f5..3ee3904572 100644 --- a/cmd/internal/ssh_server_test.go +++ b/cmd/internal/ssh_server_test.go @@ -174,22 +174,6 @@ func TestIgnoreServerClosed(t *testing.T) { } } -func TestRunActivityHeartbeatExitsOnContextCancel(t *testing.T) { - path := filepath.Join(t.TempDir(), "activity") - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - runActivityHeartbeat(ctx, path) - close(done) - }() - cancel() - select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatal("heartbeat did not exit within 2s of context cancel") - } -} - func TestEnsureActivityFile_CreatesWorldWritableFile(t *testing.T) { path := filepath.Join(t.TempDir(), "devsy.activity") diff --git a/e2e/tests/up/provider_kubernetes_inactivity.go b/e2e/tests/up/provider_kubernetes_inactivity.go new file mode 100644 index 0000000000..4902a323dc --- /dev/null +++ b/e2e/tests/up/provider_kubernetes_inactivity.go @@ -0,0 +1,424 @@ +package up + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" +) + +const ( + flagIDEOption = "--ide" + inactivityLabel = "up-provider-kubernetes-inactivity" + inactivityImage = "ghcr.io/devsy-org/test-images/go:1" + inactivityPollEvery = 3 * time.Second + sentinelName = "idle-sentinel" +) + +// idleEnv is one isolated namespace, provider and workspace. +type idleEnv struct { + f *framework.Framework + ns string + dir string + wsID string +} + +func newIdleEnv(ctx context.Context, initialDir string, providerOpts ...string) *idleEnv { + e := &idleEnv{ + f: framework.NewDefaultFramework(initialDir + "/bin"), + ns: fmt.Sprintf("devsy-idle-%d", time.Now().UnixNano()%1_000_000_000), + } + e.dir = framework.CopyTestdata("tests/up/testdata/kubernetes", initialDir) + + opts := []string{ + "-o", + "KUBERNETES_NAMESPACE=" + e.ns, + "-o", + "CREATE_NAMESPACE=true", + "-o", + "POD_TIMEOUT=5m", + } + for _, o := range providerOpts { + opts = append(opts, "-o", o) + } + _ = e.f.DevsyProviderDelete(ctx, "kubernetes") + framework.ExpectNoError(e.f.DevsyProviderAdd(ctx, append([]string{"kubernetes"}, opts...)...)) + + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + c, cancel := framework.DetachedCleanupContext(cleanupCtx, 3*time.Minute) + defer cancel() + _ = e.f.DevsyWorkspaceDelete(c, e.dir) + _ = e.f.DevsyProviderDelete(c, "kubernetes") + _ = exec.CommandContext(c, "kubectl", "delete", "namespace", e.ns, // #nosec G204 -- test-owned namespace + "--ignore-not-found", "--wait=false"). + Run() + }) + return e +} + +func (e *idleEnv) writeDevcontainer(content string) { + dir := filepath.Join(e.dir, ".devcontainer") + framework.ExpectNoError(os.MkdirAll(dir, 0o750)) + framework.ExpectNoError( + os.WriteFile(filepath.Join(dir, "devcontainer.json"), []byte(content), 0o600), + ) +} + +func (e *idleEnv) up(ctx context.Context) error { + return e.f.DevsyUp(ctx, e.dir) +} + +func (e *idleEnv) workspaceID(ctx context.Context) string { + if e.wsID == "" { + ws, err := e.f.FindWorkspace(ctx, e.dir) + framework.ExpectNoError(err) + e.wsID = ws.ID + } + return e.wsID +} + +func (e *idleEnv) kubectlJSON(ctx context.Context, into any, args ...string) error { + full := append(append([]string{}, args...), "-n", e.ns, "-o", "json") + cmd := exec.CommandContext(ctx, "kubectl", full...) // #nosec G204 -- test-owned kubectl args + out, err := cmd.Output() + if err != nil { + return err + } + return json.Unmarshal(out, into) +} + +func (e *idleEnv) pod(ctx context.Context) (*corev1.Pod, error) { + list := &corev1.PodList{} + if err := e.kubectlJSON(ctx, list, "get", "pods", "-l", "devsy.sh/created=true"); err != nil { + return nil, err + } + if len(list.Items) != 1 { + return nil, fmt.Errorf("want exactly 1 workspace pod, got %d", len(list.Items)) + } + return &list.Items[0], nil +} + +func (e *idleEnv) pvcUID(ctx context.Context) string { + list := &corev1.PersistentVolumeClaimList{} + framework.ExpectNoError(e.kubectlJSON(ctx, list, "get", "pvc")) + gomega.Expect(list.Items).To(gomega.HaveLen(1)) + return string(list.Items[0].UID) +} + +func (e *idleEnv) ssh(ctx context.Context, command string) string { + out, err := e.f.DevsySSH(ctx, e.dir, command) + framework.ExpectNoError(err) + return strings.TrimSpace(out) +} + +// holdSSH keeps one tracked SSH session open until the returned stop is called. +func (e *idleEnv) holdSSH(ctx context.Context) (stop func()) { + cmd := exec.CommandContext( + ctx, + filepath.Join(e.f.DevsyBinDir, e.f.DevsyBinName), // #nosec G204 -- test binary + cmdWorkspace, + cmdSSH, + e.dir, + flagCommand, + "sleep 3600", + ) + framework.ExpectNoError(cmd.Start()) + stop = func() { + if cmd.Process != nil { + _ = cmd.Process.Kill() + } + _ = cmd.Wait() + } + ginkgo.DeferCleanup(stop) + return stop +} + +func podTerminal(pod *corev1.Pod) bool { + return pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed +} + +func devsyRestarts(pod *corev1.Pod) int32 { + var n int32 + for _, c := range pod.Status.ContainerStatuses { + n += c.RestartCount + } + return n +} + +// waitStopped observes the pod through the Kubernetes API, never through +// workspace ssh, which would itself keep the workspace active. +func (e *idleEnv) waitStopped(ctx context.Context, uid string, within time.Duration) *corev1.Pod { + var last *corev1.Pod + gomega.Eventually(func() bool { + pod, err := e.pod(ctx) + if err != nil { + return false + } + last = pod + return string(pod.UID) == uid && podTerminal(pod) + }, within, inactivityPollEvery).Should(gomega.BeTrue(), "workspace pod should reach a terminal phase") + + gomega.Expect(last.Status.Phase).ToNot(gomega.Equal(corev1.PodRunning)) + gomega.Expect(devsyRestarts(last)).To(gomega.BeZero(), "the stopped pod must not restart") + for _, c := range last.Status.ContainerStatuses { + gomega.Expect(c.State.Terminated). + ToNot(gomega.BeNil(), "container %s should be terminated", c.Name) + } + return last +} + +// expectRunningFor asserts the same pod instance keeps running for the whole window. +func (e *idleEnv) expectRunningFor(ctx context.Context, uid string, window time.Duration) { + gomega.Consistently(func() string { + pod, err := e.pod(ctx) + if err != nil || string(pod.UID) != uid { + return "missing" + } + return string(pod.Status.Phase) + }, window, inactivityPollEvery).Should(gomega.Equal(string(corev1.PodRunning))) +} + +func devcontainerWith(extra string) string { + return fmt.Sprintf(`{"name":"idle","image":%q%s}`, inactivityImage, extra) +} + +const sidecarTemplate = "spec:\n containers:\n - name: sidecar\n image: " + inactivityImage + + "\n command: [sleep, infinity]\n" + +const nativeSidecarTemplate = "spec:\n initContainers:\n - name: mesh\n image: " + inactivityImage + + "\n command: [sleep, infinity]\n restartPolicy: Always\n" + +var _ = ginkgo.Describe( + "kubernetes workspace inactivity timeout", + ginkgo.Label(inactivityLabel), + ginkgo.Serial, + func() { + var initialDir string + + ginkgo.BeforeEach(func() { + var err error + initialDir, err = os.Getwd() + framework.ExpectNoError(err) + }) + + ginkgo.It( + "stops an idle workspace and resumes it with the volume intact", + func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir, "INACTIVITY_TIMEOUT=1m") + framework.ExpectNoError(e.up(ctx)) + + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + gomega.Expect(pod.Spec.Containers).To(gomega.HaveLen(1)) + pvc := e.pvcUID(ctx) + workspaceUID := e.workspaceUID(ctx) + sentinel := "/workspaces/" + e.workspaceID(ctx) + "/" + sentinelName + + for cycle := 1; cycle <= 2; cycle++ { + ginkgo.By(fmt.Sprintf("cycle %d: writing a sentinel and going idle", cycle)) + e.ssh(ctx, fmt.Sprintf("echo cycle-%d > %s", cycle, sentinel)) + podUID := string(pod.UID) + + e.waitStopped(ctx, podUID, 4*time.Minute) + gomega.Expect(e.pvcUID(ctx)).To(gomega.Equal(pvc)) + + ginkgo.By(fmt.Sprintf("cycle %d: resuming with a plain up", cycle)) + framework.ExpectNoError(e.up(ctx)) + pod, err = e.pod(ctx) + framework.ExpectNoError(err) + gomega.Expect(string(pod.UID)). + ToNot(gomega.Equal(podUID), "a new pod replaced the stopped one") + gomega.Expect(pod.Status.Phase).To(gomega.Equal(corev1.PodRunning)) + gomega.Expect(e.pvcUID(ctx)).To(gomega.Equal(pvc)) + gomega.Expect(e.workspaceUID(ctx)).To(gomega.Equal(workspaceUID)) + gomega.Expect(e.ssh(ctx, "cat "+sentinel)). + To(gomega.Equal(fmt.Sprintf("cycle-%d", cycle))) + } + }, + ginkgo.SpecTimeout(framework.TimeoutVeryLong()), + ) + + ginkgo.It("stays running while a tracked SSH session is open and stops after it ends", + func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir, "INACTIVITY_TIMEOUT=30s") + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + + stop := e.holdSSH(ctx) + e.expectRunningFor(ctx, string(pod.UID), 75*time.Second) + + stop() + e.waitStopped(ctx, string(pod.UID), 3*time.Minute) + }, ginkgo.SpecTimeout(framework.TimeoutLong())) + + ginkgo.It("does not stop when shutdownAction is none", func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir, "INACTIVITY_TIMEOUT=30s") + e.writeDevcontainer(devcontainerWith(`,"shutdownAction":"none"`)) + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + + e.expectRunningFor(ctx, string(pod.UID), 90*time.Second) + }, ginkgo.SpecTimeout(framework.TimeoutLong())) + + ginkgo.It("does not stop when the timeout is unset", func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir) + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + + e.expectRunningFor(ctx, string(pod.UID), 75*time.Second) + }, ginkgo.SpecTimeout(framework.TimeoutLong())) + + ginkgo.It( + "keeps the workspace alive during a long postAttachCommand", + func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir, "INACTIVITY_TIMEOUT=30s") + e.writeDevcontainer(devcontainerWith(`,"postAttachCommand":"sleep 80"`)) + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + + e.expectRunningFor(ctx, string(pod.UID), 60*time.Second) + e.waitStopped(ctx, string(pod.UID), 4*time.Minute) + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + + ginkgo.It( + "recreates a terminal pod with unchanged options on the next up", + func(ctx context.Context) { + e := newIdleEnv(ctx, initialDir) + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + pvc := e.pvcUID(ctx) + + out, err := exec.CommandContext(ctx, "kubectl", "exec", "-n", e.ns, pod.Name, // #nosec G204 -- test pod + "-c", "devsy", "--", "sh", "-c", "kill -TERM 1"). + CombinedOutput() + framework.ExpectNoError(err, string(out)) + e.waitStopped(ctx, string(pod.UID), 2*time.Minute) + + framework.ExpectNoError(e.up(ctx)) + fresh, err := e.pod(ctx) + framework.ExpectNoError(err) + gomega.Expect(fresh.UID).ToNot(gomega.Equal(pod.UID)) + gomega.Expect(e.pvcUID(ctx)).To(gomega.Equal(pvc)) + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + + ginkgo.It( + "stops the whole pod when a native sidecar is present", + func(ctx context.Context) { + e := newIdleEnv( + ctx, + initialDir, + "INACTIVITY_TIMEOUT=30s", + "POD_MANIFEST_TEMPLATE="+nativeSidecarTemplate, + ) + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + + stopped := e.waitStopped(ctx, string(pod.UID), 4*time.Minute) + for _, c := range stopped.Status.InitContainerStatuses { + gomega.Expect(c.State.Terminated). + ToNot(gomega.BeNil(), "sidecar %s should end with the pod", c.Name) + } + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + + ginkgo.It( + "rejects extra regular containers only when a timeout is enabled", + func(ctx context.Context) { + e := newIdleEnv( + ctx, + initialDir, + "INACTIVITY_TIMEOUT=30s", + "POD_MANIFEST_TEMPLATE="+sidecarTemplate, + ) + + _, stderr, err := e.f.ExecCommandCapture(ctx, + []string{cmdWorkspace, "up", e.dir, flagIDEOption, kubernetesIgnoreIDE}) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(stderr).To(gomega.ContainSubstring("sidecar")) + pvcs := &corev1.PersistentVolumeClaimList{} + framework.ExpectNoError(e.kubectlJSON(ctx, pvcs, "get", "pvc")) + gomega.Expect(pvcs.Items). + To(gomega.BeEmpty(), "nothing is created for a rejected pod") + + framework.ExpectNoError(e.f.DevsyWorkspaceDelete(ctx, e.dir)) + framework.ExpectNoError(os.Chdir(initialDir)) + e = newIdleEnv( + ctx, + initialDir, + "INACTIVITY_TIMEOUT=0", + "POD_MANIFEST_TEMPLATE="+sidecarTemplate, + ) + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + gomega.Expect(pod.Spec.Containers).To(gomega.HaveLen(2)) + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + + ginkgo.It( + "stops and resumes under a restricted non-root security context", + func(ctx context.Context) { + framework.ExpectNoError(labelNamespaceRestricted(ctx)) + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + _ = exec.CommandContext(cleanupCtx, "kubectl", "delete", "namespace", restrictedNamespace, + "--ignore-not-found", "--wait=false"). + Run() + }) + e := newIdleEnv(ctx, initialDir, + "INACTIVITY_TIMEOUT=30s", + "STRICT_SECURITY=true", + "AGENT_SECURITY_CONTEXT="+restrictedSecurityContextYAML, + "POD_MANIFEST_TEMPLATE="+restrictedPodManifestTemplate, + "AGENT_INSTALL_PATH=/tmp/devsy", + ) + e.ns = restrictedNamespace + framework.ExpectNoError(e.f.DevsyProviderUse( + ctx, + "kubernetes", + "-o", + "KUBERNETES_NAMESPACE="+restrictedNamespace, + "-o", + "CREATE_NAMESPACE=false", + )) + + framework.ExpectNoError(e.up(ctx)) + pod, err := e.pod(ctx) + framework.ExpectNoError(err) + e.waitStopped(ctx, string(pod.UID), 4*time.Minute) + + framework.ExpectNoError(e.up(ctx)) + fresh, err := e.pod(ctx) + framework.ExpectNoError(err) + gomega.Expect(fresh.UID).ToNot(gomega.Equal(pod.UID)) + }, + ginkgo.SpecTimeout(framework.TimeoutVeryLong()), + ) + }, +) + +func (e *idleEnv) workspaceUID(ctx context.Context) string { + ws, err := e.f.FindWorkspace(ctx, e.dir) + framework.ExpectNoError(err) + return ws.UID +} diff --git a/go.mod b/go.mod index 7b5ebb2c81..bd15860750 100644 --- a/go.mod +++ b/go.mod @@ -64,6 +64,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.12.1 github.com/tailscale/hujson v0.0.0-20260727124030-b80ff77dac4f + github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc github.com/tidwall/gjson v1.19.0 github.com/tonistiigi/fsutil v0.0.0-20260819142231-83cac42c1c52 github.com/u-root/u-root v0.16.0 @@ -456,7 +457,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/tailscale/certstore v0.1.1-0.20260409135935-3638fb84b77d // indirect github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 // indirect - github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc // indirect github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 // indirect github.com/tailscale/wireguard-go v0.0.0-20260730222847-4affce44577c // indirect github.com/theupdateframework/go-tuf v0.7.0 // indirect diff --git a/hack/ci/suites.yaml b/hack/ci/suites.yaml index db546ab55b..169cc47b1d 100644 --- a/hack/ci/suites.yaml +++ b/hack/ci/suites.yaml @@ -219,6 +219,7 @@ e2e: - { group: integration, label: up-private-token, requires_secret: true } - { group: integration, label: up-provider-kubernetes, free_disk_space: true, install_kind: true } - { group: integration, label: up-provider-kubernetes-restricted-scc, free_disk_space: true, install_kind: true } + - { group: integration, label: up-provider-kubernetes-inactivity, free_disk_space: true, install_kind: true, test_timeout: 60m, job_timeout_minutes: 75 } - { group: integration, label: up-provider-docker, install_kind: true } - { group: integration, label: snapshot, host_setup: [insecure-registry] } - group: integration diff --git a/pkg/config/paths.go b/pkg/config/paths.go index 40074189dc..d334676d27 100644 --- a/pkg/config/paths.go +++ b/pkg/config/paths.go @@ -45,6 +45,9 @@ const ( // ContainerActivityFile is touched by SSH/fleet servers to record container liveness. ContainerActivityFile = "/tmp/" + BinaryName + ".activity" + // ContainerLifecycleDir holds the managed container daemon's owner and control records. + ContainerLifecycleDir = "/tmp/" + BinaryName + "-container-lifecycle" + // WorkspaceBusyFile is the per-workspace lock file written under the workspace folder. WorkspaceBusyFile = "workspace.lock" ) diff --git a/pkg/daemon/agent/activity.go b/pkg/daemon/agent/activity.go new file mode 100644 index 0000000000..78fcfdeafd --- /dev/null +++ b/pkg/daemon/agent/activity.go @@ -0,0 +1,187 @@ +package agent + +import ( + "context" + "errors" + "os" + "sync" + "time" + + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/log" + "github.com/devsy-org/devsy/pkg/sharedfile" +) + +const ( + DefaultActivityInterval = 10 * time.Second + activityFileMode = 0o666 + minTickInterval = 10 * time.Millisecond +) + +func TouchActivity(path string, now time.Time) error { + if err := sharedfile.EnsureMode(path, activityFileMode); err != nil { + return err + } + err := os.Chtimes(path, now, now) + if errors.Is(err, os.ErrPermission) { + return touchByWrite(path) + } + return err +} + +func touchByWrite(path string) error { + f, err := os.OpenFile(path, os.O_WRONLY, 0) // #nosec G304 -- fixed activity file path + if err != nil { + return err + } + _, writeErr := f.WriteAt([]byte{0}, 0) + return errors.Join(writeErr, f.Close()) +} + +// ActivityWriter retains file heartbeats for non-managed daemons. Managed work +// is protected by its acknowledged connection, including when all writes fail. +type ActivityWriter struct { + Path string + LifecycleDir string + RequireOwner bool + Interval time.Duration + Touch func(string, time.Time) error + Now func() time.Time + Tick func(time.Duration) (<-chan time.Time, func()) +} + +// Start returns only after registering with any live managed owner. The caller +// must use workCtx and stop its work if that context is cancelled. +func (w ActivityWriter) Start(ctx context.Context) (context.Context, func(), error) { + client, err := w.connect(ctx) + if err != nil { + return nil, nil, err + } + if client != nil { + return startRegisteredActivity(ctx, client) + } + w = w.withDefaults() + workCtx, cancel := context.WithCancel(ctx) + lastErr := w.record(nil) + done := make(chan struct{}) + go func() { defer close(done); w.run(workCtx, lastErr) }() + var once sync.Once + return workCtx, func() { once.Do(func() { cancel(); <-done }) }, nil +} + +func startRegisteredActivity( + ctx context.Context, + client *LifecycleClient, +) (context.Context, func(), error) { + workCtx, cancel := client.ProtectedContext(ctx) + if err := client.Send(LifecycleStateActivity, "", ""); err != nil { + cancel(err) + _ = client.Close() + return nil, nil, err + } + // The live connection itself is the activity. No periodic file write or + // health publication is needed, even when storage is completely unavailable. + var once sync.Once + return workCtx, func() { + once.Do(func() { + cancel(context.Canceled) + if err := client.Send(LifecycleStateRelease, "", ""); err != nil { + log.Debugf("activity release failed; closing its protection connection: %v", err) + } + _ = client.Close() + }) + }, nil +} + +// Run records legacy file activity without claiming managed protection. Managed +// callers use Start so registration completes before their work begins. +func (w ActivityWriter) Run(ctx context.Context) { + w = w.withDefaults() + w.run(ctx, w.record(nil)) +} + +func (w ActivityWriter) connect(ctx context.Context) (*LifecycleClient, error) { + if w.LifecycleDir == "" { + if w.RequireOwner { + return nil, ErrLifecycleOwnerMissing + } + return nil, nil + } + owner, err := liveOwner(w.LifecycleDir, os.Geteuid()) + if err != nil { + return nil, err + } + if owner != nil { + return connectOwner(ctx, w.LifecycleDir, *owner) + } + if w.RequireOwner { + return nil, ErrLifecycleOwnerMissing + } + // A discovery gap in an existing managed runtime is not a legacy workspace. + if _, err := os.Lstat(w.LifecycleDir); !errors.Is(err, os.ErrNotExist) { + return nil, ErrLifecycleOwnerMissing + } + return nil, nil +} + +func (w ActivityWriter) run(ctx context.Context, lastErr error) { + ticks, stop := w.Tick(w.Interval) + defer stop() + for { + select { + case <-ctx.Done(): + _ = w.record(lastErr) + return + case <-ticks: + lastErr = w.record(lastErr) + } + } +} + +func (w ActivityWriter) record(lastErr error) error { + err := w.Touch(w.Path, w.Now()) + if (err == nil) != (lastErr == nil) { + if err != nil { + log.Errorf("activity writer: %v", err) + } else { + log.Infof("activity writer recovered") + } + } + return err +} + +func (w ActivityWriter) withDefaults() ActivityWriter { + if w.Touch == nil { + w.Touch = TouchActivity + } + if w.Now == nil { + w.Now = time.Now + } + if w.Tick == nil { + w.Tick = realTick + } + if w.Interval <= 0 { + w.Interval = DefaultActivityInterval + } + return w +} + +func realTick(d time.Duration) (<-chan time.Time, func()) { + t := time.NewTicker(d) + return t.C, t.Stop +} + +func StartActivityWriter( + ctx context.Context, + interval time.Duration, +) (context.Context, func(), error) { + daemon, err := LoadContainerDaemonConfig() + if err != nil { + return nil, nil, err + } + required := daemon != nil && daemon.Lifecycle.Managed() + return (ActivityWriter{ + Path: config.ContainerActivityFile, LifecycleDir: config.ContainerLifecycleDir, + RequireOwner: required, Interval: interval, + }).Start(ctx) +} diff --git a/pkg/daemon/agent/container_config.go b/pkg/daemon/agent/container_config.go new file mode 100644 index 0000000000..158de82a23 --- /dev/null +++ b/pkg/daemon/agent/container_config.go @@ -0,0 +1,40 @@ +package agent + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/devsy-org/devsy/pkg/config" +) + +const ContainerDaemonConfigPath = "/var/run/secrets/" + config.BinaryName + "/daemon_config" + +func LoadContainerDaemonConfig() (*DaemonConfig, error) { + encoded, err := readContainerDaemonConfig(ContainerDaemonConfigPath) + if err != nil { + return nil, err + } + if strings.TrimSpace(encoded) == "" { + return nil, nil + } + return DecodeDaemonConfig(encoded) +} + +func readContainerDaemonConfig(path string) (string, error) { + // Projected Kubernetes secrets use symlinks and take precedence over the + // process environment, matching the daemon's existing config source. + raw, err := os.ReadFile(path) // #nosec G304 -- fixed secret path or isolated test fixture + if errors.Is(err, os.ErrNotExist) { + return os.Getenv(config.EnvWorkspaceDaemonConfig), nil + } + if err != nil { + return "", fmt.Errorf("read container daemon config: %w", err) + } + encoded := string(raw) + if strings.TrimSpace(encoded) == "" { + return "", errors.New("mounted container daemon config is empty") + } + return encoded, nil +} diff --git a/pkg/daemon/agent/container_lifecycle.go b/pkg/daemon/agent/container_lifecycle.go new file mode 100644 index 0000000000..f5a1c40f2f --- /dev/null +++ b/pkg/daemon/agent/container_lifecycle.go @@ -0,0 +1,201 @@ +package agent + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/devsy-org/devsy/pkg/devcontainer/config" +) + +const ( + canonicalTimeoutField = "agent.containerInactivityTimeout" + legacyTimeoutField = "agent.kubernetes.inactivityTimeout" + + // LifecycleOwnerManagedDaemon marks a container whose main process is the + // Devsy daemon and owns inactivity shutdown. + LifecycleOwnerManagedDaemon = "managedDaemon" +) + +// ErrInactivityExpired is returned by the lifecycle loop when the idle timeout +// elapsed. It is the normal way a managed container asks to stop. +var ErrInactivityExpired = errors.New("inactivity timeout reached") + +// LifecycleConfig tells the daemon who owns inactivity shutdown. +type LifecycleConfig struct { + // Owner is LifecycleOwnerManagedDaemon when the daemon is the container's main + // process and owns the timer. + Owner string `json:"owner,omitempty"` + // SetupArmsTimer means the daemon starts unarmed and container setup arms it. + SetupArmsTimer bool `json:"setupArmsTimer,omitempty"` +} + +// Managed reports whether the daemon owns the inactivity timer. +func (l LifecycleConfig) Managed() bool { + return l.Owner == LifecycleOwnerManagedDaemon +} + +// LifecycleOptions are the explicit lifecycle inputs for building a daemon config. +type LifecycleOptions struct { + // Managed marks the daemon as the container's main process and timer owner. + Managed bool + // Timeout is the effective inactivity timeout; zero disables it. + Timeout time.Duration +} + +// ResolveManagedLifecycle selects ownership for a managed-capable driver's topology. +// Dockerless owns the pod process; its nested workspace cannot stop that pod. +func ResolveManagedLifecycle( + timeout time.Duration, + dockerless, imageEntrypoint bool, +) (LifecycleOptions, error) { + if dockerless { + if timeout > 0 { + return LifecycleOptions{}, fmt.Errorf( + "inactivity timeout %s is not supported with dockerless builds because the nested "+ + "workspace container does not end the pod when it exits; unset the inactivity "+ + "timeout or disable dockerless", timeout) + } + return LifecycleOptions{}, nil + } + if imageEntrypoint { + if timeout > 0 { + return LifecycleOptions{}, fmt.Errorf( + "inactivity timeout %s requires the Devsy daemon to be the container's main "+ + "process, but overrideCommand is false and the image entrypoint would run "+ + "instead; set overrideCommand to true or unset the inactivity timeout", timeout) + } + return LifecycleOptions{}, nil + } + return LifecycleOptions{Managed: true, Timeout: timeout}, nil +} + +// TimeoutInputs are the configured values that decide the effective timeout. +type TimeoutInputs struct { + // Canonical is agent.containerInactivityTimeout. + Canonical string + // Legacy is agent.kubernetes.inactivityTimeout, used only when Canonical is empty. + Legacy string + ShutdownAction string + DisableDaemon bool +} + +// ResolveInactivityTimeout returns the single effective inactivity timeout for a +// managed container. Zero means disabled. +func ResolveInactivityTimeout(in TimeoutInputs) (time.Duration, error) { + timeout, err := configuredTimeout(in) + if err != nil { + return 0, err + } + if timeout == 0 || in.DisableDaemon || in.ShutdownAction == config.ShutdownActionNone { + return 0, nil + } + if in.ShutdownAction == config.ShutdownActionStopCompose { + return 0, fmt.Errorf( + "inactivity timeout %s is set but shutdownAction %q is not supported by the kubernetes driver", + timeout, + config.ShutdownActionStopCompose, + ) + } + return timeout, nil +} + +// configuredTimeout merges the canonical field and its legacy alias. +func configuredTimeout(in TimeoutInputs) (time.Duration, error) { + canonical, err := parseTimeoutField(canonicalTimeoutField, in.Canonical) + if err != nil { + return 0, err + } + legacy, err := parseTimeoutField(legacyTimeoutField, in.Legacy) + if err != nil { + return 0, err + } + + hasCanonical := strings.TrimSpace(in.Canonical) != "" + if !hasCanonical { + return legacy, nil + } + if strings.TrimSpace(in.Legacy) != "" && canonical != legacy { + return 0, fmt.Errorf( + "%s (%q) conflicts with deprecated alias %s (%q); set only %s", + canonicalTimeoutField, + in.Canonical, + legacyTimeoutField, + in.Legacy, + canonicalTimeoutField, + ) + } + return canonical, nil +} + +func parseTimeoutField(field, value string) (time.Duration, error) { + value = strings.TrimSpace(value) + if value == "" { + return 0, nil + } + d, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("parse %s %q: %w", field, value, err) + } + if d < 0 { + return 0, fmt.Errorf("%s %q must not be negative", field, value) + } + return d, nil +} + +// FormatTimeout renders a timeout for DaemonConfig/ContainerWorkspaceInfo; zero is "". +func FormatTimeout(d time.Duration) string { + if d <= 0 { + return "" + } + return d.String() +} + +// DecodeDaemonConfig parses the base64 JSON produced by GetEncodedWorkspaceDaemonConfig. +func DecodeDaemonConfig(encoded string) (*DaemonConfig, error) { + raw, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil, fmt.Errorf("decode daemon config: %w", err) + } + cfg := &DaemonConfig{} + if err := json.Unmarshal(raw, cfg); err != nil { + return nil, fmt.Errorf("unmarshal daemon config: %w", err) + } + return cfg, nil +} + +// EncodeDaemonConfig is the inverse of DecodeDaemonConfig. +func EncodeDaemonConfig(cfg *DaemonConfig) (string, error) { + raw, err := json.Marshal(cfg) + if err != nil { + return "", fmt.Errorf("marshal daemon config: %w", err) + } + return base64.StdEncoding.EncodeToString(raw), nil +} + +// RefreshLifecycle rewrites the generated lifecycle policy of a stored encoded +// daemon config, leaving platform and SSH settings untouched. An empty stored +// config yields a minimal timeout-only config. +func RefreshLifecycle( + encoded string, + lifecycle LifecycleOptions, + shutdownAction string, +) (string, error) { + cfg := &DaemonConfig{} + if encoded != "" { + var err error + if cfg, err = DecodeDaemonConfig(encoded); err != nil { + return "", err + } + } + cfg.Timeout = FormatTimeout(lifecycle.Timeout) + cfg.ShutdownAction = shutdownAction + cfg.Lifecycle = LifecycleConfig{} + if lifecycle.Managed { + cfg.Lifecycle = LifecycleConfig{Owner: LifecycleOwnerManagedDaemon, SetupArmsTimer: true} + } + return EncodeDaemonConfig(cfg) +} diff --git a/pkg/daemon/agent/container_lifecycle_test.go b/pkg/daemon/agent/container_lifecycle_test.go new file mode 100644 index 0000000000..bdeaf7e3b1 --- /dev/null +++ b/pkg/daemon/agent/container_lifecycle_test.go @@ -0,0 +1,171 @@ +package agent + +import ( + "testing" + "time" + + "github.com/devsy-org/api/pkg/devsy" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + provider2 "github.com/devsy-org/devsy/pkg/provider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testWorkdir = "/workspaces/app" + +func TestResolveInactivityTimeout(t *testing.T) { + tests := []struct { + name string + in TimeoutInputs + want time.Duration + }{ + {name: "empty disables", in: TimeoutInputs{}}, + {name: "zero disables", in: TimeoutInputs{Canonical: "0"}}, + {name: "zero duration disables", in: TimeoutInputs{Canonical: "0s"}}, + {name: "positive", in: TimeoutInputs{Canonical: "10m"}, want: 10 * time.Minute}, + {name: "legacy alone", in: TimeoutInputs{Legacy: "5m"}, want: 5 * time.Minute}, + { + name: "equal aliases", + in: TimeoutInputs{Canonical: "1m", Legacy: "60s"}, + want: time.Minute, + }, + { + name: "shutdown none disables", + in: TimeoutInputs{Canonical: "1m", ShutdownAction: config.ShutdownActionNone}, + }, + {name: "disabled daemon disables", in: TimeoutInputs{Canonical: "1m", DisableDaemon: true}}, + { + name: "stopContainer enabled", + in: TimeoutInputs{ + Canonical: "1m", + ShutdownAction: config.ShutdownActionStopContainer, + }, + want: time.Minute, + }, + { + name: "stopCompose without timeout is fine", + in: TimeoutInputs{ShutdownAction: config.ShutdownActionStopCompose}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ResolveInactivityTimeout(tt.in) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestResolveInactivityTimeoutRejectsBadConfiguration(t *testing.T) { + const malformed = "soon" + tests := []struct { + name string + in TimeoutInputs + wantErr string + }{ + {"malformed", TimeoutInputs{Canonical: malformed}, "agent.containerInactivityTimeout"}, + { + "malformed legacy", + TimeoutInputs{Legacy: malformed}, + "agent.kubernetes.inactivityTimeout", + }, + { + "malformed legacy beside canonical", + TimeoutInputs{Canonical: "10m", Legacy: malformed}, + "agent.kubernetes.inactivityTimeout", + }, + {"negative", TimeoutInputs{Canonical: "-1m"}, "must not be negative"}, + { + "conflicting aliases", + TimeoutInputs{Canonical: "1m", Legacy: "2m"}, + "conflicts with deprecated alias", + }, + { + "stopCompose rejected", + TimeoutInputs{Canonical: "1m", ShutdownAction: config.ShutdownActionStopCompose}, + "not supported by the kubernetes driver", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := ResolveInactivityTimeout(tt.in) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestBuildWorkspaceDaemonConfig_ManagedWithoutPlatform(t *testing.T) { + merged := &config.MergedDevContainerConfig{} + merged.ShutdownAction = config.ShutdownActionStopContainer + merged.RemoteUser = "vscode" + + cfg, err := BuildWorkspaceDaemonConfig( + devsy.PlatformOptions{}, + &provider2.Workspace{}, + &config.SubstitutionContext{ContainerWorkspaceFolder: testWorkdir}, + merged, + LifecycleOptions{Managed: true, Timeout: time.Minute}, + ) + require.NoError(t, err) + + assert.Equal(t, "1m0s", cfg.Timeout) + assert.Equal(t, config.ShutdownActionStopContainer, cfg.ShutdownAction) + assert.True(t, cfg.Lifecycle.Managed()) + assert.True(t, cfg.Lifecycle.SetupArmsTimer) + assert.Empty(t, cfg.Ssh, "a timeout-only config must not request the embedded SSH server") + assert.Empty(t, cfg.Platform.AccessKey) +} + +func TestBuildWorkspaceDaemonConfig_PlatformKeepsNetworking(t *testing.T) { + merged := &config.MergedDevContainerConfig{} + merged.ShutdownAction = config.ShutdownActionNone + + cfg, err := BuildWorkspaceDaemonConfig( + devsy.PlatformOptions{AccessKey: "key", PlatformHost: "p", WorkspaceHost: "w"}, + &provider2.Workspace{}, + &config.SubstitutionContext{ContainerWorkspaceFolder: testWorkdir}, + merged, + LifecycleOptions{}, + ) + require.NoError(t, err) + + assert.Equal(t, "key", cfg.Platform.AccessKey) + assert.Equal(t, SshConfig{Workdir: testWorkdir, User: "root"}, cfg.Ssh) + assert.Equal(t, config.ShutdownActionNone, cfg.ShutdownAction) + assert.False(t, cfg.Lifecycle.Managed()) +} + +func TestRefreshLifecycleKeepsPlatformAndReplacesPolicy(t *testing.T) { + stored, err := EncodeDaemonConfig(&DaemonConfig{ + Platform: devsy.PlatformOptions{AccessKey: "key"}, + Ssh: SshConfig{User: "root"}, + Timeout: "5m", + ShutdownAction: config.ShutdownActionStopContainer, + }) + require.NoError(t, err) + + refreshed, err := RefreshLifecycle( + stored, LifecycleOptions{Managed: true, Timeout: 2 * time.Minute}, + config.ShutdownActionStopContainer, + ) + require.NoError(t, err) + cfg, err := DecodeDaemonConfig(refreshed) + require.NoError(t, err) + + assert.Equal(t, "2m0s", cfg.Timeout) + assert.True(t, cfg.Lifecycle.Managed()) + assert.Equal(t, "key", cfg.Platform.AccessKey) + assert.Equal(t, "root", cfg.Ssh.User) + assert.Equal(t, config.ShutdownActionStopContainer, cfg.ShutdownAction) +} + +func TestRefreshLifecycleKeepsZeroTimeoutManaged(t *testing.T) { + refreshed, err := RefreshLifecycle("", LifecycleOptions{Managed: true}, "") + require.NoError(t, err) + cfg, err := DecodeDaemonConfig(refreshed) + require.NoError(t, err) + + assert.Empty(t, cfg.Timeout) + assert.True(t, cfg.Lifecycle.Managed()) + assert.Empty(t, cfg.Ssh) +} diff --git a/pkg/daemon/agent/daemon.go b/pkg/daemon/agent/daemon.go index c8f16cc6e8..99c41aba9a 100644 --- a/pkg/daemon/agent/daemon.go +++ b/pkg/daemon/agent/daemon.go @@ -1,8 +1,6 @@ package agent import ( - "encoding/base64" - "encoding/json" "fmt" "os" "os/exec" @@ -32,6 +30,7 @@ type DaemonConfig struct { Ssh SshConfig `json:"ssh"` Timeout string `json:"timeout"` ShutdownAction string `json:"shutdownAction,omitempty"` + Lifecycle LifecycleConfig `json:"lifecycle,omitzero"` } func BuildWorkspaceDaemonConfig( @@ -39,6 +38,7 @@ func BuildWorkspaceDaemonConfig( workspaceConfig *provider2.Workspace, substitutionContext *config.SubstitutionContext, mergedConfig *config.MergedDevContainerConfig, + lifecycle LifecycleOptions, ) (*DaemonConfig, error) { var workdir string if workspaceConfig.Source.GitSubPath != "" { @@ -55,25 +55,28 @@ func BuildWorkspaceDaemonConfig( workdir = substitutionContext.ContainerWorkspaceFolder } - // Get remote user; default to "root" if empty. - user := mergedConfig.RemoteUser - if user == "" { - user = "root" + daemonConfig := &DaemonConfig{ + Platform: platformOptions, + Timeout: FormatTimeout(lifecycle.Timeout), + ShutdownAction: mergedConfig.ShutdownAction, + } + if lifecycle.Managed { + daemonConfig.Lifecycle = LifecycleConfig{ + Owner: LifecycleOwnerManagedDaemon, + SetupArmsTimer: true, + } } - // build info isn't required in the workspace and can be omitted - platformOptions.Build = nil - - shutdownAction := mergedConfig.ShutdownAction - - daemonConfig := &DaemonConfig{ - Platform: platformOptions, - Ssh: SshConfig{ - Workdir: workdir, - User: user, - }, - ShutdownAction: shutdownAction, + // A timeout-only config must not request the embedded SSH or network server. + if platformOptions.AccessKey != "" { + user := mergedConfig.RemoteUser + if user == "" { + user = "root" + } + daemonConfig.Ssh = SshConfig{Workdir: workdir, User: user} } + // build info isn't required in the workspace and can be omitted + daemonConfig.Platform.Build = nil return daemonConfig, nil } @@ -83,23 +86,20 @@ func GetEncodedWorkspaceDaemonConfig( workspaceConfig *provider2.Workspace, substitutionContext *config.SubstitutionContext, mergedConfig *config.MergedDevContainerConfig, + lifecycle LifecycleOptions, ) (string, error) { daemonConfig, err := BuildWorkspaceDaemonConfig( platformOptions, workspaceConfig, substitutionContext, mergedConfig, + lifecycle, ) if err != nil { return "", err } - data, err := json.Marshal(daemonConfig) - if err != nil { - return "", err - } - encoded := base64.StdEncoding.EncodeToString(data) - return encoded, nil + return EncodeDaemonConfig(daemonConfig) } const systemdDir = "/etc/systemd/system" diff --git a/pkg/daemon/agent/daemon_test.go b/pkg/daemon/agent/daemon_test.go index a19c3b10f5..d205f3b424 100644 --- a/pkg/daemon/agent/daemon_test.go +++ b/pkg/daemon/agent/daemon_test.go @@ -155,6 +155,7 @@ func TestBuildWorkspaceDaemonConfig_ShutdownAction(t *testing.T) { &provider2.Workspace{}, &config.SubstitutionContext{}, merged, + LifecycleOptions{}, ) if err != nil { t.Fatalf("unexpected error: %v", err) diff --git a/pkg/daemon/agent/lifecycle_admission_linux_test.go b/pkg/daemon/agent/lifecycle_admission_linux_test.go new file mode 100644 index 0000000000..fcf5d9059c --- /dev/null +++ b/pkg/daemon/agent/lifecycle_admission_linux_test.go @@ -0,0 +1,232 @@ +//go:build linux + +package agent + +import ( + "bufio" + "context" + "encoding/json" + "io" + "net" + "os" + "os/exec" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type capacityPeerReport struct { + lifecyclePeerIdentity + Live int `json:"live"` +} + +type capacityPeer struct { + cmd *exec.Cmd + input io.WriteCloser + scanner *bufio.Scanner + report capacityPeerReport +} + +func startCapacityPeer(t *testing.T, dir string, uid uint32, mode string) *capacityPeer { + t.Helper() + binary, err := os.Executable() + require.NoError(t, err) + // #nosec G204 -- re-execute the current test binary with fixed test selection. + cmd := exec.Command(binary, "-test.run=^TestLifecycleCapacityPeerChild$") + cmd.Env = append(os.Environ(), "DEVSY_CAPACITY_DIR="+dir, "DEVSY_CAPACITY_MODE="+mode) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: uid, Gid: uid, Groups: []uint32{uid}}, + Pdeathsig: syscall.SIGKILL, + } + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + input, err := cmd.StdinPipe() + require.NoError(t, err) + cmd.Stderr = os.Stderr + require.NoError(t, cmd.Start()) + peer := &capacityPeer{cmd: cmd, input: input, scanner: bufio.NewScanner(stdout)} + t.Cleanup(func() { _ = peer.input.Close(); _ = cmd.Process.Kill(); _ = cmd.Wait() }) + require.NoError(t, json.Unmarshal([]byte(readPeerLine(t, peer.scanner)), &peer.report)) + require.Equal(t, int(uid), peer.report.UID) + require.Equal(t, int(uid), peer.report.GID) + require.Equal(t, []int{int(uid)}, peer.report.Groups) + require.Equal(t, cmd.Process.Pid, peer.report.PID) + return peer +} + +func (p *capacityPeer) close(t *testing.T) { + t.Helper() + _, err := io.WriteString(p.input, "exit\n") + require.NoError(t, err) + require.NoError(t, p.cmd.Wait()) +} + +func TestLowerTrustPendingClientsPreservePIDUIDAndOwnerFairness(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires real cross-UID clients under a root owner") + } + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + pending := startCapacityPeer(t, dir, 65534, lifecyclePendingMode) + require.Equal(t, maxLifecyclePIDClients, pending.report.Live) + allowed := startCapacityPeer(t, dir, 65534, "activity") + require.Equal(t, maxLifecyclePIDClients, allowed.report.Live) + denied := startCapacityPeer(t, dir, 65534, "activity") + require.Zero(t, denied.report.Live, "UID exhaustion must not begin unregistered work") + otherUID := startCapacityPeer(t, dir, 65533, "activity") + require.Equal(t, maxLifecyclePIDClients, otherUID.report.Live) + control := connectNow(t, dir) + require.NoError(t, control.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, control.Send(LifecycleStateReady, "1m", "")) + for _, peer := range []*capacityPeer{pending, allowed, denied, otherUID} { + peer.close(t) + } + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + time.Second, + time.Millisecond, + ) +} + +func TestRootAdmissionSurvivesLowerTrustReleaseOnlySaturationAndDeath(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires real cross-UID clients under a root owner") + } + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + var peers []*capacityPeer + for i := range maxOtherLifecycleClients / maxLifecyclePIDClients { + peer := startCapacityPeer(t, dir, uint32(65000+i), "release") + require.Equal(t, maxLifecyclePIDClients, peer.report.Live) + peers = append(peers, peer) + } + denied := startCapacityPeer(t, dir, 64999, "activity") + require.Zero(t, denied.report.Live, "lower-trust saturation must not begin unregistered work") + control := connectNow(t, dir) + require.NoError(t, control.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, control.Send(LifecycleStateReady, "1m", "")) + assert.True(t, idleForAnHour(monitor).Expired, "release-only sockets do not retain work") + require.NoError(t, peers[0].cmd.Process.Kill()) + require.Error(t, peers[0].cmd.Wait()) + owner.pruneDeadClients(monitor, time.Now()) + replacement := startCapacityPeer(t, dir, 64999, "activity") + require.Equal(t, maxLifecyclePIDClients, replacement.report.Live) + for _, peer := range append(peers[1:], denied, replacement) { + peer.close(t) + } + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + time.Second, + time.Millisecond, + ) +} + +func TestLifecycleCapacityPeerChild(t *testing.T) { + mode := os.Getenv("DEVSY_CAPACITY_MODE") + if mode == "" { + return + } + dir := os.Getenv("DEVSY_CAPACITY_DIR") + var clients []*LifecycleClient + var pending []net.Conn + if mode == lifecyclePendingMode { + pending = openPendingCapacityConnections(t, dir) + } else { + clients = openRegisteredCapacityConnections(dir, mode) + } + defer func() { + for _, conn := range pending { + _ = conn.Close() + } + for _, client := range clients { + _ = client.Close() + } + }() + groups, err := os.Getgroups() + require.NoError(t, err) + report := capacityPeerReport{lifecyclePeerIdentity: lifecyclePeerIdentity{ + UID: os.Getuid(), GID: os.Getgid(), Groups: groups, PID: os.Getpid(), + }, Live: len(clients)} + if mode == lifecyclePendingMode { + report.Live = countOpenPendingConnections(pending) + } + require.NoError(t, json.NewEncoder(os.Stdout).Encode(report)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if mode == "release" { + go chatterReleaseClients(ctx, clients) + } + _, err = bufio.NewReader(os.Stdin).ReadString('\n') + require.NoError(t, err) +} + +func openPendingCapacityConnections(t *testing.T, dir string) []net.Conn { + t.Helper() + var conns []net.Conn + for range maxLifecycleClients { + // #nosec G704 -- Unix IPC in this test's private fixture, not a network destination. + conn, err := net.DialTimeout("unix", dir+"/"+socketFileName, time.Second) + require.NoError(t, err) + conns = append(conns, conn) + } + return conns +} + +func countOpenPendingConnections(conns []net.Conn) int { + live := 0 + for _, conn := range conns { + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Millisecond)) + var buf [1]byte + _, err := conn.Read(buf[:]) + if timeout, ok := err.(net.Error); ok && timeout.Timeout() { + live++ + } + } + return live +} + +func openRegisteredCapacityConnections(dir, mode string) []*LifecycleClient { + var clients []*LifecycleClient + state := LifecycleStateActivity + if mode == "release" { + state = LifecycleStateRelease + } + for range maxLifecycleClients { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + client, err := ConnectLifecycle(ctx, dir) + cancel() + if err != nil { + break + } + if err := client.Send(state, "", ""); err != nil { + _ = client.Close() + break + } + clients = append(clients, client) + } + return clients +} + +func chatterReleaseClients(ctx context.Context, clients []*LifecycleClient) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + for _, client := range clients { + _ = client.Send(LifecycleStateRelease, "", "") + } + } + } +} diff --git a/pkg/daemon/agent/lifecycle_deadline_test.go b/pkg/daemon/agent/lifecycle_deadline_test.go new file mode 100644 index 0000000000..6d883dffae --- /dev/null +++ b/pkg/daemon/agent/lifecycle_deadline_test.go @@ -0,0 +1,147 @@ +package agent + +import ( + "encoding/json" + "io" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const lifecyclePendingMode = "pending" + +type lifecycleDeadlineProbe struct { + opened time.Time + closed <-chan time.Time +} + +func TestUnprotectedChatterAndExactRetriesCannotExtendConnectionLifetime(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + fresh := newLifecycleDeadlineProbe(t, owner, dir, "fresh") + retry := newLifecycleDeadlineProbe(t, owner, dir, "retry") + pending := newLifecycleDeadlineProbe(t, owner, dir, lifecyclePendingMode) + protected := newLifecycleDeadlineProbe(t, owner, dir, "invalid-ready") + slow := newLifecycleDeadlineProbe(t, owner, dir, "slowloris") + for _, probe := range []*lifecycleDeadlineProbe{fresh, retry, pending, slow} { + select { + case closed := <-probe.closed: + assert.WithinDuration(t, probe.opened.Add(ackWaitTimeout), closed, 2*time.Second) + case <-time.After(ackWaitTimeout + 2*time.Second): + t.Fatal("unprotected connection outlived its original admission deadline") + } + } + select { + case <-protected.closed: + t.Fatal("a failed policy update removed an earlier valid setup hold") + default: + } + assert.False(t, idleForAnHour(monitor).Expired, "live protected work has no connection lease") +} + +func newLifecycleDeadlineProbe( + t *testing.T, + owner *LifecycleOwner, + dir, mode string, +) *lifecycleDeadlineProbe { + t.Helper() + opened := time.Now() + conn, err := net.DialTimeout("unix", filepath.Join(dir, socketFileName), time.Second) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + record := ControlRecord{ + Version: LifecycleProtocolVersion, Generation: owner.generation, + ClientID: mode, Sequence: 1, PID: os.Getpid(), StartTicks: procStartTicks(os.Getpid()), + State: LifecycleStateRelease, + } + if mode != lifecyclePendingMode && mode != "slowloris" { + initializeDeadlineProbe(t, conn, &record, mode) + } + closed := make(chan time.Time, 1) + done := make(chan struct{}) + go func() { _, _ = io.Copy(io.Discard, conn); closed <- time.Now(); close(done) }() + if mode == "fresh" || mode == "retry" { + go chatterLifecycleWire(conn, record, mode == "retry", done) + } + if mode == "slowloris" { + go chatterIncompleteLifecycleFrame(conn, done) + } + return &lifecycleDeadlineProbe{opened: opened, closed: closed} +} + +func initializeDeadlineProbe(t *testing.T, conn net.Conn, record *ControlRecord, mode string) { + t.Helper() + if mode == "invalid-ready" { + record.State, record.Timeout = LifecycleStateSetup, "1m" + } + require.NoError(t, json.NewEncoder(conn).Encode(record)) + var ack AckRecord + require.NoError(t, json.NewDecoder(conn).Decode(&ack)) + require.Empty(t, ack.Error) + if mode != "invalid-ready" { + return + } + record.State, record.Timeout, record.Sequence = LifecycleStateReady, "invalid", 2 + require.NoError(t, json.NewEncoder(conn).Encode(record)) + require.NoError(t, json.NewDecoder(conn).Decode(&ack)) + require.NotEmpty(t, ack.Error, "the policy transition must actually be rejected") +} + +func chatterLifecycleWire(conn net.Conn, record ControlRecord, retry bool, done <-chan struct{}) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + encoder := json.NewEncoder(conn) + for { + select { + case <-done: + return + case <-ticker.C: + if !retry { + record.Sequence++ + } + if err := encoder.Encode(record); err != nil { + return + } + } + } +} + +func TestMalformedLifecycleFrameDoesNotStopTheOwner(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + conn, err := net.DialTimeout("unix", filepath.Join(dir, socketFileName), time.Second) + require.NoError(t, err) + defer func() { _ = conn.Close() }() + _, err = io.WriteString(conn, "{\n") + require.NoError(t, err) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(time.Second))) + var buf [1]byte + _, err = conn.Read(buf[:]) + assert.ErrorIs(t, err, io.EOF) + client := connectNow(t, dir) + require.NoError(t, client.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, client.Send(LifecycleStateReady, "1m", "")) +} + +func chatterIncompleteLifecycleFrame(conn net.Conn, done <-chan struct{}) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-done: + return + case <-ticker.C: + if _, err := io.WriteString(conn, " "); err != nil { + return + } + } + } +} diff --git a/pkg/daemon/agent/lifecycle_identity_linux_test.go b/pkg/daemon/agent/lifecycle_identity_linux_test.go new file mode 100644 index 0000000000..c6bf28a86c --- /dev/null +++ b/pkg/daemon/agent/lifecycle_identity_linux_test.go @@ -0,0 +1,207 @@ +//go:build linux + +package agent + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "net" + "os" + "os/exec" + "strconv" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type inheritedPeerReport struct { + PID int `json:"pid"` + HolderPID int `json:"holderPid"` +} + +func TestInheritedSocketCannotRetainActivityAfterOriginalProcessDies(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires an actual unprivileged peer and inherited descriptor") + } + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + binary, err := os.Executable() + require.NoError(t, err) + // #nosec G204 -- re-execute the current test binary with fixed test selection. + cmd := exec.Command(binary, "-test.run=^TestInheritedSocketParentChild$") + cmd.Env = append(os.Environ(), "DEVSY_INHERITED_SOCKET_DIR="+dir) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: 65534, Gid: 65534, Groups: []uint32{65534}}, + Pdeathsig: syscall.SIGKILL, + } + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + input, err := cmd.StdinPipe() + require.NoError(t, err) + defer func() { _ = input.Close() }() + cmd.Stderr = os.Stderr + require.NoError(t, cmd.Start()) + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + var report inheritedPeerReport + require.NoError(t, json.Unmarshal([]byte(readPeerLine(t, bufio.NewScanner(stdout))), &report)) + require.Equal(t, cmd.Process.Pid, report.PID) + t.Cleanup(func() { + _ = syscall.Kill(report.HolderPID, syscall.SIGKILL) + _, _ = syscall.Wait4(report.HolderPID, nil, 0, nil) + }) + assert.False(t, idleForAnHour(monitor).Expired) + require.NoError(t, cmd.Process.Kill()) + require.Eventually( + t, + func() bool { return isUnreapedZombie(report.PID) }, + time.Second, + time.Millisecond, + "the opener must be dead but still owned by this living parent", + ) + assertZombieReleaseBeforeReap(t, monitor, dir, report) + require.Error(t, cmd.Wait()) +} + +func assertZombieReleaseBeforeReap( + t *testing.T, + monitor *LifecycleMonitor, + dir string, + report inheritedPeerReport, +) { + t.Helper() + require.True( + t, + processExists(report.HolderPID), + "the inherited socket holder must remain alive", + ) + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + 2*time.Second, + time.Millisecond, + ) + require.True( + t, + processExists(report.HolderPID), + "EOF from holder exit must not explain release", + ) + assertReclaimedUIDBudget(t, dir) + require.True( + t, + isUnreapedZombie(report.PID), + "release and capacity recovery must precede reaping", + ) +} + +func assertReclaimedUIDBudget(t *testing.T, dir string) { + t.Helper() + first := startCapacityPeer(t, dir, 65534, "activity") + second := startCapacityPeer(t, dir, 65534, "activity") + third := startCapacityPeer(t, dir, 65534, "activity") + require.Equal(t, maxLifecyclePIDClients, first.report.Live) + require.Equal(t, maxLifecyclePIDClients, second.report.Live) + require.Zero(t, third.report.Live, "prune followed by EOF must not decrement UID usage twice") + first.close(t) + second.close(t) + third.close(t) +} + +func TestInheritedSocketParentChild(t *testing.T) { + dir := os.Getenv("DEVSY_INHERITED_SOCKET_DIR") + if dir == "" { + return + } + client, err := ConnectLifecycle(context.Background(), dir) + require.NoError(t, err) + defer func() { _ = client.Close() }() + require.NoError(t, client.Send(LifecycleStateActivity, "", "")) + file, err := client.conn.(*net.UnixConn).File() + require.NoError(t, err) + defer func() { _ = file.Close() }() + binary, err := os.Executable() + require.NoError(t, err) + // #nosec G204 -- re-execute the current test binary with fixed test selection. + holder := exec.Command(binary, "-test.run=^TestInheritedSocketHolderChild$") + holder.Env = append(os.Environ(), "DEVSY_INHERITED_SOCKET_HOLDER=1") + holder.ExtraFiles = []*os.File{file} + stdout, err := holder.StdoutPipe() + require.NoError(t, err) + holder.Stderr = os.Stderr + require.NoError(t, holder.Start()) + defer func() { _ = holder.Process.Kill(); _ = holder.Wait() }() + require.Equal(t, "holding", readPeerLine(t, bufio.NewScanner(stdout))) + require.NoError( + t, + json.NewEncoder(os.Stdout). + Encode(inheritedPeerReport{PID: os.Getpid(), HolderPID: holder.Process.Pid}), + ) + _, err = bufio.NewReader(os.Stdin).ReadString('\n') + require.NoError(t, err) +} + +func TestInheritedSocketHolderChild(t *testing.T) { + if os.Getenv("DEVSY_INHERITED_SOCKET_HOLDER") == "" { + return + } + file := os.NewFile(3, "inherited-lifecycle-socket") + require.NotNil(t, file) + defer func() { _ = file.Close() }() + info, err := file.Stat() + require.NoError(t, err) + require.Equal(t, os.ModeSocket, info.Mode().Type()) + _, err = os.Stdout.WriteString("holding\n") + require.NoError(t, err) + time.Sleep(time.Minute) +} + +func TestProcessStartIdentityRejectsReusedPID(t *testing.T) { + ticks, err := processStartIdentity(os.Getpid()) + require.NoError(t, err) + require.NotEmpty(t, ticks) + assert.True(t, sameProcess(os.Getpid(), ticks)) + assert.False( + t, + sameProcess(os.Getpid(), ""), + "a missing start identity must not reduce to PID liveness", + ) + assert.False( + t, + sameProcess(os.Getpid(), ticks+"0"), + "a live PID does not validate another process generation", + ) +} + +func TestLinuxLifecycleFailsClosedWithoutReadableProcIdentity(t *testing.T) { + if os.Getenv("DEVSY_MASKED_PROC_TEST") == "" { + t.Skip("run in the isolated container with /proc masked") + } + require.Empty(t, procStartTicks(os.Getpid())) + assert.False(t, sameProcess(os.Getpid(), "previous-process-generation")) + dir, err := os.MkdirTemp("/tmp", "ds-proc-") + require.NoError(t, err) + defer func() { _ = os.RemoveAll(dir) }() + owner, err := NewLifecycleOwner(dir) + if owner != nil { + _ = owner.Close() + } + assert.ErrorIs( + t, + err, + ErrLifecycleOwnerMissing, + "unverifiable process identity must not start an owner", + ) +} + +func isUnreapedZombie(pid int) bool { + raw, err := os.ReadFile( + "/proc/" + strconv.Itoa(pid) + "/status", + ) // #nosec G304 -- actual test child PID + return err == nil && bytes.Contains(raw, []byte("\nState:\tZ")) +} diff --git a/pkg/daemon/agent/lifecycle_monitor.go b/pkg/daemon/agent/lifecycle_monitor.go new file mode 100644 index 0000000000..e5faa58a16 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_monitor.go @@ -0,0 +1,241 @@ +package agent + +import ( + "context" + "errors" + "fmt" + "os" + "sync" + "time" + + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/log" +) + +const ( + idleCheckInterval = 500 * time.Millisecond + maxCheckInterval = time.Second + unreadableLogAfter = 3 +) + +// Decision is the outcome of one inactivity check. +type Decision struct { + Expired bool + ActivityUnreadable bool + Failures int +} + +// LifecycleMonitor serializes protection changes with the decision to stop. +type LifecycleMonitor struct { + mu sync.Mutex + timeout time.Duration + armed bool + armedAt time.Time + failures int + protected map[string]struct{} + stopping bool +} + +// Apply changes one client's protection. Acknowledgements are sent only after +// this method returns; no filesystem write is needed to retain protection. +func (m *LifecycleMonitor) Apply(rec ControlRecord, now time.Time) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.stopping { + return false, ErrLifecycleOwnerMissing + } + if rec.State == LifecycleStateActivity { + m.protect(rec.ClientID) + return false, nil + } + if rec.State == LifecycleStateRelease { + m.release(rec.ClientID, now) + return false, nil + } + timeout, err := parseLifecyclePolicy(rec) + if err != nil { + return false, err + } + m.timeout = timeout + m.armed = timeout > 0 && rec.ShutdownAction != config.ShutdownActionNone + if rec.State == LifecycleStateSetup { + m.protect(rec.ClientID) + return false, nil + } + delete(m.protected, rec.ClientID) + m.armedAt = now + m.failures = 0 + return m.armed, nil +} + +func parseLifecyclePolicy(rec ControlRecord) (time.Duration, error) { + switch rec.State { + case LifecycleStateSetup, LifecycleStateReady, LifecycleStateFailed: + default: + return 0, fmt.Errorf("unknown lifecycle state %q", rec.State) + } + if rec.Timeout == "" { + return 0, nil + } + timeout, err := time.ParseDuration(rec.Timeout) + if err != nil || timeout < 0 { + return 0, fmt.Errorf("invalid lifecycle timeout %q", rec.Timeout) + } + return timeout, nil +} + +// Release ends only this client's protection and starts a fresh idle interval. +func (m *LifecycleMonitor) Release(id string, now time.Time) { + m.mu.Lock() + defer m.mu.Unlock() + m.release(id, now) +} + +// ArmNow arms daemons that have no managed setup handshake. +func (m *LifecycleMonitor) ArmNow(timeout time.Duration, now time.Time) { + m.mu.Lock() + defer m.mu.Unlock() + m.timeout, m.armed, m.armedAt = timeout, timeout > 0, now +} + +// Evaluate observes the timer without committing the daemon to shutdown. +func (m *LifecycleMonitor) Evaluate(now, modTime time.Time, statErr error) Decision { + m.mu.Lock() + defer m.mu.Unlock() + return m.evaluate(now, modTime, statErr) +} + +// CheckInterval bounds the delay before observing idle time or process death. +func (m *LifecycleMonitor) CheckInterval() time.Duration { + m.mu.Lock() + defer m.mu.Unlock() + if !m.armed { + return idleCheckInterval + } + return min(maxCheckInterval, max(m.timeout/4, minTickInterval)) +} + +func (m *LifecycleMonitor) protect(id string) { + if m.protected == nil { + m.protected = make(map[string]struct{}) + } + m.protected[id] = struct{}{} +} + +func (m *LifecycleMonitor) release(id string, now time.Time) { + if _, ok := m.protected[id]; ok { + delete(m.protected, id) + m.armedAt = now + } +} + +func (m *LifecycleMonitor) evaluate(now, modTime time.Time, statErr error) Decision { + if !m.armed || len(m.protected) > 0 { + return Decision{} + } + if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { + m.failures++ + return Decision{ActivityUnreadable: true, Failures: m.failures} + } + m.failures = 0 + last := m.armedAt + if statErr == nil && modTime.After(last) { + last = modTime + } + return Decision{Expired: now.Sub(last) >= m.timeout} +} + +func (m *LifecycleMonitor) commitExpiry(now, modTime time.Time, statErr error) Decision { + m.mu.Lock() + defer m.mu.Unlock() + d := m.evaluate(now, modTime, statErr) + // A racing registration must be rejected once shutdown has won this lock. + m.stopping = m.stopping || d.Expired + return d +} + +// LifecycleLoop uses live registrations for managed containers and the legacy +// activity file only for daemons without a managed owner. +type LifecycleLoop struct { + Owner *LifecycleOwner + Monitor *LifecycleMonitor + ActivityPath string +} + +func lifecycleContextError(ctx context.Context) error { + cause := context.Cause(ctx) + if errors.Is(cause, context.Canceled) || errors.Is(cause, context.DeadlineExceeded) { + return nil + } + return cause +} + +func (l *LifecycleLoop) Run(ctx context.Context) error { + ctx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + stopOwner := l.startOwner(ctx, cancel) + defer stopOwner() + var unreadable bool + timer := time.NewTimer(0) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return lifecycleContextError(ctx) + case <-timer.C: + } + d, err := l.observe(time.Now()) + unreadable = logUnreadable(d, err, unreadable) + if d.Expired { + log.Infof("container idle for the configured inactivity timeout, shutting down") + return ErrInactivityExpired + } + timer.Reset(l.Monitor.CheckInterval()) + } +} + +func (l *LifecycleLoop) startOwner(ctx context.Context, cancel context.CancelCauseFunc) func() { + if l.Owner == nil { + return func() {} + } + stopped := make(chan struct{}) + go func() { + defer close(stopped) + err := l.Owner.Serve(ctx, l.Monitor) + if ctx.Err() == nil { + if err == nil { + err = ErrLifecycleOwnerMissing + } + cancel(err) + } + }() + return func() { cancel(nil); _ = l.Owner.Close(); <-stopped } +} + +func (l *LifecycleLoop) observe(now time.Time) (Decision, error) { + if l.Owner != nil { + l.Owner.pruneDeadClients(l.Monitor, now) + return l.Monitor.commitExpiry(now, time.Time{}, nil), nil + } + fi, err := os.Stat(l.ActivityPath) + var mod time.Time + if err == nil { + mod = fi.ModTime() + } + return l.Monitor.commitExpiry(now, mod, err), err +} + +func logUnreadable(d Decision, err error, was bool) bool { + switch { + case d.ActivityUnreadable && d.Failures >= unreadableLogAfter && !was: + log.Errorf( + "cannot read activity file; inactivity timeout is not effective until it is readable: %v", + err, + ) + return true + case !d.ActivityUnreadable && was: + log.Infof("activity file readable again; inactivity timeout is effective") + return false + } + return was +} diff --git a/pkg/daemon/agent/lifecycle_monitor_test.go b/pkg/daemon/agent/lifecycle_monitor_test.go new file mode 100644 index 0000000000..3351d1e4cd --- /dev/null +++ b/pkg/daemon/agent/lifecycle_monitor_test.go @@ -0,0 +1,173 @@ +package agent + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var t0 = time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + +func control(state LifecycleState, timeout, action string) ControlRecord { + return ControlRecord{ + Version: LifecycleProtocolVersion, + ClientID: "client", + State: state, + Timeout: timeout, + ShutdownAction: action, + } +} + +func TestMonitorStartsUnarmed(t *testing.T) { + assert.False(t, (&LifecycleMonitor{}).Evaluate(t0.Add(24*time.Hour), t0, nil).Expired) +} + +func TestMonitorSuspendedDuringSetupThenArmed(t *testing.T) { + m := &LifecycleMonitor{} + reset, err := m.Apply(control(LifecycleStateSetup, "1m", ""), t0) + require.NoError(t, err) + assert.False(t, reset) + assert.False(t, m.Evaluate(t0.Add(time.Hour), t0, nil).Expired) + ready := t0.Add(10 * time.Minute) + _, err = m.Apply(control(LifecycleStateReady, "1m", ""), ready) + require.NoError(t, err) + assert.False(t, m.Evaluate(ready.Add(59*time.Second), t0, nil).Expired) + assert.True(t, m.Evaluate(ready.Add(time.Minute), t0, nil).Expired) +} + +func TestMonitorLegacyActivityKeepsWorkspaceAlive(t *testing.T) { + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) + require.NoError(t, err) + recent := t0.Add(50 * time.Second) + assert.False(t, m.Evaluate(t0.Add(100*time.Second), recent, nil).Expired) + assert.True(t, m.Evaluate(t0.Add(111*time.Second), recent, nil).Expired) +} + +func TestMonitorFailedSetupReleasesSuspensionWithFreshInterval(t *testing.T) { + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateSetup, "1m", ""), t0) + require.NoError(t, err) + failedAt := t0.Add(30 * time.Minute) + _, err = m.Apply(control(LifecycleStateFailed, "1m", ""), failedAt) + require.NoError(t, err) + assert.False(t, m.Evaluate(failedAt.Add(30*time.Second), t0, nil).Expired) + assert.True(t, m.Evaluate(failedAt.Add(time.Minute), t0, nil).Expired) +} + +func TestMonitorReleaseIsIdempotentAndStartsIdleAtDisconnect(t *testing.T) { + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateSetup, "1m", ""), t0) + require.NoError(t, err) + m.Release("client", t0.Add(time.Hour)) + m.Release("client", t0.Add(time.Hour+30*time.Second)) + assert.False(t, m.Evaluate(t0.Add(time.Hour+59*time.Second), t0, nil).Expired) + assert.True(t, m.Evaluate(t0.Add(time.Hour+time.Minute), t0, nil).Expired) +} + +func TestMonitorDisabledPolicyDisarms(t *testing.T) { + for _, rec := range []ControlRecord{ + control(LifecycleStateReady, "", ""), control(LifecycleStateReady, "1m", "none"), + } { + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) + require.NoError(t, err) + _, err = m.Apply(rec, t0) + require.NoError(t, err) + assert.False(t, m.Evaluate(t0.Add(24*time.Hour), t0, nil).Expired) + } +} + +func TestMonitorInvalidPolicyDoesNotReplaceCurrentDeadline(t *testing.T) { + m := &LifecycleMonitor{} + _, err := m.Apply(control(LifecycleStateReady, "1m", ""), t0) + require.NoError(t, err) + for _, rec := range []ControlRecord{control(LifecycleStateReady, "later", ""), control("bogus", "1h", "")} { + _, err := m.Apply(rec, t0.Add(30*time.Second)) + require.Error(t, err) + } + assert.True(t, m.Evaluate(t0.Add(time.Minute), t0, nil).Expired) +} + +func TestMonitorUnreadableLegacyActivityFailsSafe(t *testing.T) { + m := &LifecycleMonitor{} + m.ArmNow(time.Minute, t0) + denied := &fs.PathError{Op: "stat", Path: "activity", Err: fs.ErrPermission} + d := m.Evaluate(t0.Add(time.Hour), time.Time{}, denied) + assert.False(t, d.Expired) + assert.True(t, d.ActivityUnreadable) +} + +func TestMonitorRejectsNewWorkOnceShutdownIsCommitted(t *testing.T) { + m := &LifecycleMonitor{} + m.ArmNow(time.Minute, t0) + require.True(t, m.commitExpiry(t0.Add(time.Minute), t0, nil).Expired) + _, err := m.Apply(control(LifecycleStateSetup, "1m", ""), t0.Add(time.Minute)) + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) +} + +func TestMonitorRegistrationPreventsCommittingShutdown(t *testing.T) { + m := &LifecycleMonitor{} + m.ArmNow(time.Minute, t0) + _, err := m.Apply(control(LifecycleStateActivity, "", ""), t0) + require.NoError(t, err) + assert.False(t, m.commitExpiry(t0.Add(time.Hour), t0, nil).Expired) + m.Release("client", t0.Add(time.Hour)) + assert.True(t, m.commitExpiry(t0.Add(time.Hour+time.Minute), t0, nil).Expired) +} + +func TestLifecycleLoopStopsIdleManagedDaemon(t *testing.T) { + owner, dir := newTestOwner(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- (&LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}}).Run(ctx) }() + client := connectNow(t, dir) + require.NoError(t, client.Send(LifecycleStateReady, "300ms", "")) + assert.ErrorIs(t, <-done, ErrInactivityExpired) +} + +func TestActivityWriterTouchesImmediatelyPeriodicallyAndOnExit(t *testing.T) { + ticks := make(chan time.Time) + var mu sync.Mutex + var touched []time.Time + count := func() int { mu.Lock(); defer mu.Unlock(); return len(touched) } + w := ActivityWriter{ + Path: "activity", Interval: time.Second, + Touch: func(_ string, now time.Time) error { + mu.Lock() + defer mu.Unlock() + touched = append(touched, now) + return nil + }, + Now: func() time.Time { return t0 }, + Tick: func(time.Duration) (<-chan time.Time, func()) { return ticks, func() {} }, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { w.Run(ctx); close(done) }() + defer func() { cancel(); <-done }() + require.Eventually(t, func() bool { return count() == 1 }, time.Second, time.Millisecond) + ticks <- time.Time{} + require.Eventually(t, func() bool { return count() == 2 }, time.Second, time.Millisecond) + cancel() + <-done + assert.Equal(t, 3, count()) +} + +func TestTouchByWriteAdvancesModTimeForNonOwners(t *testing.T) { + path := filepath.Join(t.TempDir(), "activity") + require.NoError(t, TouchActivity(path, t0)) + require.NoError(t, os.Chtimes(path, t0, t0)) + require.NoError(t, touchByWrite(path)) + info, err := os.Stat(path) + require.NoError(t, err) + assert.WithinDuration(t, time.Now(), info.ModTime(), 2*time.Second) +} diff --git a/pkg/daemon/agent/lifecycle_peer_linux_test.go b/pkg/daemon/agent/lifecycle_peer_linux_test.go new file mode 100644 index 0000000000..372b048665 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_peer_linux_test.go @@ -0,0 +1,135 @@ +//go:build linux + +package agent + +import ( + "bufio" + "context" + "encoding/json" + "os" + "os/exec" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type lifecyclePeerIdentity struct { + UID int `json:"uid"` + GID int `json:"gid"` + Groups []int `json:"groups"` + PID int `json:"pid"` +} + +func TestNonrootLifecyclePeerChild(t *testing.T) { + dir := os.Getenv("DEVSY_NONROOT_LIFECYCLE_CHILD") + if dir == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client, err := ConnectLifecycle(ctx, dir) + require.NoError(t, err) + defer func() { _ = client.Close() }() + require.NoError(t, client.Send(LifecycleStateActivity, "", "")) + groups, err := os.Getgroups() + require.NoError(t, err) + require.NoError(t, json.NewEncoder(os.Stdout).Encode(lifecyclePeerIdentity{ + UID: os.Getuid(), GID: os.Getgid(), Groups: groups, PID: os.Getpid(), + })) + _, err = bufio.NewReader(os.Stdin).ReadString('\n') + require.NoError(t, err) + require.Error(t, client.Send(LifecycleStateReady, "0", "none")) + _, err = os.Stdout.WriteString("rejected\n") + require.NoError(t, err) +} + +func readPeerLine(t *testing.T, scanner *bufio.Scanner) string { + t.Helper() + line := make(chan string, 1) + go func() { scanner.Scan(); line <- scanner.Text() }() + select { + case got := <-line: + return got + case <-time.After(5 * time.Second): + t.Fatal("nonroot peer did not report its state") + return "" + } +} + +func TestRootOwnerAllowsNonrootActivityButRejectsControl(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires a root owner to exercise the actual cross-UID boundary") + } + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + binary, err := os.Executable() + require.NoError(t, err) + cmd := exec.Command( + binary, + "-test.run=^TestNonrootLifecyclePeerChild$", + ) // #nosec G204 -- current test binary + cmd.Env = append(os.Environ(), "DEVSY_NONROOT_LIFECYCLE_CHILD="+dir) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: 65534, Gid: 65534, Groups: []uint32{65534}}, + Pdeathsig: syscall.SIGKILL, + } + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + stdin, err := cmd.StdinPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + scanner := bufio.NewScanner(stdout) + var identity lifecyclePeerIdentity + require.NoError(t, json.Unmarshal([]byte(readPeerLine(t, scanner)), &identity)) + assert.Equal(t, 65534, identity.UID) + assert.Equal(t, 65534, identity.GID) + assert.Equal(t, []int{65534}, identity.Groups) + assert.Equal(t, cmd.Process.Pid, identity.PID) + assert.False( + t, + idleForAnHour(monitor).Expired, + "actual nonroot process holds activity under a root owner", + ) + + assertNonrootControlReleasesCapacity(t, dir, func() { + _, err = stdin.Write([]byte("control\n")) + require.NoError(t, err) + require.Equal(t, "rejected", readPeerLine(t, scanner)) + require.NoError(t, cmd.Wait()) + }) + + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + time.Second, + time.Millisecond, + "nonowner control cannot disable the timer or leave a permanent hold", + ) +} + +func assertNonrootControlReleasesCapacity(t *testing.T, dir string, reject func()) { + t.Helper() + for range maxLifecycleClients - 1 { + client := connectNow(t, dir) + require.NoError(t, client.Send(LifecycleStateSetup, "1m", "")) + defer func() { _ = client.Close() }() + } + writer := ActivityWriter{LifecycleDir: dir, RequireOwner: true} + _, _, err := writer.Start(context.Background()) + require.Error(t, err, "a full owner must refuse untracked work") + reject() + require.Eventually(t, func() bool { + _, stop, err := writer.Start(context.Background()) + if err != nil { + return false + } + stop() + return true + }, time.Second, time.Millisecond, "nonroot control rejection must return client capacity") +} diff --git a/pkg/daemon/agent/lifecycle_proc_linux.go b/pkg/daemon/agent/lifecycle_proc_linux.go new file mode 100644 index 0000000000..0beee53290 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_proc_linux.go @@ -0,0 +1,50 @@ +//go:build linux + +package agent + +import ( + "fmt" + "os" + "strconv" + "strings" +) + +// procStartTicks returns the start identity of a live process from /proc. +func procStartTicks(pid int) string { + raw, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat") // #nosec G304 + if err != nil { + return "" + } + stat := string(raw) + // The command name may contain spaces; fields resume after the last ')'. + end := strings.LastIndexByte(stat, ')') + if end < 0 { + return "" + } + fields := strings.Fields(stat[end+1:]) + // fields[0] is state (field 3), so start time (field 22) is fields[19]. + if len(fields) < 20 { + return "" + } + // Zombies retain both PID and start time until their living parent reaps them. + if fields[0] == "Z" || fields[0] == "X" || fields[0] == "x" { + return "" + } + return fields[19] +} + +func processStartIdentity(pid int) (string, error) { + ticks := procStartTicks(pid) + if ticks == "" { + return "", fmt.Errorf( + "%w: cannot establish live process identity from /proc/%d/stat", + ErrLifecycleOwnerMissing, + pid, + ) + } + return ticks, nil +} + +func sameProcess(pid int, startTicks string) bool { + return startTicks != "" && processExists(pid) && procStartTicks(pid) == startTicks +} diff --git a/pkg/daemon/agent/lifecycle_proc_other.go b/pkg/daemon/agent/lifecycle_proc_other.go new file mode 100644 index 0000000000..60e42344e5 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_proc_other.go @@ -0,0 +1,11 @@ +//go:build !linux + +package agent + +func procStartTicks(int) string { return "" } + +func processStartIdentity(int) (string, error) { return "", nil } + +func sameProcess(pid int, startTicks string) bool { + return processExists(pid) && (startTicks == "" || procStartTicks(pid) == startTicks) +} diff --git a/pkg/daemon/agent/lifecycle_protocol.go b/pkg/daemon/agent/lifecycle_protocol.go new file mode 100644 index 0000000000..b897f762cb --- /dev/null +++ b/pkg/daemon/agent/lifecycle_protocol.go @@ -0,0 +1,490 @@ +package agent + +import ( + "bufio" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strconv" + "sync" + "time" + + "github.com/tailscale/peercred" +) + +const ( + LifecycleProtocolVersion = 3 + ownerFileName = "owner.json" + socketFileName = "owner.sock" + ackWaitTimeout = 10 * time.Second + ackPollEvery = 100 * time.Millisecond + maxRecordBytes = 4096 + connectPollEvery = 100 * time.Millisecond + runtimeDirMode = 0o755 + recordFileMode = 0o644 +) + +var ( + ErrLifecycleOwnerMissing = errors.New( + "container lifecycle owner unavailable; stop and start the workspace to replace its pod", + ) + ErrLifecycleNotAcknowledged = errors.New( + "container lifecycle owner did not acknowledge the update", + ) + ErrLifecycleOwnerIncompatible = errors.New( + "container lifecycle owner is incompatible; stop and start the workspace to replace its pod", + ) +) + +type LifecycleState string + +const ( + LifecycleStateSetup LifecycleState = "setup" + LifecycleStateReady LifecycleState = "ready" + LifecycleStateFailed LifecycleState = "failed" + LifecycleStateActivity LifecycleState = "activity" + LifecycleStateRelease LifecycleState = "release" +) + +// OwnerRecord identifies the daemon instance, not a lease on work. +type OwnerRecord struct { + Version int `json:"version"` + Generation string `json:"generation"` + PID int `json:"pid"` + StartTicks string `json:"startTicks,omitempty"` +} + +// ControlRecord belongs to one connection and operation. PID/start identity +// prevents inherited descriptors or PID reuse from orphaning setup protection. +type ControlRecord struct { + Version int `json:"version"` + Generation string `json:"generation"` + ClientID string `json:"clientID"` + Sequence uint64 `json:"sequence"` + PID int `json:"pid"` + StartTicks string `json:"startTicks,omitempty"` + Timeout string `json:"timeout"` + ShutdownAction string `json:"shutdownAction"` + State LifecycleState `json:"state"` +} + +type AckRecord struct { + Version int `json:"version"` + Generation string `json:"generation"` + ClientID string `json:"clientID"` + Sequence uint64 `json:"sequence"` + Error string `json:"error,omitempty"` +} + +func trustedOwner(uid, euid int) bool { return uid == euid || uid == 0 } + +func checkDir(dir string, euid int) error { + fi, err := os.Lstat(dir) + if err != nil { + return err + } + if !fi.IsDir() { + return fmt.Errorf("lifecycle runtime path %s is not a directory", dir) + } + if uid, ok := fileOwner(fi); ok && !trustedOwner(uid, euid) { + return fmt.Errorf("lifecycle runtime dir %s is owned by uid %d", dir, uid) + } + if fi.Mode().Perm()&0o022 != 0 { + return fmt.Errorf("lifecycle runtime dir %s is writable by others", dir) + } + return nil +} + +func writeRecord(dir, name string, v any, euid int) error { + if err := checkDir(dir, euid); err != nil { + return err + } + raw, err := json.Marshal(v) + if err != nil { + return err + } + suffix := make([]byte, 6) + if _, err := rand.Read(suffix); err != nil { + return err + } + tmp := filepath.Join(dir, "."+name+"."+hex.EncodeToString(suffix)+".tmp") + f, err := openNoFollow(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, recordFileMode) + if err != nil { + return fmt.Errorf("create lifecycle record: %w", err) + } + defer func() { _ = os.Remove(tmp) }() + if _, err := f.Write(raw); err != nil { + _ = f.Close() + return fmt.Errorf("write lifecycle record: %w", err) + } + if err := f.Close(); err != nil { + return err + } + if err := os.Rename(tmp, filepath.Join(dir, name)); err != nil { + return fmt.Errorf("publish lifecycle record: %w", err) + } + return nil +} + +type recordReader struct { + dir string + euid int + strict bool +} + +func (r recordReader) read(name string, out any) error { + if r.strict { + if err := checkDir(r.dir, r.euid); err != nil { + return err + } + } + f, err := openNoFollow(filepath.Join(r.dir, name), os.O_RDONLY, 0) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + if err := r.checkFile(f, name); err != nil { + return err + } + raw, err := io.ReadAll(io.LimitReader(f, maxRecordBytes)) + if err != nil { + return err + } + return json.Unmarshal(raw, out) +} + +func (r recordReader) checkFile(f *os.File, name string) error { + fi, err := f.Stat() + if err != nil { + return err + } + if !fi.Mode().IsRegular() || fi.Size() > maxRecordBytes { + return fmt.Errorf("lifecycle record %s is not a small regular file", name) + } + if uid, ok := fileOwner(fi); r.strict && ok && !trustedOwner(uid, r.euid) { + return fmt.Errorf("lifecycle record %s is owned by uid %d", name, uid) + } + return nil +} + +// NewLifecycleOwner binds before publishing discovery. All subsequent safety +// decisions and acknowledgements use the connection, never writable files. +func NewLifecycleOwner(dir string) (*LifecycleOwner, error) { + euid := os.Geteuid() + ticks, err := processStartIdentity(os.Getpid()) + if err != nil { + return nil, err + } + if err := prepareRuntimeDir(dir, euid); err != nil { + return nil, err + } + listener, err := net.Listen("unix", filepath.Join(dir, socketFileName)) + if err != nil { + return nil, err + } + // #nosec G302 -- container-local activity accepts workspace users; controls require peer ownership. + if err := os.Chmod(filepath.Join(dir, socketFileName), 0o666); err != nil { + _ = listener.Close() + return nil, err + } + o := &LifecycleOwner{ + euid: euid, generation: rand.Text(), listener: listener, + clients: make(map[net.Conn]*lifecycleSession), + } + o.writeAck = writeSocketAck + err = writeRecord(dir, ownerFileName, OwnerRecord{ + Version: LifecycleProtocolVersion, Generation: o.generation, + PID: os.Getpid(), StartTicks: ticks, + }, euid) + if err != nil { + _ = o.Close() + return nil, err + } + return o, nil +} + +func createRuntimeDir(dir string, euid int) error { + if err := os.MkdirAll(filepath.Dir(dir), runtimeDirMode); err != nil { + return err + } + if err := os.Mkdir(dir, runtimeDirMode); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + if err := requireOwnedDir(dir, euid); err != nil { + return err + } + if err := os.Chmod(dir, runtimeDirMode); err != nil { + return err + } + return checkDir(dir, euid) +} + +func prepareRuntimeDir(dir string, euid int) error { + if err := createRuntimeDir(dir, euid); err != nil { + return err + } + for _, name := range []string{ownerFileName, socketFileName} { + if err := os.Remove( + filepath.Join(dir, name), + ); err != nil && + !errors.Is(err, os.ErrNotExist) { + return err + } + } + return nil +} + +func requireOwnedDir(dir string, euid int) error { + fi, err := os.Lstat(dir) + if err != nil { + return err + } + if !fi.IsDir() { + return fmt.Errorf("lifecycle runtime path %s is not a directory", dir) + } + if uid, ok := fileOwner(fi); ok && uid != euid { + return fmt.Errorf("lifecycle runtime dir %s is owned by uid %d, not %d", dir, uid, euid) + } + return nil +} + +func (o *LifecycleOwner) Generation() string { return o.generation } + +// LifecycleClient never reconnects silently: losing protection cancels the work. +// A fresh operation must explicitly connect to the current owner instance. +type LifecycleClient struct { + conn net.Conn + owner OwnerRecord + clientID string + mu sync.Mutex + sequence uint64 + ackTimeout time.Duration + acks chan AckRecord + done chan struct{} + closeOnce sync.Once +} + +func ConnectLifecycle(ctx context.Context, dir string) (*LifecycleClient, error) { + for { + owner, err := liveOwner(dir, os.Geteuid()) + if err != nil { + return nil, err + } + if owner != nil { + return connectOwner(ctx, dir, *owner) + } + select { + case <-ctx.Done(): + return nil, ErrLifecycleOwnerMissing + case <-time.After(connectPollEvery): + } + } +} + +func connectOwner(ctx context.Context, dir string, owner OwnerRecord) (*LifecycleClient, error) { + ctx, cancel := context.WithTimeout(ctx, ackWaitTimeout) + defer cancel() + conn, err := (&net.Dialer{}).DialContext(ctx, "unix", filepath.Join(dir, socketFileName)) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrLifecycleOwnerMissing, err) + } + creds, err := peercred.Get(conn) + if err != nil { + _ = conn.Close() + return nil, err + } + uid, ok := creds.UserID() + pid, hasPID := creds.PID() + if !ok || (uid != "0" && uid != strconv.Itoa(os.Geteuid())) || (hasPID && pid != owner.PID) { + _ = conn.Close() + return nil, fmt.Errorf( + "%w: socket owner differs from discovery record", + ErrLifecycleOwnerIncompatible, + ) + } + c := &LifecycleClient{ + conn: conn, owner: owner, clientID: rand.Text(), ackTimeout: ackWaitTimeout, + acks: make(chan AckRecord, 8), done: make(chan struct{}), + } + go c.readAcks() + return c, nil +} + +func liveOwner(dir string, euid int) (*OwnerRecord, error) { + var owner OwnerRecord + err := recordReader{dir: dir, euid: euid, strict: true}.read(ownerFileName, &owner) + switch { + case errors.Is(err, os.ErrNotExist): + return nil, nil + case err != nil: + return nil, fmt.Errorf("%w: %w", ErrLifecycleOwnerMissing, err) + case owner.Version != LifecycleProtocolVersion: + return nil, fmt.Errorf( + "%w: protocol version %d, want %d", + ErrLifecycleOwnerIncompatible, + owner.Version, + LifecycleProtocolVersion, + ) + case !ownerAlive(owner): + return nil, ErrLifecycleOwnerMissing + } + return &owner, nil +} + +func ownerAlive(o OwnerRecord) bool { return o.PID > 0 && sameProcess(o.PID, o.StartTicks) } + +func (c *LifecycleClient) Done() <-chan struct{} { return c.done } + +func (c *LifecycleClient) Close() error { + var err error + c.closeOnce.Do(func() { err = c.conn.Close(); close(c.done) }) + return err +} + +// ProtectedContext aborts work on socket loss instead of leaving it untracked. +func (c *LifecycleClient) ProtectedContext( + parent context.Context, +) (context.Context, context.CancelCauseFunc) { + ctx, cancel := context.WithCancelCause(parent) + go func() { + select { + case <-parent.Done(): + cancel(context.Cause(parent)) + _ = c.Close() + case <-c.done: + cancel(ErrLifecycleOwnerMissing) + case <-ctx.Done(): + } + }() + return ctx, cancel +} + +// Send retries the same operation while the connection survives. The owner's +// cached result makes acknowledgement recovery independent of applying state. +func (c *LifecycleClient) Send( + state LifecycleState, + timeout, shutdownAction string, +) (sendErr error) { + c.mu.Lock() + defer c.mu.Unlock() + defer func() { + if sendErr != nil { + _ = c.Close() + } + }() + c.sequence++ + rec := ControlRecord{ + Version: LifecycleProtocolVersion, Generation: c.owner.Generation, + ClientID: c.clientID, Sequence: c.sequence, + PID: os.Getpid(), StartTicks: procStartTicks(os.Getpid()), + State: state, Timeout: timeout, ShutdownAction: shutdownAction, + } + return c.sendRecord(rec) +} + +func (c *LifecycleClient) readAcks() { + defer func() { _ = c.Close() }() + scanner := bufio.NewScanner(c.conn) + scanner.Buffer(make([]byte, 512), maxRecordBytes) + for scanner.Scan() { + var ack AckRecord + if json.Unmarshal(scanner.Bytes(), &ack) != nil || + !c.validAck(ack) { + return + } + if !c.deliverAck(ack) { + return + } + } +} + +// deliverAck queues ack for the sender. Every resend makes the owner repeat the +// same ACK and nothing drains them between sends, so a full queue holds stale +// duplicates: drop the oldest instead of closing a live protected connection. +func (c *LifecycleClient) deliverAck(ack AckRecord) bool { + for { + select { + case c.acks <- ack: + return true + case <-c.done: + return false + default: + select { + case <-c.acks: + default: + } + } + } +} + +func (c *LifecycleClient) sendRecord(rec ControlRecord) error { + raw, err := json.Marshal(rec) + if err != nil { + return err + } + if len(raw) >= maxRecordBytes { + return errors.New("lifecycle message exceeds size limit") + } + raw = append(raw, '\n') + deadline := time.Now().Add(c.ackTimeout) + for { + ack, err := c.exchange(raw, rec.Sequence, deadline) + if err != nil { + return err + } + if ack == nil { + continue + } + if ack.Error != "" { + return fmt.Errorf("container lifecycle owner rejected the update: %s", ack.Error) + } + return nil + } +} + +func (c *LifecycleClient) exchange( + raw []byte, + sequence uint64, + deadline time.Time, +) (*AckRecord, error) { + if err := c.conn.SetWriteDeadline(deadline); err != nil { + return nil, fmt.Errorf("%w: %w", ErrLifecycleOwnerMissing, err) + } + if _, err := c.conn.Write(raw); err != nil { + return nil, fmt.Errorf("%w: %w", ErrLifecycleOwnerMissing, err) + } + return c.awaitAck(sequence, deadline) +} + +func (c *LifecycleClient) awaitAck(sequence uint64, deadline time.Time) (*AckRecord, error) { + timer := time.NewTimer(min(ackPollEvery, max(time.Until(deadline), 0))) + defer timer.Stop() + for { + select { + case ack := <-c.acks: + if ack.Sequence == sequence { + return &ack, nil + } + case <-c.done: + return nil, ErrLifecycleOwnerMissing + case <-timer.C: + if !time.Now().Before(deadline) { + return nil, ErrLifecycleNotAcknowledged + } + return nil, nil + } + } +} + +func (c *LifecycleClient) validAck(ack AckRecord) bool { + return ack.Version == LifecycleProtocolVersion && + ack.Generation == c.owner.Generation && + ack.ClientID == c.clientID +} diff --git a/pkg/daemon/agent/lifecycle_protocol_test.go b/pkg/daemon/agent/lifecycle_protocol_test.go new file mode 100644 index 0000000000..200934194d --- /dev/null +++ b/pkg/daemon/agent/lifecycle_protocol_test.go @@ -0,0 +1,614 @@ +package agent + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "net" + "os" + "os/exec" + "path/filepath" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestOwner(t *testing.T) (*LifecycleOwner, string) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("managed Kubernetes lifecycle uses Unix peer identity") + } + // Darwin's Unix socket path limit is shorter than testing.TempDir's names. + dir, err := os.MkdirTemp("/tmp", "devsy-lifecycle-") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + owner, err := NewLifecycleOwner(dir) + require.NoError(t, err) + t.Cleanup(func() { _ = owner.Close() }) + return owner, dir +} + +func connectNow(t *testing.T, dir string) *LifecycleClient { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client, err := ConnectLifecycle(ctx, dir) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + return client +} + +func runLoop(t *testing.T, loop *LifecycleLoop) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- loop.Run(ctx) }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) + } + }) +} + +func idleForAnHour(m *LifecycleMonitor) Decision { + return m.Evaluate(time.Now().Add(time.Hour), time.Time{}, nil) +} + +func TestSetupIsProtectedBeforeSendReturnsAndReadyResetsIt(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + client := connectNow(t, dir) + for range 2 { + require.NoError(t, client.Send(LifecycleStateSetup, "1m", "")) + assert.False(t, idleForAnHour(monitor).Expired) + require.NoError(t, client.Send(LifecycleStateReady, "1m", "")) + assert.True(t, idleForAnHour(monitor).Expired) + } +} + +func TestAcknowledgementFailureRetriesWithoutReapplyingReady(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + firstApplied := make(chan time.Time, 1) + var writes atomic.Int32 + owner.writeAck = func(conn net.Conn, ack AckRecord) error { + if writes.Add(1) == 1 { + monitor.mu.Lock() + firstApplied <- monitor.armedAt + monitor.mu.Unlock() + return os.ErrPermission + } + return writeSocketAck(conn, ack) + } + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + client := connectNow(t, dir) + + require.NoError(t, client.Send(LifecycleStateReady, "1m", "")) + + appliedAt := <-firstApplied + assert.True(t, monitor.Evaluate(appliedAt.Add(time.Minute), time.Time{}, nil).Expired, + "retrying the ACK must not shift the original idle deadline") +} + +func TestUnacknowledgedSetupCanBeClosedWithoutOrphaningProtection(t *testing.T) { + owner, dir := newTestOwner(t) + owner.writeAck = func(net.Conn, AckRecord) error { return os.ErrPermission } + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + client := connectNow(t, dir) + client.ackTimeout = 250 * time.Millisecond + + require.ErrorIs(t, client.Send(LifecycleStateSetup, "1m", ""), ErrLifecycleNotAcknowledged) + require.NoError(t, client.Close()) + + require.Eventually(t, func() bool { return idleForAnHour(monitor).Expired }, + time.Second, time.Millisecond, "failed handshake cleanup must release setup") +} + +func TestLiveSlowSetupHasNoLeaseThatExpiresIt(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + var loopErr error + go func() { defer close(done); loopErr = (&LifecycleLoop{Owner: owner, Monitor: monitor}).Run(ctx) }() + defer func() { cancel(); <-done }() + client := connectNow(t, dir) + require.NoError(t, client.Send(LifecycleStateSetup, "150ms", "")) + + assert.Never(t, func() bool { return monitor.Evaluate(time.Now(), time.Time{}, nil).Expired }, + 500*time.Millisecond, 10*time.Millisecond) + require.NoError(t, client.Close()) + select { + case <-done: + assert.ErrorIs(t, loopErr, ErrInactivityExpired) + case <-time.After(5 * time.Second): + t.Fatal("disconnected setup remained suspended") + } +} + +func TestOneSetupCannotReleaseAnotherSetupsProtection(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + first, second := connectNow(t, dir), connectNow(t, dir) + require.NoError(t, first.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, second.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, first.Send(LifecycleStateReady, "1m", "")) + assert.False(t, idleForAnHour(monitor).Expired) + require.NoError(t, second.Send(LifecycleStateFailed, "1m", "")) + assert.True(t, idleForAnHour(monitor).Expired) +} + +func TestStorageFailureCannotHideARegisteredLiveWriter(t *testing.T) { + owner, dir := newTestOwner(t) + if os.Geteuid() == 0 { + t.Skip("read-only directory permissions require an unprivileged writer") + } + // Stat remains readable with an old timestamp, but touching the malformed + // activity file and creating ANY health/publication inode both fail. + activity := filepath.Join(dir, "activity") + require.NoError(t, os.Mkdir(activity, 0o700)) + old := time.Now().Add(-time.Hour) + require.NoError(t, os.Chtimes(activity, old, old)) + // #nosec G302 -- readable but unwritable storage is the regression scenario. + require.NoError(t, os.Chmod(dir, 0o555)) + t.Cleanup(func() { + // #nosec G302 -- restore owner-write access so temporary state can be removed. + _ = os.Chmod(dir, 0o755) + }) + require.Error(t, TouchActivity(activity, time.Now())) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor, ActivityPath: activity}) + client := connectNow(t, dir) + require.NoError(t, client.Send(LifecycleStateReady, "1m", "")) + require.Error( + t, + writeRecord( + dir, + "unwritable-health.json", + control(LifecycleStateReady, "1m", ""), + os.Geteuid(), + ), + ) + + workCtx, stop, err := (ActivityWriter{Path: activity, LifecycleDir: dir, Interval: 10 * time.Millisecond}).Start( + context.Background(), + ) + require.NoError(t, err) + defer stop() + assert.NoError(t, workCtx.Err()) + assert.False(t, monitor.Evaluate(time.Now().Add(time.Hour), old, nil).Expired, + "neither failed file write can remove in-memory live registration") + + stop() + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + time.Second, + time.Millisecond, + ) +} + +func TestReleaseAcknowledgementRecoveryDoesNotResetIdleTwice(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + var dropped atomic.Bool + firstRelease := make(chan time.Time, 1) + owner.writeAck = func(conn net.Conn, ack AckRecord) error { + if ack.Sequence == 2 && dropped.CompareAndSwap(false, true) { + monitor.mu.Lock() + firstRelease <- monitor.armedAt + monitor.mu.Unlock() + return os.ErrPermission + } + return writeSocketAck(conn, ack) + } + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + require.NoError(t, connectNow(t, dir).Send(LifecycleStateReady, "1m", "")) + _, stop, err := (ActivityWriter{Path: filepath.Join(dir, "activity"), LifecycleDir: dir}).Start( + context.Background(), + ) + require.NoError(t, err) + defer stop() + + stop() + + appliedAt := <-firstRelease + assert.True(t, monitor.Evaluate(appliedAt.Add(time.Minute), time.Time{}, nil).Expired, + "release must retry after publication failure without another idle reset") +} + +func TestOwnerLossCancelsProtectedWriterAndRejectsFurtherOperations(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + workCtx, stop, err := (ActivityWriter{Path: filepath.Join(dir, "activity"), LifecycleDir: dir}).Start( + context.Background(), + ) + require.NoError(t, err) + defer stop() + + require.NoError(t, owner.Close()) + + select { + case <-workCtx.Done(): + assert.ErrorIs(t, context.Cause(workCtx), ErrLifecycleOwnerMissing) + case <-time.After(time.Second): + t.Fatal("work continued after losing its lifecycle owner") + } + _, _, err = (ActivityWriter{Path: filepath.Join(dir, "activity"), LifecycleDir: dir}).Start( + context.Background(), + ) + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) +} + +func TestLifecycleChildProcess(t *testing.T) { + dir := os.Getenv("DEVSY_LIFECYCLE_TEST_CHILD") + if dir == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client, err := ConnectLifecycle(ctx, dir) + if err != nil { + t.Fatal(err) + } + defer func() { _ = client.Close() }() + if err := client.Send(LifecycleStateSetup, "1m", ""); err != nil { + t.Fatal(err) + } + if _, err := os.Stdout.WriteString("registered\n"); err != nil { + t.Fatal(err) + } + <-client.Done() +} + +func TestKilledSetupProcessReleasesProtectionAndCanReconnect(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + binary, err := os.Executable() + require.NoError(t, err) + cmd := exec.Command( + binary, + "-test.run=^TestLifecycleChildProcess$", + ) // #nosec G204 -- current test binary + cmd.Env = append(os.Environ(), "DEVSY_LIFECYCLE_TEST_CHILD="+dir) + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + registered := make(chan string, 1) + go func() { scanner := bufio.NewScanner(stdout); scanner.Scan(); registered <- scanner.Text() }() + select { + case line := <-registered: + require.Equal(t, "registered", line) + case <-time.After(5 * time.Second): + t.Fatal("child did not establish acknowledged setup") + } + assert.False(t, idleForAnHour(monitor).Expired) + + require.NoError(t, cmd.Process.Kill()) + require.Error(t, cmd.Wait()) + + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + time.Second, + time.Millisecond, + ) + reconnected := connectNow(t, dir) + require.NoError(t, reconnected.Send(LifecycleStateSetup, "1m", "")) + assert.False(t, idleForAnHour(monitor).Expired) + require.NoError(t, reconnected.Send(LifecycleStateReady, "1m", "")) + assert.True(t, idleForAnHour(monitor).Expired) +} + +func TestOwnerRestartInvalidatesExistingConnectionAndNewOwnerStartsUnarmed(t *testing.T) { + first, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: first, Monitor: monitor}) + old := connectNow(t, dir) + require.NoError(t, old.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, first.Close()) + second, err := NewLifecycleOwner(dir) + require.NoError(t, err) + t.Cleanup(func() { _ = second.Close() }) + fresh := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: second, Monitor: fresh}) + + assert.ErrorIs(t, old.Send(LifecycleStateReady, "1m", ""), ErrLifecycleOwnerMissing) + assert.False(t, idleForAnHour(fresh).Expired) + require.NoError(t, connectNow(t, dir).Send(LifecycleStateReady, "1m", "")) + assert.True(t, idleForAnHour(fresh).Expired) +} + +func TestConnectLifecycleRejectsIncompatibleOwner(t *testing.T) { + _, dir := newTestOwner(t) + require.NoError(t, writeRecord(dir, ownerFileName, OwnerRecord{ + Version: LifecycleProtocolVersion - 1, Generation: "old", PID: os.Getpid(), + }, os.Geteuid())) + _, err := ConnectLifecycle(context.Background(), dir) + assert.ErrorIs(t, err, ErrLifecycleOwnerIncompatible) +} + +func TestConnectLifecycleMissingOwnerIsBounded(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + _, err := ConnectLifecycle(ctx, filepath.Join(t.TempDir(), "absent")) + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) +} + +func TestClientSeesRejectedPolicyWithoutArming(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + err := connectNow(t, dir).Send(LifecycleStateSetup, "soon", "") + assert.ErrorContains(t, err, "invalid lifecycle timeout") + assert.False(t, idleForAnHour(monitor).Expired) +} + +func TestMissingManagedDiscoveryNeverFallsBackToUntrackedWork(t *testing.T) { + owner, dir := newTestOwner(t) + require.NoError(t, owner.Close()) + require.NoError(t, os.Remove(filepath.Join(dir, ownerFileName))) + + ctx, stop, err := (ActivityWriter{ + Path: filepath.Join(dir, "activity"), LifecycleDir: dir, + }).Start(context.Background()) + if stop != nil { + defer stop() + } + + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) + assert.Nil(t, ctx) +} + +func TestDeadOwnerNeverFallsBackToLegacyActivity(t *testing.T) { + _, dir := newTestOwner(t) + require.NoError(t, writeRecord(dir, ownerFileName, OwnerRecord{ + Version: LifecycleProtocolVersion, Generation: "dead", PID: 0x7ffffffe, + }, os.Geteuid())) + + _, stop, err := (ActivityWriter{ + Path: filepath.Join(dir, "activity"), LifecycleDir: dir, + }).Start(context.Background()) + if stop != nil { + defer stop() + } + + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) +} + +func TestNewOwnerRejectsSymlinkWithoutChangingTargetPermissions(t *testing.T) { + _, dir := newTestOwner(t) + target := filepath.Join(dir, "private") + require.NoError(t, os.Mkdir(target, 0o700)) + link := filepath.Join(dir, "link") + require.NoError(t, os.Symlink(target, link)) + + _, err := NewLifecycleOwner(link) + + require.Error(t, err) + info, err := os.Stat(target) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o700), info.Mode().Perm()) +} + +func TestConnectRejectsSymlinkedDiscovery(t *testing.T) { + _, dir := newTestOwner(t) + target := filepath.Join(dir, "untrusted") + require.NoError(t, os.WriteFile(target, []byte(`{}`), 0o600)) + require.NoError(t, os.Remove(filepath.Join(dir, ownerFileName))) + require.NoError(t, os.Symlink(target, filepath.Join(dir, ownerFileName))) + + _, err := ConnectLifecycle(context.Background(), dir) + + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) +} + +func TestManagedWorkCannotFallBackDuringInitialOwnerStartup(t *testing.T) { + _, dir := newTestOwner(t) + activity := filepath.Join(dir, "activity") + _, stop, err := (ActivityWriter{ + Path: activity, LifecycleDir: filepath.Join(dir, "not-started"), RequireOwner: true, + }).Start(context.Background()) + if stop != nil { + defer stop() + } + + assert.ErrorIs(t, err, ErrLifecycleOwnerMissing) + _, err = os.Stat(activity) + assert.ErrorIs(t, err, os.ErrNotExist, "no untracked activity loop should be started") +} + +func rawLifecycleConnection(t *testing.T, dir string) net.Conn { + t.Helper() + conn, err := net.Dial("unix", filepath.Join(dir, socketFileName)) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(time.Second))) + return conn +} + +func requireRejectedConnection(t *testing.T, conn net.Conn) { + t.Helper() + var buf [1]byte + n, err := conn.Read(buf[:]) + require.Error(t, err) + assert.Zero(t, n) + if networkErr, ok := err.(net.Error); ok { + assert.False( + t, + networkErr.Timeout(), + "the owner must reject, not leave an unbounded pending request", + ) + } +} + +func TestSocketRejectsForgedProcessIdentityWithoutDisarming(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + conn := rawLifecycleConnection(t, dir) + require.NoError(t, json.NewEncoder(conn).Encode(ControlRecord{ + Version: LifecycleProtocolVersion, Generation: owner.Generation(), + ClientID: "forged", Sequence: 1, PID: os.Getpid() + 1, + State: LifecycleStateReady, Timeout: "0", + })) + + requireRejectedConnection(t, conn) + assert.True(t, idleForAnHour(monitor).Expired, "a forged peer cannot disable the current timer") + require.NoError(t, connectNow(t, dir).Send(LifecycleStateSetup, "1m", "")) + assert.False(t, idleForAnHour(monitor).Expired, "the authenticated owner must remain usable") +} + +func TestSocketMessageBoundRejectsUnterminatedOversizedFrame(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + conn := rawLifecycleConnection(t, dir) + _, err := conn.Write(bytes.Repeat([]byte{'x'}, maxRecordBytes+1)) + require.NoError(t, err) + + requireRejectedConnection(t, conn) + assert.True(t, idleForAnHour(monitor).Expired) + require.NoError(t, connectNow(t, dir).Send(LifecycleStateSetup, "1m", "")) + assert.False(t, idleForAnHour(monitor).Expired) +} + +func TestLiveClientBoundRefusesUnprotectedWorkAndRecoversCapacity(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + var first *LifecycleClient + for range maxLifecycleClients { + client := connectNow(t, dir) + if first == nil { + first = client + } + require.NoError(t, client.Send(LifecycleStateSetup, "1m", "")) + } + writer := ActivityWriter{LifecycleDir: dir, RequireOwner: true} + ctx, stop, err := writer.Start(context.Background()) + if stop != nil { + defer stop() + } + assert.Error(t, err, "capacity exhaustion must not silently fall back to legacy tracking") + assert.Nil(t, ctx) + + require.NoError(t, first.Close()) + + var regainedStop func() + require.Eventually(t, func() bool { + _, release, err := writer.Start(context.Background()) + if err != nil { + return false + } + regainedStop = release + return true + }, 3*time.Second, 10*time.Millisecond) + defer regainedStop() + assert.False(t, idleForAnHour(monitor).Expired) +} + +func TestClientVerifiesSocketPeerAgainstDiscoveryIdentity(t *testing.T) { + owner, dir := newTestOwner(t) + if os.Getppid() <= 0 { + t.Skip("a distinct live parent is needed for the forged discovery identity") + } + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: &LifecycleMonitor{}}) + require.NoError(t, writeRecord(dir, ownerFileName, OwnerRecord{ + Version: LifecycleProtocolVersion, Generation: owner.Generation(), PID: os.Getppid(), + StartTicks: procStartTicks(os.Getppid()), + }, os.Geteuid())) + + _, err := ConnectLifecycle(context.Background(), dir) + + assert.ErrorIs(t, err, ErrLifecycleOwnerIncompatible) +} + +func TestOwnerActivityCannotConsumeReservedControlAdmission(t *testing.T) { + owner, dir := newTestOwner(t) + monitor := &LifecycleMonitor{} + monitor.ArmNow(time.Minute, time.Now()) + runLoop(t, &LifecycleLoop{Owner: owner, Monitor: monitor}) + writer := ActivityWriter{LifecycleDir: dir, RequireOwner: true} + var stops []func() + for range maxOtherLifecycleClients { + _, stop, err := writer.Start(context.Background()) + require.NoError(t, err) + stops = append(stops, stop) + t.Cleanup(stop) + } + _, stop, err := writer.Start(context.Background()) + if stop != nil { + stop() + } + require.Error(t, err, "owner-UID activity must not consume the control reserve") + control := connectNow(t, dir) + require.NoError(t, control.Send(LifecycleStateSetup, "1m", "")) + require.NoError(t, control.Send(LifecycleStateReady, "1m", "")) + for _, stop := range stops { + stop() + } + require.Eventually( + t, + func() bool { return idleForAnHour(monitor).Expired }, + time.Second, + time.Millisecond, + ) +} + +func TestDuplicateAcknowledgementsDoNotCloseALiveConnection(t *testing.T) { + const ( + generation = "ack-generation" + clientID = "ack-client" + ) + owner, client := net.Pipe() + c := &LifecycleClient{ + conn: client, + owner: OwnerRecord{Generation: generation}, + clientID: clientID, + acks: make(chan AckRecord, 8), + done: make(chan struct{}), + } + go c.readAcks() + defer func() { _ = c.Close() }() + defer func() { _ = owner.Close() }() + + raw, err := json.Marshal(AckRecord{ + Version: LifecycleProtocolVersion, Generation: generation, + ClientID: clientID, Sequence: 1, + }) + require.NoError(t, err) + raw = append(raw, '\n') + for range 4 * cap(c.acks) { + _, err := owner.Write(raw) + require.NoError(t, err) + } + + select { + case <-c.done: + t.Fatal("stale duplicate acknowledgements must not close a protected connection") + case <-time.After(200 * time.Millisecond): + } + select { + case ack := <-c.acks: + assert.Equal(t, uint64(1), ack.Sequence, "a matching acknowledgement is still delivered") + default: + t.Fatal("expected a buffered acknowledgement") + } +} diff --git a/pkg/daemon/agent/lifecycle_server.go b/pkg/daemon/agent/lifecycle_server.go new file mode 100644 index 0000000000..2d50f91a6c --- /dev/null +++ b/pkg/daemon/agent/lifecycle_server.go @@ -0,0 +1,342 @@ +package agent + +import ( + "bufio" + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "net" + "strconv" + "sync" + "time" + + "github.com/devsy-org/devsy/pkg/log" + "github.com/tailscale/peercred" +) + +const ( + maxLifecycleClients = 128 + reservedLifecycleClients = 32 + maxOtherLifecycleClients = maxLifecycleClients - reservedLifecycleClients + maxLifecycleUIDClients = 32 + maxLifecyclePIDClients = 16 +) + +var errLifecycleClientLimit = errors.New("lifecycle owner connection limit reached") + +type lifecycleSession struct { + conn net.Conn + pid int + startTicks string + uid string + privileged bool + sharedSlot bool + protected bool + readDeadline time.Time + clientID string + protectionID string + applied ControlRecord + result AckRecord +} + +// LifecycleOwner keeps live work and applied operations in memory. Discovery +// and policy files are never evidence that a registered writer is healthy. +type LifecycleOwner struct { + euid int + generation string + listener net.Listener + mu sync.Mutex + closed bool + clients map[net.Conn]*lifecycleSession + otherClients int + clientsByUID map[string]int + clientsByPID map[int]int + writeAck func(net.Conn, AckRecord) error +} + +func (o *LifecycleOwner) Close() error { + o.mu.Lock() + if o.closed { + o.mu.Unlock() + return nil + } + o.closed = true + conns := make([]net.Conn, 0, len(o.clients)) + for conn := range o.clients { + conns = append(conns, conn) + } + o.mu.Unlock() + err := o.listener.Close() + for _, conn := range conns { + _ = conn.Close() + } + if errors.Is(err, net.ErrClosed) { + return nil + } + return err +} + +func (o *LifecycleOwner) Serve(ctx context.Context, monitor *LifecycleMonitor) error { + stop := context.AfterFunc(ctx, func() { _ = o.Close() }) + defer stop() + var workers sync.WaitGroup + defer func() { _ = o.Close(); workers.Wait() }() + for { + conn, err := o.listener.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) || ctx.Err() != nil { + return nil + } + return fmt.Errorf("accept lifecycle client: %w", err) + } + session, err := o.addClient(conn) + if err != nil { + _ = conn.Close() + continue + } + workers.Go(func() { o.serveClient(session, monitor) }) + } +} + +func (o *LifecycleOwner) addClient(conn net.Conn) (*lifecycleSession, error) { + pid, uid, err := readLifecyclePeer(conn) + if err != nil { + return nil, err + } + privileged := uid == "0" || uid == strconv.Itoa(o.euid) + o.mu.Lock() + defer o.mu.Unlock() + if o.closed || !o.canAdmit(uid, pid, privileged) { + return nil, errLifecycleClientLimit + } + ticks, err := processStartIdentity(pid) + if err != nil { + return nil, err + } + s := &lifecycleSession{ + conn: conn, pid: pid, uid: uid, privileged: privileged, sharedSlot: !privileged, + startTicks: ticks, protectionID: rand.Text(), + readDeadline: time.Now().Add(ackWaitTimeout), + } + o.clients[conn] = s + o.countClient(s) + return s, nil +} + +func readLifecyclePeer(conn net.Conn) (int, string, error) { + creds, err := peercred.Get(conn) + if err != nil { + return 0, "", err + } + pid, hasPID := creds.PID() + uid, hasUID := creds.UserID() + if !hasPID || !hasUID || pid <= 0 { + return 0, "", errors.New("lifecycle peer identity is unavailable") + } + return pid, uid, nil +} + +func (o *LifecycleOwner) removeClient(s *lifecycleSession, monitor *LifecycleMonitor) { + _ = s.conn.Close() + o.mu.Lock() + o.removeClientLocked(s) + id := s.protectionID + o.mu.Unlock() + monitor.Release(id, time.Now()) +} + +func (o *LifecycleOwner) pruneDeadClients(monitor *LifecycleMonitor, now time.Time) { + o.mu.Lock() + dead := make([]*lifecycleSession, 0) + for _, s := range o.clients { + if !sameProcess(s.pid, s.startTicks) { + dead = append(dead, s) + } + } + for _, s := range dead { + o.removeClientLocked(s) + monitor.Release(s.protectionID, now) + } + o.mu.Unlock() + for _, s := range dead { + _ = s.conn.Close() + } +} + +func (s *lifecycleSession) validateIdentity(rec ControlRecord) error { + if rec.ClientID == "" || rec.Sequence == 0 || rec.PID != s.pid || + (rec.StartTicks != "" && s.startTicks != "" && rec.StartTicks != s.startTicks) { + return errors.New("lifecycle client identity does not match its connection") + } + return nil +} + +func (o *LifecycleOwner) authorize(s *lifecycleSession, rec ControlRecord) error { + if rec.State == LifecycleStateActivity || rec.State == LifecycleStateRelease { + return nil + } + if s.privileged { + return nil + } + return errors.New("lifecycle policy update requires the daemon's user or root") +} + +func (o *LifecycleOwner) validate(s *lifecycleSession, rec ControlRecord) error { + if rec.Version != LifecycleProtocolVersion || rec.Generation != o.generation { + return ErrLifecycleOwnerIncompatible + } + if err := s.validateIdentity(rec); err != nil { + return err + } + if err := o.authorize(s, rec); err != nil { + return err + } + return o.validateSession(s, rec) +} + +func (o *LifecycleOwner) validateSession(s *lifecycleSession, rec ControlRecord) error { + o.mu.Lock() + defer o.mu.Unlock() + if o.closed || o.clients[s.conn] != s { + return ErrLifecycleOwnerMissing + } + if s.clientID != "" && s.clientID != rec.ClientID { + return errors.New("lifecycle connection changed client identity") + } + s.clientID = rec.ClientID + return o.classifyActivity(s, rec.State) +} + +func (o *LifecycleOwner) serveClient(s *lifecycleSession, monitor *LifecycleMonitor) { + defer o.removeClient(s, monitor) + // An unauthenticated idle socket is not protected work and must not consume + // a connection slot indefinitely. Registered live work has no time lease. + _ = s.conn.SetReadDeadline(s.readDeadline) + scanner := bufio.NewScanner(s.conn) + scanner.Buffer(make([]byte, 512), maxRecordBytes) + // Applied results are connection-owned and retained independently of ACKs. + for scanner.Scan() { + var rec ControlRecord + if json.Unmarshal(scanner.Bytes(), &rec) != nil { + return + } + if err := o.validate(s, rec); err != nil { + return + } + result, err := o.applyOperation(s, rec, monitor) + if err != nil { + return + } + if err := o.writeAck(s.conn, result); err != nil { + log.Debugf("lifecycle ACK delivery failed; retaining applied result for retry: %v", err) + } + } +} + +func (o *LifecycleOwner) applyOperation( + s *lifecycleSession, + rec ControlRecord, + monitor *LifecycleMonitor, +) (AckRecord, error) { + if rec == s.applied { + return s.result, nil + } + if rec.Sequence != s.applied.Sequence+1 { + return AckRecord{}, errors.New("lifecycle sequence is not consecutive") + } + state := rec + state.ClientID = s.protectionID + now := time.Now() + _, applyErr := monitor.Apply(state, now) + if applyErr == nil { + s.transitionProtection(rec.State, now) + } + s.applied = rec + s.result = AckRecord{ + Version: LifecycleProtocolVersion, Generation: o.generation, + ClientID: rec.ClientID, Sequence: rec.Sequence, + } + if applyErr != nil { + s.result.Error = applyErr.Error() + } + return s.result, nil +} + +func writeSocketAck(conn net.Conn, ack AckRecord) error { + if err := conn.SetWriteDeadline(time.Now().Add(ackPollEvery)); err != nil { + return err + } + return json.NewEncoder(conn).Encode(ack) +} + +func (o *LifecycleOwner) canAdmit(uid string, pid int, privileged bool) bool { + if len(o.clients) >= maxLifecycleClients { + return false + } + return privileged || (o.otherClients < maxOtherLifecycleClients && + o.clientsByUID[uid] < maxLifecycleUIDClients && + o.clientsByPID[pid] < maxLifecyclePIDClients) +} + +func (o *LifecycleOwner) countClient(s *lifecycleSession) { + if s.sharedSlot { + o.otherClients++ + } + if s.privileged { + return + } + if o.clientsByUID == nil { + o.clientsByUID = make(map[string]int) + o.clientsByPID = make(map[int]int) + } + o.clientsByUID[s.uid]++ + o.clientsByPID[s.pid]++ +} + +func (o *LifecycleOwner) removeClientLocked(s *lifecycleSession) { + if _, present := o.clients[s.conn]; !present { + return + } + delete(o.clients, s.conn) + if s.sharedSlot { + o.otherClients-- + } + if s.privileged { + return + } + o.clientsByUID[s.uid]-- + o.clientsByPID[s.pid]-- + if o.clientsByUID[s.uid] == 0 { + delete(o.clientsByUID, s.uid) + } + if o.clientsByPID[s.pid] == 0 { + delete(o.clientsByPID, s.pid) + } +} + +func (s *lifecycleSession) transitionProtection(state LifecycleState, now time.Time) { + protected := state == LifecycleStateSetup || state == LifecycleStateActivity + if protected == s.protected { + return + } + s.protected = protected + s.readDeadline = time.Time{} + if !protected { + s.readDeadline = now.Add(ackWaitTimeout) + } + _ = s.conn.SetReadDeadline(s.readDeadline) +} + +func (o *LifecycleOwner) classifyActivity(s *lifecycleSession, state LifecycleState) error { + if s.sharedSlot || (state != LifecycleStateActivity && state != LifecycleStateRelease) { + return nil + } + if o.otherClients >= maxOtherLifecycleClients { + return errLifecycleClientLimit + } + s.sharedSlot = true + o.otherClients++ + return nil +} diff --git a/pkg/daemon/agent/lifecycle_unix.go b/pkg/daemon/agent/lifecycle_unix.go new file mode 100644 index 0000000000..0ab165f623 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_unix.go @@ -0,0 +1,27 @@ +//go:build !windows + +package agent + +import ( + "os" + "syscall" +) + +func fileOwner(fi os.FileInfo) (int, bool) { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, false + } + return int(st.Uid), true +} + +// openNoFollow refuses a trailing symlink and never blocks on a FIFO. +func openNoFollow(path string, flag int, mode os.FileMode) (*os.File, error) { + //nolint:gosec // callers pass fixed lifecycle record paths + return os.OpenFile(path, flag|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, mode) +} + +func processExists(pid int) bool { + err := syscall.Kill(pid, 0) + return err == nil || err == syscall.EPERM +} diff --git a/pkg/daemon/agent/lifecycle_windows.go b/pkg/daemon/agent/lifecycle_windows.go new file mode 100644 index 0000000000..07bf1d0342 --- /dev/null +++ b/pkg/daemon/agent/lifecycle_windows.go @@ -0,0 +1,14 @@ +//go:build windows + +package agent + +import "os" + +func fileOwner(os.FileInfo) (int, bool) { return 0, false } + +func openNoFollow(path string, flag int, mode os.FileMode) (*os.File, error) { + //nolint:gosec // callers pass fixed lifecycle record paths + return os.OpenFile(path, flag, mode) +} + +func processExists(int) bool { return false } diff --git a/pkg/devcontainer/config/container_details.go b/pkg/devcontainer/config/container_details.go index 38aa970a3b..72f25a83ef 100644 --- a/pkg/devcontainer/config/container_details.go +++ b/pkg/devcontainer/config/container_details.go @@ -36,6 +36,9 @@ type ContainerDetails struct { ImageID string `json:"Image,omitempty"` ImageManifestDescriptor *ocispec.Descriptor `json:"ImageManifestDescriptor,omitempty"` Created string `json:"Created,omitempty"` + Dockerless bool `json:"Dockerless,omitempty"` + ImageEntrypoint bool `json:"ImageEntrypoint,omitempty"` + LifecycleTopologyError string `json:"LifecycleTopologyError,omitempty"` State ContainerDetailsState `json:"State"` Config ContainerDetailsConfig `json:"Config"` Mounts []ContainerMount `json:"Mounts,omitempty"` diff --git a/pkg/devcontainer/kubernetes_recreate_preflight_test.go b/pkg/devcontainer/kubernetes_recreate_preflight_test.go new file mode 100644 index 0000000000..ee4d5fc5a4 --- /dev/null +++ b/pkg/devcontainer/kubernetes_recreate_preflight_test.go @@ -0,0 +1,296 @@ +package devcontainer + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/devcontainer/metadata" + kubernetesdriver "github.com/devsy-org/devsy/pkg/driver/kubernetes" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/status" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubescheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +const ( + kubeRecreateWorkspace = "policy-workspace" + kubeRecreateNamespace = "policy-test" + kubeRecreateUID = "policy-owner" + kubeRecreateContext = "fixture" + kubeRecreateArchitecture = "amd64" + regularSidecarTemplate = "spec:\n containers:\n - name: helper\n image: example/helper:1\n" +) + +type kubeTemplatePolicyCase struct { + name string + template string + timeout string + metadataNone bool + disableDaemon bool + reject bool +} + +func TestKubernetesRecreateTemplatePreflightPreservesExistingPod(t *testing.T) { + for _, tc := range []kubeTemplatePolicyCase{ + {name: "positive regular sidecar rejected", template: regularSidecarTemplate, timeout: "1m", reject: true}, + {name: "zero regular sidecar allowed", template: regularSidecarTemplate}, + { + name: "resolved image shutdown none allows sidecar", + template: regularSidecarTemplate, timeout: "1m", metadataNone: true, + }, + {name: "current disable daemon allows sidecar", template: regularSidecarTemplate, timeout: "1m", disableDaemon: true}, + { + name: "positive normal init allowed", timeout: "1m", + template: "spec:\n initContainers:\n - name: prepare\n image: example/init:1\n", + }, + { + name: "positive native sidecar allowed", timeout: "1m", + template: "spec:\n initContainers:\n - name: helper\n" + + " image: example/helper:1\n restartPolicy: Always\n", + }, + } { + t.Run(tc.name, func(t *testing.T) { + r, params, api := newKubernetesTemplateRecreateFixture(t, tc) + before, _, fixtureErr := api.snapshot() + require.NoError(t, fixtureErr) + + _, _, err := r.buildNewContainerConfig(t.Context(), params) + + after, writes, fixtureErr := api.snapshot() + require.NoError(t, fixtureErr) + if tc.reject { + assert.Error(t, err) + assert.Equal( + t, + before, + after, + "unsupported template must not stop the existing pod", + ) + assert.Empty(t, writes, "preflight must not change API state") + return + } + require.NoError(t, err) + assert.Nil( + t, + after, + "supported explicit recreation reaches the real Kubernetes stop boundary", + ) + assert.Equal(t, []string{http.MethodDelete + " " + api.podPath}, writes) + }) + } +} + +func newKubernetesTemplateRecreateFixture( + t *testing.T, + tc kubeTemplatePolicyCase, +) (*runner, *resolveParams, *kubeRecreateAPI) { + t.Helper() + api := newKubeRecreateAPI() + server := httptest.NewServer(api) + t.Cleanup(server.Close) + info := newKubeRecreateWorkspaceInfo(t, server.URL, tc) + d, err := kubernetesdriver.NewKubernetesDriver(info) + require.NoError(t, err) + r := newTestRunner(d) + r.id, r.workspaceConfig, r.reporter = kubeRecreateWorkspace, info, status.Nop() + imageMetadata := &config.ImageMetadataConfig{} + if tc.metadataNone { + imageConfig := &config.DevContainerConfig{} + imageConfig.ShutdownAction = config.ShutdownActionNone + config.AddConfigToImageMetadata(imageConfig, imageMetadata) + } + labels, err := json.Marshal(imageMetadata.Raw) + require.NoError(t, err) + image := &config.ImageDetails{ID: "fixture-image", Config: config.ImageDetailsConfig{ + Labels: map[string]string{metadata.ImageMetadataLabel: string(labels)}, + }} + r.imageBackend = &separateImages{ + details: image, + buildInfo: &config.BuildInfo{ + ImageName: "fixture-image", + ImageDetails: image, + ImageMetadata: imageMetadata, + }, + } + params := recreateResolveParams() + params.options.Recreate = true + params.parsedConfig.Config.Image = "example/workspace:1" + params.parsedConfig.Raw = config.CloneDevContainerConfig(params.parsedConfig.Config) + params.substitutionContext = &config.SubstitutionContext{ + ContainerWorkspaceFolder: managedWorkspaceFolder, + } + return r, params, api +} + +func newKubeRecreateWorkspaceInfo( + t *testing.T, + serverURL string, + tc kubeTemplatePolicyCase, +) *provider.AgentWorkspaceInfo { + t.Helper() + dir := t.TempDir() + kubeconfig := filepath.Join(dir, "kubeconfig") + data, err := clientcmd.Write(clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{ + kubeRecreateContext: {Server: serverURL}, + }, + AuthInfos: map[string]*clientcmdapi.AuthInfo{kubeRecreateContext: {}}, + Contexts: map[string]*clientcmdapi.Context{kubeRecreateContext: { + Cluster: kubeRecreateContext, AuthInfo: kubeRecreateContext, + Namespace: kubeRecreateNamespace, + }}, + CurrentContext: kubeRecreateContext, + }) + require.NoError(t, err) + require.NoError(t, os.WriteFile(kubeconfig, data, 0o600)) + template := filepath.Join(dir, "pod.yaml") + require.NoError(t, os.WriteFile(template, []byte(tc.template), 0o600)) + return &provider.AgentWorkspaceInfo{ + Workspace: &provider.Workspace{ID: kubeRecreateWorkspace, UID: kubeRecreateUID}, + Agent: provider.ProviderAgentConfig{ + Driver: provider.KubernetesDriver, + ContainerTimeout: tc.timeout, + Kubernetes: provider.ProviderKubernetesDriverConfig{ + KubernetesConfig: kubeconfig, + KubernetesNamespace: kubeRecreateNamespace, + PodManifestTemplate: template, + Architecture: kubeRecreateArchitecture, + }, + }, + CLIOptions: provider.CLIOptions{DisableDaemon: tc.disableDaemon}, + } +} + +type kubeRecreateAPI struct { + mu sync.Mutex + podPath string + pod *corev1.Pod + writes []string + err error +} + +func newKubeRecreateAPI() *kubeRecreateAPI { + name := "devsy-" + kubeRecreateWorkspace + return &kubeRecreateAPI{ + podPath: "/api/v1/namespaces/" + kubeRecreateNamespace + "/pods/" + name, + pod: &corev1.Pod{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: kubeRecreateNamespace, + UID: "existing-pod", + ResourceVersion: "7", + Labels: map[string]string{ + kubernetesdriver.DevsyCreatedLabel: stringTrue, + kubernetesdriver.DevsyWorkspaceUIDLabel: kubeRecreateUID, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: kubernetesdriver.DevContainerName, Image: "example/old:1"}, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + } +} + +func (s *kubeRecreateAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + if r.Method != http.MethodGet && r.Method != http.MethodHead { + s.writes = append(s.writes, r.Method+" "+r.URL.Path) + } + if r.URL.Path != s.podPath || s.pod == nil { + s.writeStatus(w, http.StatusNotFound, metav1.StatusReasonNotFound) + return + } + s.servePod(w, r) +} + +func (s *kubeRecreateAPI) servePod(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + s.writeObject(w, http.StatusOK, s.pod) + case http.MethodDelete: + if !s.deleteMatches(w, r) { + return + } + s.pod = nil + s.writeObject(w, http.StatusOK, &metav1.Status{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Status", + }, + Status: metav1.StatusSuccess, + }) + default: + s.writeStatus(w, http.StatusMethodNotAllowed, metav1.StatusReasonMethodNotAllowed) + } +} + +func (s *kubeRecreateAPI) deleteMatches(w http.ResponseWriter, r *http.Request) bool { + body, err := io.ReadAll(r.Body) + if err != nil { + s.err = err + s.writeStatus(w, http.StatusBadRequest, metav1.StatusReasonBadRequest) + return false + } + options := &metav1.DeleteOptions{} + if _, _, err := kubescheme.Codecs.UniversalDeserializer(). + Decode(body, nil, options); err != nil { + s.err = err + s.writeStatus(w, http.StatusBadRequest, metav1.StatusReasonBadRequest) + return false + } + if preconditions := options.Preconditions; preconditions != nil { + if preconditions.UID != nil && *preconditions.UID != s.pod.UID { + s.writeStatus(w, http.StatusConflict, metav1.StatusReasonConflict) + return false + } + if preconditions.ResourceVersion != nil && + *preconditions.ResourceVersion != s.pod.ResourceVersion { + s.writeStatus(w, http.StatusConflict, metav1.StatusReasonConflict) + return false + } + } + return true +} + +func (s *kubeRecreateAPI) writeStatus( + w http.ResponseWriter, + code int32, + reason metav1.StatusReason, +) { + s.writeObject(w, code, &metav1.Status{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Status"}, + Status: metav1.StatusFailure, Reason: reason, Code: code, + }) +} + +func (s *kubeRecreateAPI) writeObject(w http.ResponseWriter, code int32, object runtime.Object) { + w.Header().Set("Content-Type", runtime.ContentTypeJSON) + w.WriteHeader(int(code)) + if err := json.NewEncoder(w).Encode(object); err != nil { + s.err = err + } +} + +func (s *kubeRecreateAPI) snapshot() (*corev1.Pod, []string, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.pod.DeepCopy(), append([]string(nil), s.writes...), s.err +} diff --git a/pkg/devcontainer/managed_lifecycle_test.go b/pkg/devcontainer/managed_lifecycle_test.go new file mode 100644 index 0000000000..5e0624d435 --- /dev/null +++ b/pkg/devcontainer/managed_lifecycle_test.go @@ -0,0 +1,449 @@ +package devcontainer + +import ( + "context" + "testing" + "time" + + "github.com/devsy-org/api/pkg/devsy" + pkgagent "github.com/devsy-org/devsy/pkg/agent" + pkgconfig "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/status" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + managedWorkspaceHost = "workspace" + managedWorkspaceFolder = "/workspaces/app" + testEntrypoint = "/app" +) + +type managedMockDriver struct { + *mockDriver + shutdownAction string +} + +func (*managedMockDriver) ManagedLifecycle() bool { return true } + +func (d *managedMockDriver) SetLifecycleShutdownAction(action string) { + d.shutdownAction = action +} + +func managedRunner(timeout string) (*runner, *managedMockDriver) { + d := &managedMockDriver{mockDriver: &mockDriver{}} + r := newTestRunner(d) + r.workspaceConfig.Workspace = &provider.Workspace{} + r.workspaceConfig.Agent.ContainerTimeout = timeout + return r, d +} + +func resolveParamsFor(platform devsy.PlatformOptions) *resolveParams { + return &resolveParams{ + substitutionContext: &config.SubstitutionContext{ + ContainerWorkspaceFolder: managedWorkspaceFolder, + }, + options: UpOptions{CLIOptions: provider.CLIOptions{Platform: platform}}, + } +} + +func mergedWithEnv() *config.MergedDevContainerConfig { + merged := &config.MergedDevContainerConfig{} + merged.ContainerEnv = map[string]string{} + merged.ShutdownAction = config.ShutdownActionStopContainer + return merged +} + +func injectedConfig(t *testing.T, merged *config.MergedDevContainerConfig) *agentd.DaemonConfig { + t.Helper() + encoded, ok := merged.ContainerEnv[pkgconfig.EnvWorkspaceDaemonConfig] + require.True(t, ok, "daemon config was not injected") + cfg, err := agentd.DecodeDaemonConfig(encoded) + require.NoError(t, err) + return cfg +} + +func TestInjectDaemonEntrypointManagedWithoutPlatform(t *testing.T) { + r, _ := managedRunner("1m") + merged := mergedWithEnv() + + require.NoError( + t, + r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged, false, false), + ) + + cfg := injectedConfig(t, merged) + assert.Equal(t, "1m0s", cfg.Timeout) + assert.Equal(t, config.ShutdownActionStopContainer, cfg.ShutdownAction) + assert.True(t, cfg.Lifecycle.Managed()) + assert.Empty(t, cfg.Ssh, "no embedded SSH for a timeout-only config") + assert.Empty(t, cfg.Platform.AccessKey) +} + +func TestInjectDaemonEntrypointManagedWithPlatformKeepsNetworking(t *testing.T) { + r, _ := managedRunner("1m") + merged := mergedWithEnv() + platform := devsy.PlatformOptions{AccessKey: "key", PlatformHost: "p", WorkspaceHost: "w"} + + require.NoError(t, r.injectDaemonEntrypoint(resolveParamsFor(platform), merged, false, false)) + + cfg := injectedConfig(t, merged) + assert.Equal(t, "key", cfg.Platform.AccessKey) + assert.Equal(t, "root", cfg.Ssh.User) + assert.Equal(t, "1m0s", cfg.Timeout) +} + +func TestInjectDaemonEntrypointUnmanagedWithoutPlatformIsUntouched(t *testing.T) { + r := newTestRunner(&mockDriver{}) + r.workspaceConfig.Workspace = &provider.Workspace{} + merged := mergedWithEnv() + + require.NoError( + t, + r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged, false, false), + ) + + assert.Empty(t, merged.ContainerEnv) +} + +func TestInjectDaemonEntrypointResolvesLegacyAlias(t *testing.T) { + r, _ := managedRunner("") + r.workspaceConfig.Agent.Kubernetes.InactivityTimeout = "5m" + merged := mergedWithEnv() + + require.NoError( + t, + r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged, false, false), + ) + + assert.Equal(t, "5m0s", injectedConfig(t, merged).Timeout) +} + +func TestInjectDaemonEntrypointPropagatesConfigErrors(t *testing.T) { + r, _ := managedRunner("1m") + r.workspaceConfig.Agent.Kubernetes.InactivityTimeout = "2m" + + err := r.injectDaemonEntrypoint( + resolveParamsFor(devsy.PlatformOptions{}), + mergedWithEnv(), + false, + false, + ) + + assert.ErrorContains(t, err, "conflicts") +} + +func TestInjectDaemonEntrypointShutdownNoneDisablesTimeout(t *testing.T) { + r, _ := managedRunner("1m") + merged := mergedWithEnv() + merged.ShutdownAction = config.ShutdownActionNone + + require.NoError( + t, + r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged, false, false), + ) + + cfg := injectedConfig(t, merged) + assert.Empty(t, cfg.Timeout) + assert.Equal(t, config.ShutdownActionNone, cfg.ShutdownAction) +} + +func TestWorkspaceInfoCarriesEffectiveTimeoutAndOwner(t *testing.T) { + r, _ := managedRunner("") + r.workspaceConfig.Agent.Kubernetes.InactivityTimeout = "90s" + + compressed, err := r.compressWorkspaceConfig(mergedWithEnv(), false, false) + require.NoError(t, err) + info, _, err := pkgagent.DecodeContainerWorkspaceInfo(compressed) + require.NoError(t, err) + + assert.Equal(t, "1m30s", info.ContainerTimeout) + assert.Equal(t, provider.ContainerLifecycleOwnerManagedDaemon, info.LifecycleOwner) +} + +func TestManagedTimeoutConflict(t *testing.T) { + no, yes := false, true + withEntrypoint := &config.BuildInfo{ImageDetails: &config.ImageDetails{ + Config: config.ImageDetailsConfig{Entrypoint: []string{testEntrypoint}}, + }} + empty := &config.BuildInfo{ImageDetails: &config.ImageDetails{}} + + tests := []struct { + name string + override *bool + build *config.BuildInfo + wantErr string + }{ + {name: "default override", build: withEntrypoint}, + {name: "explicit override", override: &yes, build: withEntrypoint}, + { + name: "user entrypoint bypasses daemon", + override: &no, + build: withEntrypoint, + wantErr: "overrideCommand", + }, + {name: "no user entrypoint to bypass", override: &no, build: empty}, + { + name: "dockerless nested container", + build: &config.BuildInfo{Dockerless: &config.BuildInfoDockerless{}}, + wantErr: "dockerless", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + merged := &config.MergedDevContainerConfig{} + merged.OverrideCommand = tt.override + + err := managedTimeoutConflict(time.Minute, merged, tt.build) + + if tt.wantErr == "" { + assert.NoError(t, err) + return + } + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestFindRunningContainerOrFailRejectsTerminalObjectsForManagedDrivers(t *testing.T) { + r, d := managedRunner("1m") + d.findResult = &config.ContainerDetails{ + State: config.ContainerDetailsState{Status: config.ContainerStatusExited}, + } + + _, err := r.findRunningContainerOrFail(context.Background(), "start") + + assert.ErrorContains(t, err, "not running") + + plain := newTestRunner(&mockDriver{findResult: d.findResult}) + details, err := plain.findRunningContainerOrFail(context.Background(), "start") + require.NoError(t, err) + assert.NotNil(t, details, "other drivers keep their existing behavior") +} + +func TestDockerlessBootstrapDoesNotRequireManagedOwner(t *testing.T) { + for _, platform := range []devsy.PlatformOptions{ + {}, + {AccessKey: "key", PlatformHost: "platform", WorkspaceHost: managedWorkspaceHost}, + } { + name := "without platform" + if platform.AccessKey != "" { + name = "with platform" + } + t.Run(name, func(t *testing.T) { + r, _ := managedRunner("") + merged := mergedWithEnv() + params := resolveParamsFor(platform) + require.NoError(t, r.injectDaemonEntrypoint(params, merged, true, false)) + + info, err := r.prepareSetupInfo(&setupContainerParams{ + rawConfig: &config.DevContainerConfig{}, + containerDetails: &config.ContainerDetails{ + Dockerless: true, + State: config.ContainerDetailsState{Status: config.ContainerStatusRunning}, + }, + mergedConfig: merged, + substitutionContext: params.substitutionContext, + }) + require.NoError(t, err) + workspace, _, err := pkgagent.DecodeContainerWorkspaceInfo( + info.workspaceConfigCompressed, + ) + require.NoError(t, err) + assert.Empty(t, workspace.LifecycleOwner) + assert.Empty(t, workspace.ContainerTimeout) + + daemon := injectedConfig(t, merged) + assert.Equal(t, merged.ShutdownAction, daemon.ShutdownAction) + assert.False(t, daemon.Lifecycle.Managed()) + assert.False(t, daemon.Lifecycle.SetupArmsTimer) + assert.Empty(t, daemon.Timeout) + if platform.AccessKey == "" { + assert.Empty(t, daemon.Platform.AccessKey) + assert.Empty(t, daemon.Ssh) + return + } + assert.Equal(t, platform, daemon.Platform) + assert.Equal( + t, + agentd.SshConfig{User: containerRootUser, Workdir: managedWorkspaceFolder}, + daemon.Ssh, + ) + }) + } +} + +func TestDockerlessProducerRejectsEnabledTimeout(t *testing.T) { + for _, legacy := range []bool{false, true} { + name := "canonical" + if legacy { + name = "legacy" + } + t.Run(name, func(t *testing.T) { + r, _ := managedRunner("1m") + if legacy { + r.workspaceConfig.Agent.ContainerTimeout = "" + r.workspaceConfig.Agent.Kubernetes.InactivityTimeout = "1m" + } + merged := mergedWithEnv() + err := r.injectDaemonEntrypoint( + resolveParamsFor(devsy.PlatformOptions{}), + merged, + true, + false, + ) + require.ErrorContains(t, err, "dockerless") + assert.NotContains(t, merged.ContainerEnv, pkgconfig.EnvWorkspaceDaemonConfig) + + _, err = r.compressWorkspaceConfig(merged, true, false) + assert.ErrorContains(t, err, "dockerless") + }) + } +} + +func TestDockerlessProducerHonorsDisabledTimeout(t *testing.T) { + for _, disableDaemon := range []bool{false, true} { + name := "shutdown none" + if disableDaemon { + name = "disabled daemon" + } + t.Run(name, func(t *testing.T) { + r, _ := managedRunner("1m") + merged := mergedWithEnv() + if disableDaemon { + r.workspaceConfig.CLIOptions.DisableDaemon = true + } else { + merged.ShutdownAction = config.ShutdownActionNone + } + require.NoError( + t, + r.injectDaemonEntrypoint( + resolveParamsFor(devsy.PlatformOptions{}), + merged, + true, + false, + ), + ) + compressed, err := r.compressWorkspaceConfig(merged, true, false) + require.NoError(t, err) + workspace, _, err := pkgagent.DecodeContainerWorkspaceInfo(compressed) + require.NoError(t, err) + assert.Empty(t, workspace.LifecycleOwner) + assert.Empty(t, workspace.ContainerTimeout) + daemon := injectedConfig(t, merged) + assert.Equal(t, merged.ShutdownAction, daemon.ShutdownAction) + assert.False(t, daemon.Lifecycle.Managed()) + assert.Empty(t, daemon.Timeout) + }) + } +} + +func TestZeroTimeoutKubernetesStillRequiresManagedOwner(t *testing.T) { + r, _ := managedRunner("") + merged := mergedWithEnv() + require.NoError( + t, + r.injectDaemonEntrypoint(resolveParamsFor(devsy.PlatformOptions{}), merged, false, false), + ) + assert.True(t, injectedConfig(t, merged).Lifecycle.Managed()) + + compressed, err := r.compressWorkspaceConfig(merged, false, false) + require.NoError(t, err) + workspace, _, err := pkgagent.DecodeContainerWorkspaceInfo(compressed) + require.NoError(t, err) + assert.Equal(t, provider.ContainerLifecycleOwnerManagedDaemon, workspace.LifecycleOwner) + assert.Empty(t, workspace.ContainerTimeout) +} + +func TestZeroTimeoutOwnershipFollowsActualImageMain(t *testing.T) { + no := false + for _, tt := range []struct { + name string + override *bool + image config.ImageDetailsConfig + managed bool + }{ + { + name: "default override retains the daemon", + image: config.ImageDetailsConfig{Entrypoint: []string{testEntrypoint}}, + managed: true, + }, + {name: "empty image main retains the daemon", override: &no, managed: true}, + { + name: "external entrypoint owns the container", override: &no, + image: config.ImageDetailsConfig{Entrypoint: []string{testEntrypoint}}, + }, + { + name: "external command owns the container", override: &no, + image: config.ImageDetailsConfig{Cmd: []string{testEntrypoint}}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + r, _ := managedRunner("") + merged := mergedWithEnv() + merged.OverrideCommand = tt.override + image := &config.ImageDetails{Config: tt.image} + external := usesImageEntrypoint(merged, image) + require.NoError(t, r.injectDaemonEntrypoint( + resolveParamsFor(devsy.PlatformOptions{}), merged, false, external, + )) + assert.Equal(t, tt.managed, injectedConfig(t, merged).Lifecycle.Managed()) + + _, args := GetContainerEntrypointAndArgs(merged, image) + if external { + want := append([]string{}, tt.image.Entrypoint...) + want = append(want, tt.image.Cmd...) + assert.Equal(t, want, args[3:]) + } else { + assert.Empty(t, args[3:]) + } + compressed, err := r.compressWorkspaceConfig(merged, false, external) + require.NoError(t, err) + info, _, err := pkgagent.DecodeContainerWorkspaceInfo(compressed) + require.NoError(t, err) + if tt.managed { + assert.Equal(t, provider.ContainerLifecycleOwnerManagedDaemon, info.LifecycleOwner) + } else { + assert.Empty(t, info.LifecycleOwner) + } + }) + } +} + +func TestUnsupportedManagedRecreatePreservesExistingContainer(t *testing.T) { + const ( + workspaceUID = "workspace-uid" + inspectedImageID = "image" + builtImageName = "built-image" + baseImageName = "alpine" + ) + r, d := managedRunner("1m") + r.workspaceConfig.Workspace = &provider.Workspace{ID: r.id, UID: workspaceUID} + image := &config.ImageDetails{ID: inspectedImageID, Config: config.ImageDetailsConfig{ + Entrypoint: []string{testEntrypoint}, + }} + r.imageBackend = &separateImages{ + details: image, + buildInfo: &config.BuildInfo{ + ImageName: builtImageName, ImageDetails: image, + ImageMetadata: &config.ImageMetadataConfig{}, + }, + } + r.reporter = status.Nop() + no := false + params := recreateResolveParams() + params.options.Recreate = true + params.parsedConfig.Config.Image = baseImageName + params.parsedConfig.Config.OverrideCommand = &no + params.parsedConfig.Raw = config.CloneDevContainerConfig(params.parsedConfig.Config) + params.substitutionContext = &config.SubstitutionContext{} + + _, _, err := r.buildNewContainerConfig(context.Background(), params) + + require.Error(t, err) + assert.False(t, d.stopCalled) + assert.False(t, d.deleteCalled) +} diff --git a/pkg/devcontainer/setup.go b/pkg/devcontainer/setup.go index f457cb682a..9e5e3c8721 100644 --- a/pkg/devcontainer/setup.go +++ b/pkg/devcontainer/setup.go @@ -18,6 +18,7 @@ import ( "github.com/devsy-org/devsy/pkg/agent/tunnelserver" "github.com/devsy-org/devsy/pkg/compress" pkgconfig "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/devcontainer/crane" "github.com/devsy-org/devsy/pkg/devcontainer/sshtunnel" @@ -309,6 +310,12 @@ func (r *runner) legacyInject(ctx context.Context, timeout time.Duration) error } func (r *runner) prepareSetupInfo(params *setupContainerParams) (*setupInfo, error) { + if params.containerDetails.LifecycleTopologyError != "" { + return nil, fmt.Errorf( + "workspace lifecycle topology: %s", + params.containerDetails.LifecycleTopologyError, + ) + } result := r.buildResult(params) compressed, err := r.compressResult(result) @@ -316,7 +323,11 @@ func (r *runner) prepareSetupInfo(params *setupContainerParams) (*setupInfo, err return nil, err } - workspaceConfigCompressed, err := r.compressWorkspaceConfig() + workspaceConfigCompressed, err := r.compressWorkspaceConfig( + result.MergedConfig, + params.containerDetails.Dockerless, + params.containerDetails.ImageEntrypoint, + ) if err != nil { return nil, err } @@ -375,7 +386,10 @@ func (r *runner) compressResult(result *config.Result) (string, error) { return compressed, nil } -func (r *runner) compressWorkspaceConfig() (string, error) { +func (r *runner) compressWorkspaceConfig( + mergedConfig *config.MergedDevContainerConfig, + dockerless, imageEntrypoint bool, +) (string, error) { cliOptions := r.workspaceConfig.CLIOptions cliOptions.SecretsEnv = nil cliOptions.SecretsMount = nil @@ -396,6 +410,16 @@ func (r *runner) compressWorkspaceConfig() (string, error) { Agent: r.workspaceConfig.Agent, ContentFolder: r.workspaceConfig.ContentFolder, } + lifecycle, err := r.lifecycleOptions(mergedConfig, dockerless, imageEntrypoint) + if err != nil { + return "", err + } + if r.managedLifecycle() { + workspaceConfig.ContainerTimeout = agentd.FormatTimeout(lifecycle.Timeout) + } + if lifecycle.Managed { + workspaceConfig.LifecycleOwner = provider2.ContainerLifecycleOwnerManagedDaemon + } workspaceConfig.PullFromInsideContainer = resolvePullFromInsideContainer( r.workspaceConfig.CLIOptions, r.workspaceConfig.Workspace.Source.GitRepository, diff --git a/pkg/devcontainer/setup_secrets_test.go b/pkg/devcontainer/setup_secrets_test.go index db61ba5fb8..7696922b59 100644 --- a/pkg/devcontainer/setup_secrets_test.go +++ b/pkg/devcontainer/setup_secrets_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/devsy-org/devsy/pkg/compress" + "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/provider" ) @@ -31,7 +32,7 @@ func TestCompressWorkspaceConfig_StripsSecretValues(t *testing.T) { }, } - compressed, err := r.compressWorkspaceConfig() + compressed, err := r.compressWorkspaceConfig(&config.MergedDevContainerConfig{}, false, false) if err != nil { t.Fatal(err) } diff --git a/pkg/devcontainer/single.go b/pkg/devcontainer/single.go index 4c256eae2d..282f65951e 100644 --- a/pkg/devcontainer/single.go +++ b/pkg/devcontainer/single.go @@ -7,7 +7,9 @@ import ( "maps" "os" "strings" + "time" + "github.com/devsy-org/api/pkg/devsy" "github.com/devsy-org/devsy/pkg/agent/delivery" "github.com/devsy-org/devsy/pkg/clierr" "github.com/devsy-org/devsy/pkg/command" @@ -263,7 +265,17 @@ func (r *runner) resolveExistingContainer( r.recovering = true } - containerDetails, err := r.ensureRunning(ctx, containerDetails) + var mergedConfig *config.MergedDevContainerConfig + var err error + if d, ok := r.driver.(driver.ManagedLifecycleDriver); ok && d.ManagedLifecycle() { + mergedConfig, err = r.mergeExistingContainerConfig(ctx, containerDetails, p) + if err != nil { + return nil, err + } + d.SetLifecycleShutdownAction(mergedConfig.ShutdownAction) + } + + containerDetails, err = r.ensureRunning(ctx, containerDetails) if err != nil { return nil, err } @@ -273,9 +285,11 @@ func (r *runner) resolveExistingContainer( p.substitutionContext.ContainerWorkspaceFolder = containerDetails.Config.WorkingDir } - mergedConfig, err := r.mergeExistingContainerConfig(ctx, containerDetails, p) - if err != nil { - return nil, err + if mergedConfig == nil { + mergedConfig, err = r.mergeExistingContainerConfig(ctx, containerDetails, p) + if err != nil { + return nil, err + } } containerDetails, err = r.reprovisionIfNeeded(ctx, containerDetails) @@ -390,7 +404,13 @@ func (r *runner) resolveNewContainer( return nil, err } - r.injectDaemonEntrypoint(p, mergedConfig) + imageEntrypoint := buildInfo.Dockerless == nil && + usesImageEntrypoint(mergedConfig, buildInfo.ImageDetails) + if err := r.injectDaemonEntrypoint( + p, mergedConfig, buildInfo.Dockerless != nil, imageEntrypoint, + ); err != nil { + return nil, err + } r.attemptPreStartDelivery(ctx, mergedConfig, p, buildInfo) @@ -407,6 +427,9 @@ func (r *runner) resolveNewContainer( if err != nil { return nil, err } + // New run options take precedence over metadata from an existing workspace volume. + containerDetails.Dockerless = buildInfo.Dockerless != nil + containerDetails.ImageEntrypoint = imageEntrypoint if w := r.lingerWarning(ctx); w != "" { hostWarnings = append(hostWarnings, w) @@ -482,7 +505,6 @@ func (r *runner) buildNewContainerConfig( return nil, nil, err } - _, validatesImageRun := r.driver.(driver.ImageRunValidator) mergedConfig, err := config.MergeConfiguration( activeConfig.Config, buildInfo.ImageMetadata.Config, @@ -496,10 +518,8 @@ func (r *runner) buildNewContainerConfig( return nil, nil, err } config.MergeExtraRemoteEnvConfig(mergedConfig, overlay) - if validatesImageRun { - if err := r.validateImageRun(p, activeConfig.Config, mergedConfig, buildInfo); err != nil { - return nil, nil, err - } + if err := r.validateImageRun(p, activeConfig.Config, mergedConfig, buildInfo); err != nil { + return nil, nil, err } if p.options.Recreate { if err := r.deleteForRecreate(ctx); err != nil { @@ -544,6 +564,16 @@ func (r *runner) validateImageRun( mergedConfig *config.MergedDevContainerConfig, buildInfo *config.BuildInfo, ) error { + if d, ok := r.driver.(driver.ManagedLifecycleDriver); ok && d.ManagedLifecycle() { + d.SetLifecycleShutdownAction(mergedConfig.ShutdownAction) + } + if err := r.validateManagedTimeoutTopology(mergedConfig, buildInfo); err != nil { + return err + } + validator, validates := r.driver.(driver.ImageRunValidator) + if !validates { + return nil + } var options *driver.RunOptions var err error if buildInfo.Dockerless != nil { @@ -564,7 +594,7 @@ func (r *runner) validateImageRun( if err != nil { return fmt.Errorf("validate image run options: %w", err) } - return r.driver.(driver.ImageRunValidator).ValidateRunImageDevContainer( + return validator.ValidateRunImageDevContainer( &driver.RunImageDevContainerParams{ WorkspaceID: r.id, Options: options, ParsedConfig: parsedConfig, }, @@ -675,27 +705,95 @@ func (r *runner) deleteForRecreate(ctx context.Context) error { } // injectDaemonEntrypoint adds the workspace daemon config to the container -// environment when platform configuration is provided. +// environment for Platform workspaces and for drivers whose main process is the +// managed lifecycle daemon. func (r *runner) injectDaemonEntrypoint( p *resolveParams, mergedConfig *config.MergedDevContainerConfig, -) { - if p.options.Platform.AccessKey == "" { - return + dockerless, imageEntrypoint bool, +) error { + lifecycle, err := r.lifecycleOptions(mergedConfig, dockerless, imageEntrypoint) + if err != nil { + return err + } + platform := p.options.Platform + if platform.AccessKey == "" { + platform = devsy.PlatformOptions{} + } + if !r.managedLifecycle() && platform.AccessKey == "" { + return nil } - log.Debugf("Platform config detected, injecting Devsy daemon entrypoint.") data, err := agent.GetEncodedWorkspaceDaemonConfig( - p.options.Platform, + platform, r.workspaceConfig.Workspace, p.substitutionContext, mergedConfig, + lifecycle, ) if err != nil { - log.Errorf("Failed to marshal daemon config: %v", err) - return + return fmt.Errorf("build daemon config: %w", err) } mergedConfig.ContainerEnv[pkgconfig.EnvWorkspaceDaemonConfig] = data + return nil +} + +// validateManagedTimeoutTopology rejects configurations in which the managed +// daemon would not be the process whose exit stops the container. +func (r *runner) validateManagedTimeoutTopology( + mergedConfig *config.MergedDevContainerConfig, + buildInfo *config.BuildInfo, +) error { + if !r.managedLifecycle() { + return nil + } + timeout, err := r.effectiveContainerTimeout(mergedConfig.ShutdownAction) + if err != nil || timeout == 0 { + return err + } + return managedTimeoutConflict(timeout, mergedConfig, buildInfo) +} + +func managedTimeoutConflict( + timeout time.Duration, + mergedConfig *config.MergedDevContainerConfig, + buildInfo *config.BuildInfo, +) error { + _, err := agent.ResolveManagedLifecycle( + timeout, + buildInfo.Dockerless != nil, + usesImageEntrypoint(mergedConfig, buildInfo.ImageDetails), + ) + return err +} + +func (r *runner) managedLifecycle() bool { + d, ok := r.driver.(driver.ManagedLifecycleDriver) + return ok && d.ManagedLifecycle() +} + +func (r *runner) lifecycleOptions( + mergedConfig *config.MergedDevContainerConfig, + dockerless, imageEntrypoint bool, +) (agent.LifecycleOptions, error) { + if !r.managedLifecycle() { + return agent.LifecycleOptions{}, nil + } + timeout, err := r.effectiveContainerTimeout(mergedConfig.ShutdownAction) + if err != nil { + return agent.LifecycleOptions{}, err + } + return agent.ResolveManagedLifecycle(timeout, dockerless, imageEntrypoint) +} + +func (r *runner) effectiveContainerTimeout(shutdownAction string) (time.Duration, error) { + agentConfig := r.workspaceConfig.Agent + return agent.ResolveInactivityTimeout(agent.TimeoutInputs{ + Canonical: agentConfig.ContainerTimeout, + Legacy: agentConfig.Kubernetes.InactivityTimeout, + ShutdownAction: shutdownAction, + DisableDaemon: r.workspaceConfig.CLIOptions.DisableDaemon, + }) } // findRunningContainerOrFail re-fetches container details from the driver and @@ -713,6 +811,13 @@ func (r *runner) findRunningContainerOrFail( if details == nil { return nil, fmt.Errorf("dev container %s not found after %s", r.id, operation) } + // A terminal object must not satisfy startup when the daemon owns the lifecycle. + if r.managedLifecycle() && details.State.Status != config.ContainerStatusRunning { + return nil, fmt.Errorf( + "dev container %s is %s after %s, not running", + r.id, details.State.Status, operation, + ) + } return details, nil } @@ -1040,25 +1145,26 @@ func (r *runner) getRunOptions( } return &driver.RunOptions{ - UID: r.workspaceUID(), - Image: buildInfo.ImageName, - ImageBuilt: buildInfo.BuiltLocally, - User: user, - RemoteUser: effectiveRemoteUser(mergedConfig, user), - Entrypoint: entrypoint, - Cmd: cmd, - Env: mergedConfig.ContainerEnv, - CapAdd: mergedConfig.CapAdd, - Labels: labels, - Privileged: mergedConfig.Privileged, - Init: mergedConfig.Init, - WorkspaceMount: workspaceMountPtr, - SecurityOpt: mergedConfig.SecurityOpt, - Mounts: mounts, - Userns: substitutionContext.Userns, - UidMap: substitutionContext.UidMap, - GidMap: substitutionContext.GidMap, - Platform: r.workspaceConfig.CLIOptions.RunPlatform, + UID: r.workspaceUID(), + Image: buildInfo.ImageName, + ImageBuilt: buildInfo.BuiltLocally, + User: user, + RemoteUser: effectiveRemoteUser(mergedConfig, user), + ImageEntrypoint: usesImageEntrypoint(mergedConfig, buildInfo.ImageDetails), + Entrypoint: entrypoint, + Cmd: cmd, + Env: mergedConfig.ContainerEnv, + CapAdd: mergedConfig.CapAdd, + Labels: labels, + Privileged: mergedConfig.Privileged, + Init: mergedConfig.Init, + WorkspaceMount: workspaceMountPtr, + SecurityOpt: mergedConfig.SecurityOpt, + Mounts: mounts, + Userns: substitutionContext.Userns, + UidMap: substitutionContext.UidMap, + GidMap: substitutionContext.GidMap, + Platform: r.workspaceConfig.CLIOptions.RunPlatform, }, nil } @@ -1194,6 +1300,15 @@ func GetStartScript(mergedConfig *config.MergedDevContainerConfig) string { return joinShellStatements(statements...) } +func usesImageEntrypoint( + mergedConfig *config.MergedDevContainerConfig, + imageDetails *config.ImageDetails, +) bool { + return imageDetails != nil && mergedConfig.OverrideCommand != nil && + !*mergedConfig.OverrideCommand && + len(imageDetails.Config.Entrypoint)+len(imageDetails.Config.Cmd) > 0 +} + func GetContainerEntrypointAndArgs( mergedConfig *config.MergedDevContainerConfig, imageDetails *config.ImageDetails, @@ -1203,8 +1318,7 @@ func GetContainerEntrypointAndArgs( GetStartScript(mergedConfig), "-", } // `wait $!` allows for the `trap` to run (synchronous `sleep` would not). - if imageDetails != nil && mergedConfig.OverrideCommand != nil && - !*mergedConfig.OverrideCommand { + if usesImageEntrypoint(mergedConfig, imageDetails) { cmd = append(cmd, imageDetails.Config.Entrypoint...) cmd = append(cmd, imageDetails.Config.Cmd...) } diff --git a/pkg/driver/kubernetes/client.go b/pkg/driver/kubernetes/client.go index b631520dc8..7f1a3932e2 100644 --- a/pkg/driver/kubernetes/client.go +++ b/pkg/driver/kubernetes/client.go @@ -17,7 +17,7 @@ import ( ) type Client struct { - client *kubernetes.Clientset + client kubernetes.Interface config *rest.Config } @@ -63,7 +63,7 @@ func NewClient(kubeConfig, kubeContext string) (*Client, string, error) { }, namespace, nil } -func (c *Client) Client() *kubernetes.Clientset { +func (c *Client) Client() kubernetes.Interface { return c.client } diff --git a/pkg/driver/kubernetes/driver.go b/pkg/driver/kubernetes/driver.go index 22d56e6f02..3cbb0e2bb0 100644 --- a/pkg/driver/kubernetes/driver.go +++ b/pkg/driver/kubernetes/driver.go @@ -39,10 +39,11 @@ func NewKubernetesDriver( log.Debugf("Use Kubernetes Namespace %q", namespace) return &KubernetesDriver{ - client: client, - namespace: namespace, - options: &options, - agentConfig: &workspaceInfo.Agent, + client: client, + namespace: namespace, + options: &options, + agentConfig: &workspaceInfo.Agent, + disableDaemon: workspaceInfo.CLIOptions.DisableDaemon, }, nil } @@ -74,14 +75,22 @@ type KubernetesDriver struct { client *Client - options *provider2.ProviderKubernetesDriverConfig - agentConfig *provider2.ProviderAgentConfig + options *provider2.ProviderKubernetesDriverConfig + agentConfig *provider2.ProviderAgentConfig + disableDaemon bool + shutdownAction string + hasShutdownAction bool } func (k *KubernetesDriver) CanReprovision() bool { return true } +func (k *KubernetesDriver) SetLifecycleShutdownAction(shutdownAction string) { + k.shutdownAction = shutdownAction + k.hasShutdownAction = true +} + func (k *KubernetesDriver) SupportsMountType(mountType string) bool { switch mountType { case driver.MountTypeBind, driver.MountTypeVolume, driver.MountTypeTmpfs: diff --git a/pkg/driver/kubernetes/find.go b/pkg/driver/kubernetes/find.go index 3e579c169f..0f63328126 100644 --- a/pkg/driver/kubernetes/find.go +++ b/pkg/driver/kubernetes/find.go @@ -2,10 +2,10 @@ package kubernetes import ( "context" + "fmt" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/log" - corev1 "k8s.io/api/core/v1" ) func (k *KubernetesDriver) FindDevContainer( @@ -27,17 +27,12 @@ func (k *KubernetesDriver) FindDevContainer( // check pod pod, err := k.getPod(ctx, pvc.Name) if err != nil { - log.Debugf("Error finding pod: %v", err) - log.Warn( - "If the pod does not come up automatically it is stuck in an error state. " + - "Recreate the workspace to recover from this", - ) - pod = nil + return nil, fmt.Errorf("find pod for workspace %q: %w", workspaceId, err) } // determine status status := config.ContainerStatusExited - if pod != nil && isPodRunning(pod) { + if pod != nil && classifyPod(pod) == podRunning { status = config.ContainerStatusRunning } @@ -46,20 +41,35 @@ func (k *KubernetesDriver) FindDevContainer( if pod != nil { startedAt = pod.CreationTimestamp.String() } + // A volume without saved options still has to be found so stop and delete can clean it up. + var ( + dockerless, imageEntrypoint bool + topologyError string + labels = map[string]string{} + ) + if containerInfo.Options == nil { + topologyError = fmt.Sprintf("workspace volume %q has no saved run options", workspaceId) + } else { + var topologyErr error + dockerless, imageEntrypoint, topologyErr = observedPodTopology(pod, containerInfo.Options) + if topologyErr != nil { + topologyError = topologyErr.Error() + } + labels = config.ListToObject(containerInfo.Options.Labels) + } return &config.ContainerDetails{ - ID: pvc.Name, - Created: pvc.CreationTimestamp.String(), + ID: pvc.Name, + Created: pvc.CreationTimestamp.String(), + Dockerless: dockerless, + ImageEntrypoint: imageEntrypoint, + LifecycleTopologyError: topologyError, State: config.ContainerDetailsState{ Status: status, StartedAt: startedAt, }, Config: config.ContainerDetailsConfig{ - Labels: config.ListToObject(containerInfo.Options.Labels), + Labels: labels, }, }, nil } - -func isPodRunning(pod *corev1.Pod) bool { - return pod.Status.Phase == corev1.PodRunning -} diff --git a/pkg/driver/kubernetes/pod_lifecycle.go b/pkg/driver/kubernetes/pod_lifecycle.go new file mode 100644 index 0000000000..2965109c21 --- /dev/null +++ b/pkg/driver/kubernetes/pod_lifecycle.go @@ -0,0 +1,429 @@ +package kubernetes + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strings" + "time" + + pkgconfig "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/driver" + "github.com/devsy-org/devsy/pkg/log" + provider2 "github.com/devsy-org/devsy/pkg/provider" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" +) + +const maxReplaceAttempts = 3 + +var ( + podPollInterval = time.Second + podDeleteTimeout = 2 * time.Minute +) + +// errPodChanged means the pod observed before a delete is no longer the pod +// that exists, so the caller must observe again. +var errPodChanged = errors.New("pod changed since it was observed") + +type podState int + +const ( + podPending podState = iota + podRunning + podTerminating + podTerminal + podUnknown +) + +func classifyPod(pod *corev1.Pod) podState { + switch { + case pod.DeletionTimestamp != nil: + return podTerminating + case pod.Status.Phase == corev1.PodFailed, pod.Status.Phase == corev1.PodSucceeded: + return podTerminal + case pod.Status.Phase == corev1.PodPending: + return podPending + case pod.Status.Phase == corev1.PodRunning: + if devsyContainerTerminated(pod) { + return podTerminal + } + return podRunning + default: + return podUnknown + } +} + +func devsyContainerTerminated(pod *corev1.Pod) bool { + for i := range pod.Status.ContainerStatuses { + c := &pod.Status.ContainerStatuses[i] + if c.Name == DevContainerName { + return IsTerminated(c) + } + } + return false +} + +// verifyOwnedPod refuses to touch a pod that Devsy did not create for this +// workspace. Pods that predate the workspace UID label are matched by the +// workspace volume claim. +func verifyOwnedPod(pod *corev1.Pod, id, workspaceUID string) error { + if pod.Name != id || pod.Labels[DevsyCreatedLabel] != "true" { + return fmt.Errorf("pod %q exists but was not created by Devsy; refusing to replace it", id) + } + if uid := pod.Labels[DevsyWorkspaceUIDLabel]; uid != "" { + if workspaceUID != "" && uid != workspaceUID { + return fmt.Errorf( + "pod %q belongs to workspace %q, not %q; refusing to replace it", + id, uid, workspaceUID, + ) + } + return nil + } + if !usesWorkspaceVolume(pod, id) { + return fmt.Errorf("pod %q does not use the workspace volume; refusing to replace it", id) + } + return nil +} + +func usesWorkspaceVolume(pod *corev1.Pod, claim string) bool { + for _, v := range pod.Spec.Volumes { + if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName == claim { + return true + } + } + return false +} + +// reconcileExistingPod makes sure no stale pod blocks creating the workspace +// pod. It reports true when a healthy pod with unchanged options is reused. +func (k *KubernetesDriver) reconcileExistingPod( + ctx context.Context, + id string, + options *driver.RunOptions, +) (bool, error) { + for range maxReplaceAttempts { + pod, err := k.getPod(ctx, id) + if err != nil { + return false, fmt.Errorf("get pod: %s: %w", id, err) + } + if pod == nil { + return false, nil + } + if err := verifyOwnedPod(pod, id, options.UID); err != nil { + return false, err + } + + reused, err := k.reconcileDesiredPod(ctx, pod, options) + if errors.Is(err, errPodChanged) { + continue + } + if err != nil || reused { + return reused, err + } + } + return false, fmt.Errorf("pod %q kept changing while it was being replaced", id) +} + +// reconcileObservedPod handles one observation. It returns (false, nil) after +// removing the pod, so the caller observes again and then creates. +func (k *KubernetesDriver) reconcileObservedPod( + ctx context.Context, + id string, + pod *corev1.Pod, +) (bool, error) { + switch classifyPod(pod) { + case podTerminating: + log.Debugf("Pod %q is terminating, waiting for it to be deleted", id) + return false, k.waitPodGone(ctx, id, pod.UID) + case podTerminal: + log.Infof( + "Replacing stopped pod %q (phase %s), workspace volume is kept", + id, + pod.Status.Phase, + ) + return false, k.deleteObservedPod(ctx, id, pod.UID) + case podUnknown: + return false, fmt.Errorf( + "pod %q is in phase %q; its node may be unreachable, not replacing it", + id, pod.Status.Phase, + ) + } + + if !optionsEqual(storedProviderOptions(pod), k.options) { + log.Debug("Provider options changed") + return false, k.deleteObservedPod(ctx, id, pod.UID) + } + log.Debugf("Pod %q already exists and nothing changed, skipping update", id) + if classifyPod(pod) == podPending { + if _, err := k.waitPodRunning(ctx, id); err != nil { + return false, err + } + } + return true, nil +} + +func (k *KubernetesDriver) reconcileDesiredPod( + ctx context.Context, + pod *corev1.Pod, + options *driver.RunOptions, +) (bool, error) { + // A pending pod has not started its containers, so replacing it loses nothing. + if state := classifyPod(pod); (state == podRunning || state == podPending) && + podLaunchDiffers(pod, options) { + return false, k.deleteObservedPod(ctx, pod.Name, pod.UID) + } + reused, err := k.reconcileObservedPod(ctx, pod.Name, pod) + if err != nil || !reused { + return reused, err + } + // A pod admitted while the timeout was off can hold sidecars that outlive the daemon. + timeout, err := lifecycleTimeout(options) + if err != nil { + return false, fmt.Errorf("read lifecycle policy: %w", err) + } + if err := validatePodTopology(pod, timeout); err != nil { + return false, err + } + return true, nil +} + +func storedProviderOptions(pod *corev1.Pod) *provider2.ProviderKubernetesDriverConfig { + existing := &provider2.ProviderKubernetesDriverConfig{} + if raw := pod.GetAnnotations()[DevsyLastAppliedAnnotation]; raw != "" { + if err := json.Unmarshal([]byte(raw), existing); err != nil { + log.Errorf("Error unmarshalling existing provider options, continuing...: %s", err) + } + } + return existing +} + +// deleteObservedPod deletes exactly the observed pod instance and waits until +// the API server confirms it is gone. Finalizers are never removed. +func (k *KubernetesDriver) deleteObservedPod(ctx context.Context, id string, uid types.UID) error { + err := k.client.Client().CoreV1().Pods(k.namespace).Delete(ctx, id, metav1.DeleteOptions{ + GracePeriodSeconds: new(int64(10)), + Preconditions: &metav1.Preconditions{UID: &uid}, + }) + switch { + case err == nil: + case kerrors.IsNotFound(err): + return nil + case kerrors.IsConflict(err): + return errPodChanged + default: + return fmt.Errorf("delete pod %q: %w", id, err) + } + return k.waitPodGone(ctx, id, uid) +} + +// waitPodGone polls until the pod instance uid is absent. Only NotFound counts +// as absent; any other API error is returned. +func (k *KubernetesDriver) waitPodGone(ctx context.Context, id string, uid types.UID) error { + err := wait.PollUntilContextTimeout( + ctx, + podPollInterval, + podDeleteTimeout, + true, + func(ctx context.Context) (bool, error) { + pod, err := k.client.Client(). + CoreV1(). + Pods(k.namespace). + Get(ctx, id, metav1.GetOptions{}) + switch { + case kerrors.IsNotFound(err): + return true, nil + case err != nil: + return false, fmt.Errorf("check pod %q deletion: %w", id, err) + } + return uid != "" && pod.UID != uid, nil + }, + ) + if err != nil && ctx.Err() == nil && wait.Interrupted(err) { + return fmt.Errorf("timeout waiting for pod %q to be deleted: %w", id, err) + } + return err +} + +// ManagedLifecycle reports that the workspace pod's main process is the Devsy +// daemon, which owns inactivity shutdown. +func (k *KubernetesDriver) ManagedLifecycle() bool { return true } + +var ( + _ driver.ManagedLifecycleDriver = (*KubernetesDriver)(nil) + _ driver.ImageRunValidator = (*KubernetesDriver)(nil) +) + +// effectiveTimeout resolves the current provider timeout for a stored shutdown action. +func (k *KubernetesDriver) effectiveTimeout(shutdownAction string) (time.Duration, error) { + var canonical string + if k.agentConfig != nil { + canonical = k.agentConfig.ContainerTimeout + } + return agentd.ResolveInactivityTimeout(agentd.TimeoutInputs{ + Canonical: canonical, + Legacy: k.options.InactivityTimeout, + ShutdownAction: shutdownAction, + DisableDaemon: k.disableDaemon, + }) +} + +// refreshStoredLifecycle regenerates the lifecycle policy in run options that +// were persisted with the volume, so a replacement pod uses current provider +// values rather than those from when the workspace was created. +func (k *KubernetesDriver) refreshStoredLifecycle( + options *driver.RunOptions, +) (*driver.RunOptions, error) { + if options == nil { + return nil, nil + } + stored := options.Env[pkgconfig.EnvWorkspaceDaemonConfig] + action := "" + if stored != "" { + cfg, err := agentd.DecodeDaemonConfig(stored) + if err != nil { + return nil, err + } + action = cfg.ShutdownAction + } + if k.hasShutdownAction { + action = k.shutdownAction + } + timeout, err := k.effectiveTimeout(action) + if err != nil { + return nil, err + } + lifecycle, err := agentd.ResolveManagedLifecycle( + timeout, options.Dockerless, options.ImageEntrypoint, + ) + if err != nil { + return nil, err + } + refreshed, err := agentd.RefreshLifecycle(stored, lifecycle, action) + if err != nil { + return nil, err + } + out := *options + out.Env = make(map[string]string, len(options.Env)+1) + maps.Copy(out.Env, options.Env) + out.Env[pkgconfig.EnvWorkspaceDaemonConfig] = refreshed + return &out, nil +} + +// lifecycleTimeout returns the positive timeout the generated daemon config +// asks the managed daemon to enforce, or zero. +func lifecycleTimeout(options *driver.RunOptions) (time.Duration, error) { + raw := options.Env[pkgconfig.EnvWorkspaceDaemonConfig] + if raw == "" { + return 0, nil + } + cfg, err := agentd.DecodeDaemonConfig(raw) + if err != nil { + return 0, err + } + if !cfg.Lifecycle.Managed() || cfg.Timeout == "" || + cfg.ShutdownAction == config.ShutdownActionNone { + return 0, nil + } + return time.ParseDuration(cfg.Timeout) +} + +// validatePodTopology rejects pods in which another regular container could +// keep the pod Running after the Devsy daemon exits. Native sidecars +// (restartable init containers) end with the main container and are allowed. +func validatePodTopology(pod *corev1.Pod, timeout time.Duration) error { + if timeout <= 0 { + return nil + } + hasDevsy := false + for i := range pod.Spec.Containers { + hasDevsy = hasDevsy || pod.Spec.Containers[i].Name == DevContainerName + } + if !hasDevsy { + return fmt.Errorf( + "inactivity timeout %s needs a %q container in the pod", + timeout, + DevContainerName, + ) + } + return validateNoExtraContainers(pod, timeout) +} + +// validateNoExtraContainers is the part of the topology check that can run on +// the pod template, before the devsy container is added to it. +func validateNoExtraContainers(pod *corev1.Pod, timeout time.Duration) error { + if timeout <= 0 { + return nil + } + var extra []string + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name != DevContainerName { + extra = append(extra, pod.Spec.Containers[i].Name) + } + } + if len(extra) > 0 { + return fmt.Errorf( + "inactivity timeout %s cannot stop a pod with extra regular containers (%s) because "+ + "the pod keeps running while they do; move them to native sidecars "+ + "(init containers with restartPolicy Always) or unset the inactivity timeout", + timeout, strings.Join(extra, ", "), + ) + } + return nil +} + +// launchCommandChanged reports whether admission rewrote what the devsy +// container executes, which could displace the managed daemon. +func launchCommandChanged(requested, admitted *corev1.Pod) bool { + want, got := devsyContainer(requested), devsyContainer(admitted) + if want == nil || got == nil { + return false + } + return !slices.Equal(want.Command, got.Command) || !slices.Equal(want.Args, got.Args) +} + +func devsyContainer(pod *corev1.Pod) *corev1.Container { + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == DevContainerName { + return &pod.Spec.Containers[i] + } + } + return nil +} + +// rejectAdmittedPod removes a freshly created pod whose admitted topology +// violates the stop contract. The workspace volume is kept. +func (k *KubernetesDriver) rejectAdmittedPod( + ctx context.Context, + created, requested *corev1.Pod, + timeout time.Duration, +) error { + topologyErr := validatePodTopology(created, timeout) + if topologyErr == nil && timeout > 0 && launchCommandChanged(requested, created) { + topologyErr = fmt.Errorf( + "the %q container no longer starts the Devsy daemon as its main process", + DevContainerName, + ) + } + if topologyErr == nil { + return nil + } + cleanup := k.deleteObservedPod(ctx, created.Name, created.UID) + if cleanup != nil { + return fmt.Errorf( + "%w (admission changed the pod; removing it also failed: %w)", topologyErr, cleanup, + ) + } + return fmt.Errorf( + "admission changed the pod and the inactivity timeout cannot be enforced: %w", + topologyErr, + ) +} diff --git a/pkg/driver/kubernetes/pod_lifecycle_test.go b/pkg/driver/kubernetes/pod_lifecycle_test.go new file mode 100644 index 0000000000..519838efa8 --- /dev/null +++ b/pkg/driver/kubernetes/pod_lifecycle_test.go @@ -0,0 +1,1155 @@ +package kubernetes + +import ( + "context" + "encoding/json" + "errors" + "os" + "sync/atomic" + "testing" + "time" + + pkgconfig "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/driver" + provider2 "github.com/devsy-org/devsy/pkg/provider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" +) + +const ( + testNamespace = "devsy-test" + testPodID = "devsy-ws" + testWorkspace = "ws-uid" + podResourceName = "pods" + preservedCustomerMetadata = "keep" +) + +var podsResource = schema.GroupResource{Resource: podResourceName} + +type fakeCluster struct { + cs *fake.Clientset + k *KubernetesDriver +} + +func newFakeCluster(t *testing.T, objects ...runtime.Object) *fakeCluster { + t.Helper() + oldPoll, oldTimeout := podPollInterval, podDeleteTimeout + podPollInterval, podDeleteTimeout = 5*time.Millisecond, 2*time.Second + t.Cleanup(func() { podPollInterval, podDeleteTimeout = oldPoll, oldTimeout }) + + cs := fake.NewClientset(objects...) + return &fakeCluster{ + cs: cs, + k: &KubernetesDriver{ + client: &Client{client: cs}, + namespace: testNamespace, + options: &provider2.ProviderKubernetesDriverConfig{ + PodTimeout: "2s", + DiskSize: "1Gi", + }, + agentConfig: &provider2.ProviderAgentConfig{}, + }, + } +} + +func (c *fakeCluster) opts() *driver.RunOptions { + return &driver.RunOptions{UID: testWorkspace} +} + +type podSpec struct { + uid string + phase corev1.PodPhase + options *provider2.ProviderKubernetesDriverConfig + deleting bool + foreign bool + devsy *corev1.ContainerStatus +} + +func (c *fakeCluster) pod(t *testing.T, s podSpec) *corev1.Pod { + t.Helper() + if s.options == nil { + s.options = c.k.options + } + raw, err := json.Marshal(s.options) + require.NoError(t, err) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: testPodID, + Namespace: testNamespace, + UID: types.UID(s.uid), + Labels: map[string]string{DevsyWorkspaceUIDLabel: testWorkspace}, + Annotations: map[string]string{DevsyLastAppliedAnnotation: string(raw)}, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: DevContainerName}}}, + Status: corev1.PodStatus{Phase: s.phase}, + } + if !s.foreign { + pod.Labels[DevsyCreatedLabel] = pkgconfig.BoolTrue + } + if s.deleting { + now := metav1.Now() + pod.DeletionTimestamp = &now + pod.Finalizers = []string{"test/hold"} + } + if s.devsy != nil { + s.devsy.Name = DevContainerName + pod.Status.ContainerStatuses = []corev1.ContainerStatus{*s.devsy} + } + return pod +} + +func (c *fakeCluster) putPod(t *testing.T, pod *corev1.Pod) { + t.Helper() + pods := c.cs.CoreV1().Pods(testNamespace) + _, err := pods.Get(context.Background(), pod.Name, metav1.GetOptions{}) + if kerrors.IsNotFound(err) { + _, err = pods.Create(context.Background(), pod, metav1.CreateOptions{}) + } else { + _, err = pods.Update(context.Background(), pod, metav1.UpdateOptions{}) + } + require.NoError(t, err) +} + +func (c *fakeCluster) podExists(t *testing.T) bool { + t.Helper() + _, err := c.cs.CoreV1(). + Pods(testNamespace). + Get(context.Background(), testPodID, metav1.GetOptions{}) + if kerrors.IsNotFound(err) { + return false + } + require.NoError(t, err) + return true +} + +func (c *fakeCluster) deleteCalls() int { + n := 0 + for _, a := range c.cs.Actions() { + if a.Matches("delete", podResourceName) { + n++ + } + } + return n +} + +func (c *fakeCluster) reconcile(ctx context.Context) (bool, error) { + return c.k.reconcileExistingPod(ctx, testPodID, c.opts()) +} + +func TestReconcileNoPodAllowsCreate(t *testing.T) { + c := newFakeCluster(t) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.False(t, reused) +} + +func TestReconcileReusesRunningPodWithSameOptions(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{ + uid: "u1", + phase: corev1.PodRunning, + devsy: &corev1.ContainerStatus{ + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }, + })) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.True(t, reused) + assert.Zero(t, c.deleteCalls()) +} + +func TestReconcileReplacesTerminalPodEvenWithSameOptions(t *testing.T) { + for _, phase := range []corev1.PodPhase{corev1.PodFailed, corev1.PodSucceeded} { + t.Run(string(phase), func(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: phase})) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.False(t, reused, "a terminal pod must not be reused") + assert.False( + t, + c.podExists(t), + "the terminal pod is gone before the new one is created", + ) + }) + } +} + +func TestReconcileReplacesPodWhoseDevsyContainerExited(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{ + uid: "u1", phase: corev1.PodRunning, + devsy: &corev1.ContainerStatus{State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}, + }}, + })) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.False(t, reused) + assert.False(t, c.podExists(t)) +} + +func TestReconcileReplacesPodWhenOptionsChanged(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{ + uid: "u1", phase: corev1.PodRunning, + options: &provider2.ProviderKubernetesDriverConfig{PodTimeout: "2s", DiskSize: "9Gi"}, + })) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.False(t, reused) + assert.False(t, c.podExists(t)) +} + +func TestReconcilePendingPodWaitsForReadiness(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending})) + var ready atomic.Bool + go func() { + time.Sleep(40 * time.Millisecond) + running := c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning}) + running.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: DevContainerName, Ready: true, + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }} + c.putPod(t, running) + ready.Store(true) + }() + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.True(t, reused) + assert.True(t, ready.Load(), "success must wait for the pod, not just its object") +} + +func TestReconcileTerminatingPodWaitsForDeletionThenAllowsCreate(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning, deleting: true})) + go func() { + time.Sleep(40 * time.Millisecond) + _ = c.cs.CoreV1().Pods(testNamespace).Delete( + context.Background(), testPodID, metav1.DeleteOptions{}, + ) + }() + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.False(t, reused) + assert.False(t, c.podExists(t)) + assert.Zero(t, c.deleteCalls()-1, "a terminating pod is waited on, not deleted again") +} + +func TestReconcileForbiddenLookupIsNotAbsence(t *testing.T) { + c := newFakeCluster(t) + c.cs.PrependReactor( + "get", + podResourceName, + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, kerrors.NewForbidden(podsResource, testPodID, errors.New("rbac")) + }, + ) + + reused, err := c.reconcile(context.Background()) + + require.Error(t, err) + assert.True(t, kerrors.IsForbidden(err), "forbidden must surface") + assert.Contains(t, err.Error(), "forbidden") + assert.False(t, reused) +} + +func TestReconcileDeleteForbiddenSurfaces(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodFailed})) + c.cs.PrependReactor( + "delete", + podResourceName, + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, kerrors.NewForbidden(podsResource, testPodID, errors.New("rbac")) + }, + ) + + _, err := c.reconcile(context.Background()) + + require.Error(t, err) + assert.Contains(t, err.Error(), "forbidden") + assert.True(t, c.podExists(t), "nothing is assumed deleted") +} + +func TestWaitPodGoneDoesNotTreatLookupErrorsAsDeleted(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodFailed})) + c.cs.PrependReactor( + "get", + podResourceName, + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("connection refused") + }, + ) + + err := c.k.waitPodGone(context.Background(), testPodID, "u1") + + require.Error(t, err) + assert.Contains(t, err.Error(), "connection refused") +} + +func TestReconcileUIDConflictReobservesReplacementPod(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "old", phase: corev1.PodFailed})) + var conflicted atomic.Bool + c.cs.PrependReactor( + "delete", + podResourceName, + func(a clienttesting.Action) (bool, runtime.Object, error) { + if conflicted.Swap(true) { + return false, nil, nil + } + // Someone replaced the pod between our observation and our delete. + replacement := c.pod(t, podSpec{ + uid: "new", + phase: corev1.PodRunning, + devsy: &corev1.ContainerStatus{ + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }, + }) + require.NoError(t, c.cs.Tracker().Update( + corev1.SchemeGroupVersion.WithResource(podResourceName), replacement, testNamespace, + )) + return true, nil, kerrors.NewConflict(podsResource, testPodID, errors.New("uid")) + }, + ) + + reused, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.True(t, reused, "the healthy replacement is kept, not deleted by stale knowledge") + assert.True(t, c.podExists(t)) +} + +func TestReconcileDeleteCarriesObservedUID(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "observed", phase: corev1.PodFailed})) + var got *types.UID + c.cs.PrependReactor( + "delete", + podResourceName, + func(a clienttesting.Action) (bool, runtime.Object, error) { + if del, ok := a.(clienttesting.DeleteActionImpl); ok && + del.DeleteOptions.Preconditions != nil { + got = del.DeleteOptions.Preconditions.UID + } + return false, nil, nil + }, + ) + + _, err := c.reconcile(context.Background()) + + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, types.UID("observed"), *got) +} + +func TestReconcileCancellationWhileWaitingForDeletion(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning, deleting: true})) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err := c.reconcile(ctx) + + require.Error(t, err) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.True(t, c.podExists(t)) +} + +func TestReconcileRefusesPodsNotOwnedByDevsy(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodFailed, foreign: true})) + + _, err := c.reconcile(context.Background()) + + require.Error(t, err) + assert.Contains(t, err.Error(), "not created by Devsy") + assert.True(t, c.podExists(t)) +} + +func TestReconcileRefusesPodOfAnotherWorkspace(t *testing.T) { + c := newFakeCluster(t) + pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodFailed}) + pod.Labels[DevsyWorkspaceUIDLabel] = "someone-else" + c.putPod(t, pod) + + _, err := c.reconcile(context.Background()) + + require.Error(t, err) + assert.True(t, c.podExists(t)) +} + +func TestReconcileAdoptsOldPodThroughWorkspaceVolume(t *testing.T) { + c := newFakeCluster(t) + pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodFailed}) + delete(pod.Labels, DevsyWorkspaceUIDLabel) + pod.Spec.Volumes = []corev1.Volume{{ + Name: DevContainerName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: testPodID}, + }, + }} + c.putPod(t, pod) + + _, err := c.reconcile(context.Background()) + + require.NoError(t, err) + assert.False(t, c.podExists(t)) +} + +func TestReplacementPreservesWorkspaceVolume(t *testing.T) { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: testPodID, Namespace: testNamespace, UID: "pvc-uid"}, + } + c := newFakeCluster(t, pvc) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodSucceeded})) + + _, err := c.reconcile(context.Background()) + require.NoError(t, err) + + got, err := c.cs.CoreV1().PersistentVolumeClaims(testNamespace). + Get(context.Background(), testPodID, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, types.UID("pvc-uid"), got.UID) + for _, a := range c.cs.Actions() { + assert.False(t, a.Matches("delete", "persistentvolumeclaims"), "PVC must never be deleted") + } +} + +func TestFindDevContainerReportsLookupErrors(t *testing.T) { + c := newFakeCluster(t, workspacePVC(t)) + c.cs.PrependReactor( + "get", + podResourceName, + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, kerrors.NewForbidden(podsResource, testPodID, errors.New("rbac")) + }, + ) + + details, err := c.k.FindDevContainer(context.Background(), "ws") + + require.Error(t, err) + assert.Nil(t, details) +} + +func TestFindDevContainerTreatsTerminalPodAsStopped(t *testing.T) { + c := newFakeCluster(t, workspacePVC(t)) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodSucceeded})) + + details, err := c.k.FindDevContainer(context.Background(), "ws") + + require.NoError(t, err) + assert.Equal(t, config.ContainerStatusExited, details.State.Status) +} + +func TestFindDevContainerKeepsVolumeWithoutSavedOptionsFindable(t *testing.T) { + pvc := workspacePVC(t) + raw, err := json.Marshal(&DevContainerInfo{WorkspaceID: "ws"}) + require.NoError(t, err) + pvc.Annotations[DevsyInfoAnnotation] = string(raw) + c := newFakeCluster(t, pvc) + + details, err := c.k.FindDevContainer(context.Background(), "ws") + + require.NoError(t, err, "stop and delete must still find a volume with no saved options") + require.NotNil(t, details) + assert.Equal(t, config.ContainerStatusExited, details.State.Status) + assert.Contains(t, details.LifecycleTopologyError, "no saved run options") +} + +func TestReusedRunningPodWithExtraContainerIsRejectedWhenTimeoutEnabled(t *testing.T) { + managed := agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon} + c := newFakeCluster(t) + pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning}) + pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{Name: "istio-proxy"}) + c.putPod(t, pod) + + reused, err := c.k.reconcileDesiredPod( + context.Background(), pod, + daemonOptions(t, agentd.DaemonConfig{Timeout: "1m", Lifecycle: managed}), + ) + + require.ErrorContains(t, err, "istio-proxy") + assert.False(t, reused) + assert.True(t, c.podExists(t), "a rejected reuse must not delete the pod") +} + +func TestReusedRunningPodWithExtraContainerIsKeptWhenTimeoutDisabled(t *testing.T) { + managed := agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon} + c := newFakeCluster(t) + pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning}) + pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{Name: "istio-proxy"}) + c.putPod(t, pod) + + reused, err := c.k.reconcileDesiredPod( + context.Background(), pod, + daemonOptions(t, agentd.DaemonConfig{Lifecycle: managed}), + ) + + require.NoError(t, err) + assert.True(t, reused) +} + +func TestVerifyDesiredPodReportsMissingPodWithoutInternalSentinel(t *testing.T) { + options := canonicalTopologyOptions(t, false) + c := newFakeCluster(t, savedRunVolume(t, options)) + + err := c.k.verifyDesiredPod(context.Background(), testPodID, options) + + require.Error(t, err) + assert.NotErrorIs(t, err, errPodChanged) + assert.Contains(t, err.Error(), "retry the operation") +} + +func workspacePVC(t *testing.T) *corev1.PersistentVolumeClaim { + t.Helper() + info, err := json.Marshal( + &DevContainerInfo{WorkspaceID: "ws", Options: &driver.RunOptions{UID: testWorkspace}}, + ) + require.NoError(t, err) + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: testPodID, Namespace: testNamespace, + Annotations: map[string]string{DevsyInfoAnnotation: string(info)}, + }} +} + +func daemonOptions(t *testing.T, cfg agentd.DaemonConfig) *driver.RunOptions { + t.Helper() + encoded, err := agentd.EncodeDaemonConfig(&cfg) + require.NoError(t, err) + return &driver.RunOptions{Env: map[string]string{"DEVSY_WORKSPACE_DAEMON_CONFIG": encoded}} +} + +func TestTopologyValidation(t *testing.T) { + regular := func(names ...string) *corev1.Pod { + pod := &corev1.Pod{} + for _, n := range names { + pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{Name: n}) + } + return pod + } + always := corev1.ContainerRestartPolicyAlways + + t.Run("single devsy container", func(t *testing.T) { + assert.NoError(t, validatePodTopology(regular(DevContainerName), time.Minute)) + }) + t.Run("native sidecar and init container allowed", func(t *testing.T) { + pod := regular(DevContainerName) + pod.Spec.InitContainers = []corev1.Container{ + {Name: "mesh", RestartPolicy: &always}, + {Name: "setup"}, + } + assert.NoError(t, validatePodTopology(pod, time.Minute)) + }) + t.Run("extra regular containers rejected by name", func(t *testing.T) { + err := validatePodTopology(regular(DevContainerName, "istio-proxy", "logger"), time.Minute) + require.Error(t, err) + assert.Contains(t, err.Error(), "istio-proxy") + assert.Contains(t, err.Error(), "logger") + }) + t.Run("extra containers fine when timeout disabled", func(t *testing.T) { + assert.NoError(t, validatePodTopology(regular(DevContainerName, "istio-proxy"), 0)) + }) + t.Run("missing devsy container rejected", func(t *testing.T) { + assert.Error(t, validatePodTopology(regular("other"), time.Minute)) + }) +} + +func TestLifecycleTimeoutFromGeneratedConfig(t *testing.T) { + managed := agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon} + + got, err := lifecycleTimeout( + daemonOptions(t, agentd.DaemonConfig{Timeout: "2m", Lifecycle: managed}), + ) + require.NoError(t, err) + assert.Equal(t, 2*time.Minute, got) + + got, err = lifecycleTimeout(daemonOptions(t, agentd.DaemonConfig{ + Timeout: "2m", Lifecycle: managed, ShutdownAction: config.ShutdownActionNone, + })) + require.NoError(t, err) + assert.Zero(t, got, "shutdownAction none disables the timeout") + + got, err = lifecycleTimeout(&driver.RunOptions{}) + require.NoError(t, err) + assert.Zero(t, got) +} + +func TestRefreshStoredLifecycleUsesCurrentProviderValues(t *testing.T) { + c := newFakeCluster(t) + c.k.agentConfig.ContainerTimeout = "7m" + stored := daemonOptions(t, agentd.DaemonConfig{ + Timeout: "1m", + ShutdownAction: config.ShutdownActionStopContainer, + Ssh: agentd.SshConfig{User: "root"}, + }) + + refreshed, err := c.k.refreshStoredLifecycle(stored) + require.NoError(t, err) + + cfg, err := agentd.DecodeDaemonConfig(refreshed.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"]) + require.NoError(t, err) + assert.Equal(t, "7m0s", cfg.Timeout) + assert.True(t, cfg.Lifecycle.Managed()) + assert.Equal(t, "root", cfg.Ssh.User) + oldCfg, err := agentd.DecodeDaemonConfig(stored.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"]) + require.NoError(t, err) + assert.Equal(t, "1m", oldCfg.Timeout, "stored options are not mutated") +} + +func TestRefreshStoredLifecycleLegacyAliasAndConflict(t *testing.T) { + c := newFakeCluster(t) + c.k.options.InactivityTimeout = "3m" + + refreshed, err := c.k.refreshStoredLifecycle(&driver.RunOptions{}) + require.NoError(t, err) + cfg, err := agentd.DecodeDaemonConfig(refreshed.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"]) + require.NoError(t, err) + assert.Equal(t, "3m0s", cfg.Timeout) + + c.k.agentConfig.ContainerTimeout = "4m" + _, err = c.k.refreshStoredLifecycle(&driver.RunOptions{}) + assert.ErrorContains(t, err, "conflicts") +} + +func TestStoredLifecycleOwnershipFollowsDockerlessTopology(t *testing.T) { + for _, dockerless := range []bool{false, true} { + name := "ordinary image keeps managed ownership" + if dockerless { + name = "dockerless clears stale managed ownership" + } + t.Run(name, func(t *testing.T) { + c := newFakeCluster(t) + cfg := agentd.DaemonConfig{ + Timeout: "1m", + ShutdownAction: config.ShutdownActionStopContainer, + Lifecycle: agentd.LifecycleConfig{ + Owner: agentd.LifecycleOwnerManagedDaemon, SetupArmsTimer: true, + }, + Ssh: agentd.SshConfig{User: "vscode", Workdir: "/workspaces/app"}, + } + cfg.Platform.AccessKey = "key" + cfg.Platform.PlatformHost = "platform" + cfg.Platform.WorkspaceHost = "workspace" + stored := daemonOptions(t, cfg) + stored.Dockerless = dockerless + original := stored.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"] + + refreshed, err := c.k.refreshStoredLifecycle(stored) + require.NoError(t, err) + got, err := agentd.DecodeDaemonConfig(refreshed.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"]) + require.NoError(t, err) + assert.Equal(t, !dockerless, got.Lifecycle.Managed()) + assert.Equal(t, !dockerless, got.Lifecycle.SetupArmsTimer) + assert.Empty(t, got.Timeout) + assert.Equal(t, cfg.Platform, got.Platform) + assert.Equal(t, cfg.Ssh, got.Ssh) + assert.Equal(t, cfg.ShutdownAction, got.ShutdownAction) + assert.Equal(t, original, stored.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"]) + }) + } +} + +func TestDockerlessResumeRejectsEnabledTimeoutBeforeMutation(t *testing.T) { + for _, legacy := range []bool{false, true} { + name := "canonical" + if legacy { + name = "legacy" + } + t.Run(name, func(t *testing.T) { + pvc := workspacePVC(t) + info, err := json.Marshal(&DevContainerInfo{ + WorkspaceID: "ws", + Options: &driver.RunOptions{UID: testWorkspace, Dockerless: true}, + }) + require.NoError(t, err) + pvc.Annotations[DevsyInfoAnnotation] = string(info) + c := newFakeCluster(t, pvc) + if legacy { + c.k.options.InactivityTimeout = "1m" + } else { + c.k.agentConfig.ContainerTimeout = "1m" + } + + err = c.k.RunDevContainer(context.Background(), "ws", nil) + require.ErrorContains(t, err, "dockerless") + for _, action := range c.cs.Actions() { + assert.Equal( + t, + "get", + action.GetVerb(), + "invalid resume must not mutate cluster state", + ) + } + }) + } +} + +func TestDockerlessStoredLifecycleHonorsShutdownNone(t *testing.T) { + c := newFakeCluster(t) + c.k.agentConfig.ContainerTimeout = "1m" + stored := daemonOptions(t, agentd.DaemonConfig{ + ShutdownAction: config.ShutdownActionNone, + }) + stored.Dockerless = true + + refreshed, err := c.k.refreshStoredLifecycle(stored) + require.NoError(t, err) + cfg, err := agentd.DecodeDaemonConfig(refreshed.Env["DEVSY_WORKSPACE_DAEMON_CONFIG"]) + require.NoError(t, err) + assert.False(t, cfg.Lifecycle.Managed()) + assert.Empty(t, cfg.Timeout) + assert.Equal(t, config.ShutdownActionNone, cfg.ShutdownAction) +} + +func TestAdmittedPodWithInjectedContainerIsRemovedAndVolumeKept(t *testing.T) { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: testPodID, Namespace: testNamespace}, + } + c := newFakeCluster(t, pvc) + requested := c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending}) + admitted := requested.DeepCopy() + admitted.Spec.Containers = append(admitted.Spec.Containers, corev1.Container{Name: "injected"}) + c.putPod(t, admitted) + + err := c.k.rejectAdmittedPod(context.Background(), admitted, requested, time.Minute) + + require.Error(t, err) + assert.Contains(t, err.Error(), "injected") + assert.False(t, c.podExists(t)) + _, getErr := c.cs.CoreV1().PersistentVolumeClaims(testNamespace). + Get(context.Background(), testPodID, metav1.GetOptions{}) + assert.NoError(t, getErr) +} + +func TestAdmittedPodAcceptedWhenTopologyHolds(t *testing.T) { + c := newFakeCluster(t) + pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending}) + c.putPod(t, pod) + + err := c.k.rejectAdmittedPod(context.Background(), pod, pod.DeepCopy(), time.Minute) + + require.NoError(t, err) + assert.True(t, c.podExists(t)) +} + +func TestAdmittedPodWithRewrittenCommandIsRemovedWhenTimeoutEnabled(t *testing.T) { + requested := newFakeCluster(t).pod(t, podSpec{uid: "u1", phase: corev1.PodPending}) + requested.Spec.Containers[0].Command = []string{"/bin/sh"} + admitted := requested.DeepCopy() + admitted.Spec.Containers[0].Command = []string{"/webhook/entrypoint"} + + t.Run("enabled", func(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, admitted) + + err := c.k.rejectAdmittedPod(context.Background(), admitted, requested, time.Minute) + + require.ErrorContains(t, err, "Devsy daemon as its main process") + assert.False(t, c.podExists(t)) + }) + t.Run("disabled", func(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, admitted) + + require.NoError(t, c.k.rejectAdmittedPod(context.Background(), admitted, requested, 0)) + assert.True(t, c.podExists(t), "no stop contract is at stake without a timeout") + }) +} + +func TestPendingPodWithDifferentLaunchRecipeIsReplaced(t *testing.T) { + c := newFakeCluster(t) + pod := c.pod(t, podSpec{uid: "u1", phase: corev1.PodPending}) + pod.Spec.Containers[0].Image = "registry.example/old:1" + c.putPod(t, pod) + options := &driver.RunOptions{Image: "registry.example/new:2"} + + reused, err := c.k.reconcileDesiredPod(context.Background(), pod, options) + + require.NoError(t, err) + assert.False(t, reused, "the caller creates the requested pod") + assert.False(t, c.podExists(t), "a stale pending pod must not be waited on") +} + +func TestSameRunOptionsTreatsNilAndEmptyCollectionsAsEqual(t *testing.T) { + const image = "registry.example/app:1" + nilForm := &driver.RunOptions{Image: image} + emptyForm := &driver.RunOptions{ + Image: image, Cmd: []string{}, Env: map[string]string{}, Labels: []string{}, + } + + assert.True(t, sameRunOptions(nilForm, emptyForm)) + assert.False(t, sameRunOptions(nilForm, &driver.RunOptions{Image: "other"})) + assert.False(t, sameRunOptions(nilForm, &driver.RunOptions{Image: image, Cmd: []string{"x"}})) + assert.True(t, sameRunOptions(nil, nil)) + assert.False(t, sameRunOptions(nilForm, nil)) +} + +func TestExplicitStopWaitsForPodToBeGone(t *testing.T) { + c := newFakeCluster(t) + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning})) + + require.NoError(t, c.k.StopDevContainer(context.Background(), "ws")) + + assert.False(t, c.podExists(t)) +} + +func TestWaitPodDeletedIgnoresAlreadyAbsentPod(t *testing.T) { + c := newFakeCluster(t) + + require.NoError(t, c.k.waitPodDeleted(context.Background(), testPodID)) +} + +func TestWaitPodDeletedTimesOutWhenPodStaysTerminating(t *testing.T) { + c := newFakeCluster(t) + podDeleteTimeout = 50 * time.Millisecond + c.putPod(t, c.pod(t, podSpec{uid: "u1", phase: corev1.PodRunning, deleting: true})) + c.cs.PrependReactor( + "delete", + podResourceName, + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, nil + }, + ) + + err := c.k.waitPodDeleted(context.Background(), testPodID) + + require.Error(t, err) + assert.ErrorContains(t, err, "timeout waiting for pod") + assert.True(t, c.podExists(t)) +} + +func runOptionsForWorkspace(timeout string) *driver.RunOptions { + cfg, _ := agentd.EncodeDaemonConfig(&agentd.DaemonConfig{ + Timeout: timeout, + ShutdownAction: config.ShutdownActionStopContainer, + Lifecycle: agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon}, + }) + return &driver.RunOptions{ + UID: testWorkspace, + Image: "example/image:1", + Entrypoint: "/bin/sh", + Env: map[string]string{"DEVSY_WORKSPACE_DAEMON_CONFIG": cfg}, + WorkspaceMount: &config.Mount{Type: "bind", Target: "/workspaces/app"}, + } +} + +func readyOnCreate(c *fakeCluster, uid types.UID) { + nextUID := uid + c.cs.PrependReactor( + "create", + podResourceName, + func(a clienttesting.Action) (bool, runtime.Object, error) { + pod := a.(clienttesting.CreateAction).GetObject().(*corev1.Pod) + pod.UID = nextUID + nextUID += "-next" + pod.Status.Phase = corev1.PodRunning + pod.Status.ContainerStatuses = nil + for _, container := range pod.Spec.Containers { + pod.Status.ContainerStatuses = append( + pod.Status.ContainerStatuses, + corev1.ContainerStatus{ + Name: container.Name, Ready: true, + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }, + ) + } + return false, nil, nil + }, + ) +} + +func TestRunContainerRecreatesTerminalPodAndKeepsVolume(t *testing.T) { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: testPodID, Namespace: testNamespace, UID: "pvc-uid"}, + } + c := newFakeCluster(t, pvc) + c.putPod(t, c.pod(t, podSpec{uid: "stopped", phase: corev1.PodSucceeded})) + readyOnCreate(c, "fresh") + + err := c.k.runContainer(context.Background(), testPodID, runOptionsForWorkspace("1m"), false) + + require.NoError(t, err) + pod, err := c.cs.CoreV1(). + Pods(testNamespace). + Get(context.Background(), testPodID, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, types.UID("fresh"), pod.UID, "a new pod replaced the terminal one") + assert.Equal(t, corev1.RestartPolicyNever, pod.Spec.RestartPolicy) + gotPVC, err := c.cs.CoreV1().PersistentVolumeClaims(testNamespace). + Get(context.Background(), testPodID, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, types.UID("pvc-uid"), gotPVC.UID) + for _, a := range c.cs.Actions() { + assert.False(t, a.Matches("delete", "persistentvolumeclaims")) + } +} + +func TestRunContainerRejectsExtraContainersBeforeTouchingTheCluster(t *testing.T) { + c := newFakeCluster(t) + template := t.TempDir() + "/pod.yaml" + require.NoError( + t, + writeFile(template, "spec:\n containers:\n - name: istio-proxy\n image: proxy\n"), + ) + c.k.options.PodManifestTemplate = template + + err := c.k.runContainer(context.Background(), testPodID, runOptionsForWorkspace("1m"), false) + + require.Error(t, err) + assert.Contains(t, err.Error(), "istio-proxy") + for _, a := range c.cs.Actions() { + assert.False(t, + a.Matches("create", podResourceName) || + a.Matches("delete", podResourceName) || + a.Matches("create", "secrets"), + "nothing is created or deleted: %v", a, + ) + } +} + +func TestRunContainerAllowsExtraContainersWhenTimeoutDisabled(t *testing.T) { + c := newFakeCluster(t) + template := t.TempDir() + "/pod.yaml" + require.NoError( + t, + writeFile(template, "spec:\n containers:\n - name: istio-proxy\n image: proxy\n"), + ) + c.k.options.PodManifestTemplate = template + readyOnCreate(c, "fresh") + + err := c.k.runContainer(context.Background(), testPodID, runOptionsForWorkspace(""), false) + + assert.NoError(t, err) +} + +func TestRunContainerRemovesPodWhoseAdmittedTopologyBreaksTheContract(t *testing.T) { + c := newFakeCluster(t) + c.cs.PrependReactor( + "create", + podResourceName, + func(a clienttesting.Action) (bool, runtime.Object, error) { + pod := a.(clienttesting.CreateAction).GetObject().(*corev1.Pod) + pod.UID = "admitted" + pod.Spec.Containers = append( + pod.Spec.Containers, + corev1.Container{Name: "webhook-injected"}, + ) + return false, nil, nil + }, + ) + + err := c.k.runContainer(context.Background(), testPodID, runOptionsForWorkspace("1m"), false) + + require.Error(t, err) + assert.Contains(t, err.Error(), "webhook-injected") + assert.False(t, c.podExists(t)) +} + +func TestDockerlessSuppressorsSurviveRepeatAndResume(t *testing.T) { + for _, disableDaemon := range []bool{false, true} { + name := "shutdown none" + if disableDaemon { + name = "current disable-daemon flag" + } + t.Run(name, func(t *testing.T) { + c := newFakeCluster(t) + c.k.agentConfig.ContainerTimeout = "1m" + c.k.disableDaemon = disableDaemon + options := topologyRunOptions(t, true) + action := config.ShutdownActionStopContainer + if !disableDaemon { + action = config.ShutdownActionNone + } + options.Env = daemonOptions(t, agentd.DaemonConfig{ + ShutdownAction: action, + }).Env + readyOnCreate(c, "initial") + ctx := context.Background() + + require.NoError(t, c.k.RunDevContainer(ctx, "ws", options)) + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + assertRunningTopology(t, c, options) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + require.NoError(t, c.k.StartDevContainer(ctx, "ws")) + assertRunningTopology(t, c, options) + + c.k.disableDaemon = false + c.k.SetLifecycleShutdownAction(config.ShutdownActionStopContainer) + c.cs.ClearActions() + err := c.k.RunDevContainer(ctx, "ws", nil) + require.ErrorContains(t, err, "dockerless") + assertOnlyReads(t, c) + }) + } +} + +func TestDockerlessRejectsNewPositivePolicyBeforeResumeMutation(t *testing.T) { + c := newFakeCluster(t) + options := topologyRunOptions(t, true) + readyOnCreate(c, "initial") + ctx := context.Background() + require.NoError(t, c.k.RunDevContainer(ctx, "ws", options)) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + c.k.agentConfig.ContainerTimeout = "1m" + c.cs.ClearActions() + + err := c.k.StartDevContainer(ctx, "ws") + + require.ErrorContains(t, err, "dockerless") + assertOnlyReads(t, c) +} + +func TestExternalImageMainStaysUnmanagedAcrossResume(t *testing.T) { + c := newFakeCluster(t) + options := runOptionsForWorkspace("") + options.ImageEntrypoint = true + options.Cmd = []string{"-c", `exec "$@"`, "-", "/app"} + options.Env = daemonOptions(t, agentd.DaemonConfig{ + ShutdownAction: config.ShutdownActionStopContainer, + }).Env + ctx := context.Background() + readyOnCreate(c, "external") + require.NoError(t, c.k.RunDevContainer(ctx, "ws", options)) + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + assertRunningTopology(t, c, options) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + require.NoError(t, c.k.StartDevContainer(ctx, "ws")) + assertRunningTopology(t, c, options) + c.k.agentConfig.ContainerTimeout = "1m" + c.cs.ClearActions() + + err := c.k.RunDevContainer(ctx, "ws", nil) + + require.ErrorContains(t, err, "overrideCommand") + assertOnlyReads(t, c) +} + +func TestLegacyDockerlessResumeDoesNotRequireGenerationMetadata(t *testing.T) { + options := topologyRunOptions(t, true) + options.Env = map[string]string{"DOCKERLESS": pkgconfig.BoolTrue} + pvc := workspacePVC(t) + pvc.Annotations["customer.example/annotation"] = preservedCustomerMetadata + raw, err := json.Marshal(&DevContainerInfo{WorkspaceID: "ws", Options: options}) + require.NoError(t, err) + pvc.Annotations[DevsyInfoAnnotation] = string(raw) + c := newFakeCluster(t, pvc) + c.putPod(t, c.pod(t, podSpec{uid: "legacy", phase: corev1.PodSucceeded})) + readyOnCreate(c, "resumed") + + require.NoError(t, c.k.RunDevContainer(context.Background(), "ws", nil)) + + assertRunningTopology(t, c, options) + kept, err := c.cs.CoreV1().PersistentVolumeClaims(testNamespace). + Get(context.Background(), testPodID, metav1.GetOptions{}) + require.NoError(t, err) + assertPreservedVolume(t, pvc, kept) +} + +func TestCurrentShutdownActionOverridesStoredNoneOnResume(t *testing.T) { + c := newFakeCluster(t) + c.k.agentConfig.ContainerTimeout = "1m" + options := runOptionsForWorkspace("") + options.Env = daemonOptions(t, agentd.DaemonConfig{ + ShutdownAction: config.ShutdownActionNone, + Lifecycle: agentd.LifecycleConfig{Owner: agentd.LifecycleOwnerManagedDaemon}, + }).Env + ctx := context.Background() + readyOnCreate(c, "disabled") + require.NoError(t, c.k.RunDevContainer(ctx, "ws", options)) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + c.k.SetLifecycleShutdownAction(config.ShutdownActionStopContainer) + + require.NoError(t, c.k.StartDevContainer(ctx, "ws")) + + secret, err := c.cs.CoreV1().Secrets(testNamespace). + Get(ctx, getDaemonSecretName(testPodID), metav1.GetOptions{}) + require.NoError(t, err) + cfg, err := agentd.DecodeDaemonConfig(string(secret.Data[DaemonConfigKey])) + require.NoError(t, err) + timeout, err := lifecycleTimeout(daemonOptions(t, *cfg)) + require.NoError(t, err) + assert.Equal(t, time.Minute, timeout) + assert.Equal(t, config.ShutdownActionStopContainer, cfg.ShutdownAction) +} + +func topologyRunOptions(t *testing.T, dockerless bool) *driver.RunOptions { + t.Helper() + options := runOptionsForWorkspace("") + options.Dockerless = dockerless + if dockerless { + options.Image = "example/dockerless:1" + options.Entrypoint = "/.dockerless/dockerless" + options.Cmd = []string{"start", "--wait"} + options.Env = daemonOptions(t, agentd.DaemonConfig{ + ShutdownAction: config.ShutdownActionStopContainer, + }).Env + } + return options +} + +func assertRunningTopology(t *testing.T, c *fakeCluster, options *driver.RunOptions) { + t.Helper() + ctx := context.Background() + details, err := c.k.FindDevContainer(ctx, "ws") + require.NoError(t, err) + assert.Equal(t, options.Dockerless, details.Dockerless) + assert.Equal(t, options.ImageEntrypoint, details.ImageEntrypoint) + pod, err := c.cs.CoreV1().Pods(testNamespace).Get(ctx, testPodID, metav1.GetOptions{}) + require.NoError(t, err) + var main *corev1.Container + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == DevContainerName { + main = &pod.Spec.Containers[i] + } + } + require.NotNil(t, main) + assert.Equal(t, options.Image, main.Image) + assert.Equal(t, []string{options.Entrypoint}, main.Command) + assert.Equal(t, options.Cmd, main.Args) + secret, err := c.cs.CoreV1().Secrets(testNamespace). + Get(ctx, getDaemonSecretName(testPodID), metav1.GetOptions{}) + require.NoError(t, err) + cfg, err := agentd.DecodeDaemonConfig(string(secret.Data[DaemonConfigKey])) + require.NoError(t, err) + assert.Equal(t, !options.Dockerless && !options.ImageEntrypoint, cfg.Lifecycle.Managed()) + assert.Empty(t, cfg.Timeout) +} + +func assertOnlyReads(t *testing.T, c *fakeCluster) { + t.Helper() + for _, action := range c.cs.Actions() { + assert.Equal(t, "get", action.GetVerb(), "policy rejection must not mutate cluster state") + } +} + +func writeFile(path, content string) error { + return os.WriteFile(path, []byte(content), 0o600) +} diff --git a/pkg/driver/kubernetes/pod_topology.go b/pkg/driver/kubernetes/pod_topology.go new file mode 100644 index 0000000000..e69f244d1f --- /dev/null +++ b/pkg/driver/kubernetes/pod_topology.go @@ -0,0 +1,181 @@ +package kubernetes + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "slices" + "strconv" + + "github.com/devsy-org/devsy/pkg/driver" + corev1 "k8s.io/api/core/v1" +) + +type podTopology struct { + dockerless bool + imageMain bool +} + +func decodePodTopology(pod *corev1.Pod) (podTopology, bool, error) { + rawDockerless, hasDockerless := pod.Annotations[DevsyDockerlessAnnotation] + rawImageMain, hasImageMain := pod.Annotations[DevsyImageEntrypointAnnotation] + if !hasDockerless && !hasImageMain { + return podTopology{}, false, nil + } + dockerless, err := strconv.ParseBool(rawDockerless) + if err != nil { + return podTopology{}, false, fmt.Errorf( + "pod %q has invalid Dockerless topology metadata", + pod.Name, + ) + } + imageMain, err := strconv.ParseBool(rawImageMain) + if err != nil { + return podTopology{}, false, fmt.Errorf( + "pod %q has invalid image-main topology metadata", + pod.Name, + ) + } + return podTopology{dockerless: dockerless, imageMain: imageMain}, true, nil +} + +func podLaunchDiffers(pod *corev1.Pod, options *driver.RunOptions) bool { + topology, known, err := decodePodTopology(pod) + if err == nil && known && + (topology.dockerless != options.Dockerless || topology.imageMain != options.ImageEntrypoint) { + return true + } + return podCommandDiffers(pod, options) +} + +func podCommandDiffers(pod *corev1.Pod, options *driver.RunOptions) bool { + var command []string + if options.Entrypoint != "" { + command = []string{options.Entrypoint} + } + for _, container := range pod.Spec.Containers { + if container.Name == DevContainerName { + return container.Image != options.Image || + !slices.Equal( + container.Command, + command, + ) || !slices.Equal(container.Args, options.Cmd) + } + } + return true +} + +func observedPodTopology(pod *corev1.Pod, options *driver.RunOptions) (bool, bool, error) { + if pod == nil { + return options.Dockerless, options.ImageEntrypoint, validateImageMainRecipe(options) + } + if err := verifyOwnedPod(pod, pod.Name, options.UID); err != nil { + return false, false, err + } + topology, known, err := decodePodTopology(pod) + if err != nil { + return false, false, err + } + if known { + return topology.dockerless, topology.imageMain, nil + } + if podCommandDiffers(pod, options) { + return false, false, fmt.Errorf( + "legacy pod %q does not match the saved launch recipe; run devsy workspace up --recreate", + pod.Name, + ) + } + return options.Dockerless, options.ImageEntrypoint, validateImageMainRecipe(options) +} + +func validateImageMainRecipe(options *driver.RunOptions) error { + // The legacy producer framed the script with three shell arguments, then + // appended the image main. Missing facts on that specific recipe are ambiguous. + if !options.Dockerless && !options.ImageEntrypoint && options.Entrypoint == "/bin/sh" && + len(options.Cmd) > 3 && options.Cmd[0] == "-c" && options.Cmd[2] == "-" { + return fmt.Errorf( + "saved shell recipe carries an external command without its image-main fact; run devsy workspace up --recreate", + ) + } + return nil +} + +// sameRunOptions compares the persisted JSON form, in which a nil and an empty +// collection are the same value. +func sameRunOptions(a, b *driver.RunOptions) bool { + if a == nil || b == nil { + return a == b + } + left, leftErr := json.Marshal(a) + right, rightErr := json.Marshal(b) + return leftErr == nil && rightErr == nil && bytes.Equal(left, right) +} + +func (k *KubernetesDriver) verifyDesiredPod( + ctx context.Context, + id string, + options *driver.RunOptions, +) error { + pvc, info, err := k.getDevContainerPvc(ctx, id) + if err != nil { + return err + } + if pvc == nil { + return fmt.Errorf("workspace volume %q disappeared during startup", id) + } + if err := verifyVolumeOwner(pvc, info, options.UID); err != nil { + return err + } + if !sameRunOptions(info.Options, options) { + return fmt.Errorf( + "workspace %q desired run configuration changed during startup; retry the operation", + id, + ) + } + pod, err := k.getPod(ctx, id) + if err != nil { + return err + } + if pod == nil { + return fmt.Errorf( + "pod for workspace %q is gone right after it started; it may have been deleted "+ + "or already stopped for inactivity; retry the operation", + id, + ) + } + return verifyDesiredTopology(pod, options) +} + +func verifyDesiredTopology(pod *corev1.Pod, options *driver.RunOptions) error { + dockerless, imageMain, err := observedPodTopology(pod, options) + if err != nil { + return err + } + if dockerless != options.Dockerless || imageMain != options.ImageEntrypoint || + podCommandDiffers(pod, options) { + return fmt.Errorf( + "pod %q no longer matches the desired launch topology; run devsy workspace up --recreate", + pod.Name, + ) + } + return nil +} + +func (k *KubernetesDriver) verifyExistingPodOwner( + ctx context.Context, + id, workspaceUID string, +) error { + pod, err := k.getPod(ctx, id) + if err != nil { + return err + } + if pod == nil { + return nil + } + if err := verifyOwnedPod(pod, id, workspaceUID); err != nil { + return err + } + _, _, err = decodePodTopology(pod) + return err +} diff --git a/pkg/driver/kubernetes/pvc.go b/pkg/driver/kubernetes/pvc.go index 43c11718fb..b1122285a2 100644 --- a/pkg/driver/kubernetes/pvc.go +++ b/pkg/driver/kubernetes/pvc.go @@ -10,6 +10,7 @@ import ( "github.com/devsy-org/devsy/pkg/driver" "github.com/devsy-org/devsy/pkg/log" corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -124,3 +125,79 @@ func (k *KubernetesDriver) getDevContainerInformation( return string(containerInfo), nil } + +func (k *KubernetesDriver) resolveRunOptions( + saved *DevContainerInfo, + options *driver.RunOptions, +) (*driver.RunOptions, error) { + if options == nil && saved != nil { + options = saved.Options + } + if options == nil { + return nil, fmt.Errorf("no options provided and no saved run configuration available") + } + resolved, err := k.refreshStoredLifecycle(options) + if err != nil { + return nil, err + } + if err := validateImageMainRecipe(resolved); err != nil { + return nil, err + } + if err := k.preflightTemplate(resolved); err != nil { + return nil, err + } + return resolved, nil +} + +// A single resource-version-guarded update publishes intent before pod/config changes. +func (k *KubernetesDriver) persistRunOptions( + ctx context.Context, + pvc *corev1.PersistentVolumeClaim, + saved *DevContainerInfo, + options *driver.RunOptions, +) error { + if err := verifyVolumeOwner(pvc, saved, options.UID); err != nil { + return err + } + if sameRunOptions(saved.Options, options) { + return nil + } + next := *saved + next.Options = options + raw, err := json.Marshal(&next) + if err != nil { + return err + } + updated := pvc.DeepCopy() + updated.Annotations[DevsyInfoAnnotation] = string(raw) + _, err = k.client.Client().CoreV1().PersistentVolumeClaims(k.namespace). + Update(ctx, updated, metav1.UpdateOptions{}) + if kerrors.IsForbidden(err) { + return fmt.Errorf( + "persist workspace run configuration: Kubernetes permission update on "+ + "persistentvolumeclaims in namespace %q is required for changed run configuration: %w", + k.namespace, err, + ) + } + if err != nil { + return fmt.Errorf( + "persist workspace run configuration; no pod/config changes were made: %w", + err, + ) + } + return nil +} + +func verifyVolumeOwner( + pvc *corev1.PersistentVolumeClaim, + info *DevContainerInfo, + workspaceUID string, +) error { + if uid := pvc.Labels[DevsyWorkspaceUIDLabel]; uid != "" && uid != workspaceUID { + return fmt.Errorf("workspace volume %q belongs to a different workspace", pvc.Name) + } + if info.Options != nil && info.Options.UID != "" && info.Options.UID != workspaceUID { + return fmt.Errorf("workspace volume %q has run options for a different workspace", pvc.Name) + } + return nil +} diff --git a/pkg/driver/kubernetes/pvc_metadata_test.go b/pkg/driver/kubernetes/pvc_metadata_test.go new file mode 100644 index 0000000000..8868a3d591 --- /dev/null +++ b/pkg/driver/kubernetes/pvc_metadata_test.go @@ -0,0 +1,536 @@ +package kubernetes + +import ( + "context" + "encoding/json" + "errors" + "testing" + + pkgconfig "github.com/devsy-org/devsy/pkg/config" + agentd "github.com/devsy-org/devsy/pkg/daemon/agent" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/driver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + clienttesting "k8s.io/client-go/testing" +) + +const ( + parentMountRecreateOperation = "recreate" + parentMountReuseOperation = "reuse" +) + +var pvcResource = schema.GroupVersionResource{Version: "v1", Resource: "persistentvolumeclaims"} + +func canonicalTopologyOptions(t *testing.T, dockerless bool) *driver.RunOptions { + t.Helper() + options := topologyRunOptions(t, dockerless) + if !dockerless { + options.Env = daemonOptions(t, agentd.DaemonConfig{ + ShutdownAction: config.ShutdownActionStopContainer, + Lifecycle: agentd.LifecycleConfig{ + Owner: agentd.LifecycleOwnerManagedDaemon, SetupArmsTimer: true, + }, + }).Env + } + return options +} + +func savedRunVolume(t *testing.T, options *driver.RunOptions) *corev1.PersistentVolumeClaim { + t.Helper() + pvc := workspacePVC(t) + pvc.UID, pvc.ResourceVersion = "workspace-volume", "4" + pvc.Labels = map[string]string{ + DevsyWorkspaceUIDLabel: testWorkspace, + "customer.example/label": preservedCustomerMetadata, + } + pvc.Annotations["customer.example/annotation"] = preservedCustomerMetadata + pvc.Spec.VolumeName = "customer-volume" + pvc.Finalizers = []string{"kubernetes.io/pvc-protection"} + raw, err := json.Marshal(&DevContainerInfo{WorkspaceID: "ws", Options: options}) + require.NoError(t, err) + pvc.Annotations[DevsyInfoAnnotation] = string(raw) + return pvc +} + +func assertPreservedVolume(t *testing.T, original, current *corev1.PersistentVolumeClaim) { + t.Helper() + assert.Equal(t, original.UID, current.UID) + assert.Equal(t, original.Labels, current.Labels) + assert.Equal(t, original.Spec, current.Spec) + assert.Equal(t, original.Finalizers, current.Finalizers) + assert.Equal( + t, + original.Annotations["customer.example/annotation"], + current.Annotations["customer.example/annotation"], + ) +} + +func TestRecreatedTopologyControlsRepeatAndResume(t *testing.T) { + for _, dockerless := range []bool{false, true} { + name := "ordinary to dockerless" + if !dockerless { + name = "dockerless to ordinary" + } + t.Run(name, func(t *testing.T) { + initial := canonicalTopologyOptions(t, !dockerless) + volume := savedRunVolume(t, initial) + c := newFakeCluster(t, volume) + ctx := context.Background() + readyOnCreate(c, "initial") + require.NoError(t, c.k.RunDevContainer(ctx, "ws", initial)) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + next := canonicalTopologyOptions(t, dockerless) + require.NoError(t, c.k.RunDevContainer(ctx, "ws", next)) + assertRunningTopology(t, c, next) + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + assertRunningTopology(t, c, next) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + require.NoError(t, c.k.StartDevContainer(ctx, "ws")) + assertRunningTopology(t, c, next) + current, info, err := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, err) + assertPreservedVolume(t, volume, current) + assert.Equal(t, next, info.Options) + }) + } +} + +func TestUnchangedRunOptionsDoNotRequireVolumeUpdate(t *testing.T) { + options := canonicalTopologyOptions(t, false) + c := newFakeCluster(t, savedRunVolume(t, options)) + c.cs.PrependReactor( + "update", + "persistentvolumeclaims", + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, kerrors.NewForbidden( + pvcResource.GroupResource(), + testPodID, + errors.New("update denied"), + ) + }, + ) + readyOnCreate(c, "stable") + ctx := context.Background() + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + first, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + reused, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + assert.Equal(t, first.UID, reused.UID) + require.NoError(t, c.k.StopDevContainer(ctx, "ws")) + require.NoError(t, c.k.StartDevContainer(ctx, "ws")) + resumed, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + assert.NotEqual(t, first.UID, resumed.UID) + assertRunningTopology(t, c, options) +} + +func TestVolumePublicationErrorsLeavePreviousRecordAndNoNewPod(t *testing.T) { + for _, cause := range []error{ + kerrors.NewForbidden(pvcResource.GroupResource(), testPodID, errors.New("update denied")), + kerrors.NewConflict(pvcResource.GroupResource(), testPodID, errors.New("concurrent owner")), + } { + t.Run(cause.Error(), func(t *testing.T) { + old := savedRunVolume(t, canonicalTopologyOptions(t, false)) + c := newFakeCluster(t, old) + c.cs.PrependReactor( + "update", + "persistentvolumeclaims", + func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, cause + }, + ) + err := c.k.RunDevContainer( + context.Background(), + "ws", + canonicalTopologyOptions(t, true), + ) + require.ErrorIs(t, err, cause) + if kerrors.IsForbidden(cause) { + assert.ErrorContains(t, err, "permission update") + } + current, _, getErr := c.k.getDevContainerPvc(context.Background(), testPodID) + require.NoError(t, getErr) + assert.Equal(t, old, current) + for _, action := range c.cs.Actions() { + if action.GetVerb() != "get" { + assert.True( + t, + action.Matches("update", "persistentvolumeclaims"), + "no pod/config changes after failed publication: %v", + action, + ) + } + } + }) + } +} + +func TestFailedPodCreationRetainsNewDesiredOptionsForRetry(t *testing.T) { + old := savedRunVolume(t, canonicalTopologyOptions(t, false)) + c := newFakeCluster(t, old) + next := canonicalTopologyOptions(t, true) + readyOnCreate(c, "retried") + reject := true + c.cs.PrependReactor( + "create", + podResourceName, + func(clienttesting.Action) (bool, runtime.Object, error) { + if reject { + reject = false + return true, nil, errors.New("pod admission unavailable") + } + return false, nil, nil + }, + ) + ctx := context.Background() + require.ErrorContains(t, c.k.RunDevContainer(ctx, "ws", next), "pod admission unavailable") + current, info, err := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, err) + assertPreservedVolume(t, old, current) + assert.Equal(t, next, info.Options) + assert.False(t, c.podExists(t)) + c.cs.ClearActions() + + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + + assertRunningTopology(t, c, next) + for _, action := range c.cs.Actions() { + assert.False( + t, + action.Matches("update", "persistentvolumeclaims"), + "retry must use the published desired state", + ) + } +} + +func TestStaleVolumeSnapshotCannotOverwriteReplacedVolume(t *testing.T) { + old := savedRunVolume(t, canonicalTopologyOptions(t, false)) + c := newFakeCluster(t, old) + stale, info, err := c.k.getDevContainerPvc(context.Background(), testPodID) + require.NoError(t, err) + replacement := old.DeepCopy() + replacement.UID, replacement.ResourceVersion = "replacement-volume", "5" + require.NoError(t, c.cs.Tracker().Update(pvcResource, replacement, testNamespace)) + c.cs.PrependReactor( + "update", + "persistentvolumeclaims", + func(a clienttesting.Action) (bool, runtime.Object, error) { + requested := a.(clienttesting.UpdateAction).GetObject().(*corev1.PersistentVolumeClaim) + if requested.UID != replacement.UID || + requested.ResourceVersion != replacement.ResourceVersion { + return true, nil, kerrors.NewConflict( + pvcResource.GroupResource(), + testPodID, + errors.New("volume changed"), + ) + } + return false, nil, nil + }, + ) + + err = c.k.persistRunOptions( + context.Background(), + stale, + info, + canonicalTopologyOptions(t, true), + ) + + require.True(t, kerrors.IsConflict(err)) + current, _, err := c.k.getDevContainerPvc(context.Background(), testPodID) + require.NoError(t, err) + assert.Equal(t, replacement, current) +} + +func TestForeignWorkspaceVolumeIsNotUpdated(t *testing.T) { + old := savedRunVolume(t, canonicalTopologyOptions(t, false)) + old.Labels[DevsyWorkspaceUIDLabel] = "another-workspace" + c := newFakeCluster(t, old) + + err := c.k.RunDevContainer(context.Background(), "ws", canonicalTopologyOptions(t, true)) + + require.ErrorContains(t, err, "different workspace") + assertOnlyReads(t, c) +} + +func TestLivePodTopologyOverridesPendingDesiredRecord(t *testing.T) { + old := canonicalTopologyOptions(t, false) + c := newFakeCluster(t, savedRunVolume(t, old)) + ctx := context.Background() + readyOnCreate(c, "running") + require.NoError(t, c.k.RunDevContainer(ctx, "ws", old)) + volume, info, err := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, err) + next := canonicalTopologyOptions(t, true) + require.NoError(t, c.k.persistRunOptions(ctx, volume, info, next)) + + details, err := c.k.FindDevContainer(ctx, "ws") + + require.NoError(t, err) + assert.False(t, details.Dockerless, "the live pod is still the ordinary generation") + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + assertRunningTopology(t, c, next) +} + +func TestForeignPodPreventsVolumeAndConfigMutation(t *testing.T) { + c := newFakeCluster(t, savedRunVolume(t, canonicalTopologyOptions(t, false))) + c.putPod(t, c.pod(t, podSpec{uid: "foreign", phase: corev1.PodRunning, foreign: true})) + c.cs.ClearActions() + + err := c.k.RunDevContainer(context.Background(), "ws", canonicalTopologyOptions(t, true)) + + require.Error(t, err) + assertOnlyReads(t, c) +} + +func TestLegacyRunningDockerlessPodIsReusedWithoutNewAnnotations(t *testing.T) { + options := topologyRunOptions(t, true) + options.Env = map[string]string{"DOCKERLESS": pkgconfig.BoolTrue} + c := newFakeCluster(t, savedRunVolume(t, options)) + pod := c.pod(t, podSpec{uid: "legacy-running", phase: corev1.PodRunning}) + pod.Spec.Containers[0].Image = options.Image + pod.Spec.Containers[0].Command = []string{options.Entrypoint} + pod.Spec.Containers[0].Args = options.Cmd + c.putPod(t, pod) + ctx := context.Background() + + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + + current, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + assert.Equal(t, pod.UID, current.UID) + assertRunningTopology(t, c, options) +} + +func TestMalformedLiveTopologyFailsClosedWithoutDeletingWork(t *testing.T) { + options := canonicalTopologyOptions(t, false) + c := newFakeCluster(t, savedRunVolume(t, options)) + readyOnCreate(c, "protected") + ctx := context.Background() + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + pod, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + pod.Annotations[DevsyDockerlessAnnotation] = "not-a-boolean" + require.NoError(t, c.cs.Tracker().Update( + schema.GroupVersionResource{Version: "v1", Resource: podResourceName}, pod, testNamespace, + )) + + err = c.k.RunDevContainer(ctx, "ws", nil) + + require.Error(t, err) + current, getErr := c.k.getPod(ctx, testPodID) + require.NoError(t, getErr) + assert.Equal(t, pod.UID, current.UID) +} + +func TestPositiveUnsupportedTopologyDoesNotPublishDesiredState(t *testing.T) { + for _, dockerless := range []bool{false, true} { + name := "external image main" + if dockerless { + name = "dockerless" + } + t.Run(name, func(t *testing.T) { + volume := savedRunVolume(t, canonicalTopologyOptions(t, false)) + c := newFakeCluster(t, volume) + c.k.agentConfig.ContainerTimeout = "1m" + next := canonicalTopologyOptions(t, dockerless) + if !dockerless { + next.ImageEntrypoint = true + next.Cmd = []string{"-c", `exec "$@"`, "-", "/app"} + } + + err := c.k.RunDevContainer(context.Background(), "ws", next) + + require.Error(t, err) + assertOnlyReads(t, c) + current, _, getErr := c.k.getDevContainerPvc(context.Background(), testPodID) + require.NoError(t, getErr) + assert.Equal(t, volume, current) + }) + } +} + +func TestConcurrentUserMetadataConflictIsNotOverwritten(t *testing.T) { + volume := savedRunVolume(t, canonicalTopologyOptions(t, false)) + c := newFakeCluster(t, volume) + concurrent := volume.DeepCopy() + concurrent.ResourceVersion = "5" + concurrent.Labels["customer.example/new"] = "concurrent" + c.cs.PrependReactor( + "update", + "persistentvolumeclaims", + func(clienttesting.Action) (bool, runtime.Object, error) { + err := c.cs.Tracker().Update(pvcResource, concurrent, testNamespace) + if err != nil { + return true, nil, err + } + return true, nil, kerrors.NewConflict( + pvcResource.GroupResource(), + testPodID, + errors.New("concurrent metadata"), + ) + }, + ) + + err := c.k.RunDevContainer(context.Background(), "ws", canonicalTopologyOptions(t, true)) + + require.True(t, kerrors.IsConflict(err)) + current, _, err := c.k.getDevContainerPvc(context.Background(), testPodID) + require.NoError(t, err) + assert.Equal(t, concurrent, current) + assert.False(t, c.podExists(t)) +} + +func TestParentWorkspaceVolumeMountLifecycle(t *testing.T) { + for _, operation := range []string{"initial", parentMountReuseOperation, parentMountRecreateOperation, "resume"} { + t.Run(operation, func(t *testing.T) { + options := canonicalTopologyOptions(t, false) + c, volume := newParentMountLifecycleCluster(t, operation, options) + ctx := context.Background() + var err error + switch operation { + case "resume": + err = c.k.StartDevContainer(ctx, "ws") + case parentMountReuseOperation: + err = c.k.RunDevContainer(ctx, "ws", nil) + default: + err = c.k.RunDevContainer(ctx, "ws", options) + } + require.NoError(t, err) + assertParentMountedWorkspace(t, c, options.Image) + pod, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + if operation == parentMountReuseOperation { + assert.Equal(t, types.UID("existing"), pod.UID) + } else { + assert.Equal(t, types.UID("replacement"), pod.UID) + } + current, _, err := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, err) + if volume != nil { + assertPreservedVolume(t, volume, current) + } + }) + } +} + +func newParentMountLifecycleCluster( + t *testing.T, + operation string, + options *driver.RunOptions, +) (*fakeCluster, *corev1.PersistentVolumeClaim) { + t.Helper() + var volume *corev1.PersistentVolumeClaim + var c *fakeCluster + if operation == "initial" { + c = newFakeCluster(t) + } else { + volume = savedRunVolume(t, options) + c = newFakeCluster(t, volume) + } + c.k.options.WorkspaceVolumeMount = "/workspaces" + if operation == parentMountReuseOperation || operation == parentMountRecreateOperation { + pod := c.pod(t, podSpec{uid: "existing", phase: corev1.PodRunning}) + pod.Spec.Containers[0].Image = options.Image + pod.Spec.Containers[0].Command = []string{options.Entrypoint} + pod.Spec.Containers[0].Args = options.Cmd + pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ + {Name: DevContainerName, MountPath: "/workspaces", SubPath: "devsy/0"}, + } + c.putPod(t, pod) + } + readyOnCreate(c, "replacement") + if operation == parentMountRecreateOperation { + require.NoError(t, c.k.StopDevContainer(context.Background(), "ws")) + options.Image = "example/recreated:2" + } + return c, volume +} + +func assertParentMountedWorkspace(t *testing.T, c *fakeCluster, image string) { + t.Helper() + ctx := context.Background() + pod, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + var mounted *corev1.VolumeMount + for i := range pod.Spec.Containers { + container := &pod.Spec.Containers[i] + if container.Name != DevContainerName { + continue + } + assert.Equal(t, image, container.Image) + for j := range container.VolumeMounts { + if container.VolumeMounts[j].Name == DevContainerName { + mounted = &container.VolumeMounts[j] + } + } + } + require.NotNil(t, mounted) + assert.Equal(t, "/workspaces", mounted.MountPath) + assert.Equal(t, "devsy/0", mounted.SubPath) + _, info, err := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, err) + assert.Equal(t, "/workspaces/app", info.Options.WorkspaceMount.Target) + assert.Equal(t, image, info.Options.Image) +} + +func TestMalformedLiveTopologyRejectsProviderAndCommandDriftBeforeMutation(t *testing.T) { + for _, drift := range []string{"provider options", "command", "partial topology"} { + t.Run(drift, func(t *testing.T) { + options := canonicalTopologyOptions(t, false) + c := newFakeCluster(t, savedRunVolume(t, options)) + readyOnCreate(c, "protected") + ctx := context.Background() + require.NoError(t, c.k.RunDevContainer(ctx, "ws", nil)) + pod, err := c.k.getPod(ctx, testPodID) + require.NoError(t, err) + pod.Annotations[DevsyDockerlessAnnotation] = "not-a-boolean" + if drift == "partial topology" { + pod.Annotations[DevsyDockerlessAnnotation] = "false" + delete(pod.Annotations, DevsyImageEntrypointAnnotation) + } + require.NoError(t, c.cs.Tracker().Update( + schema.GroupVersionResource{ + Version: "v1", + Resource: podResourceName, + }, pod, testNamespace, + ), + ) + beforeVolume, _, err := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, err) + beforeSecret, err := c.cs.CoreV1().Secrets(testNamespace). + Get(ctx, getDaemonSecretName(testPodID), metav1.GetOptions{}) + require.NoError(t, err) + next := canonicalTopologyOptions(t, false) + if drift == "command" { + next.Cmd = []string{"-c", "exec sleep 3600"} + } else { + c.k.options.DiskSize = "2Gi" + } + c.cs.ClearActions() + + err = c.k.RunDevContainer(ctx, "ws", next) + + require.Error(t, err) + assertOnlyReads(t, c) + currentPod, getErr := c.k.getPod(ctx, testPodID) + require.NoError(t, getErr) + assert.Equal(t, pod, currentPod) + currentVolume, _, getErr := c.k.getDevContainerPvc(ctx, testPodID) + require.NoError(t, getErr) + assert.Equal(t, beforeVolume, currentVolume) + currentSecret, getErr := c.cs.CoreV1().Secrets(testNamespace). + Get(ctx, getDaemonSecretName(testPodID), metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Equal(t, beforeSecret, currentSecret) + }) + } +} diff --git a/pkg/driver/kubernetes/pvc_test.go b/pkg/driver/kubernetes/pvc_test.go index fbd1d20ab1..1ab9851bd0 100644 --- a/pkg/driver/kubernetes/pvc_test.go +++ b/pkg/driver/kubernetes/pvc_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/http/httptest" + "strconv" "strings" "sync" "testing" @@ -18,6 +19,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -100,6 +102,10 @@ func (m *mockPvcServer) dispatch( m.handlePost(w, r, encoder, decoder) return true } + if r.Method == http.MethodPut && len(parts) == 6 { + m.handlePut(w, r, encoder, decoder) + return true + } return false } @@ -150,6 +156,8 @@ func (m *mockPvcServer) handlePost( w.WriteHeader(http.StatusBadRequest) return } + pvc.UID = types.UID("pvc-" + pvc.Name) + pvc.ResourceVersion = "1" m.mu.Lock() m.pvcs[pvc.Name] = pvc m.mu.Unlock() @@ -158,6 +166,51 @@ func (m *mockPvcServer) handlePost( _ = encoder.Encode(pvc, w) } +func (m *mockPvcServer) handlePut( + w http.ResponseWriter, + r *http.Request, + encoder runtime.Encoder, + decoder runtime.Decoder, +) { + body, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + obj, _, err := decoder.Decode(body, nil, nil) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + pvc, ok := obj.(*corev1.PersistentVolumeClaim) + if !ok { + w.WriteHeader(http.StatusBadRequest) + return + } + m.mu.Lock() + defer m.mu.Unlock() + current := m.pvcs[pvc.Name] + w.Header().Set("Content-Type", runtime.ContentTypeJSON) + if current == nil || current.UID != pvc.UID || current.ResourceVersion != pvc.ResourceVersion { + w.WriteHeader(http.StatusConflict) + _ = encoder.Encode(&metav1.Status{ + TypeMeta: metav1.TypeMeta{Kind: "Status", APIVersion: "v1"}, + Status: metav1.StatusFailure, Reason: metav1.StatusReasonConflict, + Code: http.StatusConflict, + }, w) + return + } + version, err := strconv.Atoi(current.ResourceVersion) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + pvc.ResourceVersion = strconv.Itoa(version + 1) + m.pvcs[pvc.Name] = pvc + w.WriteHeader(http.StatusOK) + _ = encoder.Encode(pvc, w) +} + func newMockPvcDriver( t *testing.T, initialPvcs map[string]*corev1.PersistentVolumeClaim, @@ -167,7 +220,10 @@ func newMockPvcDriver( pvcs: make(map[string]*corev1.PersistentVolumeClaim), } for k, v := range initialPvcs { - mock.pvcs[k] = v.DeepCopy() + pvc := v.DeepCopy() + pvc.UID = types.UID("pvc-" + pvc.Name) + pvc.ResourceVersion = "1" + mock.pvcs[k] = pvc } srv := httptest.NewServer(mock) t.Cleanup(srv.Close) @@ -219,9 +275,6 @@ func testEnsureMissingPvcWithoutOptions(t *testing.T) { if opts != nil { t.Errorf("opts = %v, want nil", opts) } - if !strings.Contains(err.Error(), "no options provided") { - t.Errorf("unexpected error message: %v", err) - } } func testEnsureMissingPvcWithOptions(t *testing.T) { @@ -230,16 +283,13 @@ func testEnsureMissingPvcWithOptions(t *testing.T) { UID: testWorkspaceID, Image: "golang:1.24", } - initialize, opts, err := k.ensureDevContainerPvc(context.Background(), testWorkspaceID, options) + initialize, _, err := k.ensureDevContainerPvc(context.Background(), testWorkspaceID, options) if err != nil { t.Fatalf("ensureDevContainerPvc: %v", err) } if !initialize { t.Errorf("initialize = false, want true") } - if opts != options { - t.Errorf("opts = %v, want %v", opts, options) - } mock.mu.Lock() created, exists := mock.pvcs[testWorkspaceID] mock.mu.Unlock() @@ -273,18 +323,6 @@ func runExistingPvcTest( return opts, err } -func testEnsureExistingPvcWithOptions(t *testing.T) { - savedOptions := &driver.RunOptions{UID: testWorkspaceID, Image: testSavedImage} - callerOptions := &driver.RunOptions{UID: testWorkspaceID, Image: "caller-image"} - opts, err := runExistingPvcTest(t, savedOptions, callerOptions) - if err != nil { - t.Fatalf("ensureDevContainerPvc: %v", err) - } - if opts != callerOptions { - t.Errorf("opts = %v, want caller options %v", opts, callerOptions) - } -} - func testEnsureExistingPvcWithoutCallerOptions(t *testing.T) { savedOptions := &driver.RunOptions{UID: testWorkspaceID, Image: testSavedImage} opts, err := runExistingPvcTest(t, savedOptions, nil) @@ -296,29 +334,11 @@ func testEnsureExistingPvcWithoutCallerOptions(t *testing.T) { } } -func testEnsureExistingPvcWithNilSavedOptions(t *testing.T) { - opts, err := runExistingPvcTest(t, nil, nil) - if err != nil { - t.Fatalf("ensureDevContainerPvc: %v", err) - } - if opts != nil { - t.Errorf("opts = %v, want nil", opts) - } -} - func TestEnsureDevContainerPvc(t *testing.T) { t.Run("missing PVC without options errors", testEnsureMissingPvcWithoutOptions) t.Run("missing PVC with options creates PVC and initializes", testEnsureMissingPvcWithOptions) - t.Run( - "existing PVC with options retains options and does not initialize", - testEnsureExistingPvcWithOptions, - ) t.Run( "existing PVC without caller options reuses saved options", testEnsureExistingPvcWithoutCallerOptions, ) - t.Run( - "existing PVC with nil saved options preserves nil options", - testEnsureExistingPvcWithNilSavedOptions, - ) } diff --git a/pkg/driver/kubernetes/run.go b/pkg/driver/kubernetes/run.go index b1fcbdb188..6833d17f8c 100644 --- a/pkg/driver/kubernetes/run.go +++ b/pkg/driver/kubernetes/run.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strconv" "strings" + "time" pkgconfig "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/devcontainer/config" @@ -24,11 +25,13 @@ const ( DevContainerName = pkgconfig.BinaryName InitContainerName = pkgconfig.BinaryName + "-init" - DevsyCreatedLabel = pkgconfig.K8sCreatedLabel - DevsyWorkspaceLabel = pkgconfig.K8sWorkspaceLabel - DevsyWorkspaceUIDLabel = pkgconfig.K8sWorkspaceUIDLabel - DevsyInfoAnnotation = pkgconfig.K8sInfoAnnotation - DevsyLastAppliedAnnotation = pkgconfig.K8sLastAppliedAnnotation + DevsyCreatedLabel = pkgconfig.K8sCreatedLabel + DevsyWorkspaceLabel = pkgconfig.K8sWorkspaceLabel + DevsyWorkspaceUIDLabel = pkgconfig.K8sWorkspaceUIDLabel + DevsyInfoAnnotation = pkgconfig.K8sInfoAnnotation + DevsyLastAppliedAnnotation = pkgconfig.K8sLastAppliedAnnotation + DevsyDockerlessAnnotation = DevsyInfoAnnotation + "-dockerless" + DevsyImageEntrypointAnnotation = DevsyInfoAnnotation + "-image-entrypoint" ) var ExtraDevsyLabels = map[string]string{ @@ -48,16 +51,39 @@ func (k *KubernetesDriver) RunDevContainer( log.Debugf("Running devcontainer for workspace %q", workspaceId) workspaceId = getID(workspaceId) - if err := k.ensureNamespace(ctx); err != nil { + initialize, options, err := k.ensureDevContainerPvc(ctx, workspaceId, options) + if err != nil { return err } - initialize, options, err := k.ensureDevContainerPvc(ctx, workspaceId, options) - if err != nil { + if err := k.runContainer(ctx, workspaceId, options, initialize); err != nil { return err } + return k.verifyDesiredPod(ctx, workspaceId, options) +} + +func (k *KubernetesDriver) ValidateRunImageDevContainer( + params *driver.RunImageDevContainerParams, +) error { + _, err := k.resolveRunOptions(nil, params.Options) + return err +} - return k.runContainer(ctx, workspaceId, options, initialize) +// preflightTemplate rejects an unusable pod template before any cluster object, +// including the workspace volume, is created. +func (k *KubernetesDriver) preflightTemplate(options *driver.RunOptions) error { + if options == nil { + return nil + } + pod, err := loadPodTemplate(k.options.PodManifestTemplate) + if err != nil { + return err + } + timeout, err := lifecycleTimeout(options) + if err != nil { + return fmt.Errorf("read lifecycle policy: %w", err) + } + return validateNoExtraContainers(pod, timeout) } func (k *KubernetesDriver) ensureNamespace(ctx context.Context) error { @@ -77,12 +103,16 @@ func (k *KubernetesDriver) ensureDevContainerPvc( if err != nil { return false, nil, err } + options, err = k.resolveRunOptions(containerInfo, options) + if err != nil { + return false, nil, err + } + if err := k.verifyExistingPodOwner(ctx, workspaceId, options.UID); err != nil { + return false, nil, err + } if pvc == nil { - if options == nil { - return false, nil, fmt.Errorf( - "no options provided and no persistent volume claim found for workspace %q", - workspaceId, - ) + if err := k.ensureNamespace(ctx); err != nil { + return false, nil, err } if err := k.createPersistentVolumeClaim(ctx, workspaceId, options); err != nil { @@ -92,8 +122,8 @@ func (k *KubernetesDriver) ensureDevContainerPvc( return true, options, nil } - if options == nil && containerInfo != nil { - options = containerInfo.Options + if err := k.persistRunOptions(ctx, pvc, containerInfo, options); err != nil { + return false, nil, err } return false, options, nil @@ -114,7 +144,15 @@ func (k *KubernetesDriver) runContainer( return err } - skip, err := k.reconcileExistingPod(ctx, id) + timeout, err := lifecycleTimeout(options) + if err != nil { + return fmt.Errorf("read lifecycle policy: %w", err) + } + if err := validatePodTopology(pod, timeout); err != nil { + return err + } + + skip, err := k.reconcileExistingPod(ctx, id, options) if err != nil { return err } @@ -122,7 +160,7 @@ func (k *KubernetesDriver) runContainer( return nil } - return k.runPod(ctx, id, pod) + return k.runPod(ctx, id, pod, timeout) } func (k *KubernetesDriver) buildPod( @@ -140,6 +178,14 @@ func (k *KubernetesDriver) buildPod( if err != nil { return nil, err } + // Reject before any cluster object is created for this run. + timeout, err := lifecycleTimeout(options) + if err != nil { + return nil, fmt.Errorf("read lifecycle policy: %w", err) + } + if err := validateNoExtraContainers(pod, timeout); err != nil { + return nil, err + } initContainers, err := k.getInitContainers(options, pod, initialize) if err != nil { @@ -180,6 +226,11 @@ func (k *KubernetesDriver) buildPod( }); err != nil { return nil, err } + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[DevsyDockerlessAnnotation] = strconv.FormatBool(options.Dockerless) + pod.Annotations[DevsyImageEntrypointAnnotation] = strconv.FormatBool(options.ImageEntrypoint) return pod, nil } @@ -261,7 +312,9 @@ func (k *KubernetesDriver) resolveWorkspaceMount( mount.Target, ) default: - mount.Target = k.options.WorkspaceVolumeMount + podMount := *mount + podMount.Target = k.options.WorkspaceVolumeMount + mount = &podMount log.Debugf("Using workspace volume mount: %s", k.options.WorkspaceVolumeMount) } @@ -493,41 +546,12 @@ func (k *KubernetesDriver) finalizePodSpec(pod *corev1.Pod, id string, pullSecre pod.Spec.RestartPolicy = corev1.RestartPolicyNever } -func (k *KubernetesDriver) reconcileExistingPod(ctx context.Context, id string) (bool, error) { - existingPod, err := k.getPod(ctx, id) - if err != nil { - return false, fmt.Errorf("get pod: %s: %w", id, err) - } - if existingPod == nil { - return false, nil - } - - existingOptions := &provider2.ProviderKubernetesDriverConfig{} - if raw := existingPod.GetAnnotations()[DevsyLastAppliedAnnotation]; raw != "" { - if err = json.Unmarshal([]byte(raw), existingOptions); err != nil { - log.Errorf("Error unmarshalling existing provider options, continuing...: %s", err) - } - } - - // Nothing changed, can safely return - if optionsEqual(existingOptions, k.options) { - log.Debugf( - "Pod %q already exists and nothing changed, skipping update", - existingPod.Name, - ) - return true, nil - } - - // Stop the current pod - log.Debug("Provider options changed") - if err := k.waitPodDeleted(ctx, id); err != nil { - return false, fmt.Errorf("stop devcontainer: %s: %w", id, err) - } - - return false, nil -} - -func (k *KubernetesDriver) runPod(ctx context.Context, id string, pod *corev1.Pod) error { +func (k *KubernetesDriver) runPod( + ctx context.Context, + id string, + pod *corev1.Pod, + timeout time.Duration, +) error { var err error // set configuration before creating the pod @@ -552,10 +576,16 @@ func (k *KubernetesDriver) runPod(ctx context.Context, id string, pod *corev1.Po // create the pod log.Debugf("Create Pod %q", id) - _, err = k.client.Client().CoreV1().Pods(k.namespace).Create(ctx, pod, metav1.CreateOptions{}) + created, err := k.client.Client(). + CoreV1(). + Pods(k.namespace). + Create(ctx, pod, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("create pod: %w", err) } + if err := k.rejectAdmittedPod(ctx, created, pod, timeout); err != nil { + return err + } // wait for pod running log.Debugf("Waiting for DevContainer Pod %q to come up", id) @@ -703,20 +733,7 @@ func (k *KubernetesDriver) StartDevContainer(ctx context.Context, workspaceId st log.Debugf("Starting devcontainer for workspace %q", workspaceId) defer log.Debugf("Done starting devcontainer for workspace %q", workspaceId) - workspaceId = getID(workspaceId) - _, containerInfo, err := k.getDevContainerPvc(ctx, workspaceId) - if err != nil { - return err - } else if containerInfo == nil { - return fmt.Errorf("persistent volume %q not found", workspaceId) - } - - return k.runContainer( - ctx, - workspaceId, - containerInfo.Options, - false, - ) + return k.RunDevContainer(ctx, workspaceId, nil) } func getID(workspaceID string) string { diff --git a/pkg/driver/kubernetes/wait.go b/pkg/driver/kubernetes/wait.go index eae366c09b..e624dae93c 100644 --- a/pkg/driver/kubernetes/wait.go +++ b/pkg/driver/kubernetes/wait.go @@ -2,13 +2,11 @@ package kubernetes import ( "context" - "errors" "fmt" "strings" "time" "github.com/devsy-org/devsy/pkg/driver/kubernetes/throttledlogger" - "github.com/devsy-org/devsy/pkg/log" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,7 +25,7 @@ func (k *KubernetesDriver) waitPodRunning(ctx context.Context, id string) (*core var pod *corev1.Pod err = wait.PollUntilContextTimeout( ctx, - time.Second, + podPollInterval, timeoutDuration, true, func(ctx context.Context) (bool, error) { @@ -36,7 +34,7 @@ func (k *KubernetesDriver) waitPodRunning(ctx context.Context, id string) (*core if err != nil { return false, err } else if pod == nil { - return true, nil + return false, fmt.Errorf("pod %q disappeared while waiting for it to start", id) } return k.evaluatePodStatus(ctx, pod, &podStatusParams{ @@ -225,17 +223,6 @@ func (k *KubernetesDriver) checkContainerStatus( c *corev1.ContainerStatus, throttledLogger *throttledlogger.ThrottledLogger, ) (bool, error) { - // delete succeeded pods - if IsTerminated(c) && Succeeded(c) { - // delete pod that is succeeded - log.Debugf("Delete Pod %q because it is succeeded", id) - if err := k.waitPodDeleted(ctx, id); err != nil { - return false, err - } - - return false, nil - } - if IsWaiting(c) { return handleContainerWaiting(id, c, throttledLogger) } @@ -335,6 +322,8 @@ func restartableInitContainer(p *corev1.ContainerRestartPolicy) bool { return p != nil && *p == corev1.ContainerRestartPolicyAlways } +// waitPodDeleted deletes the pod for an explicit stop and waits until the API +// server reports it gone. func (k *KubernetesDriver) waitPodDeleted(ctx context.Context, id string) error { err := k.client.Client().CoreV1().Pods(k.namespace).Delete(ctx, id, metav1.DeleteOptions{ GracePeriodSeconds: &[]int64{10}[0], @@ -347,23 +336,5 @@ func (k *KubernetesDriver) waitPodDeleted(ctx context.Context, id string) error return fmt.Errorf("delete pod: %w", err) } - err = wait.PollUntilContextTimeout( - ctx, - time.Second, - time.Minute*2, - true, - func(ctx context.Context) (bool, error) { - _, err := k.client.Client().CoreV1().Pods(k.namespace).Get(ctx, id, metav1.GetOptions{}) - if err != nil { - return true, nil - } - - return false, nil - }, - ) - if err != nil { - return errors.New("timeout waiting for pod to be deleted") - } - - return nil + return k.waitPodGone(ctx, id, "") } diff --git a/pkg/driver/types.go b/pkg/driver/types.go index 590fe9588c..b961acc283 100644 --- a/pkg/driver/types.go +++ b/pkg/driver/types.go @@ -54,6 +54,13 @@ type WorkspaceChowner interface { RequiresWorkspaceChown() bool } +// ManagedLifecycleDriver identifies drivers that can run the container's main +// process as the Devsy daemon. Actual ownership also depends on the run topology. +type ManagedLifecycleDriver interface { + ManagedLifecycle() bool + SetLifecycleShutdownAction(shutdownAction string) +} + // ReprovisioningDriver is a capability interface for drivers that can reprovision // an existing devcontainer in place. Reprovisioning re-runs the container, so it // embeds RunOptionsDriver. @@ -178,6 +185,9 @@ type RunOptions struct { // Dockerless builds the developer filesystem inside the runner after creation. Dockerless bool `json:"dockerless,omitempty"` + // ImageEntrypoint is set from inspected image metadata, never reconstructed from argv. + ImageEntrypoint bool `json:"imageEntrypoint,omitempty"` + // AllowRecreate authorizes replacing an existing VM and discarding its root disk. AllowRecreate bool `json:"allowRecreate,omitempty"` diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 2bfd3c446c..7c9855b227 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -92,12 +92,12 @@ type ProviderAgentConfig struct { // DownloadURL is the base url where to download the agent from DownloadURL string `json:"downloadURL,omitempty"` - // Timeout is the timeout in minutes to wait until the agent tries - // to turn of the server. + // Timeout is the Go duration (e.g. 10m) of inactivity after which the agent + // tries to turn off the server. Timeout string `json:"inactivityTimeout,omitempty"` - // ContainerTimeout is the timeout in minutes to wait until the agent tries - // to delete the container. + // ContainerTimeout is the Go duration (e.g. 10m) of inactivity after which the + // container is stopped. Empty or zero disables it. ContainerTimeout string `json:"containerInactivityTimeout,omitempty"` // InjectGitCredentials signals Devsy if git credentials should get synced into @@ -440,7 +440,9 @@ type ProviderKubernetesDriverConfig struct { ClusterRole string `json:"clusterRole,omitempty"` ServiceAccount string `json:"serviceAccount,omitempty"` - Architecture string `json:"architecture,omitempty"` + Architecture string `json:"architecture,omitempty"` + // InactivityTimeout is a deprecated alias of Agent.ContainerTimeout, used only + // when the canonical field is empty. InactivityTimeout string `json:"inactivityTimeout,omitempty"` StorageClass string `json:"storageClass,omitempty"` diff --git a/pkg/provider/workspace.go b/pkg/provider/workspace.go index ea8edc910a..c5cf1a4b95 100644 --- a/pkg/provider/workspace.go +++ b/pkg/provider/workspace.go @@ -154,6 +154,13 @@ type WorkspaceSource struct { Snapshot string `json:"snapshot,omitempty"` } +// ContainerLifecycleOwner identifies who owns inactivity shutdown in a container. +type ContainerLifecycleOwner string + +// ContainerLifecycleOwnerManagedDaemon means the container's main process is the +// Devsy daemon, which owns the inactivity timer. +const ContainerLifecycleOwnerManagedDaemon ContainerLifecycleOwner = "managedDaemon" + type ContainerWorkspaceInfo struct { // IDE holds the ide config options IDE WorkspaceIDEConfig `json:"ide"` @@ -164,10 +171,14 @@ type ContainerWorkspaceInfo struct { // Dockerless holds custom dockerless configuration Dockerless ProviderDockerlessOptions `json:"dockerless"` - // ContainerTimeout is the timeout in minutes to wait until the agent tries - // to delete the container. + // ContainerTimeout is the effective Go duration of inactivity after which + // the container is stopped. Empty disables it. ContainerTimeout string `json:"containerInactivityTimeout,omitempty"` + // LifecycleOwner names the process that owns inactivity shutdown inside the + // container. Empty means the legacy detached daemon. + LifecycleOwner ContainerLifecycleOwner `json:"lifecycleOwner,omitempty"` + // Source is a WorkspaceSource to be used inside the container Source WorkspaceSource `json:"source"` diff --git a/providers/kubernetes/provider.yaml b/providers/kubernetes/provider.yaml index 30b2397d7c..a4873121fe 100644 --- a/providers/kubernetes/provider.yaml +++ b/providers/kubernetes/provider.yaml @@ -61,7 +61,7 @@ options: description: If defined, Devsy will use the given service account for the dev container. global: true INACTIVITY_TIMEOUT: - description: "If defined, will automatically stop the pod after the inactivity period. Examples: 10m, 1h" + description: "If defined, the workspace pod stops after this much inactivity (no tracked SSH session or setup work). Empty or 0 disables it. The finished pod may stay visible until the next up, which replaces it and keeps the volume. Examples: 10m, 1h" POD_TIMEOUT: description: "Determines how long the provider waits for the workspace pod to come up. Examples: 10m, 1h" default: 10m @@ -135,7 +135,6 @@ agent: clusterRole: ${CLUSTER_ROLE} serviceAccount: ${SERVICE_ACCOUNT} architecture: ${ARCHITECTURE} - inactivityTimeout: ${INACTIVITY_TIMEOUT} storageClass: ${STORAGE_CLASS} pvcAccessMode: ${PVC_ACCESS_MODE} pvcAnnotations: ${PVC_ANNOTATIONS} diff --git a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx index 318644e0ac..02a00174cf 100644 --- a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/inactivity-timeout.mdx @@ -9,11 +9,13 @@ Providers can stop idle workspaces to save resources. The workspace keeps its da Providers that support it expose an `INACTIVITY_TIMEOUT` option. Built-in providers with this option are Apple, Docker, Kubernetes, Microsandbox, and Podman. Values use Go duration format, for example `10m` or `1h`. See [Set provider options](../managing-providers/manage-providers.mdx#set-provider-options). -If the option is unset, a machine provider's own default applies. Devsy uses 20 minutes when the provider sets no default. If `INACTIVITY_TIMEOUT` is unset, Docker, Kubernetes, Podman, Apple, and Microsandbox containers never time out. +If the option is unset, a machine provider's own default applies. Devsy uses 20 minutes when the provider sets no default. If `INACTIVITY_TIMEOUT` is unset or `0`, Docker, Kubernetes, Podman, Apple, and Microsandbox containers never time out. ## How it works -**Non-machine providers** such as Docker, Podman, Kubernetes, Apple, and Microsandbox start a daemon in the container that watches for activity. When the timeout passes with no activity, that daemon exits. For Microsandbox, Devsy also passes the timeout to the microsandbox CLI as `--idle-timeout`. +**Kubernetes** runs the Devsy daemon as the main process of the workspace pod. Setup registers a persistent local connection and waits for the daemon to acknowledge suspension before starting protected work. A live setup has no expiring lease. Finishing or losing that setup connection releases its suspension and starts a fresh idle interval. When the timeout passes with no activity, the daemon stops its children, logs that the inactivity timeout was reached, and exits normally. The pod ends, and the workspace volume is kept. See [Kubernetes](#kubernetes). + +**Other non-machine providers** such as Docker, Podman, Apple, and Microsandbox start a detached daemon in the container that watches for activity. Its exit does not by itself stop the container. For Microsandbox, Devsy also passes the timeout to the microsandbox CLI as `--idle-timeout`. The Apple provider's option is documented to delete the container instead of stopping it. Files outside mounted volumes are lost when a container is deleted. @@ -23,6 +25,57 @@ The Apple provider's option is documented to delete the container instead of sto See the [agent guide](../developing-providers/agent.mdx#machine-providers) for how providers implement this, and [Machine diagnostics](../managing-machines/machine-diagnostics.mdx) for debugging. +## What counts as activity + +A workspace is active while any of these hold: + +- A tracked `devsy workspace ssh` session is connected. Activity is recorded when the session connects, periodically while it stays open, and when it ends, so the idle period starts at disconnect. +- Setup work is running: the synchronous `devsy workspace up` setup, deferred lifecycle hooks, and `postAttachCommand`. + +CPU use and processes you started yourself do not count. A command such as `nohup make &` in a closed session does not keep the workspace alive. + +The daemon checks idle time about once a second at most, so a workspace stops up to a few seconds after the timeout passes. On Kubernetes, SSH sessions and hooks register with the managed daemon before doing work and remain active for their connection's lifetime. Registration and shutdown decisions stay in memory: failed activity-file writes or unavailable disk space cannot hide registered live work. Losing the daemon connection aborts the protected work rather than continuing without activity tracking. Other container daemons still use legacy activity-file timestamps; unreadable activity fails safe. + +Managed clients also read the daemon configuration mounted at `/var/run/secrets/devsy/daemon_config`; they do not rely only on the process environment or runtime directory. Missing discovery in a known managed workspace is an error, not a reason to fall back to file heartbeats. Linux process-start identity must remain readable through `/proc`; otherwise registration fails or the affected protected connection is cancelled. + +All Unix users in the same container intentionally retain the ability to inhibit idle stop through a live activity connection, as they could through the earlier shared heartbeat. This is not inter-user isolation inside a container. Only root or the daemon's own UID can change setup and timeout policy. + + +## Kubernetes + +Set `INACTIVITY_TIMEOUT` on the Kubernetes provider, for example `1h`. The deprecated `inactivityTimeout` field under `agent.kubernetes` in a custom provider is an alias used only when `containerInactivityTimeout` is empty. If both are set, they must match. + +The timeout is off when it is empty or `0`, when `shutdownAction` is `none`, or when `devsy workspace up` runs with `--disable-daemon`. A negative or unparsable value is an error. `stopCompose` with a positive timeout is rejected on Kubernetes. + +`--disable-daemon` applies only to that invocation; it is not saved as a permanent timeout exemption. A later invocation without the flag uses the current provider timeout and shutdown action again. + +The Kubernetes credentials Devsy uses on your machine need `update` permission on `persistentvolumeclaims` in the workspace namespace when the saved run configuration changes, including a recreation that changes the workspace image or Dockerless topology. Devsy atomically updates only its run-information annotation before changing daemon configuration or creating the replacement pod. Unchanged run options do not cause an update. Devsy does not change your RBAC roles automatically. A denied or conflicting update prevents those subsequent changes and retains the volume and its previous record. An explicit `--recreate` can already have stopped the old pod through the existing recreation flow; it does not promise that the old process remains running after an error. + +When the workspace stops: + +- The pod object can stay visible in the `Succeeded` or `Failed` phase until the next `devsy workspace up` or an explicit `devsy workspace stop`. The next `up` deletes that finished pod and creates a new one that mounts the same persistent volume claim. No recreate flag is needed. +- Files on the workspace volume persist. Files written to the container filesystem outside mounted volumes are lost, because the new pod starts from the image. +- `devsy workspace stop` still deletes the pod through the Kubernetes API from your machine. The pod gets no Kubernetes credentials of its own. + +When `WORKSPACE_VOLUME_MOUNT` selects a parent of the workspace folder, the pod mounts that parent while the saved logical workspace folder remains unchanged. Reuse and resume keep the same workspace subdirectory on the volume. + +Pod layout rules Devsy enforces: + +- One regular `devsy` container, any number of normal init containers, and native sidecars (init containers with `restartPolicy: Always`). Native sidecars end with the pod. +- With a positive timeout, extra regular containers in `spec.containers`, from a pod manifest template or injected by an admission webhook, are rejected. The pod would keep running while they run. Devsy names the containers in the error. Without a timeout, or with `shutdownAction: none`, extra containers are allowed. +- With a positive timeout, the Devsy daemon must be the container's main process. A `devcontainer.json` with `overrideCommand: false` and an image entrypoint or command is rejected. +- Dockerless builds do not support a positive timeout. + +The configured pod manifest template is checked against the merged shutdown action and the current invocation's `--disable-daemon` before an explicit `--recreate` stops the existing pod. Admission-webhook changes can only be checked after pod creation. + +At zero effective timeout, Dockerless and an external image main (`overrideCommand: false` with an image entrypoint or command) do not advertise a managed lifecycle owner. The ordinary default entrypoint still uses the managed daemon at zero, so setup and connection tracking retain the same owner contract. + + +The enforced restrictions above do not establish end-to-end support for every pod layout. Restricted non-root security, native sidecars/additional init containers, admission-webhook layouts, Devsy Platform routes, and upgrades from an earlier Devsy version require separate cluster validation. Test your layout on a non-production workspace before relying on the timeout. + + +Existing pods whose running daemon uses an older lifecycle protocol must be replaced once. `devsy workspace up` reports the incompatible owner and asks you to stop and start the workspace. Run `devsy workspace stop`, then `devsy workspace up`. The persistent volume is retained; replacing the agent binary alone does not upgrade the daemon process that already owns the pod. + ## Shutdown action Set `shutdownAction` in `devcontainer.json`; it is not a provider option. @@ -30,4 +83,4 @@ Set `shutdownAction` in `devcontainer.json`; it is not a provider option. - `none` disables the container timeout monitor. For machines, it also prevents `agent.exec.shutdown` from running. - Any other value enables the container timeout monitor. For machines, `agent.exec.shutdown` runs when the timeout passes, if the command is set. -When `shutdownAction` is unset, the default is `stopCompose` for Compose workspaces and `stopContainer` otherwise. The inactivity timeout path only checks whether the value is `none`; the page does not describe per-value container effects. +When `shutdownAction` is unset, the default is `stopCompose` for Compose workspaces and `stopContainer` otherwise. Apart from `none`, and `stopCompose` being rejected on Kubernetes, the value does not change what the container timeout does. diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx index 4cece38afc..aecb7a57b0 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx @@ -33,7 +33,7 @@ Option values can be used in this section. - **local**: set to `true` when Devsy runs directly on this machine instead of reaching a remote one. The built-in providers do this. - **driver**: how the container is deployed. One of `docker` (default), `kubernetes`, `apple`, `microsandbox`, `custom` or `external`. See [Drivers](./driver.mdx). - **inactivityTimeout**: how long to wait before stopping an idle machine. For machine providers. -- **containerInactivityTimeout**: how long to wait before stopping an idle container. For non-machine providers. +- **containerInactivityTimeout**: how long the container may be idle before it stops. A Go duration such as `10m`; empty or `0` disables it. For non-machine providers. - **injectGitCredentials**, **injectDockerCredentials**: whether to copy those credentials into the environment. - **exec.shutdown**: the command that stops the machine once it is idle. Option values are available in it. - **binaries**: helper binaries for `exec.shutdown`. Same format as [provider binaries](./binaries.mdx). @@ -44,7 +44,18 @@ Devsy tracks whether anyone is connected and stops the environment after the tim ### Non-machine providers -The agent ends the container's main process, which stops the container. Set the timeout with `containerInactivityTimeout`. +Set the timeout with `containerInactivityTimeout`. It is a Go duration such as `10m`; empty or `0` disables it. + +On the `kubernetes` driver, the Devsy daemon is the pod's main process. SSH, hooks and setup obtain acknowledged protection over a persistent Unix socket before doing work. Setup suspension belongs to that connection, so a dead setup process cannot leave the timer suspended indefinitely; loss of the owner connection aborts the client work. The daemon exits normally when the workspace has been idle for the timeout, ending the pod while keeping its volume. Other container drivers start a detached daemon whose exit does not by itself stop the container. See [Auto-Inactivity Timeout](../developing-in-workspaces/inactivity-timeout.mdx#kubernetes) for activity rules and pod-layout restrictions. + +The managed activity socket is container-local, not host remote-access IPC. Its `0666` permissions intentionally preserve the [existing cross-user heartbeat authority](https://github.com/devsy-org/devsy/blob/fe22f9a1225d6c1547001755e94c95bb2cdbc8db/cmd/internal/agentcontainer/daemon.go): any same-container UID may inhibit inactivity through its own live activity connection. This grants no general policy/control authority. The owner validates kernel peer UID/PID and process-start identity; only root or the daemon's UID can change setup or timeout policy. These checks do not isolate processes sharing a UID or add authentication to the existing SSH bridge. Do not expose the lifecycle socket outside its container. Host remote-access owner IPC retains its separate `0600` policy and context-plus-workspace-UID authorization. + +Admission is bounded to 128 connections. Non-owner pending and registered connections share a 96-slot activity budget, with at most 32 per UID and 16 per PID. Owner-UID activity and release requests also enter that shared budget, so an embedded SSH carrier cannot consume the remaining setup/control reserve. Owner/root pending admission is provisional, not proof of a control role; deliberate abuse by that already-privileged UID is outside this boundary. Unprotected connections have a nonrenewable 10-second deadline, restarted only when protected work ends; release-only chatter and exact ACK retries cannot renew it. Live acknowledged setup and activity have no time lease. A denied activity registration closes only that incoming SSH connection, not the listener or existing sessions. + +An exited peer, including an unreaped zombie, loses its registration even if another process inherited its socket. Linux liveness uses process state and start time from the same `/proc` snapshot; it does not rely only on the PID remaining present. + +Managed-mode detection uses the same mounted daemon configuration as the daemon itself, ahead of the process environment. An unreadable or malformed configuration aborts registration; absent runtime discovery cannot select legacy mode when managed configuration is present. Linux registration and liveness checks fail closed when the process-start identity cannot be established. + ### Machine providers diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx index 8d2ff1a22c..ef1e91dae9 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/driver.mdx @@ -89,7 +89,7 @@ Example Kubernetes provider that uses local kubectl to run a workspace in the cu name: simple-kubernetes version: v0.0.1 agent: - containerInactivityTimeout: 10m # Pod will automatically kill itself after timeout + containerInactivityTimeout: 10m # Pod stops itself after being idle for the timeout path: ${DEVSY} driver: kubernetes kubernetes: