Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
1ca0720
fix: stop idle Kubernetes workspaces and recreate terminal pods (#1698)
skevetter Oct 25, 2026
835a4a7
fix: satisfy lint in kubernetes inactivity e2e (#1698)
skevetter Oct 25, 2026
5d5ef32
test: avoid shadowing builtin real in lifecycle test (#1698)
skevetter Oct 25, 2026
71d152b
test: cover waitPodDeleted absent and timeout paths (#1698)
skevetter Oct 26, 2026
c7a46bb
fix: make Kubernetes inactivity lifecycle fail safe (#1698)
skevetter Oct 26, 2026
f48183a
fix: bind inactivity protection to live daemon connections
skevetter Oct 26, 2026
e916237
test: cover nonroot lifecycle client capacity release
skevetter Oct 26, 2026
6e5545d
fix: bound lifecycle activity without starving managed control
skevetter Oct 26, 2026
94e3322
fix: release lifecycle protection for zombie peers
skevetter Oct 26, 2026
be9ac00
style: satisfy lifecycle lint across supported builds
skevetter Oct 26, 2026
1b5e307
fix(kubernetes): preserve lifecycle intent across recreate and resume
skevetter Oct 27, 2026
5e8a47b
chore: merge validated main for Kubernetes inactivity fix
skevetter Oct 27, 2026
d753560
test(kubernetes): avoid shadowing builtin copy in pvc test
skevetter Oct 27, 2026
06159cc
fix(kubernetes): keep volumes without saved options findable and clar…
skevetter Oct 27, 2026
acee274
fix(kubernetes): confirm hook registration before arming and validate…
skevetter Oct 27, 2026
217cd95
fix(kubernetes): await hook workers only after a successful launch
skevetter Oct 27, 2026
f44155e
fix(kubernetes): replace stale pending pods, compare options semantic…
skevetter Oct 27, 2026
b15e12a
fix(daemon): drop stale duplicate acknowledgements instead of closing…
skevetter Oct 28, 2026
79af40f
test(daemon): use distinct constants in the duplicate acknowledgement…
skevetter Oct 28, 2026
1616756
fix(ide): hold managed lifecycle activity for connected Fleet sessions
skevetter Oct 28, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 89 additions & 76 deletions cmd/internal/agentcontainer/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,11 @@ package agentcontainer

import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"os/signal"
"strings"
"syscall"
"time"

Expand All @@ -26,10 +23,7 @@ import (
"golang.org/x/sync/errgroup"
)

const (
RootDir = config2.ContainerDataDir
DaemonConfigPath = "/var/run/secrets/" + config2.BinaryName + "/daemon_config"
)
const RootDir = config2.ContainerDataDir

type DaemonCmd struct {
Config *agentd.DaemonConfig
Expand Down Expand Up @@ -77,16 +71,30 @@ func (cmd *DaemonCmd) Run(c *cobra.Command, args []string) error {
ctx, stop := signal.NotifyContext(c.Context(), os.Interrupt, syscall.SIGTERM)

g, ctx := errgroup.WithContext(ctx)
cmd.startDaemonTasks(ctx, g, timeoutDuration)
if err := cmd.startDaemonTasks(ctx, g, timeoutDuration); err != nil {
stop()
return err
}

err = g.Wait()
stop() // Restore default signal handling before exiting.
if err != nil {
os.Exit(daemonExitCode(err))
return nil // Unreachable but needed.
}

// daemonExitCode maps the task group result to a process exit code. Inactivity
// expiry is the normal way a managed container stops, not a failure.
func daemonExitCode(err error) int {
switch {
case err == nil:
return 0
case errors.Is(err, agentd.ErrInactivityExpired):
log.Infof("inactivity timeout reached, daemon exiting")
return 0
default:
log.Errorf("daemon error: %v", err)
os.Exit(1)
return 1
}
os.Exit(0)
return nil // Unreachable but needed.
}

func (cmd *DaemonCmd) setupTimeout() (time.Duration, error) {
Expand All @@ -111,10 +119,11 @@ func (cmd *DaemonCmd) startDaemonTasks(
ctx context.Context,
g *errgroup.Group,
timeoutDuration time.Duration,
) {
) error {
var tasksStarted bool

// Start process reaper.
// Start process reaper only when this process really is the init process of
// its PID namespace; otherwise it would steal other processes' children.
if os.Getpid() == 1 {
g.Go(func() error {
agentd.RunProcessReaper(ctx)
Expand All @@ -130,11 +139,14 @@ func (cmd *DaemonCmd) startDaemonTasks(
})
}

// Start timeout monitor unless shutdownAction is "none".
if timeoutDuration > 0 && cmd.Config.ShutdownAction != config.ShutdownActionNone {
loop, err := cmd.newLifecycleLoop(timeoutDuration)
if err != nil {
return err
}
if loop != nil {
tasksStarted = true
g.Go(func() error {
return runTimeoutMonitor(ctx, timeoutDuration)
return loop.Run(ctx)
})
}

Expand All @@ -153,43 +165,56 @@ func (cmd *DaemonCmd) startDaemonTasks(
return nil
})
}
return nil
}

// newLifecycleLoop builds the inactivity loop. A managed daemon starts unarmed
// and waits for setup; any other daemon with a timeout is armed immediately.
func (cmd *DaemonCmd) newLifecycleLoop(timeout time.Duration) (*agentd.LifecycleLoop, error) {
monitor := &agentd.LifecycleMonitor{}
loop := &agentd.LifecycleLoop{Monitor: monitor, ActivityPath: config2.ContainerActivityFile}
enabled := timeout > 0 && cmd.Config.ShutdownAction != config.ShutdownActionNone

if !cmd.Config.Lifecycle.Managed() {
if !enabled {
return nil, nil
}
monitor.ArmNow(timeout, time.Now())
return loop, nil
}

owner, err := agentd.NewLifecycleOwner(config2.ContainerLifecycleDir)
if err != nil {
return nil, fmt.Errorf("publish container lifecycle owner: %w", err)
}
if err := sharedfile.EnsureMode(config2.ContainerActivityFile, 0o666); err != nil {
_ = owner.Close()
return nil, fmt.Errorf("ensure activity file: %w", err)
}
loop.Owner = owner
if enabled && !cmd.Config.Lifecycle.SetupArmsTimer {
monitor.ArmNow(timeout, time.Now())
}
return loop, nil
}

// loadConfig loads the daemon configuration from base64-encoded JSON.
// If a CLI-provided timeout exists, it will override the timeout in the config.
func (cmd *DaemonCmd) loadConfig() error {
// check local file
encodedCfg := ""
configBytes, err := os.ReadFile(DaemonConfigPath)
cfg, err := agentd.LoadContainerDaemonConfig()
if err != nil {
if errors.Is(err, os.ErrNotExist) {
// check environment variable
encodedCfg = os.Getenv(config2.EnvWorkspaceDaemonConfig)
} else {
return fmt.Errorf("get daemon config file %s: %w", DaemonConfigPath, err)
}
} else {
encodedCfg = string(configBytes)
return err
}

if strings.TrimSpace(encodedCfg) != "" {
decoded, err := base64.StdEncoding.DecodeString(encodedCfg)
if err != nil {
return fmt.Errorf("error decoding daemon config: %w", err)
}
var cfg agentd.DaemonConfig
if err = json.Unmarshal(decoded, &cfg); err != nil {
return fmt.Errorf("error unmarshalling daemon config: %w", err)
}
if cmd.Config.Timeout != "" {
cfg.Timeout = cmd.Config.Timeout
}
if cmd.Config.ShutdownAction != "" {
cfg.ShutdownAction = cmd.Config.ShutdownAction
}
cmd.Config = &cfg
if cfg == nil {
return nil
}

if cmd.Config.Timeout != "" {
cfg.Timeout = cmd.Config.Timeout
}
if cmd.Config.ShutdownAction != "" {
cfg.ShutdownAction = cmd.Config.ShutdownAction
}
cmd.Config = cfg
return nil
}

Expand All @@ -205,29 +230,6 @@ func (cmd *DaemonCmd) shouldRunSsh() bool {
return cmd.Config.Ssh.Workdir != "" || cmd.Config.Ssh.User != ""
}

// runTimeoutMonitor monitors the activity file and signals an error if the timeout is exceeded.
func runTimeoutMonitor(
ctx context.Context,
duration time.Duration,
) error {
ticker := time.NewTicker(10 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return nil
case <-ticker.C:
stat, err := os.Stat(config2.ContainerActivityFile)
if err != nil {
continue
}
if !stat.ModTime().Add(duration).After(time.Now()) {
return errors.New("timeout reached, terminating daemon")
}
}
}
}

// runNetworkServer starts the network server.
func runNetworkServer(
ctx context.Context,
Expand Down Expand Up @@ -260,6 +262,23 @@ func runNetworkServer(
return nil
}

// sshServerArgs builds the embedded SSH server command line. A managed daemon
// asks it to record connection activity so Platform and direct SSH routes keep
// the workspace alive.
func sshServerArgs(cfg *agentd.DaemonConfig) []string {
args := []string{cmdInternal, cmdAgent, cmdContainer, "ssh-server"}
if cfg.Ssh.Workdir != "" {
args = append(args, names.Flag(names.Workdir), cfg.Ssh.Workdir)
}
if cfg.Ssh.User != "" {
args = append(args, names.Flag(names.RemoteUser), cfg.Ssh.User)
}
if cfg.Lifecycle.Managed() {
args = append(args, names.Flag(names.TrackActivity))
}
return args
}

// runSshServer starts the SSH server, sending SIGTERM on context cancellation
// with a grace period before force-killing.
func runSshServer(ctx context.Context, cmd *DaemonCmd) error {
Expand All @@ -268,13 +287,7 @@ func runSshServer(ctx context.Context, cmd *DaemonCmd) error {
return err
}

args := []string{cmdInternal, cmdAgent, cmdContainer, "ssh-server"}
if cmd.Config.Ssh.Workdir != "" {
args = append(args, names.Flag(names.Workdir), cmd.Config.Ssh.Workdir)
}
if cmd.Config.Ssh.User != "" {
args = append(args, names.Flag(names.RemoteUser), cmd.Config.Ssh.User)
}
args := sshServerArgs(cmd.Config)

sshCmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204
sshCmd.Stdout = os.Stdout
Expand Down
6 changes: 6 additions & 0 deletions cmd/internal/agentcontainer/deferred_hooks.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,12 @@ func (cmd *DeferredHooksCmd) Run(ctx context.Context) error {
return err
}

ctx, stopActivity, err := startWorkerActivity(ctx)
if err != nil {
return fmt.Errorf("register deferred-hook activity: %w", err)
}
defer stopActivity()

log.Debugf("running deferred lifecycle hooks")
deferred, err := setup.RunPreAttachHooks(ctx, setupInfo, setup.PreAttachOptions{
Prebuild: cmd.Prebuild,
Expand Down
Loading
Loading