From 8ec17425dcf1d2206635419c21aa88879749036c Mon Sep 17 00:00:00 2001 From: Samuel K Date: Wed, 26 Aug 2026 12:43:05 +0000 Subject: [PATCH 01/17] fix: replace deprecated tailscale n.NetMap usage WatchNetmap subscribes to NotifyInitialStatus and reacts to SelfChange and peer deltas, fetching a fresh ipnstate.Status on demand via LocalClient.Status instead of reading the deprecated ipn.Notify.NetMap bus field (staticcheck SA1019). Both consumers write the same netmap.json debug snapshot. --- pkg/daemon/platform/daemon.go | 8 ++++---- pkg/ts/util.go | 32 +++++++++++++++++++++++--------- pkg/ts/workspace_server.go | 10 +++++----- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/pkg/daemon/platform/daemon.go b/pkg/daemon/platform/daemon.go index 0e3fdc2b11..ba95f85c2a 100644 --- a/pkg/daemon/platform/daemon.go +++ b/pkg/daemon/platform/daemon.go @@ -17,8 +17,8 @@ import ( "github.com/devsy-org/devsy/pkg/platform/client" "github.com/devsy-org/devsy/pkg/ts" "tailscale.com/client/local" + "tailscale.com/ipn/ipnstate" "tailscale.com/tsnet" - "tailscale.com/types/netmap" ) type Daemon struct { @@ -166,10 +166,10 @@ func (d *Daemon) watchNetmap(ctx context.Context) error { return err } - return ts.WatchNetmap(ctx, lc, func(netMap *netmap.NetworkMap) { - nm, err := json.Marshal(netMap) + return ts.WatchNetmap(ctx, lc, func(status *ipnstate.Status) { + nm, err := json.Marshal(status) if err != nil { - log.Errorf("Failed to marshal netmap: %v", err) + log.Errorf("failed to marshal netmap: %v", err) } else { _ = os.WriteFile(filepath.Join(d.rootDir, "netmap.json"), nm, 0o644) } diff --git a/pkg/ts/util.go b/pkg/ts/util.go index bc148846c5..ec64ec5cba 100644 --- a/pkg/ts/util.go +++ b/pkg/ts/util.go @@ -12,7 +12,7 @@ import ( "github.com/devsy-org/devsy/pkg/log" "tailscale.com/client/local" "tailscale.com/ipn" - "tailscale.com/types/netmap" + "tailscale.com/ipn/ipnstate" ) // DevsyTSNetDomain is the MagicDNS suffix for the Devsy tailnet. @@ -80,21 +80,24 @@ func WaitHostReachable( return fmt.Errorf("host %s not reachable", addr.String()) } +// WatchNetmap invokes netmapChangedFn whenever the tailnet state changes: +// first with the initial status, then on self or peer updates. The full +// snapshot is fetched on demand via LocalClient.Status instead of reading +// the deprecated ipn.Notify.NetMap bus field. func WatchNetmap( ctx context.Context, lc *local.Client, - netmapChangedFn func(nm *netmap.NetworkMap), + netmapChangedFn func(status *ipnstate.Status), ) error { watcher, err := lc.WatchIPNBus( ctx, - ipn.NotifyInitialNetMap|ipn.NotifyRateLimit|ipn.NotifyWatchEngineUpdates, + ipn.NotifyInitialStatus|ipn.NotifyWatchEngineUpdates, ) if err != nil { return err } defer func() { _ = watcher.Close() }() - var netMap *netmap.NetworkMap for { n, err := watcher.Next() if err != nil { @@ -103,11 +106,22 @@ func WatchNetmap( if n.ErrMessage != nil { return fmt.Errorf("tailscale error: %w", errors.New(*n.ErrMessage)) } - if n.NetMap != nil { - if n.NetMap != netMap { - netMap = n.NetMap - netmapChangedFn(netMap) - } + if !netmapChanged(n) { + continue } + status, err := lc.Status(ctx) + if err != nil { + return fmt.Errorf("fetch status: %w", err) + } + netmapChangedFn(status) } } + +// netmapChanged reports whether a bus notification indicates a tailnet +// state change, mirroring how upstream consumers react to InitialStatus, +// SelfChange, and peer deltas. +func netmapChanged(n ipn.Notify) bool { + return n.InitialStatus != nil || n.SelfChange != nil || + len(n.PeersChanged) > 0 || len(n.PeersRemoved) > 0 || + len(n.PeerChangedPatch) > 0 +} diff --git a/pkg/ts/workspace_server.go b/pkg/ts/workspace_server.go index 2836db2aaf..01da5c28ba 100644 --- a/pkg/ts/workspace_server.go +++ b/pkg/ts/workspace_server.go @@ -20,9 +20,9 @@ import ( sshServer "github.com/devsy-org/devsy/pkg/ssh/server" "tailscale.com/client/local" "tailscale.com/envknob" + "tailscale.com/ipn/ipnstate" "tailscale.com/ipn/store/mem" "tailscale.com/tsnet" - "tailscale.com/types/netmap" ) const ( @@ -87,20 +87,20 @@ func (s *WorkspaceServer) Start(ctx context.Context) error { go func() { lastUpdate := time.Now() - if err := WatchNetmap(ctx, lc, func(netMap *netmap.NetworkMap) { + if err := WatchNetmap(ctx, lc, func(status *ipnstate.Status) { if time.Since(lastUpdate) < netMapCooldown { return } lastUpdate = time.Now() - nm, err := json.Marshal(netMap) + nm, err := json.Marshal(status) if err != nil { - log.Errorf("Failed to marshal netmap: %v", err) + log.Errorf("failed to marshal netmap: %v", err) } else { _ = os.WriteFile(filepath.Join(s.config.RootDir, "netmap.json"), nm, 0o644) } }); err != nil { - log.Errorf("Failed to watch netmap: %v", err) + log.Errorf("failed to watch netmap: %v", err) } }() From 5955c29de15e6328ec6e7cfb6e22082031dfb245 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Wed, 26 Aug 2026 11:15:36 -0500 Subject: [PATCH 02/17] style: update comment --- pkg/ts/util.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/ts/util.go b/pkg/ts/util.go index ec64ec5cba..d72375983d 100644 --- a/pkg/ts/util.go +++ b/pkg/ts/util.go @@ -80,10 +80,7 @@ func WaitHostReachable( return fmt.Errorf("host %s not reachable", addr.String()) } -// WatchNetmap invokes netmapChangedFn whenever the tailnet state changes: -// first with the initial status, then on self or peer updates. The full -// snapshot is fetched on demand via LocalClient.Status instead of reading -// the deprecated ipn.Notify.NetMap bus field. +// WatchNetmap invokes netmapChangedFn whenever the tailnet state changes. func WatchNetmap( ctx context.Context, lc *local.Client, @@ -118,8 +115,7 @@ func WatchNetmap( } // netmapChanged reports whether a bus notification indicates a tailnet -// state change, mirroring how upstream consumers react to InitialStatus, -// SelfChange, and peer deltas. +// state change. func netmapChanged(n ipn.Notify) bool { return n.InitialStatus != nil || n.SelfChange != nil || len(n.PeersChanged) > 0 || len(n.PeersRemoved) > 0 || From a4c19cec6619e7f0cbfa7d719dde02832aa32f43 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Thu, 27 Aug 2026 21:06:37 -0500 Subject: [PATCH 03/17] fix: subscribe to NotifyPeerChanges in WatchNetmap Peer-only membership changes (join/leave/rename without SelfChange) never refreshed netmap.json: the watch mask lacked ipn.NotifyPeerChanges, so tailscaled strips PeersChanged/ PeersRemoved/PeerChangedPatch before delivery, making the peer-delta checks in netmapChanged dead code. Adds ipn.NotifyPeerChanges to the WatchIPNBus mask in pkg/ts.WatchNetmap. --- pkg/ts/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ts/util.go b/pkg/ts/util.go index d72375983d..96eb248d5a 100644 --- a/pkg/ts/util.go +++ b/pkg/ts/util.go @@ -88,7 +88,7 @@ func WatchNetmap( ) error { watcher, err := lc.WatchIPNBus( ctx, - ipn.NotifyInitialStatus|ipn.NotifyWatchEngineUpdates, + ipn.NotifyInitialStatus|ipn.NotifyWatchEngineUpdates|ipn.NotifyPeerChanges, ) if err != nil { return err From b892b105e74731b124c1131772bd58507f251568 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Thu, 27 Aug 2026 21:21:00 -0500 Subject: [PATCH 04/17] fix: decouple netmap status fetch from IPN bus drain WatchNetmap blocked in LocalClient.Status while calling watcher.Next() in the same loop. A burst of notifications (routine with NotifyPeerChanges now enabled) can outrun the IPN bus's 128-entry queue during that blocking fetch, causing tailscaled to close the watch ("IPN bus consumer fell behind") and killing the daemon/ workspace server, since neither caller retries WatchNetmap. watchNetmap now drains watcher.Next() on a dedicated goroutine and coalesces bursts into a single follow-up status fetch via a buffered(1) trigger channel, so notification consumption never stalls behind a status fetch. Adds a regression test simulating a 129- notification burst while a status fetch is blocked. --- pkg/ts/util.go | 56 +++++++++++++++++++--- pkg/ts/util_test.go | 111 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 pkg/ts/util_test.go diff --git a/pkg/ts/util.go b/pkg/ts/util.go index 96eb248d5a..b575c61663 100644 --- a/pkg/ts/util.go +++ b/pkg/ts/util.go @@ -80,6 +80,11 @@ func WaitHostReachable( return fmt.Errorf("host %s not reachable", addr.String()) } +// ipnWatcher is the subset of *local.IPNBusWatcher used by watchNetmap. +type ipnWatcher interface { + Next() (ipn.Notify, error) +} + // WatchNetmap invokes netmapChangedFn whenever the tailnet state changes. func WatchNetmap( ctx context.Context, @@ -95,22 +100,61 @@ func WatchNetmap( } defer func() { _ = watcher.Close() }() + return watchNetmap(ctx, watcher, lc.Status, netmapChangedFn) +} + +// watchNetmap drains watcher on a dedicated goroutine so notifications keep +// flowing while fetchStatus is in flight, coalescing any changes that arrive +// during a fetch into a single follow-up call. Without this, a burst of +// notifications (e.g. NotifyPeerChanges deltas) can fill the IPN bus's +// 128-entry queue while netmapChangedFn's caller blocks in fetchStatus, +// causing tailscaled to close the watch ("IPN bus consumer fell behind"). +func watchNetmap( + ctx context.Context, + watcher ipnWatcher, + fetchStatus func(context.Context) (*ipnstate.Status, error), + netmapChangedFn func(status *ipnstate.Status), +) error { + trigger := make(chan struct{}, 1) + errc := make(chan error, 1) + + go drainNotifications(watcher, trigger, errc) + + for { + select { + case err := <-errc: + return err + case <-trigger: + status, err := fetchStatus(ctx) + if err != nil { + return fmt.Errorf("fetch status: %w", err) + } + netmapChangedFn(status) + } + } +} + +// drainNotifications continuously reads watcher.Next(), coalescing every +// relevant change into a non-blocking signal on trigger, until watcher +// reports a terminal error (or a bus-side ErrMessage) on errc. +func drainNotifications(watcher ipnWatcher, trigger chan<- struct{}, errc chan<- error) { for { n, err := watcher.Next() if err != nil { - return fmt.Errorf("watch ipn: %w", err) + errc <- fmt.Errorf("watch ipn: %w", err) + return } if n.ErrMessage != nil { - return fmt.Errorf("tailscale error: %w", errors.New(*n.ErrMessage)) + errc <- fmt.Errorf("tailscale error: %w", errors.New(*n.ErrMessage)) + return } if !netmapChanged(n) { continue } - status, err := lc.Status(ctx) - if err != nil { - return fmt.Errorf("fetch status: %w", err) + select { + case trigger <- struct{}{}: + default: } - netmapChangedFn(status) } } diff --git a/pkg/ts/util_test.go b/pkg/ts/util_test.go new file mode 100644 index 0000000000..eb6840fef7 --- /dev/null +++ b/pkg/ts/util_test.go @@ -0,0 +1,111 @@ +package ts + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" +) + +// waitUntil polls cond until it reports true, failing the test if it does +// not become true within timeout. +func waitUntil(t *testing.T, timeout time.Duration, msg string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for !cond() { + if time.Now().After(deadline) { + t.Fatal(msg) + } + time.Sleep(time.Millisecond) + } +} + +// fakeIPNWatcher replays a fixed sequence of notifications, then blocks until +// the test signals it to report the watch as closed. +type fakeIPNWatcher struct { + notifs []ipn.Notify + idx atomic.Int32 + closed chan struct{} +} + +func (w *fakeIPNWatcher) Next() (ipn.Notify, error) { + i := w.idx.Load() + if int(i) < len(w.notifs) { + w.idx.Add(1) + return w.notifs[i], nil + } + <-w.closed + return ipn.Notify{}, errors.New("watcher closed") +} + +// TestWatchNetmapDrainsBurstWhileStatusFetchBlocked verifies that a burst of +// notifications arriving while fetchStatus is in flight does not stall +// notification draining: watchNetmap must keep calling watcher.Next() so a +// burst larger than the IPN bus's 128-entry queue never causes tailscaled to +// close the watch, and must coalesce the burst into a single follow-up fetch. +func TestWatchNetmapDrainsBurstWhileStatusFetchBlocked(t *testing.T) { + const burst = 129 + + notifs := make([]ipn.Notify, burst) + for i := range notifs { + notifs[i] = ipn.Notify{SelfChange: &tailcfg.Node{}} + } + watcher := &fakeIPNWatcher{notifs: notifs, closed: make(chan struct{})} + + firstFetchStarted := make(chan struct{}) + unblockFirstFetch := make(chan struct{}) + var fetchCount atomic.Int32 + + fetchStatus := func(context.Context) (*ipnstate.Status, error) { //nolint:unparam // exercises the success path only + n := fetchCount.Add(1) + if n == 1 { + close(firstFetchStarted) + <-unblockFirstFetch + } + return &ipnstate.Status{}, nil + } + + var callbackCount atomic.Int32 + callback := func(*ipnstate.Status) { callbackCount.Add(1) } + + errc := make(chan error, 1) + go func() { + errc <- watchNetmap(context.Background(), watcher, fetchStatus, callback) + }() + + select { + case <-firstFetchStarted: + case <-time.After(5 * time.Second): + t.Fatal("first status fetch never started") + } + + waitUntil(t, 5*time.Second, "watcher stalled while status fetch was blocked", func() bool { + return watcher.idx.Load() == burst + }) + + close(unblockFirstFetch) + + waitUntil(t, 5*time.Second, "expected a coalesced follow-up fetch", func() bool { + return fetchCount.Load() >= 2 + }) + + if got := fetchCount.Load(); got != 2 { + t.Errorf( + "fetchStatus called %d times, want exactly 2 (initial + one coalesced follow-up)", + got, + ) + } + if got := callbackCount.Load(); got != 2 { + t.Errorf("callback invoked %d times, want 2", got) + } + + close(watcher.closed) + if err := <-errc; err == nil { + t.Fatal("watchNetmap returned nil error after watcher closed, want an error") + } +} From 3969f4d2bdd7a532acc6f993c8fa09950155bc31 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Thu, 27 Aug 2026 22:42:20 -0500 Subject: [PATCH 05/17] fix: eliminate test race in burst-drain regression test Gate fakeIPNWatcher.Next after the first notification until the test observes the first fetchStatus call starting. Without this, the drain goroutine could exhaust all 129 notifications before watchNetmap's select loop ever ran, leaving no notification produced after the first one is consumed and no second trigger to coalesce -- an intermittent failure against correct code. --- pkg/ts/util_test.go | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/pkg/ts/util_test.go b/pkg/ts/util_test.go index eb6840fef7..e24c5fdeff 100644 --- a/pkg/ts/util_test.go +++ b/pkg/ts/util_test.go @@ -25,16 +25,37 @@ func waitUntil(t *testing.T, timeout time.Duration, msg string, cond func() bool } } +func waitFor(t *testing.T, timeout time.Duration, msg string, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(timeout): + t.Fatal(msg) + } +} + +func selfChangeNotifs(n int) []ipn.Notify { + notifs := make([]ipn.Notify, n) + for i := range notifs { + notifs[i] = ipn.Notify{SelfChange: &tailcfg.Node{}} + } + return notifs +} + // fakeIPNWatcher replays a fixed sequence of notifications, then blocks until // the test signals it to report the watch as closed. type fakeIPNWatcher struct { - notifs []ipn.Notify - idx atomic.Int32 - closed chan struct{} + notifs []ipn.Notify + idx atomic.Int32 + afterFirst chan struct{} + closed chan struct{} } func (w *fakeIPNWatcher) Next() (ipn.Notify, error) { i := w.idx.Load() + if i == 1 { + <-w.afterFirst + } if int(i) < len(w.notifs) { w.idx.Add(1) return w.notifs[i], nil @@ -51,19 +72,18 @@ func (w *fakeIPNWatcher) Next() (ipn.Notify, error) { func TestWatchNetmapDrainsBurstWhileStatusFetchBlocked(t *testing.T) { const burst = 129 - notifs := make([]ipn.Notify, burst) - for i := range notifs { - notifs[i] = ipn.Notify{SelfChange: &tailcfg.Node{}} + watcher := &fakeIPNWatcher{ + notifs: selfChangeNotifs(burst), + afterFirst: make(chan struct{}), + closed: make(chan struct{}), } - watcher := &fakeIPNWatcher{notifs: notifs, closed: make(chan struct{})} firstFetchStarted := make(chan struct{}) unblockFirstFetch := make(chan struct{}) var fetchCount atomic.Int32 fetchStatus := func(context.Context) (*ipnstate.Status, error) { //nolint:unparam // exercises the success path only - n := fetchCount.Add(1) - if n == 1 { + if fetchCount.Add(1) == 1 { close(firstFetchStarted) <-unblockFirstFetch } @@ -78,11 +98,8 @@ func TestWatchNetmapDrainsBurstWhileStatusFetchBlocked(t *testing.T) { errc <- watchNetmap(context.Background(), watcher, fetchStatus, callback) }() - select { - case <-firstFetchStarted: - case <-time.After(5 * time.Second): - t.Fatal("first status fetch never started") - } + waitFor(t, 5*time.Second, "first status fetch never started", firstFetchStarted) + close(watcher.afterFirst) waitUntil(t, 5*time.Second, "watcher stalled while status fetch was blocked", func() bool { return watcher.idx.Load() == burst From c3d2040963d95fc04ae03a809de3370c33a41f97 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 28 Aug 2026 10:32:49 -0500 Subject: [PATCH 06/17] style: cleanup comments --- pkg/ts/util.go | 16 +---------- pkg/ts/util_test.go | 9 ------ pkg/ts/workspace_server.go | 59 ++++++++++++++------------------------ 3 files changed, 23 insertions(+), 61 deletions(-) diff --git a/pkg/ts/util.go b/pkg/ts/util.go index b575c61663..cf0bbaa05b 100644 --- a/pkg/ts/util.go +++ b/pkg/ts/util.go @@ -15,11 +15,6 @@ import ( "tailscale.com/ipn/ipnstate" ) -// DevsyTSNetDomain is the MagicDNS suffix for the Devsy tailnet. -// The literal value matches the configured Tailscale tailnet and must stay -// in sync with operator network configuration. -const DevsyTSNetDomain = "ts.loft" - func GetClientHostname(userName string) (string, error) { osHostname, err := os.Hostname() if err != nil { @@ -103,12 +98,6 @@ func WatchNetmap( return watchNetmap(ctx, watcher, lc.Status, netmapChangedFn) } -// watchNetmap drains watcher on a dedicated goroutine so notifications keep -// flowing while fetchStatus is in flight, coalescing any changes that arrive -// during a fetch into a single follow-up call. Without this, a burst of -// notifications (e.g. NotifyPeerChanges deltas) can fill the IPN bus's -// 128-entry queue while netmapChangedFn's caller blocks in fetchStatus, -// causing tailscaled to close the watch ("IPN bus consumer fell behind"). func watchNetmap( ctx context.Context, watcher ipnWatcher, @@ -134,9 +123,6 @@ func watchNetmap( } } -// drainNotifications continuously reads watcher.Next(), coalescing every -// relevant change into a non-blocking signal on trigger, until watcher -// reports a terminal error (or a bus-side ErrMessage) on errc. func drainNotifications(watcher ipnWatcher, trigger chan<- struct{}, errc chan<- error) { for { n, err := watcher.Next() @@ -158,7 +144,7 @@ func drainNotifications(watcher ipnWatcher, trigger chan<- struct{}, errc chan<- } } -// netmapChanged reports whether a bus notification indicates a tailnet +// netmapChanged reports whether a notification indicates a tailnet // state change. func netmapChanged(n ipn.Notify) bool { return n.InitialStatus != nil || n.SelfChange != nil || diff --git a/pkg/ts/util_test.go b/pkg/ts/util_test.go index e24c5fdeff..f71722869e 100644 --- a/pkg/ts/util_test.go +++ b/pkg/ts/util_test.go @@ -12,8 +12,6 @@ import ( "tailscale.com/tailcfg" ) -// waitUntil polls cond until it reports true, failing the test if it does -// not become true within timeout. func waitUntil(t *testing.T, timeout time.Duration, msg string, cond func() bool) { t.Helper() deadline := time.Now().Add(timeout) @@ -42,8 +40,6 @@ func selfChangeNotifs(n int) []ipn.Notify { return notifs } -// fakeIPNWatcher replays a fixed sequence of notifications, then blocks until -// the test signals it to report the watch as closed. type fakeIPNWatcher struct { notifs []ipn.Notify idx atomic.Int32 @@ -64,11 +60,6 @@ func (w *fakeIPNWatcher) Next() (ipn.Notify, error) { return ipn.Notify{}, errors.New("watcher closed") } -// TestWatchNetmapDrainsBurstWhileStatusFetchBlocked verifies that a burst of -// notifications arriving while fetchStatus is in flight does not stall -// notification draining: watchNetmap must keep calling watcher.Next() so a -// burst larger than the IPN bus's 128-entry queue never causes tailscaled to -// close the watch, and must coalesce the burst into a single follow-up fetch. func TestWatchNetmapDrainsBurstWhileStatusFetchBlocked(t *testing.T) { const burst = 129 diff --git a/pkg/ts/workspace_server.go b/pkg/ts/workspace_server.go index 01da5c28ba..b85e131a1d 100644 --- a/pkg/ts/workspace_server.go +++ b/pkg/ts/workspace_server.go @@ -29,6 +29,9 @@ const ( // TSPortForwardPort is the fixed port on which the workspace WebSocket reverse proxy listens. TSPortForwardPort string = "12051" + // DevsyTSNetDomain is the MagicDNS suffix for the Devsy tailnet. + DevsyTSNetDomain = "ts.devsy" + RunnerProxySocket string = "runner-proxy.sock" netMapCooldown = 30 * time.Second @@ -67,7 +70,6 @@ func NewWorkspaceServer(config *WorkspaceServerConfig) *WorkspaceServer { func (s *WorkspaceServer) Start(ctx context.Context) error { log.Infof("Starting workspace server") - // Perform TSNet initialization (validation, control URL, server startup, hostname parsing) workspaceName, projectName, err := s.setupTSNet(ctx) if err != nil { return err @@ -77,10 +79,8 @@ func (s *WorkspaceServer) Start(ctx context.Context) error { return err } - // send heartbeats go s.sendHeartbeats(ctx, projectName, workspaceName, lc) - // Start both SSH and HTTP reverse proxy listeners if err := s.startListeners(ctx, projectName, workspaceName, lc); err != nil { return err } @@ -104,7 +104,6 @@ func (s *WorkspaceServer) Start(ctx context.Context) error { } }() - // Wait until the context is canceled. <-ctx.Done() return nil } @@ -174,7 +173,7 @@ func (s *WorkspaceServer) setupControlURL(ctx context.Context) (*url.URL, error) func (s *WorkspaceServer) initTsServer(ctx context.Context, controlURL *url.URL) error { store, _ := mem.New(s.config.LogF, "") envknob.Setenv("TS_DEBUG_TLS_DIAL_INSECURE_SKIP_VERIFY", "true") - log.Infof("Connecting to control URL - %s/coordinator/", controlURL.String()) + log.Infof("connecting to control URL - %s/coordinator/", controlURL.String()) s.tsServer = &tsnet.Server{ Hostname: s.config.WorkspaceHost, Logf: s.config.LogF, @@ -205,15 +204,13 @@ func (s *WorkspaceServer) startListeners( projectName, workspaceName string, lc *local.Client, ) error { - // Create and start the SSH listener. - log.Infof("Starting SSH listener") + log.Infof("starting SSH listener") sshListener, err := s.createListener(fmt.Sprintf(":%d", sshServer.DefaultUserPort)) if err != nil { return err } - // Create and start the HTTP reverse proxy listener. - log.Infof("Starting HTTP reverse proxy listener on TSNet port %s", TSPortForwardPort) + log.Infof("starting HTTP reverse proxy listener on TSNet port %s", TSPortForwardPort) wsListener, err := s.createListener(fmt.Sprintf(":%s", TSPortForwardPort)) if err != nil { return fmt.Errorf("failed to create listener on TS port %s: %w", TSPortForwardPort, err) @@ -221,7 +218,7 @@ func (s *WorkspaceServer) startListeners( // Create and start the platform HTTP git credentials listener runnerProxySocket := filepath.Join(s.config.RootDir, RunnerProxySocket) - log.Infof("Starting runner proxy socket on %s", runnerProxySocket) + log.Infof("starting runner proxy socket on %s", runnerProxySocket) _ = os.Remove(runnerProxySocket) runnerProxyListener, err := net.Listen("unix", runnerProxySocket) if err != nil { @@ -233,10 +230,8 @@ func (s *WorkspaceServer) startListeners( 0o777, ) // #nosec G302 -- required so all users can connect to the unix socket - // add all listeners to the list s.listeners = append(s.listeners, sshListener, wsListener, runnerProxyListener) - // Setup HTTP handler for git and docker credentials on the runner proxy. go func() { mux := http.NewServeMux() transport := &http.Transport{DialContext: s.tsServer.Dial} @@ -249,7 +244,6 @@ func (s *WorkspaceServer) startListeners( serveMux(runnerProxyListener, mux, "HTTP runner proxy server error: %v") }() - // Setup HTTP handler for port forwarding. go func() { mux := http.NewServeMux() mux.HandleFunc("/portforward", s.httpPortForwardHandler) @@ -260,7 +254,6 @@ func (s *WorkspaceServer) startListeners( ) }() - // Start handling SSH connections. go s.handleSSHConnections(ctx, sshListener) return nil @@ -306,17 +299,16 @@ func (s *WorkspaceServer) gitCredentialsHandler( ) { log.Infof("Received git credentials request from %s", r.RemoteAddr) - // create a new http client with a custom transport discoveredRunner, err := s.discoverRunner(r.Context(), lc) if err != nil { http.Error(w, "failed to discover runner", http.StatusInternalServerError) return } - // build the runner URL runnerURL := fmt.Sprintf( - "http://%s.ts.loft/devsy/%s/%s/workspace-git-credentials", + "http://%s.%s/devsy/%s/%s/workspace-git-credentials", discoveredRunner, + DevsyTSNetDomain projectName, workspaceName, ) @@ -347,19 +339,18 @@ func (s *WorkspaceServer) dockerCredentialsHandler( transport *http.Transport, projectName, workspaceName string, ) { - log.Infof("Received docker credentials request from %s", r.RemoteAddr) + log.Infof("received docker credentials request from %s", r.RemoteAddr) - // create a new http client with a custom transport discoveredRunner, err := s.discoverRunner(r.Context(), lc) if err != nil { http.Error(w, "failed to discover runner", http.StatusInternalServerError) return } - // build the runner URL runnerURL := fmt.Sprintf( - "http://%s.ts.loft/devsy/%s/%s/workspace-docker-credentials", + "http://%s.%s/devsy/%s/%s/workspace-docker-credentials", discoveredRunner, + DevsyTSNetDomain projectName, workspaceName, ) @@ -389,20 +380,18 @@ func (s *WorkspaceServer) httpPortForwardHandler(w http.ResponseWriter, r *http. defer s.removeConnection() log.Debugf("httpPortForwardHandler: starting") - // Retrieve required custom headers. - targetPort := r.Header.Get("X-Loft-Forward-Port") - baseForwardStr := r.Header.Get("X-Loft-Forward-Url") + targetPort := r.Header.Get("X-Devsy-Forward-Port") + baseForwardStr := r.Header.Get("X-Devsy-Forward-Url") if targetPort == "" || baseForwardStr == "" { - http.Error(w, "missing required X-Loft headers", http.StatusBadRequest) + http.Error(w, "missing required X-Devsy headers", http.StatusBadRequest) return } log.Debugf( - "httpPortForwardHandler: received headers: X-Loft-Forward-Port=%s, X-Loft-Forward-Url=%s", + "httpPortForwardHandler: received headers: X-Devsy-Forward-Port=%s, X-Devsy-Forward-Url=%s", targetPort, baseForwardStr, ) - // Parse and modify the URL to target the local endpoint. parsedURL, err := url.Parse(baseForwardStr) if err != nil { log.Errorf("httpPortForwardHandler: failed to parse base URL: %v", err) @@ -419,9 +408,9 @@ func (s *WorkspaceServer) httpPortForwardHandler(w http.ResponseWriter, r *http. pr.Out.URL = &dest pr.Out.Host = dest.Host // Remove custom headers so they are not forwarded. - pr.Out.Header.Del("X-Loft-Forward-Port") - pr.Out.Header.Del("X-Loft-Forward-Url") - pr.Out.Header.Del("X-Loft-Forward-Authorization") + pr.Out.Header.Del("X-Devsy-Forward-Port") + pr.Out.Header.Del("X-Devsy-Forward-Url") + pr.Out.Header.Del("X-Devsy-Forward-Authorization") addForwardedFor(pr) }, } @@ -485,7 +474,6 @@ func (s *WorkspaceServer) handleSSHConnection(clientConn net.Conn) { } defer func() { _ = backendConn.Close() }() - // Start bidirectional copy between client and backend. go func() { defer func() { _ = clientConn.Close() }() defer func() { _ = backendConn.Close() }() @@ -499,11 +487,9 @@ func (s *WorkspaceServer) sendHeartbeats( projectName, workspaceName string, lc *local.Client, ) { - // create a new http client with a custom transport transport := &http.Transport{DialContext: s.tsServer.Dial} client := &http.Client{Transport: transport, Timeout: 10 * time.Second} - // create a ticker to send heartbeats every 10 seconds ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() for { @@ -511,12 +497,10 @@ func (s *WorkspaceServer) sendHeartbeats( case <-ctx.Done(): return case <-ticker.C: - // get the current number of connections s.connectionCounterMu.Lock() connections := s.connectionCounter s.connectionCounterMu.Unlock() - // send a heartbeat if there are connections if connections > 0 { err := s.sendHeartbeat(ctx, client, projectName, workspaceName, lc, connections) if err != nil { @@ -543,13 +527,14 @@ func (s *WorkspaceServer) sendHeartbeat( } heartbeatURL := fmt.Sprintf( - "http://%s.ts.loft/devsy/%s/%s/heartbeat", + "http://%s.%s/devsy/%s/%s/heartbeat", discoveredRunner, + DevsyTSNetDomain, projectName, workspaceName, ) log.Infof( - "Sending heartbeat to %s, because there are %d active connections", + "sending heartbeat to %s, because there are %d active connections", heartbeatURL, connections, ) From 56bde7f58b0b9f5c2c01846a916c0d0d0d0250db Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 28 Aug 2026 11:43:40 -0500 Subject: [PATCH 07/17] refactor: consolidate workspace_server.go around a runnerClient Fixes a build-breaking syntax error (missing comma in the runner URL fmt.Sprintf calls) introduced by the prior commit, then refactors: - Introduce runnerClient, bundling the local client, access key, and project/workspace names needed to locate the platform runner and address its endpoints. Replaces threading (lc, projectName, workspaceName) through every handler and sendHeartbeat/sendHeartbeats, fixing the revive argument-limit violation on sendHeartbeat. - Collapse gitCredentialsHandler/dockerCredentialsHandler, identical but for one path segment, into a single runnerProxyHandler factory parameterized by path. Fixes the dupl finding. - Extract selectRunnerPeer as a pure function over the peer map so runner selection is unit-testable without a live *local.Client. - Replace connectionCounter+sync.Mutex with atomic.Int64. - Extract watchNetmap/persistNetmap and createRunnerProxySocket out of Start/startListeners for single-responsibility; netmap.json now written 0600 instead of 0644 (gosec G306), chmod/write failures are logged instead of silently dropped. - Drop the createListener doc-comment claiming connection tracking it never did. Adds workspace_server_test.go covering selectRunnerPeer and runnerClient.url. No behavior change to the public WorkspaceServer API. --- pkg/ts/workspace_server.go | 381 ++++++++++++++++---------------- pkg/ts/workspace_server_test.go | 67 ++++++ 2 files changed, 253 insertions(+), 195 deletions(-) create mode 100644 pkg/ts/workspace_server_test.go diff --git a/pkg/ts/workspace_server.go b/pkg/ts/workspace_server.go index b85e131a1d..6b6c2f4da7 100644 --- a/pkg/ts/workspace_server.go +++ b/pkg/ts/workspace_server.go @@ -12,7 +12,7 @@ import ( "os" "path/filepath" "strings" - "sync" + "sync/atomic" "time" "github.com/devsy-org/devsy/pkg/log" @@ -23,6 +23,7 @@ import ( "tailscale.com/ipn/ipnstate" "tailscale.com/ipn/store/mem" "tailscale.com/tsnet" + "tailscale.com/types/key" ) const ( @@ -35,6 +36,8 @@ const ( RunnerProxySocket string = "runner-proxy.sock" netMapCooldown = 30 * time.Second + + netmapFileName = "netmap.json" ) // WorkspaceServer holds the TSNet server and its listeners. @@ -42,8 +45,7 @@ type WorkspaceServer struct { tsServer *tsnet.Server listeners []net.Listener - connectionCounter int - connectionCounterMu sync.Mutex + connectionCount atomic.Int64 config *WorkspaceServerConfig } @@ -79,30 +81,19 @@ func (s *WorkspaceServer) Start(ctx context.Context) error { return err } - go s.sendHeartbeats(ctx, projectName, workspaceName, lc) - - if err := s.startListeners(ctx, projectName, workspaceName, lc); err != nil { - return err + runner := &runnerClient{ + lc: lc, + accessKey: s.config.AccessKey, + projectName: projectName, + workspaceName: workspaceName, } - go func() { - lastUpdate := time.Now() - if err := WatchNetmap(ctx, lc, func(status *ipnstate.Status) { - if time.Since(lastUpdate) < netMapCooldown { - return - } - lastUpdate = time.Now() + go s.sendHeartbeats(ctx, runner) + go s.watchNetmap(ctx, lc) - nm, err := json.Marshal(status) - if err != nil { - log.Errorf("failed to marshal netmap: %v", err) - } else { - _ = os.WriteFile(filepath.Join(s.config.RootDir, "netmap.json"), nm, 0o644) - } - }); err != nil { - log.Errorf("failed to watch netmap: %v", err) - } - }() + if err := s.startListeners(ctx, runner); err != nil { + return err + } <-ctx.Done() return nil @@ -189,7 +180,7 @@ func (s *WorkspaceServer) initTsServer(ctx context.Context, controlURL *url.URL) return nil } -// parseHostname extracts workspace and project names from the hostname. +// parseWorkspaceHostname extracts workspace and project names from the hostname. func (s *WorkspaceServer) parseWorkspaceHostname() (workspace, project string, err error) { parts := strings.Split(s.config.WorkspaceHost, ".") if len(parts) < 4 { @@ -198,12 +189,35 @@ func (s *WorkspaceServer) parseWorkspaceHostname() (workspace, project string, e return parts[1], parts[2], nil } +// watchNetmap persists the tailnet status to netmap.json for debugging, +// throttled to once per netMapCooldown. +func (s *WorkspaceServer) watchNetmap(ctx context.Context, lc *local.Client) { + lastUpdate := time.Now() + err := WatchNetmap(ctx, lc, func(status *ipnstate.Status) { + if time.Since(lastUpdate) < netMapCooldown { + return + } + lastUpdate = time.Now() + s.persistNetmap(status) + }) + if err != nil { + log.Errorf("failed to watch netmap: %v", err) + } +} + +func (s *WorkspaceServer) persistNetmap(status *ipnstate.Status) { + nm, err := json.Marshal(status) + if err != nil { + log.Errorf("failed to marshal netmap: %v", err) + return + } + if err := os.WriteFile(filepath.Join(s.config.RootDir, netmapFileName), nm, 0o600); err != nil { + log.Errorf("failed to write netmap: %v", err) + } +} + // startListeners creates and starts the SSH and HTTP reverse proxy listeners. -func (s *WorkspaceServer) startListeners( - ctx context.Context, - projectName, workspaceName string, - lc *local.Client, -) error { +func (s *WorkspaceServer) startListeners(ctx context.Context, runner *runnerClient) error { log.Infof("starting SSH listener") sshListener, err := s.createListener(fmt.Sprintf(":%d", sshServer.DefaultUserPort)) if err != nil { @@ -216,47 +230,69 @@ func (s *WorkspaceServer) startListeners( return fmt.Errorf("failed to create listener on TS port %s: %w", TSPortForwardPort, err) } - // Create and start the platform HTTP git credentials listener + runnerProxyListener, err := s.createRunnerProxySocket() + if err != nil { + return err + } + + s.listeners = append(s.listeners, sshListener, wsListener, runnerProxyListener) + + transport := &http.Transport{DialContext: s.tsServer.Dial} + go s.serveRunnerProxy(runnerProxyListener, runner, transport) + go s.servePortForward(wsListener) + go s.handleSSHConnections(ctx, sshListener) + + return nil +} + +// createRunnerProxySocket creates the unix socket the platform runner uses to +// reach the workspace's credential-proxy endpoints. +func (s *WorkspaceServer) createRunnerProxySocket() (net.Listener, error) { runnerProxySocket := filepath.Join(s.config.RootDir, RunnerProxySocket) log.Infof("starting runner proxy socket on %s", runnerProxySocket) + _ = os.Remove(runnerProxySocket) - runnerProxyListener, err := net.Listen("unix", runnerProxySocket) + listener, err := net.Listen("unix", runnerProxySocket) if err != nil { - return fmt.Errorf("failed to create listener on TS port %s: %w", TSPortForwardPort, err) + return nil, fmt.Errorf( + "failed to create runner proxy socket %s: %w", + runnerProxySocket, + err, + ) } - _ = os.Chmod( + chmodErr := os.Chmod( runnerProxySocket, 0o777, - ) // #nosec G302 -- required so all users can connect to the unix socket - - s.listeners = append(s.listeners, sshListener, wsListener, runnerProxyListener) - - go func() { - mux := http.NewServeMux() - transport := &http.Transport{DialContext: s.tsServer.Dial} - mux.HandleFunc("/git-credentials", func(w http.ResponseWriter, r *http.Request) { - s.gitCredentialsHandler(w, r, lc, transport, projectName, workspaceName) - }) - mux.HandleFunc("/docker-credentials", func(w http.ResponseWriter, r *http.Request) { - s.dockerCredentialsHandler(w, r, lc, transport, projectName, workspaceName) - }) - serveMux(runnerProxyListener, mux, "HTTP runner proxy server error: %v") - }() + ) // #nosec G302 -- all users must reach the unix socket + if chmodErr != nil { + log.Errorf("failed to chmod runner proxy socket %s: %v", runnerProxySocket, chmodErr) + } - go func() { - mux := http.NewServeMux() - mux.HandleFunc("/portforward", s.httpPortForwardHandler) - serveMux( - wsListener, - mux, - fmt.Sprintf("HTTP server error on TS port %s: %%v", TSPortForwardPort), - ) - }() + return listener, nil +} - go s.handleSSHConnections(ctx, sshListener) +func (s *WorkspaceServer) serveRunnerProxy( + listener net.Listener, + runner *runnerClient, + transport *http.Transport, +) { + mux := http.NewServeMux() + mux.HandleFunc( + "/git-credentials", + s.runnerProxyHandler(runner, transport, "workspace-git-credentials"), + ) + mux.HandleFunc( + "/docker-credentials", + s.runnerProxyHandler(runner, transport, "workspace-docker-credentials"), + ) + serveMux(listener, mux, "runner proxy server error: %v") +} - return nil +func (s *WorkspaceServer) servePortForward(listener net.Listener) { + mux := http.NewServeMux() + mux.HandleFunc("/portforward", s.httpPortForwardHandler) + serveMux(listener, mux, fmt.Sprintf("HTTP server error on TS port %s: %%v", TSPortForwardPort)) } func serveMux(listener net.Listener, mux *http.ServeMux, errFormat string) { @@ -266,111 +302,57 @@ func serveMux(listener net.Listener, mux *http.ServeMux, errFormat string) { } } -// createListener creates a raw listener and wraps it with connection tracking. +// createListener creates a listener bound to the TSNet server. func (s *WorkspaceServer) createListener(addr string) (net.Listener, error) { l, err := s.tsServer.Listen("tcp", addr) if err != nil { return nil, fmt.Errorf("failed to listen on %s: %w", addr, err) } - - // create a new tracked listener to track the number of connections return l, nil } func (s *WorkspaceServer) addConnection() { - s.connectionCounterMu.Lock() - defer s.connectionCounterMu.Unlock() - s.connectionCounter++ + s.connectionCount.Add(1) } func (s *WorkspaceServer) removeConnection() { - s.connectionCounterMu.Lock() - defer s.connectionCounterMu.Unlock() - s.connectionCounter-- -} - -// gitCredentialsHandler is the handler for git credentials requests for workspace. -func (s *WorkspaceServer) gitCredentialsHandler( - w http.ResponseWriter, - r *http.Request, - lc *local.Client, - transport *http.Transport, - projectName, workspaceName string, -) { - log.Infof("Received git credentials request from %s", r.RemoteAddr) - - discoveredRunner, err := s.discoverRunner(r.Context(), lc) - if err != nil { - http.Error(w, "failed to discover runner", http.StatusInternalServerError) - return - } - - runnerURL := fmt.Sprintf( - "http://%s.%s/devsy/%s/%s/workspace-git-credentials", - discoveredRunner, - DevsyTSNetDomain - projectName, - workspaceName, - ) - parsedURL, err := url.Parse(runnerURL) - if err != nil { - http.Error(w, "failed to parse runner URL", http.StatusInternalServerError) - return - } - - proxy := &httputil.ReverseProxy{ - Rewrite: func(pr *httputil.ProxyRequest) { - dest := *parsedURL - pr.Out.URL = &dest - pr.Out.Host = dest.Host - pr.Out.Header.Set("Authorization", "Bearer "+s.config.AccessKey) - addForwardedFor(pr) - }, - } - proxy.Transport = transport - proxy.ServeHTTP(w, r) + s.connectionCount.Add(-1) } -// dockerCredentialsHandler is the handler for docker credentials requests for workspace. -func (s *WorkspaceServer) dockerCredentialsHandler( - w http.ResponseWriter, - r *http.Request, - lc *local.Client, +// runnerProxyHandler builds a reverse-proxy handler that discovers the +// runner peer and forwards the request to path on it. +func (s *WorkspaceServer) runnerProxyHandler( + runner *runnerClient, transport *http.Transport, - projectName, workspaceName string, -) { - log.Infof("received docker credentials request from %s", r.RemoteAddr) + path string, +) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + log.Infof("received %s request from %s", path, r.RemoteAddr) - discoveredRunner, err := s.discoverRunner(r.Context(), lc) - if err != nil { - http.Error(w, "failed to discover runner", http.StatusInternalServerError) - return - } + runnerHost, err := runner.discoverRunner(r.Context()) + if err != nil { + http.Error(w, "failed to discover runner", http.StatusInternalServerError) + return + } - runnerURL := fmt.Sprintf( - "http://%s.%s/devsy/%s/%s/workspace-docker-credentials", - discoveredRunner, - DevsyTSNetDomain - projectName, - workspaceName, - ) - parsedURL, err := url.Parse(runnerURL) - if err != nil { - http.Error(w, "failed to parse runner URL", http.StatusInternalServerError) - return - } + parsedURL, err := url.Parse(runner.url(runnerHost, path)) + if err != nil { + http.Error(w, "failed to parse runner URL", http.StatusInternalServerError) + return + } - proxy := &httputil.ReverseProxy{ - Rewrite: func(pr *httputil.ProxyRequest) { - dest := *parsedURL - pr.Out.URL = &dest - pr.Out.Host = dest.Host - pr.Out.Header.Set("Authorization", "Bearer "+s.config.AccessKey) - addForwardedFor(pr) - }, + proxy := &httputil.ReverseProxy{ + Transport: transport, + Rewrite: func(pr *httputil.ProxyRequest) { + dest := *parsedURL + pr.Out.URL = &dest + pr.Out.Host = dest.Host + pr.Out.Header.Set("Authorization", "Bearer "+runner.accessKey) + addForwardedFor(pr) + }, + } + proxy.ServeHTTP(w, r) } - proxy.Transport = transport - proxy.ServeHTTP(w, r) } // httpPortForwardHandler is the HTTP reverse proxy handler for workspace. @@ -403,6 +385,7 @@ func (s *WorkspaceServer) httpPortForwardHandler(w http.ResponseWriter, r *http. log.Debugf("httpPortForwardHandler: final target URL=%s", parsedURL.String()) proxy := &httputil.ReverseProxy{ + Transport: http.DefaultTransport, Rewrite: func(pr *httputil.ProxyRequest) { dest := *parsedURL pr.Out.URL = &dest @@ -414,7 +397,6 @@ func (s *WorkspaceServer) httpPortForwardHandler(w http.ResponseWriter, r *http. addForwardedFor(pr) }, } - proxy.Transport = http.DefaultTransport log.Infof( "httpPortForwardHandler: final proxied request: %s %s", @@ -482,13 +464,11 @@ func (s *WorkspaceServer) handleSSHConnection(clientConn net.Conn) { _, err = io.Copy(clientConn, backendConn) } -func (s *WorkspaceServer) sendHeartbeats( - ctx context.Context, - projectName, workspaceName string, - lc *local.Client, -) { - transport := &http.Transport{DialContext: s.tsServer.Dial} - client := &http.Client{Transport: transport, Timeout: 10 * time.Second} +func (s *WorkspaceServer) sendHeartbeats(ctx context.Context, runner *runnerClient) { + client := &http.Client{ + Transport: &http.Transport{DialContext: s.tsServer.Dial}, + Timeout: 10 * time.Second, + } ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() @@ -497,15 +477,11 @@ func (s *WorkspaceServer) sendHeartbeats( case <-ctx.Done(): return case <-ticker.C: - s.connectionCounterMu.Lock() - connections := s.connectionCounter - s.connectionCounterMu.Unlock() - - if connections > 0 { - err := s.sendHeartbeat(ctx, client, projectName, workspaceName, lc, connections) - if err != nil { - log.Errorf("Failed to send heartbeat: %v", err) - } + if s.connectionCount.Load() <= 0 { + continue + } + if err := s.sendHeartbeat(ctx, client, runner); err != nil { + log.Errorf("Failed to send heartbeat: %v", err) } } } @@ -514,36 +490,29 @@ func (s *WorkspaceServer) sendHeartbeats( func (s *WorkspaceServer) sendHeartbeat( ctx context.Context, client *http.Client, - projectName, workspaceName string, - lc *local.Client, - connections int, + runner *runnerClient, ) error { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - discoveredRunner, err := s.discoverRunner(ctx, lc) + runnerHost, err := runner.discoverRunner(ctx) if err != nil { return fmt.Errorf("failed to discover runner: %w", err) } - heartbeatURL := fmt.Sprintf( - "http://%s.%s/devsy/%s/%s/heartbeat", - discoveredRunner, - DevsyTSNetDomain, - projectName, - workspaceName, - ) + heartbeatURL := runner.url(runnerHost, "heartbeat") log.Infof( "sending heartbeat to %s, because there are %d active connections", heartbeatURL, - connections, + s.connectionCount.Load(), ) - req, err := http.NewRequestWithContext(ctx, "GET", heartbeatURL, nil) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, heartbeatURL, nil) if err != nil { return fmt.Errorf("failed to create request for %s: %w", heartbeatURL, err) } + req.Header.Set("Authorization", "Bearer "+runner.accessKey) - req.Header.Set("Authorization", "Bearer "+s.config.AccessKey) resp, err := client.Do(req) if err != nil { return fmt.Errorf("request to %s failed: %w", heartbeatURL, err) @@ -551,34 +520,56 @@ func (s *WorkspaceServer) sendHeartbeat( defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("received response from %s - Status: %d", heartbeatURL, resp.StatusCode) + return fmt.Errorf("received response from %s - status: %d", heartbeatURL, resp.StatusCode) } - log.Infof("received response from %s - Status: %d", heartbeatURL, resp.StatusCode) + log.Infof("received response from %s - status: %d", heartbeatURL, resp.StatusCode) return nil } -// discoverRunner attempts to find the runner peer from the TSNet status. -func (s *WorkspaceServer) discoverRunner(ctx context.Context, lc *local.Client) (string, error) { - status, err := lc.Status(ctx) +// runnerClient bundles what's needed to locate the platform runner peer and +// address its endpoints on the tailnet. +type runnerClient struct { + lc *local.Client + accessKey string + projectName string + workspaceName string +} + +// discoverRunner finds the runner peer's hostname from the TSNet status. +func (rc *runnerClient) discoverRunner(ctx context.Context) (string, error) { + status, err := rc.lc.Status(ctx) if err != nil { return "", fmt.Errorf("failed to get status: %w", err) } - var runner string - for _, peer := range status.Peer { - if peer == nil || peer.HostName == "" { - continue - } + runnerHost, ok := selectRunnerPeer(status.Peer) + if !ok { + return "", fmt.Errorf("no active runner found") + } + log.Infof("discoverRunner: selected runner = %s", runnerHost) + return runnerHost, nil +} - if strings.HasSuffix(peer.HostName, "runner") { - runner = peer.HostName - break +// selectRunnerPeer picks the platform runner from the tailnet's peer set, +// identified by a "runner"-suffixed hostname. +func selectRunnerPeer(peers map[key.NodePublic]*ipnstate.PeerStatus) (string, bool) { + for _, peer := range peers { + if peer != nil && strings.HasSuffix(peer.HostName, "runner") { + return peer.HostName, true } } - if runner == "" { - return "", fmt.Errorf("no active runner found") - } + return "", false +} - log.Infof("discoverRunner: selected runner = %s", runner) - return runner, nil +// url builds the runner-relative URL for path, e.g. "heartbeat" or +// "workspace-git-credentials". +func (rc *runnerClient) url(runnerHost, path string) string { + return fmt.Sprintf( + "http://%s.%s/devsy/%s/%s/%s", + runnerHost, + DevsyTSNetDomain, + rc.projectName, + rc.workspaceName, + path, + ) } diff --git a/pkg/ts/workspace_server_test.go b/pkg/ts/workspace_server_test.go new file mode 100644 index 0000000000..a0d3557064 --- /dev/null +++ b/pkg/ts/workspace_server_test.go @@ -0,0 +1,67 @@ +package ts + +import ( + "testing" + + "tailscale.com/ipn/ipnstate" + "tailscale.com/types/key" +) + +func TestSelectRunnerPeer(t *testing.T) { + cases := []struct { + name string + peers map[key.NodePublic]*ipnstate.PeerStatus + wantHost string + wantFound bool + }{ + { + name: "no peers", + peers: map[key.NodePublic]*ipnstate.PeerStatus{}, + wantFound: false, + }, + { + name: "no runner peer", + peers: map[key.NodePublic]*ipnstate.PeerStatus{ + {}: {HostName: "some-client"}, + }, + wantFound: false, + }, + { + name: "nil peer entry is skipped", + peers: map[key.NodePublic]*ipnstate.PeerStatus{ + {}: nil, + }, + wantFound: false, + }, + { + name: "runner peer found", + peers: map[key.NodePublic]*ipnstate.PeerStatus{ + {}: {HostName: "acme-project-abc123-runner"}, + }, + wantHost: "acme-project-abc123-runner", + wantFound: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + host, found := selectRunnerPeer(tc.peers) + if found != tc.wantFound { + t.Fatalf("selectRunnerPeer() found = %v, want %v", found, tc.wantFound) + } + if found && host != tc.wantHost { + t.Errorf("selectRunnerPeer() host = %q, want %q", host, tc.wantHost) + } + }) + } +} + +func TestRunnerClientURL(t *testing.T) { + rc := &runnerClient{projectName: "acme", workspaceName: "ws1"} + + got := rc.url("acme-ws1-runner", "workspace-git-credentials") + want := "http://acme-ws1-runner.ts.devsy/devsy/acme/ws1/workspace-git-credentials" + if got != want { + t.Errorf("url() = %q, want %q", got, want) + } +} From d4e3a335bc9c21884f382b54b438b32d165528c2 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 28 Aug 2026 11:56:24 -0500 Subject: [PATCH 08/17] fix: resource leaks, TLS-skip inconsistency, and lint debt in pkg/ts Bugs: - WaitHostReachable (util.go) deferred cancel() inside its retry loop, leaking up to maxRetries contexts/timers until the whole call returned instead of releasing each one after its own iteration. - CheckDerpConnection (derp.go) never closed the probe response body, and wrapped a nil error with %w when the request succeeded but returned a non-200 status, producing a garbled message instead of reporting the actual status code. Security consistency: - CheckDerpConnection unconditionally set InsecureSkipVerify, and WorkspaceServer.initTsServer unconditionally set TS_DEBUG_TLS_DIAL_INSECURE_SKIP_VERIFY, while pkg/daemon/platform's newTSServer gated the same env var behind an explicit insecure bool. Added WorkspaceServerConfig.Insecure and threaded it through both call sites; the one construction site (cmd/internal/agentcontainer) sets it true, preserving current behavior explicitly instead of unconditionally. Lint (all findings in pkg/ts now resolved): - WaitForSSHClient/newSSHClient (ssh.go): bundled 5-6 positional args into SSHDialConfig (revive argument-limit). Updated the one external caller (daemonclient/client.go). - WaitHostReachable's uint16(addr.Port()) conversion (gosec G115): added toUint16Port with an explicit range check instead of a silent truncation. - ssh.go's InsecureIgnoreHostKey (gosec G106): justified with #nosec -- the tailnet, not host-key trust, is this connection's security boundary. Dead code: - Removed ConnTrackingFunc (derp.go), unreferenced anywhere. - Removed WorkspaceServerConfig.Client, set by its one caller but never read in pkg/ts. Duplication: - Extracted PersistNetmapStatus (util.go), replacing the netmap.json marshal+write duplicated between pkg/daemon/platform/daemon.go and workspace_server.go's persistNetmap. Both now write 0600 (was 0644 in daemon.go, gosec G306). --- cmd/internal/agentcontainer/daemon.go | 2 +- .../daemonclient/client.go | 8 +++- pkg/daemon/platform/daemon.go | 10 +---- pkg/daemon/platform/ts_server.go | 2 +- pkg/ts/derp.go | 24 +++++++---- pkg/ts/ssh.go | 42 +++++++++---------- pkg/ts/util.go | 40 ++++++++++++++++-- pkg/ts/workspace_server.go | 28 ++++--------- 8 files changed, 91 insertions(+), 65 deletions(-) diff --git a/cmd/internal/agentcontainer/daemon.go b/cmd/internal/agentcontainer/daemon.go index bfd74e1e97..41d017153c 100644 --- a/cmd/internal/agentcontainer/daemon.go +++ b/cmd/internal/agentcontainer/daemon.go @@ -248,8 +248,8 @@ func runNetworkServer( AccessKey: cmd.Config.Platform.AccessKey, PlatformHost: ts.RemoveProtocol(cmd.Config.Platform.PlatformHost), WorkspaceHost: cmd.Config.Platform.WorkspaceHost, - Client: baseClient, RootDir: RootDir, + Insecure: true, LogF: func(format string, args ...any) { log.Infof(format, args...) }, diff --git a/pkg/client/clientimplementation/daemonclient/client.go b/pkg/client/clientimplementation/daemonclient/client.go index c82733e1bd..6e7a8ee08c 100644 --- a/pkg/client/clientimplementation/daemonclient/client.go +++ b/pkg/client/clientimplementation/daemonclient/client.go @@ -183,11 +183,15 @@ func (c *client) SSHClients( return c.tsClient.DialTCP(ctx, addressParts[0], uint16(port)) } - toolClient, err = ts.WaitForSSHClient(ctx, dial, "tcp", address, "root", time.Second*10) + toolClient, err = ts.WaitForSSHClient(ctx, ts.SSHDialConfig{ + Dialer: dial, Network: "tcp", Address: address, User: "root", Timeout: time.Second * 10, + }) if err != nil { return nil, nil, fmt.Errorf("create SSH tool client: %w", err) } - userClient, err = ts.WaitForSSHClient(ctx, dial, "tcp", address, user, time.Second*10) + userClient, err = ts.WaitForSSHClient(ctx, ts.SSHDialConfig{ + Dialer: dial, Network: "tcp", Address: address, User: user, Timeout: time.Second * 10, + }) if err != nil { return nil, nil, fmt.Errorf("create SSH user client: %w", err) } diff --git a/pkg/daemon/platform/daemon.go b/pkg/daemon/platform/daemon.go index ba95f85c2a..98151730d8 100644 --- a/pkg/daemon/platform/daemon.go +++ b/pkg/daemon/platform/daemon.go @@ -3,13 +3,10 @@ package daemon import ( "bufio" "context" - "encoding/json" "errors" "fmt" "io" "net" - "os" - "path/filepath" "time" "github.com/devsy-org/devsy/pkg/config" @@ -167,12 +164,7 @@ func (d *Daemon) watchNetmap(ctx context.Context) error { } return ts.WatchNetmap(ctx, lc, func(status *ipnstate.Status) { - nm, err := json.Marshal(status) - if err != nil { - log.Errorf("failed to marshal netmap: %v", err) - } else { - _ = os.WriteFile(filepath.Join(d.rootDir, "netmap.json"), nm, 0o644) - } + ts.PersistNetmapStatus(d.rootDir, status) }) } diff --git a/pkg/daemon/platform/ts_server.go b/pkg/daemon/platform/ts_server.go index 6a93ec79c7..5694116b48 100644 --- a/pkg/daemon/platform/ts_server.go +++ b/pkg/daemon/platform/ts_server.go @@ -26,7 +26,7 @@ func newTSServer( Scheme: ts.GetEnvOrDefault("DEVSY_TSNET_SCHEME", "https"), Host: ts.RemoveProtocol(host), } - if err := ts.CheckDerpConnection(ctx, &baseUrl); err != nil { + if err := ts.CheckDerpConnection(ctx, &baseUrl, insecure); err != nil { return nil, nil, fmt.Errorf("failed to verify DERP connection: %w", err) } if insecure { diff --git a/pkg/ts/derp.go b/pkg/ts/derp.go index 9370f9a22a..896a95bf75 100644 --- a/pkg/ts/derp.go +++ b/pkg/ts/derp.go @@ -11,17 +11,18 @@ import ( "time" ) -type ConnTrackingFunc func(address string) - -// CheckDerpConnection validates the DERP connection. -func CheckDerpConnection(ctx context.Context, baseUrl *url.URL) error { - newTransport := http.DefaultTransport.(*http.Transport).Clone() - newTransport.TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, +// CheckDerpConnection validates the DERP connection. insecure skips TLS +// certificate verification, for coordinators known to use self-signed certs. +func CheckDerpConnection(ctx context.Context, baseUrl *url.URL, insecure bool) error { + transport := http.DefaultTransport.(*http.Transport).Clone() + if insecure { + transport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } // #nosec G402 -- opt-in via explicit insecure flag } client := &http.Client{ - Transport: newTransport, + Transport: transport, Timeout: 5 * time.Second, } @@ -34,9 +35,14 @@ func CheckDerpConnection(ctx context.Context, baseUrl *url.URL) error { } res, err := client.Do(req) - if err != nil || (res != nil && res.StatusCode != http.StatusOK) { + if err != nil { return fmt.Errorf("failed to reach the coordinator server: %w", err) } + defer func() { _ = res.Body.Close() }() + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("coordinator server returned status %d", res.StatusCode) + } return nil } diff --git a/pkg/ts/ssh.go b/pkg/ts/ssh.go index e6357c0898..6ceccc81c1 100644 --- a/pkg/ts/ssh.go +++ b/pkg/ts/ssh.go @@ -12,22 +12,26 @@ import ( type Dialer func(ctx context.Context, network, address string) (net.Conn, error) -func WaitForSSHClient( - ctx context.Context, - dialer Dialer, - network, address string, - user string, - timeout time.Duration, -) (*ssh.Client, error) { - deadline := time.Now().Add(timeout) +// SSHDialConfig bundles the parameters needed to establish and retry an SSH +// connection over a tailnet dialer. +type SSHDialConfig struct { + Dialer Dialer + Network string + Address string + User string + Timeout time.Duration +} + +func WaitForSSHClient(ctx context.Context, cfg SSHDialConfig) (*ssh.Client, error) { + deadline := time.Now().Add(cfg.Timeout) var ( c *ssh.Client err error ) - log.Debugf("Attempting to establish SSH connection with %s as user %s", address, user) + log.Debugf("Attempting to establish SSH connection with %s as user %s", cfg.Address, cfg.User) for time.Now().Before(deadline) { - c, err = newSSHClient(ctx, dialer, network, address, user) + c, err = newSSHClient(ctx, cfg) if err == nil { return c, nil } @@ -43,24 +47,20 @@ func WaitForSSHClient( return c, err } -func newSSHClient( - ctx context.Context, - dialer Dialer, - network, address string, - user string, -) (*ssh.Client, error) { - conn, err := dialer(ctx, network, address) +func newSSHClient(ctx context.Context, cfg SSHDialConfig) (*ssh.Client, error) { + conn, err := cfg.Dialer(ctx, cfg.Network, cfg.Address) if err != nil { - return nil, fmt.Errorf("dial %s: %w", address, err) + return nil, fmt.Errorf("dial %s: %w", cfg.Address, err) } clientConfig := &ssh.ClientConfig{ - User: user, - Auth: []ssh.AuthMethod{}, // The SSH server is only reachable through the tailnet + User: cfg.User, + Auth: []ssh.AuthMethod{}, // The SSH server is only reachable through the tailnet + // #nosec G106 -- the tailnet, not TLS/host-key trust, is this connection's security boundary HostKeyCallback: ssh.InsecureIgnoreHostKey(), } - sshConn, channels, requests, err := ssh.NewClientConn(conn, address, clientConfig) + sshConn, channels, requests, err := ssh.NewClientConn(conn, cfg.Address, clientConfig) if err != nil { return nil, fmt.Errorf("establish SSH connection: %w", err) } diff --git a/pkg/ts/util.go b/pkg/ts/util.go index cf0bbaa05b..3d4116697b 100644 --- a/pkg/ts/util.go +++ b/pkg/ts/util.go @@ -2,9 +2,12 @@ package ts import ( "context" + "encoding/json" "errors" "fmt" + "math" "os" + "path/filepath" "strings" "time" @@ -54,11 +57,16 @@ func WaitHostReachable( addr Addr, maxRetries int, ) error { + port, err := toUint16Port(addr.Port()) + if err != nil { + return fmt.Errorf("host %s: %w", addr.String(), err) + } + for i := range maxRetries { timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - conn, err := lc.DialTCP(timeoutCtx, addr.Host(), uint16(addr.Port())) - if err == nil { + conn, dialErr := lc.DialTCP(timeoutCtx, addr.Host(), port) + cancel() + if dialErr == nil { _ = conn.Close() return nil // Host is reachable } @@ -75,6 +83,14 @@ func WaitHostReachable( return fmt.Errorf("host %s not reachable", addr.String()) } +// toUint16Port validates that port fits the uint16 range TCP ports use. +func toUint16Port(port int) (uint16, error) { + if port < 0 || port > math.MaxUint16 { + return 0, fmt.Errorf("port %d out of range", port) + } + return uint16(port), nil // #nosec G115 -- bounds-checked above +} + // ipnWatcher is the subset of *local.IPNBusWatcher used by watchNetmap. type ipnWatcher interface { Next() (ipn.Notify, error) @@ -151,3 +167,21 @@ func netmapChanged(n ipn.Notify) bool { len(n.PeersChanged) > 0 || len(n.PeersRemoved) > 0 || len(n.PeerChangedPatch) > 0 } + +// netmapFileName is the debug snapshot both the daemon and workspace +// server's WatchNetmap callbacks write on every tailnet state change. +const netmapFileName = "netmap.json" + +// PersistNetmapStatus marshals status and writes it to netmap.json under +// rootDir for debugging, logging rather than returning any failure since +// callers invoke this from a WatchNetmap callback with no error path. +func PersistNetmapStatus(rootDir string, status *ipnstate.Status) { + nm, err := json.Marshal(status) + if err != nil { + log.Errorf("failed to marshal netmap: %v", err) + return + } + if err := os.WriteFile(filepath.Join(rootDir, netmapFileName), nm, 0o600); err != nil { + log.Errorf("failed to write netmap: %v", err) + } +} diff --git a/pkg/ts/workspace_server.go b/pkg/ts/workspace_server.go index 6b6c2f4da7..55cfb45270 100644 --- a/pkg/ts/workspace_server.go +++ b/pkg/ts/workspace_server.go @@ -2,7 +2,6 @@ package ts import ( "context" - "encoding/json" "fmt" "io" "net" @@ -16,7 +15,6 @@ import ( "time" "github.com/devsy-org/devsy/pkg/log" - "github.com/devsy-org/devsy/pkg/platform/client" sshServer "github.com/devsy-org/devsy/pkg/ssh/server" "tailscale.com/client/local" "tailscale.com/envknob" @@ -36,8 +34,6 @@ const ( RunnerProxySocket string = "runner-proxy.sock" netMapCooldown = 30 * time.Second - - netmapFileName = "netmap.json" ) // WorkspaceServer holds the TSNet server and its listeners. @@ -56,8 +52,11 @@ type WorkspaceServerConfig struct { PlatformHost string WorkspaceHost string LogF func(format string, args ...any) - Client client.Client RootDir string + // Insecure skips TLS certificate verification for the DERP probe and + // the TSNet control-plane connection. Only set for coordinators known + // to use self-signed certificates. + Insecure bool } // NewWorkspaceServer creates a new TSNet server instance. @@ -154,7 +153,7 @@ func (s *WorkspaceServer) setupControlURL(ctx context.Context) (*url.URL, error) Scheme: GetEnvOrDefault("DEVSY_TSNET_SCHEME", "https"), Host: s.config.PlatformHost, } - if err := CheckDerpConnection(ctx, baseURL); err != nil { + if err := CheckDerpConnection(ctx, baseURL, s.config.Insecure); err != nil { return nil, fmt.Errorf("failed to verify DERP connection: %w", err) } return baseURL, nil @@ -163,7 +162,9 @@ func (s *WorkspaceServer) setupControlURL(ctx context.Context) (*url.URL, error) // initTsServer initializes the TSNet server. func (s *WorkspaceServer) initTsServer(ctx context.Context, controlURL *url.URL) error { store, _ := mem.New(s.config.LogF, "") - envknob.Setenv("TS_DEBUG_TLS_DIAL_INSECURE_SKIP_VERIFY", "true") + if s.config.Insecure { + envknob.Setenv("TS_DEBUG_TLS_DIAL_INSECURE_SKIP_VERIFY", "true") + } log.Infof("connecting to control URL - %s/coordinator/", controlURL.String()) s.tsServer = &tsnet.Server{ Hostname: s.config.WorkspaceHost, @@ -198,24 +199,13 @@ func (s *WorkspaceServer) watchNetmap(ctx context.Context, lc *local.Client) { return } lastUpdate = time.Now() - s.persistNetmap(status) + PersistNetmapStatus(s.config.RootDir, status) }) if err != nil { log.Errorf("failed to watch netmap: %v", err) } } -func (s *WorkspaceServer) persistNetmap(status *ipnstate.Status) { - nm, err := json.Marshal(status) - if err != nil { - log.Errorf("failed to marshal netmap: %v", err) - return - } - if err := os.WriteFile(filepath.Join(s.config.RootDir, netmapFileName), nm, 0o600); err != nil { - log.Errorf("failed to write netmap: %v", err) - } -} - // startListeners creates and starts the SSH and HTTP reverse proxy listeners. func (s *WorkspaceServer) startListeners(ctx context.Context, runner *runnerClient) error { log.Infof("starting SSH listener") From 9ba8a18683e942c37465a374f85c0b3e76b0bf32 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 28 Aug 2026 12:11:03 -0500 Subject: [PATCH 09/17] fix: close port-conversion gap missed by independent review An independent code-reviewer subagent re-audited the full diff and verification claims from the prior commits. No Critical/Major bugs; three Minor gaps: - Export ToUint16Port (was toUint16Port) and add Addr.PortUint16(), reusing the same bounds check WaitHostReachable already had. Wires it into daemonclient/client.go's two remaining unguarded uint16(port) conversions (SSHClients' dial closure, DirectTunnel) -- same gosec G115 pattern this PR already fixed once in pkg/ts but left live at these two sibling call sites on the same Addr type. - Add TestToUint16Port covering negative/zero/max/over-max. - Add TestNetmapChanged covering all five branches (InitialStatus, SelfChange, PeersChanged, PeersRemoved, PeerChangedPatch) plus the all-empty case; previously only SelfChange was exercised indirectly via the burst-drain test. --- .../daemonclient/client.go | 12 +++- pkg/ts/addr.go | 6 ++ pkg/ts/util.go | 6 +- pkg/ts/util_test.go | 60 +++++++++++++++++++ 4 files changed, 79 insertions(+), 5 deletions(-) diff --git a/pkg/client/clientimplementation/daemonclient/client.go b/pkg/client/clientimplementation/daemonclient/client.go index 6e7a8ee08c..d443bbaa41 100644 --- a/pkg/client/clientimplementation/daemonclient/client.go +++ b/pkg/client/clientimplementation/daemonclient/client.go @@ -179,8 +179,12 @@ func (c *client) SSHClients( if err != nil { return nil, fmt.Errorf("invalid port: %s", addressParts[1]) } + port16, err := ts.ToUint16Port(port) + if err != nil { + return nil, fmt.Errorf("invalid port: %w", err) + } - return c.tsClient.DialTCP(ctx, addressParts[0], uint16(port)) + return c.tsClient.DialTCP(ctx, addressParts[0], port16) } toolClient, err = ts.WaitForSSHClient(ctx, ts.SSHDialConfig{ @@ -204,7 +208,11 @@ func (c *client) DirectTunnel(ctx context.Context, stdin io.Reader, stdout io.Wr if err != nil { return fmt.Errorf("resolve workspace hostname: %w", err) } - conn, err := c.tsClient.DialTCP(ctx, wAddr.Host(), uint16(wAddr.Port())) + port, err := wAddr.PortUint16() + if err != nil { + return fmt.Errorf("invalid workspace port: %w", err) + } + conn, err := c.tsClient.DialTCP(ctx, wAddr.Host(), port) if err != nil { return fmt.Errorf("failed to connect to SSH server in proxy mode: %w", err) } diff --git a/pkg/ts/addr.go b/pkg/ts/addr.go index b2de733ce8..cb83fdf51a 100644 --- a/pkg/ts/addr.go +++ b/pkg/ts/addr.go @@ -28,3 +28,9 @@ func (a Addr) Host() string { func (a Addr) Port() int { return a.port } + +// PortUint16 returns Port bounds-checked for use with APIs that take a +// uint16 port, such as *local.Client.DialTCP. +func (a Addr) PortUint16() (uint16, error) { + return ToUint16Port(a.port) +} diff --git a/pkg/ts/util.go b/pkg/ts/util.go index 3d4116697b..0a42d0b2a3 100644 --- a/pkg/ts/util.go +++ b/pkg/ts/util.go @@ -57,7 +57,7 @@ func WaitHostReachable( addr Addr, maxRetries int, ) error { - port, err := toUint16Port(addr.Port()) + port, err := ToUint16Port(addr.Port()) if err != nil { return fmt.Errorf("host %s: %w", addr.String(), err) } @@ -83,8 +83,8 @@ func WaitHostReachable( return fmt.Errorf("host %s not reachable", addr.String()) } -// toUint16Port validates that port fits the uint16 range TCP ports use. -func toUint16Port(port int) (uint16, error) { +// ToUint16Port validates that port fits the uint16 range TCP ports use. +func ToUint16Port(port int) (uint16, error) { if port < 0 || port > math.MaxUint16 { return 0, fmt.Errorf("port %d out of range", port) } diff --git a/pkg/ts/util_test.go b/pkg/ts/util_test.go index f71722869e..395f037a28 100644 --- a/pkg/ts/util_test.go +++ b/pkg/ts/util_test.go @@ -3,6 +3,7 @@ package ts import ( "context" "errors" + "math" "sync/atomic" "testing" "time" @@ -117,3 +118,62 @@ func TestWatchNetmapDrainsBurstWhileStatusFetchBlocked(t *testing.T) { t.Fatal("watchNetmap returned nil error after watcher closed, want an error") } } + +func TestToUint16Port(t *testing.T) { + cases := []struct { + name string + port int + want uint16 + wantErr bool + }{ + {name: "negative", port: -1, wantErr: true}, + {name: "zero", port: 0, want: 0}, + {name: "max", port: math.MaxUint16, want: math.MaxUint16}, + {name: "over max", port: math.MaxUint16 + 1, wantErr: true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ToUint16Port(tc.port) + if tc.wantErr { + if err == nil { + t.Fatalf("ToUint16Port(%d) = %d, nil; want an error", tc.port, got) + } + return + } + if err != nil { + t.Fatalf("ToUint16Port(%d) returned unexpected error: %v", tc.port, err) + } + if got != tc.want { + t.Errorf("ToUint16Port(%d) = %d, want %d", tc.port, got, tc.want) + } + }) + } +} + +func TestNetmapChanged(t *testing.T) { + cases := []struct { + name string + n ipn.Notify + want bool + }{ + {name: "empty notify", n: ipn.Notify{}, want: false}, + {name: "initial status", n: ipn.Notify{InitialStatus: &ipnstate.Status{}}, want: true}, + {name: "self change", n: ipn.Notify{SelfChange: &tailcfg.Node{}}, want: true}, + {name: "peers changed", n: ipn.Notify{PeersChanged: []*tailcfg.Node{{}}}, want: true}, + {name: "peers removed", n: ipn.Notify{PeersRemoved: []tailcfg.NodeID{1}}, want: true}, + { + name: "peer changed patch", + n: ipn.Notify{PeerChangedPatch: []*tailcfg.PeerChange{{}}}, + want: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := netmapChanged(tc.n); got != tc.want { + t.Errorf("netmapChanged(%+v) = %v, want %v", tc.n, got, tc.want) + } + }) + } +} From 01c3af0f68354460749e63695afaf01998dc962e Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 28 Aug 2026 14:02:05 -0500 Subject: [PATCH 10/17] style: lowercase log messages; document runner-proxy socket permission CodeRabbit review on 9ba8a186: - Lowercase log messages per repo convention (AGENTS.md), including two more it missed in functions this PR already touched. - Flagged runnerProxySocket's 0o777 chmod as world-reachable, exposing the git/docker credential proxy to any local account. Investigated: the daemon runs as the container's default user (root, PID 1), but git/docker commands invoked from an interactive session run as whatever devcontainer remoteUser pkg/ssh/server su'd to -- a different UID, by design. Tightening the socket would break credential proxying for that (normal) case, so left 0o777 and documented why instead of narrowing it blind. --- pkg/ts/ssh.go | 4 ++-- pkg/ts/util.go | 2 +- pkg/ts/workspace_server.go | 19 ++++++++++--------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/pkg/ts/ssh.go b/pkg/ts/ssh.go index 6ceccc81c1..2844e343da 100644 --- a/pkg/ts/ssh.go +++ b/pkg/ts/ssh.go @@ -29,7 +29,7 @@ func WaitForSSHClient(ctx context.Context, cfg SSHDialConfig) (*ssh.Client, erro c *ssh.Client err error ) - log.Debugf("Attempting to establish SSH connection with %s as user %s", cfg.Address, cfg.User) + log.Debugf("attempting to establish SSH connection with %s as user %s", cfg.Address, cfg.User) for time.Now().Before(deadline) { c, err = newSSHClient(ctx, cfg) if err == nil { @@ -42,7 +42,7 @@ func WaitForSSHClient(ctx context.Context, cfg SSHDialConfig) (*ssh.Client, erro time.Sleep(100 * time.Millisecond) } } - log.Debugf("Failed to establish SSH connection %v", err) + log.Debugf("failed to establish SSH connection %v", err) return c, err } diff --git a/pkg/ts/util.go b/pkg/ts/util.go index 0a42d0b2a3..34d4cd127d 100644 --- a/pkg/ts/util.go +++ b/pkg/ts/util.go @@ -70,7 +70,7 @@ func WaitHostReachable( _ = conn.Close() return nil // Host is reachable } - log.Debugf("Host %s not reachable, retrying (%d/%d)", addr.String(), i+1, maxRetries) + log.Debugf("host %s not reachable, retrying (%d/%d)", addr.String(), i+1, maxRetries) time.Sleep(200 * time.Millisecond) select { diff --git a/pkg/ts/workspace_server.go b/pkg/ts/workspace_server.go index 55cfb45270..9156feb6c1 100644 --- a/pkg/ts/workspace_server.go +++ b/pkg/ts/workspace_server.go @@ -69,7 +69,7 @@ func NewWorkspaceServer(config *WorkspaceServerConfig) *WorkspaceServer { // Start initializes the TSNet server, sets up listeners for SSH and HTTP // reverse proxy traffic, and waits until the given context is canceled. func (s *WorkspaceServer) Start(ctx context.Context) error { - log.Infof("Starting workspace server") + log.Infof("starting workspace server") workspaceName, projectName, err := s.setupTSNet(ctx) if err != nil { @@ -251,10 +251,11 @@ func (s *WorkspaceServer) createRunnerProxySocket() (net.Listener, error) { ) } - chmodErr := os.Chmod( - runnerProxySocket, - 0o777, - ) // #nosec G302 -- all users must reach the unix socket + // The daemon runs as the container's default (often root) user, but git/docker + // commands invoking the credential helper run as whatever devcontainer remoteUser + // an interactive session su'd to (see pkg/ssh/server), a different UID. All local + // users must therefore reach this socket. + chmodErr := os.Chmod(runnerProxySocket, 0o777) // #nosec G302 -- see comment above if chmodErr != nil { log.Errorf("failed to chmod runner proxy socket %s: %v", runnerProxySocket, chmodErr) } @@ -282,7 +283,7 @@ func (s *WorkspaceServer) serveRunnerProxy( func (s *WorkspaceServer) servePortForward(listener net.Listener) { mux := http.NewServeMux() mux.HandleFunc("/portforward", s.httpPortForwardHandler) - serveMux(listener, mux, fmt.Sprintf("HTTP server error on TS port %s: %%v", TSPortForwardPort)) + serveMux(listener, mux, fmt.Sprintf("http server error on TS port %s: %%v", TSPortForwardPort)) } func serveMux(listener net.Listener, mux *http.ServeMux, errFormat string) { @@ -425,7 +426,7 @@ func (s *WorkspaceServer) handleSSHConnections(ctx context.Context, listener net if ctx.Err() != nil { return } - log.Errorf("Failed to accept connection: %v", err) + log.Errorf("failed to accept connection: %v", err) continue } go s.handleSSHConnection(clientConn) @@ -441,7 +442,7 @@ func (s *WorkspaceServer) handleSSHConnection(clientConn net.Conn) { localAddr := fmt.Sprintf("127.0.0.1:%d", sshServer.DefaultUserPort) backendConn, err := net.Dial("tcp", localAddr) if err != nil { - log.Errorf("Failed to connect to local address %s: %v", localAddr, err) + log.Errorf("failed to connect to local address %s: %v", localAddr, err) return } defer func() { _ = backendConn.Close() }() @@ -471,7 +472,7 @@ func (s *WorkspaceServer) sendHeartbeats(ctx context.Context, runner *runnerClie continue } if err := s.sendHeartbeat(ctx, client, runner); err != nil { - log.Errorf("Failed to send heartbeat: %v", err) + log.Errorf("failed to send heartbeat: %v", err) } } } From fae81a443e9d907dbbac89f97daa7b0d13e7796d Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 28 Aug 2026 14:05:46 -0500 Subject: [PATCH 11/17] refactor: replace hand-rolled retry loop with wait.PollUntilContextTimeout k8s.io/apimachinery/pkg/util/wait is already a direct dependency and the established retry/poll idiom elsewhere in the repo (e.g. pkg/driver/docker/lifecycle.go's waitForLocalImage). WaitHostReachable had its own hand-rolled loop with the same category of bug already fixed once in this function (a context/timer leak) -- delegating to the shared, already-trusted implementation removes that whole class of risk (e.g. the ctx.Done() check running only after a 200ms sleep instead of being respected immediately). WaitHostReachable's maxRetries int becomes timeout time.Duration, since the wait package is duration- not count-bounded. Updated both call sites (daemonclient/client.go's CheckWorkspaceReachable): - 5 retries (~26s worst case) -> 30s - 20 retries (~104s worst case) -> 2m --- .../daemonclient/client.go | 4 +- pkg/ts/util.go | 39 ++++++++++--------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/pkg/client/clientimplementation/daemonclient/client.go b/pkg/client/clientimplementation/daemonclient/client.go index d443bbaa41..c8e1a6aaa2 100644 --- a/pkg/client/clientimplementation/daemonclient/client.go +++ b/pkg/client/clientimplementation/daemonclient/client.go @@ -132,7 +132,7 @@ func (c *client) CheckWorkspaceReachable(ctx context.Context) error { if err != nil { return fmt.Errorf("resolve workspace hostname: %w", err) } - err = ts.WaitHostReachable(ctx, c.tsClient, wAddr, 5) + err = ts.WaitHostReachable(ctx, c.tsClient, wAddr, 30*time.Second) if err == nil { log.Debugf("Host %s is reachable. Proceeding with SSH session", wAddr.Host()) return nil @@ -149,7 +149,7 @@ func (c *client) CheckWorkspaceReachable(ctx context.Context) error { time.Sleep(2 * time.Second) // let's try again - err = ts.WaitHostReachable(ctx, c.tsClient, wAddr, 20) + err = ts.WaitHostReachable(ctx, c.tsClient, wAddr, 2*time.Minute) if err == nil { return nil } diff --git a/pkg/ts/util.go b/pkg/ts/util.go index 34d4cd127d..63a0b46590 100644 --- a/pkg/ts/util.go +++ b/pkg/ts/util.go @@ -13,6 +13,7 @@ import ( "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/log" + "k8s.io/apimachinery/pkg/util/wait" "tailscale.com/client/local" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" @@ -50,37 +51,37 @@ func GetURL(host string, port int) string { return fmt.Sprintf("%s.%s:%d", host, DevsyTSNetDomain, port) } -// WaitHostReachable polls until the given host is reachable via ts. +// WaitHostReachable polls until the given host is reachable via ts, or +// timeout elapses. func WaitHostReachable( ctx context.Context, lc *local.Client, addr Addr, - maxRetries int, + timeout time.Duration, ) error { port, err := ToUint16Port(addr.Port()) if err != nil { return fmt.Errorf("host %s: %w", addr.String(), err) } - for i := range maxRetries { - timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - conn, dialErr := lc.DialTCP(timeoutCtx, addr.Host(), port) - cancel() - if dialErr == nil { + var lastErr error + pollErr := wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, timeout, true, + func(ctx context.Context) (bool, error) { + dialCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + conn, dialErr := lc.DialTCP(dialCtx, addr.Host(), port) + if dialErr != nil { + lastErr = dialErr + log.Debugf("host %s not reachable: %v", addr.String(), dialErr) + return false, nil // keep polling; not fatal + } _ = conn.Close() - return nil // Host is reachable - } - log.Debugf("host %s not reachable, retrying (%d/%d)", addr.String(), i+1, maxRetries) - time.Sleep(200 * time.Millisecond) - - select { - case <-ctx.Done(): - return ctx.Err() - default: - } + return true, nil + }) + if pollErr != nil { + return fmt.Errorf("host %s not reachable: %w", addr.String(), lastErr) } - - return fmt.Errorf("host %s not reachable", addr.String()) + return nil } // ToUint16Port validates that port fits the uint16 range TCP ports use. From b46b76602ac483264764b98f1d459f0e152f0cb0 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 28 Aug 2026 14:15:01 -0500 Subject: [PATCH 12/17] refactor: collapse CheckWorkspaceReachable to a single poll The two sequential ts.WaitHostReachable calls (30s quick check, then a fresh 2m budget after launching the desktop app) were two genuinely different retry budgets, not duplication -- but the phase transition (check daemon health, maybe launch the desktop app) was hidden between them as inline branching rather than expressed as part of the poll itself. Collapsed into one wait.PollUntilContextTimeout call. The condition function (workspaceReachabilityCheck.step, extracted to a separate type to keep cyclomatic complexity down) dials on every tick and, once daemonHealthCheckDelay (30s) has passed without success, checks whether the daemon is down and launches the desktop app -- returning errGiveUpReachingWorkspace to stop early once retrying cannot help (daemon confirmed up, or the desktop app itself failed to launch). Total budget (150s) preserves the original worst-case (30s + 2s sleep + 2m). The explicit 2s "give the app a chance to start" sleep is subsumed by the 200ms poll interval, which was already retrying during that window. pkg/ts.WaitHostReachable is now unused (this was its only caller) and removed, along with its now-dead time/wait imports. --- .../daemonclient/client.go | 105 ++++++++++++++---- pkg/ts/util.go | 35 ------ 2 files changed, 85 insertions(+), 55 deletions(-) diff --git a/pkg/client/clientimplementation/daemonclient/client.go b/pkg/client/clientimplementation/daemonclient/client.go index c8e1a6aaa2..2b457e33fb 100644 --- a/pkg/client/clientimplementation/daemonclient/client.go +++ b/pkg/client/clientimplementation/daemonclient/client.go @@ -27,10 +27,77 @@ import ( sshServer "github.com/devsy-org/devsy/pkg/ssh/server" "github.com/devsy-org/devsy/pkg/ts" "golang.org/x/crypto/ssh" + "k8s.io/apimachinery/pkg/util/wait" "tailscale.com/client/local" "tailscale.com/tailcfg" ) +// checkWorkspaceReachableTimeout is CheckWorkspaceReachable's overall retry +// budget, covering both the initial reachability check and, if the daemon +// needed restarting, however long the desktop app takes to bring the +// workspace back up. +const checkWorkspaceReachableTimeout = 150 * time.Second + +// daemonHealthCheckDelay is how long CheckWorkspaceReachable retries a plain +// dial before it checks whether the daemon itself is down and, if so, +// launches the desktop app to restart it. Checking immediately would fire on +// every transient dial failure; the delay gives a live daemon a chance to +// become reachable on its own first. +const daemonHealthCheckDelay = 30 * time.Second + +// errGiveUpReachingWorkspace stops CheckWorkspaceReachable's poll early once +// retrying can no longer help: either the daemon is confirmed to be running +// and the host is unreachable for some other reason, or launching the +// desktop app itself failed. +var errGiveUpReachingWorkspace = errors.New("give up reaching workspace") + +// workspaceReachabilityCheck holds CheckWorkspaceReachable's poll state +// across attempts: the last dial error, whether the desktop app has already +// been launched, and whatever workspace status that launch decision was +// based on (reused in the final error if every attempt fails). +type workspaceReachabilityCheck struct { + client *client + wAddr ts.Addr + port uint16 + start time.Time + + desktopOpened bool + reachErr error + getWorkspaceErr error + instance *managementv1.DevsyWorkspaceInstance +} + +// step is one poll attempt: dial the workspace, and once +// daemonHealthCheckDelay has passed without success, check whether the +// daemon itself is down and launch the desktop app to restart it. +func (r *workspaceReachabilityCheck) step(ctx context.Context) (bool, error) { + dialCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + conn, dialErr := r.client.tsClient.DialTCP(dialCtx, r.wAddr.Host(), r.port) + if dialErr == nil { + _ = conn.Close() + return true, nil + } + r.reachErr = dialErr + + if r.desktopOpened || time.Since(r.start) < daemonHealthCheckDelay { + return false, nil + } + + r.instance, r.getWorkspaceErr = r.client.localClient.GetWorkspace(ctx, r.client.workspace.UID) + if !daemon.IsDaemonNotAvailableError(r.getWorkspaceErr) { + // Daemon is reachable (or failed for an unrelated reason); no amount + // of retrying fixes an unreachable host in that case. + return false, errGiveUpReachingWorkspace + } + + r.desktopOpened = true + if openErr := r.client.openDesktopApp(); openErr != nil { + return false, errGiveUpReachingWorkspace // inform user about daemon state + } + return false, nil +} + func New( devsyConfig *config.Config, prov *provider.ProviderConfig, @@ -132,31 +199,29 @@ func (c *client) CheckWorkspaceReachable(ctx context.Context) error { if err != nil { return fmt.Errorf("resolve workspace hostname: %w", err) } - err = ts.WaitHostReachable(ctx, c.tsClient, wAddr, 30*time.Second) - if err == nil { - log.Debugf("Host %s is reachable. Proceeding with SSH session", wAddr.Host()) - return nil + port, err := wAddr.PortUint16() + if err != nil { + return fmt.Errorf("resolve workspace hostname: %w", err) } - instance, getWorkspaceErr := c.localClient.GetWorkspace(ctx, c.workspace.UID) - // if we can't reach the daemon try to start the desktop app - if daemon.IsDaemonNotAvailableError(getWorkspaceErr) { - openErr := c.openDesktopApp() - if openErr != nil { - return getWorkspaceErr // inform user about daemon state - } - // give desktop app a chance to start - time.Sleep(2 * time.Second) + check := &workspaceReachabilityCheck{client: c, wAddr: wAddr, port: port, start: time.Now()} + pollErr := wait.PollUntilContextTimeout( + ctx, + 200*time.Millisecond, + checkWorkspaceReachableTimeout, + true, + check.step, + ) + if pollErr == nil { + log.Debugf("host %s is reachable, proceeding with SSH session", wAddr.Host()) + return nil + } - // let's try again - err = ts.WaitHostReachable(ctx, c.tsClient, wAddr, 2*time.Minute) - if err == nil { - return nil - } + instance, getWorkspaceErr := check.instance, check.getWorkspaceErr + if instance == nil && getWorkspaceErr == nil { instance, getWorkspaceErr = c.localClient.GetWorkspace(ctx, c.workspace.UID) } - - return c.workspaceUnreachableError(instance, getWorkspaceErr, err) + return c.workspaceUnreachableError(instance, getWorkspaceErr, check.reachErr) } func (c *client) SSHClients( diff --git a/pkg/ts/util.go b/pkg/ts/util.go index 63a0b46590..a28301dd0d 100644 --- a/pkg/ts/util.go +++ b/pkg/ts/util.go @@ -9,11 +9,9 @@ import ( "os" "path/filepath" "strings" - "time" "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/log" - "k8s.io/apimachinery/pkg/util/wait" "tailscale.com/client/local" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" @@ -51,39 +49,6 @@ func GetURL(host string, port int) string { return fmt.Sprintf("%s.%s:%d", host, DevsyTSNetDomain, port) } -// WaitHostReachable polls until the given host is reachable via ts, or -// timeout elapses. -func WaitHostReachable( - ctx context.Context, - lc *local.Client, - addr Addr, - timeout time.Duration, -) error { - port, err := ToUint16Port(addr.Port()) - if err != nil { - return fmt.Errorf("host %s: %w", addr.String(), err) - } - - var lastErr error - pollErr := wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, timeout, true, - func(ctx context.Context) (bool, error) { - dialCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - conn, dialErr := lc.DialTCP(dialCtx, addr.Host(), port) - if dialErr != nil { - lastErr = dialErr - log.Debugf("host %s not reachable: %v", addr.String(), dialErr) - return false, nil // keep polling; not fatal - } - _ = conn.Close() - return true, nil - }) - if pollErr != nil { - return fmt.Errorf("host %s not reachable: %w", addr.String(), lastErr) - } - return nil -} - // ToUint16Port validates that port fits the uint16 range TCP ports use. func ToUint16Port(port int) (uint16, error) { if port < 0 || port > math.MaxUint16 { From 90410070d52295d422dc7aa8a4b945c3ad9e6774 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 28 Aug 2026 14:20:00 -0500 Subject: [PATCH 13/17] style: comment cleanup -- add missing rationale, remove restating comments Added the one genuinely missing comment: watchNetmap/drainNotifications had no explanation for why notifications are drained on a separate goroutine (avoiding tailscaled's 128-entry IPN bus queue overflow while fetchStatus is in flight) -- the least obvious design decision in the file had zero documentation. Removed ten doc comments that only restated the function name and signature with no added information: NewWorkspaceServer, Dial, validateConfig, initTsServer, parseWorkspaceHostname, createListener (workspace_server.go), GetEnvOrDefault, RemoveProtocol (derp.go), WatchNetmap, netmapChanged (util.go). --- pkg/ts/derp.go | 2 -- pkg/ts/util.go | 7 ++++--- pkg/ts/workspace_server.go | 6 ------ 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/pkg/ts/derp.go b/pkg/ts/derp.go index 896a95bf75..9b91594fa9 100644 --- a/pkg/ts/derp.go +++ b/pkg/ts/derp.go @@ -47,7 +47,6 @@ func CheckDerpConnection(ctx context.Context, baseUrl *url.URL, insecure bool) e return nil } -// Utility function to get environment variable or default. func GetEnvOrDefault(envVar, defaultVal string) string { if val := os.Getenv(envVar); val != "" { return val @@ -55,7 +54,6 @@ func GetEnvOrDefault(envVar, defaultVal string) string { return defaultVal } -// RemoveProtocol removes protocol from URL. func RemoveProtocol(hostPath string) string { if _, after, ok := strings.Cut(hostPath, "://"); ok { return after diff --git a/pkg/ts/util.go b/pkg/ts/util.go index a28301dd0d..0e08d524d0 100644 --- a/pkg/ts/util.go +++ b/pkg/ts/util.go @@ -62,7 +62,6 @@ type ipnWatcher interface { Next() (ipn.Notify, error) } -// WatchNetmap invokes netmapChangedFn whenever the tailnet state changes. func WatchNetmap( ctx context.Context, lc *local.Client, @@ -80,6 +79,10 @@ func WatchNetmap( return watchNetmap(ctx, watcher, lc.Status, netmapChangedFn) } +// watchNetmap drains notifications on a separate goroutine (drainNotifications) +// so a burst never outruns tailscaled's 128-entry IPN bus queue while +// fetchStatus is in flight below; bursts are coalesced into a single +// follow-up fetch via the buffered trigger channel. func watchNetmap( ctx context.Context, watcher ipnWatcher, @@ -126,8 +129,6 @@ func drainNotifications(watcher ipnWatcher, trigger chan<- struct{}, errc chan<- } } -// netmapChanged reports whether a notification indicates a tailnet -// state change. func netmapChanged(n ipn.Notify) bool { return n.InitialStatus != nil || n.SelfChange != nil || len(n.PeersChanged) > 0 || len(n.PeersRemoved) > 0 || diff --git a/pkg/ts/workspace_server.go b/pkg/ts/workspace_server.go index 9156feb6c1..c286ca5b15 100644 --- a/pkg/ts/workspace_server.go +++ b/pkg/ts/workspace_server.go @@ -59,7 +59,6 @@ type WorkspaceServerConfig struct { Insecure bool } -// NewWorkspaceServer creates a new TSNet server instance. func NewWorkspaceServer(config *WorkspaceServerConfig) *WorkspaceServer { return &WorkspaceServer{ config: config, @@ -112,7 +111,6 @@ func (s *WorkspaceServer) Stop() { log.Info("Tailscale server stopped") } -// Dial dials the given address using the TSNet server. func (s *WorkspaceServer) Dial(ctx context.Context, network, addr string) (net.Conn, error) { if s.tsServer == nil { return nil, fmt.Errorf("tailscale server is not running") @@ -139,7 +137,6 @@ func (s *WorkspaceServer) setupTSNet(ctx context.Context) (workspace, project st return s.parseWorkspaceHostname() } -// validateConfig ensures required configuration values are set. func (s *WorkspaceServer) validateConfig() error { if s.config.AccessKey == "" || s.config.PlatformHost == "" || s.config.WorkspaceHost == "" { return fmt.Errorf("access key, host, or hostname cannot be empty") @@ -159,7 +156,6 @@ func (s *WorkspaceServer) setupControlURL(ctx context.Context) (*url.URL, error) return baseURL, nil } -// initTsServer initializes the TSNet server. func (s *WorkspaceServer) initTsServer(ctx context.Context, controlURL *url.URL) error { store, _ := mem.New(s.config.LogF, "") if s.config.Insecure { @@ -181,7 +177,6 @@ func (s *WorkspaceServer) initTsServer(ctx context.Context, controlURL *url.URL) return nil } -// parseWorkspaceHostname extracts workspace and project names from the hostname. func (s *WorkspaceServer) parseWorkspaceHostname() (workspace, project string, err error) { parts := strings.Split(s.config.WorkspaceHost, ".") if len(parts) < 4 { @@ -293,7 +288,6 @@ func serveMux(listener net.Listener, mux *http.ServeMux, errFormat string) { } } -// createListener creates a listener bound to the TSNet server. func (s *WorkspaceServer) createListener(addr string) (net.Listener, error) { l, err := s.tsServer.Listen("tcp", addr) if err != nil { From f21165d1065ac1e0d4eed4c95d03ce574704f685 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 28 Aug 2026 15:39:36 -0500 Subject: [PATCH 14/17] fix: refresh workspace state after desktop-app launch before final error CheckWorkspaceReachable's step() only checks daemon health and launches the desktop app once, ~30s into the poll, caching getWorkspaceErr/ instance at that point. If the remaining ~120s budget then times out, the final error used that stale daemon-not-available error instead of querying the workspace's current state -- by then the daemon has usually restarted, and the real cause (e.g. workspace actually stopped) was never reported. The pre-refactor code re-queried GetWorkspace a second time after its second retry phase failed; the single-poll collapse dropped that second query. Re-query whenever desktopOpened is true, not just when the cached state is nil. CodeRabbit review on 90410070: a second "Critical" finding claiming DevsyTSNetDomain was removed while GetURL still references it is a false positive -- verified the constant is still declared (workspace_server.go:32) and go build ./... passes clean. --- pkg/client/clientimplementation/daemonclient/client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/client/clientimplementation/daemonclient/client.go b/pkg/client/clientimplementation/daemonclient/client.go index 2b457e33fb..b6eff61d3e 100644 --- a/pkg/client/clientimplementation/daemonclient/client.go +++ b/pkg/client/clientimplementation/daemonclient/client.go @@ -218,7 +218,9 @@ func (c *client) CheckWorkspaceReachable(ctx context.Context) error { } instance, getWorkspaceErr := check.instance, check.getWorkspaceErr - if instance == nil && getWorkspaceErr == nil { + // check.getWorkspaceErr predates the desktop-app launch; re-query once the + // daemon has had a chance to come back so the reported cause is current. + if check.desktopOpened || (instance == nil && getWorkspaceErr == nil) { instance, getWorkspaceErr = c.localClient.GetWorkspace(ctx, c.workspace.UID) } return c.workspaceUnreachableError(instance, getWorkspaceErr, check.reachErr) From b954b8aef9fbf89ef6fad6c4dded09d5283f242b Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 28 Aug 2026 15:47:58 -0500 Subject: [PATCH 15/17] refactor: split workspaceReachabilityCheck by responsibility step() mixed three unrelated subsystems in one function: tailscale dial reachability (network), daemon health via GetWorkspace (local platform API), and launching the desktop app (OS-level side effect). Split into: - workspaceDialer: dial reachability only, no daemon/desktop knowledge. - daemonRecovery: daemon-health-check-and-launch-desktop-app decision only, no dial knowledge. attempt() is idempotent once opened is set. - workspaceReachabilityCheck: thin orchestrator wiring the two together for the poll step. Each type now has exactly one reason to change. No behavior change -- CheckWorkspaceReachable's poll semantics, timeouts, and the stale-getWorkspaceErr re-query fix are unchanged, just re-homed onto check.recovery. --- .../daemonclient/client.go | 99 ++++++++++++------- 1 file changed, 65 insertions(+), 34 deletions(-) diff --git a/pkg/client/clientimplementation/daemonclient/client.go b/pkg/client/clientimplementation/daemonclient/client.go index b6eff61d3e..b6b8117984 100644 --- a/pkg/client/clientimplementation/daemonclient/client.go +++ b/pkg/client/clientimplementation/daemonclient/client.go @@ -51,49 +51,76 @@ const daemonHealthCheckDelay = 30 * time.Second // desktop app itself failed. var errGiveUpReachingWorkspace = errors.New("give up reaching workspace") -// workspaceReachabilityCheck holds CheckWorkspaceReachable's poll state -// across attempts: the last dial error, whether the desktop app has already -// been launched, and whatever workspace status that launch decision was -// based on (reused in the final error if every attempt fails). -type workspaceReachabilityCheck struct { +// workspaceDialer checks tailnet reachability only -- no daemon-health or +// desktop-app concerns. +type workspaceDialer struct { + tsClient *local.Client + wAddr ts.Addr + port uint16 +} + +func (d workspaceDialer) reachable(ctx context.Context) error { + dialCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + conn, err := d.tsClient.DialTCP(dialCtx, d.wAddr.Host(), d.port) + if err != nil { + return err + } + _ = conn.Close() + return nil +} + +// daemonRecovery decides, once daemonHealthCheckDelay has passed without a +// reachable host, whether the daemon itself is down and, if so, launches the +// desktop app to restart it -- no network-reachability concerns. attempt is +// idempotent after opened is set: later calls are no-ops until the daemon's +// state needs re-checking (which the caller does once polling ends). +type daemonRecovery struct { client *client - wAddr ts.Addr - port uint16 start time.Time - desktopOpened bool - reachErr error + opened bool getWorkspaceErr error instance *managementv1.DevsyWorkspaceInstance } -// step is one poll attempt: dial the workspace, and once -// daemonHealthCheckDelay has passed without success, check whether the -// daemon itself is down and launch the desktop app to restart it. -func (r *workspaceReachabilityCheck) step(ctx context.Context) (bool, error) { - dialCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - conn, dialErr := r.client.tsClient.DialTCP(dialCtx, r.wAddr.Host(), r.port) - if dialErr == nil { - _ = conn.Close() - return true, nil - } - r.reachErr = dialErr - - if r.desktopOpened || time.Since(r.start) < daemonHealthCheckDelay { - return false, nil +// attempt returns errGiveUpReachingWorkspace once retrying can no longer +// help: the daemon is confirmed running (so an unreachable host has some +// other cause), or launching the desktop app itself failed. +func (r *daemonRecovery) attempt(ctx context.Context) error { + if r.opened || time.Since(r.start) < daemonHealthCheckDelay { + return nil } r.instance, r.getWorkspaceErr = r.client.localClient.GetWorkspace(ctx, r.client.workspace.UID) if !daemon.IsDaemonNotAvailableError(r.getWorkspaceErr) { - // Daemon is reachable (or failed for an unrelated reason); no amount - // of retrying fixes an unreachable host in that case. - return false, errGiveUpReachingWorkspace + return errGiveUpReachingWorkspace } - r.desktopOpened = true + r.opened = true if openErr := r.client.openDesktopApp(); openErr != nil { - return false, errGiveUpReachingWorkspace // inform user about daemon state + return errGiveUpReachingWorkspace // inform user about daemon state + } + return nil +} + +// workspaceReachabilityCheck is CheckWorkspaceReachable's poll step: dial, +// and on failure hand off to daemonRecovery. +type workspaceReachabilityCheck struct { + dial workspaceDialer + recovery *daemonRecovery + reachErr error +} + +func (r *workspaceReachabilityCheck) step(ctx context.Context) (bool, error) { + dialErr := r.dial.reachable(ctx) + if dialErr == nil { + return true, nil + } + r.reachErr = dialErr + + if err := r.recovery.attempt(ctx); err != nil { + return false, err } return false, nil } @@ -204,7 +231,10 @@ func (c *client) CheckWorkspaceReachable(ctx context.Context) error { return fmt.Errorf("resolve workspace hostname: %w", err) } - check := &workspaceReachabilityCheck{client: c, wAddr: wAddr, port: port, start: time.Now()} + check := &workspaceReachabilityCheck{ + dial: workspaceDialer{tsClient: c.tsClient, wAddr: wAddr, port: port}, + recovery: &daemonRecovery{client: c, start: time.Now()}, + } pollErr := wait.PollUntilContextTimeout( ctx, 200*time.Millisecond, @@ -217,10 +247,11 @@ func (c *client) CheckWorkspaceReachable(ctx context.Context) error { return nil } - instance, getWorkspaceErr := check.instance, check.getWorkspaceErr - // check.getWorkspaceErr predates the desktop-app launch; re-query once the - // daemon has had a chance to come back so the reported cause is current. - if check.desktopOpened || (instance == nil && getWorkspaceErr == nil) { + instance, getWorkspaceErr := check.recovery.instance, check.recovery.getWorkspaceErr + // check.recovery's cached state predates the desktop-app launch; re-query + // once the daemon has had a chance to come back so the reported cause is + // current. + if check.recovery.opened || (instance == nil && getWorkspaceErr == nil) { instance, getWorkspaceErr = c.localClient.GetWorkspace(ctx, c.workspace.UID) } return c.workspaceUnreachableError(instance, getWorkspaceErr, check.reachErr) From 46d6d0d4569945ab8c1c280eff15c92b9a4cc0c3 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 28 Aug 2026 15:51:28 -0500 Subject: [PATCH 16/17] style: restore missing rationale on the burst-drain regression test Audited every file this PR touches (scoped to each file's actual diff against main, not just current content) against the self-documenting/ minimal-comments standard: cmd/internal/agentcontainer/daemon.go and pkg/daemon/platform/ts_server.go have only 1-2 line diffs that don't touch any comments, so left as-is (out of this PR's scope). pkg/ts/ {util,derp,workspace_server}.go and daemonclient/client.go were already clean from prior passes. Found one real gap: fakeIPNWatcher and TestWatchNetmapDrainsBurstWhileStatusFetchBlocked (util_test.go) are new code from this PR and had zero comments -- the non-obvious "why" (afterFirst exists specifically to force the burst to race a blocked status fetch, per the earlier CodeRabbit-caught timing bug) wasn't otherwise derivable. Added two one-line comments; no other changes. --- pkg/ts/util_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/ts/util_test.go b/pkg/ts/util_test.go index 395f037a28..4b1210cdda 100644 --- a/pkg/ts/util_test.go +++ b/pkg/ts/util_test.go @@ -41,6 +41,8 @@ func selfChangeNotifs(n int) []ipn.Notify { return notifs } +// fakeIPNWatcher blocks Next's second call on afterFirst so a test can force +// a burst to arrive while a status fetch is still in flight. type fakeIPNWatcher struct { notifs []ipn.Notify idx atomic.Int32 @@ -61,6 +63,8 @@ func (w *fakeIPNWatcher) Next() (ipn.Notify, error) { return ipn.Notify{}, errors.New("watcher closed") } +// Regression test for the burst-drain/coalescing bug: draining must not +// stall behind a blocked status fetch. func TestWatchNetmapDrainsBurstWhileStatusFetchBlocked(t *testing.T) { const burst = 129 From fd445ebf2c5cef33fcd316224b096369cb4e28bc Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 28 Aug 2026 17:46:08 -0500 Subject: [PATCH 17/17] fix: don't abandon reachability poll on a transient GetWorkspace error daemonRecovery.attempt treated any GetWorkspace error other than DaemonNotAvailableError as terminal (errGiveUpReachingWorkspace), including plain backend errors (a non-200 HTTP status, a JSON decode failure) that only mean the daemon's own call to the management API hiccuped -- not that the daemon is confirmed healthy and the host is unreachable for some unfixable reason. A single transient failure could throw away the remaining ~120s of dial budget. This pre-existed the single-poll refactor (the old two-phase code made the same one-shot classification after its own first WaitHostReachable call) rather than being a regression, but it's a real bug: separate three outcomes now -- GetWorkspace succeeding (daemon up, reported the actual state, genuinely terminal), DaemonNotAvailableError (launch the app), and any other error (inconclusive, re-check after another daemonHealthCheckDelay instead of giving up). Re-checks are throttled via lastCheck so a recurring transient error can't turn into a 200ms-interval hot loop against the daemon's HTTP endpoint. --- .../daemonclient/client.go | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/pkg/client/clientimplementation/daemonclient/client.go b/pkg/client/clientimplementation/daemonclient/client.go index b6b8117984..835be374fe 100644 --- a/pkg/client/clientimplementation/daemonclient/client.go +++ b/pkg/client/clientimplementation/daemonclient/client.go @@ -39,10 +39,12 @@ import ( const checkWorkspaceReachableTimeout = 150 * time.Second // daemonHealthCheckDelay is how long CheckWorkspaceReachable retries a plain -// dial before it checks whether the daemon itself is down and, if so, +// dial before it first checks whether the daemon itself is down and, if so, // launches the desktop app to restart it. Checking immediately would fire on // every transient dial failure; the delay gives a live daemon a chance to -// become reachable on its own first. +// become reachable on its own first. Also the minimum interval between +// re-checks after an inconclusive one (see attempt), so a recurring +// transient GetWorkspace error can't turn into a hot loop. const daemonHealthCheckDelay = 30 * time.Second // errGiveUpReachingWorkspace stops CheckWorkspaceReachable's poll early once @@ -80,22 +82,38 @@ type daemonRecovery struct { start time.Time opened bool + lastCheck time.Time getWorkspaceErr error instance *managementv1.DevsyWorkspaceInstance } // attempt returns errGiveUpReachingWorkspace once retrying can no longer -// help: the daemon is confirmed running (so an unreachable host has some -// other cause), or launching the desktop app itself failed. +// help: GetWorkspace succeeded (so the daemon is up and reported the actual, +// unfixable-by-retrying workspace state), or launching the desktop app +// failed. A GetWorkspace error other than DaemonNotAvailableError (e.g. a +// transient backend error) is inconclusive, not terminal: re-check after +// another daemonHealthCheckDelay instead of giving up. func (r *daemonRecovery) attempt(ctx context.Context) error { - if r.opened || time.Since(r.start) < daemonHealthCheckDelay { + if r.opened { return nil } + now := time.Now() + nextCheck := r.start.Add(daemonHealthCheckDelay) + if !r.lastCheck.IsZero() { + nextCheck = r.lastCheck.Add(daemonHealthCheckDelay) + } + if now.Before(nextCheck) { + return nil + } + r.lastCheck = now r.instance, r.getWorkspaceErr = r.client.localClient.GetWorkspace(ctx, r.client.workspace.UID) - if !daemon.IsDaemonNotAvailableError(r.getWorkspaceErr) { + if r.getWorkspaceErr == nil { return errGiveUpReachingWorkspace } + if !daemon.IsDaemonNotAvailableError(r.getWorkspaceErr) { + return nil + } r.opened = true if openErr := r.client.openDesktopApp(); openErr != nil {