diff --git a/cmd/internal/agentcontainer/daemon.go b/cmd/internal/agentcontainer/daemon.go index bfd74e1e97..41d017153c 100644 --- a/cmd/internal/agentcontainer/daemon.go +++ b/cmd/internal/agentcontainer/daemon.go @@ -248,8 +248,8 @@ func runNetworkServer( AccessKey: cmd.Config.Platform.AccessKey, PlatformHost: ts.RemoveProtocol(cmd.Config.Platform.PlatformHost), WorkspaceHost: cmd.Config.Platform.WorkspaceHost, - Client: baseClient, RootDir: RootDir, + Insecure: true, LogF: func(format string, args ...any) { log.Infof(format, args...) }, diff --git a/pkg/client/clientimplementation/daemonclient/client.go b/pkg/client/clientimplementation/daemonclient/client.go index c82733e1bd..835be374fe 100644 --- a/pkg/client/clientimplementation/daemonclient/client.go +++ b/pkg/client/clientimplementation/daemonclient/client.go @@ -27,10 +27,122 @@ import ( sshServer "github.com/devsy-org/devsy/pkg/ssh/server" "github.com/devsy-org/devsy/pkg/ts" "golang.org/x/crypto/ssh" + "k8s.io/apimachinery/pkg/util/wait" "tailscale.com/client/local" "tailscale.com/tailcfg" ) +// checkWorkspaceReachableTimeout is CheckWorkspaceReachable's overall retry +// budget, covering both the initial reachability check and, if the daemon +// needed restarting, however long the desktop app takes to bring the +// workspace back up. +const checkWorkspaceReachableTimeout = 150 * time.Second + +// daemonHealthCheckDelay is how long CheckWorkspaceReachable retries a plain +// dial before it first checks whether the daemon itself is down and, if so, +// launches the desktop app to restart it. Checking immediately would fire on +// every transient dial failure; the delay gives a live daemon a chance to +// become reachable on its own first. Also the minimum interval between +// re-checks after an inconclusive one (see attempt), so a recurring +// transient GetWorkspace error can't turn into a hot loop. +const daemonHealthCheckDelay = 30 * time.Second + +// errGiveUpReachingWorkspace stops CheckWorkspaceReachable's poll early once +// retrying can no longer help: either the daemon is confirmed to be running +// and the host is unreachable for some other reason, or launching the +// desktop app itself failed. +var errGiveUpReachingWorkspace = errors.New("give up reaching workspace") + +// workspaceDialer checks tailnet reachability only -- no daemon-health or +// desktop-app concerns. +type workspaceDialer struct { + tsClient *local.Client + wAddr ts.Addr + port uint16 +} + +func (d workspaceDialer) reachable(ctx context.Context) error { + dialCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + conn, err := d.tsClient.DialTCP(dialCtx, d.wAddr.Host(), d.port) + if err != nil { + return err + } + _ = conn.Close() + return nil +} + +// daemonRecovery decides, once daemonHealthCheckDelay has passed without a +// reachable host, whether the daemon itself is down and, if so, launches the +// desktop app to restart it -- no network-reachability concerns. attempt is +// idempotent after opened is set: later calls are no-ops until the daemon's +// state needs re-checking (which the caller does once polling ends). +type daemonRecovery struct { + client *client + start time.Time + + opened bool + lastCheck time.Time + getWorkspaceErr error + instance *managementv1.DevsyWorkspaceInstance +} + +// attempt returns errGiveUpReachingWorkspace once retrying can no longer +// help: GetWorkspace succeeded (so the daemon is up and reported the actual, +// unfixable-by-retrying workspace state), or launching the desktop app +// failed. A GetWorkspace error other than DaemonNotAvailableError (e.g. a +// transient backend error) is inconclusive, not terminal: re-check after +// another daemonHealthCheckDelay instead of giving up. +func (r *daemonRecovery) attempt(ctx context.Context) error { + if r.opened { + return nil + } + now := time.Now() + nextCheck := r.start.Add(daemonHealthCheckDelay) + if !r.lastCheck.IsZero() { + nextCheck = r.lastCheck.Add(daemonHealthCheckDelay) + } + if now.Before(nextCheck) { + return nil + } + r.lastCheck = now + + r.instance, r.getWorkspaceErr = r.client.localClient.GetWorkspace(ctx, r.client.workspace.UID) + if r.getWorkspaceErr == nil { + return errGiveUpReachingWorkspace + } + if !daemon.IsDaemonNotAvailableError(r.getWorkspaceErr) { + return nil + } + + r.opened = true + if openErr := r.client.openDesktopApp(); openErr != nil { + return errGiveUpReachingWorkspace // inform user about daemon state + } + return nil +} + +// workspaceReachabilityCheck is CheckWorkspaceReachable's poll step: dial, +// and on failure hand off to daemonRecovery. +type workspaceReachabilityCheck struct { + dial workspaceDialer + recovery *daemonRecovery + reachErr error +} + +func (r *workspaceReachabilityCheck) step(ctx context.Context) (bool, error) { + dialErr := r.dial.reachable(ctx) + if dialErr == nil { + return true, nil + } + r.reachErr = dialErr + + if err := r.recovery.attempt(ctx); err != nil { + return false, err + } + return false, nil +} + func New( devsyConfig *config.Config, prov *provider.ProviderConfig, @@ -132,31 +244,35 @@ func (c *client) CheckWorkspaceReachable(ctx context.Context) error { if err != nil { return fmt.Errorf("resolve workspace hostname: %w", err) } - err = ts.WaitHostReachable(ctx, c.tsClient, wAddr, 5) - if err == nil { - log.Debugf("Host %s is reachable. Proceeding with SSH session", wAddr.Host()) - return nil + port, err := wAddr.PortUint16() + if err != nil { + return fmt.Errorf("resolve workspace hostname: %w", err) } - instance, getWorkspaceErr := c.localClient.GetWorkspace(ctx, c.workspace.UID) - // if we can't reach the daemon try to start the desktop app - if daemon.IsDaemonNotAvailableError(getWorkspaceErr) { - openErr := c.openDesktopApp() - if openErr != nil { - return getWorkspaceErr // inform user about daemon state - } - // give desktop app a chance to start - time.Sleep(2 * time.Second) + check := &workspaceReachabilityCheck{ + dial: workspaceDialer{tsClient: c.tsClient, wAddr: wAddr, port: port}, + recovery: &daemonRecovery{client: c, start: time.Now()}, + } + pollErr := wait.PollUntilContextTimeout( + ctx, + 200*time.Millisecond, + checkWorkspaceReachableTimeout, + true, + check.step, + ) + if pollErr == nil { + log.Debugf("host %s is reachable, proceeding with SSH session", wAddr.Host()) + return nil + } - // let's try again - err = ts.WaitHostReachable(ctx, c.tsClient, wAddr, 20) - if err == nil { - return nil - } + instance, getWorkspaceErr := check.recovery.instance, check.recovery.getWorkspaceErr + // check.recovery's cached state predates the desktop-app launch; re-query + // once the daemon has had a chance to come back so the reported cause is + // current. + if check.recovery.opened || (instance == nil && getWorkspaceErr == nil) { instance, getWorkspaceErr = c.localClient.GetWorkspace(ctx, c.workspace.UID) } - - return c.workspaceUnreachableError(instance, getWorkspaceErr, err) + return c.workspaceUnreachableError(instance, getWorkspaceErr, check.reachErr) } func (c *client) SSHClients( @@ -179,15 +295,23 @@ func (c *client) SSHClients( if err != nil { return nil, fmt.Errorf("invalid port: %s", addressParts[1]) } + port16, err := ts.ToUint16Port(port) + if err != nil { + return nil, fmt.Errorf("invalid port: %w", err) + } - return c.tsClient.DialTCP(ctx, addressParts[0], uint16(port)) + return c.tsClient.DialTCP(ctx, addressParts[0], port16) } - toolClient, err = ts.WaitForSSHClient(ctx, dial, "tcp", address, "root", time.Second*10) + toolClient, err = ts.WaitForSSHClient(ctx, ts.SSHDialConfig{ + Dialer: dial, Network: "tcp", Address: address, User: "root", Timeout: time.Second * 10, + }) if err != nil { return nil, nil, fmt.Errorf("create SSH tool client: %w", err) } - userClient, err = ts.WaitForSSHClient(ctx, dial, "tcp", address, user, time.Second*10) + userClient, err = ts.WaitForSSHClient(ctx, ts.SSHDialConfig{ + Dialer: dial, Network: "tcp", Address: address, User: user, Timeout: time.Second * 10, + }) if err != nil { return nil, nil, fmt.Errorf("create SSH user client: %w", err) } @@ -200,7 +324,11 @@ func (c *client) DirectTunnel(ctx context.Context, stdin io.Reader, stdout io.Wr if err != nil { return fmt.Errorf("resolve workspace hostname: %w", err) } - conn, err := c.tsClient.DialTCP(ctx, wAddr.Host(), uint16(wAddr.Port())) + port, err := wAddr.PortUint16() + if err != nil { + return fmt.Errorf("invalid workspace port: %w", err) + } + conn, err := c.tsClient.DialTCP(ctx, wAddr.Host(), port) if err != nil { return fmt.Errorf("failed to connect to SSH server in proxy mode: %w", err) } diff --git a/pkg/daemon/platform/daemon.go b/pkg/daemon/platform/daemon.go index 0e3fdc2b11..98151730d8 100644 --- a/pkg/daemon/platform/daemon.go +++ b/pkg/daemon/platform/daemon.go @@ -3,13 +3,10 @@ package daemon import ( "bufio" "context" - "encoding/json" "errors" "fmt" "io" "net" - "os" - "path/filepath" "time" "github.com/devsy-org/devsy/pkg/config" @@ -17,8 +14,8 @@ import ( "github.com/devsy-org/devsy/pkg/platform/client" "github.com/devsy-org/devsy/pkg/ts" "tailscale.com/client/local" + "tailscale.com/ipn/ipnstate" "tailscale.com/tsnet" - "tailscale.com/types/netmap" ) type Daemon struct { @@ -166,13 +163,8 @@ func (d *Daemon) watchNetmap(ctx context.Context) error { return err } - return ts.WatchNetmap(ctx, lc, func(netMap *netmap.NetworkMap) { - nm, err := json.Marshal(netMap) - if err != nil { - log.Errorf("Failed to marshal netmap: %v", err) - } else { - _ = os.WriteFile(filepath.Join(d.rootDir, "netmap.json"), nm, 0o644) - } + return ts.WatchNetmap(ctx, lc, func(status *ipnstate.Status) { + ts.PersistNetmapStatus(d.rootDir, status) }) } diff --git a/pkg/daemon/platform/ts_server.go b/pkg/daemon/platform/ts_server.go index 6a93ec79c7..5694116b48 100644 --- a/pkg/daemon/platform/ts_server.go +++ b/pkg/daemon/platform/ts_server.go @@ -26,7 +26,7 @@ func newTSServer( Scheme: ts.GetEnvOrDefault("DEVSY_TSNET_SCHEME", "https"), Host: ts.RemoveProtocol(host), } - if err := ts.CheckDerpConnection(ctx, &baseUrl); err != nil { + if err := ts.CheckDerpConnection(ctx, &baseUrl, insecure); err != nil { return nil, nil, fmt.Errorf("failed to verify DERP connection: %w", err) } if insecure { diff --git a/pkg/ts/addr.go b/pkg/ts/addr.go index b2de733ce8..cb83fdf51a 100644 --- a/pkg/ts/addr.go +++ b/pkg/ts/addr.go @@ -28,3 +28,9 @@ func (a Addr) Host() string { func (a Addr) Port() int { return a.port } + +// PortUint16 returns Port bounds-checked for use with APIs that take a +// uint16 port, such as *local.Client.DialTCP. +func (a Addr) PortUint16() (uint16, error) { + return ToUint16Port(a.port) +} diff --git a/pkg/ts/derp.go b/pkg/ts/derp.go index 9370f9a22a..9b91594fa9 100644 --- a/pkg/ts/derp.go +++ b/pkg/ts/derp.go @@ -11,17 +11,18 @@ import ( "time" ) -type ConnTrackingFunc func(address string) - -// CheckDerpConnection validates the DERP connection. -func CheckDerpConnection(ctx context.Context, baseUrl *url.URL) error { - newTransport := http.DefaultTransport.(*http.Transport).Clone() - newTransport.TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, +// CheckDerpConnection validates the DERP connection. insecure skips TLS +// certificate verification, for coordinators known to use self-signed certs. +func CheckDerpConnection(ctx context.Context, baseUrl *url.URL, insecure bool) error { + transport := http.DefaultTransport.(*http.Transport).Clone() + if insecure { + transport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } // #nosec G402 -- opt-in via explicit insecure flag } client := &http.Client{ - Transport: newTransport, + Transport: transport, Timeout: 5 * time.Second, } @@ -34,14 +35,18 @@ func CheckDerpConnection(ctx context.Context, baseUrl *url.URL) error { } res, err := client.Do(req) - if err != nil || (res != nil && res.StatusCode != http.StatusOK) { + if err != nil { return fmt.Errorf("failed to reach the coordinator server: %w", err) } + defer func() { _ = res.Body.Close() }() + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("coordinator server returned status %d", res.StatusCode) + } return nil } -// Utility function to get environment variable or default. func GetEnvOrDefault(envVar, defaultVal string) string { if val := os.Getenv(envVar); val != "" { return val @@ -49,7 +54,6 @@ func GetEnvOrDefault(envVar, defaultVal string) string { return defaultVal } -// RemoveProtocol removes protocol from URL. func RemoveProtocol(hostPath string) string { if _, after, ok := strings.Cut(hostPath, "://"); ok { return after diff --git a/pkg/ts/ssh.go b/pkg/ts/ssh.go index e6357c0898..2844e343da 100644 --- a/pkg/ts/ssh.go +++ b/pkg/ts/ssh.go @@ -12,22 +12,26 @@ import ( type Dialer func(ctx context.Context, network, address string) (net.Conn, error) -func WaitForSSHClient( - ctx context.Context, - dialer Dialer, - network, address string, - user string, - timeout time.Duration, -) (*ssh.Client, error) { - deadline := time.Now().Add(timeout) +// SSHDialConfig bundles the parameters needed to establish and retry an SSH +// connection over a tailnet dialer. +type SSHDialConfig struct { + Dialer Dialer + Network string + Address string + User string + Timeout time.Duration +} + +func WaitForSSHClient(ctx context.Context, cfg SSHDialConfig) (*ssh.Client, error) { + deadline := time.Now().Add(cfg.Timeout) var ( c *ssh.Client err error ) - log.Debugf("Attempting to establish SSH connection with %s as user %s", address, user) + log.Debugf("attempting to establish SSH connection with %s as user %s", cfg.Address, cfg.User) for time.Now().Before(deadline) { - c, err = newSSHClient(ctx, dialer, network, address, user) + c, err = newSSHClient(ctx, cfg) if err == nil { return c, nil } @@ -38,29 +42,25 @@ func WaitForSSHClient( time.Sleep(100 * time.Millisecond) } } - log.Debugf("Failed to establish SSH connection %v", err) + log.Debugf("failed to establish SSH connection %v", err) return c, err } -func newSSHClient( - ctx context.Context, - dialer Dialer, - network, address string, - user string, -) (*ssh.Client, error) { - conn, err := dialer(ctx, network, address) +func newSSHClient(ctx context.Context, cfg SSHDialConfig) (*ssh.Client, error) { + conn, err := cfg.Dialer(ctx, cfg.Network, cfg.Address) if err != nil { - return nil, fmt.Errorf("dial %s: %w", address, err) + return nil, fmt.Errorf("dial %s: %w", cfg.Address, err) } clientConfig := &ssh.ClientConfig{ - User: user, - Auth: []ssh.AuthMethod{}, // The SSH server is only reachable through the tailnet + User: cfg.User, + Auth: []ssh.AuthMethod{}, // The SSH server is only reachable through the tailnet + // #nosec G106 -- the tailnet, not TLS/host-key trust, is this connection's security boundary HostKeyCallback: ssh.InsecureIgnoreHostKey(), } - sshConn, channels, requests, err := ssh.NewClientConn(conn, address, clientConfig) + sshConn, channels, requests, err := ssh.NewClientConn(conn, cfg.Address, clientConfig) if err != nil { return nil, fmt.Errorf("establish SSH connection: %w", err) } diff --git a/pkg/ts/util.go b/pkg/ts/util.go index bc148846c5..0e08d524d0 100644 --- a/pkg/ts/util.go +++ b/pkg/ts/util.go @@ -2,24 +2,21 @@ package ts import ( "context" + "encoding/json" "errors" "fmt" + "math" "os" + "path/filepath" "strings" - "time" "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/log" "tailscale.com/client/local" "tailscale.com/ipn" - "tailscale.com/types/netmap" + "tailscale.com/ipn/ipnstate" ) -// DevsyTSNetDomain is the MagicDNS suffix for the Devsy tailnet. -// The literal value matches the configured Tailscale tailnet and must stay -// in sync with operator network configuration. -const DevsyTSNetDomain = "ts.loft" - func GetClientHostname(userName string) (string, error) { osHostname, err := os.Hostname() if err != nil { @@ -52,62 +49,106 @@ func GetURL(host string, port int) string { return fmt.Sprintf("%s.%s:%d", host, DevsyTSNetDomain, port) } -// WaitHostReachable polls until the given host is reachable via ts. -func WaitHostReachable( - ctx context.Context, - lc *local.Client, - addr Addr, - maxRetries int, -) error { - for i := range maxRetries { - timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - conn, err := lc.DialTCP(timeoutCtx, addr.Host(), uint16(addr.Port())) - if err == nil { - _ = conn.Close() - return nil // Host is reachable - } - log.Debugf("Host %s not reachable, retrying (%d/%d)", addr.String(), i+1, maxRetries) - time.Sleep(200 * time.Millisecond) - - select { - case <-ctx.Done(): - return ctx.Err() - default: - } +// ToUint16Port validates that port fits the uint16 range TCP ports use. +func ToUint16Port(port int) (uint16, error) { + if port < 0 || port > math.MaxUint16 { + return 0, fmt.Errorf("port %d out of range", port) } + return uint16(port), nil // #nosec G115 -- bounds-checked above +} - return fmt.Errorf("host %s not reachable", addr.String()) +// ipnWatcher is the subset of *local.IPNBusWatcher used by watchNetmap. +type ipnWatcher interface { + Next() (ipn.Notify, error) } func WatchNetmap( ctx context.Context, lc *local.Client, - netmapChangedFn func(nm *netmap.NetworkMap), + netmapChangedFn func(status *ipnstate.Status), ) error { watcher, err := lc.WatchIPNBus( ctx, - ipn.NotifyInitialNetMap|ipn.NotifyRateLimit|ipn.NotifyWatchEngineUpdates, + ipn.NotifyInitialStatus|ipn.NotifyWatchEngineUpdates|ipn.NotifyPeerChanges, ) if err != nil { return err } defer func() { _ = watcher.Close() }() - var netMap *netmap.NetworkMap + return watchNetmap(ctx, watcher, lc.Status, netmapChangedFn) +} + +// watchNetmap drains notifications on a separate goroutine (drainNotifications) +// so a burst never outruns tailscaled's 128-entry IPN bus queue while +// fetchStatus is in flight below; bursts are coalesced into a single +// follow-up fetch via the buffered trigger channel. +func watchNetmap( + ctx context.Context, + watcher ipnWatcher, + fetchStatus func(context.Context) (*ipnstate.Status, error), + netmapChangedFn func(status *ipnstate.Status), +) error { + trigger := make(chan struct{}, 1) + errc := make(chan error, 1) + + go drainNotifications(watcher, trigger, errc) + + for { + select { + case err := <-errc: + return err + case <-trigger: + status, err := fetchStatus(ctx) + if err != nil { + return fmt.Errorf("fetch status: %w", err) + } + netmapChangedFn(status) + } + } +} + +func drainNotifications(watcher ipnWatcher, trigger chan<- struct{}, errc chan<- error) { for { n, err := watcher.Next() if err != nil { - return fmt.Errorf("watch ipn: %w", err) + errc <- fmt.Errorf("watch ipn: %w", err) + return } if n.ErrMessage != nil { - return fmt.Errorf("tailscale error: %w", errors.New(*n.ErrMessage)) + errc <- fmt.Errorf("tailscale error: %w", errors.New(*n.ErrMessage)) + return } - if n.NetMap != nil { - if n.NetMap != netMap { - netMap = n.NetMap - netmapChangedFn(netMap) - } + if !netmapChanged(n) { + continue + } + select { + case trigger <- struct{}{}: + default: } } } + +func netmapChanged(n ipn.Notify) bool { + return n.InitialStatus != nil || n.SelfChange != nil || + len(n.PeersChanged) > 0 || len(n.PeersRemoved) > 0 || + len(n.PeerChangedPatch) > 0 +} + +// netmapFileName is the debug snapshot both the daemon and workspace +// server's WatchNetmap callbacks write on every tailnet state change. +const netmapFileName = "netmap.json" + +// PersistNetmapStatus marshals status and writes it to netmap.json under +// rootDir for debugging, logging rather than returning any failure since +// callers invoke this from a WatchNetmap callback with no error path. +func PersistNetmapStatus(rootDir string, status *ipnstate.Status) { + nm, err := json.Marshal(status) + if err != nil { + log.Errorf("failed to marshal netmap: %v", err) + return + } + if err := os.WriteFile(filepath.Join(rootDir, netmapFileName), nm, 0o600); err != nil { + log.Errorf("failed to write netmap: %v", err) + } +} diff --git a/pkg/ts/util_test.go b/pkg/ts/util_test.go new file mode 100644 index 0000000000..4b1210cdda --- /dev/null +++ b/pkg/ts/util_test.go @@ -0,0 +1,183 @@ +package ts + +import ( + "context" + "errors" + "math" + "sync/atomic" + "testing" + "time" + + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" +) + +func waitUntil(t *testing.T, timeout time.Duration, msg string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for !cond() { + if time.Now().After(deadline) { + t.Fatal(msg) + } + time.Sleep(time.Millisecond) + } +} + +func waitFor(t *testing.T, timeout time.Duration, msg string, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(timeout): + t.Fatal(msg) + } +} + +func selfChangeNotifs(n int) []ipn.Notify { + notifs := make([]ipn.Notify, n) + for i := range notifs { + notifs[i] = ipn.Notify{SelfChange: &tailcfg.Node{}} + } + return notifs +} + +// fakeIPNWatcher blocks Next's second call on afterFirst so a test can force +// a burst to arrive while a status fetch is still in flight. +type fakeIPNWatcher struct { + notifs []ipn.Notify + idx atomic.Int32 + afterFirst chan struct{} + closed chan struct{} +} + +func (w *fakeIPNWatcher) Next() (ipn.Notify, error) { + i := w.idx.Load() + if i == 1 { + <-w.afterFirst + } + if int(i) < len(w.notifs) { + w.idx.Add(1) + return w.notifs[i], nil + } + <-w.closed + return ipn.Notify{}, errors.New("watcher closed") +} + +// Regression test for the burst-drain/coalescing bug: draining must not +// stall behind a blocked status fetch. +func TestWatchNetmapDrainsBurstWhileStatusFetchBlocked(t *testing.T) { + const burst = 129 + + watcher := &fakeIPNWatcher{ + notifs: selfChangeNotifs(burst), + afterFirst: make(chan struct{}), + closed: make(chan struct{}), + } + + firstFetchStarted := make(chan struct{}) + unblockFirstFetch := make(chan struct{}) + var fetchCount atomic.Int32 + + fetchStatus := func(context.Context) (*ipnstate.Status, error) { //nolint:unparam // exercises the success path only + if fetchCount.Add(1) == 1 { + close(firstFetchStarted) + <-unblockFirstFetch + } + return &ipnstate.Status{}, nil + } + + var callbackCount atomic.Int32 + callback := func(*ipnstate.Status) { callbackCount.Add(1) } + + errc := make(chan error, 1) + go func() { + errc <- watchNetmap(context.Background(), watcher, fetchStatus, callback) + }() + + waitFor(t, 5*time.Second, "first status fetch never started", firstFetchStarted) + close(watcher.afterFirst) + + waitUntil(t, 5*time.Second, "watcher stalled while status fetch was blocked", func() bool { + return watcher.idx.Load() == burst + }) + + close(unblockFirstFetch) + + waitUntil(t, 5*time.Second, "expected a coalesced follow-up fetch", func() bool { + return fetchCount.Load() >= 2 + }) + + if got := fetchCount.Load(); got != 2 { + t.Errorf( + "fetchStatus called %d times, want exactly 2 (initial + one coalesced follow-up)", + got, + ) + } + if got := callbackCount.Load(); got != 2 { + t.Errorf("callback invoked %d times, want 2", got) + } + + close(watcher.closed) + if err := <-errc; err == nil { + t.Fatal("watchNetmap returned nil error after watcher closed, want an error") + } +} + +func TestToUint16Port(t *testing.T) { + cases := []struct { + name string + port int + want uint16 + wantErr bool + }{ + {name: "negative", port: -1, wantErr: true}, + {name: "zero", port: 0, want: 0}, + {name: "max", port: math.MaxUint16, want: math.MaxUint16}, + {name: "over max", port: math.MaxUint16 + 1, wantErr: true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ToUint16Port(tc.port) + if tc.wantErr { + if err == nil { + t.Fatalf("ToUint16Port(%d) = %d, nil; want an error", tc.port, got) + } + return + } + if err != nil { + t.Fatalf("ToUint16Port(%d) returned unexpected error: %v", tc.port, err) + } + if got != tc.want { + t.Errorf("ToUint16Port(%d) = %d, want %d", tc.port, got, tc.want) + } + }) + } +} + +func TestNetmapChanged(t *testing.T) { + cases := []struct { + name string + n ipn.Notify + want bool + }{ + {name: "empty notify", n: ipn.Notify{}, want: false}, + {name: "initial status", n: ipn.Notify{InitialStatus: &ipnstate.Status{}}, want: true}, + {name: "self change", n: ipn.Notify{SelfChange: &tailcfg.Node{}}, want: true}, + {name: "peers changed", n: ipn.Notify{PeersChanged: []*tailcfg.Node{{}}}, want: true}, + {name: "peers removed", n: ipn.Notify{PeersRemoved: []tailcfg.NodeID{1}}, want: true}, + { + name: "peer changed patch", + n: ipn.Notify{PeerChangedPatch: []*tailcfg.PeerChange{{}}}, + want: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := netmapChanged(tc.n); got != tc.want { + t.Errorf("netmapChanged(%+v) = %v, want %v", tc.n, got, tc.want) + } + }) + } +} diff --git a/pkg/ts/workspace_server.go b/pkg/ts/workspace_server.go index 2836db2aaf..c286ca5b15 100644 --- a/pkg/ts/workspace_server.go +++ b/pkg/ts/workspace_server.go @@ -2,7 +2,6 @@ package ts import ( "context" - "encoding/json" "fmt" "io" "net" @@ -12,23 +11,26 @@ import ( "os" "path/filepath" "strings" - "sync" + "sync/atomic" "time" "github.com/devsy-org/devsy/pkg/log" - "github.com/devsy-org/devsy/pkg/platform/client" sshServer "github.com/devsy-org/devsy/pkg/ssh/server" "tailscale.com/client/local" "tailscale.com/envknob" + "tailscale.com/ipn/ipnstate" "tailscale.com/ipn/store/mem" "tailscale.com/tsnet" - "tailscale.com/types/netmap" + "tailscale.com/types/key" ) const ( // TSPortForwardPort is the fixed port on which the workspace WebSocket reverse proxy listens. TSPortForwardPort string = "12051" + // DevsyTSNetDomain is the MagicDNS suffix for the Devsy tailnet. + DevsyTSNetDomain = "ts.devsy" + RunnerProxySocket string = "runner-proxy.sock" netMapCooldown = 30 * time.Second @@ -39,8 +41,7 @@ type WorkspaceServer struct { tsServer *tsnet.Server listeners []net.Listener - connectionCounter int - connectionCounterMu sync.Mutex + connectionCount atomic.Int64 config *WorkspaceServerConfig } @@ -51,11 +52,13 @@ type WorkspaceServerConfig struct { PlatformHost string WorkspaceHost string LogF func(format string, args ...any) - Client client.Client RootDir string + // Insecure skips TLS certificate verification for the DERP probe and + // the TSNet control-plane connection. Only set for coordinators known + // to use self-signed certificates. + Insecure bool } -// NewWorkspaceServer creates a new TSNet server instance. func NewWorkspaceServer(config *WorkspaceServerConfig) *WorkspaceServer { return &WorkspaceServer{ config: config, @@ -65,9 +68,8 @@ func NewWorkspaceServer(config *WorkspaceServerConfig) *WorkspaceServer { // Start initializes the TSNet server, sets up listeners for SSH and HTTP // reverse proxy traffic, and waits until the given context is canceled. func (s *WorkspaceServer) Start(ctx context.Context) error { - log.Infof("Starting workspace server") + log.Infof("starting workspace server") - // Perform TSNet initialization (validation, control URL, server startup, hostname parsing) workspaceName, projectName, err := s.setupTSNet(ctx) if err != nil { return err @@ -77,34 +79,20 @@ func (s *WorkspaceServer) Start(ctx context.Context) error { return err } - // send heartbeats - go s.sendHeartbeats(ctx, projectName, workspaceName, lc) - - // Start both SSH and HTTP reverse proxy listeners - if err := s.startListeners(ctx, projectName, workspaceName, lc); err != nil { - return err + runner := &runnerClient{ + lc: lc, + accessKey: s.config.AccessKey, + projectName: projectName, + workspaceName: workspaceName, } - go func() { - lastUpdate := time.Now() - if err := WatchNetmap(ctx, lc, func(netMap *netmap.NetworkMap) { - if time.Since(lastUpdate) < netMapCooldown { - return - } - lastUpdate = time.Now() + go s.sendHeartbeats(ctx, runner) + go s.watchNetmap(ctx, lc) - nm, err := json.Marshal(netMap) - if err != nil { - log.Errorf("Failed to marshal netmap: %v", err) - } else { - _ = os.WriteFile(filepath.Join(s.config.RootDir, "netmap.json"), nm, 0o644) - } - }); err != nil { - log.Errorf("Failed to watch netmap: %v", err) - } - }() + if err := s.startListeners(ctx, runner); err != nil { + return err + } - // Wait until the context is canceled. <-ctx.Done() return nil } @@ -123,7 +111,6 @@ func (s *WorkspaceServer) Stop() { log.Info("Tailscale server stopped") } -// Dial dials the given address using the TSNet server. func (s *WorkspaceServer) Dial(ctx context.Context, network, addr string) (net.Conn, error) { if s.tsServer == nil { return nil, fmt.Errorf("tailscale server is not running") @@ -150,7 +137,6 @@ func (s *WorkspaceServer) setupTSNet(ctx context.Context) (workspace, project st return s.parseWorkspaceHostname() } -// validateConfig ensures required configuration values are set. func (s *WorkspaceServer) validateConfig() error { if s.config.AccessKey == "" || s.config.PlatformHost == "" || s.config.WorkspaceHost == "" { return fmt.Errorf("access key, host, or hostname cannot be empty") @@ -164,17 +150,18 @@ func (s *WorkspaceServer) setupControlURL(ctx context.Context) (*url.URL, error) Scheme: GetEnvOrDefault("DEVSY_TSNET_SCHEME", "https"), Host: s.config.PlatformHost, } - if err := CheckDerpConnection(ctx, baseURL); err != nil { + if err := CheckDerpConnection(ctx, baseURL, s.config.Insecure); err != nil { return nil, fmt.Errorf("failed to verify DERP connection: %w", err) } return baseURL, nil } -// initTsServer initializes the TSNet server. func (s *WorkspaceServer) initTsServer(ctx context.Context, controlURL *url.URL) error { store, _ := mem.New(s.config.LogF, "") - envknob.Setenv("TS_DEBUG_TLS_DIAL_INSECURE_SKIP_VERIFY", "true") - log.Infof("Connecting to control URL - %s/coordinator/", controlURL.String()) + if s.config.Insecure { + envknob.Setenv("TS_DEBUG_TLS_DIAL_INSECURE_SKIP_VERIFY", "true") + } + log.Infof("connecting to control URL - %s/coordinator/", controlURL.String()) s.tsServer = &tsnet.Server{ Hostname: s.config.WorkspaceHost, Logf: s.config.LogF, @@ -190,7 +177,6 @@ func (s *WorkspaceServer) initTsServer(ctx context.Context, controlURL *url.URL) return nil } -// parseHostname extracts workspace and project names from the hostname. func (s *WorkspaceServer) parseWorkspaceHostname() (workspace, project string, err error) { parts := strings.Split(s.config.WorkspaceHost, ".") if len(parts) < 4 { @@ -199,71 +185,100 @@ func (s *WorkspaceServer) parseWorkspaceHostname() (workspace, project string, e return parts[1], parts[2], nil } +// watchNetmap persists the tailnet status to netmap.json for debugging, +// throttled to once per netMapCooldown. +func (s *WorkspaceServer) watchNetmap(ctx context.Context, lc *local.Client) { + lastUpdate := time.Now() + err := WatchNetmap(ctx, lc, func(status *ipnstate.Status) { + if time.Since(lastUpdate) < netMapCooldown { + return + } + lastUpdate = time.Now() + PersistNetmapStatus(s.config.RootDir, status) + }) + if err != nil { + log.Errorf("failed to watch netmap: %v", err) + } +} + // startListeners creates and starts the SSH and HTTP reverse proxy listeners. -func (s *WorkspaceServer) startListeners( - ctx context.Context, - projectName, workspaceName string, - lc *local.Client, -) error { - // Create and start the SSH listener. - log.Infof("Starting SSH listener") +func (s *WorkspaceServer) startListeners(ctx context.Context, runner *runnerClient) error { + log.Infof("starting SSH listener") sshListener, err := s.createListener(fmt.Sprintf(":%d", sshServer.DefaultUserPort)) if err != nil { return err } - // Create and start the HTTP reverse proxy listener. - log.Infof("Starting HTTP reverse proxy listener on TSNet port %s", TSPortForwardPort) + log.Infof("starting HTTP reverse proxy listener on TSNet port %s", TSPortForwardPort) wsListener, err := s.createListener(fmt.Sprintf(":%s", TSPortForwardPort)) if err != nil { return fmt.Errorf("failed to create listener on TS port %s: %w", TSPortForwardPort, err) } - // Create and start the platform HTTP git credentials listener - runnerProxySocket := filepath.Join(s.config.RootDir, RunnerProxySocket) - log.Infof("Starting runner proxy socket on %s", runnerProxySocket) - _ = os.Remove(runnerProxySocket) - runnerProxyListener, err := net.Listen("unix", runnerProxySocket) + runnerProxyListener, err := s.createRunnerProxySocket() if err != nil { - return fmt.Errorf("failed to create listener on TS port %s: %w", TSPortForwardPort, err) + return err } - _ = os.Chmod( - runnerProxySocket, - 0o777, - ) // #nosec G302 -- required so all users can connect to the unix socket - - // add all listeners to the list s.listeners = append(s.listeners, sshListener, wsListener, runnerProxyListener) - // Setup HTTP handler for git and docker credentials on the runner proxy. - go func() { - mux := http.NewServeMux() - transport := &http.Transport{DialContext: s.tsServer.Dial} - mux.HandleFunc("/git-credentials", func(w http.ResponseWriter, r *http.Request) { - s.gitCredentialsHandler(w, r, lc, transport, projectName, workspaceName) - }) - mux.HandleFunc("/docker-credentials", func(w http.ResponseWriter, r *http.Request) { - s.dockerCredentialsHandler(w, r, lc, transport, projectName, workspaceName) - }) - serveMux(runnerProxyListener, mux, "HTTP runner proxy server error: %v") - }() + transport := &http.Transport{DialContext: s.tsServer.Dial} + go s.serveRunnerProxy(runnerProxyListener, runner, transport) + go s.servePortForward(wsListener) + go s.handleSSHConnections(ctx, sshListener) - // Setup HTTP handler for port forwarding. - go func() { - mux := http.NewServeMux() - mux.HandleFunc("/portforward", s.httpPortForwardHandler) - serveMux( - wsListener, - mux, - fmt.Sprintf("HTTP server error on TS port %s: %%v", TSPortForwardPort), + return nil +} + +// createRunnerProxySocket creates the unix socket the platform runner uses to +// reach the workspace's credential-proxy endpoints. +func (s *WorkspaceServer) createRunnerProxySocket() (net.Listener, error) { + runnerProxySocket := filepath.Join(s.config.RootDir, RunnerProxySocket) + log.Infof("starting runner proxy socket on %s", runnerProxySocket) + + _ = os.Remove(runnerProxySocket) + listener, err := net.Listen("unix", runnerProxySocket) + if err != nil { + return nil, fmt.Errorf( + "failed to create runner proxy socket %s: %w", + runnerProxySocket, + err, ) - }() + } - // Start handling SSH connections. - go s.handleSSHConnections(ctx, sshListener) + // The daemon runs as the container's default (often root) user, but git/docker + // commands invoking the credential helper run as whatever devcontainer remoteUser + // an interactive session su'd to (see pkg/ssh/server), a different UID. All local + // users must therefore reach this socket. + chmodErr := os.Chmod(runnerProxySocket, 0o777) // #nosec G302 -- see comment above + if chmodErr != nil { + log.Errorf("failed to chmod runner proxy socket %s: %v", runnerProxySocket, chmodErr) + } - return nil + return listener, nil +} + +func (s *WorkspaceServer) serveRunnerProxy( + listener net.Listener, + runner *runnerClient, + transport *http.Transport, +) { + mux := http.NewServeMux() + mux.HandleFunc( + "/git-credentials", + s.runnerProxyHandler(runner, transport, "workspace-git-credentials"), + ) + mux.HandleFunc( + "/docker-credentials", + s.runnerProxyHandler(runner, transport, "workspace-docker-credentials"), + ) + serveMux(listener, mux, "runner proxy server error: %v") +} + +func (s *WorkspaceServer) servePortForward(listener net.Listener) { + mux := http.NewServeMux() + mux.HandleFunc("/portforward", s.httpPortForwardHandler) + serveMux(listener, mux, fmt.Sprintf("http server error on TS port %s: %%v", TSPortForwardPort)) } func serveMux(listener net.Listener, mux *http.ServeMux, errFormat string) { @@ -273,113 +288,56 @@ func serveMux(listener net.Listener, mux *http.ServeMux, errFormat string) { } } -// createListener creates a raw listener and wraps it with connection tracking. func (s *WorkspaceServer) createListener(addr string) (net.Listener, error) { l, err := s.tsServer.Listen("tcp", addr) if err != nil { return nil, fmt.Errorf("failed to listen on %s: %w", addr, err) } - - // create a new tracked listener to track the number of connections return l, nil } func (s *WorkspaceServer) addConnection() { - s.connectionCounterMu.Lock() - defer s.connectionCounterMu.Unlock() - s.connectionCounter++ + s.connectionCount.Add(1) } func (s *WorkspaceServer) removeConnection() { - s.connectionCounterMu.Lock() - defer s.connectionCounterMu.Unlock() - s.connectionCounter-- + s.connectionCount.Add(-1) } -// gitCredentialsHandler is the handler for git credentials requests for workspace. -func (s *WorkspaceServer) gitCredentialsHandler( - w http.ResponseWriter, - r *http.Request, - lc *local.Client, +// runnerProxyHandler builds a reverse-proxy handler that discovers the +// runner peer and forwards the request to path on it. +func (s *WorkspaceServer) runnerProxyHandler( + runner *runnerClient, transport *http.Transport, - projectName, workspaceName string, -) { - log.Infof("Received git credentials request from %s", r.RemoteAddr) + path string, +) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + log.Infof("received %s request from %s", path, r.RemoteAddr) - // create a new http client with a custom transport - discoveredRunner, err := s.discoverRunner(r.Context(), lc) - if err != nil { - http.Error(w, "failed to discover runner", http.StatusInternalServerError) - return - } - - // build the runner URL - runnerURL := fmt.Sprintf( - "http://%s.ts.loft/devsy/%s/%s/workspace-git-credentials", - discoveredRunner, - projectName, - workspaceName, - ) - parsedURL, err := url.Parse(runnerURL) - if err != nil { - http.Error(w, "failed to parse runner URL", http.StatusInternalServerError) - return - } - - proxy := &httputil.ReverseProxy{ - Rewrite: func(pr *httputil.ProxyRequest) { - dest := *parsedURL - pr.Out.URL = &dest - pr.Out.Host = dest.Host - pr.Out.Header.Set("Authorization", "Bearer "+s.config.AccessKey) - addForwardedFor(pr) - }, - } - proxy.Transport = transport - proxy.ServeHTTP(w, r) -} - -// dockerCredentialsHandler is the handler for docker credentials requests for workspace. -func (s *WorkspaceServer) dockerCredentialsHandler( - w http.ResponseWriter, - r *http.Request, - lc *local.Client, - transport *http.Transport, - projectName, workspaceName string, -) { - log.Infof("Received docker credentials request from %s", r.RemoteAddr) - - // create a new http client with a custom transport - discoveredRunner, err := s.discoverRunner(r.Context(), lc) - if err != nil { - http.Error(w, "failed to discover runner", http.StatusInternalServerError) - return - } + runnerHost, err := runner.discoverRunner(r.Context()) + if err != nil { + http.Error(w, "failed to discover runner", http.StatusInternalServerError) + return + } - // build the runner URL - runnerURL := fmt.Sprintf( - "http://%s.ts.loft/devsy/%s/%s/workspace-docker-credentials", - discoveredRunner, - projectName, - workspaceName, - ) - parsedURL, err := url.Parse(runnerURL) - if err != nil { - http.Error(w, "failed to parse runner URL", http.StatusInternalServerError) - return - } + parsedURL, err := url.Parse(runner.url(runnerHost, path)) + if err != nil { + http.Error(w, "failed to parse runner URL", http.StatusInternalServerError) + return + } - proxy := &httputil.ReverseProxy{ - Rewrite: func(pr *httputil.ProxyRequest) { - dest := *parsedURL - pr.Out.URL = &dest - pr.Out.Host = dest.Host - pr.Out.Header.Set("Authorization", "Bearer "+s.config.AccessKey) - addForwardedFor(pr) - }, + proxy := &httputil.ReverseProxy{ + Transport: transport, + Rewrite: func(pr *httputil.ProxyRequest) { + dest := *parsedURL + pr.Out.URL = &dest + pr.Out.Host = dest.Host + pr.Out.Header.Set("Authorization", "Bearer "+runner.accessKey) + addForwardedFor(pr) + }, + } + proxy.ServeHTTP(w, r) } - proxy.Transport = transport - proxy.ServeHTTP(w, r) } // httpPortForwardHandler is the HTTP reverse proxy handler for workspace. @@ -389,20 +347,18 @@ func (s *WorkspaceServer) httpPortForwardHandler(w http.ResponseWriter, r *http. defer s.removeConnection() log.Debugf("httpPortForwardHandler: starting") - // Retrieve required custom headers. - targetPort := r.Header.Get("X-Loft-Forward-Port") - baseForwardStr := r.Header.Get("X-Loft-Forward-Url") + targetPort := r.Header.Get("X-Devsy-Forward-Port") + baseForwardStr := r.Header.Get("X-Devsy-Forward-Url") if targetPort == "" || baseForwardStr == "" { - http.Error(w, "missing required X-Loft headers", http.StatusBadRequest) + http.Error(w, "missing required X-Devsy headers", http.StatusBadRequest) return } log.Debugf( - "httpPortForwardHandler: received headers: X-Loft-Forward-Port=%s, X-Loft-Forward-Url=%s", + "httpPortForwardHandler: received headers: X-Devsy-Forward-Port=%s, X-Devsy-Forward-Url=%s", targetPort, baseForwardStr, ) - // Parse and modify the URL to target the local endpoint. parsedURL, err := url.Parse(baseForwardStr) if err != nil { log.Errorf("httpPortForwardHandler: failed to parse base URL: %v", err) @@ -414,18 +370,18 @@ func (s *WorkspaceServer) httpPortForwardHandler(w http.ResponseWriter, r *http. log.Debugf("httpPortForwardHandler: final target URL=%s", parsedURL.String()) proxy := &httputil.ReverseProxy{ + Transport: http.DefaultTransport, Rewrite: func(pr *httputil.ProxyRequest) { dest := *parsedURL pr.Out.URL = &dest pr.Out.Host = dest.Host // Remove custom headers so they are not forwarded. - pr.Out.Header.Del("X-Loft-Forward-Port") - pr.Out.Header.Del("X-Loft-Forward-Url") - pr.Out.Header.Del("X-Loft-Forward-Authorization") + pr.Out.Header.Del("X-Devsy-Forward-Port") + pr.Out.Header.Del("X-Devsy-Forward-Url") + pr.Out.Header.Del("X-Devsy-Forward-Authorization") addForwardedFor(pr) }, } - proxy.Transport = http.DefaultTransport log.Infof( "httpPortForwardHandler: final proxied request: %s %s", @@ -464,7 +420,7 @@ func (s *WorkspaceServer) handleSSHConnections(ctx context.Context, listener net if ctx.Err() != nil { return } - log.Errorf("Failed to accept connection: %v", err) + log.Errorf("failed to accept connection: %v", err) continue } go s.handleSSHConnection(clientConn) @@ -480,12 +436,11 @@ func (s *WorkspaceServer) handleSSHConnection(clientConn net.Conn) { localAddr := fmt.Sprintf("127.0.0.1:%d", sshServer.DefaultUserPort) backendConn, err := net.Dial("tcp", localAddr) if err != nil { - log.Errorf("Failed to connect to local address %s: %v", localAddr, err) + log.Errorf("failed to connect to local address %s: %v", localAddr, err) return } defer func() { _ = backendConn.Close() }() - // Start bidirectional copy between client and backend. go func() { defer func() { _ = clientConn.Close() }() defer func() { _ = backendConn.Close() }() @@ -494,16 +449,12 @@ func (s *WorkspaceServer) handleSSHConnection(clientConn net.Conn) { _, err = io.Copy(clientConn, backendConn) } -func (s *WorkspaceServer) sendHeartbeats( - ctx context.Context, - projectName, workspaceName string, - lc *local.Client, -) { - // create a new http client with a custom transport - transport := &http.Transport{DialContext: s.tsServer.Dial} - client := &http.Client{Transport: transport, Timeout: 10 * time.Second} +func (s *WorkspaceServer) sendHeartbeats(ctx context.Context, runner *runnerClient) { + client := &http.Client{ + Transport: &http.Transport{DialContext: s.tsServer.Dial}, + Timeout: 10 * time.Second, + } - // create a ticker to send heartbeats every 10 seconds ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() for { @@ -511,17 +462,11 @@ func (s *WorkspaceServer) sendHeartbeats( case <-ctx.Done(): return case <-ticker.C: - // get the current number of connections - s.connectionCounterMu.Lock() - connections := s.connectionCounter - s.connectionCounterMu.Unlock() - - // send a heartbeat if there are connections - if connections > 0 { - err := s.sendHeartbeat(ctx, client, projectName, workspaceName, lc, connections) - if err != nil { - log.Errorf("Failed to send heartbeat: %v", err) - } + if s.connectionCount.Load() <= 0 { + continue + } + if err := s.sendHeartbeat(ctx, client, runner); err != nil { + log.Errorf("failed to send heartbeat: %v", err) } } } @@ -530,35 +475,29 @@ func (s *WorkspaceServer) sendHeartbeats( func (s *WorkspaceServer) sendHeartbeat( ctx context.Context, client *http.Client, - projectName, workspaceName string, - lc *local.Client, - connections int, + runner *runnerClient, ) error { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - discoveredRunner, err := s.discoverRunner(ctx, lc) + runnerHost, err := runner.discoverRunner(ctx) if err != nil { return fmt.Errorf("failed to discover runner: %w", err) } - heartbeatURL := fmt.Sprintf( - "http://%s.ts.loft/devsy/%s/%s/heartbeat", - discoveredRunner, - projectName, - workspaceName, - ) + heartbeatURL := runner.url(runnerHost, "heartbeat") log.Infof( - "Sending heartbeat to %s, because there are %d active connections", + "sending heartbeat to %s, because there are %d active connections", heartbeatURL, - connections, + s.connectionCount.Load(), ) - req, err := http.NewRequestWithContext(ctx, "GET", heartbeatURL, nil) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, heartbeatURL, nil) if err != nil { return fmt.Errorf("failed to create request for %s: %w", heartbeatURL, err) } + req.Header.Set("Authorization", "Bearer "+runner.accessKey) - req.Header.Set("Authorization", "Bearer "+s.config.AccessKey) resp, err := client.Do(req) if err != nil { return fmt.Errorf("request to %s failed: %w", heartbeatURL, err) @@ -566,34 +505,56 @@ func (s *WorkspaceServer) sendHeartbeat( defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("received response from %s - Status: %d", heartbeatURL, resp.StatusCode) + return fmt.Errorf("received response from %s - status: %d", heartbeatURL, resp.StatusCode) } - log.Infof("received response from %s - Status: %d", heartbeatURL, resp.StatusCode) + log.Infof("received response from %s - status: %d", heartbeatURL, resp.StatusCode) return nil } -// discoverRunner attempts to find the runner peer from the TSNet status. -func (s *WorkspaceServer) discoverRunner(ctx context.Context, lc *local.Client) (string, error) { - status, err := lc.Status(ctx) +// runnerClient bundles what's needed to locate the platform runner peer and +// address its endpoints on the tailnet. +type runnerClient struct { + lc *local.Client + accessKey string + projectName string + workspaceName string +} + +// discoverRunner finds the runner peer's hostname from the TSNet status. +func (rc *runnerClient) discoverRunner(ctx context.Context) (string, error) { + status, err := rc.lc.Status(ctx) if err != nil { return "", fmt.Errorf("failed to get status: %w", err) } - var runner string - for _, peer := range status.Peer { - if peer == nil || peer.HostName == "" { - continue - } + runnerHost, ok := selectRunnerPeer(status.Peer) + if !ok { + return "", fmt.Errorf("no active runner found") + } + log.Infof("discoverRunner: selected runner = %s", runnerHost) + return runnerHost, nil +} - if strings.HasSuffix(peer.HostName, "runner") { - runner = peer.HostName - break +// selectRunnerPeer picks the platform runner from the tailnet's peer set, +// identified by a "runner"-suffixed hostname. +func selectRunnerPeer(peers map[key.NodePublic]*ipnstate.PeerStatus) (string, bool) { + for _, peer := range peers { + if peer != nil && strings.HasSuffix(peer.HostName, "runner") { + return peer.HostName, true } } - if runner == "" { - return "", fmt.Errorf("no active runner found") - } + return "", false +} - log.Infof("discoverRunner: selected runner = %s", runner) - return runner, nil +// url builds the runner-relative URL for path, e.g. "heartbeat" or +// "workspace-git-credentials". +func (rc *runnerClient) url(runnerHost, path string) string { + return fmt.Sprintf( + "http://%s.%s/devsy/%s/%s/%s", + runnerHost, + DevsyTSNetDomain, + rc.projectName, + rc.workspaceName, + path, + ) } diff --git a/pkg/ts/workspace_server_test.go b/pkg/ts/workspace_server_test.go new file mode 100644 index 0000000000..a0d3557064 --- /dev/null +++ b/pkg/ts/workspace_server_test.go @@ -0,0 +1,67 @@ +package ts + +import ( + "testing" + + "tailscale.com/ipn/ipnstate" + "tailscale.com/types/key" +) + +func TestSelectRunnerPeer(t *testing.T) { + cases := []struct { + name string + peers map[key.NodePublic]*ipnstate.PeerStatus + wantHost string + wantFound bool + }{ + { + name: "no peers", + peers: map[key.NodePublic]*ipnstate.PeerStatus{}, + wantFound: false, + }, + { + name: "no runner peer", + peers: map[key.NodePublic]*ipnstate.PeerStatus{ + {}: {HostName: "some-client"}, + }, + wantFound: false, + }, + { + name: "nil peer entry is skipped", + peers: map[key.NodePublic]*ipnstate.PeerStatus{ + {}: nil, + }, + wantFound: false, + }, + { + name: "runner peer found", + peers: map[key.NodePublic]*ipnstate.PeerStatus{ + {}: {HostName: "acme-project-abc123-runner"}, + }, + wantHost: "acme-project-abc123-runner", + wantFound: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + host, found := selectRunnerPeer(tc.peers) + if found != tc.wantFound { + t.Fatalf("selectRunnerPeer() found = %v, want %v", found, tc.wantFound) + } + if found && host != tc.wantHost { + t.Errorf("selectRunnerPeer() host = %q, want %q", host, tc.wantHost) + } + }) + } +} + +func TestRunnerClientURL(t *testing.T) { + rc := &runnerClient{projectName: "acme", workspaceName: "ws1"} + + got := rc.url("acme-ws1-runner", "workspace-git-credentials") + want := "http://acme-ws1-runner.ts.devsy/devsy/acme/ws1/workspace-git-credentials" + if got != want { + t.Errorf("url() = %q, want %q", got, want) + } +}