diff --git a/cmd/machine/ssh.go b/cmd/machine/ssh.go index 7396bee63c..acaf349621 100644 --- a/cmd/machine/ssh.go +++ b/cmd/machine/ssh.go @@ -21,7 +21,7 @@ import ( "github.com/devsy-org/devsy/pkg/pty" devssh "github.com/devsy-org/devsy/pkg/ssh" devsshagent "github.com/devsy-org/devsy/pkg/ssh/agent" - "github.com/devsy-org/devsy/pkg/tunnel" + "github.com/devsy-org/devsy/pkg/transport" "github.com/devsy-org/devsy/pkg/workspace" "github.com/spf13/cobra" "golang.org/x/crypto/ssh" @@ -66,6 +66,7 @@ type StartSSHSessionOptions struct { Command string AgentForwarding bool SessionOptions SSHSessionOptions + OpenTransport func(context.Context) (transport.ManagedConn, error) Exec ExecFunc Stderr io.Writer } @@ -180,18 +181,36 @@ func (cmd *SSHCmd) Run(ctx context.Context, args []string) error { type ExecFunc func(ctx context.Context, stdin io.Reader, stdout io.Writer, stderr io.Writer) error func StartSSHSession(ctx context.Context, options StartSSHSessionOptions) error { - pb, err := tunnel.NewPipeBridge() + openTransport := options.OpenTransport + if openTransport == nil { + if options.Exec == nil { + return errors.New("open transport or exec function is required") + } + openTransport = func(ctx context.Context) (transport.ManagedConn, error) { + return transport.OpenCallbackConn(ctx, func( + ctx context.Context, + stdin io.Reader, + stdout io.Writer, + ) error { + return options.Exec(ctx, stdin, stdout, options.Stderr) + }, transport.CallbackConnOptions{ + LocalAddr: transport.NewAddr("machine"), + RemoteAddr: transport.NewAddr("provider"), + }) + } + } + + conn, err := openTransport(ctx) if err != nil { return err } - defer pb.Close() - - return pb.RunPair(ctx, - func(ctx context.Context, stdin, stdout *os.File) error { - return options.Exec(ctx, stdin, stdout, options.Stderr) - }, - func(ctx context.Context, stdout, stdin *os.File) error { - sshClient, err := devssh.StdioClientWithUser(stdout, stdin, options.User) + return transport.RunManaged(transport.RunManagedOptions{ + Parent: ctx, + Conn: conn, + TransportSide: transport.SideProvider, + Metadata: transport.LogMetadata{Mode: "machine", TransportImpl: "callback"}, + Handler: func(ctx context.Context) error { + sshClient, err := devssh.ClientFromConn(conn, options.User, nil) if err != nil { return err } @@ -204,7 +223,7 @@ func StartSSHSession(ctx context.Context, options StartSSHSessionOptions) error Stderr: options.Stderr, }) }, - ) + }) } func RunSSHSession(ctx context.Context, sshClient *ssh.Client, options RunSSHSessionOptions) error { diff --git a/cmd/workspace/ssh.go b/cmd/workspace/ssh.go index d7ff310979..1bbddf31df 100644 --- a/cmd/workspace/ssh.go +++ b/cmd/workspace/ssh.go @@ -22,6 +22,7 @@ import ( "github.com/devsy-org/devsy/pkg/log" "github.com/devsy-org/devsy/pkg/provider" devssh "github.com/devsy-org/devsy/pkg/ssh" + "github.com/devsy-org/devsy/pkg/transport" "github.com/devsy-org/devsy/pkg/tunnel" workspace2 "github.com/devsy-org/devsy/pkg/workspace" "github.com/spf13/cobra" @@ -355,24 +356,27 @@ func (cmd *SSHCmd) startProxyTunnel( client client2.ProxyClient, ) error { log.Debugf("start proxy tunnel") - return tunnel.NewTunnelWithMetadata( - ctx, - func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { - return client.Ssh(ctx, client2.SshOptions{ - User: cmd.User, - Stdin: stdin, - Stdout: stdout, - }) + conn, err := client.OpenSSHTransport(ctx, client2.SSHTransportOptions{User: cmd.User}) + if err != nil { + return err + } + return transport.RunManaged(transport.RunManagedOptions{ + Parent: ctx, + Conn: conn, + TransportSide: transport.SideProvider, + Metadata: transport.LogMetadata{ + Provider: client.Provider(), Mode: "proxy", + Workspace: client.Workspace(), TransportImpl: "callback", }, - func(ctx context.Context, containerClient *ssh.Client) error { + Handler: func(ctx context.Context) error { + containerClient, err := devssh.ClientFromConn(conn, cmd.User, nil) + if err != nil { + return err + } + defer func() { _ = containerClient.Close() }() return cmd.startTunnel(ctx, devsyConfig, containerClient, client) }, - tunnel.TransportLogMetadata{ - Provider: client.Provider(), - Mode: "proxy", - Workspace: client.Workspace(), - }, - ) + }) } func (cmd *SSHCmd) retrieveEnVars() (map[string]string, error) { diff --git a/cmd/workspace/up/agent.go b/cmd/workspace/up/agent.go index b463e88bde..4cf229f048 100644 --- a/cmd/workspace/up/agent.go +++ b/cmd/workspace/up/agent.go @@ -246,7 +246,7 @@ func newAgentInjectFunc( return func( cancelCtx context.Context, sshCmd string, - sshTunnelStdinReader, sshTunnelStdoutWriter *os.File, + sshTunnelStdinReader io.Reader, sshTunnelStdoutWriter io.Writer, writer io.WriteCloser, ) error { return agent.InjectAgent(cancelCtx, &agent.InjectOptions{ diff --git a/pkg/client/client.go b/pkg/client/client.go index e447757195..76d89cddf7 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -10,6 +10,7 @@ import ( "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/provider" "github.com/devsy-org/devsy/pkg/status" + "github.com/devsy-org/devsy/pkg/transport" "golang.org/x/crypto/ssh" ) @@ -56,6 +57,12 @@ type Client interface { // Command creates an SSH tunnel into the workspace Command(ctx context.Context, options CommandOptions) error + + // OpenCommandTransport starts a long-lived command as a managed connection. + OpenCommandTransport( + ctx context.Context, + options CommandTransportOptions, + ) (transport.ManagedConn, error) } // ProxyClient executes it's commands on the platform. @@ -70,6 +77,13 @@ type ProxyClient interface { // Ssh starts an ssh tunnel to the workspace container Ssh(ctx context.Context, options SshOptions) error + + // OpenSSHTransport starts an SSH transport whose stdin/stdout carry the + // persistent SSH protocol. + OpenSSHTransport( + ctx context.Context, + options SSHTransportOptions, + ) (transport.ManagedConn, error) } // DaemonClient connects to workspaces through a shared daemon. @@ -160,6 +174,11 @@ type CommandOptions struct { Stderr io.Writer } +type CommandTransportOptions struct { + Command string + Stderr io.Writer +} + type UpOptions struct { provider.CLIOptions @@ -176,6 +195,11 @@ type SshOptions struct { Stdout io.Writer } +type SSHTransportOptions struct { + User string + Stderr io.Writer +} + type Status string const ( diff --git a/pkg/client/clientimplementation/machine_client.go b/pkg/client/clientimplementation/machine_client.go index 037add70bd..5d28f2be6a 100644 --- a/pkg/client/clientimplementation/machine_client.go +++ b/pkg/client/clientimplementation/machine_client.go @@ -13,6 +13,7 @@ import ( "github.com/devsy-org/devsy/pkg/log" "github.com/devsy-org/devsy/pkg/options" "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/transport" "github.com/devsy-org/devsy/pkg/types" ) @@ -202,6 +203,24 @@ func (s *machineClient) Command(ctx context.Context, commandOptions client.Comma }) } +func (s *machineClient) OpenCommandTransport( + ctx context.Context, + opt client.CommandTransportOptions, +) (transport.ManagedConn, error) { + return transport.OpenCallbackConn( + ctx, + func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { + return s.Command(ctx, client.CommandOptions{ + Command: opt.Command, Stdin: stdin, Stdout: stdout, Stderr: opt.Stderr, + }) + }, + transport.CallbackConnOptions{ + LocalAddr: transport.NewAddr("machine"), + RemoteAddr: transport.NewAddr("provider:" + s.Provider()), + }, + ) +} + func (s *machineClient) Status( ctx context.Context, options client.StatusOptions, diff --git a/pkg/client/clientimplementation/proxy_client.go b/pkg/client/clientimplementation/proxy_client.go index 4e6d0fe746..4da49e5f5d 100644 --- a/pkg/client/clientimplementation/proxy_client.go +++ b/pkg/client/clientimplementation/proxy_client.go @@ -22,6 +22,7 @@ import ( platformclient "github.com/devsy-org/devsy/pkg/platform/client" "github.com/devsy-org/devsy/pkg/provider" "github.com/devsy-org/devsy/pkg/terminal" + "github.com/devsy-org/devsy/pkg/transport" "github.com/devsy-org/devsy/pkg/types" "github.com/gofrs/flock" ) @@ -94,7 +95,11 @@ func (e *proxyExecutor) execute(ctx context.Context, params execParams) error { // where the actionable error message usually lives. func (e *proxyExecutor) executeWithJSONLog(ctx context.Context, params execParams) error { writer, done := log.PipeJSONStream() - params.stderr = writer + if params.stderr == nil { + params.stderr = writer + } else { + params.stderr = io.MultiWriter(writer, params.stderr) + } err := e.execute(ctx, params) _ = writer.Close() <-done @@ -243,6 +248,27 @@ func (s *proxyClient) Ssh(ctx context.Context, opt client.SshOptions) error { }) } +func (s *proxyClient) OpenSSHTransport( + ctx context.Context, + opt client.SSHTransportOptions, +) (transport.ManagedConn, error) { + return transport.OpenCallbackConn( + ctx, + func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { + return s.executor.executeWithJSONLog(ctx, execParams{ + command: s.config.Exec.Proxy.Ssh, + extraEnv: EncodeOptions(client.SshOptions{User: opt.User}, config.EnvFlagsSSH), + stdin: stdin, + stdout: stdout, + }) + }, + transport.CallbackConnOptions{ + LocalAddr: transport.NewAddr("proxy"), + RemoteAddr: transport.NewAddr("provider:" + s.config.Name), + }, + ) +} + func (s *proxyClient) Stop(ctx context.Context, opt client.StopOptions) error { s.m.Lock() defer s.m.Unlock() diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index 79f72d2c10..15338031ea 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -27,6 +27,7 @@ import ( "github.com/devsy-org/devsy/pkg/ssh" "github.com/devsy-org/devsy/pkg/status" "github.com/devsy-org/devsy/pkg/task" + "github.com/devsy-org/devsy/pkg/transport" "github.com/devsy-org/devsy/pkg/types" "github.com/gofrs/flock" ) @@ -285,6 +286,30 @@ func (s *workspaceClient) Command(ctx context.Context, opt client.CommandOptions }) } +func (s *workspaceClient) OpenCommandTransport( + ctx context.Context, + opt client.CommandTransportOptions, +) (transport.ManagedConn, error) { + environ, err := s.buildEnvironment(opt.Command) + if err != nil { + return nil, err + } + return transport.OpenCallbackConn( + ctx, + func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { + return RunCommand(ctx, RunCommandOptions{ + Command: s.providerConfig.Exec.Command, + Environ: environ, + Stdin: stdin, Stdout: stdout, Stderr: opt.Stderr, + }) + }, + transport.CallbackConnOptions{ + LocalAddr: transport.NewAddr("workspace"), + RemoteAddr: transport.NewAddr("provider:" + s.Provider()), + }, + ) +} + func (s *workspaceClient) Status( ctx context.Context, opt client.StatusOptions, diff --git a/pkg/devcontainer/setup.go b/pkg/devcontainer/setup.go index 62c2bb64ef..e95bf2e3fb 100644 --- a/pkg/devcontainer/setup.go +++ b/pkg/devcontainer/setup.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "os" "path/filepath" "runtime" "strings" @@ -526,7 +525,7 @@ func (r *runner) executeSetup( agentInjectFunc := func( cancelCtx context.Context, sshCmd string, - sshTunnelStdinReader, sshTunnelStdoutWriter *os.File, + sshTunnelStdinReader io.Reader, sshTunnelStdoutWriter io.Writer, writer io.WriteCloser, ) error { return r.driver.CommandDevContainer(cancelCtx, &driver.CommandParams{ diff --git a/pkg/devcontainer/sshtunnel/sshtunnel.go b/pkg/devcontainer/sshtunnel/sshtunnel.go index 5fe9dde403..8c100c2baa 100644 --- a/pkg/devcontainer/sshtunnel/sshtunnel.go +++ b/pkg/devcontainer/sshtunnel/sshtunnel.go @@ -14,13 +14,14 @@ import ( "github.com/devsy-org/devsy/pkg/log" devssh "github.com/devsy-org/devsy/pkg/ssh" devsshagent "github.com/devsy-org/devsy/pkg/ssh/agent" + "github.com/devsy-org/devsy/pkg/transport" "github.com/devsy-org/devsy/pkg/tunnel" "golang.org/x/crypto/ssh" "k8s.io/apimachinery/pkg/util/wait" ) type ( - AgentInjectFunc func(context.Context, string, *os.File, *os.File, io.WriteCloser) error + AgentInjectFunc func(context.Context, string, io.Reader, io.Writer, io.WriteCloser) error TunnelServerFunc func(ctx context.Context, stdin io.WriteCloser, stdout io.Reader) (*config2.Result, error) ) @@ -34,18 +35,12 @@ type ExecuteCommandOptions struct { } // ExecuteCommand runs the command in an SSH Tunnel and returns the result. -// It uses two PipeBridges: sshBridge connects the helper (agent inject) to -// the SSH tunnel, and grpcBridge connects the SSH command to the gRPC server. +// The SSH transport is managed as a connection; grpcBridge remains a bounded +// duplex operation between the SSH command and the gRPC server. func ExecuteCommand(ctx context.Context, opts ExecuteCommandOptions) (*config2.Result, error) { log.Debugf("starting SSH tunnel execution: ssh=%q workspace=%q addKeys=%v", opts.SSHCommand, opts.Command, opts.AddPrivateKeys) - sshBridge, err := tunnel.NewPipeBridge() - if err != nil { - return nil, err - } - defer sshBridge.Close() - grpcBridge, err := tunnel.NewPipeBridge() if err != nil { return nil, err @@ -54,24 +49,26 @@ func ExecuteCommand(ctx context.Context, opts ExecuteCommandOptions) (*config2.R var result *config2.Result - err = sshBridge.RunPair(ctx, - func(ctx context.Context, stdin, stdout *os.File) error { + conn, err := transport.OpenCallbackConn( + ctx, + func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { return executeSSHServerHelper(ctx, opts, stdin, stdout) }, - func(ctx context.Context, stdout, stdin *os.File) error { - if opts.AddPrivateKeys { - addPrivateKeys(ctx) - } - var runErr error - result, runErr = runSSHTunnel(ctx, sshTunnelParams{ - opts: opts, - stdout: stdout, - stdin: stdin, - grpcBridge: grpcBridge, - }) - return runErr + transport.CallbackConnOptions{ + LocalAddr: transport.NewAddr("devcontainer"), + RemoteAddr: transport.NewAddr("ssh-server"), }, ) + if err != nil { + return nil, err + } + defer func() { _ = conn.Close() }() + if opts.AddPrivateKeys { + addPrivateKeys(ctx) + } + result, err = runSSHTunnel(ctx, sshTunnelParams{ + opts: opts, conn: conn, grpcBridge: grpcBridge, + }) return result, err } @@ -90,7 +87,8 @@ func isExpectedError(err error) bool { func executeSSHServerHelper( ctx context.Context, opts ExecuteCommandOptions, - stdin, stdout *os.File, + stdin io.Reader, + stdout io.Writer, ) error { defer log.Debug("done executing SSH server helper command") @@ -118,8 +116,7 @@ func addPrivateKeys(ctx context.Context) { type sshTunnelParams struct { opts ExecuteCommandOptions - stdout *os.File - stdin *os.File + conn transport.ManagedConn grpcBridge *tunnel.PipeBridge } @@ -129,7 +126,7 @@ func runSSHTunnel(ctx context.Context, p sshTunnelParams) (*config2.Result, erro defer func() { log.Infof("tunnel: setup complete elapsed=%s", time.Since(start)) }() log.Debug("creating SSH client") - sshClient, err := devssh.StdioClient(p.stdout, p.stdin) + sshClient, err := devssh.ClientFromConn(p.conn, "", nil) if err != nil { return nil, fmt.Errorf("failed to create SSH client: %w", err) } diff --git a/pkg/devcontainer/sshtunnel/sshtunnel_test.go b/pkg/devcontainer/sshtunnel/sshtunnel_test.go index 15160d3d9b..3ff7b1aefa 100644 --- a/pkg/devcontainer/sshtunnel/sshtunnel_test.go +++ b/pkg/devcontainer/sshtunnel/sshtunnel_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "os" "testing" config2 "github.com/devsy-org/devsy/pkg/devcontainer/config" @@ -175,7 +174,7 @@ func TestExecuteCommand_PipeBridgeIntegration(t *testing.T) { wantErr := fmt.Errorf("inject failed") _, err := ExecuteCommand(context.Background(), ExecuteCommandOptions{ - AgentInject: func(_ context.Context, _ string, _ *os.File, _ *os.File, _ io.WriteCloser) error { + AgentInject: func(_ context.Context, _ string, _ io.Reader, _ io.Writer, _ io.WriteCloser) error { return wantErr }, SSHCommand: "test-ssh", @@ -194,7 +193,7 @@ func TestExecuteCommand_PipeBridgeIntegration(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) _, err := ExecuteCommand(ctx, ExecuteCommandOptions{ - AgentInject: func(ctx context.Context, _ string, _ *os.File, _ *os.File, _ io.WriteCloser) error { + AgentInject: func(ctx context.Context, _ string, _ io.Reader, _ io.Writer, _ io.WriteCloser) error { cancel() <-ctx.Done() return ctx.Err() diff --git a/pkg/ssh/helper.go b/pkg/ssh/helper.go index 568c7de601..1fe7ce2b63 100644 --- a/pkg/ssh/helper.go +++ b/pkg/ssh/helper.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "net" "github.com/devsy-org/devsy/pkg/stdio" "golang.org/x/crypto/ssh" @@ -86,6 +87,14 @@ func StdioClientFromKeyBytesWithUser( user string, ) (*ssh.Client, error) { conn := stdio.NewStdioStream(reader, writer) + return ClientFromConn(conn, user, keyBytes) +} + +// ClientFromConn creates an SSH client over an existing network connection. +func ClientFromConn(conn net.Conn, user string, keyBytes []byte) (*ssh.Client, error) { + if conn == nil { + return nil, fmt.Errorf("connection is required") + } clientConfig, err := ConfigFromKeyBytes(keyBytes) if err != nil { return nil, err diff --git a/pkg/ssh/session_conn.go b/pkg/ssh/session_conn.go new file mode 100644 index 0000000000..24430a37be --- /dev/null +++ b/pkg/ssh/session_conn.go @@ -0,0 +1,153 @@ +package ssh + +import ( + "context" + "fmt" + "io" + "net" + "os" + "sync" + "time" + + "github.com/devsy-org/devsy/pkg/transport" + xssh "golang.org/x/crypto/ssh" +) + +type SessionConnOptions struct { + Command string + Env map[string]string + Stderr io.Writer +} + +type sessionConn struct { + stdin io.WriteCloser + stdout io.Reader + session *xssh.Session + cancel context.CancelFunc + + closeOnce sync.Once + waitOnce sync.Once + waitDone chan struct{} + waitErr error +} + +// startedSession bundles a started SSH session together with its stdin/stdout +// pipes, which must be created before Session.Start is called. +type startedSession struct { + session *xssh.Session + stdin io.WriteCloser + stdout io.Reader +} + +// OpenSessionConn exposes a long-lived command on an existing SSH client as a +// managed connection. Closing it never closes the parent client. +func OpenSessionConn( + ctx context.Context, + client *xssh.Client, + opts SessionConnOptions, +) (transport.ManagedConn, error) { + if err := validateSessionConn(client, opts); err != nil { + return nil, err + } + if ctx == nil { + ctx = context.Background() + } + + started, err := startSession(client, opts) + if err != nil { + return nil, err + } + + sessionCtx, cancel := context.WithCancel(ctx) + conn := &sessionConn{ + stdin: started.stdin, stdout: started.stdout, session: started.session, cancel: cancel, + waitDone: make(chan struct{}), + } + go func() { + <-sessionCtx.Done() + _ = conn.Close() + }() + return conn, nil +} + +func validateSessionConn(client *xssh.Client, opts SessionConnOptions) error { + if client == nil { + return fmt.Errorf("ssh client is required") + } + if opts.Command == "" { + return fmt.Errorf("command is required") + } + return nil +} + +func startSession( + client *xssh.Client, + opts SessionConnOptions, +) (*startedSession, error) { + session, err := client.NewSession() + if err != nil { + return nil, fmt.Errorf("create ssh session: %w", err) + } + session.Stderr = opts.Stderr + stdin, err := session.StdinPipe() + if err != nil { + _ = session.Close() + return nil, fmt.Errorf("create ssh session stdin: %w", err) + } + stdout, err := session.StdoutPipe() + if err != nil { + _ = stdin.Close() + _ = session.Close() + return nil, fmt.Errorf("create ssh session stdout: %w", err) + } + for key, value := range opts.Env { + if err := session.Setenv(key, value); err != nil { + _ = stdin.Close() + _ = session.Close() + return nil, fmt.Errorf("set ssh session environment %q: %w", key, err) + } + } + if err := session.Start(opts.Command); err != nil { + _ = stdin.Close() + _ = session.Close() + return nil, fmt.Errorf("start ssh session: %w", err) + } + return &startedSession{session: session, stdin: stdin, stdout: stdout}, nil +} + +func (c *sessionConn) Read(p []byte) (int, error) { return c.stdout.Read(p) } +func (c *sessionConn) Write(p []byte) (int, error) { return c.stdin.Write(p) } + +func (c *sessionConn) Close() error { + var err error + c.closeOnce.Do(func() { + c.cancel() + if closeErr := c.stdin.Close(); closeErr != nil { + err = closeErr + } + if closeErr := c.session.Close(); err == nil && closeErr != nil { + err = closeErr + } + }) + return err +} + +func (c *sessionConn) LocalAddr() net.Addr { return transport.NewAddr("ssh-session") } +func (c *sessionConn) RemoteAddr() net.Addr { return transport.NewAddr("ssh-session") } +func (c *sessionConn) SetDeadline(time.Time) error { return os.ErrInvalid } +func (c *sessionConn) SetReadDeadline(time.Time) error { return os.ErrInvalid } +func (c *sessionConn) SetWriteDeadline(time.Time) error { return os.ErrInvalid } +func (c *sessionConn) Wait() error { + c.waitOnce.Do(func() { + c.waitErr = c.session.Wait() + c.cancel() + close(c.waitDone) + }) + <-c.waitDone + return c.waitErr +} + +var ( + _ net.Conn = (*sessionConn)(nil) + _ transport.ManagedConn = (*sessionConn)(nil) +) diff --git a/pkg/ssh/session_conn_test.go b/pkg/ssh/session_conn_test.go new file mode 100644 index 0000000000..45069f351a --- /dev/null +++ b/pkg/ssh/session_conn_test.go @@ -0,0 +1,16 @@ +package ssh + +import ( + "context" + "testing" +) + +func TestOpenSessionConnRequiresInputs(t *testing.T) { + if _, err := OpenSessionConn( + context.Background(), + nil, + SessionConnOptions{Command: "cat"}, + ); err == nil { + t.Fatal("nil client should fail") + } +} diff --git a/pkg/transport/addr.go b/pkg/transport/addr.go new file mode 100644 index 0000000000..dd05dc865c --- /dev/null +++ b/pkg/transport/addr.go @@ -0,0 +1,29 @@ +package transport + +import "fmt" + +const networkName = "devsy-transport" + +// Addr is a stable diagnostic address for a synthetic connection. +type Addr struct { + NetworkName string + RemoteName string +} + +func NewAddr(remote string) Addr { + return Addr{NetworkName: networkName, RemoteName: remote} +} + +func (a Addr) Network() string { + if a.NetworkName == "" { + return networkName + } + return a.NetworkName +} + +func (a Addr) String() string { + if a.RemoteName == "" { + return a.Network() + } + return fmt.Sprintf("%s:%s", a.Network(), a.RemoteName) +} diff --git a/pkg/transport/callback_conn.go b/pkg/transport/callback_conn.go new file mode 100644 index 0000000000..292d36f6f3 --- /dev/null +++ b/pkg/transport/callback_conn.go @@ -0,0 +1,128 @@ +package transport + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "time" +) + +type Callback func(ctx context.Context, stdin io.Reader, stdout io.Writer) error + +type CallbackConnOptions struct { + LocalAddr net.Addr + RemoteAddr net.Addr +} + +type callbackConn struct { + local net.Conn + remote net.Conn + cancel context.CancelFunc + state *managedState +} + +func OpenCallbackConn( + ctx context.Context, + callback Callback, + opts CallbackConnOptions, +) (ManagedConn, error) { + if callback == nil { + return nil, fmt.Errorf("callback is required") + } + if ctx == nil { + ctx = context.Background() + } + + local, remote := net.Pipe() + callbackCtx, cancel := context.WithCancel(ctx) + conn := &callbackConn{ + local: local, + remote: remote, + cancel: cancel, + state: newManagedState(), + } + callbackDone := make(chan struct{}) + go func() { + select { + case <-callbackCtx.Done(): + _ = conn.Close() + case <-callbackDone: + } + }() + go func() { + defer close(callbackDone) + err := callback(callbackCtx, remote, remote) + conn.state.finish(err) + _ = remote.Close() + }() + return &addrConn{Conn: conn, local: opts.LocalAddr, remote: opts.RemoteAddr}, nil +} + +func (c *callbackConn) Read(p []byte) (int, error) { return c.local.Read(p) } +func (c *callbackConn) Write(p []byte) (int, error) { return c.local.Write(p) } + +func (c *callbackConn) Close() error { + var err error + c.state.closeOnce.Do(func() { + c.cancel() + if closeErr := c.local.Close(); closeErr != nil { + err = closeErr + } + if closeErr := c.remote.Close(); err == nil && closeErr != nil { + err = closeErr + } + }) + return err +} + +func (c *callbackConn) LocalAddr() net.Addr { return netPipeAddr{} } +func (c *callbackConn) RemoteAddr() net.Addr { return netPipeAddr{} } +func (c *callbackConn) SetDeadline(t time.Time) error { return c.local.SetDeadline(t) } +func (c *callbackConn) SetReadDeadline(t time.Time) error { return c.local.SetReadDeadline(t) } +func (c *callbackConn) SetWriteDeadline(t time.Time) error { return c.local.SetWriteDeadline(t) } +func (c *callbackConn) Wait() error { return c.state.wait() } + +type netPipeAddr struct{} + +func (netPipeAddr) Network() string { return networkName } +func (netPipeAddr) String() string { return "callback" } + +type addrConn struct { + net.Conn + local net.Addr + remote net.Addr +} + +func (c *addrConn) LocalAddr() net.Addr { + if c.local != nil { + return c.local + } + return c.Conn.LocalAddr() +} + +func (c *addrConn) RemoteAddr() net.Addr { + if c.remote != nil { + return c.remote + } + return c.Conn.RemoteAddr() +} + +func (c *addrConn) Wait() error { + return c.Conn.(ManagedConn).Wait() +} + +func (c *addrConn) CloseWrite() error { + if closeWriter, ok := c.Conn.(CloseWriter); ok { + return closeWriter.CloseWrite() + } + return fmt.Errorf("close write: %w", errors.ErrUnsupported) +} + +func (c *addrConn) PID() int { + if process, ok := c.Conn.(ProcessInfo); ok { + return process.PID() + } + return 0 +} diff --git a/pkg/transport/conn.go b/pkg/transport/conn.go new file mode 100644 index 0000000000..d6cbeb7284 --- /dev/null +++ b/pkg/transport/conn.go @@ -0,0 +1,45 @@ +package transport + +import ( + "net" + "sync" +) + +// ManagedConn is a net.Conn with an observable terminal result. +type ManagedConn interface { + net.Conn + Wait() error +} + +// CloseWriter is optionally implemented by transports that support half-close. +type CloseWriter interface { + CloseWrite() error +} + +// ProcessInfo is optionally implemented by process-backed transports. +type ProcessInfo interface { + PID() int +} + +type managedState struct { + closeOnce sync.Once + waitOnce sync.Once + waitDone chan struct{} + waitErr error +} + +func newManagedState() *managedState { + return &managedState{waitDone: make(chan struct{})} +} + +func (s *managedState) finish(err error) { + s.waitOnce.Do(func() { + s.waitErr = err + close(s.waitDone) + }) +} + +func (s *managedState) wait() error { + <-s.waitDone + return s.waitErr +} diff --git a/pkg/transport/conn_test.go b/pkg/transport/conn_test.go new file mode 100644 index 0000000000..167502664c --- /dev/null +++ b/pkg/transport/conn_test.go @@ -0,0 +1,185 @@ +package transport + +import ( + "context" + "errors" + "io" + "os/exec" + "runtime" + "testing" + "time" +) + +const testOSWindows = "windows" + +func TestCallbackConnEchoAndWait(t *testing.T) { + conn, err := OpenCallbackConn( + context.Background(), + func(_ context.Context, stdin io.Reader, stdout io.Writer) error { + _, err := io.Copy(stdout, stdin) + return err + }, + CallbackConnOptions{}, + ) + if err != nil { + t.Fatal(err) + } + defer func() { _ = conn.Close() }() + + input := []byte("managed callback") + readDone := make(chan error, 1) + go func() { + _, err := conn.Write(input) + if err != nil { + readDone <- err + return + } + output := make([]byte, len(input)) + _, err = io.ReadFull(conn, output) + if err == nil && string(output) != string(input) { + err = errors.New("echo mismatch") + } + readDone <- err + }() + select { + case err := <-readDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("callback echo timed out") + } +} + +func TestCallbackConnPreservesErrorAndUnblocksOnClose(t *testing.T) { + wantErr := errors.New("callback failed") + conn, err := OpenCallbackConn( + context.Background(), + func(_ context.Context, _ io.Reader, _ io.Writer) error { + return wantErr + }, + CallbackConnOptions{}, + ) + if err != nil { + t.Fatal(err) + } + + if err := conn.Wait(); !errors.Is(err, wantErr) { + t.Fatalf("Wait() = %v, want %v", err, wantErr) + } + if err := conn.Wait(); !errors.Is(err, wantErr) { + t.Fatalf("second Wait() = %v, want %v", err, wantErr) + } + if err := conn.Close(); err != nil { + t.Fatalf("Close() = %v", err) + } +} + +func TestCallbackConnConcurrentClose(t *testing.T) { + conn, err := OpenCallbackConn( + context.Background(), + func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { + <-ctx.Done() + return ctx.Err() + }, + CallbackConnOptions{}, + ) + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + for range 32 { + go func() { + _ = conn.Close() + done <- struct{}{} + }() + } + for range 32 { + <-done + } + if err := conn.Wait(); !errors.Is(err, context.Canceled) { + t.Fatalf("Wait() = %v, want context.Canceled", err) + } +} + +func TestProcessConnEchoAndExit(t *testing.T) { + conn := openProcessTestConn(t, "cat") + input := []byte("managed process") + if _, err := conn.Write(input); err != nil { + t.Fatal(err) + } + output := make([]byte, len(input)) + if _, err := io.ReadFull(conn, output); err != nil { + t.Fatal(err) + } + if string(output) != string(input) { + t.Fatalf("output = %q, want %q", output, input) + } + if err := conn.Close(); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- conn.Wait() }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("process did not terminate after close") + } +} + +func openProcessTestConn(t *testing.T, command string) ManagedConn { + t.Helper() + if runtime.GOOS == testOSWindows { + t.Skip("shell command differs on Windows") + } + conn, err := StartProcessConn(context.Background(), ProcessSpec{ + Command: []string{"sh", "-c", command}, + }) + if err != nil { + t.Fatal(err) + } + return conn +} + +func TestProcessConnReportsExitError(t *testing.T) { + if runtime.GOOS == testOSWindows { + t.Skip("shell command differs on Windows") + } + + conn, err := StartProcessConn(context.Background(), ProcessSpec{ + Command: []string{"sh", "-c", "exit 7"}, + }) + if err != nil { + t.Fatal(err) + } + err = conn.Wait() + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("Wait() = %v, want exec.ExitError", err) + } + if exitErr.ExitCode() != 7 { + t.Fatalf("exit code = %d, want 7", exitErr.ExitCode()) + } +} + +func TestProcessConnCancellationTerminatesProcess(t *testing.T) { + if runtime.GOOS == testOSWindows { + t.Skip("shell command differs on Windows") + } + ctx, cancel := context.WithCancel(context.Background()) + conn, err := StartProcessConn(ctx, ProcessSpec{ + Command: []string{"sh", "-c", "sleep 30"}, + }) + if err != nil { + t.Fatal(err) + } + cancel() + _ = conn.Close() + done := make(chan error, 1) + go func() { done <- conn.Wait() }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("process did not terminate after cancellation") + } +} diff --git a/pkg/transport/lifecycle.go b/pkg/transport/lifecycle.go new file mode 100644 index 0000000000..3a6331336b --- /dev/null +++ b/pkg/transport/lifecycle.go @@ -0,0 +1,187 @@ +package transport + +import ( + "context" + "errors" + "io" + "sync" + + "github.com/devsy-org/devsy/pkg/log" +) + +type CloseReason string + +const ( + CloseUnknown CloseReason = "unknown" + ClosePeerEOF CloseReason = "peer_eof" + CloseContextCancelled CloseReason = "context_cancelled" + CloseLocalShutdown CloseReason = "local_shutdown" + CloseProviderExit CloseReason = "provider_exit" + CloseProviderError CloseReason = "provider_error" + CloseSSHTransportError CloseReason = "ssh_transport_error" + CloseSessionExit CloseReason = "session_exit" + CloseKeepaliveTimeout CloseReason = "keepalive_timeout" +) + +type Side string + +const ( + SideUnknown Side = "unknown" + SideProvider Side = "provider" + SideSSH Side = "ssh" + SideParent Side = "parent" +) + +const ( + TransportSideUnknown = SideUnknown + TransportSideProvider = SideProvider + TransportSideSSH = SideSSH + TransportSideParent = SideParent +) + +const ( + TransportCloseUnknown = CloseUnknown + TransportClosePeerEOF = ClosePeerEOF + TransportCloseContextCancelled = CloseContextCancelled + TransportCloseLocalShutdown = CloseLocalShutdown + TransportCloseProviderExit = CloseProviderExit + TransportCloseProviderError = CloseProviderError + TransportCloseSSHTransportError = CloseSSHTransportError + TransportCloseSessionExit = CloseSessionExit + TransportCloseKeepaliveTimeout = CloseKeepaliveTimeout +) + +type CloseInfo struct { + Reason CloseReason + Err error + Side Side +} + +type PersistentLifecycle struct { + cancel context.CancelFunc + done chan struct{} + once sync.Once + mu sync.RWMutex + info CloseInfo +} + +// NewPersistentLifecycle derives a cancellable context from parent and +// returns it alongside the lifecycle that controls it. +func NewPersistentLifecycle(parent context.Context) (*PersistentLifecycle, context.Context) { + if parent == nil { + parent = context.Background() + } + ctx, cancel := context.WithCancel(parent) + lifecycle := &PersistentLifecycle{ + cancel: cancel, done: make(chan struct{}), + info: CloseInfo{Reason: CloseUnknown, Side: SideUnknown}, + } + return lifecycle, ctx +} + +func (l *PersistentLifecycle) Done() <-chan struct{} { return l.done } + +func (l *PersistentLifecycle) Close(info CloseInfo) { + l.once.Do(func() { + l.mu.Lock() + l.info = info + l.mu.Unlock() + l.cancel() + close(l.done) + }) +} + +func (l *PersistentLifecycle) CloseInfo() CloseInfo { + l.mu.RLock() + defer l.mu.RUnlock() + return l.info +} + +func Classify(side Side, err error, parent context.Context) CloseInfo { + if parent != nil && parent.Err() != nil { + return CloseInfo{Reason: CloseContextCancelled, Side: SideParent, Err: parent.Err()} + } + if err == nil { + return classifyNil(side) + } + if errors.Is(err, io.EOF) { + return CloseInfo{Reason: ClosePeerEOF, Side: side, Err: err} + } + return classifyError(side, err) +} + +func classifyNil(side Side) CloseInfo { + switch side { + case SideProvider: + return CloseInfo{Reason: CloseProviderExit, Side: side} + case SideSSH: + return CloseInfo{Reason: CloseSessionExit, Side: side} + default: + return CloseInfo{Reason: CloseLocalShutdown, Side: side} + } +} + +func classifyError(side Side, err error) CloseInfo { + switch side { + case SideProvider: + return CloseInfo{Reason: CloseProviderError, Side: side, Err: err} + case SideSSH: + return CloseInfo{Reason: CloseSSHTransportError, Side: side, Err: err} + default: + return CloseInfo{Reason: CloseUnknown, Side: side, Err: err} + } +} + +type LogMetadata struct { + Provider string + Mode string + Workspace string + TransportImpl string +} + +func LogClose(info CloseInfo, metadata LogMetadata) { + log.Debugw("ssh transport closed", + "reason", info.Reason, "side", info.Side, "error", info.Err, + "provider", metadata.Provider, "mode", metadata.Mode, + "workspace", metadata.Workspace, "transport_impl", metadata.TransportImpl, + ) +} + +type RunManagedOptions struct { + Parent context.Context + Conn ManagedConn + Handler func(context.Context) error + Metadata LogMetadata + TransportSide Side +} + +func RunManaged(opts RunManagedOptions) error { + if opts.Parent == nil { + opts.Parent = context.Background() + } + if opts.Conn == nil { + return errors.New("managed connection is required") + } + if opts.Handler == nil { + return errors.New("handler is required") + } + lifecycle, ctx := NewPersistentLifecycle(opts.Parent) + handlerDone := make(chan error, 1) + go func() { handlerDone <- opts.Handler(ctx) }() + connDone := make(chan error, 1) + go func() { connDone <- opts.Conn.Wait() }() + + var info CloseInfo + select { + case err := <-connDone: + info = Classify(opts.TransportSide, err, opts.Parent) + case err := <-handlerDone: + info = Classify(SideSSH, err, opts.Parent) + case <-opts.Parent.Done(): + info = Classify(SideParent, opts.Parent.Err(), opts.Parent) + } + lifecycle.Close(info) + _ = opts.Conn.Close() + LogClose(lifecycle.CloseInfo(), opts.Metadata) + return lifecycle.CloseInfo().Err +} diff --git a/pkg/transport/lifecycle_test.go b/pkg/transport/lifecycle_test.go new file mode 100644 index 0000000000..7de355db2a --- /dev/null +++ b/pkg/transport/lifecycle_test.go @@ -0,0 +1,123 @@ +package transport + +import ( + "context" + "errors" + "net" + "testing" + "time" +) + +type testManagedConn struct { + net.Conn + waitErr error + waitDone chan struct{} + closed chan struct{} +} + +func newTestManagedConn(waitErr error) *testManagedConn { + conn := &testManagedConn{ + Conn: &stubConn{}, + waitErr: waitErr, + waitDone: make(chan struct{}), + closed: make(chan struct{}), + } + if waitErr != nil { + close(conn.waitDone) + } + return conn +} + +func (c *testManagedConn) Wait() error { + <-c.waitDone + return c.waitErr +} + +func (c *testManagedConn) Close() error { + select { + case <-c.closed: + default: + close(c.closed) + select { + case <-c.waitDone: + default: + close(c.waitDone) + } + } + return nil +} + +func TestRunManagedReturnsHandlerError(t *testing.T) { + wantErr := errors.New("handler failed") + conn := newTestManagedConn(nil) + err := RunManaged(RunManagedOptions{ + Parent: context.Background(), + Conn: conn, + Handler: func(context.Context) error { + return wantErr + }, + TransportSide: SideProvider, + }) + if !errors.Is(err, wantErr) { + t.Fatalf("RunManaged() = %v, want %v", err, wantErr) + } + select { + case <-conn.closed: + default: + t.Fatal("connection was not closed") + } +} + +func TestRunManagedReturnsTransportError(t *testing.T) { + wantErr := errors.New("provider failed") + conn := newTestManagedConn(wantErr) + err := RunManaged(RunManagedOptions{ + Parent: context.Background(), + Conn: conn, + Handler: func(ctx context.Context) error { + <-ctx.Done() + return ctx.Err() + }, + TransportSide: SideProvider, + }) + if !errors.Is(err, wantErr) { + t.Fatalf("RunManaged() = %v, want %v", err, wantErr) + } +} + +func TestRunManagedReturnsParentCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + conn := newTestManagedConn(nil) + done := make(chan error, 1) + go func() { + done <- RunManaged(RunManagedOptions{ + Parent: ctx, + Conn: conn, + Handler: func(ctx context.Context) error { + <-ctx.Done() + return ctx.Err() + }, + TransportSide: SideProvider, + }) + }() + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("RunManaged() = %v, want context.Canceled", err) + } + case <-time.After(time.Second): + t.Fatal("RunManaged did not stop after cancellation") + } +} + +type stubConn struct{} + +func (*stubConn) Read([]byte) (int, error) { return 0, net.ErrClosed } +func (*stubConn) Write([]byte) (int, error) { return 0, net.ErrClosed } +func (*stubConn) Close() error { return nil } +func (*stubConn) LocalAddr() net.Addr { return Addr{} } +func (*stubConn) RemoteAddr() net.Addr { return Addr{} } +func (*stubConn) SetDeadline(time.Time) error { return nil } +func (*stubConn) SetReadDeadline(time.Time) error { return nil } +func (*stubConn) SetWriteDeadline(time.Time) error { return nil } diff --git a/pkg/transport/process_conn.go b/pkg/transport/process_conn.go new file mode 100644 index 0000000000..782e75d0c5 --- /dev/null +++ b/pkg/transport/process_conn.go @@ -0,0 +1,108 @@ +package transport + +import ( + "context" + "fmt" + "io" + "net" + "os" + "os/exec" + "time" +) + +type ProcessSpec struct { + Command []string + Env []string + Dir string + Stderr io.Writer + + LocalAddr net.Addr + RemoteAddr net.Addr +} + +type processConn struct { + stdin *os.File + stdout *os.File + cmd *exec.Cmd + cancel context.CancelFunc + state *managedState +} + +func StartProcessConn(ctx context.Context, spec ProcessSpec) (ManagedConn, error) { + if len(spec.Command) == 0 || spec.Command[0] == "" { + return nil, fmt.Errorf("command is required") + } + if ctx == nil { + ctx = context.Background() + } + + childStdin, parentStdin, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("create process stdin pipe: %w", err) + } + parentStdout, childStdout, err := os.Pipe() + if err != nil { + _ = childStdin.Close() + _ = parentStdin.Close() + return nil, fmt.Errorf("create process stdout pipe: %w", err) + } + + processCtx, cancel := context.WithCancel(ctx) + // Command and arguments are supplied by the caller + cmd := exec.CommandContext(processCtx, spec.Command[0], spec.Command[1:]...) // #nosec G204 + cmd.Stdin = childStdin + cmd.Stdout = childStdout + cmd.Stderr = spec.Stderr + cmd.Env = spec.Env + cmd.Dir = spec.Dir + if err := cmd.Start(); err != nil { + cancel() + _ = childStdin.Close() + _ = parentStdin.Close() + _ = parentStdout.Close() + _ = childStdout.Close() + return nil, fmt.Errorf("start process: %w", err) + } + _ = childStdin.Close() + _ = childStdout.Close() + + conn := &processConn{ + stdin: parentStdin, + stdout: parentStdout, + cmd: cmd, + cancel: cancel, + state: newManagedState(), + } + return &addrConn{Conn: conn, local: spec.LocalAddr, remote: spec.RemoteAddr}, nil +} + +func (c *processConn) Read(p []byte) (int, error) { return c.stdout.Read(p) } +func (c *processConn) Write(p []byte) (int, error) { return c.stdin.Write(p) } + +func (c *processConn) Close() error { + var err error + c.state.closeOnce.Do(func() { + c.cancel() + if closeErr := c.stdin.Close(); closeErr != nil { + err = closeErr + } + if closeErr := c.stdout.Close(); err == nil && closeErr != nil { + err = closeErr + } + }) + return err +} + +func (c *processConn) LocalAddr() net.Addr { return NewAddr("process") } +func (c *processConn) RemoteAddr() net.Addr { return NewAddr("process") } +func (c *processConn) SetDeadline(time.Time) error { return os.ErrInvalid } +func (c *processConn) SetReadDeadline(time.Time) error { return os.ErrInvalid } +func (c *processConn) SetWriteDeadline(time.Time) error { return os.ErrInvalid } +func (c *processConn) PID() int { return c.cmd.Process.Pid } +func (c *processConn) Wait() error { + c.state.waitOnce.Do(func() { + c.state.waitErr = c.cmd.Wait() + close(c.state.waitDone) + }) + return c.state.wait() +} diff --git a/pkg/tunnel/container.go b/pkg/tunnel/container.go index 3096f3b91c..39a9be375f 100644 --- a/pkg/tunnel/container.go +++ b/pkg/tunnel/container.go @@ -8,7 +8,7 @@ import ( "context" "fmt" "io" - "os" + "sync" "time" "github.com/devsy-org/devsy/pkg/agent" @@ -17,6 +17,7 @@ import ( "github.com/devsy-org/devsy/pkg/log" "github.com/devsy-org/devsy/pkg/provider" devssh "github.com/devsy-org/devsy/pkg/ssh" + "github.com/devsy-org/devsy/pkg/transport" "golang.org/x/crypto/ssh" ) @@ -53,50 +54,68 @@ func (c *ContainerTunnel) Run( timeout := config.ParseTimeOption(cfg, config.ContextOptionAgentInjectTimeout) - pb, err := NewPipeBridge() + conn, err := transport.OpenCallbackConn( + ctx, + func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { + return c.runHostTunnel(ctx, stdin, stdout, timeout) + }, + transport.CallbackConnOptions{ + LocalAddr: transport.NewAddr("workspace"), + RemoteAddr: transport.NewAddr("provider:" + c.client.Provider()), + }, + ) if err != nil { return err } - defer pb.Close() - - info, err := runPersistentPair( - ctx, - pb, - func(ctx context.Context, stdin, stdout *os.File) error { - return c.runHostTunnel(ctx, stdin, stdout, timeout) + defer func() { _ = conn.Close() }() + return transport.RunManaged(transport.RunManagedOptions{ + Parent: ctx, + Conn: conn, + TransportSide: transport.SideProvider, + Metadata: transport.LogMetadata{ + Provider: c.client.Provider(), Mode: workspaceSubcommand, + Workspace: c.client.Workspace(), TransportImpl: "callback", }, - func(ctx context.Context, stdout, stdin *os.File) error { - sshClient, err := devssh.StdioClient(stdout, stdin) + Handler: func(ctx context.Context) error { + sshClient, err := devssh.ClientFromConn(conn, "", nil) if err != nil { return fmt.Errorf("create ssh client: %w", err) } - defer func() { _ = sshClient.Close() }() defer log.Debugf("connection to container closed") log.Debugf("connected to host") - + updateCtx, cancelUpdate := context.WithCancel(ctx) + var updateWG sync.WaitGroup if c.updateConfigInterval > 0 { - go c.updateConfig(ctx, sshClient) + updateWG.Go(func() { + c.updateConfig(updateCtx, sshClient) + }) } - + defer stopUpdateThenClose(cancelUpdate, &updateWG, sshClient) if err := c.runInContainer(ctx, sshClient, handler, envVars); err != nil { return fmt.Errorf("run in container: %w", err) } return nil }, - ) - LogTransportClose(info, TransportLogMetadata{ - Provider: c.client.Provider(), - Mode: workspaceSubcommand, - Workspace: c.client.Workspace(), }) - return err +} + +// stopUpdateThenClose cancels the periodic config-update and waits +// for it to fully exit before closing sshClient. +func stopUpdateThenClose( + cancelUpdate context.CancelFunc, + updateWG *sync.WaitGroup, + sshClient io.Closer, +) { + cancelUpdate() + updateWG.Wait() + _ = sshClient.Close() } // runHostTunnel injects the devsy agent onto the host and starts the SSH server, // forwarding stdio through the provided pipes. func (c *ContainerTunnel) runHostTunnel( ctx context.Context, - stdinReader, stdoutWriter *os.File, + stdinReader io.Reader, stdoutWriter io.Writer, timeout time.Duration, ) error { writer, done := log.PipeJSONStreamWithFallback(log.PassthroughWriter()) @@ -189,83 +208,45 @@ func (c *ContainerTunnel) runInContainer( return err } - pb, err := NewPipeBridge() - if err != nil { - return err - } - defer pb.Close() - - cancelCtx, cancel := context.WithCancel(ctx) - defer cancel() - - // tunnel to container - tunnelDone := make(chan error, 1) - go func() { - tunnelDone <- c.runContainerTunnel(cancelCtx, containerTunnelOpts{ - sshClient: sshClient, - workspaceInfo: workspaceInfo, - stdinReader: pb.StdinReader, - stdoutWriter: pb.StdoutWriter, - envVars: envVars, - }) - }() - - containerClient, err := devssh.StdioClient(pb.StdoutReader, pb.StdinWriter) - if err != nil { - select { // check if the tunnel goroutine has already returned an error - case tunnelErr := <-tunnelDone: - if tunnelErr != nil { - return tunnelErr - } - default: - } - return fmt.Errorf("ssh client: %w", err) - } - defer func() { _ = containerClient.Close() }() - log.Debugf("connected to container") - - return handler(cancelCtx, containerClient) -} - -type containerTunnelOpts struct { - sshClient *ssh.Client - workspaceInfo string - stdinReader *os.File - stdoutWriter *os.File - envVars map[string]string -} - -// runContainerTunnel runs the container tunnel SSH command. It closes -// stdoutWriter on exit so StdioClient gets EOF when the tunnel dies. -func (c *ContainerTunnel) runContainerTunnel(ctx context.Context, opts containerTunnelOpts) error { - writer, done := log.PipeJSONStream() + writer, writerDone := log.PipeJSONStream() defer func() { _ = writer.Close() - <-done + <-writerDone }() - defer func() { _ = opts.stdoutWriter.Close() }() - - log.Debugf("Run container tunnel") - defer log.Debugf("Container tunnel exited") command := fmt.Sprintf( "%q internal agent container-tunnel --workspace-info %q", c.client.AgentPath(), - opts.workspaceInfo, + workspaceInfo, ) if log.DebugEnabled() { command += " --debug" } - err := devssh.Run(ctx, devssh.RunOptions{ - Client: opts.sshClient, + containerConn, err := devssh.OpenSessionConn(ctx, sshClient, devssh.SessionConnOptions{ Command: command, - Stdin: opts.stdinReader, - Stdout: opts.stdoutWriter, + Env: envVars, Stderr: writer, - EnvVars: opts.envVars, }) - if err != nil && ctx.Err() == nil { - return fmt.Errorf("container tunnel: %w", err) + if err != nil { + return fmt.Errorf("open container transport: %w", err) } - return nil + defer func() { _ = containerConn.Close() }() + return transport.RunManaged(transport.RunManagedOptions{ + Parent: ctx, + Conn: containerConn, + TransportSide: transport.SideProvider, + Metadata: transport.LogMetadata{ + Mode: workspaceSubcommand, + TransportImpl: "ssh_session", + }, + Handler: func(ctx context.Context) error { + containerClient, err := devssh.ClientFromConn(containerConn, "", nil) + if err != nil { + return fmt.Errorf("ssh client: %w", err) + } + defer func() { _ = containerClient.Close() }() + log.Debugf("connected to container") + return handler(ctx, containerClient) + }, + }) } diff --git a/pkg/tunnel/container_test.go b/pkg/tunnel/container_test.go new file mode 100644 index 0000000000..aa83a6c343 --- /dev/null +++ b/pkg/tunnel/container_test.go @@ -0,0 +1,73 @@ +package tunnel + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" +) + +type fakeCloser struct { + closed atomic.Bool + inFlight *atomic.Bool +} + +func (f *fakeCloser) Close() error { + f.closed.Store(true) + if f.inFlight != nil && f.inFlight.Load() { + panic("sshClient closed while updateConfig goroutine still running") + } + return nil +} + +func TestStopUpdateThenCloseWaitsForGoroutine(t *testing.T) { + updateCtx, cancelUpdate := context.WithCancel(context.Background()) + defer cancelUpdate() + + var inFlight atomic.Bool + closer := &fakeCloser{inFlight: &inFlight} + + started := make(chan struct{}) + release := make(chan struct{}) + + var updateWG sync.WaitGroup + updateWG.Go(func() { + inFlight.Store(true) + close(started) + <-updateCtx.Done() + <-release + inFlight.Store(false) + }) + + <-started + + done := make(chan struct{}) + go func() { + stopUpdateThenClose(cancelUpdate, &updateWG, closer) + close(done) + }() + + select { + case <-done: + t.Fatal("stopUpdateThenClose returned while updateConfig goroutine was still in flight") + case <-time.After(50 * time.Millisecond): + // stopUpdateThenClose is blocked on updateWG.Wait(); the goroutine + // is still in flight and Close must not have run. + } + if closer.closed.Load() { + t.Fatal("sshClient closed before updateConfig goroutine finished") + } + + close(release) + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("stopUpdateThenClose did not return after goroutine finished") + } + + if !closer.closed.Load() { + t.Fatal("expected sshClient to be closed") + } +} diff --git a/pkg/tunnel/direct.go b/pkg/tunnel/direct.go index 33d956479c..21fbf286da 100644 --- a/pkg/tunnel/direct.go +++ b/pkg/tunnel/direct.go @@ -3,61 +3,43 @@ package tunnel import ( "context" "io" - "os" devssh "github.com/devsy-org/devsy/pkg/ssh" + "github.com/devsy-org/devsy/pkg/transport" ) // Tunnel defines the function to create an "outer" tunnel. type Tunnel func(ctx context.Context, stdin io.Reader, stdout io.Writer) error -// NewTunnel creates a tunnel to the devcontainer using generic functions -// to establish the "outer" and "inner" tunnel, used by proxy clients. -// The tunnel will be an SSH connection with its STDIO as arguments and the -// handler will be the function to execute the command using the -// connected SSH client. +// NewTunnel creates a managed SSH tunnel using generic transport callbacks. func NewTunnel(ctx context.Context, tunnel Tunnel, handler Handler) error { - return newTunnel(ctx, tunnel, handler, TransportLogMetadata{}) -} - -// NewTunnelWithMetadata is NewTunnel with context for the canonical transport -// close event. -func NewTunnelWithMetadata( - ctx context.Context, - tunnel Tunnel, - handler Handler, - metadata TransportLogMetadata, -) error { - return newTunnel(ctx, tunnel, handler, metadata) -} - -func newTunnel( - ctx context.Context, - tunnel Tunnel, - handler Handler, - metadata TransportLogMetadata, -) error { - pb, err := NewPipeBridge() + conn, err := transport.OpenCallbackConn( + ctx, + func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { + return tunnel(ctx, stdin, stdout) + }, + transport.CallbackConnOptions{ + LocalAddr: transport.NewAddr("tunnel"), + RemoteAddr: transport.NewAddr("provider"), + }, + ) if err != nil { return err } - defer pb.Close() + defer func() { _ = conn.Close() }() - info, err := runPersistentPair( - ctx, - pb, - func(ctx context.Context, stdin, stdout *os.File) error { - return tunnel(ctx, stdin, stdout) - }, - func(ctx context.Context, stdout, stdin *os.File) error { - sshClient, err := devssh.StdioClient(stdout, stdin) + return transport.RunManaged(transport.RunManagedOptions{ + Parent: ctx, + Conn: conn, + TransportSide: transport.SideProvider, + Metadata: transport.LogMetadata{TransportImpl: "callback"}, + Handler: func(ctx context.Context) error { + sshClient, err := devssh.ClientFromConn(conn, "", nil) if err != nil { return err } defer func() { _ = sshClient.Close() }() return handler(ctx, sshClient) }, - ) - LogTransportClose(info, metadata) - return err + }) } diff --git a/pkg/tunnel/pipebridge.go b/pkg/tunnel/pipebridge.go index e0b501edfa..dff7df33fd 100644 --- a/pkg/tunnel/pipebridge.go +++ b/pkg/tunnel/pipebridge.go @@ -13,8 +13,7 @@ import ( const joinTimeout = 5 * time.Second // PipeBridge is a duplex bridge between two concurrent units of work that share -// resources. It is used to implement the bridge between the caller's stdio and -// the exec's stdio. +// resources. type PipeBridge struct { StdoutReader *os.File StdoutWriter *os.File diff --git a/pkg/tunnel/transport.go b/pkg/tunnel/transport.go deleted file mode 100644 index 55d68201fc..0000000000 --- a/pkg/tunnel/transport.go +++ /dev/null @@ -1,208 +0,0 @@ -package tunnel - -import ( - "context" - "errors" - "io" - "os" - "sync" - - "github.com/devsy-org/devsy/pkg/log" -) - -// TransportCloseReason describes why a persistent transport ended. The reason -// is recorded by the transport owner before any shared-resource cleanup runs. -type TransportCloseReason string - -const ( - CloseUnknown TransportCloseReason = "unknown" - ClosePeerEOF TransportCloseReason = "peer_eof" - CloseContextCancelled TransportCloseReason = "context_cancelled" - CloseLocalShutdown TransportCloseReason = "local_shutdown" - CloseProviderExit TransportCloseReason = "provider_exit" - CloseProviderError TransportCloseReason = "provider_error" - CloseSSHTransportError TransportCloseReason = "ssh_transport_error" - CloseSessionExit TransportCloseReason = "session_exit" - CloseKeepaliveTimeout TransportCloseReason = "keepalive_timeout" -) - -// TransportSide identifies the participant that first caused transport -// shutdown. -type TransportSide string - -const ( - TransportSideUnknown TransportSide = "unknown" - TransportSideProvider TransportSide = "provider" - TransportSideSSH TransportSide = "ssh" - TransportSideParent TransportSide = "parent" -) - -// TransportCloseInfo is the immutable, first-close-wins result of a -// persistent transport's lifecycle. Err is the initiating error, not a later -// cleanup error caused by shutting down the other side. -type TransportCloseInfo struct { - Reason TransportCloseReason - Err error - Side TransportSide -} - -// PersistentTransport owns lifecycle state shared by a persistent transport. -// It provides a transport-scoped context, a one-shot completion signal, and the -// authoritative close reason. It deliberately does not own the underlying -// sockets, subprocesses, or PipeBridge; the caller remains responsible for -// those resources while using this object as the lifecycle owner. -type PersistentTransport struct { - ctx context.Context - cancel context.CancelFunc - - done chan struct{} - once sync.Once - - mu sync.RWMutex - info TransportCloseInfo -} - -// NewPersistentTransport creates a persistent transport whose context is a -// child of parent. Parent cancellation propagates to Context immediately. -func NewPersistentTransport(parent context.Context) *PersistentTransport { - ctx, cancel := context.WithCancel(parent) - return &PersistentTransport{ - ctx: ctx, - cancel: cancel, - done: make(chan struct{}), - info: TransportCloseInfo{Reason: CloseUnknown, Side: TransportSideUnknown}, - } -} - -// Context returns the context shared by persistent transport work. -func (t *PersistentTransport) Context() context.Context { - return t.ctx -} - -// Done is closed exactly once when Close records the transport's terminal -// state. -func (t *PersistentTransport) Done() <-chan struct{} { - return t.done -} - -// Close records info and shuts down the transport. The first call wins; -// repeated calls are safe and cannot replace CloseInfo or panic on Done. -func (t *PersistentTransport) Close(info TransportCloseInfo) { - t.once.Do(func() { - t.mu.Lock() - t.info = info - t.mu.Unlock() - - t.cancel() - close(t.done) - }) -} - -// CloseInfo returns the first recorded close information. Before Close it is -// the zero value, whose Reason is CloseUnknown. -func (t *PersistentTransport) CloseInfo() TransportCloseInfo { - t.mu.RLock() - defer t.mu.RUnlock() - return t.info -} - -// classifyTransportError maps a side completion into the semantic close -// reason for a persistent transport. Parent cancellation takes precedence over -// side errors because cancellation is intentional caller shutdown. -func classifyTransportError( - side TransportSide, - err error, - parentCtx context.Context, -) TransportCloseInfo { - if parentCtx.Err() != nil { - return TransportCloseInfo{ - Reason: CloseContextCancelled, - Side: TransportSideParent, - Err: parentCtx.Err(), - } - } - - if err == nil { - return TransportCloseInfo{ - Reason: CloseLocalShutdown, - Side: side, - } - } - - if errors.Is(err, io.EOF) { - return TransportCloseInfo{ - Reason: ClosePeerEOF, - Side: side, - Err: err, - } - } - - switch side { - case TransportSideProvider: - return TransportCloseInfo{Reason: CloseProviderError, Side: side, Err: err} - case TransportSideSSH: - return TransportCloseInfo{Reason: CloseSSHTransportError, Side: side, Err: err} - default: - return TransportCloseInfo{Reason: CloseUnknown, Side: side, Err: err} - } -} - -// TransportLogMetadata supplies optional context for the canonical close log. -type TransportLogMetadata struct { - Provider string - Mode string - Workspace string -} - -// LogTransportClose emits the single canonical close event for a persistent -// SSH transport. Cleanup diagnostics remain separate from this event. -func LogTransportClose(info TransportCloseInfo, metadata TransportLogMetadata) { - log.Debugw("ssh transport closed", - "reason", info.Reason, - "side", info.Side, - "error", info.Err, - "provider", metadata.Provider, - "mode", metadata.Mode, - "workspace", metadata.Workspace, - ) -} - -// runPersistentPair gives a PipeBridge pair an explicit persistent lifecycle -// owner without changing PipeBridge's byte movement or join semantics. -func runPersistentPair( - parent context.Context, - pb *PipeBridge, - tunnelFn TunnelFunc, - handlerFn HandlerFunc, -) (TransportCloseInfo, error) { - transport := NewPersistentTransport(parent) - - go func() { - select { - case <-parent.Done(): - transport.Close(classifyTransportError(TransportSideParent, parent.Err(), parent)) - case <-transport.Done(): - } - }() - - err := pb.RunPair( - transport.Context(), - func(ctx context.Context, stdin, stdout *os.File) error { - err := tunnelFn(ctx, stdin, stdout) - transport.Close(classifyTransportError(TransportSideProvider, err, parent)) - return err - }, - func(ctx context.Context, stdout, stdin *os.File) error { - err := handlerFn(ctx, stdout, stdin) - transport.Close(classifyTransportError(TransportSideSSH, err, parent)) - return err - }, - ) - - select { - case <-transport.Done(): - default: - transport.Close(classifyTransportError(TransportSideUnknown, err, parent)) - } - return transport.CloseInfo(), err -} diff --git a/pkg/tunnel/transport_test.go b/pkg/tunnel/transport_test.go deleted file mode 100644 index 6d6487f2ef..0000000000 --- a/pkg/tunnel/transport_test.go +++ /dev/null @@ -1,269 +0,0 @@ -package tunnel - -import ( - "context" - "errors" - "io" - "os" - "sync" - "testing" - "time" -) - -func TestPersistentTransportStartsUnknown(t *testing.T) { - t.Parallel() - - transport := NewPersistentTransport(context.Background()) - info := transport.CloseInfo() - if info.Reason != CloseUnknown || info.Side != TransportSideUnknown || info.Err != nil { - t.Fatalf("initial CloseInfo() = %+v, want unknown", info) - } -} - -func TestPersistentTransportFirstCloseWins(t *testing.T) { - t.Parallel() - - transport := NewPersistentTransport(context.Background()) - firstErr := errors.New("provider stopped") - transport.Close(TransportCloseInfo{ - Reason: CloseProviderError, - Side: TransportSideProvider, - Err: firstErr, - }) - transport.Close(TransportCloseInfo{ - Reason: CloseSSHTransportError, - Side: TransportSideSSH, - Err: errors.New("ssh stopped"), - }) - - info := transport.CloseInfo() - if info.Reason != CloseProviderError || - info.Side != TransportSideProvider || - !errors.Is(info.Err, firstErr) { - t.Fatalf("CloseInfo() = %+v, want first close", info) - } - select { - case <-transport.Done(): - default: - t.Fatal("Done() is still open after Close") - } - select { - case <-transport.Context().Done(): - default: - t.Fatal("transport context is still active after Close") - } -} - -func TestPersistentTransportCloseIsSafeConcurrently(t *testing.T) { - t.Parallel() - - transport := NewPersistentTransport(context.Background()) - var wg sync.WaitGroup - for range 32 { - wg.Go(func() { - transport.Close(TransportCloseInfo{Reason: CloseLocalShutdown, Side: TransportSideSSH}) - }) - } - wg.Wait() - select { - case <-transport.Done(): - case <-time.After(time.Second): - t.Fatal("Done() did not close") - } -} - -func TestPersistentTransportParentCancellationPropagates(t *testing.T) { - t.Parallel() - - parent, cancel := context.WithCancel(context.Background()) - transport := NewPersistentTransport(parent) - cancel() - - select { - case <-transport.Context().Done(): - case <-time.After(time.Second): - t.Fatal("transport context did not inherit parent cancellation") - } - if !errors.Is(transport.Context().Err(), context.Canceled) { - t.Fatalf("transport context error = %v, want context.Canceled", transport.Context().Err()) - } -} - -func TestClassifyTransportError(t *testing.T) { - t.Parallel() - - for _, tt := range transportErrorTests { - t.Run(tt.name, func(t *testing.T) { - parentCtx := context.Background() - if tt.cancelParent { - var cancel context.CancelFunc - parentCtx, cancel = context.WithCancel(parentCtx) - cancel() - } - got := classifyTransportError(tt.side, tt.err, parentCtx) - if got.Reason != tt.wantReason || got.Side != tt.wantSide { - t.Fatalf( - "classifyTransportError() = %+v, want reason=%s side=%s", - got, - tt.wantReason, - tt.wantSide, - ) - } - if tt.wantErr != nil && !errors.Is(got.Err, tt.wantErr) { - t.Fatalf("classified error = %v, want %v", got.Err, tt.wantErr) - } - }) - } -} - -type fmtWrappedEOF struct{} - -func (fmtWrappedEOF) Error() string { return "peer: EOF" } -func (fmtWrappedEOF) Unwrap() error { return io.EOF } - -type transportErrorTest struct { - name string - side TransportSide - err error - cancelParent bool - wantReason TransportCloseReason - wantSide TransportSide - wantErr error -} - -var ( - errProviderExit = errors.New("provider exited") - errSSHTransport = errors.New("ssh failed") - transportErrorTests = []transportErrorTest{ - { - name: "nil provider completion", - side: TransportSideProvider, - wantReason: CloseLocalShutdown, - wantSide: TransportSideProvider, - }, - { - name: "peer eof", - side: TransportSideSSH, - err: fmtWrappedEOF{}, - wantReason: ClosePeerEOF, - wantSide: TransportSideSSH, - }, - { - name: "parent cancellation", - side: TransportSideProvider, - err: errProviderExit, - cancelParent: true, - wantReason: CloseContextCancelled, - wantSide: TransportSideParent, - wantErr: context.Canceled, - }, - { - name: "provider error", - side: TransportSideProvider, - err: errProviderExit, - wantReason: CloseProviderError, - wantSide: TransportSideProvider, - wantErr: errProviderExit, - }, - { - name: "ssh error", - side: TransportSideSSH, - err: errSSHTransport, - wantReason: CloseSSHTransportError, - wantSide: TransportSideSSH, - wantErr: errSSHTransport, - }, - { - name: "unknown side", - side: TransportSideUnknown, - err: errSSHTransport, - wantReason: CloseUnknown, - wantSide: TransportSideUnknown, - wantErr: errSSHTransport, - }, - } -) - -func TestRunPersistentPairAttributesProviderFailure(t *testing.T) { - t.Parallel() - - pb, err := NewPipeBridge() - if err != nil { - t.Fatal(err) - } - defer pb.Close() - providerErr := errors.New("provider failed") - - info, _ := runPersistentPair( - context.Background(), - pb, - func(context.Context, *os.File, *os.File) error { return providerErr }, - func(ctx context.Context, _ *os.File, _ *os.File) error { - <-ctx.Done() - return ctx.Err() - }, - ) - if info.Reason != CloseProviderError || - info.Side != TransportSideProvider || - !errors.Is(info.Err, providerErr) { - t.Fatalf("close info = %+v, want provider failure", info) - } -} - -func TestRunPersistentPairAttributesSSHFailure(t *testing.T) { - t.Parallel() - - pb, err := NewPipeBridge() - if err != nil { - t.Fatal(err) - } - defer pb.Close() - sshErr := errors.New("ssh failed") - - info, _ := runPersistentPair( - context.Background(), - pb, - func(ctx context.Context, _ *os.File, _ *os.File) error { - <-ctx.Done() - return ctx.Err() - }, - func(context.Context, *os.File, *os.File) error { return sshErr }, - ) - if info.Reason != CloseSSHTransportError || - info.Side != TransportSideSSH || - !errors.Is(info.Err, sshErr) { - t.Fatalf("close info = %+v, want ssh failure", info) - } -} - -func TestRunPersistentPairAttributesParentCancellation(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - pb, err := NewPipeBridge() - if err != nil { - t.Fatal(err) - } - defer pb.Close() - - done := make(chan TransportCloseInfo, 1) - go func() { - info, _ := runPersistentPair( - ctx, - pb, - func(ctx context.Context, _ *os.File, _ *os.File) error { <-ctx.Done(); return ctx.Err() }, - func(ctx context.Context, _ *os.File, _ *os.File) error { <-ctx.Done(); return ctx.Err() }, - ) - done <- info - }() - cancel() - - select { - case info := <-done: - if info.Reason != CloseContextCancelled || info.Side != TransportSideParent { - t.Fatalf("close info = %+v, want parent cancellation", info) - } - case <-time.After(2 * time.Second): - t.Fatal("persistent pair did not stop after parent cancellation") - } -}