Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 0 additions & 11 deletions pkg/devcontainer/sshtunnel/sshtunnel.go
Original file line number Diff line number Diff line change
Expand Up @@ -257,25 +257,14 @@ func runCommandInSSHTunnel(ctx context.Context, p sshCommandParams, stdin, stdou
Stderr: streamer,
})
if err != nil {
_ = streamer.Close()
if out := streamer.ErrorOutput(); out != "" {
return fmt.Errorf("run agent command failed: %w\n%s", err, out)
}
return fmt.Errorf("run agent command failed: %w", err)
}
return nil
}

const (
maxLogLines = 25
maxLogBytes = 256 * 1024
)

func newSSHTunnelJSONLogStreamer() *log.JSONLogStreamer {
return log.NewJSONLogStreamer(log.StreamerOptions{
FallbackLevel: log.LevelDebug,
CaptureLines: maxLogLines,
CaptureBytes: maxLogBytes,
DetectLevelPrefixes: true,
TreatUnknownJSONAsDebug: true,
})
Expand Down
21 changes: 21 additions & 0 deletions pkg/devcontainer/sshtunnel/streamer_safety_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package sshtunnel

import (
"testing"

"github.com/stretchr/testify/suite"
)

type streamerSafetySuite struct{ suite.Suite }

func TestStreamerSafetySuite(t *testing.T) {
suite.Run(t, new(streamerSafetySuite))
}

func (s *streamerSafetySuite) TestAgentOutputIsNotRetainedForErrors() {
streamer := newSSHTunnelJSONLogStreamer()
_, err := streamer.Write([]byte("synthetic-workspace-credential\n"))
s.Require().NoError(err)
s.Require().NoError(streamer.Close())
s.Empty(streamer.ErrorOutput())
}
35 changes: 30 additions & 5 deletions pkg/tunnel/pipebridge.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,13 @@ type (
HandlerFunc func(ctx context.Context, stdout *os.File, stdin *os.File) error
)

type pairCompletion struct {
tunnel bool
err error
}

func (*pairCompletion) Error() string { return "pipe bridge completed" }

// RunPair runs the tunnel and handler sides of the bridge concurrently and
// returns once both have finished (or the slower side is abandoned after
// joinTimeout).
Expand All @@ -68,13 +75,13 @@ func (pb *PipeBridge) RunPair(
tunnelFn TunnelFunc,
handlerFn HandlerFunc,
) error {
pairCtx, cancel := context.WithCancel(ctx)
defer cancel()
pairCtx, cancel := context.WithCancelCause(ctx)
defer cancel(nil)

var stopOnce sync.Once
stop := func() {
stopOnce.Do(func() {
cancel()
cancel(nil)
_ = pb.StdoutWriter.Close()
_ = pb.StdinWriter.Close()
})
Expand All @@ -95,15 +102,33 @@ func (pb *PipeBridge) RunPair(
}
}()

tunnelSide := func() error { return tunnelFn(pairCtx, pb.StdinReader, pb.StdoutWriter) }
tunnelSide := func() error {
err := tunnelFn(pairCtx, pb.StdinReader, pb.StdoutWriter)
cancel(&pairCompletion{tunnel: true, err: err})
return err
}
handlerSide := func() error {
return handlerFn(pairCtx, pb.StdoutReader, pb.StdinWriter)
err := handlerFn(pairCtx, pb.StdoutReader, pb.StdinWriter)
cancel(&pairCompletion{err: err})
return err
}

// Run the two sides concurrently and wait for both to finish (or the slower
// side to be abandoned after joinTimeout).
tunnelErr, handlerErr := iojoin.Join(tunnelSide, handlerSide, joinTimeout, stop)
classifiedErr := ClassifyTunnelErrors(tunnelErr, handlerErr)
var first *pairCompletion
if ctx.Err() != nil {
classifiedErr = ctx.Err()
} else if errors.As(context.Cause(pairCtx), &first) && first.tunnel && !IsEOF(handlerErr) {
if first.err == nil && errors.Is(handlerErr, context.Canceled) {
classifiedErr = errors.New("tunnel closed before handler completed")
} else if first.err != nil && !errors.Is(first.err, context.Canceled) &&
!errors.Is(first.err, context.DeadlineExceeded) && handlerErr != nil {
classifiedErr = first.err
}
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

internalTunnelCancel := errors.Is(tunnelErr, context.Canceled) && ctx.Err() == nil
if classifiedErr == nil && ctx.Err() == nil && (tunnelErr == nil || internalTunnelCancel) {
log.Debug("pipe bridge completed successfully")
Expand Down
197 changes: 197 additions & 0 deletions pkg/tunnel/pipebridge_classification_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
package tunnel

import (
"context"
"errors"
"fmt"
"io"
"os"
"testing"
"time"

config2 "github.com/devsy-org/devsy/pkg/devcontainer/config"
"github.com/stretchr/testify/suite"
)

type pipeBridgeClassificationSuite struct{ suite.Suite }

func TestPipeBridgeClassificationSuite(t *testing.T) {
suite.Run(t, new(pipeBridgeClassificationSuite))
}

func (s *pipeBridgeClassificationSuite) TestTunnelErrorWinsOverInternalCancellation() {
agentErr := errors.New("agent command failed")
err := s.runPair(
context.Background(),
func(context.Context, *os.File, *os.File) error { return agentErr },
func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return ctx.Err() },
)
s.ErrorIs(err, agentErr)
Comment thread
skevetter marked this conversation as resolved.
Resolved
}

func (s *pipeBridgeClassificationSuite) TestTunnelErrorAfterTeardownDoesNotOverrideHandler() {
err := s.runPair(
context.Background(),
func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return errors.New("teardown failed") },
func(context.Context, *os.File, *os.File) error { return context.Canceled },
)
s.ErrorIs(err, context.Canceled)
}

func (s *pipeBridgeClassificationSuite) TestEarlierTunnelFailureWinsOverHandlerFailure() {
agentErr := errors.New("agent command failed")
handlerErr := errors.New("handler failed")
err := s.runPair(
context.Background(),
func(context.Context, *os.File, *os.File) error { return agentErr },
func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return handlerErr },
)
s.ErrorIs(err, agentErr)
s.NotErrorIs(err, handlerErr)
}

func (s *pipeBridgeClassificationSuite) TestTunnelErrorWinsOverClosedHandlerPipe() {
agentErr := errors.New("agent command failed")
err := s.runPair(context.Background(),
func(context.Context, *os.File, *os.File) error { return agentErr },
func(ctx context.Context, _, _ *os.File) error {
<-ctx.Done()
return fmt.Errorf("write: %w", os.ErrClosed)
},
)
s.ErrorIs(err, agentErr)
}

func (s *pipeBridgeClassificationSuite) TestCompletedHandlerResultSurvivesTunnelExit() {
var result *config2.Result
resultSent := make(chan struct{})
err := s.runPair(
context.Background(),
func(context.Context, *os.File, *os.File) error {
<-resultSent
return errors.New("SSH command failed with exit code 1")
},
func(ctx context.Context, _, _ *os.File) error {
result = &config2.Result{Error: "workspace changed; use --recreate"}
close(resultSent)
<-ctx.Done()
return nil
},
)
s.NoError(err)
s.Require().NotNil(result)
s.EqualError(result.Err(), "workspace changed; use --recreate")
}

func (s *pipeBridgeClassificationSuite) TestHandlerFailureRecordedBeforeTunnelError() {
tunnelStarted := make(chan struct{})
handlerErr := errors.New("handler failed")
err := s.runPair(context.Background(),
func(ctx context.Context, _, _ *os.File) error {
close(tunnelStarted)
<-ctx.Done()
return errors.New("closed resource")
},
func(context.Context, *os.File, *os.File) error {
<-tunnelStarted
return handlerErr
},
)
s.ErrorIs(err, handlerErr)
s.Equal("tunnel to container: handler failed", err.Error())
}

func (s *pipeBridgeClassificationSuite) TestSuccessfulTunnelBeforeHandlerReportsIncompleteOperation() {
err := s.runPair(context.Background(),
func(context.Context, *os.File, *os.File) error { return nil },
func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return ctx.Err() },
)
s.EqualError(err, "tunnel closed before handler completed")
s.NotErrorIs(err, context.Canceled)
}

func (s *pipeBridgeClassificationSuite) TestHandlerDeadlineAfterSuccessfulTunnel() {
err := s.runPair(
context.Background(),
func(context.Context, *os.File, *os.File) error { return nil },
func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return context.DeadlineExceeded },
)
s.ErrorIs(err, context.DeadlineExceeded)
}

func (s *pipeBridgeClassificationSuite) TestParentCancellationWins() {
ctx, cancel := context.WithCancel(context.Background())
agentErr := errors.New("agent command failed")
err := s.runPair(
ctx,
func(context.Context, *os.File, *os.File) error { cancel(); return agentErr },
func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return ctx.Err() },
)
s.ErrorIs(err, context.Canceled)
}

func (s *pipeBridgeClassificationSuite) TestHandlerDeadlinePreservesTunnelError() {
agentErr := errors.New("agent command failed")
err := s.runPair(
context.Background(),
func(context.Context, *os.File, *os.File) error { return agentErr },
func(ctx context.Context, _, _ *os.File) error {
<-ctx.Done()
return fmt.Errorf("handler: %w", context.DeadlineExceeded)
},
)
s.ErrorIs(err, agentErr)
}

func (s *pipeBridgeClassificationSuite) TestParentDeadlineWins() {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
defer cancel()
err := s.runPair(
ctx,
func(context.Context, *os.File, *os.File) error { return errors.New("agent failed") },
func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return ctx.Err() },
)
s.ErrorIs(err, context.DeadlineExceeded)
}

func (s *pipeBridgeClassificationSuite) TestWrappedTunnelCancellationDoesNotReplaceHandlerError() {
for _, tunnelErr := range []error{context.Canceled, context.DeadlineExceeded} {
err := s.runPair(
context.Background(),
func(context.Context, *os.File, *os.File) error { return fmt.Errorf("tunnel: %w", tunnelErr) },
func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return context.Canceled },
)
s.Equal("tunnel to container: context canceled", err.Error())
}
}

func (s *pipeBridgeClassificationSuite) TestBothNil() {
err := s.runPair(
context.Background(),
func(context.Context, *os.File, *os.File) error { return nil },
func(context.Context, *os.File, *os.File) error { return nil },
)
s.NoError(err)
}

func (s *pipeBridgeClassificationSuite) TestConnectionErrorClassification() {
agentErr := errors.New("dial failed")
err := s.runPair(
context.Background(),
func(context.Context, *os.File, *os.File) error { return agentErr },
func(context.Context, *os.File, *os.File) error { return io.EOF },
)
s.ErrorIs(err, agentErr)
s.Equal("connect to server: dial failed", err.Error())
}

func (s *pipeBridgeClassificationSuite) runPair(
ctx context.Context,
tunnelFn TunnelFunc,
handlerFn HandlerFunc,
) error {
bridge, err := NewPipeBridge()
s.Require().NoError(err)
defer bridge.Close()
return bridge.RunPair(ctx, tunnelFn, handlerFn)
}
67 changes: 67 additions & 0 deletions pkg/tunnel/pipebridge_result_integration_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package tunnel_test

import (
"context"
"errors"
"net"
"os"
"testing"
"time"

agenttunnel "github.com/devsy-org/devsy/pkg/agent/tunnel"
"github.com/devsy-org/devsy/pkg/agent/tunnelserver"
config2 "github.com/devsy-org/devsy/pkg/devcontainer/config"
"github.com/devsy-org/devsy/pkg/stdio"
"github.com/devsy-org/devsy/pkg/tunnel"
"github.com/stretchr/testify/suite"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)

type bridgeResultSuite struct{ suite.Suite }

func TestBridgeResultSuite(t *testing.T) { suite.Run(t, new(bridgeResultSuite)) }

func (s *bridgeResultSuite) TestForwardedAgentErrorSurvivesSSHExit() {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
bridge, err := tunnel.NewPipeBridge()
s.Require().NoError(err)
defer bridge.Close()
var result *config2.Result
err = bridge.RunPair(ctx,
func(ctx context.Context, stdin, stdout *os.File) error {
conn, err := grpc.NewClient(
"passthrough:///agent",
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithContextDialer(
func(context.Context, string) (net.Conn, error) { return stdio.NewStdioStream(stdin, stdout), nil },
),
)
if err != nil {
return err
}
defer func() { _ = conn.Close() }()
_, err = tunnelserver.ReportResult(
ctx,
agenttunnel.NewTunnelClient(conn),
func(context.Context) (*config2.Result, error) {
return nil, errors.New("workspace changed; use --recreate")
},
)
if err == nil {
return errors.New("expected agent failure")
}
return errors.New("SSH command failed with exit code 1")
},
func(ctx context.Context, stdout, stdin *os.File) error {
var handlerErr error
result, handlerErr = tunnelserver.New().RunWithResult(ctx, stdout, stdin)
<-ctx.Done()
return handlerErr
},
)
s.Require().NoError(err)
s.Require().NotNil(result)
s.EqualError(result.Err(), "workspace changed; use --recreate")
}
Loading
Loading