diff --git a/pkg/devcontainer/sshtunnel/sshtunnel.go b/pkg/devcontainer/sshtunnel/sshtunnel.go index 1fe2a521e..f0cf3d64e 100644 --- a/pkg/devcontainer/sshtunnel/sshtunnel.go +++ b/pkg/devcontainer/sshtunnel/sshtunnel.go @@ -257,25 +257,14 @@ func runCommandInSSHTunnel(ctx context.Context, p sshCommandParams, stdin, stdou Stderr: streamer, }) if err != nil { - _ = streamer.Close() - if out := streamer.ErrorOutput(); out != "" { - return fmt.Errorf("run agent command failed: %w\n%s", err, out) - } return fmt.Errorf("run agent command failed: %w", err) } return nil } -const ( - maxLogLines = 25 - maxLogBytes = 256 * 1024 -) - func newSSHTunnelJSONLogStreamer() *log.JSONLogStreamer { return log.NewJSONLogStreamer(log.StreamerOptions{ FallbackLevel: log.LevelDebug, - CaptureLines: maxLogLines, - CaptureBytes: maxLogBytes, DetectLevelPrefixes: true, TreatUnknownJSONAsDebug: true, }) diff --git a/pkg/devcontainer/sshtunnel/streamer_safety_test.go b/pkg/devcontainer/sshtunnel/streamer_safety_test.go new file mode 100644 index 000000000..404af9ca1 --- /dev/null +++ b/pkg/devcontainer/sshtunnel/streamer_safety_test.go @@ -0,0 +1,21 @@ +package sshtunnel + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type streamerSafetySuite struct{ suite.Suite } + +func TestStreamerSafetySuite(t *testing.T) { + suite.Run(t, new(streamerSafetySuite)) +} + +func (s *streamerSafetySuite) TestAgentOutputIsNotRetainedForErrors() { + streamer := newSSHTunnelJSONLogStreamer() + _, err := streamer.Write([]byte("synthetic-workspace-credential\n")) + s.Require().NoError(err) + s.Require().NoError(streamer.Close()) + s.Empty(streamer.ErrorOutput()) +} diff --git a/pkg/tunnel/pipebridge.go b/pkg/tunnel/pipebridge.go index 3d8acfdcb..88962ce15 100644 --- a/pkg/tunnel/pipebridge.go +++ b/pkg/tunnel/pipebridge.go @@ -60,6 +60,13 @@ type ( HandlerFunc func(ctx context.Context, stdout *os.File, stdin *os.File) error ) +type pairCompletion struct { + tunnel bool + err error +} + +func (*pairCompletion) Error() string { return "pipe bridge completed" } + // RunPair runs the tunnel and handler sides of the bridge concurrently and // returns once both have finished (or the slower side is abandoned after // joinTimeout). @@ -68,13 +75,13 @@ func (pb *PipeBridge) RunPair( tunnelFn TunnelFunc, handlerFn HandlerFunc, ) error { - pairCtx, cancel := context.WithCancel(ctx) - defer cancel() + pairCtx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) var stopOnce sync.Once stop := func() { stopOnce.Do(func() { - cancel() + cancel(nil) _ = pb.StdoutWriter.Close() _ = pb.StdinWriter.Close() }) @@ -95,15 +102,33 @@ func (pb *PipeBridge) RunPair( } }() - tunnelSide := func() error { return tunnelFn(pairCtx, pb.StdinReader, pb.StdoutWriter) } + tunnelSide := func() error { + err := tunnelFn(pairCtx, pb.StdinReader, pb.StdoutWriter) + cancel(&pairCompletion{tunnel: true, err: err}) + return err + } handlerSide := func() error { - return handlerFn(pairCtx, pb.StdoutReader, pb.StdinWriter) + err := handlerFn(pairCtx, pb.StdoutReader, pb.StdinWriter) + cancel(&pairCompletion{err: err}) + return err } // Run the two sides concurrently and wait for both to finish (or the slower // side to be abandoned after joinTimeout). tunnelErr, handlerErr := iojoin.Join(tunnelSide, handlerSide, joinTimeout, stop) classifiedErr := ClassifyTunnelErrors(tunnelErr, handlerErr) + var first *pairCompletion + if ctx.Err() != nil { + classifiedErr = ctx.Err() + } else if errors.As(context.Cause(pairCtx), &first) && first.tunnel && !IsEOF(handlerErr) { + if first.err == nil && errors.Is(handlerErr, context.Canceled) { + classifiedErr = errors.New("tunnel closed before handler completed") + } else if first.err != nil && !errors.Is(first.err, context.Canceled) && + !errors.Is(first.err, context.DeadlineExceeded) && handlerErr != nil { + classifiedErr = first.err + } + } + internalTunnelCancel := errors.Is(tunnelErr, context.Canceled) && ctx.Err() == nil if classifiedErr == nil && ctx.Err() == nil && (tunnelErr == nil || internalTunnelCancel) { log.Debug("pipe bridge completed successfully") diff --git a/pkg/tunnel/pipebridge_classification_test.go b/pkg/tunnel/pipebridge_classification_test.go new file mode 100644 index 000000000..17de1b1ac --- /dev/null +++ b/pkg/tunnel/pipebridge_classification_test.go @@ -0,0 +1,197 @@ +package tunnel + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "testing" + "time" + + config2 "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/stretchr/testify/suite" +) + +type pipeBridgeClassificationSuite struct{ suite.Suite } + +func TestPipeBridgeClassificationSuite(t *testing.T) { + suite.Run(t, new(pipeBridgeClassificationSuite)) +} + +func (s *pipeBridgeClassificationSuite) TestTunnelErrorWinsOverInternalCancellation() { + agentErr := errors.New("agent command failed") + err := s.runPair( + context.Background(), + func(context.Context, *os.File, *os.File) error { return agentErr }, + func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return ctx.Err() }, + ) + s.ErrorIs(err, agentErr) +} + +func (s *pipeBridgeClassificationSuite) TestTunnelErrorAfterTeardownDoesNotOverrideHandler() { + err := s.runPair( + context.Background(), + func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return errors.New("teardown failed") }, + func(context.Context, *os.File, *os.File) error { return context.Canceled }, + ) + s.ErrorIs(err, context.Canceled) +} + +func (s *pipeBridgeClassificationSuite) TestEarlierTunnelFailureWinsOverHandlerFailure() { + agentErr := errors.New("agent command failed") + handlerErr := errors.New("handler failed") + err := s.runPair( + context.Background(), + func(context.Context, *os.File, *os.File) error { return agentErr }, + func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return handlerErr }, + ) + s.ErrorIs(err, agentErr) + s.NotErrorIs(err, handlerErr) +} + +func (s *pipeBridgeClassificationSuite) TestTunnelErrorWinsOverClosedHandlerPipe() { + agentErr := errors.New("agent command failed") + err := s.runPair(context.Background(), + func(context.Context, *os.File, *os.File) error { return agentErr }, + func(ctx context.Context, _, _ *os.File) error { + <-ctx.Done() + return fmt.Errorf("write: %w", os.ErrClosed) + }, + ) + s.ErrorIs(err, agentErr) +} + +func (s *pipeBridgeClassificationSuite) TestCompletedHandlerResultSurvivesTunnelExit() { + var result *config2.Result + resultSent := make(chan struct{}) + err := s.runPair( + context.Background(), + func(context.Context, *os.File, *os.File) error { + <-resultSent + return errors.New("SSH command failed with exit code 1") + }, + func(ctx context.Context, _, _ *os.File) error { + result = &config2.Result{Error: "workspace changed; use --recreate"} + close(resultSent) + <-ctx.Done() + return nil + }, + ) + s.NoError(err) + s.Require().NotNil(result) + s.EqualError(result.Err(), "workspace changed; use --recreate") +} + +func (s *pipeBridgeClassificationSuite) TestHandlerFailureRecordedBeforeTunnelError() { + tunnelStarted := make(chan struct{}) + handlerErr := errors.New("handler failed") + err := s.runPair(context.Background(), + func(ctx context.Context, _, _ *os.File) error { + close(tunnelStarted) + <-ctx.Done() + return errors.New("closed resource") + }, + func(context.Context, *os.File, *os.File) error { + <-tunnelStarted + return handlerErr + }, + ) + s.ErrorIs(err, handlerErr) + s.Equal("tunnel to container: handler failed", err.Error()) +} + +func (s *pipeBridgeClassificationSuite) TestSuccessfulTunnelBeforeHandlerReportsIncompleteOperation() { + err := s.runPair(context.Background(), + func(context.Context, *os.File, *os.File) error { return nil }, + func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return ctx.Err() }, + ) + s.EqualError(err, "tunnel closed before handler completed") + s.NotErrorIs(err, context.Canceled) +} + +func (s *pipeBridgeClassificationSuite) TestHandlerDeadlineAfterSuccessfulTunnel() { + err := s.runPair( + context.Background(), + func(context.Context, *os.File, *os.File) error { return nil }, + func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return context.DeadlineExceeded }, + ) + s.ErrorIs(err, context.DeadlineExceeded) +} + +func (s *pipeBridgeClassificationSuite) TestParentCancellationWins() { + ctx, cancel := context.WithCancel(context.Background()) + agentErr := errors.New("agent command failed") + err := s.runPair( + ctx, + func(context.Context, *os.File, *os.File) error { cancel(); return agentErr }, + func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return ctx.Err() }, + ) + s.ErrorIs(err, context.Canceled) +} + +func (s *pipeBridgeClassificationSuite) TestHandlerDeadlinePreservesTunnelError() { + agentErr := errors.New("agent command failed") + err := s.runPair( + context.Background(), + func(context.Context, *os.File, *os.File) error { return agentErr }, + func(ctx context.Context, _, _ *os.File) error { + <-ctx.Done() + return fmt.Errorf("handler: %w", context.DeadlineExceeded) + }, + ) + s.ErrorIs(err, agentErr) +} + +func (s *pipeBridgeClassificationSuite) TestParentDeadlineWins() { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + err := s.runPair( + ctx, + func(context.Context, *os.File, *os.File) error { return errors.New("agent failed") }, + func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return ctx.Err() }, + ) + s.ErrorIs(err, context.DeadlineExceeded) +} + +func (s *pipeBridgeClassificationSuite) TestWrappedTunnelCancellationDoesNotReplaceHandlerError() { + for _, tunnelErr := range []error{context.Canceled, context.DeadlineExceeded} { + err := s.runPair( + context.Background(), + func(context.Context, *os.File, *os.File) error { return fmt.Errorf("tunnel: %w", tunnelErr) }, + func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return context.Canceled }, + ) + s.Equal("tunnel to container: context canceled", err.Error()) + } +} + +func (s *pipeBridgeClassificationSuite) TestBothNil() { + err := s.runPair( + context.Background(), + func(context.Context, *os.File, *os.File) error { return nil }, + func(context.Context, *os.File, *os.File) error { return nil }, + ) + s.NoError(err) +} + +func (s *pipeBridgeClassificationSuite) TestConnectionErrorClassification() { + agentErr := errors.New("dial failed") + err := s.runPair( + context.Background(), + func(context.Context, *os.File, *os.File) error { return agentErr }, + func(context.Context, *os.File, *os.File) error { return io.EOF }, + ) + s.ErrorIs(err, agentErr) + s.Equal("connect to server: dial failed", err.Error()) +} + +func (s *pipeBridgeClassificationSuite) runPair( + ctx context.Context, + tunnelFn TunnelFunc, + handlerFn HandlerFunc, +) error { + bridge, err := NewPipeBridge() + s.Require().NoError(err) + defer bridge.Close() + return bridge.RunPair(ctx, tunnelFn, handlerFn) +} diff --git a/pkg/tunnel/pipebridge_result_integration_test.go b/pkg/tunnel/pipebridge_result_integration_test.go new file mode 100644 index 000000000..d51bbb86d --- /dev/null +++ b/pkg/tunnel/pipebridge_result_integration_test.go @@ -0,0 +1,67 @@ +package tunnel_test + +import ( + "context" + "errors" + "net" + "os" + "testing" + "time" + + agenttunnel "github.com/devsy-org/devsy/pkg/agent/tunnel" + "github.com/devsy-org/devsy/pkg/agent/tunnelserver" + config2 "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/stdio" + "github.com/devsy-org/devsy/pkg/tunnel" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type bridgeResultSuite struct{ suite.Suite } + +func TestBridgeResultSuite(t *testing.T) { suite.Run(t, new(bridgeResultSuite)) } + +func (s *bridgeResultSuite) TestForwardedAgentErrorSurvivesSSHExit() { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + bridge, err := tunnel.NewPipeBridge() + s.Require().NoError(err) + defer bridge.Close() + var result *config2.Result + err = bridge.RunPair(ctx, + func(ctx context.Context, stdin, stdout *os.File) error { + conn, err := grpc.NewClient( + "passthrough:///agent", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer( + func(context.Context, string) (net.Conn, error) { return stdio.NewStdioStream(stdin, stdout), nil }, + ), + ) + if err != nil { + return err + } + defer func() { _ = conn.Close() }() + _, err = tunnelserver.ReportResult( + ctx, + agenttunnel.NewTunnelClient(conn), + func(context.Context) (*config2.Result, error) { + return nil, errors.New("workspace changed; use --recreate") + }, + ) + if err == nil { + return errors.New("expected agent failure") + } + return errors.New("SSH command failed with exit code 1") + }, + func(ctx context.Context, stdout, stdin *os.File) error { + var handlerErr error + result, handlerErr = tunnelserver.New().RunWithResult(ctx, stdout, stdin) + <-ctx.Done() + return handlerErr + }, + ) + s.Require().NoError(err) + s.Require().NotNil(result) + s.EqualError(result.Err(), "workspace changed; use --recreate") +} diff --git a/pkg/tunnel/pipebridge_test.go b/pkg/tunnel/pipebridge_test.go index e46e8ba94..4005aaa68 100644 --- a/pkg/tunnel/pipebridge_test.go +++ b/pkg/tunnel/pipebridge_test.go @@ -182,8 +182,8 @@ func TestRunPairContextCancellation(t *testing.T) { } err = pb.RunPair(ctx, tunnel, handler) - if err != nil { - t.Errorf("RunPair() error = %v, want nil", err) + if !errors.Is(err, context.Canceled) { + t.Errorf("RunPair() error = %v, want context canceled", err) } } @@ -313,8 +313,8 @@ func TestRunPairLogsParentCancellationDetails(t *testing.T) { return nil }, ) - if err != nil { - t.Fatalf("RunPair() error = %v, want nil for classified shutdown", err) + if !errors.Is(err, context.Canceled) { + t.Fatalf("RunPair() error = %v, want context canceled", err) } entries := logs.FilterMessageSnippet("parent_err=context canceled")