diff --git a/e2e/framework/exec.go b/e2e/framework/exec.go index b23aeddfe..7fce2515d 100644 --- a/e2e/framework/exec.go +++ b/e2e/framework/exec.go @@ -11,8 +11,8 @@ import ( "strings" "time" - "github.com/devsy-org/devsy/e2e/framework/internal/livetee" "github.com/devsy-org/devsy/pkg/docker" + "github.com/devsy-org/devsy/pkg/livetee" ) // execWaitDelay bounds how long Wait may block on I/O after the process exits diff --git a/e2e/framework/internal/livetee/tee.go b/e2e/framework/internal/livetee/tee.go deleted file mode 100644 index 3301227bb..000000000 --- a/e2e/framework/internal/livetee/tee.go +++ /dev/null @@ -1,93 +0,0 @@ -package livetee - -import ( - "io" - "sync" - "time" -) - -const ( - chunkSize = 8 << 10 - queueChunks = 128 - drainTimeout = 100 * time.Millisecond -) - -type Tee struct { - capture io.Writer - chunks chan []byte - done chan struct{} - mu sync.Mutex - queued int64 - dropped int64 - stopped bool -} - -func New(capture, live io.Writer) *Tee { - t := &Tee{capture: capture} - if live != nil { - t.chunks = make(chan []byte, queueChunks) - t.done = make(chan struct{}) - go t.run(live) - } - return t -} - -func (t *Tee) Write(p []byte) (int, error) { - n, err := t.capture.Write(p) - if t.chunks == nil { - return n, err - } - for remaining := p[:n]; len(remaining) > 0; { - size := min(len(remaining), chunkSize) - chunk := make([]byte, size) - copy(chunk, remaining[:size]) - t.mu.Lock() - select { - case t.chunks <- chunk: - t.queued += int64(size) - default: - t.dropped += int64(size) - } - t.mu.Unlock() - remaining = remaining[size:] - } - return n, err -} - -func (t *Tee) Close() int64 { - if t.chunks != nil { - close(t.chunks) - timer := time.NewTimer(drainTimeout) - defer timer.Stop() - select { - case <-t.done: - case <-timer.C: - t.mu.Lock() - t.stopped = true - t.dropped += t.queued - t.queued = 0 - t.mu.Unlock() - } - } - t.mu.Lock() - defer t.mu.Unlock() - return t.dropped -} - -func (t *Tee) run(live io.Writer) { - defer close(t.done) - for chunk := range t.chunks { - t.mu.Lock() - if t.stopped { - t.mu.Unlock() - return - } - t.queued -= int64(len(chunk)) - t.mu.Unlock() - n, _ := live.Write(chunk) - n = max(0, min(n, len(chunk))) - t.mu.Lock() - t.dropped += int64(len(chunk) - n) - t.mu.Unlock() - } -} diff --git a/e2e/framework/internal/livetee/tee_test.go b/e2e/framework/internal/livetee/tee_test.go deleted file mode 100644 index df06a2cf1..000000000 --- a/e2e/framework/internal/livetee/tee_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package livetee - -import ( - "bytes" - "io" - "sync/atomic" - "testing" - "time" - - "github.com/stretchr/testify/suite" -) - -type teeSuite struct{ suite.Suite } - -func TestTeeSuite(t *testing.T) { suite.Run(t, new(teeSuite)) } - -type blockedWriter struct { - entered chan struct{} - release chan struct{} - writes atomic.Int64 -} - -func (w *blockedWriter) Write(p []byte) (int, error) { - w.writes.Add(1) - select { - case <-w.entered: - default: - close(w.entered) - } - <-w.release - return len(p), nil -} - -func (s *teeSuite) TestBlockedDropsAndReleases() { - w := &blockedWriter{entered: make(chan struct{}), release: make(chan struct{})} - var captured bytes.Buffer - tee := New(&captured, w) - _, err := tee.Write([]byte("first")) - s.Require().NoError(err) - select { - case <-w.entered: - case <-time.After(5 * time.Second): - s.FailNow("writer not entered") - } - payload := bytes.Repeat([]byte("x"), (queueChunks+2)*chunkSize) - _, err = tee.Write(payload) - s.Require().NoError(err) - s.Equal(int64((queueChunks+2)*chunkSize), tee.Close()) - s.Equal(append([]byte("first"), payload...), captured.Bytes()) - close(w.release) - select { - case <-tee.done: - s.Equal(int64(1), w.writes.Load()) - case <-time.After(5 * time.Second): - s.FailNow("writer did not finish") - } -} - -func (s *teeSuite) TestLiveOrder() { - var captured, live bytes.Buffer - tee := New(&captured, &live) - for _, part := range []string{"one", "two", "three"} { - _, err := tee.Write([]byte(part)) - s.Require().NoError(err) - } - s.Zero(tee.Close()) - s.Equal("onetwothree", captured.String()) - s.Equal(captured.String(), live.String()) -} - -type shortWriter struct{} - -func (shortWriter) Write(p []byte) (int, error) { return len(p) / 2, io.ErrShortWrite } - -func (s *teeSuite) TestShortWriteCountsDroppedBytes() { - var captured bytes.Buffer - tee := New(&captured, shortWriter{}) - _, err := tee.Write([]byte("123456")) - s.Require().NoError(err) - s.Equal(int64(3), tee.Close()) - s.Equal("123456", captured.String()) -} diff --git a/e2e/framework/runenv/exec.go b/e2e/framework/runenv/exec.go index bdb42a577..8d7f4c210 100644 --- a/e2e/framework/runenv/exec.go +++ b/e2e/framework/runenv/exec.go @@ -12,8 +12,8 @@ import ( "strings" "time" - "github.com/devsy-org/devsy/e2e/framework/internal/livetee" "github.com/devsy-org/devsy/pkg/docker" + "github.com/devsy-org/devsy/pkg/livetee" ) const ( diff --git a/hack/ci/suites.yaml b/hack/ci/suites.yaml index 01c55c77e..2c2ff83fb 100644 --- a/hack/ci/suites.yaml +++ b/hack/ci/suites.yaml @@ -81,7 +81,7 @@ unit: description: E2E runtime helper packages packages: - ./e2e/framework/runenv/... - - ./e2e/framework/internal/livetee + - ./pkg/livetee - ./e2e/framework/dockerwsl - ./e2e/framework/ginkgoreport - ./e2e/cmd/devsy-ci diff --git a/pkg/compose/blocked_repro_test.go b/pkg/compose/blocked_repro_test.go new file mode 100644 index 000000000..ab997adfb --- /dev/null +++ b/pkg/compose/blocked_repro_test.go @@ -0,0 +1,138 @@ +package compose + +import ( + "bytes" + "context" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/docker" + "github.com/devsy-org/devsy/pkg/livetee" + "github.com/stretchr/testify/suite" +) + +type liveOutputSuite struct{ suite.Suite } + +func TestLiveOutputSuite(t *testing.T) { suite.Run(t, new(liveOutputSuite)) } + +type blockedLiveWriter struct{ entered, release chan struct{} } + +func (w *blockedLiveWriter) Write(p []byte) (int, error) { + select { + case <-w.entered: + default: + close(w.entered) + } + <-w.release + return len(p), nil +} + +func (s *liveOutputSuite) TestBlockedCancellationAndCapture() { + h := s.helper("printf 'stderr line\n' >&2\nprintf 'stdout line\n'\nexec sleep 60\n") + w := &blockedLiveWriter{entered: make(chan struct{}), release: make(chan struct{})} + closed := make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + captured := completeCapture{complete: make(chan struct{})} + done := make(chan struct{}) + var err error + go func() { + err = h.RunWithOutput( + ctx, + nil, + docker.Streams{Stdout: w, Stderr: w}, + Output{Capture: &captured, Close: func() error { close(closed); return nil }}, + ) + close(done) + }() + s.wait(w.entered) + s.wait(captured.complete) + start := time.Now() + cancel() + s.wait(done) + s.Less(time.Since(start), livetee.DrainTimeout+time.Second) + s.Require().ErrorContains(err, "drain abandoned: true") + s.Contains(err.Error(), "stderr line: signal: killed") + s.Contains(captured.String(), "stdout line\n") + s.Contains(captured.String(), "stderr line\n") + select { + case <-closed: + s.Fail("closed before Write returned") + default: + } + close(w.release) + s.wait(closed) + s.T().Log("returned within 1100ms after cancellation; stderr and recorder capture complete") +} + +func (s *liveOutputSuite) TestSuccessAndFailureOutput() { + for _, code := range []string{"0", "7"} { + s.Run(code, func() { + h := s.helper( + "printf 'one\ntwo\nthree\n'\nprintf 'failure detail\n' >&2\nexit " + code + "\n", + ) + var out, stderr bytes.Buffer + err := h.Run(context.Background(), nil, docker.Streams{Stdout: &out, Stderr: &stderr}) + if code == "0" { + s.Require().NoError(err) + } else { + s.EqualError(err, "failure detail: exit status 7") + } + s.Equal("one\ntwo\nthree\n", out.String()) + s.Equal("failure detail\n", stderr.String()) + }) + } +} + +func (s *liveOutputSuite) TestRedactsFailedStderr() { + secret := strings.Join([]string{"privateprefix", "privatesuffix"}, "") + h := s.helper("printf 'privateprefix' >&2\nprintf 'privatesuffix' >&2\nexit 7\n") + h.Docker = &docker.DockerHelper{Environment: []string{"DEVSY_TEST_TOKEN=" + secret}} + var stderr, capture bytes.Buffer + err := h.RunWithOutput( + context.Background(), + nil, + docker.Streams{Stderr: &stderr}, + Output{Capture: &capture}, + ) + s.EqualError(err, "***: exit status 7") + s.Equal("***", stderr.String()) + s.Equal("***", capture.String()) +} + +func (s *liveOutputSuite) wait(ch <-chan struct{}) { + select { + case <-ch: + case <-time.After(5 * time.Second): + s.FailNow("channel deadline exceeded") + } +} + +func (s *liveOutputSuite) helper(script string) *ComposeHelper { + if runtime.GOOS == "windows" { + s.T().Skip("requires a POSIX shell") + } + path := filepath.Join(s.T().TempDir(), "fake-compose") + s.Require().NoError(os.WriteFile(path, []byte(script), 0o600)) + return &ComposeHelper{Command: "/bin/sh", Args: []string{path}} +} + +type completeCapture struct { + bytes.Buffer + complete chan struct{} + once sync.Once +} + +func (w *completeCapture) Write(p []byte) (int, error) { + n, err := w.Buffer.Write(p) + if bytes.Contains(w.Bytes(), []byte("stdout line\n")) && + bytes.Contains(w.Bytes(), []byte("stderr line\n")) { + w.once.Do(func() { close(w.complete) }) + } + return n, err +} diff --git a/pkg/compose/helper.go b/pkg/compose/helper.go index 88d2437f4..01f666744 100644 --- a/pkg/compose/helper.go +++ b/pkg/compose/helper.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "maps" @@ -12,6 +13,7 @@ import ( "regexp" "runtime" "strings" + "sync" "github.com/blang/semver/v4" composecli "github.com/compose-spec/compose-go/v2/cli" @@ -20,7 +22,10 @@ import ( pkgconfig "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/docker" + "github.com/devsy-org/devsy/pkg/livetee" "github.com/devsy-org/devsy/pkg/log" + "github.com/devsy-org/devsy/pkg/secrets" + "github.com/devsy-org/devsy/pkg/subprocess" "github.com/devsy-org/devsy/pkg/wslg" ) @@ -276,23 +281,52 @@ func (h *ComposeHelper) Run( ctx context.Context, args []string, streams docker.Streams, +) error { + return h.RunWithOutput(ctx, args, streams, Output{}) +} + +type Output struct { + Capture io.Writer + Close func() error +} + +type captureWriter struct { + mu sync.Mutex + next io.Writer +} + +func (w *captureWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.next.Write(p) +} + +func (h *ComposeHelper) RunWithOutput( + ctx context.Context, + args []string, + streams docker.Streams, + output Output, ) error { cmd := h.buildCmd(ctx, args...) cmd.Stdin = streams.Stdin - cmd.Stdout = streams.Stdout - + tee := livetee.NewWithClose(io.Discard, nil, output.Close) + redactor := secrets.NewEnvironmentRedactor(cmd.Environ()) + stdout, stderr := tee.Writer(streams.Stdout), tee.Writer(streams.Stderr) + if output.Capture != nil { + capture := &captureWriter{next: output.Capture} + stdout = io.MultiWriter(capture, stdout) + stderr = io.MultiWriter(capture, stderr) + } + outStream := &subprocess.StreamingRedactingWriter{Next: stdout, Redactor: redactor} + errStream := &subprocess.StreamingRedactingWriter{Next: stderr, Redactor: redactor} + cmd.Stdout = outStream var stderrBuf strings.Builder - if streams.Stderr != nil { - cmd.Stderr = io.MultiWriter(streams.Stderr, &stderrBuf) - } else { - cmd.Stderr = &stderrBuf - } - + cmd.Stderr = io.MultiWriter(&stderrBuf, errStream) err := cmd.Run() if err != nil && stderrBuf.Len() > 0 { - return fmt.Errorf("%s: %w", strings.TrimSpace(stderrBuf.String()), err) + err = fmt.Errorf("%s: %w", strings.TrimSpace(redactor.Redact(stderrBuf.String())), err) } - return err + return errors.Join(err, outStream.Flush(), errStream.Flush(), tee.Finish()) } func (h *ComposeHelper) Stop(ctx context.Context, projectName string, args []string) error { diff --git a/pkg/devcontainer/compose.go b/pkg/devcontainer/compose.go index fca65309e..53f289cc9 100644 --- a/pkg/devcontainer/compose.go +++ b/pkg/devcontainer/compose.go @@ -677,11 +677,11 @@ func (r *runner) composeUpExistingProject( upArgs = r.onlyRunServices(upArgs, params.parsedConfig) writer := log.Writer(log.LevelInfo) - defer func() { _ = writer.Close() }() - if err := params.composeHelper.Run( + if err := params.composeHelper.RunWithOutput( ctx, upArgs, docker.Streams{Stdout: writer, Stderr: writer}, + compose.Output{Close: writer.Close}, ); err != nil { log.Debugf("could not reuse persisted compose project: %v", err) return nil, err @@ -1414,11 +1414,11 @@ func (r *runner) composeUpAndFindContainer( upArgs = r.onlyRunServices(upArgs, params.parsedConfig) writer := log.Writer(log.LevelInfo) - defer func() { _ = writer.Close() }() - if err := params.composeHelper.Run( + if err := params.composeHelper.RunWithOutput( ctx, upArgs, docker.Streams{Stdout: writer, Stderr: writer}, + compose.Output{Close: writer.Close}, ); err != nil { return nil, fmt.Errorf("docker-compose run: %w", err) } diff --git a/pkg/devcontainer/compose_build.go b/pkg/devcontainer/compose_build.go index 51bade246..24050b97b 100644 --- a/pkg/devcontainer/compose_build.go +++ b/pkg/devcontainer/compose_build.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "os" "path" "path/filepath" @@ -419,13 +420,12 @@ func (r *runner) runComposeBuild( buildArgs []string, ) error { logWriter := log.Writer(log.LevelInfo) - defer func() { _ = logWriter.Close() }() - // One shared writer keeps stdout and stderr on a single pipe, so the - // recorded text is combined like the logged text. step := buildkit.StartStep(ctx, "compose") - writer := step.Tee(logWriter, workspacelogs.StreamCombined) + writer := step.Tee(io.Discard, workspacelogs.StreamCombined) log.Debugf("Run %s %s", composeHelper.Command, strings.Join(buildArgs, " ")) - err := composeHelper.Run(ctx, buildArgs, docker.Streams{Stdout: writer, Stderr: writer}) + err := composeHelper.RunWithOutput(ctx, buildArgs, + docker.Streams{Stdout: logWriter, Stderr: logWriter}, + compose.Output{Capture: writer, Close: logWriter.Close}) step.Finish(ctx, err) return err } diff --git a/pkg/devcontainer/compose_resume.go b/pkg/devcontainer/compose_resume.go index 4be2707a1..b0f0e5e0d 100644 --- a/pkg/devcontainer/compose_resume.go +++ b/pkg/devcontainer/compose_resume.go @@ -114,11 +114,11 @@ func (r *runner) startComposeInstance( } args = r.onlyRunServices(append(args, "start"), params.runParams.parsedConfig) writer := log.Writer(log.LevelInfo) - defer func() { _ = writer.Close() }() - if err := params.composeHelper.Run( + if err := params.composeHelper.RunWithOutput( ctx, args, docker.Streams{Stdout: writer, Stderr: writer}, + compose.Output{Close: writer.Close}, ); err != nil { return nil, fmt.Errorf("start existing compose workspace: %w", err) } diff --git a/pkg/git/blocked_repro_test.go b/pkg/git/blocked_repro_test.go new file mode 100644 index 000000000..80bdb6bbe --- /dev/null +++ b/pkg/git/blocked_repro_test.go @@ -0,0 +1,152 @@ +package git + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/livetee" + "github.com/devsy-org/devsy/pkg/secrets" + "github.com/devsy-org/devsy/pkg/subprocess" + "github.com/stretchr/testify/suite" +) + +const liveTestShell = "/bin/sh" + +type liveOutputSuite struct{ suite.Suite } + +func TestLiveOutputSuite(t *testing.T) { suite.Run(t, new(liveOutputSuite)) } + +type blockedLiveWriter struct { + entered chan struct{} + release chan struct{} + bytes.Buffer +} + +func (w *blockedLiveWriter) Write(p []byte) (int, error) { + select { + case <-w.entered: + default: + close(w.entered) + } + <-w.release + return w.Buffer.Write(p) +} + +func (s *liveOutputSuite) TestBlockedCancellation() { + binary := s.binary("printf 'stderr line\n' >&2\nprintf 'stdout line\n'\nexec sleep 60\n") + w := &blockedLiveWriter{entered: make(chan struct{}), release: make(chan struct{})} + defer close(w.release) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + var result RunResult + var err error + go func() { + result, err = (execRunner{}).Run( + ctx, + RunOptions{Binary: liveTestShell, Args: []string{binary}, Stdout: w}, + ) + close(done) + }() + s.wait(w.entered) + start := time.Now() + cancel() + s.wait(done) + s.Less(time.Since(start), livetee.DrainTimeout+time.Second) + s.Equal("stdout line\n", string(result.Stdout)) + s.Equal("stderr line\n", string(result.Stderr)) + s.Require().ErrorContains(err, "drain abandoned: true") + s.T().Log("returned within 1100ms after cancellation; captured stdout and stderr complete") +} + +func (s *liveOutputSuite) TestSplitSecretBeforeDroppedQueue() { + secret := strings.Join([]string{"privateprefix", "privatesuffix"}, "") + redactor := secrets.NewEnvironmentRedactor([]string{"DEVSY_TEST_TOKEN=" + secret}) + captured := subprocess.NewBuffer(redactor) + w := &blockedLiveWriter{entered: make(chan struct{}), release: make(chan struct{})} + released := make(chan struct{}) + tee := livetee.NewWithClose(io.Discard, nil, func() error { close(released); return nil }) + var dst io.Writer + stream := attachStream(&dst, captured, tee.Writer(w), redactor) + _, err := dst.Write([]byte(strings.Repeat("safe line\n", 20))) + s.Require().NoError(err) + s.wait(w.entered) + for _, part := range []string{strings.Repeat("x", 2<<20), "privateprefix", "privatesuffix", "\n"} { + _, err = dst.Write([]byte(part)) + s.Require().NoError(err) + } + s.Require().NoError(stream.Flush()) + s.Require().ErrorContains(tee.Finish(), "drain abandoned: true") + s.NotContains(captured.String(), "privateprefix") + s.NotContains(captured.String(), "privatesuffix") + s.True(captured.String() == strings.Repeat("x", subprocess.MaxCapturedBytes-4)+"***\n") + s.True(captured.Truncated()) + close(w.release) + s.wait(released) + s.NotContains(w.String(), "privateprefix") + s.NotContains(w.String(), "privatesuffix") +} + +func (s *liveOutputSuite) TestRedactionAndOrdering() { + secret := strings.Join([]string{"credential", "value"}, "") + binary := s.binary( + "printf 'one\ntwo\nthree\n%s\n' \"$DEVSY_TEST_TOKEN\"\nprintf 'error %s\n' \"$DEVSY_TEST_TOKEN\" >&2\n", + ) + var out, stderr bytes.Buffer + result, err := (execRunner{}).Run( + context.Background(), + RunOptions{ + Binary: liveTestShell, Args: []string{binary}, + Env: []string{"DEVSY_TEST_TOKEN=" + secret}, + Stdout: &out, + Stderr: &stderr, + }, + ) + s.Require().NoError(err) + s.Equal("one\ntwo\nthree\n***\n", out.String()) + s.Equal(out.String(), string(result.Stdout)) + s.Equal("error ***\n", stderr.String()) + s.Equal(stderr.String(), string(result.Stderr)) +} + +func (s *liveOutputSuite) TestCredentialCapture() { + binary := s.binary("printf 'protocol=https\npassword=%s\n' \"$DEVSY_TEST_TOKEN\"\n") + secret := strings.Join([]string{"credential", "value"}, "") + var out bytes.Buffer + result, err := (execRunner{}).Run( + context.Background(), + RunOptions{ + Binary: liveTestShell, Args: []string{binary}, + Env: []string{"DEVSY_TEST_TOKEN=" + secret}, + Stdout: &out, + UnredactedStdout: true, + }, + ) + s.Require().NoError(err) + s.Equal("protocol=https\npassword="+secret+"\n", string(result.Stdout)) + s.Equal(string(result.Stdout), out.String()) +} + +func (s *liveOutputSuite) wait(ch <-chan struct{}) { + select { + case <-ch: + case <-time.After(5 * time.Second): + s.FailNow("channel deadline exceeded") + } +} + +func (s *liveOutputSuite) binary(script string) string { + if runtime.GOOS == "windows" { + s.T().Skip("requires a POSIX shell") + } + path := filepath.Join(s.T().TempDir(), "fake-git") + s.Require().NoError(os.WriteFile(path, []byte(script), 0o600)) + return path +} diff --git a/pkg/git/installer.go b/pkg/git/installer.go index 492d1e839..94e16b84b 100644 --- a/pkg/git/installer.go +++ b/pkg/git/installer.go @@ -117,15 +117,14 @@ func (s *pkgManagerStrategy) usable() bool { func (s *pkgManagerStrategy) install(ctx context.Context, t tool) error { log.Debugf("installing %s with %s", t.pkg, s.manager) - w := log.Writer(log.LevelInfo) - defer func() { _ = w.Close() }() - for _, args := range s.installArgs(t.pkg) { + w := log.Writer(log.LevelInfo) if _, err := s.runner.Run(ctx, RunOptions{ - Binary: s.manager, - Args: args, - Stdout: w, - Stderr: w, + Binary: s.manager, + Args: args, + Stdout: w, + Stderr: w, + CloseOutput: w.Close, }); err != nil { return err } diff --git a/pkg/git/repo.go b/pkg/git/repo.go index e6ace4f42..73b31a4a1 100644 --- a/pkg/git/repo.go +++ b/pkg/git/repo.go @@ -247,21 +247,20 @@ func (r *Repo) run(ctx context.Context, args ...string) (RunResult, error) { // runLogged executes a git subcommand streaming its output to the logs. func (r *Repo) runLogged(ctx context.Context, args ...string) error { w := log.Writer(log.LevelInfo) - defer func() { _ = w.Close() }() _, err := r.runner.Run(ctx, RunOptions{ - Dir: r.path, - Env: r.env, - Args: args, - Stdout: w, - Stderr: w, + Dir: r.path, + Env: r.env, + Args: args, + Stdout: w, + Stderr: w, + CloseOutput: w.Close, }) return err } func (r *Repo) cloneWith(ctx context.Context, repository string, c cloneConfig) error { w := log.Writer(log.LevelInfo) - defer func() { _ = w.Close() }() pw := newProgressWriter(w) @@ -272,14 +271,12 @@ func (r *Repo) cloneWith(ctx context.Context, repository string, c cloneConfig) args = append(args, cloneArgsForLFS()...) _, err := r.runner.Run(ctx, RunOptions{ - Env: env, - Args: args, - Stdout: pw, - Stderr: pw, + Env: env, + Args: args, + Stdout: pw, + Stderr: pw, + CloseOutput: func() error { return errors.Join(pw.Close(), w.Close()) }, }) - if closeErr := pw.Close(); err == nil { - err = closeErr - } return err } diff --git a/pkg/git/runner.go b/pkg/git/runner.go index bdfdbe888..a74ec9f98 100644 --- a/pkg/git/runner.go +++ b/pkg/git/runner.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/devsy-org/devsy/pkg/command" + "github.com/devsy-org/devsy/pkg/livetee" "github.com/devsy-org/devsy/pkg/secrets" "github.com/devsy-org/devsy/pkg/subprocess" ) @@ -44,13 +45,14 @@ type Runner interface { // RunOptions describes one command invocation. type RunOptions struct { - Binary string - Dir string - Env []string - Args []string - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer + Binary string + Dir string + Env []string + Args []string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + CloseOutput func() error // UnredactedStdout preserves machine-readable output that may contain a // credential, such as the response from `git credential fill`. UnredactedStdout bool @@ -68,12 +70,13 @@ type execRunner struct{} var _ Runner = execRunner{} func (execRunner) Run(ctx context.Context, opts RunOptions) (RunResult, error) { + tee := livetee.NewWithClose(io.Discard, nil, opts.CloseOutput) binary := opts.Binary if binary == "" { binary = binGit } if !command.Exists(binary) { - return RunResult{}, missingBinaryError(binary, opts.Args) + return RunResult{}, errors.Join(missingBinaryError(binary, opts.Args), tee.Finish()) } cmd := exec.CommandContext( @@ -90,13 +93,13 @@ func (execRunner) Run(ctx context.Context, opts RunOptions) (RunResult, error) { stdoutRedactor = nil } outBuf := subprocess.NewBuffer(stdoutRedactor) - streamOut := attachStream(&cmd.Stdout, outBuf, opts.Stdout, stdoutRedactor) + streamOut := attachStream(&cmd.Stdout, outBuf, tee.Writer(opts.Stdout), stdoutRedactor) errBuf := subprocess.NewBuffer(redactor) - streamErr := attachStream(&cmd.Stderr, errBuf, opts.Stderr, redactor) + streamErr := attachStream(&cmd.Stderr, errBuf, tee.Writer(opts.Stderr), redactor) err := cmd.Run() result := RunResult{Stdout: []byte(outBuf.String()), Stderr: []byte(errBuf.String())} - flushErr := errors.Join(flushStream(streamOut), flushStream(streamErr)) + flushErr := errors.Join(flushStream(streamOut), flushStream(streamErr), tee.Finish()) return result, finishRun(opts.Args, errBuf.String(), err, flushErr) } @@ -141,7 +144,7 @@ func attachStream( return nil } stream := &subprocess.StreamingRedactingWriter{Next: live, Redactor: redactor} - *dst = io.MultiWriter(stream, buf) + *dst = io.MultiWriter(buf, stream) return stream } diff --git a/pkg/livetee/tee.go b/pkg/livetee/tee.go new file mode 100644 index 000000000..95e695037 --- /dev/null +++ b/pkg/livetee/tee.go @@ -0,0 +1,181 @@ +package livetee + +import ( + "errors" + "fmt" + "io" + "sync" + "time" +) + +const ( + chunkSize = 8 << 10 + queueChunks = 128 + DrainTimeout = 100 * time.Millisecond +) + +type chunk struct { + data []byte + live io.Writer +} + +type Tee struct { + capture io.Writer + live io.Writer + chunks chan chunk + done chan struct{} + mu sync.Mutex + queued int64 + dropped int64 + stopped bool + closed bool + closeErr error + writeErr error +} + +func New(capture, live io.Writer) *Tee { + return NewWithClose(capture, live, nil) +} + +func NewWithClose(capture, live io.Writer, closeLive func() error) *Tee { + t := &Tee{ + capture: capture, + live: live, + chunks: make(chan chunk, queueChunks), + done: make(chan struct{}), + } + go t.run(closeLive) + return t +} + +func (t *Tee) Writer(live io.Writer) io.Writer { + return streamWriter{tee: t, live: live} +} + +type streamWriter struct { + tee *Tee + live io.Writer +} + +func (w streamWriter) Write(p []byte) (int, error) { + w.tee.enqueue(p, w.live) + return len(p), nil +} + +func (t *Tee) Write(p []byte) (int, error) { + n, err := t.capture.Write(p) + t.enqueue(p[:n], t.live) + return n, err +} + +func (t *Tee) Close() int64 { + dropped, _ := t.CloseWithStatus() + return dropped +} + +func (t *Tee) CloseWithStatus() (int64, bool) { + t.mu.Lock() + if !t.closed { + t.closed = true + close(t.chunks) + } + t.mu.Unlock() + timer := time.NewTimer(DrainTimeout) + defer timer.Stop() + abandoned := t.waitForDrain(timer.C) + if abandoned { + t.mu.Lock() + t.stopped = true + t.dropped += t.queued + t.queued = 0 + t.mu.Unlock() + } + + t.mu.Lock() + defer t.mu.Unlock() + return t.dropped, abandoned +} + +func (t *Tee) Finish() error { + dropped, abandoned := t.CloseWithStatus() + t.mu.Lock() + defer t.mu.Unlock() + var outputErr error + if dropped != 0 || abandoned { + outputErr = fmt.Errorf( + "live output: %d bytes dropped, drain abandoned: %t", + dropped, + abandoned, + ) + } + return errors.Join(outputErr, t.writeErr, t.closeErr) +} + +func (t *Tee) run(closeLive func() error) { + defer close(t.done) + if closeLive != nil { + defer func() { + err := closeLive() + t.mu.Lock() + t.closeErr = err + t.mu.Unlock() + }() + } + for chunk := range t.chunks { + t.mu.Lock() + stopped := t.stopped + t.mu.Unlock() + if stopped { + return + } + n, err := chunk.live.Write(chunk.data) + n = max(0, min(n, len(chunk.data))) + t.mu.Lock() + if t.writeErr == nil { + t.writeErr = err + } + if !t.stopped { + t.queued -= int64(len(chunk.data)) + t.dropped += int64(len(chunk.data) - n) + } + t.mu.Unlock() + } +} + +func (t *Tee) enqueue(p []byte, live io.Writer) { + if live == nil { + return + } + t.mu.Lock() + defer t.mu.Unlock() + for len(p) > 0 { + size := min(len(p), chunkSize) + if t.closed { + t.dropped += int64(len(p)) + return + } + if len(t.chunks) == cap(t.chunks) { + t.dropped += int64(size) + } else { + data := append([]byte(nil), p[:size]...) + t.chunks <- chunk{data: data, live: live} + t.queued += int64(size) + } + + p = p[size:] + } +} + +func (t *Tee) waitForDrain(deadline <-chan time.Time) bool { + select { + case <-t.done: + return false + case <-deadline: + select { + case <-t.done: + return false + default: + return true + } + } +} diff --git a/pkg/livetee/tee_test.go b/pkg/livetee/tee_test.go new file mode 100644 index 000000000..03ccd0d85 --- /dev/null +++ b/pkg/livetee/tee_test.go @@ -0,0 +1,192 @@ +package livetee + +import ( + "bytes" + "errors" + "io" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type teeSuite struct{ suite.Suite } + +func TestTeeSuite(t *testing.T) { suite.Run(t, new(teeSuite)) } + +type blockedWriter struct { + entered chan struct{} + release chan struct{} + writes atomic.Int64 +} + +func (w *blockedWriter) Write(p []byte) (int, error) { + w.writes.Add(1) + select { + case <-w.entered: + default: + close(w.entered) + } + <-w.release + return len(p), nil +} + +func (s *teeSuite) TestBlockedDropsAndReleases() { + w := &blockedWriter{entered: make(chan struct{}), release: make(chan struct{})} + var captured bytes.Buffer + tee := New(&captured, w) + _, err := tee.Write([]byte("first")) + s.Require().NoError(err) + select { + case <-w.entered: + case <-time.After(5 * time.Second): + s.FailNow("writer not entered") + } + payload := bytes.Repeat([]byte("x"), (queueChunks+2)*chunkSize) + _, err = tee.Write(payload) + s.Require().NoError(err) + s.Equal(int64(len("first")+(queueChunks+2)*chunkSize), tee.Close()) + s.Equal(append([]byte("first"), payload...), captured.Bytes()) + close(w.release) + select { + case <-tee.done: + s.Equal(int64(1), w.writes.Load()) + case <-time.After(5 * time.Second): + s.FailNow("writer did not finish") + } +} + +func (s *teeSuite) TestLiveOrder() { + var captured, live bytes.Buffer + tee := New(&captured, &live) + for _, part := range []string{"one", "two", "three"} { + _, err := tee.Write([]byte(part)) + s.Require().NoError(err) + } + s.Zero(tee.Close()) + s.Equal("onetwothree", captured.String()) + s.Equal(captured.String(), live.String()) +} + +type shortWriter struct{} + +func (shortWriter) Write(p []byte) (int, error) { return len(p) / 2, io.ErrShortWrite } + +func (s *teeSuite) TestShortWriteCountsDroppedBytes() { + var captured bytes.Buffer + tee := New(&captured, shortWriter{}) + _, err := tee.Write([]byte("123456")) + s.Require().NoError(err) + s.Equal(int64(3), tee.Close()) + s.Equal("123456", captured.String()) +} + +func (s *teeSuite) TestAbandonedCloseRunsOnce() { + w := &blockedWriter{entered: make(chan struct{}), release: make(chan struct{})} + var captured bytes.Buffer + closed := make(chan struct{}) + var count atomic.Int64 + tee := NewWithClose(&captured, w, func() error { count.Add(1); close(closed); return nil }) + _, err := tee.Write([]byte("test")) + s.Require().NoError(err) + select { + case <-w.entered: + case <-time.After(5 * time.Second): + s.FailNow("writer not entered") + } + _, abandoned := tee.CloseWithStatus() + s.True(abandoned) + s.Zero(count.Load()) + close(w.release) + select { + case <-closed: + case <-time.After(5 * time.Second): + s.FailNow("writer not closed") + } + s.Equal(int64(1), count.Load()) + select { + case <-tee.done: + case <-time.After(5 * time.Second): + s.FailNow("goroutine did not exit") + } + s.Equal(int64(4), tee.Close()) + s.Equal(int64(1), count.Load()) +} + +func (s *teeSuite) TestSharedWriterOrderAndClose() { + var live bytes.Buffer + var count atomic.Int64 + tee := NewWithClose(io.Discard, nil, func() error { count.Add(1); return nil }) + out, stderr := tee.Writer(&live), tee.Writer(&live) + _, err := out.Write([]byte("one")) + s.Require().NoError(err) + _, err = stderr.Write([]byte("two")) + s.Require().NoError(err) + s.Require().NoError(tee.Finish()) + s.Equal("onetwo", live.String()) + s.Equal(int64(1), count.Load()) + _, err = out.Write([]byte("late")) + s.Require().NoError(err) + s.Equal(int64(4), tee.Close()) + s.Equal("onetwo", live.String()) +} + +func (s *teeSuite) TestCompletedDrainWinsExpiredDeadline() { + done := make(chan struct{}) + close(done) + deadline := make(chan time.Time) + close(deadline) + tee := &Tee{done: done} + for range 128 { + s.False(tee.waitForDrain(deadline)) + } +} + +func (s *teeSuite) TestDropsPreserveWriteAndCloseErrors() { + closeErr := errors.New("close failed") + tee := NewWithClose(io.Discard, shortWriter{}, func() error { return closeErr }) + _, err := tee.Write([]byte("123456")) + s.Require().NoError(err) + err = tee.Finish() + s.Require().ErrorIs(err, closeErr) + s.ErrorIs(err, io.ErrShortWrite) + s.ErrorContains(err, "3 bytes dropped") +} + +type fullErrorWriter struct{ err error } + +func (w fullErrorWriter) Write(p []byte) (int, error) { return len(p), w.err } + +func (s *teeSuite) TestFullWritePreservesError() { + writeErr := errors.New("live write failed") + tee := New(io.Discard, fullErrorWriter{err: writeErr}) + _, err := tee.Write([]byte("123456")) + s.Require().NoError(err) + s.ErrorIs(tee.Finish(), writeErr) + s.Zero(tee.Close()) +} + +func (s *teeSuite) TestBlockedCloseHookIsBounded() { + entered, release := make(chan struct{}), make(chan struct{}) + tee := NewWithClose(io.Discard, nil, func() error { close(entered); <-release; return nil }) + result := make(chan error, 1) + go func() { result <- tee.Finish() }() + select { + case <-entered: + case <-time.After(5 * time.Second): + s.FailNow("close hook not entered") + } + select { + case err := <-result: + s.Require().ErrorContains(err, "0 bytes dropped, drain abandoned: true") + case <-time.After(DrainTimeout + time.Second): + s.FailNow("blocked close hook held Finish") + } + close(release) + select { + case <-tee.done: + case <-time.After(5 * time.Second): + s.FailNow("close hook did not finish") + } +}