diff --git a/cmd/config/apply.go b/cmd/config/apply.go index f082505e2..3da077f05 100644 --- a/cmd/config/apply.go +++ b/cmd/config/apply.go @@ -384,14 +384,16 @@ func (cmd *ApplyCmd) copyAndExecFeatures( ) error { featureSets, featureStageDir := stage.featureSets, stage.dir containerFeaturesPath := "/tmp/build-features" - streams := docker.Streams{Stdout: os.Stdout, Stderr: os.Stderr} + streams := docker.Streams{LiveStdout: os.Stdout, LiveStderr: os.Stderr} if emitJSON { // Raw Docker output must never share stdout with status and result // envelopes. The configured logger preserves the selected machine // encoding on stderr instead. outputWriter := log.Writer(log.LevelInfo) defer func() { _ = outputWriter.Close() }() + streams.LiveStdout = nil streams.Stdout = outputWriter + streams.LiveStderr = nil streams.Stderr = outputWriter } diff --git a/cmd/root.go b/cmd/root.go index 08aec9dcb..c607e65f4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -250,6 +250,9 @@ func exitCodeForError(err error, machineMode bool) int { return exitcode.Success } + if code, ok := unsafeOutputExitCode(err); ok { + return code + } cliErr := clierr.Classify(err) // Stay transparent for unclassified child-process exits (e.g. `devsy ssh -- cmd`). @@ -269,7 +272,31 @@ func exitCodeForError(err error, machineMode bool) int { return exitcode.Failure } +func unsafeOutputExitCode(err error) (int, bool) { + unsafe, ok := errors.AsType[interface { + error + SkipFinalStatus() bool + }](err) + if !ok || !unsafe.SkipFinalStatus() { + return 0, false + } + if commandExit, ok := errors.AsType[*workspace.CommandExitError](err); ok { + return commandExit.Code, true + } + return exitcode.Failure, true +} + +func commandExitCode(commandExit *workspace.CommandExitError, machineMode bool) int { + if machineMode || commandExit.Cause != nil { + renderCLIError(clierr.Classify(commandExit), machineMode) + } + return commandExit.Code +} + func passthroughExitCode(err error, machineMode bool) (int, bool) { + if commandExit, ok := errors.AsType[*workspace.CommandExitError](err); ok { + return commandExitCode(commandExit, machineMode), true + } if sshExitErr, ok := errors.AsType[*ssh.ExitError](err); ok { if machineMode { renderCLIError( @@ -294,12 +321,6 @@ func passthroughExitCode(err error, machineMode bool) (int, bool) { } return execExitErr.ExitCode(), true } - if commandExit, ok := errors.AsType[*workspace.CommandExitError](err); ok { - if machineMode { - renderCLIError(clierr.Classify(commandExit), true) - } - return commandExit.Code, true - } return 0, false } diff --git a/cmd/root_live_output_test.go b/cmd/root_live_output_test.go new file mode 100644 index 000000000..46088f2cc --- /dev/null +++ b/cmd/root_live_output_test.go @@ -0,0 +1,42 @@ +package cmd + +import ( + "context" + "errors" + "io" + "os" + "testing" + + "github.com/devsy-org/devsy/pkg/docker" + "github.com/devsy-org/devsy/pkg/workspace" + "github.com/stretchr/testify/suite" +) + +type rootLiveOutputSuite struct{ suite.Suite } + +type canceledUnsafeOutput struct{} + +func (canceledUnsafeOutput) Error() string { return context.Canceled.Error() } +func (canceledUnsafeOutput) Unwrap() error { return context.Canceled } +func (canceledUnsafeOutput) SkipFinalStatus() bool { return true } + +func TestRootLiveOutputSuite(t *testing.T) { suite.Run(t, new(rootLiveOutputSuite)) } + +func (s *rootLiveOutputSuite) TestOutputErrorDoesNotWriteToStderr() { + reader, writer, err := os.Pipe() + s.Require().NoError(err) + defer func() { _ = reader.Close() }() + defer func() { _ = writer.Close() }() + original := os.Stderr + os.Stderr = writer + defer func() { os.Stderr = original }() + outputErr := &docker.LiveOutputError{Err: errors.New("live output: 12 bytes dropped")} + code := exitCodeForError(&workspace.CommandExitError{Code: 7, Cause: outputErr}, false) + s.Equal(7, code) + s.Equal(1, exitCodeForError(errors.Join(context.Canceled, outputErr), false)) + s.Equal(1, exitCodeForError(canceledUnsafeOutput{}, false)) + s.Require().NoError(writer.Close()) + data, err := io.ReadAll(reader) + s.Require().NoError(err) + s.Empty(data) +} diff --git a/cmd/workspace/exec.go b/cmd/workspace/exec.go index 8689f879b..7382b06c9 100644 --- a/cmd/workspace/exec.go +++ b/cmd/workspace/exec.go @@ -3,6 +3,7 @@ package workspace import ( "context" "fmt" + "math" "os" "strings" @@ -353,16 +354,26 @@ func (cmd *ExecCmd) execInContainer(ctx context.Context, opts execOpts, args []s stdoutIsTerminal := term.IsTerminal( int(os.Stdout.Fd()), ) // #nosec G115 -- valid standard descriptor + stderrFD := os.Stderr.Fd() + stderrIsTerminal := stderrFD <= math.MaxInt && term.IsTerminal(int(stderrFD)) code, err := opts.runtime.Exec(ctx, workspace2.ExecRequest{ - Target: opts.target, Workdir: opts.workdir, Env: opts.envMap, Argv: args, - Stdin: os.Stdin, Stdout: os.Stdout, Stderr: os.Stderr, - TTY: stdinIsTerminal && stdoutIsTerminal, + Target: opts.target, + Workdir: opts.workdir, + Env: opts.envMap, + Argv: args, + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + TTY: stdinIsTerminal && + stdoutIsTerminal, + LiveStdout: stdoutIsTerminal, + LiveStderr: stderrIsTerminal, }) - if err != nil { + if err != nil && code < 0 { return err } - if code != 0 { - return &workspace2.CommandExitError{Code: code} + if code != 0 || err != nil { + return &workspace2.CommandExitError{Code: code, Cause: err} } return nil } diff --git a/cmd/workspace/exec_live_status_test.go b/cmd/workspace/exec_live_status_test.go new file mode 100644 index 000000000..4b70a9f3c --- /dev/null +++ b/cmd/workspace/exec_live_status_test.go @@ -0,0 +1,58 @@ +package workspace + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/devsy-org/devsy/cmd/flags" + devcconfig "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/docker" + "github.com/devsy-org/devsy/pkg/status" + workspace2 "github.com/devsy-org/devsy/pkg/workspace" + "github.com/stretchr/testify/suite" +) + +type liveStatusSuite struct{ suite.Suite } + +func TestLiveStatusSuite(t *testing.T) { suite.Run(t, new(liveStatusSuite)) } + +type failedLiveRuntime struct{ workspace2.ContainerRuntime } + +func (failedLiveRuntime) Exec(context.Context, workspace2.ExecRequest) (int, error) { + return 7, &docker.LiveOutputError{Err: errors.New("12 bytes dropped")} +} + +type blockingFinalReporter struct { + calls int + release chan struct{} +} + +func (r *blockingFinalReporter) Report(status.Event) { + r.calls++ + if r.calls > 1 { + <-r.release + } +} + +func (s *liveStatusSuite) TestOutputErrorDoesNotBlockFinalStatus() { + reporter := &blockingFinalReporter{release: make(chan struct{})} + defer close(reporter.release) + command := &ExecCmd{ + GlobalFlags: &flags.GlobalFlags{ResultFormat: formatPlain}, + statusReporter: reporter, + } + done := make(chan error, 1) + go func() { + opts := execOpts{runtime: failedLiveRuntime{}} + done <- command.execAndReport(context.Background(), opts, nil, devcconfig.ResultEnvelope{}) + }() + select { + case err := <-done: + s.Require().ErrorContains(err, "12 bytes dropped") + s.Equal(1, reporter.calls) + case <-time.After(time.Second): + s.FailNow("final status did not return") + } +} diff --git a/cmd/workspace/exec_tty_unix_test.go b/cmd/workspace/exec_tty_unix_test.go index 886c90ff4..f140ecd11 100644 --- a/cmd/workspace/exec_tty_unix_test.go +++ b/cmd/workspace/exec_tty_unix_test.go @@ -48,26 +48,28 @@ func TestExecInContainerRequestsTTYOnlyWhenInputAndOutputAreTerminals(t *testing runtime := &execTTYCaptureRuntime{} target := workspace2.ContainerTarget{ContainerID: "container-id", User: "1001:1002"} env := map[string]string{"LANG": "C.UTF-8"} - originalStdin, originalStdout := os.Stdin, os.Stdout - os.Stdin, os.Stdout = stdin, stdout + originalStdin, originalStdout, originalStderr := os.Stdin, os.Stdout, os.Stderr + os.Stdin, os.Stdout, os.Stderr = stdin, stdout, stdin t.Cleanup(func() { - os.Stdin, os.Stdout = originalStdin, originalStdout + os.Stdin, os.Stdout, os.Stderr = originalStdin, originalStdout, originalStderr }) err := (&ExecCmd{}).execInContainer( t.Context(), execOpts{runtime: runtime, target: target, workdir: "/workspace", envMap: env}, []string{"printf", "hello"}, ) - os.Stdin, os.Stdout = originalStdin, originalStdout + os.Stdin, os.Stdout, os.Stderr = originalStdin, originalStdout, originalStderr require.NoError(t, err) require.Equal(t, test.wantTTY, runtime.request.TTY) + require.Equal(t, test.stdoutTTY, runtime.request.LiveStdout) + require.Equal(t, test.stdinTTY, runtime.request.LiveStderr) require.Equal(t, target, runtime.request.Target) require.Equal(t, "/workspace", runtime.request.Workdir) require.Equal(t, env, runtime.request.Env) require.Equal(t, []string{"printf", "hello"}, runtime.request.Argv) require.Same(t, stdin, runtime.request.Stdin) require.Same(t, stdout, runtime.request.Stdout) - require.Same(t, os.Stderr, runtime.request.Stderr) + require.Same(t, stdin, runtime.request.Stderr) }) } } diff --git a/cmd/workspace/lifecycle.go b/cmd/workspace/lifecycle.go index 7d9d015c2..7c72c7bd0 100644 --- a/cmd/workspace/lifecycle.go +++ b/cmd/workspace/lifecycle.go @@ -72,6 +72,10 @@ type LifecycleExecParams struct { // EnvArgsRedactor masks sensitive values passed to docker exec as -e entries. func EnvArgsRedactor(envArgs []string) *secrets.Redactor { + return secrets.NewEnvironmentRedactor(envArgsEntries(envArgs)) +} + +func envArgsEntries(envArgs []string) []string { entries := make([]string, 0, len(envArgs)/2) for _, arg := range envArgs { if arg != "-e" { @@ -79,7 +83,7 @@ func EnvArgsRedactor(envArgs []string) *secrets.Redactor { } } - return secrets.NewEnvironmentRedactor(entries) + return entries } // ExecLifecycleHook runs a single devcontainer lifecycle hook against a container. @@ -104,9 +108,12 @@ func ExecLifecycleHook(params *LifecycleExecParams, name string, hook types.Life rec := newLifecycleRecording(params, name, key) started := time.Now() err := params.Helper.Run(params.Ctx, args, docker.Streams{ - Stdin: os.Stdin, - Stdout: rec.tee(os.Stdout, workspacelogs.StreamStdout), - Stderr: rec.tee(os.Stderr, workspacelogs.StreamStderr), + Stdin: os.Stdin, + Stdout: rec.capture(workspacelogs.StreamStdout), + LiveStdout: os.Stdout, + Stderr: rec.capture(workspacelogs.StreamStderr), + LiveStderr: os.Stderr, + RedactionEnv: envArgsEntries(params.EnvArgs), }) rec.finish(params.Ctx, time.Since(started), err) if err != nil { @@ -144,16 +151,14 @@ func newLifecycleRecording(params *LifecycleExecParams, hook, key string) *lifec return r } -// tee keeps dst as the only functional destination when nothing is recorded. -func (r *lifecycleRecording) tee(dst io.Writer, stream workspacelogs.Stream) io.Writer { +func (r *lifecycleRecording) capture(stream workspacelogs.Stream) io.Writer { if r == nil { - return dst + return nil } if stream == workspacelogs.StreamStderr { - return io.MultiWriter(dst, r.stderr) + return r.stderr } - - return io.MultiWriter(dst, r.stdout) + return r.stdout } func (r *lifecycleRecording) record(state workspacelogs.InvocationState, reason string) { @@ -180,7 +185,7 @@ func (r *lifecycleRecording) finish(ctx context.Context, elapsed time.Duration, code := exitErr.ExitCode() inv.ExitCode = &code } - } else if runErr == nil { + } else if _, outputOnly := runErr.(*docker.LiveOutputError); runErr == nil || outputOnly { code := 0 inv.ExitCode = &code } @@ -194,6 +199,9 @@ func lifecycleOutcome(ctx context.Context, runErr error) (workspacelogs.Invocati case ctx != nil && ctx.Err() != nil: return workspacelogs.InvocationCanceled, "" } + if _, ok := errors.AsType[*docker.LiveOutputError](runErr); ok { + return workspacelogs.InvocationFailed, "output_failed" + } if _, ok := errors.AsType[*exec.ExitError](runErr); !ok { return workspacelogs.InvocationFailed, "start_failed" } diff --git a/cmd/workspace/lifecycle_record_test.go b/cmd/workspace/lifecycle_record_test.go index 87544330b..986922963 100644 --- a/cmd/workspace/lifecycle_record_test.go +++ b/cmd/workspace/lifecycle_record_test.go @@ -4,7 +4,9 @@ package workspace import ( "context" + "errors" "os" + "os/exec" "path/filepath" "testing" "time" @@ -14,6 +16,7 @@ import ( "github.com/devsy-org/devsy/pkg/workspacelogs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) const fakeDockerScript = `#!/bin/sh @@ -204,3 +207,51 @@ func TestExecLifecycleHookWithoutRecorderIsUnchanged(t *testing.T) { require.Error(t, err) assert.NotContains(t, err.Error(), "full output") } + +type lifecycleOutputSuite struct{ suite.Suite } + +func TestLifecycleOutputSuite(t *testing.T) { suite.Run(t, new(lifecycleOutputSuite)) } + +func (s *lifecycleOutputSuite) TestSuccessfulCommandWithLostOutput() { + run := newRecordedLifecycle(s.T()) + rec := newLifecycleRecording(execParams(s.T(), run.rec), "postCreateCommand", sourceSetup) + rec.finish( + context.Background(), + time.Second, + &docker.LiveOutputError{Err: errors.New("bytes dropped")}, + ) + inv := lastInvocation(run.records(s.T())) + s.Require().NotNil(inv) + s.Equal(workspacelogs.InvocationFailed, inv.State) + s.Equal("output_failed", inv.Reason) + s.Require().NotNil(inv.ExitCode) + s.Zero(*inv.ExitCode) +} + +func (s *lifecycleOutputSuite) TestFileLiveOutputPreservesRawHookOutput() { + stdoutPath, _ := redirectStdio(s.T()) + params := execParams(s.T(), nil) + params.EnvArgs = []string{"-e", "API_TOKEN=fake-out"} + s.Require(). + NoError(ExecLifecycleHook(params, "postCreateCommand", types.LifecycleHook{sourceSetup: {"echo hi"}})) + s.Equal("fake-out\n", readFile(s.T(), stdoutPath)) +} + +func (s *lifecycleOutputSuite) TestFailedCommandWithLostOutput() { + run := newRecordedLifecycle(s.T()) + rec := newLifecycleRecording(execParams(s.T(), run.rec), "postCreateCommand", sourceSetup) + cmd := exec.Command("sh", "-c", "exit 7") + exitErr := cmd.Run() + s.Require().Error(exitErr) + rec.finish( + context.Background(), + time.Second, + errors.Join(exitErr, &docker.LiveOutputError{Err: errors.New("bytes dropped")}), + ) + inv := lastInvocation(run.records(s.T())) + s.Require().NotNil(inv) + s.Equal(workspacelogs.InvocationFailed, inv.State) + s.Equal("output_failed", inv.Reason) + s.Require().NotNil(inv.ExitCode) + s.Equal(7, *inv.ExitCode) +} diff --git a/pkg/docker/README.md b/pkg/docker/README.md new file mode 100644 index 000000000..472d80584 --- /dev/null +++ b/pkg/docker/README.md @@ -0,0 +1,7 @@ +# Docker command output + +`Streams.Stdout` and `Streams.Stderr` are synchronous capture destinations. Non-file live destinations use bounded delivery and are redacted before the live queue. + +A live `*os.File` without a capture destination is passed directly to the Docker process. This preserves terminal detection and resizing, but the helper does not redact that output. A live `*os.File` combined with capture is written through a synchronous `io.MultiWriter`. The Docker process sees a pipe rather than a terminal, so a TTY cannot be combined with capture. A blocked file in this combination can also block command return. + +When a destination is used for both capture and live output, it is kept on the synchronous capture side. A blocked capture destination can prevent command return. diff --git a/pkg/docker/helper.go b/pkg/docker/helper.go index a2df3403a..90f1b1720 100644 --- a/pkg/docker/helper.go +++ b/pkg/docker/helper.go @@ -9,16 +9,21 @@ import ( "io" "os" "os/exec" + "reflect" "runtime" "strings" + "sync" "sync/atomic" "time" "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/image" + "github.com/devsy-org/devsy/pkg/livetee" "github.com/devsy-org/devsy/pkg/log" "github.com/devsy-org/devsy/pkg/scanner" + "github.com/devsy-org/devsy/pkg/secrets" + "github.com/devsy-org/devsy/pkg/subprocess" "k8s.io/apimachinery/pkg/util/wait" ) @@ -442,9 +447,12 @@ func (r *DockerHelper) Remove(ctx context.Context, id string) error { } type Streams struct { - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + LiveStdout io.Writer + LiveStderr io.Writer + RedactionEnv []string } func (r *DockerHelper) Run(ctx context.Context, args []string, streams Streams) error { @@ -459,10 +467,7 @@ func (r *DockerHelper) RunWithDir( ) error { cmd := r.buildCmd(ctx, args...) cmd.Dir = dir - cmd.Stdin = streams.Stdin - cmd.Stdout = streams.Stdout - cmd.Stderr = streams.Stderr - return runCmd(ctx, cmd) + return runStreams(ctx, cmd, streams) } // RunWithEnv runs a command with extra environment variables for this @@ -480,10 +485,133 @@ func (r *DockerHelper) RunWithEnv( } cmd.Env = append(cmd.Env, extraEnv...) } + return runStreams(ctx, cmd, streams) +} + +type synchronizedCapture struct { + mu *sync.Mutex + writer io.Writer +} + +func (w synchronizedCapture) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.writer.Write(p) +} + +func attachDockerStream( + capture, live io.Writer, + tee *livetee.Tee, + redactor *secrets.Redactor, +) (io.Writer, *subprocess.StreamingRedactingWriter) { + if live == nil { + return capture, nil + } + if file, ok := live.(*os.File); ok { + if capture == nil { + return file, nil + } + return io.MultiWriter(capture, file), nil + } + redacted := &subprocess.StreamingRedactingWriter{Next: tee.Writer(live), Redactor: redactor} + if capture == nil { + return redacted, redacted + } + return io.MultiWriter(capture, redacted), redacted +} + +func sameDockerWriter(a, b io.Writer) bool { + return a != nil && b != nil && reflect.ValueOf(a).Comparable() && + reflect.ValueOf(b).Comparable() && + a == b +} + +func captureSharedLive(capture, otherCapture, live io.Writer) (io.Writer, io.Writer, bool) { + if sameDockerWriter(live, capture) { + return capture, nil, true + } + if !sameDockerWriter(live, otherCapture) { + return capture, live, false + } + if capture == nil { + return live, nil, true + } + return io.MultiWriter(capture, live), nil, true +} + +func synchronizedStreams(streams Streams) Streams { + var stdoutShared, stderrShared bool + originalStdout := streams.Stdout + streams.Stdout, streams.LiveStdout, stdoutShared = captureSharedLive( + streams.Stdout, + streams.Stderr, + streams.LiveStdout, + ) + streams.Stderr, streams.LiveStderr, stderrShared = captureSharedLive( + streams.Stderr, + originalStdout, + streams.LiveStderr, + ) + if (stdoutShared || stderrShared || streams.LiveStdout != nil || streams.LiveStderr != nil) && + streams.Stdout != nil && + streams.Stderr != nil { + mu := new(sync.Mutex) + streams.Stdout = synchronizedCapture{mu: mu, writer: streams.Stdout} + streams.Stderr = synchronizedCapture{mu: mu, writer: streams.Stderr} + } + return streams +} + +func flushDockerStream(stream *subprocess.StreamingRedactingWriter) error { + if stream == nil { + return nil + } + return stream.Flush() +} + +type LiveOutputError struct{ Err error } + +func (e *LiveOutputError) Error() string { return e.Err.Error() } + +func (e *LiveOutputError) Unwrap() error { return e.Err } + +func (e *LiveOutputError) SkipFinalStatus() bool { return true } + +type canceledFileOutputError struct{ err error } + +func (e *canceledFileOutputError) Error() string { return e.err.Error() } + +func (e *canceledFileOutputError) Unwrap() error { return e.err } + +func (e *canceledFileOutputError) SkipFinalStatus() bool { return true } + +func hasLiveFile(streams Streams) bool { + _, stdout := streams.LiveStdout.(*os.File) + _, stderr := streams.LiveStderr.(*os.File) + return stdout || stderr +} + +func runStreams(ctx context.Context, cmd *exec.Cmd, streams Streams) error { + streams = synchronizedStreams(streams) cmd.Stdin = streams.Stdin - cmd.Stdout = streams.Stdout - cmd.Stderr = streams.Stderr - return runCmd(ctx, cmd) + tee := livetee.New(io.Discard, nil) + redactor := secrets.NewEnvironmentRedactor(append(cmd.Environ(), streams.RedactionEnv...)) + var stdout, stderr *subprocess.StreamingRedactingWriter + cmd.Stdout, stdout = attachDockerStream(streams.Stdout, streams.LiveStdout, tee, redactor) + cmd.Stderr, stderr = attachDockerStream(streams.Stderr, streams.LiveStderr, tee, redactor) + err := runCmd(ctx, cmd) + if err != nil && ctx.Err() != nil && hasLiveFile(streams) { + err = &canceledFileOutputError{err: err} + } + outputErr := errors.Join(flushDockerStream(stdout), flushDockerStream(stderr), tee.Finish()) + if outputErr != nil { + liveErr := &LiveOutputError{Err: outputErr} + if err == nil { + return liveErr + } + return errors.Join(err, liveErr) + } + return err } func (r *DockerHelper) StartContainer(ctx context.Context, containerId string) error { diff --git a/pkg/docker/live_output_test.go b/pkg/docker/live_output_test.go new file mode 100644 index 000000000..d8eb4247e --- /dev/null +++ b/pkg/docker/live_output_test.go @@ -0,0 +1,276 @@ +package docker + +import ( + "bufio" + "bytes" + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/livetee" + "github.com/devsy-org/devsy/pkg/subprocess" + "github.com/stretchr/testify/suite" +) + +const liveOutputWindows = "windows" + +type liveOutputSuite struct { + suite.Suite + script string +} + +func TestLiveOutputSuite(t *testing.T) { suite.Run(t, new(liveOutputSuite)) } + +type blockedOutput struct { + entered chan struct{} + release chan struct{} + finished chan struct{} + once sync.Once + onceFinished sync.Once + mu sync.Mutex + received string +} + +func (w *blockedOutput) Write(p []byte) (int, error) { + w.once.Do(func() { close(w.entered) }) + <-w.release + w.mu.Lock() + w.received += string(p) + w.mu.Unlock() + w.onceFinished.Do(func() { close(w.finished) }) + return len(p), nil +} + +type safeBuffer struct { + mu sync.Mutex + bytes.Buffer +} + +func (b *safeBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.Buffer.Write(p) +} +func (b *safeBuffer) String() string { b.mu.Lock(); defer b.mu.Unlock(); return b.Buffer.String() } + +func (s *liveOutputSuite) TestRunBlocked() { + s.blocked(func(ctx context.Context, h *DockerHelper, streams Streams) error { + return h.Run(ctx, []string{s.script}, streams) + }) +} + +func (s *liveOutputSuite) TestRunWithDirBlocked() { + s.blocked(func(ctx context.Context, h *DockerHelper, streams Streams) error { + return h.RunWithDir(ctx, "", []string{s.script}, streams) + }) +} + +func (s *liveOutputSuite) TestRunWithEnvBlocked() { + s.blocked(func(ctx context.Context, h *DockerHelper, streams Streams) error { + return h.RunWithEnv(ctx, nil, []string{s.script}, streams) + }) +} + +func (s *liveOutputSuite) TestCaptureOnly() { + h := s.helper("printf 'one\\ntwo\\nthree\\n'; printf 'error\\n' >&2\n") + var out, stderr bytes.Buffer + s.Require(). + NoError(h.Run(context.Background(), []string{s.script}, Streams{Stdout: &out, Stderr: &stderr})) + s.Equal("one\ntwo\nthree\n", out.String()) + s.Equal("error\n", stderr.String()) +} + +func (s *liveOutputSuite) TestSharedCapture() { + h := s.helper( + "i=0; while [ $i -lt 200 ]; do printf 'out\\n'; printf 'err\\n' >&2; i=$((i+1)); done\n", + ) + var capture bytes.Buffer + err := h.Run(context.Background(), []string{s.script}, Streams{ + Stdout: &capture, Stderr: &capture, LiveStdout: io.Discard, LiveStderr: io.Discard, + }) + s.Require().NoError(err) + s.Equal(200, strings.Count(capture.String(), "out\n")) + s.Equal(200, strings.Count(capture.String(), "err\n")) +} + +func (s *liveOutputSuite) TestSharedCaptureAndLiveDestination() { + h := s.helper( + "i=0; while [ $i -lt 200 ]; do printf 'out\\n'; printf 'err\\n' >&2; i=$((i+1)); done\n", + ) + var capture bytes.Buffer + err := h.Run( + context.Background(), + []string{s.script}, + Streams{Stdout: &capture, LiveStderr: &capture}, + ) + s.Require().NoError(err) + s.Equal(200, strings.Count(capture.String(), "out\n")) + s.Equal(200, strings.Count(capture.String(), "err\n")) +} + +func (s *liveOutputSuite) TestSameCaptureAndLiveStreamWritesOnce() { + h := s.helper("printf 'once\\n'\n") + var captured bytes.Buffer + s.Require(). + NoError(h.Run(context.Background(), []string{s.script}, Streams{Stdout: &captured, LiveStdout: &captured})) + s.Equal("once\n", captured.String()) +} + +func (s *liveOutputSuite) TestLiveRedaction() { + h := s.helper("printf 'secretprefix'; printf 'secretsuffix\\n'\n") + var captured, live bytes.Buffer + err := h.RunWithEnv( + context.Background(), + nil, + []string{s.script}, + Streams{ + Stdout: &captured, + LiveStdout: &live, + Stderr: io.Discard, + RedactionEnv: []string{"DEVSY_TEST_TOKEN=secretprefixsecretsuffix"}, + }, + ) + s.Require().NoError(err) + s.Equal("secretprefixsecretsuffix\n", captured.String()) + s.NotContains(live.String(), "secretprefixsecretsuffix") + s.True(strings.Contains(live.String(), "***"), live.String()) +} + +type sliceOutput []byte + +func (w sliceOutput) Write(p []byte) (int, error) { return len(p), nil } + +type nestedSliceOutput struct{ io.Writer } + +func (s *liveOutputSuite) TestNonComparableWriterDoesNotPanic() { + h := s.helper("printf 'output\\n'\n") + writer := nestedSliceOutput{Writer: sliceOutput{1}} + s.NotPanics(func() { + s.Require(). + NoError(h.Run(context.Background(), []string{s.script}, Streams{Stdout: writer, LiveStdout: writer})) + }) +} + +func (s *liveOutputSuite) TestFileLiveOutputUsesFileDirectly() { + file, err := os.CreateTemp(s.T().TempDir(), "output") + s.Require().NoError(err) + defer func() { _ = file.Close() }() + tee := livetee.New(io.Discard, nil) + cmd := &exec.Cmd{} + var stream *subprocess.StreamingRedactingWriter + cmd.Stdout, stream = attachDockerStream(nil, file, tee, nil) + s.Same(file, cmd.Stdout) + s.Nil(stream) + s.NoError(tee.Finish()) +} + +func (s *liveOutputSuite) TestFileLiveCancellationSkipsFinalStatus() { + h := s.helper("printf 'ready\\n'; exec sleep 60\n") + reader, writer, err := os.Pipe() + s.Require().NoError(err) + defer func() { _ = reader.Close(); _ = writer.Close() }() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- h.Run(ctx, []string{s.script}, Streams{LiveStdout: writer}) }() + ready, err := bufio.NewReader(reader).ReadString('\n') + s.Require().NoError(err) + s.Equal("ready\n", ready) + cancel() + runErr := s.waitError(done) + s.ErrorIs(runErr, context.Canceled) + marker, ok := errors.AsType[interface { + error + SkipFinalStatus() bool + }](runErr) + s.Require().True(ok) + s.True(marker.SkipFinalStatus()) +} + +func (s *liveOutputSuite) TestNonFileLiveOutputUsesTee() { + var live bytes.Buffer + tee := livetee.New(io.Discard, nil) + cmd := &exec.Cmd{} + var stream *subprocess.StreamingRedactingWriter + cmd.Stdout, stream = attachDockerStream(nil, &live, tee, nil) + s.NotSame(&live, cmd.Stdout) + s.NotNil(stream) + _, err := cmd.Stdout.Write([]byte("output")) + s.Require().NoError(err) + s.Require().NoError(stream.Flush()) + s.Require().NoError(tee.Finish()) + s.Equal("output", live.String()) +} + +func (s *liveOutputSuite) helper(script string) *DockerHelper { + if runtime.GOOS == liveOutputWindows { + s.T().Skip("shell fixture requires Unix") + } + p := filepath.Join(s.T().TempDir(), "docker") + s.Require().NoError(os.WriteFile(p, []byte(script), 0o600)) + s.script = p + return &DockerHelper{DockerCommand: "/bin/sh"} +} + +func (s *liveOutputSuite) blocked(run func(context.Context, *DockerHelper, Streams) error) { + h := s.helper("printf 'stdout line\\n'; printf 'stderr line\\n' >&2; exec sleep 60\n") + w := &blockedOutput{ + entered: make(chan struct{}), + release: make(chan struct{}), + finished: make(chan struct{}), + } + var out, stderr safeBuffer + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- run(ctx, h, Streams{Stdout: &out, Stderr: &stderr, LiveStdout: w, LiveStderr: w}) }() + var release sync.Once + s.T().Cleanup(func() { release.Do(func() { close(w.release) }) }) + s.wait(w.entered) + s.Require(). + Eventually(func() bool { + return out.String() == "stdout line\n" && stderr.String() == "stderr line\n" + }, 5*time.Second, 10*time.Millisecond) + start := time.Now() + cancel() + err := s.waitError(done) + s.Less(time.Since(start), time.Second) + s.Require().ErrorContains(err, "bytes dropped") + s.Contains(err.Error(), "drain abandoned: true") + s.Equal("stdout line\n", out.String()) + s.Equal("stderr line\n", stderr.String()) + release.Do(func() { close(w.release) }) + s.wait(w.finished) + w.mu.Lock() + received := w.received + w.mu.Unlock() + s.NotContains(received, "bytes dropped") + s.T(). + Log("returned within 1s after cancellation; stdout and stderr capture complete; dropped bytes reported") +} + +func (s *liveOutputSuite) wait(ch <-chan struct{}) { + select { + case <-ch: + case <-time.After(5 * time.Second): + s.FailNow("channel deadline exceeded") + } +} + +func (s *liveOutputSuite) waitError(ch <-chan error) error { + select { + case err := <-ch: + return err + case <-time.After(2 * time.Second): + s.FailNow("command did not return") + return nil + } +} diff --git a/pkg/status/status.go b/pkg/status/status.go index 17fe9e6fe..32b595cb3 100644 --- a/pkg/status/status.go +++ b/pkg/status/status.go @@ -4,6 +4,7 @@ package status import ( "context" + "errors" "fmt" "maps" "sync" @@ -160,6 +161,11 @@ type Operation struct { Step string } +type skipFinalStatus interface { + error + SkipFinalStatus() bool +} + // Run reports an operation lifecycle and returns the callback error. func Run( ctx context.Context, @@ -198,6 +204,9 @@ func Run( } }() err := fn(childCtx) + if skip, ok := errors.AsType[skipFinalStatus](err); ok && skip.SkipFinalStatus() { + return err + } done := Event{ Pipeline: operation.Pipeline, OperationID: id, diff --git a/pkg/status/status_live_output_test.go b/pkg/status/status_live_output_test.go new file mode 100644 index 000000000..228ca36b1 --- /dev/null +++ b/pkg/status/status_live_output_test.go @@ -0,0 +1,51 @@ +package status + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type liveStatusSuite struct{ suite.Suite } + +func TestLiveStatusSuite(t *testing.T) { suite.Run(t, new(liveStatusSuite)) } + +type undeliverableOutput struct{} + +func (undeliverableOutput) Error() string { return "12 bytes dropped" } +func (undeliverableOutput) SkipFinalStatus() bool { return true } + +type blockedFinalReporter struct { + calls int + release chan struct{} +} + +func (r *blockedFinalReporter) Report(e Event) { + r.calls++ + if e.State != StateStarted { + <-r.release + } +} + +func (s *liveStatusSuite) TestNestedOperationsSkipFinalStatus() { + reporter := &blockedFinalReporter{release: make(chan struct{})} + defer close(reporter.release) + done := make(chan error, 1) + go func() { + done <- Run(context.Background(), reporter, Operation{}, func(ctx context.Context) error { + return Run(ctx, reporter, Operation{}, func(context.Context) error { + return fmt.Errorf("command: %w", undeliverableOutput{}) + }) + }) + }() + select { + case err := <-done: + s.Require().ErrorContains(err, "12 bytes dropped") + s.Equal(2, reporter.calls) + case <-time.After(time.Second): + s.FailNow("final status did not return") + } +} diff --git a/pkg/workspace/exec.go b/pkg/workspace/exec.go index 5123416b7..f153279eb 100644 --- a/pkg/workspace/exec.go +++ b/pkg/workspace/exec.go @@ -220,13 +220,22 @@ type ContainerRuntime interface { // CommandExitError preserves a guest command's exit status at the CLI boundary. type CommandExitError struct { - Code int + Code int + Cause error } func (e *CommandExitError) Error() string { + if e.Cause != nil { + if e.Code == 0 { + return fmt.Sprintf("workspace command output: %v", e.Cause) + } + return fmt.Sprintf("workspace command failed with exit code %d: %v", e.Code, e.Cause) + } return fmt.Sprintf("workspace command failed with exit code %d", e.Code) } +func (e *CommandExitError) Unwrap() error { return e.Cause } + // CommandRuntime exposes the native CLI used by legacy raw-command callers. type CommandRuntime interface { ContainerRuntime @@ -241,14 +250,16 @@ type CommandRuntime interface { // ExecRequest is the per-call input to ContainerRuntime.Exec. type ExecRequest struct { - Target ContainerTarget - Workdir string - Env map[string]string - Argv []string - Stdout io.Writer - Stderr io.Writer - Stdin io.Reader - TTY bool + Target ContainerTarget + Workdir string + Env map[string]string + Argv []string + Stdout io.Writer + Stderr io.Writer + Stdin io.Reader + TTY bool + LiveStdout bool + LiveStderr bool } // ContainerTarget identifies a container and the user to exec as. @@ -310,6 +321,9 @@ func (r *DockerRuntime) FindRunning( } func (r *DockerRuntime) Exec(ctx context.Context, req ExecRequest) (int, error) { + if req.LiveStdout || req.LiveStderr { + return execWithRunner(ctx, req, r.liveRunner(req.LiveStdout, req.LiveStderr, req.Env)) + } return execWithRunner(ctx, req, r.runner()) } @@ -321,6 +335,31 @@ func (r *DockerRuntime) ProbeEnv( return probeEnvWithRunner(ctx, target, probe, r.runner()) } +func (r *DockerRuntime) liveRunner( + liveStdout, liveStderr bool, + env map[string]string, +) containerRunFunc { + redactionEnv := make([]string, 0, len(env)) + for key, value := range env { + redactionEnv = append(redactionEnv, key+"="+value) + } + return func(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error { + streams := docker.Streams{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + RedactionEnv: redactionEnv, + } + if liveStdout { + streams.Stdout, streams.LiveStdout = nil, stdout + } + if liveStderr { + streams.Stderr, streams.LiveStderr = nil, stderr + } + return r.helper.Run(ctx, args, streams) + } +} + // runner adapts the helper's Streams-based Run to containerRunFunc. func (r *DockerRuntime) runner() containerRunFunc { return func(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error { @@ -380,8 +419,14 @@ func execWithRunner(ctx context.Context, req ExecRequest, run containerRunFunc) return -1, fmt.Errorf("exec in container %s: %w", req.Target.ContainerID, err) } if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { + if outputErr, found := errors.AsType[*docker.LiveOutputError](err); found { + return exitErr.ExitCode(), outputErr + } return exitErr.ExitCode(), nil } + if outputErr, found := err.(*docker.LiveOutputError); found { + return 0, outputErr + } return -1, fmt.Errorf("exec in container %s: %w", req.Target.ContainerID, err) } diff --git a/pkg/workspace/exec_live_output_test.go b/pkg/workspace/exec_live_output_test.go new file mode 100644 index 000000000..e4612c9ca --- /dev/null +++ b/pkg/workspace/exec_live_output_test.go @@ -0,0 +1,97 @@ +package workspace + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "os/exec" + "testing" + + "github.com/devsy-org/devsy/pkg/docker" + "github.com/stretchr/testify/suite" +) + +type execLiveOutputSuite struct{ suite.Suite } + +func TestExecLiveOutputSuite(t *testing.T) { suite.Run(t, new(execLiveOutputSuite)) } + +func (s *execLiveOutputSuite) TestExitPreservesOutputError() { + executable, err := os.Executable() + s.Require().NoError(err) + process, err := os.StartProcess( + executable, + []string{executable, "-test.run=^TestExecLiveOutputChild$"}, + &os.ProcAttr{ + Env: append(os.Environ(), "DEVSY_TEST_EXIT_CHILD=1"), + Files: []*os.File{os.Stdin, os.Stdout, os.Stderr}, + }, + ) + s.Require().NoError(err) + state, err := process.Wait() + s.Require().NoError(err) + exitErr := &exec.ExitError{ProcessState: state} + exit := error(exitErr) + outputErr := &docker.LiveOutputError{Err: errors.New("live output: 12 bytes dropped")} + run := func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { + return errors.Join(exit, outputErr) + } + code, err := execWithRunner(context.Background(), ExecRequest{}, run) + s.Equal(exitErr.ExitCode(), code) + s.ErrorIs(err, outputErr) + s.ErrorContains(err, "12 bytes dropped") +} + +func TestExecLiveOutputChild(t *testing.T) { + if os.Getenv("DEVSY_TEST_EXIT_CHILD") == "1" { + os.Exit(7) + } +} + +func (s *execLiveOutputSuite) TestSuccessfulExitPreservesOutputError() { + outputErr := &docker.LiveOutputError{Err: errors.New("live output: 12 bytes dropped")} + run := func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { return outputErr } + code, err := execWithRunner(context.Background(), ExecRequest{}, run) + s.Zero(code) + s.ErrorIs(err, outputErr) +} + +func (s *execLiveOutputSuite) TestMachineryErrorWithOutputError() { + outputErr := &docker.LiveOutputError{Err: errors.New("live output: 12 bytes dropped")} + commandErr := errors.New("copy failed") + run := func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { + return errors.Join(commandErr, outputErr) + } + code, err := execWithRunner(context.Background(), ExecRequest{}, run) + s.Equal(-1, code) + s.ErrorIs(err, commandErr) + s.ErrorIs(err, outputErr) +} + +func TestMain(m *testing.M) { + if os.Getenv("DEVSY_TEST_DOCKER_HELPER") == "1" { + _, _ = io.WriteString(os.Stdout, "secretprefixsecretsuffix") + os.Exit(0) + } + os.Exit(m.Run()) +} + +func (s *execLiveOutputSuite) TestRequestEnvironmentRedactsLiveOutput() { + executable, err := os.Executable() + s.Require().NoError(err) + runtime := &DockerRuntime{helper: &docker.DockerHelper{ + DockerCommand: executable, + Environment: []string{"DEVSY_TEST_DOCKER_HELPER=1"}, + }} + var live bytes.Buffer + code, err := runtime.Exec(context.Background(), ExecRequest{ + Target: ContainerTarget{ContainerID: "test"}, + Env: map[string]string{"API_TOKEN": "secretprefixsecretsuffix"}, + Stdout: &live, LiveStdout: true, + }) + s.Require().NoError(err) + s.Zero(code) + s.NotContains(live.String(), "secretprefixsecretsuffix") + s.Contains(live.String(), "***") +}