diff --git a/pkg/agent/delivery/capture.go b/pkg/agent/delivery/capture.go index 4c1e1229e9..45e231a826 100644 --- a/pkg/agent/delivery/capture.go +++ b/pkg/agent/delivery/capture.go @@ -10,12 +10,15 @@ import ( "github.com/devsy-org/devsy/pkg/subprocess" ) +// runCaptured captures with redacted diagnostics, keeping the unredacted +// stdout for the callers that parse docker output such as volume mountpoints, +// where a masked character would corrupt the path. func runCaptured(ctx context.Context, cmd *exec.Cmd) (subprocess.Result, error) { env := cmd.Env if env == nil { env = os.Environ() } - result, err := subprocess.RunCommand(cmd, secrets.NewEnvironmentRedactor(env)) + result, err := subprocess.RunCommandUnredactedStdout(cmd, secrets.NewEnvironmentRedactor(env)) result.OperationID = status.OperationID(ctx) return result, err } diff --git a/pkg/agent/delivery/local_docker.go b/pkg/agent/delivery/local_docker.go index 6b40165ffa..7f77904431 100644 --- a/pkg/agent/delivery/local_docker.go +++ b/pkg/agent/delivery/local_docker.go @@ -169,7 +169,7 @@ func (d *LocalDockerDelivery) detectVolumeVersion(ctx context.Context, volumeNam log.Debugf("failed to detect agent version in volume: %v", err) return "" } - return strings.TrimSpace(result.Stdout) + return strings.TrimSpace(result.RawStdout) } func (d *LocalDockerDelivery) populateVolume( @@ -292,7 +292,7 @@ func (d *LocalDockerDelivery) volumeMountpoint( if err != nil { return "", fmt.Errorf("%s: %w", capturedOutput(result), err) } - return strings.TrimSpace(result.Stdout), nil + return strings.TrimSpace(result.RawStdout), nil } // removeVolume force-removes a single named volume. It is safe to call for a @@ -321,7 +321,7 @@ func (d *LocalDockerDelivery) listManagedVolumes( return nil, fmt.Errorf("%s: %w", capturedOutput(result), err) } var names []string - for line := range strings.SplitSeq(strings.TrimSpace(result.Stdout), "\n") { + for line := range strings.SplitSeq(strings.TrimSpace(result.RawStdout), "\n") { if name := strings.TrimSpace(line); name != "" { names = append(names, name) } diff --git a/pkg/agent/delivery/local_docker_test.go b/pkg/agent/delivery/local_docker_test.go index c2d6f44076..4c29f4d5f6 100644 --- a/pkg/agent/delivery/local_docker_test.go +++ b/pkg/agent/delivery/local_docker_test.go @@ -557,3 +557,58 @@ func TestLocalDockerDelivery_Seed_CopyAndCleanupFailureJoined(t *testing.T) { assert.Contains(t, err.Error(), "seed workspace volume") assert.Contains(t, err.Error(), "remove partial volume") } + +// lowEntropySecretEnv is a credential-bearing variable with a one-character +// value: masking it as a literal substring corrupts captured paths and versions. +var lowEntropySecretEnv = []string{"DEVSY_DEFERRED_AUTH_BOOTSTRAP=1"} //nolint:gosec // test fixture + +func writeFakeVolumeInspectScript(t *testing.T, mountDir string) string { + t.Helper() + + scriptPath := filepath.Join(t.TempDir(), "fake-docker.sh") + script := "#!/bin/sh\n" + + "case \"$1\" in\n" + + " volume) echo \"" + mountDir + "\" ;;\n" + + " *) exit 1 ;;\n" + + "esac\n" + require.NoError(t, os.WriteFile(scriptPath, []byte(script), 0o600)) + // #nosec G302 -- test script must be executable + require.NoError(t, os.Chmod(scriptPath, 0o755)) + return scriptPath +} + +// The mountpoint is used as a filesystem path, and "001" is what a +// single-character mask would corrupt. +func TestVolumeMountpoint_UnaffectedByLowEntropySecretInEnv(t *testing.T) { + mountDir := filepath.Join(t.TempDir(), "001", "mount") + require.NoError(t, os.MkdirAll(mountDir, 0o750)) + + d := &LocalDockerDelivery{ + DockerCommand: writeFakeVolumeInspectScript(t, mountDir), + Environment: lowEntropySecretEnv, + } + + got, err := d.volumeMountpoint(context.Background(), "test-vol") + require.NoError(t, err) + assert.Equal(t, mountDir, got) +} + +// The detected version is compared verbatim, so it must survive capture intact. +func TestDetectVolumeVersion_UnaffectedByLowEntropySecretInEnv(t *testing.T) { + scriptPath := filepath.Join(t.TempDir(), "fake-docker.sh") + script := "#!/bin/sh\n" + + "case \"$1\" in\n" + + " run) echo \"v1.2.3\" ;;\n" + + " *) exit 1 ;;\n" + + "esac\n" + require.NoError(t, os.WriteFile(scriptPath, []byte(script), 0o600)) + // #nosec G302 -- test script must be executable + require.NoError(t, os.Chmod(scriptPath, 0o755)) + + d := &LocalDockerDelivery{ + DockerCommand: scriptPath, + Environment: lowEntropySecretEnv, + } + + assert.Equal(t, "v1.2.3", d.detectVolumeVersion(context.Background(), "test-vol")) +} diff --git a/pkg/agent/delivery/workspace_seed.go b/pkg/agent/delivery/workspace_seed.go index dd53a167cf..ced2050e20 100644 --- a/pkg/agent/delivery/workspace_seed.go +++ b/pkg/agent/delivery/workspace_seed.go @@ -136,7 +136,7 @@ func (d *LocalDockerDelivery) volumeSeedState( err, ) } - managedStr, seededStr, _ := strings.Cut(strings.TrimSpace(result.Stdout), ",") + managedStr, seededStr, _ := strings.Cut(strings.TrimSpace(result.RawStdout), ",") return managedStr == pkgconfig.LabelValueTrue, seededStr == pkgconfig.LabelValueTrue, nil } diff --git a/pkg/devcontainer/crane/run.go b/pkg/devcontainer/crane/run.go index 252fbf1ab1..598d263faf 100644 --- a/pkg/devcontainer/crane/run.go +++ b/pkg/devcontainer/crane/run.go @@ -65,6 +65,8 @@ func (c *command) RunContext(ctx context.Context) (string, error) { } result, err := subprocess.Run(ctx, c.cmd, c.args, subprocess.Options{ Redactor: redactor, + // The template is written to disk and must be reproduced verbatim. + UnredactedStdout: true, }) if err != nil { details := result.DiagnosticOutput() @@ -74,7 +76,7 @@ func (c *command) RunContext(ctx context.Context) (string, error) { return "", fmt.Errorf("failed to execute command: %w", err) } - return result.Stdout, nil + return result.RawStdout, nil } // ShouldUse takes CLIOptions and returns true if crane should be used. diff --git a/pkg/secrets/redact.go b/pkg/secrets/redact.go index 1223030cf5..35d3f5bdd3 100644 --- a/pkg/secrets/redact.go +++ b/pkg/secrets/redact.go @@ -10,6 +10,11 @@ import ( const redactMask = "***" +// minRedactableValueLength is the shortest value worth masking. Masking is +// literal substring replacement, so a shorter value corrupts unrelated output +// (paths, versions) instead of protecting anything. +const minRedactableValueLength = 8 + var ( credentialURLPattern = regexp.MustCompile(`(?i)(https?://)[^\s/@]+@`) authorizationPattern = regexp.MustCompile( @@ -18,23 +23,29 @@ var ( ) type Redactor struct { - replacer *strings.Replacer - values []string - maxLength int + replacer *strings.Replacer + values []string + skipped int } -// NewRedactor masks the values (not keys) of KEY=VALUE entries; empty values are ignored. +// NewRedactor masks the values (not keys) of KEY=VALUE entries, skipping empty +// values and those too short to mask safely. func NewRedactor(secretsEnv []string) *Redactor { values := make([]string, 0, len(secretsEnv)) + skipped := 0 for _, entry := range secretsEnv { _, value, ok := strings.Cut(entry, "=") if !ok || value == "" { continue } + if len(value) < minRedactableValueLength { + skipped++ + continue + } values = append(values, value) } if len(values) == 0 { - return &Redactor{} + return &Redactor{skipped: skipped} } // Mask longer values first so an overlapping prefix (e.g. "sec" of "secret") @@ -46,9 +57,9 @@ func NewRedactor(secretsEnv []string) *Redactor { } return &Redactor{ - replacer: strings.NewReplacer(pairs...), - values: values, - maxLength: len(values[0]), + replacer: strings.NewReplacer(pairs...), + values: values, + skipped: skipped, } } @@ -81,6 +92,15 @@ func NewEnvironmentRedactor(env []string) *Redactor { return NewRedactor(sensitive) } +// SkippedValueCount reports sensitive values left unmasked for being too +// short, so callers emitting captured output can surface the gap. +func (r *Redactor) SkippedValueCount() int { + if r == nil { + return 0 + } + return r.skipped +} + func isSensitiveEnvironmentKey(key string) bool { key = strings.ToUpper(key) for _, marker := range []string{ diff --git a/pkg/secrets/redact_test.go b/pkg/secrets/redact_test.go index f2d5b2d568..37edcb1749 100644 --- a/pkg/secrets/redact_test.go +++ b/pkg/secrets/redact_test.go @@ -7,16 +7,79 @@ import ( "github.com/devsy-org/devsy/pkg/secrets" ) +const ( + redactTestSecret = "sk-live-abcdef123456" + redactTestKey = "DEVSY_API_KEY=" + redactTestSecret +) + func TestRedactor_MasksValues(t *testing.T) { - r := secrets.NewRedactor([]string{"DB_PASSWORD=hunter2", "TOKEN=abc123"}) + r := secrets.NewRedactor([]string{ + "DB_PASSWORD=hunter2-correct-horse", + "TOKEN=abc123def456", + }) - got := r.Redact("connecting with hunter2 and token abc123") + got := r.Redact("connecting with hunter2-correct-horse and token abc123def456") want := "connecting with *** and token ***" if got != want { t.Errorf("Redact = %q, want %q", got, want) } } +func TestRedactor_SkipsTooShortValues(t *testing.T) { + r := secrets.NewRedactor([]string{"DEVSY_AUTH_BOOTSTRAP=1"}) + + got := r.Redact("version v1.2.3 in /var/run/001/mount") + if want := "version v1.2.3 in /var/run/001/mount"; got != want { + t.Errorf("Redact = %q, want short value left unmasked (%q)", got, want) + } + if got := r.SkippedValueCount(); got != 1 { + t.Errorf("SkippedValueCount = %d, want 1", got) + } +} + +// Maskable values alongside a too-short one are still masked, and only the +// short value is reported as skipped. +func TestRedactor_MasksLongValuesAlongsideShortOnes(t *testing.T) { + r := secrets.NewRedactor([]string{ + "DEVSY_AUTH_BOOTSTRAP=1", + redactTestKey, + }) + + got := r.Redact("key=sk-live-abcdef123456 bootstrap=1") + if want := "key=*** bootstrap=1"; got != want { + t.Errorf("Redact = %q, want %q", got, want) + } + if got := r.SkippedValueCount(); got != 1 { + t.Errorf("SkippedValueCount = %d, want 1", got) + } +} + +func TestRedactor_SkippedValueCountIsZeroWhenAllMaskable(t *testing.T) { + r := secrets.NewRedactor([]string{redactTestKey}) + if got := r.SkippedValueCount(); got != 0 { + t.Errorf("SkippedValueCount = %d, want 0", got) + } + + var nilRedactor *secrets.Redactor + if got := nilRedactor.SkippedValueCount(); got != 0 { + t.Errorf("nil SkippedValueCount = %d, want 0", got) + } +} + +// The environment redactor is the path that selects sensitive keys. +func TestEnvironmentRedactor_ShortSensitiveValueDoesNotCorruptOutput(t *testing.T) { + r := secrets.NewEnvironmentRedactor([]string{ + "DEVSY_DEFERRED_AUTH_BOOTSTRAP=1", + redactTestKey, + }) + + got := r.Redact("agent version v1.2.3 at /volumes/ws-001/_data key sk-live-abcdef123456") + want := "agent version v1.2.3 at /volumes/ws-001/_data key ***" + if got != want { + t.Errorf("Redact = %q, want %q", got, want) + } +} + func TestRedactor_IgnoresEmptyValues(t *testing.T) { r := secrets.NewRedactor([]string{"EMPTY=", "NO_EQUALS"}) @@ -51,11 +114,11 @@ func TestRedactor_ValueWithEquals(t *testing.T) { // longer value, regardless of input order. func TestRedactor_PrefixOverlap(t *testing.T) { for _, order := range [][]string{ - {"A=sec", "B=secret"}, - {"B=secret", "A=sec"}, + {"A=sec-abcd", "B=sec-abcd-efgh"}, + {"B=sec-abcd-efgh", "A=sec-abcd"}, } { r := secrets.NewRedactor(order) - got := r.Redact("value is secret here") + got := r.Redact("value is sec-abcd-efgh here") if got != "value is *** here" { t.Errorf("order %v: Redact = %q, want longer value fully masked", order, got) } diff --git a/pkg/subprocess/capture.go b/pkg/subprocess/capture.go index 8aad2940d6..1c425a9590 100644 --- a/pkg/subprocess/capture.go +++ b/pkg/subprocess/capture.go @@ -33,6 +33,9 @@ type Options struct { // subprocess output but are not represented by a sensitive environment // variable. Values are masked before capture and command rendering. SensitiveValues []string + // UnredactedStdout preserves machine-readable stdout in Result.RawStdout. + // Display channels stay redacted. + UnredactedStdout bool // OperationID links diagnostics to the semantic operation that spawned // the command. When empty, Run derives it from ctx. OperationID string @@ -120,10 +123,13 @@ type Result struct { OperationID string Stdout string Stderr string - ExitCode int - Signal string - Duration time.Duration - Truncated bool + // RawStdout holds unredacted stdout when the caller asked for + // machine-readable output, such as a volume mountpoint. Empty otherwise. + RawStdout string + ExitCode int + Signal string + Duration time.Duration + Truncated bool } // DiagnosticOutput returns the useful captured tail for a failed command. @@ -163,7 +169,7 @@ func Run(ctx context.Context, binary string, args []string, options Options) (Re } cmd.Stdin = options.Stdin redactor := commandRedactor(options) - result, err := RunCommand(cmd, redactor) + result, err := runCommand(cmd, redactor, options.UnredactedStdout) err = withContextError(ctx, err) result.OperationID = options.OperationID return result, err @@ -202,12 +208,21 @@ func withContextError(ctx context.Context, err error) error { // stdout and stderr capture. Existing command setup such as an explicit // environment, working directory, or stdin is preserved. func RunCommand(cmd *exec.Cmd, redactor *secrets.Redactor) (Result, error) { + return runCommand(cmd, redactor, false) +} + +// RunCommandUnredactedStdout is RunCommand with the stdout also preserved +// unredacted in Result.RawStdout. +func RunCommandUnredactedStdout(cmd *exec.Cmd, redactor *secrets.Redactor) (Result, error) { + return runCommand(cmd, redactor, true) +} + +func runCommand(cmd *exec.Cmd, redactor *secrets.Redactor, unredactedStdout bool) (Result, error) { if cmd == nil { return Result{}, errors.New("subprocess command is nil") } - out := NewBuffer(redactor) + out, rawOut := captureStdout(cmd, redactor, unredactedStdout) errOut := NewBuffer(redactor) - cmd.Stdout = out cmd.Stderr = errOut binary := cmd.Path args := []string(nil) @@ -229,17 +244,43 @@ func RunCommand(cmd *exec.Cmd, redactor *secrets.Redactor) (Result, error) { Truncated: out.Truncated() || errOut.Truncated(), ExitCode: 0, } + if rawOut != nil { + result.RawStdout = rawOut.String() + result.Truncated = result.Truncated || rawOut.Truncated() + } if err != nil { - result.ExitCode = -1 - if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { - result.ExitCode = exitErr.ExitCode() - result.Signal = processSignal(exitErr.ProcessState) - } + applyExitFailure(&result, err) return result, fmt.Errorf("run %s: %w", result.DisplayCommand, err) } return result, nil } +func applyExitFailure(result *Result, err error) { + result.ExitCode = -1 + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { + result.ExitCode = exitErr.ExitCode() + result.Signal = processSignal(exitErr.ProcessState) + } +} + +// captureStdout wires the redacting buffer to cmd and, when the caller needs +// the stdout verbatim, a second buffer alongside it. A buffer with no redactor +// still bounds the copy. +func captureStdout( + cmd *exec.Cmd, + redactor *secrets.Redactor, + unredacted bool, +) (display, raw *Buffer) { + display = NewBuffer(redactor) + if !unredacted { + cmd.Stdout = display + return display, nil + } + raw = NewBuffer(nil) + cmd.Stdout = io.MultiWriter(display, raw) + return display, raw +} + func processSignal(state *os.ProcessState) string { if state == nil { return "" diff --git a/pkg/subprocess/capture_test.go b/pkg/subprocess/capture_test.go index c8b3e8c878..0d0cb8f370 100644 --- a/pkg/subprocess/capture_test.go +++ b/pkg/subprocess/capture_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "io" + "os/exec" "runtime" "strings" "testing" @@ -229,6 +230,74 @@ func TestRunCapturesSignal(t *testing.T) { } } +func TestRunUnredactedStdoutKeepsMachineReadableOutputVerbatim(t *testing.T) { + if runtime.GOOS == testWindows { + t.Skip("test command uses sh") + } + secret := "sk-live-abcdef123456" //nolint:gosec // test-only redaction fixture + mountpoint := "/var/lib/docker/volumes/devsy-ws-001/_data" + result, err := Run( + context.Background(), + "sh", + []string{"-c", `printf '%s %s' "$TOKEN" "$1"`, "sh", mountpoint}, + Options{ + Env: []string{"TOKEN=" + secret}, + UnredactedStdout: true, + }, + ) + if err != nil { + t.Fatalf("Run: %v", err) + } + if !strings.Contains(result.RawStdout, mountpoint) { + t.Fatalf("RawStdout = %q, want mountpoint %q preserved", result.RawStdout, mountpoint) + } + if strings.Contains(result.Stdout, secret) { + t.Fatalf("secret escaped display stdout: %q", result.Stdout) + } + if !strings.Contains(result.Stdout, "***") { + t.Fatalf("Stdout = %q, want secret masked for display", result.Stdout) + } +} + +func TestRunUnredactedStdoutKeepsDiagnosticsRedacted(t *testing.T) { + if runtime.GOOS == testWindows { + t.Skip("test command uses sh") + } + secret := "sk-live-abcdef123456" //nolint:gosec // test-only redaction fixture + result, err := Run( + context.Background(), + "sh", + []string{"-c", `printf '%s' "$TOKEN" >&2; exit 3`, "sh"}, + Options{ + Env: []string{"TOKEN=" + secret}, + UnredactedStdout: true, + }, + ) + if err == nil { + t.Fatal("Run succeeded, want failure") + } + if strings.Contains(result.DiagnosticOutput(), secret) { + t.Fatalf("secret escaped diagnostic output: %q", result.DiagnosticOutput()) + } +} + +func TestRunCommandLeavesRawStdoutEmptyByDefault(t *testing.T) { + if runtime.GOOS == testWindows { + t.Skip("test command uses sh") + } + cmd := exec.Command("sh", "-c", "printf 'v1.2.3'") // #nosec G204 -- test-only command + result, err := RunCommand(cmd, secrets.NewRedactor([]string{captureToken + captureTestSecret})) + if err != nil { + t.Fatalf("RunCommand: %v", err) + } + if result.RawStdout != "" { + t.Fatalf("RawStdout = %q, want empty when not requested", result.RawStdout) + } + if result.Stdout != "v1.2.3" { + t.Fatalf("Stdout = %q, want v1.2.3", result.Stdout) + } +} + func TestDiagnosticOutputAnnotatesTruncatedTail(t *testing.T) { result := Result{ Stdout: "last useful line",