From 221043fd5926fb0b3e2b2e8f29aab7d003a5f0d0 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 19:38:41 -0600 Subject: [PATCH 01/11] fix(provider): keep local provider binaries on checksum mismatch --- AGENTS.md | 2 + README.md | 2 + .../sshtunnel/command_error_test.go | 35 +++++ pkg/devcontainer/sshtunnel/sshtunnel.go | 12 +- pkg/provider/download.go | 39 ++++++ pkg/provider/local_source_safety_test.go | 121 ++++++++++++++++++ pkg/provider/resolve.go | 29 +++-- pkg/tunnel/pipebridge.go | 10 +- pkg/tunnel/pipebridge_error_test.go | 27 ++++ 9 files changed, 264 insertions(+), 13 deletions(-) create mode 100644 pkg/devcontainer/sshtunnel/command_error_test.go create mode 100644 pkg/provider/local_source_safety_test.go create mode 100644 pkg/tunnel/pipebridge_error_test.go diff --git a/AGENTS.md b/AGENTS.md index a60735be0a..f57005ef36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,8 @@ Toolchain is managed by mise. Install mise using `curl https://mise.run | sh`. I - **Run unit tests**: `mise run cli:test` (writes coverage to `dist/profile.out`; a `dist` directory is required, e.g. `mkdir -p dist`) - **Build development binary**: `mise run cli:build:dev` (output under `dist/devsy-dev_linux_amd64_v1/`) +Local provider manifests can reference absolute binary paths. Regenerate their SHA-256 checksums after rebuilding. A checksum mismatch must preserve the source binary and report the expected and actual hashes. + ### Desktop (Electron/Svelte 5) Development - **Verify desktop code quality**: `mise run desktop:check` (Svelte check and TypeScript compiler) diff --git a/README.md b/README.md index 4394d75fb2..2fdf4fa08f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ Devsy gives teams and their AI coding agents separate workspaces built from one +Local provider manifests can be installed with `devsy provider add /absolute/path/provider.yaml`. If a local binary checksum changes, regenerate the manifest with its current SHA-256 before starting a workspace. Devsy preserves the source binary when verification fails. + ![Devsy Flow](sites/docs-devsy-sh/public/docs/media/devsy-flow.gif)

Downloads

diff --git a/pkg/devcontainer/sshtunnel/command_error_test.go b/pkg/devcontainer/sshtunnel/command_error_test.go new file mode 100644 index 0000000000..e99c8962cd --- /dev/null +++ b/pkg/devcontainer/sshtunnel/command_error_test.go @@ -0,0 +1,35 @@ +package sshtunnel + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/suite" +) + +type commandErrorSuite struct{ suite.Suite } + +func TestCommandErrorSuite(t *testing.T) { suite.Run(t, new(commandErrorSuite)) } + +func (s *commandErrorSuite) TestCommandFailure() { + for _, tc := range []struct { + name string + output string + wrapsError bool + }{ + {name: "agent diagnostic", output: "provider binary checksum mismatch"}, + {name: "no diagnostic", wrapsError: true}, + } { + s.Run(tc.name, func() { + exit := errors.New("exit status 1") + err := commandFailure(exit, tc.output) + s.ErrorContains(err, "exit status 1") + if tc.wrapsError { + s.ErrorIs(err, exit) + } else { + s.ErrorContains(err, tc.output) + s.NotErrorIs(err, exit) + } + }) + } +} diff --git a/pkg/devcontainer/sshtunnel/sshtunnel.go b/pkg/devcontainer/sshtunnel/sshtunnel.go index 5712903344..bb8d3bdadf 100644 --- a/pkg/devcontainer/sshtunnel/sshtunnel.go +++ b/pkg/devcontainer/sshtunnel/sshtunnel.go @@ -250,14 +250,18 @@ func runCommandInSSHTunnel(ctx context.Context, p sshCommandParams, stdin, stdou }) if err != nil { _ = streamer.Close() - if out := streamer.ErrorOutput(); out != "" { - return fmt.Errorf("run agent command failed: %w\n%s", err, out) - } - return fmt.Errorf("run agent command failed: %w", err) + return commandFailure(err, streamer.ErrorOutput()) } return nil } +func commandFailure(err error, output string) error { + if output != "" { + return fmt.Errorf("run agent command failed: %v\n%s", err, output) + } + return fmt.Errorf("run agent command failed: %w", err) +} + const ( maxLogLines = 25 maxLogBytes = 256 * 1024 diff --git a/pkg/provider/download.go b/pkg/provider/download.go index f7add5319a..f5e9c71d94 100644 --- a/pkg/provider/download.go +++ b/pkg/provider/download.go @@ -135,6 +135,9 @@ func downloadBinaryForPlatform( // check if binary is correct binaryTargetFolder := filepath.Join(targetFolder, strings.ToLower(binaryName)) binaryPath := getBinaryPath(binary, binaryTargetFolder) + if isUnownedLocalBinary(binary.Path, binaryPath) { + return verifyLocalBinary(binaryPath, binary.Checksum) + } if verifyOrRemoveBinary(binaryPath, binary.Checksum) || fromCache(binary, binaryTargetFolder) { return binaryPath, nil @@ -152,6 +155,42 @@ func downloadBinaryForPlatform( binaryName, runtime.GOOS, runtime.GOARCH) } +func isUnownedLocalBinary(source, target string) bool { + return filepath.IsAbs(source) || sameLocalFile(source, target) +} + +func verifyLocalBinary(binaryPath, checksum string) (string, error) { + if checksum == "" { + if _, err := os.Stat(binaryPath); err != nil { + return "", fmt.Errorf("stat provider binary %s: %w", binaryPath, err) + } + return binaryPath, nil + } + actual, err := hash.File(binaryPath) + if err != nil { + return "", fmt.Errorf("hash provider binary %s: %w", binaryPath, err) + } + if !strings.EqualFold(actual, checksum) { + return "", fmt.Errorf( + "provider binary %s SHA-256 mismatch: expected %s, actual %s; regenerate the manifest", + binaryPath, checksum, actual, + ) + } + return binaryPath, nil +} + +func sameLocalFile(source, target string) bool { + if isRemotePath(source) { + return false + } + sourceInfo, err := os.Stat(source) + if err != nil { + return false + } + targetInfo, err := os.Stat(target) + return err == nil && os.SameFile(sourceInfo, targetInfo) +} + func downloadWithRetry( ctx context.Context, binaryName string, diff --git a/pkg/provider/local_source_safety_test.go b/pkg/provider/local_source_safety_test.go new file mode 100644 index 0000000000..56109d2475 --- /dev/null +++ b/pkg/provider/local_source_safety_test.go @@ -0,0 +1,121 @@ +package provider + +import ( + "context" + "crypto/sha256" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/suite" +) + +type localSourceSafetySuite struct{ suite.Suite } + +func TestLocalSourceSafety(t *testing.T) { suite.Run(t, new(localSourceSafetySuite)) } + +func (s *localSourceSafetySuite) TestChecksumMismatch() { + for _, tc := range []struct { + name string + remote bool + }{{name: "absolute source"}, {name: "downloaded source", remote: true}} { + s.Run(tc.name, func() { + dir := s.T().TempDir() + source := filepath.Join(dir, "source") + contents := []byte("actual binary") + s.Require().NoError(os.WriteFile(source, contents, 0o600)) + path := source + if tc.remote { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(contents) + }), + ) + defer server.Close() + path = server.URL + "/binary" + } + expected := strings.Repeat("0", 64) + actual := fmt.Sprintf("%x", sha256.Sum256(contents)) + binary := &ProviderBinary{ + OS: runtime.GOOS, + Arch: runtime.GOARCH, + Path: path, + Checksum: expected, + } + _, err := downloadBinaryForPlatform( + context.Background(), + "TEST", + []*ProviderBinary{binary}, + filepath.Join(dir, "storage"), + ) + s.Require().Error(err) + if tc.remote { + s.NoFileExists(filepath.Join(dir, "storage", "test", "binary")) + } else { + s.FileExists(source) + s.ErrorContains(err, source) + s.ErrorContains(err, expected) + s.ErrorContains(err, actual) + s.ErrorContains(err, "regenerate") + } + }) + } +} + +func (s *localSourceSafetySuite) TestManifestPaths() { + dir := s.T().TempDir() + existing := filepath.Join(dir, "manifest") + s.Require().NoError(os.WriteFile(existing, []byte("name: test\n"), 0o600)) + for _, tc := range []struct { + name, path string + missing bool + }{ + {name: "missing yaml", path: filepath.Join(dir, "missing.yaml"), missing: true}, + {name: "missing yml", path: filepath.Join(dir, "missing.yml"), missing: true}, + {name: "existing without extension", path: existing}, + } { + s.Run(tc.name, func() { + data, source, err := ResolveProvider(context.Background(), tc.path) + if tc.missing { + s.Require().Error(err) + s.ErrorContains(err, "provider manifest file not found: "+tc.path) + return + } + s.Require().NoError(err) + s.Equal("name: test\n", string(data)) + s.Equal(tc.path, source.File) + }) + } +} + +func (s *localSourceSafetySuite) TestRelativeSourceAtDestination() { + dir := s.T().TempDir() + s.Require().NoError(os.Mkdir(filepath.Join(dir, "test"), 0o750)) + source := filepath.Join(dir, "test", "binary") + s.Require().NoError(os.WriteFile(source, []byte("local"), 0o600)) + workingDir, err := os.Getwd() + s.Require().NoError(err) + relative, err := filepath.Rel(workingDir, source) + s.Require().NoError(err) + binary := &ProviderBinary{ + OS: runtime.GOOS, + Arch: runtime.GOARCH, + Path: relative, + Checksum: strings.Repeat("0", 64), + } + _, err = downloadBinaryForPlatform(context.Background(), "TEST", []*ProviderBinary{binary}, dir) + s.Require().ErrorContains(err, "regenerate") + s.FileExists(source) +} + +func (s *localSourceSafetySuite) TestInvalidCachedBinaryIsRemoved() { + file := filepath.Join(s.T().TempDir(), "cached") + s.Require().NoError(os.WriteFile(file, []byte("stale"), 0o600)) + s.False(verifyOrRemoveBinary(file, strings.Repeat("0", 64))) + s.NoFileExists(file) +} diff --git a/pkg/provider/resolve.go b/pkg/provider/resolve.go index a740db4058..1cc99155d4 100644 --- a/pkg/provider/resolve.go +++ b/pkg/provider/resolve.go @@ -195,15 +195,14 @@ func resolveFileProvider( providerSource string, retSource *ProviderSource, ) ([]byte, bool, error) { - if !strings.HasSuffix(providerSource, ".yaml") && !strings.HasSuffix(providerSource, ".yml") { - return nil, false, nil + manifestPath := strings.HasSuffix(providerSource, ".yaml") || + strings.HasSuffix(providerSource, ".yml") + info, err := os.Stat(providerSource) + if err != nil { + return resolveFileStatError(providerSource, manifestPath, err) } - - if _, err := os.Stat(providerSource); err != nil { - if os.IsNotExist(err) { - return nil, false, nil - } - return nil, true, fmt.Errorf("stat provider file %q: %w", providerSource, err) + if !manifestPath && !info.Mode().IsRegular() { + return nil, false, nil } // #nosec G304 - providerSource is user-provided path for loading provider config @@ -220,6 +219,20 @@ func resolveFileProvider( return out, true, nil } +func resolveFileStatError(source string, manifestPath bool, err error) ([]byte, bool, error) { + if !manifestPath { + return nil, false, nil + } + if !os.IsNotExist(err) { + return nil, true, fmt.Errorf("stat provider file %q: %w", source, err) + } + absPath, absErr := filepath.Abs(source) + if absErr != nil { + return nil, true, fmt.Errorf("resolve absolute path for %q: %w", source, absErr) + } + return nil, true, fmt.Errorf("provider manifest file not found: %s", absPath) +} + func downloadProvider(ctx context.Context, url string) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { diff --git a/pkg/tunnel/pipebridge.go b/pkg/tunnel/pipebridge.go index 3d8acfdcb8..42084b879a 100644 --- a/pkg/tunnel/pipebridge.go +++ b/pkg/tunnel/pipebridge.go @@ -103,7 +103,7 @@ func (pb *PipeBridge) RunPair( // Run the two sides concurrently and wait for both to finish (or the slower // side to be abandoned after joinTimeout). tunnelErr, handlerErr := iojoin.Join(tunnelSide, handlerSide, joinTimeout, stop) - classifiedErr := ClassifyTunnelErrors(tunnelErr, handlerErr) + classifiedErr := classifyBridgeErrors(ctx, tunnelErr, handlerErr) internalTunnelCancel := errors.Is(tunnelErr, context.Canceled) && ctx.Err() == nil if classifiedErr == nil && ctx.Err() == nil && (tunnelErr == nil || internalTunnelCancel) { log.Debug("pipe bridge completed successfully") @@ -118,3 +118,11 @@ func (pb *PipeBridge) RunPair( } return classifiedErr } + +func classifyBridgeErrors(ctx context.Context, tunnelErr, handlerErr error) error { + if ctx.Err() == nil && errors.Is(handlerErr, context.Canceled) && + tunnelErr != nil && !errors.Is(tunnelErr, context.Canceled) { + return tunnelErr + } + return ClassifyTunnelErrors(tunnelErr, handlerErr) +} diff --git a/pkg/tunnel/pipebridge_error_test.go b/pkg/tunnel/pipebridge_error_test.go new file mode 100644 index 0000000000..979440ab90 --- /dev/null +++ b/pkg/tunnel/pipebridge_error_test.go @@ -0,0 +1,27 @@ +package tunnel + +import ( + "context" + "errors" + "os" + "testing" + + "github.com/stretchr/testify/suite" +) + +type pipeBridgeErrorSuite struct{ suite.Suite } + +func TestPipeBridgeErrorSuite(t *testing.T) { suite.Run(t, new(pipeBridgeErrorSuite)) } + +func (s *pipeBridgeErrorSuite) TestCommandFailureIsNotMaskedByInternalCancellation() { + bridge, err := NewPipeBridge() + s.Require().NoError(err) + defer bridge.Close() + commandErr := errors.New("provider binary checksum mismatch") + err = bridge.RunPair( + context.Background(), + func(context.Context, *os.File, *os.File) error { return commandErr }, + func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return ctx.Err() }, + ) + s.ErrorIs(err, commandErr) +} From 1a5dbcc460786bfdb52e3c7e809c3cacd0c0c361 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 19:47:48 -0600 Subject: [PATCH 02/11] fix(provider): recognize missing absolute manifest paths --- pkg/provider/local_source_safety_test.go | 1 + pkg/provider/resolve.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/provider/local_source_safety_test.go b/pkg/provider/local_source_safety_test.go index 56109d2475..0f3c1ee6da 100644 --- a/pkg/provider/local_source_safety_test.go +++ b/pkg/provider/local_source_safety_test.go @@ -77,6 +77,7 @@ func (s *localSourceSafetySuite) TestManifestPaths() { }{ {name: "missing yaml", path: filepath.Join(dir, "missing.yaml"), missing: true}, {name: "missing yml", path: filepath.Join(dir, "missing.yml"), missing: true}, + {name: "missing absolute path", path: filepath.Join(dir, "missing"), missing: true}, {name: "existing without extension", path: existing}, } { s.Run(tc.name, func() { diff --git a/pkg/provider/resolve.go b/pkg/provider/resolve.go index 1cc99155d4..be9fbe62ea 100644 --- a/pkg/provider/resolve.go +++ b/pkg/provider/resolve.go @@ -195,7 +195,7 @@ func resolveFileProvider( providerSource string, retSource *ProviderSource, ) ([]byte, bool, error) { - manifestPath := strings.HasSuffix(providerSource, ".yaml") || + manifestPath := filepath.IsAbs(providerSource) || strings.HasSuffix(providerSource, ".yaml") || strings.HasSuffix(providerSource, ".yml") info, err := os.Stat(providerSource) if err != nil { From 98a0da55f2d3ae84219ad51769c5215ceae39e68 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 20:03:21 -0600 Subject: [PATCH 03/11] fix(provider): preserve startup error handling --- .../sshtunnel/command_error_test.go | 7 ++++ pkg/devcontainer/sshtunnel/sshtunnel.go | 3 ++ pkg/provider/local_source_safety_test.go | 10 +++++ pkg/provider/resolve.go | 11 ++++- pkg/tunnel/pipebridge.go | 6 ++- pkg/tunnel/pipebridge_error_test.go | 40 +++++++++++++++++++ 6 files changed, 73 insertions(+), 4 deletions(-) diff --git a/pkg/devcontainer/sshtunnel/command_error_test.go b/pkg/devcontainer/sshtunnel/command_error_test.go index e99c8962cd..5e1344c58e 100644 --- a/pkg/devcontainer/sshtunnel/command_error_test.go +++ b/pkg/devcontainer/sshtunnel/command_error_test.go @@ -1,6 +1,7 @@ package sshtunnel import ( + "context" "errors" "testing" @@ -33,3 +34,9 @@ func (s *commandErrorSuite) TestCommandFailure() { }) } } + +func (s *commandErrorSuite) TestCommandFailurePreservesCancellation() { + err := commandFailure(context.Canceled, "agent diagnostic") + s.ErrorIs(err, context.Canceled) + s.ErrorContains(err, "agent diagnostic") +} diff --git a/pkg/devcontainer/sshtunnel/sshtunnel.go b/pkg/devcontainer/sshtunnel/sshtunnel.go index bb8d3bdadf..88da8eb90e 100644 --- a/pkg/devcontainer/sshtunnel/sshtunnel.go +++ b/pkg/devcontainer/sshtunnel/sshtunnel.go @@ -257,6 +257,9 @@ func runCommandInSSHTunnel(ctx context.Context, p sshCommandParams, stdin, stdou func commandFailure(err error, output string) error { if output != "" { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return fmt.Errorf("run agent command failed: %w\n%s", err, output) + } return fmt.Errorf("run agent command failed: %v\n%s", err, output) } return fmt.Errorf("run agent command failed: %w", err) diff --git a/pkg/provider/local_source_safety_test.go b/pkg/provider/local_source_safety_test.go index 0f3c1ee6da..0b41c61284 100644 --- a/pkg/provider/local_source_safety_test.go +++ b/pkg/provider/local_source_safety_test.go @@ -120,3 +120,13 @@ func (s *localSourceSafetySuite) TestInvalidCachedBinaryIsRemoved() { s.False(verifyOrRemoveBinary(file, strings.Repeat("0", 64))) s.NoFileExists(file) } + +func (s *localSourceSafetySuite) TestBareNameDoesNotReadLocalFile() { + file, err := os.CreateTemp(".", "provider-name-") + s.Require().NoError(err) + s.Require().NoError(file.Close()) + s.T().Cleanup(func() { _ = os.Remove(file.Name()) }) + _, ok, err := resolveFileProvider(filepath.Base(file.Name()), &ProviderSource{}) + s.NoError(err) + s.False(ok) +} diff --git a/pkg/provider/resolve.go b/pkg/provider/resolve.go index be9fbe62ea..9e6e4f57d3 100644 --- a/pkg/provider/resolve.go +++ b/pkg/provider/resolve.go @@ -195,8 +195,10 @@ func resolveFileProvider( providerSource string, retSource *ProviderSource, ) ([]byte, bool, error) { - manifestPath := filepath.IsAbs(providerSource) || strings.HasSuffix(providerSource, ".yaml") || - strings.HasSuffix(providerSource, ".yml") + manifestPath := isManifestPath(providerSource) + if !manifestPath && filepath.Base(providerSource) == providerSource { + return nil, false, nil + } info, err := os.Stat(providerSource) if err != nil { return resolveFileStatError(providerSource, manifestPath, err) @@ -219,6 +221,11 @@ func resolveFileProvider( return out, true, nil } +func isManifestPath(source string) bool { + return filepath.IsAbs(source) || strings.HasSuffix(source, ".yaml") || + strings.HasSuffix(source, ".yml") +} + func resolveFileStatError(source string, manifestPath bool, err error) ([]byte, bool, error) { if !manifestPath { return nil, false, nil diff --git a/pkg/tunnel/pipebridge.go b/pkg/tunnel/pipebridge.go index 42084b879a..078390f82c 100644 --- a/pkg/tunnel/pipebridge.go +++ b/pkg/tunnel/pipebridge.go @@ -120,8 +120,10 @@ func (pb *PipeBridge) RunPair( } func classifyBridgeErrors(ctx context.Context, tunnelErr, handlerErr error) error { - if ctx.Err() == nil && errors.Is(handlerErr, context.Canceled) && - tunnelErr != nil && !errors.Is(tunnelErr, context.Canceled) { + if ctx.Err() == nil && + (errors.Is(handlerErr, context.Canceled) || errors.Is(handlerErr, os.ErrClosed)) && + tunnelErr != nil && + !errors.Is(tunnelErr, context.Canceled) { return tunnelErr } return ClassifyTunnelErrors(tunnelErr, handlerErr) diff --git a/pkg/tunnel/pipebridge_error_test.go b/pkg/tunnel/pipebridge_error_test.go index 979440ab90..c421642412 100644 --- a/pkg/tunnel/pipebridge_error_test.go +++ b/pkg/tunnel/pipebridge_error_test.go @@ -3,6 +3,7 @@ package tunnel import ( "context" "errors" + "fmt" "os" "testing" @@ -25,3 +26,42 @@ func (s *pipeBridgeErrorSuite) TestCommandFailureIsNotMaskedByInternalCancellati ) s.ErrorIs(err, commandErr) } + +func (s *pipeBridgeErrorSuite) TestClosedPipeDoesNotMaskCommandFailure() { + commandErr := errors.New("command failed") + err := classifyBridgeErrors( + context.Background(), + commandErr, + &os.PathError{Op: "write", Path: "pipe", Err: os.ErrClosed}, + ) + s.ErrorIs(err, commandErr) +} + +func (s *pipeBridgeErrorSuite) TestCancellationClassification() { + for _, tc := range []struct { + name string + cancelParent bool + }{ + {name: "internal cancellation"}, + {name: "parent cancellation", cancelParent: true}, + } { + s.Run(tc.name, func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if tc.cancelParent { + cancel() + } + commandErr := errors.New("command failed") + err := classifyBridgeErrors( + ctx, + commandErr, + fmt.Errorf("handler: %w", context.Canceled), + ) + if tc.cancelParent { + s.ErrorIs(err, context.Canceled) + } else { + s.ErrorIs(err, commandErr) + } + }) + } +} From 109222ba41f585b3a71ad42178cd9a55b8987489 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 20:23:12 -0600 Subject: [PATCH 04/11] fix(provider): preserve source and process error semantics --- cmd/provider_diagnostic_test.go | 43 +++++++++++++++++++ cmd/root.go | 7 ++- .../sshtunnel/command_error_test.go | 2 +- pkg/devcontainer/sshtunnel/sshtunnel.go | 18 ++++++-- pkg/provider/download.go | 10 +++-- pkg/provider/local_source_safety_test.go | 28 ++++++++++++ pkg/provider/resolve.go | 14 +++++- pkg/tunnel/pipebridge.go | 3 ++ pkg/tunnel/pipebridge_error_test.go | 8 ++++ 9 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 cmd/provider_diagnostic_test.go diff --git a/cmd/provider_diagnostic_test.go b/cmd/provider_diagnostic_test.go new file mode 100644 index 0000000000..0c78be3c63 --- /dev/null +++ b/cmd/provider_diagnostic_test.go @@ -0,0 +1,43 @@ +package cmd + +import ( + "fmt" + "io" + "os" + "testing" + + "github.com/stretchr/testify/suite" + "golang.org/x/crypto/ssh" +) + +type ( + providerDiagnosticSuite struct{ suite.Suite } + diagnosticExit struct{ err error } +) + +func (e diagnosticExit) Error() string { return fmt.Sprintf("provider diagnostic: %v", e.err) } +func (e diagnosticExit) Unwrap() error { return e.err } +func (e diagnosticExit) AgentDiagnostic() {} + +func TestProviderDiagnosticSuite(t *testing.T) { suite.Run(t, new(providerDiagnosticSuite)) } + +func (s *providerDiagnosticSuite) TestExitPreservesStatusAndDiagnostic() { + for _, machine := range []bool{false, true} { + s.Run(fmt.Sprintf("machine=%t", machine), func() { + reader, writer, err := os.Pipe() + s.Require().NoError(err) + old := os.Stderr + os.Stderr = writer + exit := &ssh.ExitError{} + code, ok := passthroughExitCode(diagnosticExit{err: exit}, machine) + os.Stderr = old + s.Require().NoError(writer.Close()) + output, err := io.ReadAll(reader) + s.Require().NoError(err) + s.Require().NoError(reader.Close()) + s.True(ok) + s.Equal(exit.ExitStatus(), code) + s.Contains(string(output), "provider diagnostic") + }) + } +} diff --git a/cmd/root.go b/cmd/root.go index 47ef5805b6..9a93926c56 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -271,7 +271,12 @@ func exitCodeForError(err error, machineMode bool) int { func passthroughExitCode(err error, machineMode bool) (int, bool) { if sshExitErr, ok := errors.AsType[*ssh.ExitError](err); ok { - if machineMode { + if _, hasDiagnostic := errors.AsType[interface { + error + AgentDiagnostic() + }](err); hasDiagnostic { + renderCLIError(clierr.Classify(err), machineMode) + } else if machineMode { renderCLIError( clierr.Classify(fmt.Errorf( "SSH command failed with exit code %d", diff --git a/pkg/devcontainer/sshtunnel/command_error_test.go b/pkg/devcontainer/sshtunnel/command_error_test.go index 5e1344c58e..100b1371d2 100644 --- a/pkg/devcontainer/sshtunnel/command_error_test.go +++ b/pkg/devcontainer/sshtunnel/command_error_test.go @@ -29,7 +29,7 @@ func (s *commandErrorSuite) TestCommandFailure() { s.ErrorIs(err, exit) } else { s.ErrorContains(err, tc.output) - s.NotErrorIs(err, exit) + s.ErrorIs(err, exit) } }) } diff --git a/pkg/devcontainer/sshtunnel/sshtunnel.go b/pkg/devcontainer/sshtunnel/sshtunnel.go index 88da8eb90e..0a9761ef22 100644 --- a/pkg/devcontainer/sshtunnel/sshtunnel.go +++ b/pkg/devcontainer/sshtunnel/sshtunnel.go @@ -257,14 +257,24 @@ func runCommandInSSHTunnel(ctx context.Context, p sshCommandParams, stdin, stdou func commandFailure(err error, output string) error { if output != "" { - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return fmt.Errorf("run agent command failed: %w\n%s", err, output) - } - return fmt.Errorf("run agent command failed: %v\n%s", err, output) + return &agentCommandError{err: err, output: output} } return fmt.Errorf("run agent command failed: %w", err) } +type agentCommandError struct { + err error + output string +} + +func (e *agentCommandError) Error() string { + return fmt.Sprintf("run agent command failed: %v\n%s", e.err, e.output) +} + +func (e *agentCommandError) Unwrap() error { return e.err } + +func (e *agentCommandError) AgentDiagnostic() {} + const ( maxLogLines = 25 maxLogBytes = 256 * 1024 diff --git a/pkg/provider/download.go b/pkg/provider/download.go index f5e9c71d94..0466708710 100644 --- a/pkg/provider/download.go +++ b/pkg/provider/download.go @@ -160,10 +160,14 @@ func isUnownedLocalBinary(source, target string) bool { } func verifyLocalBinary(binaryPath, checksum string) (string, error) { + info, err := os.Stat(binaryPath) + if err != nil { + return "", fmt.Errorf("stat provider binary %s: %w", binaryPath, err) + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("provider binary %s is not a regular file", binaryPath) + } if checksum == "" { - if _, err := os.Stat(binaryPath); err != nil { - return "", fmt.Errorf("stat provider binary %s: %w", binaryPath, err) - } return binaryPath, nil } actual, err := hash.File(binaryPath) diff --git a/pkg/provider/local_source_safety_test.go b/pkg/provider/local_source_safety_test.go index 0b41c61284..32d8ff6b60 100644 --- a/pkg/provider/local_source_safety_test.go +++ b/pkg/provider/local_source_safety_test.go @@ -130,3 +130,31 @@ func (s *localSourceSafetySuite) TestBareNameDoesNotReadLocalFile() { s.NoError(err) s.False(ok) } + +func (s *localSourceSafetySuite) TestDirectoryIsNotBinary() { + dir := s.T().TempDir() + _, err := verifyLocalBinary(dir, "") + s.Require().Error(err) +} + +func (s *localSourceSafetySuite) TestGitHubSourcesDoNotResolveAsFiles() { + for _, source := range []string{"owner/repo", "owner/repo.yaml", "github.com/owner/repo.yml"} { + s.Run(source, func() { + _, ok, err := resolveFileProvider(source, &ProviderSource{}) + s.NoError(err) + s.False(ok) + }) + } +} + +func (s *localSourceSafetySuite) TestGitHubSourceDoesNotReadLocalFile() { + dir, err := os.MkdirTemp(".", "owner-") + s.Require().NoError(err) + file := filepath.Join(dir, "repo") + s.Require().NoError(os.WriteFile(file, []byte("local"), 0o600)) + s.T().Cleanup(func() { _ = os.Remove(file); _ = os.Remove(dir) }) + source := filepath.Base(dir) + "/repo" + _, ok, err := resolveFileProvider(source, &ProviderSource{}) + s.NoError(err) + s.False(ok) +} diff --git a/pkg/provider/resolve.go b/pkg/provider/resolve.go index 9e6e4f57d3..e66897d907 100644 --- a/pkg/provider/resolve.go +++ b/pkg/provider/resolve.go @@ -196,7 +196,7 @@ func resolveFileProvider( retSource *ProviderSource, ) ([]byte, bool, error) { manifestPath := isManifestPath(providerSource) - if !manifestPath && filepath.Base(providerSource) == providerSource { + if isNonFileProvider(providerSource, manifestPath) { return nil, false, nil } info, err := os.Stat(providerSource) @@ -221,6 +221,18 @@ func resolveFileProvider( return out, true, nil } +func isNonFileProvider(source string, manifestPath bool) bool { + return isGitHubShorthand(source) || (!manifestPath && filepath.Base(source) == source) +} + +func isGitHubShorthand(source string) bool { + if filepath.IsAbs(source) || strings.HasPrefix(source, "./") || + strings.HasPrefix(source, "../") { + return false + } + return strings.HasPrefix(source, "github.com/") || strings.Count(source, "/") == 1 +} + func isManifestPath(source string) bool { return filepath.IsAbs(source) || strings.HasSuffix(source, ".yaml") || strings.HasSuffix(source, ".yml") diff --git a/pkg/tunnel/pipebridge.go b/pkg/tunnel/pipebridge.go index 078390f82c..78191f3daa 100644 --- a/pkg/tunnel/pipebridge.go +++ b/pkg/tunnel/pipebridge.go @@ -124,6 +124,9 @@ func classifyBridgeErrors(ctx context.Context, tunnelErr, handlerErr error) erro (errors.Is(handlerErr, context.Canceled) || errors.Is(handlerErr, os.ErrClosed)) && tunnelErr != nil && !errors.Is(tunnelErr, context.Canceled) { + if errors.Is(handlerErr, os.ErrClosed) { + return errors.Join(tunnelErr, handlerErr) + } return tunnelErr } return ClassifyTunnelErrors(tunnelErr, handlerErr) diff --git a/pkg/tunnel/pipebridge_error_test.go b/pkg/tunnel/pipebridge_error_test.go index c421642412..acacac1d84 100644 --- a/pkg/tunnel/pipebridge_error_test.go +++ b/pkg/tunnel/pipebridge_error_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "os" "testing" @@ -65,3 +66,10 @@ func (s *pipeBridgeErrorSuite) TestCancellationClassification() { }) } } + +func (s *pipeBridgeErrorSuite) TestClosedHandlerAndTunnelFailureArePreserved() { + handlerErr := &os.PathError{Op: "write", Path: "pipe", Err: os.ErrClosed} + err := classifyBridgeErrors(context.Background(), io.EOF, handlerErr) + s.ErrorIs(err, io.EOF) + s.ErrorIs(err, os.ErrClosed) +} From 3525b9ede74b2bc68edc7af7763b905eb14298a5 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 20:31:36 -0600 Subject: [PATCH 05/11] fix(provider): retain relative manifest resolution --- pkg/provider/local_source_safety_test.go | 13 +++++++++++++ pkg/provider/resolve.go | 16 +++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/pkg/provider/local_source_safety_test.go b/pkg/provider/local_source_safety_test.go index 32d8ff6b60..99f52f19e8 100644 --- a/pkg/provider/local_source_safety_test.go +++ b/pkg/provider/local_source_safety_test.go @@ -158,3 +158,16 @@ func (s *localSourceSafetySuite) TestGitHubSourceDoesNotReadLocalFile() { s.NoError(err) s.False(ok) } + +func (s *localSourceSafetySuite) TestRelativeYAMLManifest() { + dir, err := os.MkdirTemp(".", "manifest-dir-") + s.Require().NoError(err) + file := filepath.Join(dir, "provider.yaml") + s.Require().NoError(os.WriteFile(file, []byte("name: local\n"), 0o600)) + s.T().Cleanup(func() { _ = os.Remove(file); _ = os.Remove(dir) }) + source := filepath.Base(dir) + "/provider.yaml" + data, ok, err := resolveFileProvider(source, &ProviderSource{}) + s.Require().NoError(err) + s.True(ok) + s.Equal("name: local\n", string(data)) +} diff --git a/pkg/provider/resolve.go b/pkg/provider/resolve.go index e66897d907..e283f68b3d 100644 --- a/pkg/provider/resolve.go +++ b/pkg/provider/resolve.go @@ -222,7 +222,14 @@ func resolveFileProvider( } func isNonFileProvider(source string, manifestPath bool) bool { - return isGitHubShorthand(source) || (!manifestPath && filepath.Base(source) == source) + if isGitHubShorthand(source) { + if !hasYAMLSuffix(source) { + return true + } + _, err := os.Stat(source) + return os.IsNotExist(err) + } + return !manifestPath && filepath.Base(source) == source } func isGitHubShorthand(source string) bool { @@ -234,8 +241,11 @@ func isGitHubShorthand(source string) bool { } func isManifestPath(source string) bool { - return filepath.IsAbs(source) || strings.HasSuffix(source, ".yaml") || - strings.HasSuffix(source, ".yml") + return filepath.IsAbs(source) || hasYAMLSuffix(source) +} + +func hasYAMLSuffix(source string) bool { + return strings.HasSuffix(source, ".yaml") || strings.HasSuffix(source, ".yml") } func resolveFileStatError(source string, manifestPath bool, err error) ([]byte, bool, error) { From 6f5c5f11c4a50a7e06ea6ad053963ad2c50b0579 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 21:11:37 -0600 Subject: [PATCH 06/11] fix(provider): distinguish explicit manifest paths --- cmd/provider_diagnostic_test.go | 44 +++++++++++- cmd/root.go | 39 +++++++++-- pkg/devcontainer/sshtunnel/sshtunnel.go | 2 +- pkg/provider/download.go | 18 ++++- pkg/provider/local_source_safety_test.go | 88 ++++++++++++++++++++++++ pkg/provider/resolve.go | 53 +++++++------- pkg/tunnel/pipebridge.go | 34 ++++++++- pkg/tunnel/pipebridge_error_test.go | 26 +++++++ 8 files changed, 267 insertions(+), 37 deletions(-) diff --git a/cmd/provider_diagnostic_test.go b/cmd/provider_diagnostic_test.go index 0c78be3c63..f6d048de21 100644 --- a/cmd/provider_diagnostic_test.go +++ b/cmd/provider_diagnostic_test.go @@ -6,6 +6,7 @@ import ( "os" "testing" + "github.com/devsy-org/devsy/pkg/clierr" "github.com/stretchr/testify/suite" "golang.org/x/crypto/ssh" ) @@ -15,9 +16,9 @@ type ( diagnosticExit struct{ err error } ) -func (e diagnosticExit) Error() string { return fmt.Sprintf("provider diagnostic: %v", e.err) } -func (e diagnosticExit) Unwrap() error { return e.err } -func (e diagnosticExit) AgentDiagnostic() {} +func (e diagnosticExit) Error() string { return fmt.Sprintf("provider diagnostic: %v", e.err) } +func (e diagnosticExit) Unwrap() error { return e.err } +func (e diagnosticExit) AgentDiagnostic() string { return e.Error() } func TestProviderDiagnosticSuite(t *testing.T) { suite.Run(t, new(providerDiagnosticSuite)) } @@ -41,3 +42,40 @@ func (s *providerDiagnosticSuite) TestExitPreservesStatusAndDiagnostic() { }) } } + +func (s *providerDiagnosticSuite) TestExitCodeRetainsDiagnosticForClassifiedText() { + reader, writer, err := os.Pipe() + s.Require().NoError(err) + old := os.Stderr + os.Stderr = writer + exit := &ssh.ExitError{} + wrapped := diagnosticExit{err: fmt.Errorf("Cannot connect to the Docker daemon: %w", exit)} + code := exitCodeForError(wrapped, true) + os.Stderr = old + s.Require().NoError(writer.Close()) + output, err := io.ReadAll(reader) + s.Require().NoError(err) + s.Require().NoError(reader.Close()) + s.Equal(exit.ExitStatus(), code) + s.Contains(string(output), "Cannot connect to the Docker daemon") +} + +func (s *providerDiagnosticSuite) TestClassifiedWrapperRetainsAgentDiagnostic() { + reader, writer, err := os.Pipe() + s.Require().NoError(err) + old := os.Stderr + os.Stderr = writer + exit := &ssh.ExitError{} + wrapped := clierr.WithContext( + diagnosticExit{err: fmt.Errorf("Cannot connect to the Docker daemon: %w", exit)}, + map[string]string{"run": "test"}, + ) + code := exitCodeForError(wrapped, true) + os.Stderr = old + s.Require().NoError(writer.Close()) + output, err := io.ReadAll(reader) + s.Require().NoError(err) + s.Require().NoError(reader.Close()) + s.Equal(exit.ExitStatus(), code) + s.Contains(string(output), "Cannot connect to the Docker daemon") +} diff --git a/cmd/root.go b/cmd/root.go index 9a93926c56..6c15bd6c37 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -249,6 +249,11 @@ func exitCodeForError(err error, machineMode bool) int { if err == nil { return exitcode.Success } + if shouldPassthroughAgent(err) { + if code, ok := passthroughExitCode(err, machineMode); ok { + return code + } + } cliErr := clierr.Classify(err) @@ -269,13 +274,21 @@ func exitCodeForError(err error, machineMode bool) int { return exitcode.Failure } +func shouldPassthroughAgent(err error) bool { + return isAgentDiagnostic(err) && !errors.Is(err, gocontext.Canceled) && + !errors.Is( + err, + gocontext.DeadlineExceeded, + ) && !errors.Is(err, clierr.ErrBuildFailedRecoverable) +} + func passthroughExitCode(err error, machineMode bool) (int, bool) { if sshExitErr, ok := errors.AsType[*ssh.ExitError](err); ok { - if _, hasDiagnostic := errors.AsType[interface { - error - AgentDiagnostic() - }](err); hasDiagnostic { - renderCLIError(clierr.Classify(err), machineMode) + if diagnostic, ok := agentDiagnostic(err); ok { + renderCLIError( + &clierr.CLIError{Code: clierr.CodeUnknown, Message: diagnostic}, + machineMode, + ) } else if machineMode { renderCLIError( clierr.Classify(fmt.Errorf( @@ -308,6 +321,22 @@ func passthroughExitCode(err error, machineMode bool) (int, bool) { return 0, false } +func isAgentDiagnostic(err error) bool { + _, ok := agentDiagnostic(err) + return ok +} + +func agentDiagnostic(err error) (string, bool) { + diagnostic, ok := errors.AsType[interface { + error + AgentDiagnostic() string + }](err) + if !ok { + return "", false + } + return diagnostic.AgentDiagnostic(), true +} + func renderCLIError(cliErr *clierr.CLIError, machineMode bool) { if cliErr == nil { return diff --git a/pkg/devcontainer/sshtunnel/sshtunnel.go b/pkg/devcontainer/sshtunnel/sshtunnel.go index 0a9761ef22..9ee0fdc96d 100644 --- a/pkg/devcontainer/sshtunnel/sshtunnel.go +++ b/pkg/devcontainer/sshtunnel/sshtunnel.go @@ -273,7 +273,7 @@ func (e *agentCommandError) Error() string { func (e *agentCommandError) Unwrap() error { return e.err } -func (e *agentCommandError) AgentDiagnostic() {} +func (e *agentCommandError) AgentDiagnostic() string { return e.Error() } const ( maxLogLines = 25 diff --git a/pkg/provider/download.go b/pkg/provider/download.go index 0466708710..9118c56d53 100644 --- a/pkg/provider/download.go +++ b/pkg/provider/download.go @@ -136,7 +136,7 @@ func downloadBinaryForPlatform( binaryTargetFolder := filepath.Join(targetFolder, strings.ToLower(binaryName)) binaryPath := getBinaryPath(binary, binaryTargetFolder) if isUnownedLocalBinary(binary.Path, binaryPath) { - return verifyLocalBinary(binaryPath, binary.Checksum) + return prepareLocalSource(binary, binaryPath, binaryTargetFolder) } if verifyOrRemoveBinary(binaryPath, binary.Checksum) || fromCache(binary, binaryTargetFolder) { @@ -159,6 +159,22 @@ func isUnownedLocalBinary(source, target string) bool { return filepath.IsAbs(source) || sameLocalFile(source, target) } +func prepareLocalSource(binary *ProviderBinary, source, targetFolder string) (string, error) { + result, err := verifyLocalBinary(source, binary.Checksum) + if !filepath.IsAbs(binary.Path) || !errors.Is(err, os.ErrNotExist) { + return result, err + } + target := localTargetPath(binary, targetFolder) + absTarget, absErr := filepath.Abs(target) + if absErr != nil || filepath.Clean(absTarget) == filepath.Clean(source) { + return "", err + } + if _, targetErr := verifyLocalBinary(target, binary.Checksum); targetErr == nil { + return target, nil + } + return "", err +} + func verifyLocalBinary(binaryPath, checksum string) (string, error) { info, err := os.Stat(binaryPath) if err != nil { diff --git a/pkg/provider/local_source_safety_test.go b/pkg/provider/local_source_safety_test.go index 99f52f19e8..d63d67a17b 100644 --- a/pkg/provider/local_source_safety_test.go +++ b/pkg/provider/local_source_safety_test.go @@ -171,3 +171,91 @@ func (s *localSourceSafetySuite) TestRelativeYAMLManifest() { s.True(ok) s.Equal("name: local\n", string(data)) } + +func (s *localSourceSafetySuite) TestMissingAbsoluteSourceUsesStoredCopy() { + dir := s.T().TempDir() + target := filepath.Join(dir, "test", "binary") + s.Require().NoError(os.MkdirAll(filepath.Dir(target), 0o750)) + content := []byte("installed binary") + s.Require().NoError(os.WriteFile(target, content, 0o600)) + checksum := fmt.Sprintf("%x", sha256.Sum256(content)) + binary := &ProviderBinary{ + OS: runtime.GOOS, + Arch: runtime.GOARCH, + Path: filepath.Join(dir, "missing"), + Name: "binary", + Checksum: checksum, + } + got, err := downloadBinaryForPlatform( + context.Background(), + "TEST", + []*ProviderBinary{binary}, + dir, + ) + s.Require().NoError(err) + s.Equal(target, got) + s.FileExists(target) +} + +func (s *localSourceSafetySuite) TestMissingExplicitManifestForms() { + name := filepath.Base(s.T().TempDir()) + ".yaml" + home, err := os.UserHomeDir() + s.Require().NoError(err) + cases := []struct { + name, source, want string + local bool + }{ + { + name: "dot", + source: "./" + name, + want: filepath.Join(mustWorkingDir(s), name), + local: true, + }, + { + name: "parent", + source: "../" + name, + want: filepath.Join(filepath.Dir(mustWorkingDir(s)), name), + local: true, + }, + { + name: "absolute", + source: filepath.Join(s.T().TempDir(), name), + want: filepath.Join(s.T().TempDir(), name), + local: true, + }, + {name: "home", source: "~/" + name, want: filepath.Join(home, name), local: true}, + {name: "bare repository", source: "owner/repo.yaml"}, + {name: "bare yaml", source: name}, + } + cases[2].want = cases[2].source + if runtime.GOOS == windowsOS { + drivePath := filepath.Join(filepath.VolumeName(mustWorkingDir(s))+`\`, name) + cases = append(cases, struct { + name, source, want string + local bool + }{name: "backslash", source: `.\` + name, want: filepath.Join(mustWorkingDir(s), name), local: true}) + cases = append(cases, struct { + name, source, want string + local bool + }{name: "drive", source: drivePath, want: drivePath, local: true}) + } + for _, tc := range cases { + s.Run(tc.name, func() { + _, ok, err := resolveFileProvider(tc.source, &ProviderSource{}) + if !tc.local { + s.NoError(err) + s.False(ok) + return + } + s.True(ok) + s.Require().Error(err) + s.ErrorContains(err, "provider manifest file not found: "+tc.want) + }) + } +} + +func mustWorkingDir(s *localSourceSafetySuite) string { + dir, err := os.Getwd() + s.Require().NoError(err) + return dir +} diff --git a/pkg/provider/resolve.go b/pkg/provider/resolve.go index e283f68b3d..77ef86e54a 100644 --- a/pkg/provider/resolve.go +++ b/pkg/provider/resolve.go @@ -195,25 +195,29 @@ func resolveFileProvider( providerSource string, retSource *ProviderSource, ) ([]byte, bool, error) { - manifestPath := isManifestPath(providerSource) - if isNonFileProvider(providerSource, manifestPath) { + explicit := isExplicitManifestPath(providerSource) + if skipManifestSource(providerSource, explicit) { return nil, false, nil } - info, err := os.Stat(providerSource) + filePath, err := expandManifestPath(providerSource) if err != nil { - return resolveFileStatError(providerSource, manifestPath, err) + return nil, true, err } - if !manifestPath && !info.Mode().IsRegular() { + info, err := os.Stat(filePath) + if err != nil { + return resolveFileStatError(filePath, explicit, err) + } + if !explicit && !info.Mode().IsRegular() { return nil, false, nil } // #nosec G304 - providerSource is user-provided path for loading provider config - out, err := os.ReadFile(providerSource) + out, err := os.ReadFile(filePath) if err != nil { return nil, true, fmt.Errorf("read provider file %q: %w", providerSource, err) } - absPath, err := filepath.Abs(providerSource) + absPath, err := filepath.Abs(filePath) if err != nil { return nil, true, fmt.Errorf("resolve absolute path for %q: %w", providerSource, err) } @@ -221,27 +225,28 @@ func resolveFileProvider( return out, true, nil } -func isNonFileProvider(source string, manifestPath bool) bool { - if isGitHubShorthand(source) { - if !hasYAMLSuffix(source) { - return true - } - _, err := os.Stat(source) - return os.IsNotExist(err) - } - return !manifestPath && filepath.Base(source) == source +func skipManifestSource(source string, explicit bool) bool { + return !explicit && !hasYAMLSuffix(source) } -func isGitHubShorthand(source string) bool { - if filepath.IsAbs(source) || strings.HasPrefix(source, "./") || - strings.HasPrefix(source, "../") { - return false - } - return strings.HasPrefix(source, "github.com/") || strings.Count(source, "/") == 1 +func isExplicitManifestPath(source string) bool { + return filepath.IsAbs(source) || strings.HasPrefix(source, "/") || + strings.HasPrefix(source, "./") || + strings.HasPrefix(source, "../") || + strings.HasPrefix(source, "~") || + strings.Contains(source, `\`) || + (len(source) >= 2 && source[1] == ':') } -func isManifestPath(source string) bool { - return filepath.IsAbs(source) || hasYAMLSuffix(source) +func expandManifestPath(source string) (string, error) { + if !strings.HasPrefix(source, "~") { + return source, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("resolve home directory: %w", err) + } + return filepath.Join(home, strings.TrimLeft(source[1:], `/\`)), nil } func hasYAMLSuffix(source string) bool { diff --git a/pkg/tunnel/pipebridge.go b/pkg/tunnel/pipebridge.go index 78191f3daa..e7a5b9ab93 100644 --- a/pkg/tunnel/pipebridge.go +++ b/pkg/tunnel/pipebridge.go @@ -5,6 +5,7 @@ import ( "errors" "os" "sync" + "sync/atomic" "time" "github.com/devsy-org/devsy/pkg/log" @@ -95,15 +96,22 @@ func (pb *PipeBridge) RunPair( } }() - tunnelSide := func() error { return tunnelFn(pairCtx, pb.StdinReader, pb.StdoutWriter) } + var firstSide atomic.Uint32 + tunnelSide := func() error { + err := tunnelFn(pairCtx, pb.StdinReader, pb.StdoutWriter) + firstSide.CompareAndSwap(0, 1) + return err + } handlerSide := func() error { - return handlerFn(pairCtx, pb.StdoutReader, pb.StdinWriter) + err := handlerFn(pairCtx, pb.StdoutReader, pb.StdinWriter) + firstSide.CompareAndSwap(0, 2) + return err } // Run the two sides concurrently and wait for both to finish (or the slower // side to be abandoned after joinTimeout). tunnelErr, handlerErr := iojoin.Join(tunnelSide, handlerSide, joinTimeout, stop) - classifiedErr := classifyBridgeErrors(ctx, tunnelErr, handlerErr) + classifiedErr := classifyOrderedBridgeErrors(ctx, tunnelErr, handlerErr, firstSide.Load()) internalTunnelCancel := errors.Is(tunnelErr, context.Canceled) && ctx.Err() == nil if classifiedErr == nil && ctx.Err() == nil && (tunnelErr == nil || internalTunnelCancel) { log.Debug("pipe bridge completed successfully") @@ -119,6 +127,26 @@ func (pb *PipeBridge) RunPair( return classifiedErr } +func classifyOrderedBridgeErrors( + ctx context.Context, + tunnelErr, handlerErr error, + firstSide uint32, +) error { + if isIndependentTunnelError(ctx, tunnelErr) { + if firstSide == 1 && handlerErr == nil && !IsEOF(tunnelErr) { + return tunnelErr + } + if firstSide == 2 && errors.Is(handlerErr, context.Canceled) { + return errors.Join(tunnelErr, handlerErr) + } + } + return classifyBridgeErrors(ctx, tunnelErr, handlerErr) +} + +func isIndependentTunnelError(ctx context.Context, err error) bool { + return ctx.Err() == nil && err != nil && !errors.Is(err, context.Canceled) +} + func classifyBridgeErrors(ctx context.Context, tunnelErr, handlerErr error) error { if ctx.Err() == nil && (errors.Is(handlerErr, context.Canceled) || errors.Is(handlerErr, os.ErrClosed)) && diff --git a/pkg/tunnel/pipebridge_error_test.go b/pkg/tunnel/pipebridge_error_test.go index acacac1d84..b02105eaca 100644 --- a/pkg/tunnel/pipebridge_error_test.go +++ b/pkg/tunnel/pipebridge_error_test.go @@ -73,3 +73,29 @@ func (s *pipeBridgeErrorSuite) TestClosedHandlerAndTunnelFailureArePreserved() { s.ErrorIs(err, io.EOF) s.ErrorIs(err, os.ErrClosed) } + +func (s *pipeBridgeErrorSuite) TestCommandFailureBeforeHandlerSuccess() { + bridge, err := NewPipeBridge() + s.Require().NoError(err) + defer bridge.Close() + commandErr := errors.New("command failed") + err = bridge.RunPair(context.Background(), + func(context.Context, *os.File, *os.File) error { return commandErr }, + func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return nil }, + ) + s.ErrorIs(err, commandErr) +} + +func (s *pipeBridgeErrorSuite) TestIndependentHandlerCancellationIsPreserved() { + bridge, err := NewPipeBridge() + s.Require().NoError(err) + defer bridge.Close() + commandErr := errors.New("command failed during shutdown") + err = bridge.RunPair( + context.Background(), + func(ctx context.Context, _, _ *os.File) error { <-ctx.Done(); return commandErr }, + func(context.Context, *os.File, *os.File) error { return fmt.Errorf("handler: %w", context.Canceled) }, + ) + s.ErrorIs(err, context.Canceled) + s.ErrorIs(err, commandErr) +} From a121cfabd2614693e1c4c3e5984bbe9df20824ef Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 21:19:25 -0600 Subject: [PATCH 07/11] fix(provider): report manifest filesystem errors --- pkg/provider/local_source_safety_test.go | 15 +++++++++++++++ pkg/provider/resolve.go | 6 +++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/pkg/provider/local_source_safety_test.go b/pkg/provider/local_source_safety_test.go index d63d67a17b..3e10c3aa12 100644 --- a/pkg/provider/local_source_safety_test.go +++ b/pkg/provider/local_source_safety_test.go @@ -259,3 +259,18 @@ func mustWorkingDir(s *localSourceSafetySuite) string { s.Require().NoError(err) return dir } + +func (s *localSourceSafetySuite) TestRelativeYAMLStatError() { + if runtime.GOOS == windowsOS { + s.T().Skip("Windows reports a missing path for a non-directory parent") + } + file, err := os.CreateTemp(".", "manifest-parent-") + s.Require().NoError(err) + s.Require().NoError(file.Close()) + s.T().Cleanup(func() { _ = os.Remove(file.Name()) }) + source := filepath.Join(filepath.Base(file.Name()), "provider.yaml") + _, ok, err := resolveFileProvider(source, &ProviderSource{}) + s.True(ok) + s.Require().Error(err) + s.ErrorContains(err, "stat provider file") +} diff --git a/pkg/provider/resolve.go b/pkg/provider/resolve.go index 77ef86e54a..ce49124faf 100644 --- a/pkg/provider/resolve.go +++ b/pkg/provider/resolve.go @@ -254,12 +254,12 @@ func hasYAMLSuffix(source string) bool { } func resolveFileStatError(source string, manifestPath bool, err error) ([]byte, bool, error) { - if !manifestPath { - return nil, false, nil - } if !os.IsNotExist(err) { return nil, true, fmt.Errorf("stat provider file %q: %w", source, err) } + if !manifestPath { + return nil, false, nil + } absPath, absErr := filepath.Abs(source) if absErr != nil { return nil, true, fmt.Errorf("resolve absolute path for %q: %w", source, absErr) From 1a7a27436498aa19e1b3eaaa4c35bd432ce00ca3 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 21:53:58 -0600 Subject: [PATCH 08/11] fix(provider): protect local sources and agent secrets --- cmd/workspace/up/agent.go | 15 ++++++ .../up/agent_secret_redaction_test.go | 27 ++++++++++ pkg/devcontainer/setup.go | 29 ++++++++-- .../setup_secret_redaction_test.go | 29 ++++++++++ .../sshtunnel/command_error_test.go | 38 +++++++++++++ pkg/devcontainer/sshtunnel/sshtunnel.go | 19 ++++--- pkg/devcontainer/sshtunnel/sshtunnel_test.go | 10 ++-- pkg/provider/download.go | 54 +++++++++++++------ pkg/provider/local_source_safety_test.go | 46 ++++++++++++++++ pkg/secrets/redact.go | 22 +++++++- pkg/tunnel/pipebridge.go | 14 ++++- pkg/tunnel/pipebridge_error_test.go | 20 +++++++ 12 files changed, 286 insertions(+), 37 deletions(-) create mode 100644 cmd/workspace/up/agent_secret_redaction_test.go create mode 100644 pkg/devcontainer/setup_secret_redaction_test.go diff --git a/cmd/workspace/up/agent.go b/cmd/workspace/up/agent.go index 993ef0de83..7ba7093660 100644 --- a/cmd/workspace/up/agent.go +++ b/cmd/workspace/up/agent.go @@ -16,6 +16,7 @@ import ( "github.com/devsy-org/devsy/pkg/devcontainer/sshtunnel" "github.com/devsy-org/devsy/pkg/log" provider2 "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/secrets" ) func (cmd *UpCmd) devsyUp( @@ -280,6 +281,7 @@ func (cmd *UpCmd) devsyUpMachineSSH( AgentInject: newAgentInjectFunc(client, wInfo), SSHCommand: sshTunnelCmd, Command: agentCommand, + Redactor: workspaceSecretRedactor(cmd), TunnelServerFunc: func(ctx context.Context, stdin io.WriteCloser, stdout io.Reader) (*config2.Result, error) { return tunnelserver.RunUpServer(ctx, tunnelserver.UpSpec{ Reader: stdout, @@ -293,6 +295,19 @@ func (cmd *UpCmd) devsyUpMachineSSH( }) } +func workspaceSecretRedactor(cmd *UpCmd) *secrets.Redactor { + var token []string + if cmd.GitToken != nil { + token = append(token, "TOKEN="+cmd.GitToken.Token) + } + return secrets.Combine( + secrets.NewRedactor(cmd.SecretsEnv), + secrets.NewRedactor(cmd.SecretsMount), + secrets.NewRedactor(cmd.BuildSecrets), + secrets.NewRedactor(token), + ) +} + func workspaceAgentCommand(agentPath, workspaceInfo string, debug, lifecycleTerminal bool) string { command := fmt.Sprintf( "%s internal agent workspace up --workspace-info %s", diff --git a/cmd/workspace/up/agent_secret_redaction_test.go b/cmd/workspace/up/agent_secret_redaction_test.go new file mode 100644 index 0000000000..aa250bf27a --- /dev/null +++ b/cmd/workspace/up/agent_secret_redaction_test.go @@ -0,0 +1,27 @@ +package up + +import ( + "testing" + + "github.com/devsy-org/devsy/pkg/provider" + "github.com/stretchr/testify/suite" +) + +type agentSecretRedactionSuite struct{ suite.Suite } + +func TestAgentSecretRedactionSuite(t *testing.T) { suite.Run(t, new(agentSecretRedactionSuite)) } +func (s *agentSecretRedactionSuite) TestResolvedSecretsAreRedacted() { + cmd := &UpCmd{CLIOptions: provider.CLIOptions{ + SecretsEnv: []string{"ENV=private-env-value"}, + SecretsMount: []string{"MOUNT=private-mount-value"}, + BuildSecrets: []string{"BUILD=private-build-value"}, + GitToken: &provider.GitToken{Token: "private-git-value"}, + }} + redactor := workspaceSecretRedactor(cmd) + values := []string{ + "private-env-value", "private-mount-value", "private-build-value", "private-git-value", + } + for _, value := range values { + s.Run(value, func() { s.NotContains(redactor.Redact("agent: "+value), value) }) + } +} diff --git a/pkg/devcontainer/setup.go b/pkg/devcontainer/setup.go index ec1e11336c..17a1d7a208 100644 --- a/pkg/devcontainer/setup.go +++ b/pkg/devcontainer/setup.go @@ -28,6 +28,7 @@ import ( "github.com/devsy-org/devsy/pkg/ide" "github.com/devsy-org/devsy/pkg/log" provider2 "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/secrets" "github.com/devsy-org/devsy/pkg/status" "github.com/devsy-org/devsy/pkg/types" "github.com/devsy-org/devsy/pkg/version" @@ -627,15 +628,33 @@ func (r *runner) executeSetup( } return sshtunnel.ExecuteCommand(ctx, sshtunnel.ExecuteCommandOptions{ - Client: nil, - AddPrivateKeys: false, - AgentInject: agentInjectFunc, - SSHCommand: sshTunnelCmd, - Command: setupCommand, + Client: nil, + AddPrivateKeys: false, + AgentInject: agentInjectFunc, + SSHCommand: sshTunnelCmd, + Command: setupCommand, + Redactor: setupSecretRedactor( + secretsEnv, + secretsMount, + r.workspaceConfig.CLIOptions, + ), TunnelServerFunc: runSetupServer, }) } +func setupSecretRedactor(env, mount []string, options provider2.CLIOptions) *secrets.Redactor { + var token []string + if options.GitToken != nil { + token = append(token, "TOKEN="+options.GitToken.Token) + } + return secrets.Combine( + secrets.NewRedactor(env), + secrets.NewRedactor(mount), + secrets.NewRedactor(options.BuildSecrets), + secrets.NewRedactor(token), + ) +} + func (r *runner) execSetupSSHServer( ctx context.Context, req agent.ExecRequest, diff --git a/pkg/devcontainer/setup_secret_redaction_test.go b/pkg/devcontainer/setup_secret_redaction_test.go new file mode 100644 index 0000000000..eb799a4b48 --- /dev/null +++ b/pkg/devcontainer/setup_secret_redaction_test.go @@ -0,0 +1,29 @@ +package devcontainer + +import ( + "testing" + + "github.com/devsy-org/devsy/pkg/provider" + "github.com/stretchr/testify/suite" +) + +type setupSecretRedactionSuite struct{ suite.Suite } + +func TestSetupSecretRedactionSuite(t *testing.T) { suite.Run(t, new(setupSecretRedactionSuite)) } +func (s *setupSecretRedactionSuite) TestResolvedSecretsAreRedacted() { + options := provider.CLIOptions{ + BuildSecrets: []string{"BUILD=private-build-value"}, + GitToken: &provider.GitToken{Token: "private-git-value"}, + } + redactor := setupSecretRedactor( + []string{"ENV=private-env-value"}, + []string{"MOUNT=private-mount-value"}, + options, + ) + values := []string{ + "private-env-value", "private-mount-value", "private-build-value", "private-git-value", + } + for _, value := range values { + s.Run(value, func() { s.NotContains(redactor.Redact("agent: "+value), value) }) + } +} diff --git a/pkg/devcontainer/sshtunnel/command_error_test.go b/pkg/devcontainer/sshtunnel/command_error_test.go index 100b1371d2..0b3106d76e 100644 --- a/pkg/devcontainer/sshtunnel/command_error_test.go +++ b/pkg/devcontainer/sshtunnel/command_error_test.go @@ -3,9 +3,13 @@ package sshtunnel import ( "context" "errors" + "io" "testing" + "github.com/devsy-org/devsy/pkg/log" + "github.com/devsy-org/devsy/pkg/secrets" "github.com/stretchr/testify/suite" + "go.uber.org/zap/zapcore" ) type commandErrorSuite struct{ suite.Suite } @@ -40,3 +44,37 @@ func (s *commandErrorSuite) TestCommandFailurePreservesCancellation() { s.ErrorIs(err, context.Canceled) s.ErrorContains(err, "agent diagnostic") } + +func (s *commandErrorSuite) TestStoredSecretIsRedactedFromAgentLogs() { + const secret = "stored-secret-not-in-environment-5791" + logs := log.InitTestObserved(s.T(), zapcore.DebugLevel) + opts := ExecuteCommandOptions{ + Redactor: secrets.NewRedactor([]string{"VALUE=" + secret}), + AgentInject: func(_ context.Context, _ string, _ io.Reader, _ io.Writer, stderr io.WriteCloser) error { + _, err := io.WriteString(stderr, "agent: "+secret+"\n") + return err + }, + } + s.Require().NoError(executeSSHServerHelper(context.Background(), opts, nil, io.Discard)) + for _, entry := range logs.All() { + s.NotContains(entry.Message, secret) + } +} + +func (s *commandErrorSuite) TestStoredSecretIsRedactedFromCapturedDiagnostics() { + for _, tc := range []struct{ name, value, output string }{ + {name: "plain", value: "stored-private-value-8231", output: "stored-private-value-8231"}, + {name: "json escaped", value: "stored-quote-\"-value-8231", output: `stored-quote-\"-value-8231`}, + {name: "multiline", value: "stored-first-8231\nstored-second-8231", output: "stored-first-8231\nstored-second-8231"}, + } { + s.Run(tc.name, func() { + streamer := newSSHTunnelJSONLogStreamer( + secrets.NewRedactor([]string{"VALUE=" + tc.value}), + ) + _, err := io.WriteString(streamer, tc.output+"\n") + s.Require().NoError(err) + s.Require().NoError(streamer.Close()) + s.NotContains(streamer.ErrorOutput(), "stored-") + }) + } +} diff --git a/pkg/devcontainer/sshtunnel/sshtunnel.go b/pkg/devcontainer/sshtunnel/sshtunnel.go index 9ee0fdc96d..14bca2f1c1 100644 --- a/pkg/devcontainer/sshtunnel/sshtunnel.go +++ b/pkg/devcontainer/sshtunnel/sshtunnel.go @@ -12,6 +12,7 @@ import ( client2 "github.com/devsy-org/devsy/pkg/client" config2 "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/log" + "github.com/devsy-org/devsy/pkg/secrets" devssh "github.com/devsy-org/devsy/pkg/ssh" devsshagent "github.com/devsy-org/devsy/pkg/ssh/agent" "github.com/devsy-org/devsy/pkg/transport" @@ -32,6 +33,7 @@ type ExecuteCommandOptions struct { SSHCommand string Command string TunnelServerFunc TunnelServerFunc + Redactor *secrets.Redactor } // ExecuteCommand runs the command in an SSH Tunnel and returns the result. @@ -97,7 +99,7 @@ func executeSSHServerHelper( // AgentInject's stderr carries the ssh-server command's structured // JSON logs, and for some callers (see newAgentInjectFunc) also the // injection script's plain-text preamble. - streamer := newSSHTunnelJSONLogStreamer() + streamer := newSSHTunnelJSONLogStreamer(opts.Redactor) defer func() { _ = streamer.Close() }() log.Debugf("injecting and running SSH server command: %q", opts.SSHCommand) @@ -155,8 +157,9 @@ func runSSHTunnel(ctx context.Context, p sshTunnelParams) (*config2.Result, erro err = p.grpcBridge.RunPair(ctx, func(ctx context.Context, stdin, stdout *os.File) error { return runCommandInSSHTunnel(ctx, sshCommandParams{ - session: sess, - command: p.opts.Command, + session: sess, + command: p.opts.Command, + redactor: p.opts.Redactor, }, stdin, stdout) }, func(ctx context.Context, stdout, stdin *os.File) error { @@ -233,12 +236,13 @@ func setupSSHAgentForwarding(sshClient *ssh.Client, sess *ssh.Session) { } type sshCommandParams struct { - session *ssh.Session - command string + session *ssh.Session + command string + redactor *secrets.Redactor } func runCommandInSSHTunnel(ctx context.Context, p sshCommandParams, stdin, stdout *os.File) error { - streamer := newSSHTunnelJSONLogStreamer() + streamer := newSSHTunnelJSONLogStreamer(p.redactor) defer func() { _ = streamer.Close() }() log.Debugf("running agent command in SSH tunnel: %q", p.command) @@ -280,8 +284,9 @@ const ( maxLogBytes = 256 * 1024 ) -func newSSHTunnelJSONLogStreamer() *log.JSONLogStreamer { +func newSSHTunnelJSONLogStreamer(redactor *secrets.Redactor) *log.JSONLogStreamer { return log.NewJSONLogStreamer(log.StreamerOptions{ + Redactor: redactor, FallbackLevel: log.LevelDebug, CaptureLines: maxLogLines, CaptureBytes: maxLogBytes, diff --git a/pkg/devcontainer/sshtunnel/sshtunnel_test.go b/pkg/devcontainer/sshtunnel/sshtunnel_test.go index b3c437967e..446b71128e 100644 --- a/pkg/devcontainer/sshtunnel/sshtunnel_test.go +++ b/pkg/devcontainer/sshtunnel/sshtunnel_test.go @@ -72,7 +72,7 @@ func TestLogLine_JSONPassthrough(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { logs := log.InitTestObserved(t, zapcore.DebugLevel) - streamer := newSSHTunnelJSONLogStreamer() + streamer := newSSHTunnelJSONLogStreamer(nil) defer func() { _ = streamer.Close() }() streamer.LogLine(tt.input) @@ -114,7 +114,7 @@ func TestLogLine_JSONLevelNormalization(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { logs := log.InitTestObserved(t, zapcore.DebugLevel) - streamer := newSSHTunnelJSONLogStreamer() + streamer := newSSHTunnelJSONLogStreamer(nil) defer func() { _ = streamer.Close() }() streamer.LogLine(tt.input) @@ -150,7 +150,7 @@ func TestLogLine_PlainText(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { logs := log.InitTestObserved(t, zapcore.DebugLevel) - streamer := newSSHTunnelJSONLogStreamer() + streamer := newSSHTunnelJSONLogStreamer(nil) defer func() { _ = streamer.Close() }() streamer.LogLine(tt.input) @@ -164,7 +164,7 @@ func TestLogLine_PlainText(t *testing.T) { func TestLogLine_EmptyAndWhitespace(t *testing.T) { logs := log.InitTestObserved(t, zapcore.DebugLevel) - streamer := newSSHTunnelJSONLogStreamer() + streamer := newSSHTunnelJSONLogStreamer(nil) defer func() { _ = streamer.Close() }() streamer.LogLine("") @@ -176,7 +176,7 @@ func TestLogLine_EmptyAndWhitespace(t *testing.T) { func TestLogLine_JSONWithoutMessage(t *testing.T) { logs := log.InitTestObserved(t, zapcore.DebugLevel) - streamer := newSSHTunnelJSONLogStreamer() + streamer := newSSHTunnelJSONLogStreamer(nil) defer func() { _ = streamer.Close() }() streamer.LogLine(`{"level":"info","key":"value"}`) diff --git a/pkg/provider/download.go b/pkg/provider/download.go index 9118c56d53..559ff0ec26 100644 --- a/pkg/provider/download.go +++ b/pkg/provider/download.go @@ -132,29 +132,44 @@ func downloadBinaryForPlatform( continue } - // check if binary is correct - binaryTargetFolder := filepath.Join(targetFolder, strings.ToLower(binaryName)) - binaryPath := getBinaryPath(binary, binaryTargetFolder) - if isUnownedLocalBinary(binary.Path, binaryPath) { - return prepareLocalSource(binary, binaryPath, binaryTargetFolder) - } - if verifyOrRemoveBinary(binaryPath, binary.Checksum) || - fromCache(binary, binaryTargetFolder) { - return binaryPath, nil - } - - // try to download the binary - binaryPath, err := downloadWithRetry(ctx, binaryName, binary, binaryTargetFolder) - if err != nil { - return "", err - } - return binaryPath, nil + return preparePlatformBinary(ctx, binaryName, binary, targetFolder) } return "", fmt.Errorf( "cannot download provider binary %s, because no binary location matched OS %s and ARCH %s", binaryName, runtime.GOOS, runtime.GOARCH) } +func preparePlatformBinary( + ctx context.Context, + binaryName string, + binary *ProviderBinary, + targetFolder string, +) (string, error) { + binaryTargetFolder := filepath.Join(targetFolder, strings.ToLower(binaryName)) + binaryPath := getBinaryPath(binary, binaryTargetFolder) + if err := verifyAvailableLocalSource(binary); err != nil { + return "", err + } + if isUnownedLocalBinary(binary.Path, binaryPath) { + return prepareLocalSource(binary, binaryPath, binaryTargetFolder) + } + if verifyOrRemoveBinary(binaryPath, binary.Checksum) || fromCache(binary, binaryTargetFolder) { + return binaryPath, nil + } + return downloadWithRetry(ctx, binaryName, binary, binaryTargetFolder) +} + +func verifyAvailableLocalSource(binary *ProviderBinary) error { + if isRemotePath(binary.Path) || filepath.IsAbs(binary.Path) { + return nil + } + _, err := verifyLocalBinary(binary.Path, binary.Checksum) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err +} + func isUnownedLocalBinary(source, target string) bool { return filepath.IsAbs(source) || sameLocalFile(source, target) } @@ -203,6 +218,11 @@ func sameLocalFile(source, target string) bool { if isRemotePath(source) { return false } + sourcePath, sourceErr := filepath.Abs(source) + targetPath, targetErr := filepath.Abs(target) + if sourceErr == nil && targetErr == nil && sourcePath == targetPath { + return true + } sourceInfo, err := os.Stat(source) if err != nil { return false diff --git a/pkg/provider/local_source_safety_test.go b/pkg/provider/local_source_safety_test.go index 3e10c3aa12..29798e8e77 100644 --- a/pkg/provider/local_source_safety_test.go +++ b/pkg/provider/local_source_safety_test.go @@ -274,3 +274,49 @@ func (s *localSourceSafetySuite) TestRelativeYAMLStatError() { s.Require().Error(err) s.ErrorContains(err, "stat provider file") } + +func (s *localSourceSafetySuite) TestRelativeSourceMismatchBeforeStoredCopy() { + for _, existing := range []bool{false, true} { + s.Run(fmt.Sprintf("stored=%t", existing), func() { + dir := s.T().TempDir() + source := filepath.Join(dir, "source") + actualBytes := []byte("changed source") + expectedBytes := []byte("stored binary") + s.Require().NoError(os.WriteFile(source, actualBytes, 0o600)) + cwd, err := os.Getwd() + s.Require().NoError(err) + relative, err := filepath.Rel(cwd, source) + s.Require().NoError(err) + storage := filepath.Join(dir, "storage") + stored := filepath.Join(storage, "test", "binary") + if existing { + s.Require().NoError(os.MkdirAll(filepath.Dir(stored), 0o750)) + s.Require().NoError(os.WriteFile(stored, expectedBytes, 0o600)) + } + expected := fmt.Sprintf("%x", sha256.Sum256(expectedBytes)) + actual := fmt.Sprintf("%x", sha256.Sum256(actualBytes)) + binary := &ProviderBinary{ + OS: runtime.GOOS, + Arch: runtime.GOARCH, + Path: relative, + Name: "binary", + Checksum: expected, + } + _, err = downloadBinaryForPlatform( + context.Background(), + "TEST", + []*ProviderBinary{binary}, + storage, + ) + s.Require().Error(err) + s.ErrorContains(err, relative) + s.ErrorContains(err, expected) + s.ErrorContains(err, actual) + s.ErrorContains(err, "regenerate") + s.FileExists(source) + if existing { + s.FileExists(stored) + } + }) + } +} diff --git a/pkg/secrets/redact.go b/pkg/secrets/redact.go index e50880f246..d0db015343 100644 --- a/pkg/secrets/redact.go +++ b/pkg/secrets/redact.go @@ -1,6 +1,7 @@ package secrets import ( + "encoding/json" "fmt" "regexp" "sort" @@ -27,12 +28,18 @@ type Redactor struct { // NewRedactor masks the values (not keys) of KEY=VALUE entries; empty values are ignored. func NewRedactor(secretsEnv []string) *Redactor { values := make([]string, 0, len(secretsEnv)) + seen := map[string]struct{}{} for _, entry := range secretsEnv { _, value, ok := strings.Cut(entry, "=") if !ok || value == "" { continue } - values = append(values, value) + for _, candidate := range redactionValues(value) { + if _, ok := seen[candidate]; !ok { + seen[candidate] = struct{}{} + values = append(values, candidate) + } + } } if len(values) == 0 { return &Redactor{} @@ -54,6 +61,19 @@ func NewRedactor(secretsEnv []string) *Redactor { } } +func redactionValues(value string) []string { + values := []string{value} + fragments := strings.FieldsFunc(value, func(r rune) bool { return r == '\n' || r == '\r' }) + values = append(values, fragments...) + for _, candidate := range append([]string(nil), values...) { + encoded, err := json.Marshal(candidate) + if err == nil && len(encoded) > 2 { + values = append(values, string(encoded[1:len(encoded)-1])) + } + } + return values +} + // Combine returns a redactor that masks the values known by every input. // Nil redactors are ignored. func Combine(redactors ...*Redactor) *Redactor { diff --git a/pkg/tunnel/pipebridge.go b/pkg/tunnel/pipebridge.go index e7a5b9ab93..e11ed5e582 100644 --- a/pkg/tunnel/pipebridge.go +++ b/pkg/tunnel/pipebridge.go @@ -3,6 +3,8 @@ package tunnel import ( "context" "errors" + "fmt" + "net" "os" "sync" "sync/atomic" @@ -149,13 +151,21 @@ func isIndependentTunnelError(ctx context.Context, err error) bool { func classifyBridgeErrors(ctx context.Context, tunnelErr, handlerErr error) error { if ctx.Err() == nil && - (errors.Is(handlerErr, context.Canceled) || errors.Is(handlerErr, os.ErrClosed)) && + isHandlerShutdownError(handlerErr) && tunnelErr != nil && !errors.Is(tunnelErr, context.Canceled) { - if errors.Is(handlerErr, os.ErrClosed) { + if errors.Is(handlerErr, os.ErrClosed) || errors.Is(handlerErr, net.ErrClosed) { return errors.Join(tunnelErr, handlerErr) } + if handlerErr != context.Canceled { + return fmt.Errorf("%w; handler: %v", tunnelErr, handlerErr) + } return tunnelErr } return ClassifyTunnelErrors(tunnelErr, handlerErr) } + +func isHandlerShutdownError(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, os.ErrClosed) || + errors.Is(err, net.ErrClosed) +} diff --git a/pkg/tunnel/pipebridge_error_test.go b/pkg/tunnel/pipebridge_error_test.go index b02105eaca..cff13c9f18 100644 --- a/pkg/tunnel/pipebridge_error_test.go +++ b/pkg/tunnel/pipebridge_error_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "net" "os" "testing" @@ -99,3 +100,22 @@ func (s *pipeBridgeErrorSuite) TestIndependentHandlerCancellationIsPreserved() { s.ErrorIs(err, context.Canceled) s.ErrorIs(err, commandErr) } + +func (s *pipeBridgeErrorSuite) TestNetworkCloseDoesNotMaskCommandFailure() { + commandErr := errors.New("command failed") + err := classifyBridgeErrors(context.Background(), commandErr, net.ErrClosed) + s.ErrorIs(err, commandErr) + s.ErrorIs(err, net.ErrClosed) +} + +func (s *pipeBridgeErrorSuite) TestInternalCancellationKeepsCleanupDiagnostic() { + commandErr := errors.New("command failed") + err := classifyBridgeErrors( + context.Background(), + commandErr, + fmt.Errorf("cleanup detail: %w", context.Canceled), + ) + s.ErrorIs(err, commandErr) + s.NotErrorIs(err, context.Canceled) + s.ErrorContains(err, "cleanup detail") +} From e0f5a2bb64d2a58be5cbaed175bc8fc42f8d570d Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 22:13:11 -0600 Subject: [PATCH 09/11] fix(secrets): preserve structured logs during redaction --- pkg/secrets/redact.go | 37 ++++++++++++++++++++++++++-- pkg/secrets/redact_multiline_test.go | 27 ++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 pkg/secrets/redact_multiline_test.go diff --git a/pkg/secrets/redact.go b/pkg/secrets/redact.go index d0db015343..307d2e6e1f 100644 --- a/pkg/secrets/redact.go +++ b/pkg/secrets/redact.go @@ -7,6 +7,7 @@ import ( "sort" "strings" "sync" + "unicode" ) const redactMask = "***" @@ -63,8 +64,16 @@ func NewRedactor(secretsEnv []string) *Redactor { func redactionValues(value string) []string { values := []string{value} - fragments := strings.FieldsFunc(value, func(r rune) bool { return r == '\n' || r == '\r' }) - values = append(values, fragments...) + var decoded any + if strings.ContainsAny(value, "\r\n") && json.Unmarshal([]byte(value), &decoded) == nil { + values = append(values, jsonStringValues(decoded)...) + } else { + for _, fragment := range strings.FieldsFunc(value, func(r rune) bool { return r == '\n' || r == '\r' }) { + if hasRedactableContent(fragment) { + values = append(values, fragment) + } + } + } for _, candidate := range append([]string(nil), values...) { encoded, err := json.Marshal(candidate) if err == nil && len(encoded) > 2 { @@ -74,6 +83,30 @@ func redactionValues(value string) []string { return values } +func jsonStringValues(value any) []string { + var values []string + switch value := value.(type) { + case string: + if hasRedactableContent(value) { + values = append(values, value) + } + case []any: + for _, entry := range value { + values = append(values, jsonStringValues(entry)...) + } + case map[string]any: + for _, entry := range value { + values = append(values, jsonStringValues(entry)...) + } + } + return values +} + +func hasRedactableContent(value string) bool { + return strings.IndexFunc(value, unicode.IsLetter) >= 0 || + strings.IndexFunc(value, unicode.IsNumber) >= 0 +} + // Combine returns a redactor that masks the values known by every input. // Nil redactors are ignored. func Combine(redactors ...*Redactor) *Redactor { diff --git a/pkg/secrets/redact_multiline_test.go b/pkg/secrets/redact_multiline_test.go new file mode 100644 index 0000000000..b033531cd2 --- /dev/null +++ b/pkg/secrets/redact_multiline_test.go @@ -0,0 +1,27 @@ +package secrets_test + +import ( + "encoding/json" + "testing" + + "github.com/devsy-org/devsy/pkg/secrets" + "github.com/stretchr/testify/suite" +) + +type multilineRedactionSuite struct{ suite.Suite } + +func TestMultilineRedactionSuite(t *testing.T) { suite.Run(t, new(multilineRedactionSuite)) } +func (s *multilineRedactionSuite) TestMultilineSecretsKeepUnrelatedJSONValid() { + values := []string{ + "{\n\"password\": \"private-value-123\",\n\"punctuation\": \"{\"\n}", + "{\nprivate-value-123\n}", + } + for _, value := range values { + s.Run(value, func() { + redactor := secrets.NewRedactor([]string{"TOKEN=" + value}) + output := redactor.Redact(`{"level":"info","message":"ordinary event"}`) + s.True(json.Valid([]byte(output))) + s.NotContains(redactor.Redact("agent: private-value-123"), "private-value-123") + }) + } +} From 815956e17ffaed5f886d86ff81ea32acc3436a97 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 22:43:16 -0600 Subject: [PATCH 10/11] fix(secrets): protect numeric values in multiline credentials --- .../sshtunnel/command_error_test.go | 21 ++ pkg/secrets/redact.go | 309 ++++++++++++++++-- pkg/secrets/redact_multiline_test.go | 100 ++++++ 3 files changed, 403 insertions(+), 27 deletions(-) diff --git a/pkg/devcontainer/sshtunnel/command_error_test.go b/pkg/devcontainer/sshtunnel/command_error_test.go index 0b3106d76e..4a83c26ecf 100644 --- a/pkg/devcontainer/sshtunnel/command_error_test.go +++ b/pkg/devcontainer/sshtunnel/command_error_test.go @@ -2,6 +2,7 @@ package sshtunnel import ( "context" + "encoding/json" "errors" "io" "testing" @@ -78,3 +79,23 @@ func (s *commandErrorSuite) TestStoredSecretIsRedactedFromCapturedDiagnostics() }) } } + +func (s *commandErrorSuite) TestNumericJSONSecretIsRedactedFromCapturedDiagnostics() { + fixture := "{\n\"pin\": 123456\n}" + streamer := newSSHTunnelJSONLogStreamer(secrets.NewRedactor([]string{"VALUE=" + fixture})) + _, err := io.WriteString(streamer, "\"pin\": 123456\n") + s.Require().NoError(err) + s.Require().NoError(streamer.Close()) + s.NotContains(streamer.ErrorOutput(), "123456") +} + +func (s *commandErrorSuite) TestNumericJSONSecretKeepsStructuredDiagnosticValid() { + fixture := "{\n\"pin\": 123456\n}" + streamer := newSSHTunnelJSONLogStreamer(secrets.NewRedactor([]string{"VALUE=" + fixture})) + _, err := io.WriteString(streamer, `{"level":"info","message":"ready","pin":123456}`+"\n") + s.Require().NoError(err) + s.Require().NoError(streamer.Close()) + output := streamer.ErrorOutput() + s.True(json.Valid([]byte(output))) + s.NotContains(output, "123456") +} diff --git a/pkg/secrets/redact.go b/pkg/secrets/redact.go index 307d2e6e1f..fac92c5e38 100644 --- a/pkg/secrets/redact.go +++ b/pkg/secrets/redact.go @@ -2,26 +2,37 @@ package secrets import ( "encoding/json" - "fmt" + "maps" "regexp" + "slices" "sort" + "strconv" "strings" "sync" "unicode" ) -const redactMask = "***" +const ( + redactMask = "***" + jsonRedactMask = `"***"` +) var ( credentialURLPattern = regexp.MustCompile(`(?i)(https?://)[^\s/@]+@`) authorizationPattern = regexp.MustCompile( `(?i)(authorization\s*[:=]\s*(?:bearer|basic)\s+)[^\s,]+`, ) + jsonScalarLinePattern = regexp.MustCompile( + `("(?:[^"\\]|\\.)+"\s*:\s*)(-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?|true|false|null)`, + ) ) type Redactor struct { replacer *strings.Replacer values []string + entries []string + lines map[string]string + scalars *strings.Replacer maxLength int matcher *secretNode } @@ -29,18 +40,20 @@ type Redactor struct { // NewRedactor masks the values (not keys) of KEY=VALUE entries; empty values are ignored. func NewRedactor(secretsEnv []string) *Redactor { values := make([]string, 0, len(secretsEnv)) - seen := map[string]struct{}{} + replacements := map[string]string{} + lines := map[string]string{} + scalarValues := map[string]struct{}{} + var entries []string for _, entry := range secretsEnv { _, value, ok := strings.Cut(entry, "=") if !ok || value == "" { continue } - for _, candidate := range redactionValues(value) { - if _, ok := seen[candidate]; !ok { - seen[candidate] = struct{}{} - values = append(values, candidate) - } - } + entries = append(entries, entry) + values = appendPlainCandidates(values, replacements, value) + fragments, lineValues := jsonScalarFragments(value, scalarValues) + maps.Copy(lines, lineValues) + values = appendMappedCandidates(values, replacements, fragments) } if len(values) == 0 { return &Redactor{} @@ -51,28 +64,168 @@ func NewRedactor(secretsEnv []string) *Redactor { sort.SliceStable(values, func(i, j int) bool { return len(values[i]) > len(values[j]) }) pairs := make([]string, 0, len(values)*2) for _, v := range values { - pairs = append(pairs, v, redactMask) + pairs = append(pairs, v, replacements[v]) + } + var scalarReplacer *strings.Replacer + if len(scalarValues) > 0 { + var scalarKeys []string + for scalar := range scalarValues { + scalarKeys = append(scalarKeys, scalar) + } + sort.Slice( + scalarKeys, + func(i, j int) bool { return len(scalarKeys[i]) > len(scalarKeys[j]) }, + ) + var scalarPairs []string + for _, scalar := range scalarKeys { + scalarPairs = append(scalarPairs, scalar, redactMask) + } + scalarReplacer = strings.NewReplacer(scalarPairs...) } return &Redactor{ replacer: strings.NewReplacer(pairs...), values: values, + entries: entries, + lines: lines, + scalars: scalarReplacer, maxLength: len(values[0]), matcher: newSecretMatcher(values), } } +func appendPlainCandidates(values []string, replacements map[string]string, value string) []string { + for _, candidate := range redactionValues(value) { + if _, ok := replacements[candidate]; !ok { + values = append(values, candidate) + } + replacements[candidate] = redactMask + } + return values +} + +func appendMappedCandidates(values []string, replacements, fragments map[string]string) []string { + for candidate, replacement := range fragments { + if _, ok := replacements[candidate]; !ok { + values = append(values, candidate) + replacements[candidate] = replacement + } + } + return values +} + +func jsonScalarFragments( + value string, + scalarValues map[string]struct{}, +) (map[string]string, map[string]string) { + fragments := map[string]string{} + lines := map[string]string{} + decoded, ok := decodeMultilineJSON(value) + if !ok { + return fragments, lines + } + addJSONScalarValues(lines, fragments, scalarValues, decoded) + if compact, err := json.Marshal(decoded); err == nil { + switch decoded.(type) { + case []any, map[string]any: + fragments[string(compact)] = jsonRedactMask + } + } + addJSONScalarLineFragments(fragments, lines, value) + return fragments, lines +} + +func decodeMultilineJSON(value string) (any, bool) { + if !strings.ContainsAny(value, "\r\n") || !json.Valid([]byte(value)) { + return nil, false + } + decoder := json.NewDecoder(strings.NewReader(value)) + decoder.UseNumber() + var decoded any + if err := decoder.Decode(&decoded); err != nil { + return nil, false + } + return decoded, true +} + +func addJSONScalarLineFragments(fragments, lines map[string]string, value string) { + for _, match := range jsonScalarLinePattern.FindAllStringSubmatch(value, -1) { + key := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(match[1]), ":")) + for _, prefix := range []string{match[1], key + ":"} { + candidate := prefix + match[2] + replacement := prefix + jsonRedactMask + lines[candidate] = replacement + addBoundedJSONFragments(fragments, candidate, replacement) + } + } +} + +func addBoundedJSONFragments(fragments map[string]string, candidate, replacement string) { + for _, suffix := range []string{",", "}", "]"} { + for _, space := range []string{"", " ", "\t"} { + bounded := candidate + space + suffix + fragments[bounded] = replacement + space + suffix + encoded, err := json.Marshal(bounded) + if err == nil && len(encoded) > 2 { + fragments[string(encoded[1:len(encoded)-1])] = redactMask + space + suffix + } + } + } +} + +func addJSONScalarValues( + lines, fragments map[string]string, + scalarValues map[string]struct{}, + value any, +) { + switch value := value.(type) { + case json.Number: + lines[value.String()] = jsonRedactMask + scalarValues[value.String()] = struct{}{} + case bool: + lines[strconv.FormatBool(value)] = jsonRedactMask + scalarValues[strconv.FormatBool(value)] = struct{}{} + case nil: + lines["null"] = jsonRedactMask + scalarValues["null"] = struct{}{} + case string: + addJSONScalarString(lines, fragments, scalarValues, value) + case []any: + addJSONScalarEntries(lines, fragments, scalarValues, value) + case map[string]any: + addJSONScalarEntries(lines, fragments, scalarValues, slices.Collect(maps.Values(value))) + } +} + +func addJSONScalarEntries( + lines, fragments map[string]string, + scalarValues map[string]struct{}, + entries []any, +) { + for _, entry := range entries { + addJSONScalarValues(lines, fragments, scalarValues, entry) + } +} + +func addJSONScalarString( + lines, fragments map[string]string, + scalarValues map[string]struct{}, + value string, +) { + if isJSONScalarToken(value) { + scalarValues[value] = struct{}{} + lines[value] = jsonRedactMask + encoded, _ := json.Marshal(value) + fragments[string(encoded)] = jsonRedactMask + } +} + func redactionValues(value string) []string { values := []string{value} - var decoded any - if strings.ContainsAny(value, "\r\n") && json.Unmarshal([]byte(value), &decoded) == nil { + if decoded, ok := decodeMultilineJSON(value); ok { values = append(values, jsonStringValues(decoded)...) } else { - for _, fragment := range strings.FieldsFunc(value, func(r rune) bool { return r == '\n' || r == '\r' }) { - if hasRedactableContent(fragment) { - values = append(values, fragment) - } - } + values = append(values, textSecretFragments(value)...) } for _, candidate := range append([]string(nil), values...) { encoded, err := json.Marshal(candidate) @@ -83,11 +236,21 @@ func redactionValues(value string) []string { return values } +func textSecretFragments(value string) []string { + var values []string + for _, fragment := range strings.FieldsFunc(value, func(r rune) bool { return r == '\n' || r == '\r' }) { + if hasRedactableContent(fragment) { + values = append(values, fragment) + } + } + return values +} + func jsonStringValues(value any) []string { var values []string switch value := value.(type) { case string: - if hasRedactableContent(value) { + if hasRedactableContent(value) && !isJSONScalarToken(value) { values = append(values, value) } case []any: @@ -102,6 +265,24 @@ func jsonStringValues(value any) []string { return values } +func isJSONScalarToken(value string) bool { + if !json.Valid([]byte(value)) { + return false + } + decoder := json.NewDecoder(strings.NewReader(value)) + decoder.UseNumber() + var decoded any + if err := decoder.Decode(&decoded); err != nil { + return false + } + switch decoded.(type) { + case json.Number, bool, nil: + return true + default: + return false + } +} + func hasRedactableContent(value string) bool { return strings.IndexFunc(value, unicode.IsLetter) >= 0 || strings.IndexFunc(value, unicode.IsNumber) >= 0 @@ -115,9 +296,7 @@ func Combine(redactors ...*Redactor) *Redactor { if redactor == nil { continue } - for i, value := range redactor.values { - entries = append(entries, fmt.Sprintf("DEVSY_COMBINED_%d=%s", i, value)) - } + entries = append(entries, redactor.entries...) } return NewRedactor(entries) } @@ -150,15 +329,91 @@ func isSensitiveEnvironmentKey(key string) bool { } func (r *Redactor) Redact(s string) string { - if r != nil && r.replacer != nil { - s = r.replacer.Replace(s) + return r.redact(s, true) +} + +func (r *Redactor) redact(s string, standalone bool) string { + if r != nil { + s = r.redactKnown(s, standalone) } - // Format-based masking protects credentials supplied directly in a URL or - // header and therefore not available as environment values. s = credentialURLPattern.ReplaceAllString(s, "$1***@") return authorizationPattern.ReplaceAllString(s, "$1***") } +func (r *Redactor) redactKnown(s string, standalone bool) string { + if standalone { + trimmed := strings.TrimSpace(s) + scalar := strings.TrimSpace(strings.TrimSuffix(trimmed, ",")) + if replacement, ok := r.lines[scalar]; ok { + s = strings.Replace(s, scalar, replacement, 1) + } + } + if r.replacer != nil { + s = r.replacer.Replace(s) + } + if standalone && r.scalars != nil { + s = redactScalarContent(s, r.scalars) + } + return s +} + +func redactScalarContent(text string, replacer *strings.Replacer) string { + if !json.Valid([]byte(text)) { + return replacer.Replace(text) + } + decoder := json.NewDecoder(strings.NewReader(text)) + decoder.UseNumber() + var decoded any + if err := decoder.Decode(&decoded); err != nil { + return jsonRedactMask + } + redacted, changed := redactJSONStringValues(decoded, replacer) + if !changed { + return text + } + encoded, err := json.Marshal(redacted) + if err != nil { + return jsonRedactMask + } + return string(encoded) +} + +func redactJSONStringValues(value any, replacer *strings.Replacer) (any, bool) { + switch value := value.(type) { + case string: + redacted := replacer.Replace(value) + return redacted, redacted != value + case []any: + return redactJSONArray(value, replacer) + case map[string]any: + return redactJSONObject(value, replacer) + default: + return value, false + } +} + +func redactJSONArray(value []any, replacer *strings.Replacer) (any, bool) { + changed := false + for i, entry := range value { + var entryChanged bool + value[i], entryChanged = redactJSONStringValues(entry, replacer) + changed = changed || entryChanged + } + return value, changed +} + +func redactJSONObject(value map[string]any, replacer *strings.Replacer) (any, bool) { + redacted := make(map[string]any, len(value)) + changed := false + for key, entry := range value { + redactedKey := replacer.Replace(key) + redactedValue, entryChanged := redactJSONStringValues(entry, replacer) + redacted[redactedKey] = redactedValue + changed = changed || entryChanged || redactedKey != key + } + return redacted, changed +} + // StreamingRedactor masks secrets split across text writes, buffering a short // suffix until the next write or Flush. type StreamingRedactor struct { @@ -201,7 +456,7 @@ func (r *StreamingRedactor) RedactChunk(chunk string) string { } cut = r.base.completeSecretBoundary(combined, cut) r.pending = combined[cut:] - return r.base.Redact(combined[:cut]) + return r.base.redact(combined[:cut], false) } // Follow the replacer's leftmost, longest-first matches. A complete secret @@ -304,5 +559,5 @@ func (r *StreamingRedactor) Flush() string { if r.base == nil { return pending } - return r.base.Redact(pending) + return r.base.redact(pending, false) } diff --git a/pkg/secrets/redact_multiline_test.go b/pkg/secrets/redact_multiline_test.go index b033531cd2..ec0e01fecd 100644 --- a/pkg/secrets/redact_multiline_test.go +++ b/pkg/secrets/redact_multiline_test.go @@ -8,6 +8,8 @@ import ( "github.com/stretchr/testify/suite" ) +const numericFixture = "TOKEN={\n\"pin\": 1\n}" + type multilineRedactionSuite struct{ suite.Suite } func TestMultilineRedactionSuite(t *testing.T) { suite.Run(t, new(multilineRedactionSuite)) } @@ -25,3 +27,101 @@ func (s *multilineRedactionSuite) TestMultilineSecretsKeepUnrelatedJSONValid() { }) } } + +func (s *multilineRedactionSuite) TestNumericJSONSecretIsMaskedWithoutBreakingJSON() { + redactor := secrets.NewRedactor([]string{"TOKEN={\n\"pin\": 123456\n}"}) + output := redactor.Redact(`{"level":"info","pin": 123456}`) + s.True(json.Valid([]byte(output))) + s.NotContains(output, "123456") + s.NotContains(redactor.Redact(`"pin": 123456`), "123456") +} + +func (s *multilineRedactionSuite) TestScalarValuesRemainMaskedAfterCombining() { + values := []struct{ name, value, line, hidden string }{ + { + name: "array number", + value: "[\n 123456\n]", + line: `{"values":[123456]}`, + hidden: "123456", + }, + { + name: "boolean", + value: "{\n\"enabled\": true\n}", + line: `{"enabled": true}`, + hidden: "true", + }, + { + name: "nested", + value: "{\n\"inner\": {\n\"pin\": 123456\n}\n}", + line: `{"pin": 123456}`, + hidden: "123456", + }, + } + for _, tc := range values { + s.Run(tc.name, func() { + redactor := secrets.Combine( + secrets.NewRedactor([]string{"TOKEN=" + tc.value}), + secrets.NewRedactor(nil), + ) + output := redactor.Redact(tc.line) + s.True(json.Valid([]byte(output))) + s.NotContains(output, tc.hidden) + if tc.name == "array number" { + s.NotContains(redactor.Redact(" 123456,"), tc.hidden) + } + }) + } +} + +func (s *multilineRedactionSuite) TestNumericSecretDoesNotChangeUnrelatedJSON() { + redactor := secrets.NewRedactor([]string{numericFixture}) + input := `{"schemaVersion":1,"message":"ordinary"}` + s.Equal(input, redactor.Redact(input)) +} + +func (s *multilineRedactionSuite) TestScalarSecretDoesNotPartiallyReplaceOtherNumbers() { + redactor := secrets.NewRedactor([]string{numericFixture}) + input := `{"pin":123,"schemaVersion":1}` + output := redactor.Redact(input) + s.Equal(input, output) + s.True(json.Valid([]byte(output))) +} + +func (s *multilineRedactionSuite) TestNumericSecretDoesNotChangeSplitJSONWrites() { + redactor := secrets.NewStreamingRedactor( + secrets.NewRedactor([]string{numericFixture}), + ) + output := redactor.RedactChunk( + `{"schemaVersion":`, + ) + redactor.RedactChunk( + "1", + ) + redactor.RedactChunk( + "}", + ) + redactor.Flush() + s.Equal(`{"schemaVersion":1}`, output) +} + +func (s *multilineRedactionSuite) TestLargeJSONNumberIsMasked() { + redactor := secrets.NewRedactor([]string{"TOKEN={\n\"amount\": 1e10000\n}"}) + output := redactor.Redact(`{"amount": 1e10000}`) + s.True(json.Valid([]byte(output))) + s.NotContains(output, "1e10000") +} + +func (s *multilineRedactionSuite) TestNumericStringSecretDoesNotCorruptUnrelatedJSON() { + redactor := secrets.NewRedactor([]string{"TOKEN={\n\"pin\": \"1\"\n}"}) + input := `{"schemaVersion":1,"message":"ordinary"}` + s.Equal(input, redactor.Redact(input)) + output := redactor.Redact(`{"message":"1"}`) + s.True(json.Valid([]byte(output))) + s.NotContains(output, `"1"`) +} + +func (s *multilineRedactionSuite) TestNumericSecretIsMaskedInTextAndJSONString() { + redactor := secrets.NewRedactor([]string{"TOKEN={\n\"pin\": 123456\n}"}) + s.NotContains(redactor.Redact("agent reported 123456"), "123456") + output := redactor.Redact(`{"schemaVersion":1,"message":"agent reported 123456"}`) + s.True(json.Valid([]byte(output))) + s.NotContains(output, "123456") + s.Contains(output, `"schemaVersion":1`) +} From 0d7204f456f82aed57eb68b8a367612618ce689e Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 22:57:39 -0600 Subject: [PATCH 11/11] fix(secrets): mask numeric strings in streamed logs --- pkg/secrets/redact.go | 35 ++++++++++++++++++++++++++++ pkg/secrets/redact_multiline_test.go | 29 +++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/pkg/secrets/redact.go b/pkg/secrets/redact.go index fac92c5e38..1b4b6215b6 100644 --- a/pkg/secrets/redact.go +++ b/pkg/secrets/redact.go @@ -420,6 +420,7 @@ type StreamingRedactor struct { mu sync.Mutex base *Redactor pending string + discard bool } // NewStreamingRedactor creates a chunk-safe redactor around r. @@ -434,6 +435,9 @@ func (r *StreamingRedactor) RedactChunk(chunk string) string { } r.mu.Lock() defer r.mu.Unlock() + if r.base != nil && r.base.scalars != nil { + return r.redactScalarChunk(chunk) + } combined := r.pending + chunk r.pending = "" if r.base == nil { @@ -559,5 +563,36 @@ func (r *StreamingRedactor) Flush() string { if r.base == nil { return pending } + if r.base.scalars != nil { + r.discard = false + return r.base.Redact(pending) + } return r.base.redact(pending, false) } + +func (r *StreamingRedactor) redactScalarChunk(chunk string) string { + const maxLineLength = 1 << 20 + var output strings.Builder + for len(chunk) > 0 { + line, rest, complete := strings.Cut(chunk, "\n") + chunk = rest + if !r.discard { + if len(r.pending)+len(line) > maxLineLength { + output.WriteString(redactMask) + r.pending = "" + r.discard = true + } else { + r.pending += line + } + } + if complete { + if !r.discard { + output.WriteString(r.base.Redact(r.pending)) + } + output.WriteByte('\n') + r.pending = "" + r.discard = false + } + } + return output.String() +} diff --git a/pkg/secrets/redact_multiline_test.go b/pkg/secrets/redact_multiline_test.go index ec0e01fecd..1a37ee1eb7 100644 --- a/pkg/secrets/redact_multiline_test.go +++ b/pkg/secrets/redact_multiline_test.go @@ -2,6 +2,7 @@ package secrets_test import ( "encoding/json" + "strings" "testing" "github.com/devsy-org/devsy/pkg/secrets" @@ -125,3 +126,31 @@ func (s *multilineRedactionSuite) TestNumericSecretIsMaskedInTextAndJSONString() s.NotContains(output, "123456") s.Contains(output, `"schemaVersion":1`) } + +func (s *multilineRedactionSuite) TestStreamingNumericStringSecretsAreMasked() { + for _, chunks := range [][]string{ + {"PIN 123456\n"}, + {"PIN 123", "456\n"}, + {`{"message":"PIN 123456","schemaVersion":1}`, "\n"}, + } { + redactor := secrets.NewStreamingRedactor( + secrets.NewRedactor([]string{"TOKEN={\n\"pin\":\"123456\"\n}"}), + ) + var output string + for _, chunk := range chunks { + output += redactor.RedactChunk(chunk) + } + output += redactor.Flush() + s.NotContains(output, "123456") + } +} + +func (s *multilineRedactionSuite) TestStreamingOversizedLineIsBoundedAndMasked() { + redactor := secrets.NewStreamingRedactor( + secrets.NewRedactor([]string{"TOKEN={\n\"pin\":\"123456\"\n}"}), + ) + output := redactor.RedactChunk(strings.Repeat("a", 1<<20)) + output += redactor.RedactChunk("123456\nnext line\n") + output += redactor.Flush() + s.Equal("***\nnext line\n", output) +}