diff --git a/pkg/agent/delivery/delivery.go b/pkg/agent/delivery/delivery.go index 1174b18abd..885dc2195b 100644 --- a/pkg/agent/delivery/delivery.go +++ b/pkg/agent/delivery/delivery.go @@ -43,6 +43,9 @@ type PostStartOptions struct { Arch string DownloadURL string PreferInContainerDownload bool + // SkipVersionCheck permits custom agent versions while retaining transfer + // completeness and executable validation. + SkipVersionCheck bool } // Cleaner removes the resources a delivery created for a workspace. Cleanup is diff --git a/pkg/agent/delivery/kubernetes.go b/pkg/agent/delivery/kubernetes.go index b3cf11db4f..2ad79924d5 100644 --- a/pkg/agent/delivery/kubernetes.go +++ b/pkg/agent/delivery/kubernetes.go @@ -7,7 +7,9 @@ import ( "fmt" "io" "net" + "path" "strings" + "sync" "time" "al.essio.dev/pkg/shellescape" @@ -16,6 +18,7 @@ import ( "github.com/devsy-org/devsy/pkg/driver" "github.com/devsy-org/devsy/pkg/log" "github.com/devsy-org/devsy/pkg/version" + "github.com/google/uuid" "k8s.io/apimachinery/pkg/util/wait" execerr "k8s.io/client-go/util/exec" "k8s.io/client-go/util/retry" @@ -37,13 +40,26 @@ type KubernetesDelivery struct { // InstallPath overrides where the agent binary is installed inside the // container. InstallPath string + + // execStreamIdleTimeout overrides the idle timeout in tests. + execStreamIdleTimeout time.Duration + execStreamCompletionTimeout time.Duration } const ( - noDownloadToolExitCode = 127 - downloadTimeoutSeconds = 25 - execStreamAttemptTimeout = 30 * time.Second - execStreamMaxAttempts = 2 + noDownloadToolExitCode = 127 + downloadTimeoutSeconds = 25 + execStreamIdleTimeout = 30 * time.Second + execStreamMaxAttempts = 2 + execStreamCleanupTimeout = 5 * time.Second + execStreamCompletionTimeout = 2 * time.Minute +) + +var ( + errExecStreamIdleTimeout = errors.New("exec-stream delivery stalled") + errExecStreamCompletionTimeout = errors.New( + "exec-stream delivery did not complete after source EOF", + ) ) func (d *KubernetesDelivery) Phase() DeliveryPhase { @@ -65,15 +81,22 @@ func (d *KubernetesDelivery) DeliverPostStart(ctx context.Context, opts PostStar destPath := d.destPath() // Skip delivery when the in-pod binary already matches. - expected := d.expectedVersion() - if actual := d.detectVersion(ctx, destPath); actual != "" && actual == expected { - log.Debugf("remote agent version matches expected version %s, skipping delivery", expected) - return nil + if !opts.SkipVersionCheck { + expected := d.expectedVersion() + if actual := d.detectVersion(ctx, destPath); actual != "" && actual == expected { + log.Debugf( + "remote agent version matches expected version %s, skipping delivery", + expected, + ) + return nil + } } + var downloadErr error if opts.PreferInContainerDownload { if err := d.deliverViaDownload(ctx, destPath, opts.DownloadURL, opts.Arch); err != nil { - log.Debugf( + downloadErr = err + log.Warnf( "in-container download unavailable, falling back to exec-stream delivery: %v", err, ) @@ -84,13 +107,24 @@ func (d *KubernetesDelivery) DeliverPostStart(ctx context.Context, opts PostStar } if err := d.deliverViaExecStream(ctx, destPath, opts); err != nil { - return fmt.Errorf("write binary to container: %w", err) + return deliveryFailure(downloadErr, err) } log.Debugf("delivered agent binary to pod via kubernetes exec-stream") return nil } +func deliveryFailure(downloadErr, streamErr error) error { + streamErr = fmt.Errorf("exec-stream delivery failed: %w", streamErr) + if downloadErr == nil { + return streamErr + } + return errors.Join( + fmt.Errorf("in-container download failed: %w", downloadErr), + streamErr, + ) +} + func (d *KubernetesDelivery) Cleanup(_ context.Context, _ string) error { return nil } @@ -161,6 +195,9 @@ func (d *KubernetesDelivery) deliverViaExecStream( wait.Backoff{Steps: execStreamMaxAttempts}, isTransientDeliveryError, func() error { + if err := ctx.Err(); err != nil { + return &permanentDeliveryError{err} + } attempt++ binary, err := opts.BinarySource(ctx, opts.Arch) if err != nil { @@ -168,9 +205,7 @@ func (d *KubernetesDelivery) deliverViaExecStream( } defer func() { _ = binary.Close() }() - attemptCtx, cancel := context.WithTimeout(ctx, execStreamAttemptTimeout) - defer cancel() - streamErr := d.execStreamOnce(attemptCtx, destPath, binary) + streamErr := d.execStreamOnce(ctx, destPath, binary, opts.SkipVersionCheck) if streamErr != nil && isTransientDeliveryError(streamErr) && attempt < execStreamMaxAttempts { log.Warnf( @@ -181,8 +216,7 @@ func (d *KubernetesDelivery) deliverViaExecStream( return streamErr }, ) - var perm *permanentDeliveryError - if errors.As(err, &perm) { + if perm, ok := errors.AsType[*permanentDeliveryError](err); ok { return perm.err } return err @@ -198,15 +232,237 @@ func (d *KubernetesDelivery) execStreamOnce( ctx context.Context, destPath string, binary io.Reader, + skipVersionCheck bool, ) error { + tempPath, err := transferTempPath(destPath) + if err != nil { + return &permanentDeliveryError{err} + } + + attemptCtx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + progress := newProgressReader(binary) + monitorDone := make(chan struct{}) + monitorStop := make(chan struct{}) + monitor := execStreamProgressMonitor{ + ctx: attemptCtx, + cancel: cancel, + progress: progress, + idleTimeout: d.idleTimeout(), + completionTimeout: d.completionTimeout(), + stop: monitorStop, + done: monitorDone, + } + go monitor.run() + + quotedTemp := shellescape.Quote(tempPath) + stageScript := fmt.Sprintf( + `set -e; d=$(dirname %s); mkdir -p "$d"; rm -f %s; cat > %s`, + quotedTemp, quotedTemp, quotedTemp, + ) + stageErr := d.Exec( + attemptCtx, + []string{"sh", "-c", stageScript}, + driver.Streams{Stdin: progress}, + ) + close(monitorStop) + <-monitorDone + if cause := context.Cause(attemptCtx); cause != nil { + stageErr = cause + } + if stageErr == nil { + stageErr = progress.completionError() + } + if stageErr != nil { + d.cleanupStagedBinary(ctx, tempPath) + return stageErr + } + + commitCtx, commitCancel := context.WithTimeout(ctx, d.idleTimeout()) + defer commitCancel() + expectedVersion := d.expectedVersion() + if skipVersionCheck { + expectedVersion = "" + } + commitScript := validationCommitScript(tempPath, destPath, expectedVersion, progress.size()) + var stderr bytes.Buffer + if err := d.Exec( + commitCtx, + []string{"sh", "-c", commitScript}, + driver.Streams{Stderr: &stderr}, + ); err != nil { + d.cleanupStagedBinary(ctx, tempPath) + validationErr := fmt.Errorf( + "validate staged agent: %w (%s)", + err, + strings.TrimSpace(stderr.String()), + ) + if isTransientDeliveryError(err) { + return validationErr + } + return &permanentDeliveryError{validationErr} + } + return nil +} + +func (d *KubernetesDelivery) idleTimeout() time.Duration { + if d.execStreamIdleTimeout > 0 { + return d.execStreamIdleTimeout + } + return execStreamIdleTimeout +} + +func (d *KubernetesDelivery) completionTimeout() time.Duration { + if d.execStreamCompletionTimeout > 0 { + return d.execStreamCompletionTimeout + } + return execStreamCompletionTimeout +} + +func transferTempPath(destPath string) (string, error) { + cleanDest := path.Clean(destPath) + if cleanDest == "." || cleanDest == "/" { + return "", fmt.Errorf("invalid agent destination path %q", destPath) + } + return path.Join(path.Dir(cleanDest), ".devsy-transfer-"+uuid.NewString()), nil +} + +func validationCommitScript(tempPath, destPath, expectedVersion string, expectedSize int64) string { + quotedTemp := shellescape.Quote(tempPath) quotedDest := shellescape.Quote(destPath) - script := fmt.Sprintf( - `set -e; d=$(dirname %s); mkdir -p "$d"; `+ - `t=$(mktemp %s.XXXXXX); `+ - `cat > "$t" && chmod 0755 "$t" && mv -f "$t" %s || { rm -f "$t"; exit 1; }`, - quotedDest, quotedDest, quotedDest, + quotedExpected := shellescape.Quote(expectedVersion) + return fmt.Sprintf(`set -e +actual_size="$(wc -c < %s)" +[ "$actual_size" -eq %d ] || { + rm -f %s + echo "staged Devsy agent size mismatch" >&2 + exit 1 +} +chmod 0755 %s +actual="$(%s --version 2>/dev/null)" || { + rm -f %s + echo "staged Devsy agent is not executable" >&2 + exit 1 +} +if [ -n %s ] && [ "$actual" != %s ]; then + rm -f %s + echo "staged Devsy agent version mismatch" >&2 + exit 1 +fi +mv -f %s %s`, + quotedTemp, expectedSize, quotedTemp, quotedTemp, quotedTemp, + quotedTemp, quotedExpected, quotedExpected, quotedTemp, quotedTemp, quotedDest, ) - return d.Exec(ctx, []string{"sh", "-c", script}, driver.Streams{Stdin: binary}) +} + +func (d *KubernetesDelivery) cleanupStagedBinary(parent context.Context, tempPath string) { + cleanupCtx, cancel := context.WithTimeout( + context.WithoutCancel(parent), + execStreamCleanupTimeout, + ) + defer cancel() + if err := d.Exec( + cleanupCtx, + []string{"sh", "-c", "rm -f " + shellescape.Quote(tempPath)}, + driver.Streams{}, + ); err != nil { + log.Debugf("failed to clean up staged agent %s: %v", tempPath, err) + } +} + +type progressReader struct { + reader io.Reader + mu sync.Mutex + lastProgress time.Time + bytesRead int64 + readErr error + eofAt time.Time +} + +func newProgressReader(reader io.Reader) *progressReader { + return &progressReader{reader: reader, lastProgress: time.Now()} +} + +func (r *progressReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + r.mu.Lock() + defer r.mu.Unlock() + if n > 0 { + r.lastProgress = time.Now() + r.bytesRead += int64(n) + } + if err != nil { + r.readErr = err + if errors.Is(err, io.EOF) && r.eofAt.IsZero() { + r.eofAt = time.Now() + } + } + return n, err +} + +func (r *progressReader) completionError() error { + r.mu.Lock() + defer r.mu.Unlock() + if errors.Is(r.readErr, io.EOF) && r.bytesRead > 0 { + return nil + } + if r.readErr != nil && !errors.Is(r.readErr, io.EOF) { + return &permanentDeliveryError{fmt.Errorf("read agent binary: %w", r.readErr)} + } + return fmt.Errorf("incomplete agent stream: %w", io.ErrUnexpectedEOF) +} + +func (r *progressReader) size() int64 { + r.mu.Lock() + defer r.mu.Unlock() + return r.bytesRead +} + +func (r *progressReader) timeoutError( + now time.Time, + idleTimeout, completionTimeout time.Duration, +) error { + r.mu.Lock() + defer r.mu.Unlock() + if !r.eofAt.IsZero() { + if now.Sub(r.eofAt) >= completionTimeout { + return errExecStreamCompletionTimeout + } + return nil + } + if now.Sub(r.lastProgress) >= idleTimeout { + return errExecStreamIdleTimeout + } + return nil +} + +type execStreamProgressMonitor struct { + ctx context.Context + cancel context.CancelCauseFunc + progress *progressReader + idleTimeout time.Duration + completionTimeout time.Duration + stop <-chan struct{} + done chan<- struct{} +} + +func (m *execStreamProgressMonitor) run() { + defer close(m.done) + ticker := time.NewTicker(max(m.idleTimeout/2, time.Nanosecond)) + defer ticker.Stop() + for { + select { + case <-m.ctx.Done(): + return + case <-m.stop: + return + case now := <-ticker.C: + if err := m.progress.timeoutError(now, m.idleTimeout, m.completionTimeout); err != nil { + m.cancel(err) + return + } + } + } } func isTransientDeliveryError(err error) bool { @@ -219,6 +475,9 @@ func isTransientDeliveryError(err error) bool { if errors.Is(err, context.DeadlineExceeded) { return true } + if errors.Is(err, errExecStreamIdleTimeout) || errors.Is(err, errExecStreamCompletionTimeout) { + return true + } if _, ok := errors.AsType[net.Error](err); ok { return true } diff --git a/pkg/agent/delivery/kubernetes_test.go b/pkg/agent/delivery/kubernetes_test.go index 542e38ca49..fda4626fad 100644 --- a/pkg/agent/delivery/kubernetes_test.go +++ b/pkg/agent/delivery/kubernetes_test.go @@ -3,11 +3,19 @@ package delivery import ( "bytes" "context" + "errors" "fmt" "io" + "os" + "os/exec" + "path" + "path/filepath" "strings" "testing" + "testing/iotest" + "time" + "al.essio.dev/pkg/shellescape" pkgconfig "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/driver" "github.com/stretchr/testify/assert" @@ -94,7 +102,7 @@ func TestKubernetesDelivery_DeliverPostStart_WritesBinary(t *testing.T) { }) require.NoError(t, err) - require.Len(t, exec.calls, 2) + require.Len(t, exec.calls, 3) destPath := pkgconfig.ContainerDevsyHelperLocation probeScript := strings.Join(exec.calls[0].argv, " ") @@ -102,11 +110,16 @@ func TestKubernetesDelivery_DeliverPostStart_WritesBinary(t *testing.T) { assert.Contains(t, probeScript, destPath) writeScript := strings.Join(exec.calls[1].argv, " ") - assert.Contains(t, writeScript, "mktemp") - assert.Contains(t, writeScript, "chmod 0755") - assert.Contains(t, writeScript, "mv -f") - assert.Contains(t, writeScript, destPath) + assert.Contains(t, writeScript, ".devsy-transfer-") + assert.NotContains(t, writeScript, "chmod 0755") + assert.NotContains(t, writeScript, "mv -f") + assert.Contains(t, writeScript, path.Dir(destPath)) assert.Equal(t, binaryData, exec.calls[1].stdin) + commitScript := strings.Join(exec.calls[2].argv, " ") + assert.Contains(t, commitScript, "--version") + assert.Contains(t, commitScript, testVersion) + assert.Contains(t, commitScript, "mv -f") + assert.Contains(t, commitScript, destPath) } func TestKubernetesDelivery_DeliverPostStart_SkipsWhenVersionMatches(t *testing.T) { @@ -135,7 +148,7 @@ func TestKubernetesDelivery_DeliverPostStart_DeliversWhenProbeErrors(t *testing. Arch: testArch, }) require.NoError(t, err) - assert.Len(t, probeErr.calls, 2) + assert.Len(t, probeErr.calls, 3) } func TestKubernetesDelivery_DeliverPostStart_BinarySourceError(t *testing.T) { @@ -197,7 +210,7 @@ func TestKubernetesDelivery_DeliverPostStart_FallsBackToExecStreamWhenNoDownload }) require.NoError(t, err) - require.Len(t, exec.calls, 3, "probe, failed download, then exec-stream fallback") + require.Len(t, exec.calls, 4, "probe, failed download, stage, then validation and commit") assert.Equal(t, binaryData, exec.calls[2].stdin) } @@ -212,13 +225,18 @@ func TestKubernetesDelivery_DeliverPostStart_SkipsDownloadWhenNoURLConfigured(t }) require.NoError(t, err) - require.Len(t, exec.calls, 2, "probe, then exec-stream (no download attempted)") + require.Len(t, exec.calls, 3, "probe, stage, then validation and commit") assert.Equal(t, binaryData, exec.calls[1].stdin) } func TestKubernetesDelivery_DeliverViaExecStream_RetriesTransientFailureOnce(t *testing.T) { exec := &recordingExec{ - errs: []error{fmt.Errorf("write binary to container: %w", context.DeadlineExceeded), nil}, + errs: []error{ + fmt.Errorf("write binary to container: %w", context.DeadlineExceeded), + nil, // cleanup + nil, // retry stage + nil, // retry validation and commit + }, } d := &KubernetesDelivery{Exec: exec.fn} @@ -227,7 +245,10 @@ func TestKubernetesDelivery_DeliverViaExecStream_RetriesTransientFailureOnce(t * Arch: testArch, }) require.NoError(t, err) - assert.Len(t, exec.calls, execStreamMaxAttempts, "one stalled attempt, one successful retry") + assert.Len(t, exec.calls, 4, "failed stage, cleanup, retry stage, and commit") + firstTemp := stagedPath(t, exec.calls[0]) + secondTemp := stagedPath(t, exec.calls[2]) + assert.NotEqual(t, firstTemp, secondTemp, "every retry must use an independent staging path") } func TestKubernetesDelivery_DeliverViaExecStream_DoesNotRetryPermanentFailure(t *testing.T) { @@ -240,7 +261,144 @@ func TestKubernetesDelivery_DeliverViaExecStream_DoesNotRetryPermanentFailure(t Arch: testArch, }) require.Error(t, err) - assert.Len(t, exec.calls, 1, "a permanent failure must not be retried") + assert.Len(t, exec.calls, 2, "a permanent failure is cleaned up but not retried") +} + +func TestKubernetesDelivery_InterruptedStageNeverCommits(t *testing.T) { + var calls []recordedCall + attempt := 0 + d := &KubernetesDelivery{ExpectedVersion: testVersion} + d.Exec = func(_ context.Context, argv []string, streams driver.Streams) error { + call := recordedCall{argv: argv} + if streams.Stdin != nil { + attempt++ + if attempt == 1 { + buf := make([]byte, 4) + n, _ := streams.Stdin.Read(buf) + call.stdin = string(buf[:n]) + } else { + data, err := io.ReadAll(streams.Stdin) + require.NoError(t, err) + call.stdin = string(data) + } + } + calls = append(calls, call) + if streams.Stdin != nil && attempt == 1 { + return fmt.Errorf("stream interrupted: unexpected EOF") + } + return nil + } + + err := d.deliverViaExecStream(context.Background(), "/usr/local/bin/devsy", PostStartOptions{ + BinarySource: binarySourceFrom("complete-binary"), + Arch: testArch, + }) + require.NoError(t, err) + require.Len(t, calls, 4) + assert.NotContains(t, strings.Join(calls[0].argv, " "), "mv -f") + assert.Contains(t, strings.Join(calls[1].argv, " "), "rm -f") + assert.NotEqual(t, stagedPath(t, calls[0]), stagedPath(t, calls[2])) + assert.Contains(t, strings.Join(calls[3].argv, " "), "mv -f") +} + +func TestKubernetesDelivery_ValidationFailureCleansUpWithoutPromotion(t *testing.T) { + exec := &recordingExec{ + errs: []error{ + nil, + execerr.CodeExitError{Code: 1, Err: fmt.Errorf("wrong version")}, + nil, + }, + } + d := &KubernetesDelivery{Exec: exec.fn, ExpectedVersion: testVersion} + + err := d.deliverViaExecStream(context.Background(), "/usr/local/bin/devsy", PostStartOptions{ + BinarySource: binarySourceFrom("invalid-binary"), + Arch: testArch, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "validate staged agent") + require.Len(t, exec.calls, 3) + assert.NotContains(t, strings.Join(exec.calls[0].argv, " "), "mv -f") + assert.Contains(t, strings.Join(exec.calls[2].argv, " "), "rm -f") +} + +type delayedReader struct { + chunks int + delay time.Duration +} + +func (r *delayedReader) Read(p []byte) (int, error) { + if r.chunks == 0 { + return 0, io.EOF + } + time.Sleep(r.delay) + r.chunks-- + p[0] = 'x' + return 1, nil +} + +func TestKubernetesDelivery_LongProgressingTransferSucceeds(t *testing.T) { + exec := &recordingExec{} + d := &KubernetesDelivery{ + Exec: exec.fn, + ExpectedVersion: testVersion, + execStreamIdleTimeout: 200 * time.Millisecond, + } + start := time.Now() + err := d.execStreamOnce(context.Background(), "/usr/local/bin/devsy", &delayedReader{ + chunks: 12, + delay: 30 * time.Millisecond, + }, false) + require.NoError(t, err) + assert.Greater(t, time.Since(start), 300*time.Millisecond) + assert.Len(t, exec.calls, 2) +} + +func TestKubernetesDelivery_StalledTransferCancelsWithoutCommit(t *testing.T) { + var calls []recordedCall + d := &KubernetesDelivery{ + ExpectedVersion: testVersion, + execStreamIdleTimeout: 20 * time.Millisecond, + } + d.Exec = func(ctx context.Context, argv []string, streams driver.Streams) error { + calls = append(calls, recordedCall{argv: argv}) + if streams.Stdin == nil { + return nil + } + buf := make([]byte, 1) + _, _ = streams.Stdin.Read(buf) + <-ctx.Done() + return ctx.Err() + } + + err := d.execStreamOnce( + context.Background(), "/usr/local/bin/devsy", strings.NewReader("x"), false, + ) + require.ErrorIs(t, err, errExecStreamIdleTimeout) + require.Len(t, calls, 2, "stage and cleanup only") + assert.NotContains(t, strings.Join(calls[0].argv, " "), "mv -f") + assert.Contains(t, strings.Join(calls[1].argv, " "), "rm -f") +} + +func TestKubernetesDelivery_PreservesDownloadAndStreamErrors(t *testing.T) { + exec := &recordingExec{errs: []error{ + nil, + fmt.Errorf("egress blocked"), + fmt.Errorf("stream broken"), + nil, + }} + d := &KubernetesDelivery{Exec: exec.fn, ExpectedVersion: testVersion} + err := d.DeliverPostStart(context.Background(), PostStartOptions{ + BinarySource: binarySourceFrom("binary"), + Arch: testArch, + DownloadURL: "https://example.com/releases", + PreferInContainerDownload: true, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "in-container download failed") + assert.Contains(t, err.Error(), "egress blocked") + assert.Contains(t, err.Error(), "exec-stream delivery failed") + assert.Contains(t, err.Error(), "stream broken") } func TestIsTransientDeliveryError(t *testing.T) { @@ -291,15 +449,342 @@ func TestKubernetesDelivery_DeliverPostStart_UsesInstallPathOverride(t *testing. }) require.NoError(t, err) - require.Len(t, exec.calls, 2) + require.Len(t, exec.calls, 3) probeScript := strings.Join(exec.calls[0].argv, " ") assert.Contains(t, probeScript, installPath) writeScript := strings.Join(exec.calls[1].argv, " ") - assert.Contains(t, writeScript, installPath) + assert.Contains(t, writeScript, path.Dir(installPath)) assert.NotContains(t, writeScript, pkgconfig.ContainerDevsyHelperLocation) + assert.Contains(t, strings.Join(exec.calls[2].argv, " "), installPath) +} + +func stagedPath(t *testing.T, call recordedCall) string { + t.Helper() + for field := range strings.FieldsSeq(strings.Join(call.argv, " ")) { + field = strings.Trim(field, "';") + if strings.Contains(field, ".devsy-transfer-") { + return field + } + } + t.Fatal("staging path not found") + return "" } func TestKubernetesDelivery_DestPath_DefaultsWhenInstallPathUnset(t *testing.T) { d := &KubernetesDelivery{} assert.Equal(t, pkgconfig.ContainerDevsyHelperLocation, d.destPath()) } + +func TestKubernetesDelivery_ExecStreamPreservesInstalledAgent(t *testing.T) { + sourceErr := errors.New("binary source failed") + for _, tc := range []struct { + name string + version string + truncate bool + cancel bool + sourceError bool + incomplete bool + wantExecuted bool + wantError string + }{ + {name: "complete transfer", version: testVersion, wantExecuted: true}, + {name: "truncated but executable", version: testVersion, truncate: true, wantError: "size mismatch"}, + {name: "wrong version", version: "v0.0.1", wantExecuted: true, wantError: "version mismatch"}, + {name: "cancelled stage", version: testVersion, cancel: true, wantError: "context canceled"}, + {name: "source read error", version: testVersion, sourceError: true, wantError: "binary source failed"}, + {name: "remote success before EOF", version: testVersion, incomplete: true, wantError: "incomplete agent stream"}, + } { + t.Run(tc.name, func(t *testing.T) { + dest, marker := installedAgentFixture(t) + dir := filepath.Dir(dest) + binary := fmt.Sprintf( + "#!/bin/sh\necho executed > %s\necho %s\n# trailing bytes\n", + shellescape.Quote(marker), + tc.version, + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + d := &KubernetesDelivery{ExpectedVersion: testVersion} + var cancelStage func() + if tc.cancel { + cancelStage = cancel + } + d.Exec = shellAgentExec(tc.truncate, tc.incomplete, cancelStage) + var reader io.Reader = strings.NewReader(binary) + if tc.sourceError { + reader = io.MultiReader(reader, iotest.ErrReader(sourceErr)) + } + err := d.execStreamOnce(ctx, dest, reader, false) + installed, readErr := os.ReadFile(dest) // #nosec G304 -- reads an owned test fixture + require.NoError(t, readErr) + if tc.wantError == "" { + require.NoError(t, err) + assert.Equal(t, binary, string(installed)) + } else { + require.ErrorContains(t, err, tc.wantError) + assert.Equal(t, "installed agent", string(installed)) + } + assertAgentStagingClean(t, dir, marker, tc.wantExecuted) + }) + } +} + +func TestKubernetesDelivery_CustomAgentSkipsVersionChecks(t *testing.T) { + for _, tc := range []struct { + name string + truncate bool + badCLI bool + wantErr string + }{ + {name: "complete transfer replaces matching installed agent"}, + {name: "truncated transfer is rejected", truncate: true, wantErr: "size mismatch"}, + {name: "version command failure is rejected", badCLI: true, wantErr: "not executable"}, + } { + t.Run(tc.name, func(t *testing.T) { + dest, marker := installedAgentFixture(t) + installed := "#!/bin/sh\nif [ \"$1\" = \"--version\" ]; then echo " + testVersion + "; exit 0; fi\n" + require.NoError(t, os.WriteFile(dest, []byte(installed), 0o600)) + chmodErr := os.Chmod(dest, 0o700) // #nosec G302 -- owned executable fixture + require.NoError(t, chmodErr) + + cliCommand := "echo custom-agent-version; exit 0" + if tc.badCLI { + cliCommand = "exit 1" + } + binary := fmt.Sprintf( + "#!/bin/sh\nif [ \"$1\" = \"--version\" ]; then %s; fi\necho executed > %s\n# trailing bytes\n", + cliCommand, + shellescape.Quote(marker), + ) + d := &KubernetesDelivery{ + Exec: shellAgentExec(tc.truncate, false, nil), + ExpectedVersion: testVersion, + InstallPath: dest, + } + err := d.DeliverPostStart(context.Background(), PostStartOptions{ + BinarySource: binarySourceFrom(binary), + Arch: testArch, + SkipVersionCheck: true, + }) + + actual, readErr := os.ReadFile(dest) // #nosec G304 -- reads an owned test fixture + require.NoError(t, readErr) + if tc.wantErr == "" { + require.NoError(t, err) + assert.Equal(t, binary, string(actual)) + } else { + require.ErrorContains(t, err, tc.wantErr) + assert.Equal( + t, + installed, + string(actual), + "failed custom delivery must preserve the installed agent", + ) + } + assertAgentStagingClean(t, filepath.Dir(dest), marker, false) + }) + } +} + +func TestKubernetesDelivery_ValidationIsBounded(t *testing.T) { + calls := 0 + d := &KubernetesDelivery{ + ExpectedVersion: testVersion, + execStreamIdleTimeout: 50 * time.Millisecond, + } + d.Exec = func(ctx context.Context, _ []string, streams driver.Streams) error { + calls++ + switch calls { + case 1: + _, err := io.Copy(io.Discard, streams.Stdin) + return err + case 2: + _, hasDeadline := ctx.Deadline() + require.True(t, hasDeadline) + <-ctx.Done() + return ctx.Err() + default: + require.NoError(t, ctx.Err(), "cleanup must have an independent context") + return nil + } + } + err := d.execStreamOnce( + context.Background(), + "/usr/local/bin/devsy", + strings.NewReader("binary"), + false, + ) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 3, calls) +} + +func shellAgentExec(truncate, incomplete bool, cancelStage func()) ArgvExecFunc { + return func(ctx context.Context, argv []string, streams driver.Streams) error { + if streams.Stdin != nil && incomplete { + _, err := io.CopyN(io.Discard, streams.Stdin, 1) + return err + } + if streams.Stdin != nil && truncate { + data, err := io.ReadAll(streams.Stdin) + if err != nil { + return err + } + streams.Stdin = bytes.NewReader(data[:len(data)-len("# trailing bytes\n")]) + } + cmd := exec.CommandContext( + ctx, + argv[0], + argv[1:]...) // #nosec G204 -- executes delivery-generated scripts in a test directory + cmd.Stdin, cmd.Stdout, cmd.Stderr = streams.Stdin, streams.Stdout, streams.Stderr + err := cmd.Run() + if streams.Stdin != nil && cancelStage != nil { + cancelStage() + } + return err + } +} + +func installedAgentFixture(t *testing.T) (string, string) { + t.Helper() + dir := filepath.Join(t.TempDir(), "agent directory's") + require.NoError(t, os.MkdirAll(dir, 0o700)) + dest := filepath.Join(dir, "devsy") + require.NoError(t, os.WriteFile(dest, []byte("installed agent"), 0o600)) + return dest, filepath.Join(dir, "executed") +} + +func assertAgentStagingClean(t *testing.T, dir, marker string, wantExecuted bool) { + t.Helper() + _, err := os.Stat(marker) + if wantExecuted { + require.NoError(t, err) + } else { + require.ErrorIs( + t, + err, + os.ErrNotExist, + "invalid transfers must not execute the staged binary", + ) + } + staged, err := filepath.Glob(filepath.Join(dir, ".devsy-transfer-*")) + require.NoError(t, err) + assert.Empty(t, staged) +} + +func TestKubernetesDelivery_RetriesValidationTimeout(t *testing.T) { + recorder := &recordingExec{errs: []error{nil, context.DeadlineExceeded, nil, nil, nil}} + d := &KubernetesDelivery{Exec: recorder.fn, ExpectedVersion: testVersion} + err := d.deliverViaExecStream(context.Background(), "/usr/local/bin/devsy", PostStartOptions{ + BinarySource: binarySourceFrom("binary"), + }) + require.NoError(t, err) + require.Len(t, recorder.calls, 5) + assert.NotEqual(t, stagedPath(t, recorder.calls[0]), stagedPath(t, recorder.calls[3])) +} + +func TestKubernetesDelivery_CancelledContextDoesNotAcquireBinary(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + d := &KubernetesDelivery{} + err := d.deliverViaExecStream(ctx, "/usr/local/bin/devsy", PostStartOptions{ + BinarySource: func(context.Context, string) (io.ReadCloser, error) { + t.Fatal("cancelled delivery must not acquire a binary") + return nil, nil + }, + }) + require.ErrorIs(t, err, context.Canceled) +} + +func TestProgressReaderTimeoutBoundaries(t *testing.T) { + now := time.Unix(1000, 0) + idleTimeout := time.Second + completionTimeout := time.Minute + for _, tc := range []struct { + name string + lastProgress time.Time + eofAt time.Time + want error + }{ + {name: "active source", lastProgress: now.Add(-idleTimeout / 2)}, + {name: "idle source", lastProgress: now.Add(-idleTimeout), want: errExecStreamIdleTimeout}, + {name: "draining after EOF", lastProgress: now.Add(-completionTimeout), eofAt: now.Add(-2 * idleTimeout)}, + {name: "completion deadline", eofAt: now.Add(-completionTimeout), want: errExecStreamCompletionTimeout}, + } { + t.Run(tc.name, func(t *testing.T) { + reader := &progressReader{lastProgress: tc.lastProgress, eofAt: tc.eofAt} + err := reader.timeoutError(now, idleTimeout, completionTimeout) + if tc.want == nil { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, tc.want) + } + }) + } +} + +func TestKubernetesDelivery_SlowCompletionAfterEOFSucceeds(t *testing.T) { + calls := 0 + d := &KubernetesDelivery{ + ExpectedVersion: testVersion, + execStreamIdleTimeout: 20 * time.Millisecond, + execStreamCompletionTimeout: time.Second, + } + d.Exec = func(ctx context.Context, _ []string, streams driver.Streams) error { + calls++ + if streams.Stdin == nil { + return nil + } + _, err := io.Copy(io.Discard, streams.Stdin) + if err != nil { + return err + } + timer := time.NewTimer(3 * d.idleTimeout()) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } + } + err := d.execStreamOnce( + context.Background(), + "/usr/local/bin/devsy", + strings.NewReader("binary"), + false, + ) + require.NoError(t, err) + assert.Equal(t, 2, calls, "stage completion must be followed by validation and promotion") +} + +func TestKubernetesDelivery_StalledCompletionAfterEOFCancels(t *testing.T) { + var calls []recordedCall + d := &KubernetesDelivery{ + ExpectedVersion: testVersion, + execStreamIdleTimeout: 10 * time.Millisecond, + execStreamCompletionTimeout: 30 * time.Millisecond, + } + d.Exec = func(ctx context.Context, argv []string, streams driver.Streams) error { + calls = append(calls, recordedCall{argv: argv}) + if streams.Stdin == nil { + return nil + } + _, err := io.Copy(io.Discard, streams.Stdin) + if err != nil { + return err + } + <-ctx.Done() + return ctx.Err() + } + err := d.execStreamOnce( + context.Background(), + "/usr/local/bin/devsy", + strings.NewReader("binary"), + false, + ) + require.ErrorIs(t, err, errExecStreamCompletionTimeout) + assert.True(t, isTransientDeliveryError(err)) + require.Len(t, calls, 2, "stage and cleanup only") + assert.NotContains(t, strings.Join(calls[0].argv, " "), "mv -f") + assert.Contains(t, strings.Join(calls[1].argv, " "), "rm -f") +} diff --git a/pkg/devcontainer/setup.go b/pkg/devcontainer/setup.go index 8647662240..4b9f9cceaf 100644 --- a/pkg/devcontainer/setup.go +++ b/pkg/devcontainer/setup.go @@ -3,8 +3,10 @@ package devcontainer import ( "context" "encoding/json" + "errors" "fmt" "io" + "os" "path/filepath" "runtime" "strings" @@ -27,6 +29,7 @@ import ( provider2 "github.com/devsy-org/devsy/pkg/provider" "github.com/devsy-org/devsy/pkg/status" "github.com/devsy-org/devsy/pkg/types" + "github.com/devsy-org/devsy/pkg/version" ) const ( @@ -118,16 +121,31 @@ func (r *runner) injectAgentIntoContainer(ctx context.Context, timeout time.Dura strategy := r.newAgentDelivery() if strategy.Phase() == delivery.PhasePostStart { - if err := r.deliverPostStart(ctx, strategy); err != nil { - log.Warnf("platform-native delivery failed, falling back to legacy inject: %v", err) - return r.legacyInject(ctx, timeout) - } - return nil + return fallbackAgentDelivery( + func() error { return r.deliverPostStart(ctx, strategy) }, + func() error { return r.legacyInject(ctx, timeout) }, + ) } return r.legacyInject(ctx, timeout) } +func fallbackAgentDelivery(native, legacy func() error) error { + nativeErr := native() + if nativeErr == nil { + return nil + } + log.Warnf("platform-native delivery failed, falling back to legacy inject: %v", nativeErr) + legacyErr := legacy() + if legacyErr == nil { + return nil + } + return errors.Join( + fmt.Errorf("platform-native agent delivery failed: %w", nativeErr), + fmt.Errorf("legacy agent injection failed: %w", legacyErr), + ) +} + func (r *runner) newAgentDelivery() delivery.AgentDelivery { dockerCmd := "docker" var dockerEnv []string @@ -197,6 +215,10 @@ func (r *runner) deliverPostStart(ctx context.Context, strategy delivery.AgentDe opts.BinarySource = mgr.AcquireBinary opts.Arch = arch opts.PreferInContainerDownload = !mgr.HasLocalOverride(arch) + opts.SkipVersionCheck = strings.TrimSpace(os.Getenv(pkgconfig.EnvAgentBinary)) != "" || + os.Getenv(pkgconfig.EnvAgentURL) != "" || + opts.DownloadURL != pkgconfig.DefaultAgentDownloadURL() || + version.GetVersion() == version.DevVersion } if err := strategy.DeliverPostStart(ctx, opts); err != nil { return fmt.Errorf("deliver agent (post-start): %w", err) diff --git a/pkg/devcontainer/setup_test.go b/pkg/devcontainer/setup_test.go index c4cfc41125..47da290990 100644 --- a/pkg/devcontainer/setup_test.go +++ b/pkg/devcontainer/setup_test.go @@ -3,6 +3,7 @@ package devcontainer import ( "bytes" "context" + "errors" "io" "net" "reflect" @@ -17,6 +18,7 @@ import ( "github.com/devsy-org/devsy/pkg/driver" provider2 "github.com/devsy-org/devsy/pkg/provider" "github.com/devsy-org/devsy/pkg/types" + "github.com/devsy-org/devsy/pkg/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -476,3 +478,66 @@ func TestPostStartDeliveryRetainsArchitectureForBinarySource(t *testing.T) { assert.ErrorIs(t, r.deliverPostStart(context.Background(), strategy), d.err) assert.False(t, strategy.called) } + +func TestPostStartDeliveryCustomAgentVersionPolicy(t *testing.T) { + for _, tc := range []struct { + name string + binaryPath string + envURL string + downloadURL string + wantSkip bool + }{ + {name: "release agent"}, + {name: "empty local override", binaryPath: " "}, + {name: "local override", binaryPath: "/custom/devsy", wantSkip: true}, + {name: "custom download", downloadURL: "http://localhost:8080/", wantSkip: true}, + {name: "environment download override", envURL: "http://localhost:8080", wantSkip: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(pkgconfig.EnvAgentBinary, tc.binaryPath) + t.Setenv(pkgconfig.EnvAgentURL, tc.envURL) + r := newTestRunner(&architectureDriver{arch: "arm64"}) + r.agentDownloadURL = tc.downloadURL + strategy := &capturePostStartDelivery{usesBinarySource: true} + require.NoError(t, r.deliverPostStart(context.Background(), strategy)) + wantSkip := tc.wantSkip || version.GetVersion() == version.DevVersion + assert.Equal(t, wantSkip, strategy.opts.SkipVersionCheck) + }) + } +} + +func TestFallbackAgentDeliveryPreservesBothErrors(t *testing.T) { + nativeErr := errors.New("exec-stream delivery stalled") + legacyErr := errors.New("read ping: EOF") + + err := fallbackAgentDelivery( + func() error { return nativeErr }, + func() error { return legacyErr }, + ) + require.Error(t, err) + assert.ErrorIs(t, err, nativeErr) + assert.ErrorIs(t, err, legacyErr) + assert.Contains(t, err.Error(), "platform-native agent delivery failed") + assert.Contains(t, err.Error(), "legacy agent injection failed") +} + +func TestFallbackAgentDeliverySuccessfulPaths(t *testing.T) { + for _, tc := range []struct { + name string + nativeErr error + wantLegacy bool + }{ + {name: "native success"}, + {name: "legacy recovery", nativeErr: errors.New("native failed"), wantLegacy: true}, + } { + t.Run(tc.name, func(t *testing.T) { + legacyCalled := false + err := fallbackAgentDelivery( + func() error { return tc.nativeErr }, + func() error { legacyCalled = true; return nil }, + ) + require.NoError(t, err) + assert.Equal(t, tc.wantLegacy, legacyCalled) + }) + } +}