From f48a49902f3912193517b72985b652397c83e8ad Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 10:25:24 -0600 Subject: [PATCH 1/3] ci(e2e): prefetch microsandbox provider assets and retry manifest downloads --- .github/workflows/pr-ci.yml | 58 ++++- e2e/framework/command.go | 10 +- e2e/framework/provider_cache.go | 80 +++++++ e2e/framework/retry.go | 52 +++++ e2e/framework/retry_test.go | 126 ++++++++++ hack/microsandbox/main.go | 19 +- hack/microsandbox/provider_assets.go | 272 ++++++++++++++++++++++ hack/microsandbox/provider_assets_test.go | 262 +++++++++++++++++++++ hack/microsandbox/run.go | 5 + hack/microsandbox/run_test.go | 28 +++ 10 files changed, 906 insertions(+), 6 deletions(-) create mode 100644 e2e/framework/provider_cache.go create mode 100644 hack/microsandbox/provider_assets.go create mode 100644 hack/microsandbox/provider_assets_test.go diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index a4e0bfa57d..103cbb5d9c 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -444,11 +444,59 @@ jobs: GH_CREDENTIAL_USERNAME: x-access-token run: '"$CI_TOOL" e2e run --label "${{ matrix.name }}" --runner "${{ matrix.runner }}" --e2e-dir ./e2e' + prepare-microsandbox-provider: + name: Prepare MicroSandbox provider assets + needs: [changes, build-cli] + if: needs.changes.outputs.go == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: read pinned provider version + id: pin + shell: bash + run: | + versions="$(sed -nE 's#.*devsy-provider-microsandbox@(v[0-9]+\.[0-9]+\.[0-9]+)".*#\1#p' e2e/tests/up/microsandbox_session.go)" + if [ "$(printf '%s\n' "$versions" | grep -c .)" != 1 ]; then + echo "::error::expected exactly one pinned microsandbox provider version, got: $versions" + exit 1 + fi + echo "version=$versions" >> "$GITHUB_OUTPUT" + + - name: download MicroSandbox CI helper + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: devsy-ci-microsandbox-linux + path: ${{ runner.temp }} + + - name: cache provider assets + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/microsandbox-provider + key: ${{ runner.os }}-${{ runner.arch }}-microsandbox-provider-${{ steps.pin.outputs.version }} + + - name: download and verify provider assets + shell: bash + run: | + chmod +x "${RUNNER_TEMP}/devsy-ci-microsandbox" + "${RUNNER_TEMP}/devsy-ci-microsandbox" prepare-provider \ + --version "${{ steps.pin.outputs.version }}" \ + --output-dir "${RUNNER_TEMP}/microsandbox-provider" + + - name: upload provider assets + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: microsandbox-provider-assets + path: ${{ runner.temp }}/microsandbox-provider/ + if-no-files-found: error + retention-days: 1 + # E2E tier, group "microsandbox": the MicroSandbox CI helper owns the run and # its report checks. integration-tests-microsandbox: name: ${{ matrix.title }} - needs: [changes, plan, build-cli] + needs: [changes, plan, build-cli, prepare-microsandbox-provider] if: needs.changes.outputs.go == 'true' strategy: fail-fast: false @@ -522,6 +570,12 @@ jobs: shell: bash run: '"$CI_TOOL" e2e prepare --bin-src "$RUNNER_TEMP/devsy-bin" --e2e-dir ./e2e' + - name: download provider assets + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: microsandbox-provider-assets + path: ${{ runner.temp }}/microsandbox-provider/ + - name: cache microsandbox release bundle (Linux) uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -540,6 +594,7 @@ jobs: GH_CREDENTIAL_USERNAME: x-access-token TEST_TIMEOUT: ${{ matrix.test_timeout }} DEVSY_REQUIRE_MICROSANDBOX: "true" + DEVSY_E2E_PROVIDER_DOWNLOAD_CACHE: ${{ runner.temp }}/microsandbox-provider/cache run: '"${RUNNER_TEMP}/devsy-ci-microsandbox" run --label "${{ matrix.name }}" --expected-count "${{ matrix.expected_spec_count }}" --provider-mode "${{ matrix.provider_mode }}"' - name: upload MicroSandbox test evidence @@ -870,6 +925,7 @@ jobs: - build-cli - integration-tests-unprivileged - integration-tests + - prepare-microsandbox-provider - integration-tests-microsandbox steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/e2e/framework/command.go b/e2e/framework/command.go index 405376cdfa..deab7d6485 100644 --- a/e2e/framework/command.go +++ b/e2e/framework/command.go @@ -343,7 +343,13 @@ func (f *Framework) DevsyStop(ctx context.Context, workspace string) error { func (f *Framework) DevsyProviderAdd(ctx context.Context, args ...string) error { baseArgs := []string{cmdProvider, "add"} baseArgs = append(baseArgs, args...) - _, stderr, err := f.ExecCommandCapture(ctx, baseArgs) + if err := seedProviderDownloadCache(); err != nil { + return fmt.Errorf("devsy provider add failed: %w", err) + } + capture := func(ctx context.Context) (string, string, error) { + return f.ExecCommandCapture(ctx, baseArgs) + } + stderr, err := execWithProviderAddRetry(ctx, capture) if err != nil { if strings.Contains(stderr, "already exists") { providerName := providerNameFromAddArgs(args) @@ -355,7 +361,7 @@ func (f *Framework) DevsyProviderAdd(ctx context.Context, args ...string) error ) } - _, stderr, err = f.ExecCommandCapture(ctx, baseArgs) + stderr, err = execWithProviderAddRetry(ctx, capture) } } if err != nil { diff --git a/e2e/framework/provider_cache.go b/e2e/framework/provider_cache.go new file mode 100644 index 0000000000..a30abf539a --- /dev/null +++ b/e2e/framework/provider_cache.go @@ -0,0 +1,80 @@ +package framework + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/devsy-org/devsy/pkg/config" +) + +const ( + providerDownloadCacheEnv = "DEVSY_E2E_PROVIDER_DOWNLOAD_CACHE" + providerCacheDirPerm = 0o750 + providerCacheFilePerm = 0o755 +) + +var providerCacheRelativeDir = filepath.Join("cache", "providers") + +func seedProviderDownloadCache() error { + source := os.Getenv(providerDownloadCacheEnv) + home := os.Getenv(config.EnvHome) + if source == "" || home == "" { + return nil + } + sourceRoot, err := os.OpenRoot(source) + if err != nil { + return fmt.Errorf("open %s: %w", providerDownloadCacheEnv, err) + } + defer func() { _ = sourceRoot.Close() }() + homeRoot, err := os.OpenRoot(home) + if err != nil { + return fmt.Errorf("open %s: %w", config.EnvHome, err) + } + defer func() { _ = homeRoot.Close() }() + if err := homeRoot.MkdirAll(providerCacheRelativeDir, providerCacheDirPerm); err != nil { + return fmt.Errorf("create provider download cache: %w", err) + } + return copyMissingFiles(source, sourceRoot, homeRoot) +} + +func copyMissingFiles(source string, sourceRoot, homeRoot *os.Root) error { + entries, err := os.ReadDir(source) + if err != nil { + return fmt.Errorf("read %s: %w", providerDownloadCacheEnv, err) + } + for _, entry := range entries { + if !entry.Type().IsRegular() { + continue + } + destination := filepath.Join(providerCacheRelativeDir, entry.Name()) + if err := copyIfMissing(sourceRoot, entry.Name(), homeRoot, destination); err != nil { + return fmt.Errorf("seed provider download cache: %w", err) + } + } + return nil +} + +func copyIfMissing( + sourceRoot *os.Root, name string, destinationRoot *os.Root, destination string, +) (err error) { + if _, statErr := destinationRoot.Stat(destination); statErr == nil { + return nil + } + in, err := sourceRoot.Open(name) + if err != nil { + return err + } + defer func() { err = errors.Join(err, in.Close()) }() + out, err := destinationRoot.OpenFile( + destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, providerCacheFilePerm, + ) + if err != nil { + return err + } + defer func() { err = errors.Join(err, out.Close()) }() + _, err = io.Copy(out, in) + return err +} diff --git a/e2e/framework/retry.go b/e2e/framework/retry.go index 6a1696783c..b7d692a87a 100644 --- a/e2e/framework/retry.go +++ b/e2e/framework/retry.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os/exec" + "regexp" "strings" "time" @@ -32,6 +33,26 @@ var sshBackoff = wait.Backoff{ Jitter: 0.1, } +// providerAddBackoff defines retry timing for provider manifest downloads. +// 5 total attempts with waits of ~5s, ~10s, ~20s, ~40s. +var providerAddBackoff = wait.Backoff{ + Steps: 5, + Duration: 5 * time.Second, + Factor: 2.0, + Jitter: 0.1, +} + +var providerServerErrorPattern = regexp.MustCompile( + `(?i)(received status code|download failed with status|status) 5\d\d`, +) + +var retryableProviderNetworkPatterns = []string{ + "TLS handshake timeout", + "i/o timeout", + "connection reset by peer", + "unexpected EOF", +} + // retryableDockerPatterns are stderr substrings indicating a transient Docker // registry error that is worth retrying. var retryableDockerPatterns = []string{ @@ -93,6 +114,22 @@ func isRetryableDockerError(stderr string) bool { return false } +// isRetryableProviderDownloadError returns true when a provider download +// failed on a server error or a network error. Client errors such as 404 and +// provider validation failures are never retried. +func isRetryableProviderDownloadError(_ error, stderr string) bool { + if providerServerErrorPattern.MatchString(stderr) { + return true + } + lower := strings.ToLower(stderr) + for _, pattern := range retryableProviderNetworkPatterns { + if strings.Contains(lower, strings.ToLower(pattern)) { + return true + } + } + return false +} + type execFunc func(ctx context.Context) (stdout, stderr string, err error) // retryPolicy describes one retry loop. label prefixes the retry log line and @@ -168,6 +205,21 @@ func execWithDockerRetry(ctx context.Context, fn execFunc) (string, string, erro return res.stdout, res.stderr, res.err } +// execWithProviderAddRetry runs fn and retries when the provider manifest +// download failed with a server or network error. +func execWithProviderAddRetry(ctx context.Context, fn execFunc) (string, error) { + res := runWithRetry(ctx, retryPolicy{ + backoff: providerAddBackoff, + retryable: isRetryableProviderDownloadError, + label: "[retry] provider add:", + kind: "retryable provider download error", + }, fn) + if res.gaveUp { + return res.stderr, fmt.Errorf("after %d attempts: %w", res.attempts, res.err) + } + return res.stderr, res.err +} + // execWithSSHRetry runs fn and retries if the error indicates a transient SSH // connection failure. The fn callback must return stdout, stderr, and error so // that stderr can be inspected for SSH-specific patterns. diff --git a/e2e/framework/retry_test.go b/e2e/framework/retry_test.go index 5b900f7ce0..9025a10653 100644 --- a/e2e/framework/retry_test.go +++ b/e2e/framework/retry_test.go @@ -696,3 +696,129 @@ func TestRetryFixed_ContextCanceledOnFinalAttempt(t *testing.T) { "after 2 attempts: context canceled (last error: final attempt failed)", ) } + +const ( + providerServerFailure = "received status code 504 when trying to download x" + providerClientFailure = "received status code 404 when trying to download x" +) + +var errExitOne = errors.New("exit status 1") + +func TestIsRetryableProviderDownloadError(t *testing.T) { + retryable := []string{ + "download: received status code 504 when trying to download https://github.com/o/r/provider.yaml", + "download failed with status 502", + "Get \"https://github.com\": net/http: TLS handshake timeout", + "read tcp: connection reset by peer", + } + for _, stderr := range retryable { + assert.True(t, isRetryableProviderDownloadError(errExitOne, stderr), stderr) + } + permanent := []string{ + "download: received status code 404 when trying to download https://github.com/o/r/provider.yaml", + "download failed with status 403", + "provider microsandbox already exists", + "validate: exec.command is required", + } + for _, stderr := range permanent { + assert.False(t, isRetryableProviderDownloadError(errExitOne, stderr), stderr) + } +} + +func TestExecWithProviderAddRetryRecoversFromServerErrors(t *testing.T) { + original := providerAddBackoff + providerAddBackoff = wait.Backoff{Steps: 4, Duration: time.Millisecond, Factor: 1} + t.Cleanup(func() { providerAddBackoff = original }) + attempts := 0 + _, err := execWithProviderAddRetry( + context.Background(), + func(context.Context) (string, string, error) { + attempts++ + if attempts < 3 { + return "", providerServerFailure, errExitOne + } + return "", "", nil + }, + ) + require.NoError(t, err) + assert.Equal(t, 3, attempts) +} + +func TestExecWithProviderAddRetrySkipsClientErrors(t *testing.T) { + original := providerAddBackoff + providerAddBackoff = wait.Backoff{Steps: 4, Duration: time.Millisecond, Factor: 1} + t.Cleanup(func() { providerAddBackoff = original }) + attempts := 0 + stderr, err := execWithProviderAddRetry( + context.Background(), + func(context.Context) (string, string, error) { + attempts++ + return "", providerClientFailure, errExitOne + }, + ) + require.Error(t, err) + assert.Equal(t, 1, attempts) + assert.Contains(t, stderr, "404") +} + +func TestExecWithProviderAddRetryStopsAtBudget(t *testing.T) { + original := providerAddBackoff + providerAddBackoff = wait.Backoff{Steps: 3, Duration: time.Millisecond, Factor: 1} + t.Cleanup(func() { providerAddBackoff = original }) + attempts := 0 + _, err := execWithProviderAddRetry( + context.Background(), + func(context.Context) (string, string, error) { + attempts++ + return "", providerServerFailure, errExitOne + }, + ) + require.ErrorContains(t, err, "after 3 attempts") + assert.Equal(t, 3, attempts) +} + +func TestSeedProviderDownloadCache(t *testing.T) { + source := t.TempDir() + home := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(source, "abc"), []byte("binary"), 0o600)) + require.NoError(t, os.Mkdir(filepath.Join(source, "nested"), 0o750)) + t.Setenv(providerDownloadCacheEnv, source) + t.Setenv("DEVSY_HOME", home) + + require.NoError(t, seedProviderDownloadCache()) + root, err := os.OpenRoot(home) + require.NoError(t, err) + t.Cleanup(func() { _ = root.Close() }) + got, err := root.ReadFile(filepath.Join("cache", "providers", "abc")) + require.NoError(t, err) + assert.Equal(t, "binary", string(got)) + assert.NoDirExists(t, filepath.Join(home, "cache", "providers", "nested")) +} + +func TestSeedProviderDownloadCacheKeepsExistingFiles(t *testing.T) { + source := t.TempDir() + home := t.TempDir() + cache := filepath.Join(home, "cache", "providers") + require.NoError(t, os.MkdirAll(cache, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(source, "abc"), []byte("new"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(cache, "abc"), []byte("old"), 0o600)) + t.Setenv(providerDownloadCacheEnv, source) + t.Setenv("DEVSY_HOME", home) + + require.NoError(t, seedProviderDownloadCache()) + root, err := os.OpenRoot(cache) + require.NoError(t, err) + t.Cleanup(func() { _ = root.Close() }) + got, err := root.ReadFile("abc") + require.NoError(t, err) + assert.Equal(t, "old", string(got)) +} + +func TestSeedProviderDownloadCacheIsOptional(t *testing.T) { + t.Setenv(providerDownloadCacheEnv, "") + t.Setenv("DEVSY_HOME", t.TempDir()) + require.NoError(t, seedProviderDownloadCache()) + + t.Setenv(providerDownloadCacheEnv, filepath.Join(t.TempDir(), "missing")) + require.Error(t, seedProviderDownloadCache()) +} diff --git a/hack/microsandbox/main.go b/hack/microsandbox/main.go index 4ab924fa9c..382732f59b 100644 --- a/hack/microsandbox/main.go +++ b/hack/microsandbox/main.go @@ -27,16 +27,18 @@ func runCLI(args []string, stdout, stderr io.Writer) int { if len(args) == 0 { _, _ = fmt.Fprintln( stderr, - "usage: devsy-ci-microsandbox ", + "usage: devsy-ci-microsandbox ", ) return 2 } if code, ok := runManagedPolicyCommand(args, stderr); ok { return code } + if code, ok := runProvisionCommand(args, stdout, stderr); ok { + return code + } switch args[0] { - case installLinuxCommand: - return installLinuxCLI(args[1:], stdout, stderr) case runSuiteCommand: return runSuiteCLI(args[1:], stdout, stderr) case verifyReportCommand: @@ -49,6 +51,17 @@ func runCLI(args []string, stdout, stderr io.Writer) int { } } +func runProvisionCommand(args []string, stdout, stderr io.Writer) (int, bool) { + switch args[0] { + case installLinuxCommand: + return installLinuxCLI(args[1:], stdout, stderr), true + case prepareProviderCommand: + return prepareProviderCLI(args[1:], stdout, stderr), true + default: + return 0, false + } +} + // runManagedPolicyCommand dispatches the managed-policy commands and reports // whether args[0] named one of them. func runManagedPolicyCommand(args []string, stderr io.Writer) (int, bool) { diff --git a/hack/microsandbox/provider_assets.go b/hack/microsandbox/provider_assets.go new file mode 100644 index 0000000000..3e7cba1dfa --- /dev/null +++ b/hack/microsandbox/provider_assets.go @@ -0,0 +1,272 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/devsy-org/devsy/pkg/hash" + "github.com/devsy-org/devsy/pkg/provider" +) + +const ( + prepareProviderCommand = "prepare-provider" + providerManifestName = "provider.yaml" + providerCacheDirName = "cache" + runtimeDriverBinary = "DEVSY_RUNTIME_DRIVER" + defaultProviderRepo = "devsy-org/devsy-provider-microsandbox" + defaultGithubBaseURL = "https://github.com" + providerFetchAttempts = 6 + providerFetchBaseDelay = 5 * time.Second + providerFetchMaxDelay = time.Minute + providerFetchTimeout = 5 * time.Minute +) + +type statusError struct { + code int + url string +} + +func (e *statusError) Error() string { + return fmt.Sprintf("received status code %d when trying to download %s", e.code, e.url) +} + +type providerAssets struct { + baseURL string + repository string + version string + outputDir string + goos string + goarch string + attempts int + baseDelay time.Duration + client *http.Client + sleep func(context.Context, time.Duration) error +} + +func prepareProviderCLI(args []string, stdout, stderr io.Writer) int { + flags := flag.NewFlagSet(prepareProviderCommand, flag.ContinueOnError) + flags.SetOutput(stderr) + assets := providerAssets{ + baseURL: defaultGithubBaseURL, + attempts: providerFetchAttempts, + baseDelay: providerFetchBaseDelay, + client: &http.Client{Timeout: providerFetchTimeout}, + sleep: sleepContext, + } + flags.StringVar(&assets.repository, "repository", defaultProviderRepo, "provider repository") + flags.StringVar(&assets.version, "version", "", "pinned provider release tag") + flags.StringVar(&assets.outputDir, "output-dir", "", "directory receiving the assets") + flags.StringVar(&assets.goos, "os", runtime.GOOS, "runtime binary operating system") + flags.StringVar(&assets.goarch, "arch", runtime.GOARCH, "runtime binary architecture") + if err := flags.Parse(args); err != nil { + return 2 + } + if flags.NArg() != 0 || assets.version == "" || assets.outputDir == "" { + _, _ = fmt.Fprintln( + stderr, + "prepare-provider requires --version and --output-dir and takes no arguments", + ) + return 2 + } + if err := assets.prepare(context.Background(), stdout); err != nil { + _, _ = fmt.Fprintf(stderr, "::error::%v\n", err) + return 1 + } + return 0 +} + +func (a *providerAssets) prepare(ctx context.Context, out io.Writer) error { + manifestPath := filepath.Join(a.outputDir, providerManifestName) + config, err := a.loadManifest(ctx, manifestPath) + if err != nil { + return err + } + binary, err := a.runtimeBinary(config) + if err != nil { + return err + } + target := filepath.Join(a.outputDir, providerCacheDirName, hash.String(binary.Path)) + if verifyFileSHA256(target, binary.Checksum) == nil { + _, _ = fmt.Fprintf(out, "provider %s runtime binary already prepared\n", a.version) + return nil + } + if err := a.fetch(ctx, binary.Path, target); err != nil { + return fmt.Errorf("download runtime binary: %w", err) + } + if err := verifyFileSHA256(target, binary.Checksum); err != nil { + return errors.Join(err, removeIfPresent(target)) + } + _, _ = fmt.Fprintf( + out, + "prepared provider %s runtime binary %s/%s\n", + a.version, + a.goos, + a.goarch, + ) + return nil +} + +func (a *providerAssets) loadManifest( + ctx context.Context, + path string, +) (*provider.ProviderConfig, error) { + config, err := parseManifestFile(path) + if err == nil && config.Version == a.version { + return config, nil + } + manifestURL := fmt.Sprintf( + "%s/%s/releases/download/%s/%s", + a.baseURL, a.repository, url.PathEscape(a.version), providerManifestName, + ) + if err := a.fetch(ctx, manifestURL, path); err != nil { + return nil, fmt.Errorf("download provider manifest: %w", err) + } + config, err = parseManifestFile(path) + if err != nil { + return nil, errors.Join(err, removeIfPresent(path)) + } + if config.Version != a.version { + mismatch := fmt.Errorf("provider manifest version %q, want %q", config.Version, a.version) + return nil, errors.Join(mismatch, removeIfPresent(path)) + } + return config, nil +} + +func parseManifestFile(path string) (_ *provider.ProviderConfig, err error) { + file, err := os.Open(path) // #nosec G304 -- path is the helper's own output directory + if err != nil { + return nil, err + } + defer func() { err = errors.Join(err, file.Close()) }() + return provider.ParseProvider(file) +} + +func (a *providerAssets) runtimeBinary( + config *provider.ProviderConfig, +) (*provider.ProviderBinary, error) { + for _, binary := range config.Agent.Binaries[runtimeDriverBinary] { + if binary.OS != a.goos || binary.Arch != a.goarch { + continue + } + if binary.Checksum == "" || !strings.HasPrefix(binary.Path, "https://") { + return nil, fmt.Errorf( + "provider manifest binary for %s/%s needs an https path and a checksum", + a.goos, a.goarch, + ) + } + return binary, nil + } + return nil, fmt.Errorf( + "provider manifest has no %s binary for %s/%s", + runtimeDriverBinary, a.goos, a.goarch, + ) +} + +func (a *providerAssets) fetch(ctx context.Context, rawURL, target string) error { + var err error + for attempt := 1; ; attempt++ { + if err = a.fetchOnce(ctx, rawURL, target); err == nil { + return nil + } + if attempt >= a.attempts || !retryableFetch(ctx, err) { + return fmt.Errorf("after %d attempts: %w", attempt, err) + } + if err := a.sleep(ctx, a.backoff(attempt)); err != nil { + return err + } + } +} + +func (a *providerAssets) backoff(attempt int) time.Duration { + delay := a.baseDelay << (attempt - 1) + if delay <= 0 || delay > providerFetchMaxDelay { + return providerFetchMaxDelay + } + return delay +} + +func retryableFetch(ctx context.Context, err error) bool { + if ctx.Err() != nil { + return false + } + if status, ok := errors.AsType[*statusError](err); ok { + return status.code >= http.StatusInternalServerError + } + return true +} + +func (a *providerAssets) fetchOnce(ctx context.Context, rawURL, target string) (err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return err + } + resp, err := a.client.Do(req) + if err != nil { + return err + } + defer func() { err = errors.Join(err, resp.Body.Close()) }() + if resp.StatusCode != http.StatusOK { + return &statusError{code: resp.StatusCode, url: rawURL} + } + if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { + return err + } + partial, err := os.CreateTemp(filepath.Dir(target), ".download-*") + if err != nil { + return err + } + defer func() { err = errors.Join(err, removeIfPresent(partial.Name())) }() + if _, err := io.Copy(partial, resp.Body); err != nil { + return errors.Join(err, partial.Close()) + } + if err := partial.Close(); err != nil { + return err + } + return os.Rename(partial.Name(), target) +} + +func verifyFileSHA256(path, want string) (err error) { + file, err := os.Open(path) // #nosec G304 -- path is the helper's own output directory + if err != nil { + return err + } + defer func() { err = errors.Join(err, file.Close()) }() + digest := sha256.New() + if _, err := io.Copy(digest, file); err != nil { + return err + } + if got := hex.EncodeToString(digest.Sum(nil)); !strings.EqualFold(got, want) { + return fmt.Errorf("checksum mismatch for %s: got %s, want %s", path, got, want) + } + return nil +} + +func removeIfPresent(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} + +func sleepContext(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/hack/microsandbox/provider_assets_test.go b/hack/microsandbox/provider_assets_test.go new file mode 100644 index 0000000000..ed08b15ce1 --- /dev/null +++ b/hack/microsandbox/provider_assets_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/hash" + "github.com/stretchr/testify/require" +) + +const ( + assetsRepository = "devsy-org/devsy-provider-microsandbox" + assetsVersion = "v9.9.9" + assetsBinaryName = "devsy-runtime-microsandbox-linux-amd64" +) + +type assetsServer struct { + *httptest.Server + mu sync.Mutex + requests map[string]int + failures map[string]int + status int + version string + binary []byte + checksum string +} + +func newAssetsServer(t *testing.T, binary []byte, checksum string) *assetsServer { + t.Helper() + server := &assetsServer{ + requests: map[string]int{}, + failures: map[string]int{}, + status: http.StatusGatewayTimeout, + version: assetsVersion, + binary: binary, + checksum: checksum, + } + server.Server = httptest.NewTLSServer(http.HandlerFunc(server.serve)) + t.Cleanup(server.Close) + return server +} + +func (s *assetsServer) manifest() string { + return fmt.Sprintf(`name: microsandbox +version: %s +exec: + command: "true" +agent: + driver: external + external: + binary: DEVSY_RUNTIME_DRIVER + args: [serve] + binaries: + DEVSY_RUNTIME_DRIVER: + - os: linux + arch: amd64 + path: %s/%s/releases/download/%s/%s + name: %s + checksum: %s +`, s.version, s.URL, assetsRepository, assetsVersion, assetsBinaryName, assetsBinaryName, s.checksum) +} + +func (s *assetsServer) serve(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.requests[r.URL.Path]++ + remaining := s.failures[r.URL.Path] + if remaining > 0 { + s.failures[r.URL.Path]-- + } + s.mu.Unlock() + if remaining > 0 { + http.Error(w, "failure", s.status) + return + } + prefix := "/" + assetsRepository + "/releases/download/" + assetsVersion + "/" + switch r.URL.Path { + case prefix + providerManifestName: + _, _ = w.Write([]byte(s.manifest())) + case prefix + assetsBinaryName: + _, _ = w.Write(s.binary) + default: + http.NotFound(w, r) + } +} + +func (s *assetsServer) count(name string) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.requests["/"+assetsRepository+"/releases/download/"+assetsVersion+"/"+name] +} + +func (s *assetsServer) fail(name string, times, status int) { + s.mu.Lock() + defer s.mu.Unlock() + s.failures["/"+assetsRepository+"/releases/download/"+assetsVersion+"/"+name] = times + s.status = status +} + +func (s *assetsServer) assets(t *testing.T, delays *[]time.Duration) providerAssets { + t.Helper() + return providerAssets{ + baseURL: s.URL, + repository: assetsRepository, + version: assetsVersion, + outputDir: t.TempDir(), + goos: "linux", + goarch: "amd64", + attempts: 4, + baseDelay: time.Second, + client: s.Client(), + sleep: func(_ context.Context, delay time.Duration) error { + *delays = append(*delays, delay) + return nil + }, + } +} + +func binaryFixture() ([]byte, string) { + binary := []byte("runtime-binary") + sum := sha256.Sum256(binary) + return binary, hex.EncodeToString(sum[:]) +} + +func wrongChecksum() string { + sum := sha256.Sum256([]byte("other")) + return hex.EncodeToString(sum[:]) +} + +func (s *assetsServer) cachedBinary(a providerAssets) string { + return filepath.Join( + a.outputDir, providerCacheDirName, + hash.String(fmt.Sprintf( + "%s/%s/releases/download/%s/%s", + s.URL, assetsRepository, assetsVersion, assetsBinaryName, + )), + ) +} + +func TestPrepareProviderRetriesGatewayTimeouts(t *testing.T) { + binary, checksum := binaryFixture() + server := newAssetsServer(t, binary, checksum) + server.fail(providerManifestName, 2, http.StatusGatewayTimeout) + var delays []time.Duration + assets := server.assets(t, &delays) + + require.NoError(t, assets.prepare(context.Background(), &bytes.Buffer{})) + require.Equal(t, 3, server.count(providerManifestName)) + require.Equal(t, 1, server.count(assetsBinaryName)) + require.Equal(t, []time.Duration{time.Second, 2 * time.Second}, delays) + got, err := os.ReadFile(server.cachedBinary(assets)) + require.NoError(t, err) + require.Equal(t, binary, got) +} + +func TestPrepareProviderRetriesBinaryServerErrors(t *testing.T) { + binary, checksum := binaryFixture() + server := newAssetsServer(t, binary, checksum) + server.fail(assetsBinaryName, 3, http.StatusBadGateway) + var delays []time.Duration + assets := server.assets(t, &delays) + + require.NoError(t, assets.prepare(context.Background(), &bytes.Buffer{})) + require.Equal(t, 4, server.count(assetsBinaryName)) +} + +func TestPrepareProviderStopsAfterAttemptBudget(t *testing.T) { + binary, checksum := binaryFixture() + server := newAssetsServer(t, binary, checksum) + server.fail(assetsBinaryName, 100, http.StatusGatewayTimeout) + var delays []time.Duration + assets := server.assets(t, &delays) + + err := assets.prepare(context.Background(), &bytes.Buffer{}) + require.ErrorContains(t, err, "after 4 attempts") + require.ErrorContains(t, err, "status code 504") + require.Equal(t, 4, server.count(assetsBinaryName)) +} + +func TestPrepareProviderDoesNotRetryClientErrors(t *testing.T) { + binary, checksum := binaryFixture() + server := newAssetsServer(t, binary, checksum) + server.fail(providerManifestName, 100, http.StatusNotFound) + var delays []time.Duration + assets := server.assets(t, &delays) + + err := assets.prepare(context.Background(), &bytes.Buffer{}) + require.ErrorContains(t, err, "status code 404") + require.Equal(t, 1, server.count(providerManifestName)) + require.Empty(t, delays) +} + +func TestPrepareProviderRejectsChecksumMismatchWithoutRetry(t *testing.T) { + binary, _ := binaryFixture() + server := newAssetsServer(t, binary, wrongChecksum()) + var delays []time.Duration + assets := server.assets(t, &delays) + + err := assets.prepare(context.Background(), &bytes.Buffer{}) + require.ErrorContains(t, err, "checksum mismatch") + require.Equal(t, 1, server.count(assetsBinaryName)) + require.NoFileExists(t, server.cachedBinary(assets)) +} + +func TestPrepareProviderRejectsVersionMismatch(t *testing.T) { + binary, checksum := binaryFixture() + server := newAssetsServer(t, binary, checksum) + server.version = "v9.9.8" + var delays []time.Duration + assets := server.assets(t, &delays) + + err := assets.prepare(context.Background(), &bytes.Buffer{}) + require.ErrorContains(t, err, `provider manifest version "v9.9.8", want "v9.9.9"`) + require.Zero(t, server.count(assetsBinaryName)) + require.NoFileExists(t, filepath.Join(assets.outputDir, providerManifestName)) +} + +func TestPrepareProviderReusesPreparedAssets(t *testing.T) { + binary, checksum := binaryFixture() + server := newAssetsServer(t, binary, checksum) + var delays []time.Duration + assets := server.assets(t, &delays) + require.NoError(t, assets.prepare(context.Background(), &bytes.Buffer{})) + + server.fail(providerManifestName, 100, http.StatusGatewayTimeout) + server.fail(assetsBinaryName, 100, http.StatusGatewayTimeout) + var out bytes.Buffer + require.NoError(t, assets.prepare(context.Background(), &out)) + require.Equal(t, 1, server.count(providerManifestName)) + require.Equal(t, 1, server.count(assetsBinaryName)) + require.Contains(t, out.String(), "already prepared") +} + +func TestPrepareProviderRefetchesCorruptedBinary(t *testing.T) { + binary, checksum := binaryFixture() + server := newAssetsServer(t, binary, checksum) + var delays []time.Duration + assets := server.assets(t, &delays) + require.NoError(t, assets.prepare(context.Background(), &bytes.Buffer{})) + require.NoError(t, os.WriteFile(server.cachedBinary(assets), []byte("corrupt"), 0o600)) + + require.NoError(t, assets.prepare(context.Background(), &bytes.Buffer{})) + require.Equal(t, 2, server.count(assetsBinaryName)) + got, err := os.ReadFile(server.cachedBinary(assets)) + require.NoError(t, err) + require.Equal(t, binary, got) +} + +func TestPrepareProviderCLIRequiresArguments(t *testing.T) { + var stderr bytes.Buffer + require.Equal(t, 2, prepareProviderCLI(nil, &bytes.Buffer{}, &stderr)) + require.Contains(t, stderr.String(), "--version and --output-dir") +} diff --git a/hack/microsandbox/run.go b/hack/microsandbox/run.go index 41151364fd..adfdc6a8ab 100644 --- a/hack/microsandbox/run.go +++ b/hack/microsandbox/run.go @@ -33,6 +33,8 @@ var requiredSuiteEnvironmentNames = []string{ "PATH", } +const providerDownloadCacheEnv = "DEVSY_E2E_PROVIDER_DOWNLOAD_CACHE" + var suiteLabelPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9-]*$`) func runSuite( @@ -243,6 +245,9 @@ func configuredKubeconfig() (string, error) { func suiteSpecificEnvironment(cfg suiteConfig) []string { var values []string + if value := os.Getenv(providerDownloadCacheEnv); value != "" { + values = append(values, providerDownloadCacheEnv+"="+value) + } if cfg.label == managedPolicyLabel { values = append(values, "DEVSY_REQUIRE_MSB_MANAGED_POLICY_FIXTURE=true", diff --git a/hack/microsandbox/run_test.go b/hack/microsandbox/run_test.go index 96c9d1e12d..2b12a0d03e 100644 --- a/hack/microsandbox/run_test.go +++ b/hack/microsandbox/run_test.go @@ -584,3 +584,31 @@ func exitWithStatus(status int) error { } return exec.Command("sh", "-c", "exit "+strconv.Itoa(status)).Run() } + +func TestProviderDownloadCacheIsForwardedToEverySuite(t *testing.T) { + for _, label := range []string{"up-provider-microsandbox-images", performanceSuiteLabel} { + cfg, _ := newSuiteRunFixture(t, label) + t.Setenv(providerDownloadCacheEnv, "/prepared/cache") + args, err := suiteSudoArguments(cfg) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(args, providerDownloadCacheEnv+"=/prepared/cache") { + t.Errorf("%s was not forwarded to %s", providerDownloadCacheEnv, label) + } + } +} + +func TestProviderDownloadCacheIsOmittedWhenUnset(t *testing.T) { + cfg, _ := newSuiteRunFixture(t, "up-provider-microsandbox-images") + t.Setenv(providerDownloadCacheEnv, "") + args, err := suiteSudoArguments(cfg) + if err != nil { + t.Fatal(err) + } + for _, arg := range args { + if strings.HasPrefix(arg, providerDownloadCacheEnv) { + t.Errorf("unexpected argument %q", arg) + } + } +} From 8ba0e7d969a229702c5c443a9c918c87ffac91cd Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 11:28:56 -0600 Subject: [PATCH 2/3] fix(ci): harden microsandbox provider asset retries and cache seeding --- .github/workflows/pr-ci.yml | 1 - e2e/framework/command.go | 2 +- e2e/framework/provider_cache.go | 36 ++++++++- e2e/framework/retry.go | 1 + e2e/framework/retry_test.go | 15 ++++ hack/microsandbox/provider_assets.go | 49 ++++++++++-- hack/microsandbox/provider_assets_test.go | 95 +++++++++++++++++++++-- 7 files changed, 179 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 103cbb5d9c..3df0750293 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -490,7 +490,6 @@ jobs: name: microsandbox-provider-assets path: ${{ runner.temp }}/microsandbox-provider/ if-no-files-found: error - retention-days: 1 # E2E tier, group "microsandbox": the MicroSandbox CI helper owns the run and # its report checks. diff --git a/e2e/framework/command.go b/e2e/framework/command.go index deab7d6485..1045af890a 100644 --- a/e2e/framework/command.go +++ b/e2e/framework/command.go @@ -365,7 +365,7 @@ func (f *Framework) DevsyProviderAdd(ctx context.Context, args ...string) error } } if err != nil { - return fmt.Errorf("devsy provider add failed: %s", stderr) + return fmt.Errorf("devsy provider add failed: %w: %s", err, stderr) } return nil } diff --git a/e2e/framework/provider_cache.go b/e2e/framework/provider_cache.go index a30abf539a..c3b0f27af5 100644 --- a/e2e/framework/provider_cache.go +++ b/e2e/framework/provider_cache.go @@ -6,12 +6,14 @@ import ( "io" "os" "path/filepath" + "time" "github.com/devsy-org/devsy/pkg/config" ) const ( providerDownloadCacheEnv = "DEVSY_E2E_PROVIDER_DOWNLOAD_CACHE" + providerCacheHomePerm = 0o700 providerCacheDirPerm = 0o750 providerCacheFilePerm = 0o755 ) @@ -29,7 +31,7 @@ func seedProviderDownloadCache() error { return fmt.Errorf("open %s: %w", providerDownloadCacheEnv, err) } defer func() { _ = sourceRoot.Close() }() - homeRoot, err := os.OpenRoot(home) + homeRoot, err := openHomeRoot(home) if err != nil { return fmt.Errorf("open %s: %w", config.EnvHome, err) } @@ -40,6 +42,23 @@ func seedProviderDownloadCache() error { return copyMissingFiles(source, sourceRoot, homeRoot) } +func openHomeRoot(home string) (*os.Root, error) { + parent, err := os.OpenRoot(filepath.Dir(home)) + if err != nil { + return nil, err + } + defer func() { _ = parent.Close() }() + name := filepath.Base(home) + if err := parent.Mkdir( + name, + providerCacheHomePerm, + ); err != nil && + !errors.Is(err, os.ErrExist) { + return nil, err + } + return parent.OpenRoot(name) +} + func copyMissingFiles(source string, sourceRoot, homeRoot *os.Root) error { entries, err := os.ReadDir(source) if err != nil { @@ -68,13 +87,22 @@ func copyIfMissing( return err } defer func() { err = errors.Join(err, in.Close()) }() + partial := fmt.Sprintf("%s.%d.%d.partial", destination, os.Getpid(), time.Now().UnixNano()) out, err := destinationRoot.OpenFile( - destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, providerCacheFilePerm, + partial, os.O_WRONLY|os.O_CREATE|os.O_EXCL, providerCacheFilePerm, ) if err != nil { return err } - defer func() { err = errors.Join(err, out.Close()) }() _, err = io.Copy(out, in) - return err + if closeErr := out.Close(); err == nil { + err = closeErr + } + if err != nil { + return errors.Join(err, destinationRoot.Remove(partial)) + } + if err := destinationRoot.Rename(partial, destination); err != nil { + return errors.Join(err, destinationRoot.Remove(partial)) + } + return nil } diff --git a/e2e/framework/retry.go b/e2e/framework/retry.go index b7d692a87a..e22b8a2d97 100644 --- a/e2e/framework/retry.go +++ b/e2e/framework/retry.go @@ -51,6 +51,7 @@ var retryableProviderNetworkPatterns = []string{ "i/o timeout", "connection reset by peer", "unexpected EOF", + "stalled: no progress within timeout", } // retryableDockerPatterns are stderr substrings indicating a transient Docker diff --git a/e2e/framework/retry_test.go b/e2e/framework/retry_test.go index 9025a10653..20f931656b 100644 --- a/e2e/framework/retry_test.go +++ b/e2e/framework/retry_test.go @@ -710,6 +710,7 @@ func TestIsRetryableProviderDownloadError(t *testing.T) { "download failed with status 502", "Get \"https://github.com\": net/http: TLS handshake timeout", "read tcp: connection reset by peer", + "download file: stalled: no progress within timeout", } for _, stderr := range retryable { assert.True(t, isRetryableProviderDownloadError(errExitOne, stderr), stderr) @@ -822,3 +823,17 @@ func TestSeedProviderDownloadCacheIsOptional(t *testing.T) { t.Setenv(providerDownloadCacheEnv, filepath.Join(t.TempDir(), "missing")) require.Error(t, seedProviderDownloadCache()) } + +func TestSeedProviderDownloadCacheCreatesHome(t *testing.T) { + source := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(source, "abc"), []byte("binary"), 0o600)) + home := filepath.Join(t.TempDir(), "new-home") + t.Setenv(providerDownloadCacheEnv, source) + t.Setenv("DEVSY_HOME", home) + + require.NoError(t, seedProviderDownloadCache()) + assert.FileExists(t, filepath.Join(home, "cache", "providers", "abc")) + entries, err := os.ReadDir(filepath.Join(home, "cache", "providers")) + require.NoError(t, err) + assert.Len(t, entries, 1) +} diff --git a/hack/microsandbox/provider_assets.go b/hack/microsandbox/provider_assets.go index 3e7cba1dfa..da0929286a 100644 --- a/hack/microsandbox/provider_assets.go +++ b/hack/microsandbox/provider_assets.go @@ -42,6 +42,12 @@ func (e *statusError) Error() string { return fmt.Sprintf("received status code %d when trying to download %s", e.code, e.url) } +type localError struct{ err error } + +func (e *localError) Error() string { return e.err.Error() } + +func (e *localError) Unwrap() error { return e.err } + type providerAssets struct { baseURL string repository string @@ -123,7 +129,7 @@ func (a *providerAssets) loadManifest( path string, ) (*provider.ProviderConfig, error) { config, err := parseManifestFile(path) - if err == nil && config.Version == a.version { + if err == nil && config.Version == a.version && a.boundToRelease(config) { return config, nil } manifestURL := fmt.Sprintf( @@ -137,13 +143,30 @@ func (a *providerAssets) loadManifest( if err != nil { return nil, errors.Join(err, removeIfPresent(path)) } - if config.Version != a.version { - mismatch := fmt.Errorf("provider manifest version %q, want %q", config.Version, a.version) + if config.Version != a.version || !a.boundToRelease(config) { + mismatch := fmt.Errorf( + "provider manifest version %q does not match release %s%s", + config.Version, a.releasePrefix(), providerManifestName, + ) return nil, errors.Join(mismatch, removeIfPresent(path)) } return config, nil } +func (a *providerAssets) releasePrefix() string { + return fmt.Sprintf( + "%s/%s/releases/download/%s/", + a.baseURL, + a.repository, + url.PathEscape(a.version), + ) +} + +func (a *providerAssets) boundToRelease(config *provider.ProviderConfig) bool { + binary, err := a.runtimeBinary(config) + return err == nil && strings.HasPrefix(binary.Path, a.releasePrefix()) +} + func parseManifestFile(path string) (_ *provider.ProviderConfig, err error) { file, err := os.Open(path) // #nosec G304 -- path is the helper's own output directory if err != nil { @@ -201,6 +224,9 @@ func retryableFetch(ctx context.Context, err error) bool { if ctx.Err() != nil { return false } + if _, ok := errors.AsType[*localError](err); ok { + return false + } if status, ok := errors.AsType[*statusError](err); ok { return status.code >= http.StatusInternalServerError } @@ -220,21 +246,28 @@ func (a *providerAssets) fetchOnce(ctx context.Context, rawURL, target string) ( if resp.StatusCode != http.StatusOK { return &statusError{code: resp.StatusCode, url: rawURL} } + return saveBody(resp.Body, target) +} + +func saveBody(body io.Reader, target string) (err error) { if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { - return err + return &localError{err} } partial, err := os.CreateTemp(filepath.Dir(target), ".download-*") if err != nil { - return err + return &localError{err} } defer func() { err = errors.Join(err, removeIfPresent(partial.Name())) }() - if _, err := io.Copy(partial, resp.Body); err != nil { + if _, err := io.Copy(partial, body); err != nil { return errors.Join(err, partial.Close()) } if err := partial.Close(); err != nil { - return err + return &localError{err} } - return os.Rename(partial.Name(), target) + if err := os.Rename(partial.Name(), target); err != nil { + return &localError{err} + } + return nil } func verifyFileSHA256(path, want string) (err error) { diff --git a/hack/microsandbox/provider_assets_test.go b/hack/microsandbox/provider_assets_test.go index ed08b15ce1..1ce4eefb25 100644 --- a/hack/microsandbox/provider_assets_test.go +++ b/hack/microsandbox/provider_assets_test.go @@ -10,17 +10,21 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "sync" "testing" "time" "github.com/devsy-org/devsy/pkg/hash" + "github.com/devsy-org/devsy/pkg/provider" "github.com/stretchr/testify/require" ) const ( assetsRepository = "devsy-org/devsy-provider-microsandbox" assetsVersion = "v9.9.9" + osLinux = "linux" + archAmd64 = "amd64" assetsBinaryName = "devsy-runtime-microsandbox-linux-amd64" ) @@ -31,6 +35,8 @@ type assetsServer struct { failures map[string]int status int version string + goos string + goarch string binary []byte checksum string } @@ -42,6 +48,8 @@ func newAssetsServer(t *testing.T, binary []byte, checksum string) *assetsServer failures: map[string]int{}, status: http.StatusGatewayTimeout, version: assetsVersion, + goos: osLinux, + goarch: archAmd64, binary: binary, checksum: checksum, } @@ -62,12 +70,12 @@ agent: args: [serve] binaries: DEVSY_RUNTIME_DRIVER: - - os: linux - arch: amd64 + - os: %s + arch: %s path: %s/%s/releases/download/%s/%s name: %s checksum: %s -`, s.version, s.URL, assetsRepository, assetsVersion, assetsBinaryName, assetsBinaryName, s.checksum) +`, s.version, s.goos, s.goarch, s.URL, assetsRepository, assetsVersion, assetsBinaryName, assetsBinaryName, s.checksum) } func (s *assetsServer) serve(w http.ResponseWriter, r *http.Request) { @@ -113,8 +121,8 @@ func (s *assetsServer) assets(t *testing.T, delays *[]time.Duration) providerAss repository: assetsRepository, version: assetsVersion, outputDir: t.TempDir(), - goos: "linux", - goarch: "amd64", + goos: s.goos, + goarch: s.goarch, attempts: 4, baseDelay: time.Second, client: s.Client(), @@ -219,7 +227,7 @@ func TestPrepareProviderRejectsVersionMismatch(t *testing.T) { assets := server.assets(t, &delays) err := assets.prepare(context.Background(), &bytes.Buffer{}) - require.ErrorContains(t, err, `provider manifest version "v9.9.8", want "v9.9.9"`) + require.ErrorContains(t, err, `provider manifest version "v9.9.8" does not match release`) require.Zero(t, server.count(assetsBinaryName)) require.NoFileExists(t, filepath.Join(assets.outputDir, providerManifestName)) } @@ -260,3 +268,78 @@ func TestPrepareProviderCLIRequiresArguments(t *testing.T) { require.Equal(t, 2, prepareProviderCLI(nil, &bytes.Buffer{}, &stderr)) require.Contains(t, stderr.String(), "--version and --output-dir") } + +func TestPrepareProviderDoesNotRetryLocalFileErrors(t *testing.T) { + binary, checksum := binaryFixture() + server := newAssetsServer(t, binary, checksum) + var delays []time.Duration + assets := server.assets(t, &delays) + blocker := filepath.Join(t.TempDir(), "file") + require.NoError(t, os.WriteFile(blocker, nil, 0o600)) + assets.outputDir = filepath.Join(blocker, "nested") + + require.Error(t, assets.prepare(context.Background(), &bytes.Buffer{})) + require.Equal(t, 1, server.count(providerManifestName)) + require.Empty(t, delays) +} + +func TestPrepareProviderRefetchesManifestFromAnotherRepository(t *testing.T) { + binary, checksum := binaryFixture() + server := newAssetsServer(t, binary, checksum) + var delays []time.Duration + assets := server.assets(t, &delays) + other := fmt.Sprintf( + "name: microsandbox\nversion: %s\nexec:\n command: \"true\"\nagent:\n driver: external\n"+ + " external:\n binary: DEVSY_RUNTIME_DRIVER\n binaries:\n DEVSY_RUNTIME_DRIVER:\n"+ + " - os: linux\n arch: amd64\n checksum: %s\n"+ + " path: https://github.com/other/repo/releases/download/%s/runtime\n", + assetsVersion, checksum, assetsVersion, + ) + require.NoError( + t, + os.WriteFile(filepath.Join(assets.outputDir, providerManifestName), []byte(other), 0o600), + ) + + require.NoError(t, assets.prepare(context.Background(), &bytes.Buffer{})) + require.Equal(t, 1, server.count(providerManifestName)) + require.FileExists(t, server.cachedBinary(assets)) +} + +func TestPreparedCacheSatisfiesProviderDownloader(t *testing.T) { + binary, checksum := binaryFixture() + server := newAssetsServer(t, binary, checksum) + var delays []time.Duration + server.goos, server.goarch = runtime.GOOS, runtime.GOARCH + assets := server.assets(t, &delays) + config, err := assets.loadManifest( + context.Background(), + filepath.Join(assets.outputDir, providerManifestName), + ) + require.NoError(t, err) + require.NoError(t, assets.prepare(context.Background(), &bytes.Buffer{})) + + home := t.TempDir() + cache := filepath.Join(home, "cache", "providers") + require.NoError(t, os.MkdirAll(cache, 0o750)) + entries, err := os.ReadDir(filepath.Join(assets.outputDir, providerCacheDirName)) + require.NoError(t, err) + for _, entry := range entries { + data, err := os.ReadFile( + filepath.Join(assets.outputDir, providerCacheDirName, entry.Name()), + ) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(cache, entry.Name()), data, 0o600)) + } + t.Setenv("DEVSY_HOME", home) + server.Close() + + paths, err := provider.DownloadBinaries( + context.Background(), + config.Agent.Binaries, + t.TempDir(), + ) + require.NoError(t, err) + got, err := os.ReadFile(paths[runtimeDriverBinary]) + require.NoError(t, err) + require.Equal(t, binary, got) +} From c992d958e5e5fbd530af5251d974c36d1aef53f7 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 12:37:49 -0600 Subject: [PATCH 3/3] fix(ci): treat provider asset write failures as non-retryable --- hack/microsandbox/provider_assets.go | 12 +++++++++++- hack/microsandbox/provider_assets_test.go | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/hack/microsandbox/provider_assets.go b/hack/microsandbox/provider_assets.go index da0929286a..59585bc576 100644 --- a/hack/microsandbox/provider_assets.go +++ b/hack/microsandbox/provider_assets.go @@ -48,6 +48,16 @@ func (e *localError) Error() string { return e.err.Error() } func (e *localError) Unwrap() error { return e.err } +type localWriter struct{ io.Writer } + +func (w localWriter) Write(p []byte) (int, error) { + n, err := w.Writer.Write(p) + if err != nil { + err = &localError{err} + } + return n, err +} + type providerAssets struct { baseURL string repository string @@ -258,7 +268,7 @@ func saveBody(body io.Reader, target string) (err error) { return &localError{err} } defer func() { err = errors.Join(err, removeIfPresent(partial.Name())) }() - if _, err := io.Copy(partial, body); err != nil { + if _, err := io.Copy(localWriter{partial}, body); err != nil { return errors.Join(err, partial.Close()) } if err := partial.Close(); err != nil { diff --git a/hack/microsandbox/provider_assets_test.go b/hack/microsandbox/provider_assets_test.go index 1ce4eefb25..7f1df4dcc5 100644 --- a/hack/microsandbox/provider_assets_test.go +++ b/hack/microsandbox/provider_assets_test.go @@ -5,12 +5,15 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" + "io" "net/http" "net/http/httptest" "os" "path/filepath" "runtime" + "strings" "sync" "testing" "time" @@ -343,3 +346,14 @@ func TestPreparedCacheSatisfiesProviderDownloader(t *testing.T) { require.NoError(t, err) require.Equal(t, binary, got) } + +type diskFullWriter struct{} + +func (diskFullWriter) Write([]byte) (int, error) { return 0, errors.New("disk full") } + +func TestLocalWriterMarksWriteFailuresNonRetryable(t *testing.T) { + _, err := io.Copy(localWriter{diskFullWriter{}}, strings.NewReader("data")) + require.Error(t, err) + require.False(t, retryableFetch(context.Background(), err)) + require.True(t, retryableFetch(context.Background(), io.ErrUnexpectedEOF)) +}