From c204f833f3e0090472acb2a0f81518ce284e07f4 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 19:20:56 -0600 Subject: [PATCH 1/4] ci(e2e): test the latest microsandbox provider release --- .github/workflows/pr-ci.yml | 4 ++-- AGENTS.md | 13 +++++++++++++ e2e/tests/up/microsandbox_session.go | 6 +++--- .../up/provider_microsandbox_compatibility.go | 17 ++++++++++------- hack/microsandbox/install.go | 8 ++++---- hack/microsandbox/install_test.go | 16 ++++++++-------- renovate.json | 9 +++++++++ 7 files changed, 49 insertions(+), 24 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 3df0750293..f7ee35a25c 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -578,8 +578,8 @@ jobs: - name: cache microsandbox release bundle (Linux) uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: ${{ runner.temp }}/microsandbox-linux-x86_64-v0.7.7.tar.gz - key: ${{ runner.os }}-${{ runner.arch }}-microsandbox-bundle-v0.7.7 + path: ${{ runner.temp }}/microsandbox-linux-x86_64-v0.7.13.tar.gz + key: ${{ runner.os }}-${{ runner.arch }}-microsandbox-bundle-v0.7.13 - name: install checksum-pinned microsandbox (Linux) timeout-minutes: 5 diff --git a/AGENTS.md b/AGENTS.md index a60735be0a..a48d33259c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,6 +79,19 @@ Devsy uses [Ginkgo](https://onsi.github.io/ginkgo/) for Go E2E and integration t --- +## MicroSandbox E2E Pins + +`e2e/tests/up/microsandbox_session.go` pins the external provider release. Renovate +tracks that source. When updating it, check the provider's runtime requirement +and qualified executable and firmware pair. Update `hack/microsandbox/install.go`, +its tests, and the release bundle cache in `.github/workflows/pr-ci.yml` together. +The external provider v1.0.5 uses msb 0.7.13 and `libkrunfw.so.5.6.1`. Keep the +built-in provider's compatibility expectations separate. The prefetch helper +reads runtime asset names and checksums from the published provider manifest. +Run the real VM specs on Linux with KVM before claiming parity. + +--- + ## Code Style and Quality ### Go Code Style diff --git a/e2e/tests/up/microsandbox_session.go b/e2e/tests/up/microsandbox_session.go index 77601e3ed9..254d5abef0 100644 --- a/e2e/tests/up/microsandbox_session.go +++ b/e2e/tests/up/microsandbox_session.go @@ -19,7 +19,7 @@ const ( microsandboxResourceBuiltin = provider.DeprecatedMicrosandboxSource osLinux = "linux" osDarwin = "darwin" - microsandboxExternalProvider = "github.com/devsy-org/devsy-provider-microsandbox@v0.1.10" + microsandboxExternalProvider = "github.com/devsy-org/devsy-provider-microsandbox@v1.0.5" microsandboxRootUser = "root" ) @@ -52,9 +52,9 @@ func checkMicrosandboxVersion(ctx context.Context) { version, err := semver.ParseTolerant( strings.TrimPrefix(strings.TrimSpace(string(output)), "msb "), ) - if err != nil || version.LT(semver.Version{Major: 0, Minor: 7, Patch: 7}) { + if err != nil || version.LT(semver.Version{Major: 0, Minor: 7, Patch: 13}) { microsandboxUnavailable( - "microsandbox parity requires msb v0.7.7 or newer: " + string(output), + "microsandbox parity requires msb v0.7.13 or newer: " + string(output), ) } } diff --git a/e2e/tests/up/provider_microsandbox_compatibility.go b/e2e/tests/up/provider_microsandbox_compatibility.go index c1c7047dd5..65a7069876 100644 --- a/e2e/tests/up/provider_microsandbox_compatibility.go +++ b/e2e/tests/up/provider_microsandbox_compatibility.go @@ -71,6 +71,7 @@ type compatibilityFixture struct { hostPath, sourcePath, sandbox string uid, label string workspace *providerpkg.Workspace + minimumVersion string workspaceBefore compatibilityWorkspaceState vmBefore compatibilityVMState sourceBefore compatibilityFileState @@ -83,7 +84,6 @@ func newCompatibilityFixture( ) *compatibilityFixture { realMSB := captureCompatibilityMSB() ownedHome := ginkgo.GinkgoT().TempDir() - // Pinned msb0.7.7 accepts an absent private config as an empty defaults patch. msbHome := filepath.Join(ownedHome, "microsandbox") ginkgo.GinkgoT().Setenv("MSB_HOME", msbHome) ginkgo.GinkgoT().Setenv("MSB_CONFIG_PATH", filepath.Join(msbHome, "config.json")) @@ -95,7 +95,11 @@ func newCompatibilityFixture( shim := installCompatibilityMSBShim(ginkgo.GinkgoT(), realMSB, ownedHome) fixture := &compatibilityFixture{ framework: f, shim: shim, hostPath: tempDir, - uid: encoding.CreateNewUID("", filepath.Base(tempDir)), + uid: encoding.CreateNewUID("", filepath.Base(tempDir)), + minimumVersion: "0.7.2", + } + if selection.runtime == providerpkg.ExternalDriver { + fixture.minimumVersion = "0.7.13" } fixture.sandbox = "devsy-" + fixture.uid fixture.label = "devsy.compatibility-parity=" + fixture.uid @@ -188,7 +192,7 @@ func (fixture *compatibilityFixture) rejectVersionFault(ctx context.Context, fau stdout, stderr, err := fixture.framework.DevsyUpStreams(ctx, fixture.hostPath, append(fixture.upArgs(), "--recreate")...) gomega.Expect(err).To(gomega.HaveOccurred(), "stdout: %s\nstderr: %s", stdout, stderr) - assertCompatibilityDiagnostic(fault, stdout+"\n"+stderr) + assertCompatibilityDiagnostic(fault, stdout+"\n"+stderr, fixture.minimumVersion) calls := fixture.shim.calls() foundFault := false for _, call := range calls { @@ -204,12 +208,11 @@ func (fixture *compatibilityFixture) rejectVersionFault(ctx context.Context, fau fixture.assertPreservedState(ctx) } -func assertCompatibilityDiagnostic(fault, output string) { - // The external provider may fail in Info; builtin fails in provisioning. - // Assert the underlying diagnostic without demanding identical RPC phases. +func assertCompatibilityDiagnostic(fault, output, minimumVersion string) { switch fault { case "old": - gomega.Expect(output).To(gomega.ContainSubstring("v0.7.2 or newer is required")) + gomega.Expect(output). + To(gomega.ContainSubstring("v" + minimumVersion + " or newer is required")) case "malformed": gomega.Expect(output).To(gomega.ContainSubstring("unable to parse microsandbox version")) case "failing": diff --git a/hack/microsandbox/install.go b/hack/microsandbox/install.go index 8c08b3c698..495885f4a2 100644 --- a/hack/microsandbox/install.go +++ b/hack/microsandbox/install.go @@ -14,11 +14,11 @@ import ( ) const ( - microsandboxVersion = "v0.7.7" + microsandboxVersion = "v0.7.13" microsandboxURL = "https://github.com/superradcompany/microsandbox/releases/download/" + - "v0.7.7/microsandbox-linux-x86_64.tar.gz" - microsandboxSHA256 = "b3cc4a5e3f52dfdd938a6f67ac4a9a959ddfe304bab56de4964044b8613f01bb" - microsandboxArchive = "microsandbox-linux-x86_64-v0.7.7.tar.gz" + "v0.7.13/microsandbox-linux-x86_64.tar.gz" + microsandboxSHA256 = "c41aeefc050e97f42cd55ce55cc139efc13c6ee2dc371e8d30c325a305a5c3b1" + microsandboxArchive = "microsandbox-linux-x86_64-v0.7.13.tar.gz" microsandboxLibrary = "libkrunfw.so.5.6.1" ) diff --git a/hack/microsandbox/install_test.go b/hack/microsandbox/install_test.go index b4ba93955f..f31afd41cf 100644 --- a/hack/microsandbox/install_test.go +++ b/hack/microsandbox/install_test.go @@ -24,18 +24,18 @@ const ( installerSudoCommand = "sudo" installerTestCommand = "test" installerTestURL = "https://github.com/superradcompany/microsandbox/releases/download/" + - "v0.7.7/microsandbox-linux-x86_64.tar.gz" + "v0.7.13/microsandbox-linux-x86_64.tar.gz" ) func TestInstallLinuxPinnedBundle(t *testing.T) { - require.Equal(t, "v0.7.7", microsandboxVersion) + require.Equal(t, "v0.7.13", microsandboxVersion) require.Equal(t, installerTestURL, microsandboxURL) require.Equal( t, - "b3cc4a5e3f52dfdd938a6f67ac4a9a959ddfe304bab56de4964044b8613f01bb", + "c41aeefc050e97f42cd55ce55cc139efc13c6ee2dc371e8d30c325a305a5c3b1", microsandboxSHA256, ) - require.Equal(t, "microsandbox-linux-x86_64-v0.7.7.tar.gz", microsandboxArchive) + require.Equal(t, "microsandbox-linux-x86_64-v0.7.13.tar.gz", microsandboxArchive) } func TestInstallLinuxBundleUsesCachedArchiveAndInstallsFiles(t *testing.T) { @@ -47,7 +47,7 @@ func TestInstallLinuxBundleUsesCachedArchiveAndInstallsFiles(t *testing.T) { var stdout, stderr bytes.Buffer require.NoError(t, fixture.installer(context.Background(), runner.run, commandOutput{stdout: &stdout, stderr: &stderr}).install(), stderr.String()) - require.Equal(t, "MicroSandbox v0.7.7\n", stdout.String()) + require.Equal(t, "MicroSandbox v0.7.13\n", stdout.String()) require.Equal( t, []string{ @@ -229,7 +229,7 @@ func (runner *installerTestRunner) run(cmd *exec.Cmd) error { case installerTarCommand: return installerExtractFixture(cmd.Args[1:]) case installerBinaryName: - _, _ = io.WriteString(cmd.Stdout, "MicroSandbox v0.7.7\n") + _, _ = io.WriteString(cmd.Stdout, "MicroSandbox v0.7.13\n") case installerSudoCommand: return runner.checkKVM(cmd.Args[1:]) default: @@ -283,7 +283,7 @@ func installerTarFixture(t *testing.T) []byte { mode int64 body string }{ - {name: installerBinaryName, mode: 0o755, body: "#!/bin/sh\necho MicroSandbox v0.7.7\n"}, + {name: installerBinaryName, mode: 0o755, body: "#!/bin/sh\necho MicroSandbox v0.7.13\n"}, {name: microsandboxLibrary, mode: 0o644, body: "library fixture"}, } { require.NoError(t, writer.WriteHeader(&tar.Header{ @@ -307,7 +307,7 @@ func installerExtractFixture(args []string) error { } staging := args[3] msb := filepath.Join(staging, installerBinaryName) - content := []byte("#!/bin/sh\necho MicroSandbox v0.7.7\n") + content := []byte("#!/bin/sh\necho MicroSandbox v0.7.13\n") if err := os.WriteFile(msb, content, 0o600); err != nil { return err } diff --git a/renovate.json b/renovate.json index e85432c851..03f12fdceb 100644 --- a/renovate.json +++ b/renovate.json @@ -14,6 +14,15 @@ "platformAutomerge": true, "postUpdateOptions": ["gomodTidy"], "customManagers": [ + { + "customType": "regex", + "description": "External MicroSandbox provider release used by the core e2e suites", + "managerFilePatterns": ["/^e2e/tests/up/microsandbox_session\\.go$/"], + "matchStrings": [ + "github\\.com/(?devsy-org/devsy-provider-microsandbox)@(?v[0-9]+\\.[0-9]+\\.[0-9]+)" + ], + "datasourceTemplate": "github-releases" + }, { "customType": "regex", "description": "prek hook repositories in prek.toml (Renovate has no prek manager yet)", From 1b557d60b41346e2446d1d95a87ccb7ff012e6df Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 22:08:33 -0600 Subject: [PATCH 2/4] ci(e2e): test the latest microsandbox provider release --- AGENTS.md | 2 +- e2e/tests/up/microsandbox_session.go | 2 +- .../up/provider_microsandbox_migration.go | 19 ++++++++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a48d33259c..6d709849ab 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,7 +85,7 @@ Devsy uses [Ginkgo](https://onsi.github.io/ginkgo/) for Go E2E and integration t tracks that source. When updating it, check the provider's runtime requirement and qualified executable and firmware pair. Update `hack/microsandbox/install.go`, its tests, and the release bundle cache in `.github/workflows/pr-ci.yml` together. -The external provider v1.0.5 uses msb 0.7.13 and `libkrunfw.so.5.6.1`. Keep the +The Linux CI bundle uses msb 0.7.13 and `libkrunfw.so.5.6.1`. Keep the built-in provider's compatibility expectations separate. The prefetch helper reads runtime asset names and checksums from the published provider manifest. Run the real VM specs on Linux with KVM before claiming parity. diff --git a/e2e/tests/up/microsandbox_session.go b/e2e/tests/up/microsandbox_session.go index 254d5abef0..db9b4b2365 100644 --- a/e2e/tests/up/microsandbox_session.go +++ b/e2e/tests/up/microsandbox_session.go @@ -19,7 +19,7 @@ const ( microsandboxResourceBuiltin = provider.DeprecatedMicrosandboxSource osLinux = "linux" osDarwin = "darwin" - microsandboxExternalProvider = "github.com/devsy-org/devsy-provider-microsandbox@v1.0.5" + microsandboxExternalProvider = "github.com/devsy-org/devsy-provider-microsandbox@v1.1.1" microsandboxRootUser = "root" ) diff --git a/e2e/tests/up/provider_microsandbox_migration.go b/e2e/tests/up/provider_microsandbox_migration.go index fd8fb888b4..9e0dafb8f1 100644 --- a/e2e/tests/up/provider_microsandbox_migration.go +++ b/e2e/tests/up/provider_microsandbox_migration.go @@ -250,7 +250,7 @@ func (fixture *microsandboxMigrationFixture) migrate(ctx context.Context, stoppe gomega.Expect(workspace.UID).To(gomega.Equal(fixture.uid)) gomega.Expect(workspace.Provider.Name).To(gomega.Equal("microsandbox")) gomega.Expect(snapshotCompatibilityWorkspace(ctx, workspace)). - To(gomega.Equal(fixture.workspaceState)) + To(gomega.Equal(expectedMigrationWorkspaceAfterUp(fixture.workspaceState))) vmAfterUp := inspectSafeRecreate(ctx, fixture.sandbox) gomega.Expect(vmAfterUp.Status).To(gomega.Equal("Running")) gomega.Expect(vmAfterUp.CreatedAt).To(gomega.Equal(fixture.vmState.CreatedAt)) @@ -263,6 +263,23 @@ func (fixture *microsandboxMigrationFixture) migrate(ctx context.Context, stoppe fixture.assertMarkers(ctx) } +func expectedMigrationWorkspaceAfterUp( + before compatibilityWorkspaceState, +) compatibilityWorkspaceState { + var record map[string]json.RawMessage + framework.ExpectNoError(json.Unmarshal(before.RecordWithoutLastUsed, &record)) + var providerConfig providerpkg.WorkspaceProviderConfig + framework.ExpectNoError(json.Unmarshal(record["provider"], &providerConfig)) + providerConfig.Options["MSB_PATH"] = config.OptionValue{} + providerConfig.Options["MSB_LIBKRUNFW_PATH"] = config.OptionValue{} + providerData, err := json.Marshal(providerConfig) + framework.ExpectNoError(err) + record["provider"] = providerData + before.RecordWithoutLastUsed, err = json.Marshal(record) + framework.ExpectNoError(err) + return before +} + func (fixture *microsandboxMigrationFixture) assertMarkers(ctx context.Context) { for _, path := range []string{ microsandboxMigrationRootMarker, From bd836da8c2199e9e89d109efc5e61b96d3e682d3 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 8 Nov 2026 23:48:23 -0600 Subject: [PATCH 3/4] ci(e2e): qualify microsandbox provider v1.1.2 --- e2e/framework/retry.go | 16 +++++++-- e2e/framework/retry_test.go | 34 +++++++++++++++---- e2e/tests/up/microsandbox_session.go | 2 +- .../up/provider_microsandbox_ephemeral.go | 10 ++++-- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/e2e/framework/retry.go b/e2e/framework/retry.go index e22b8a2d97..872d1a4e4c 100644 --- a/e2e/framework/retry.go +++ b/e2e/framework/retry.go @@ -34,18 +34,22 @@ var sshBackoff = wait.Backoff{ } // providerAddBackoff defines retry timing for provider manifest downloads. -// 5 total attempts with waits of ~5s, ~10s, ~20s, ~40s. var providerAddBackoff = wait.Backoff{ - Steps: 5, + Steps: 8, Duration: 5 * time.Second, Factor: 2.0, Jitter: 0.1, + Cap: time.Minute, } var providerServerErrorPattern = regexp.MustCompile( `(?i)(received status code|download failed with status|status) 5\d\d`, ) +var providerClientErrorPattern = regexp.MustCompile( + `(?i)(received status code|download failed with status|status) 4\d\d`, +) + var retryableProviderNetworkPatterns = []string{ "TLS handshake timeout", "i/o timeout", @@ -119,10 +123,13 @@ func isRetryableDockerError(stderr string) bool { // failed on a server error or a network error. Client errors such as 404 and // provider validation failures are never retried. func isRetryableProviderDownloadError(_ error, stderr string) bool { + lower := strings.ToLower(stderr) + if providerClientErrorPattern.MatchString(stderr) || strings.Contains(lower, "checksum") { + return false + } if providerServerErrorPattern.MatchString(stderr) { return true } - lower := strings.ToLower(stderr) for _, pattern := range retryableProviderNetworkPatterns { if strings.Contains(lower, strings.ToLower(pattern)) { return true @@ -209,6 +216,9 @@ func execWithDockerRetry(ctx context.Context, fn execFunc) (string, string, erro // execWithProviderAddRetry runs fn and retries when the provider manifest // download failed with a server or network error. func execWithProviderAddRetry(ctx context.Context, fn execFunc) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + res := runWithRetry(ctx, retryPolicy{ backoff: providerAddBackoff, retryable: isRetryableProviderDownloadError, diff --git a/e2e/framework/retry_test.go b/e2e/framework/retry_test.go index 20f931656b..1b1f6d258c 100644 --- a/e2e/framework/retry_test.go +++ b/e2e/framework/retry_test.go @@ -718,6 +718,10 @@ func TestIsRetryableProviderDownloadError(t *testing.T) { permanent := []string{ "download: received status code 404 when trying to download https://github.com/o/r/provider.yaml", "download failed with status 403", + "received status code 404: unexpected EOF", + "download failed with status 403 after status 502", + "external runtime executable checksum verification failed", + "checksum verification failed after connection reset by peer", "provider microsandbox already exists", "validate: exec.command is required", } @@ -728,21 +732,37 @@ func TestIsRetryableProviderDownloadError(t *testing.T) { func TestExecWithProviderAddRetryRecoversFromServerErrors(t *testing.T) { original := providerAddBackoff - providerAddBackoff = wait.Backoff{Steps: 4, Duration: time.Millisecond, Factor: 1} + providerAddBackoff.Duration = time.Millisecond + providerAddBackoff.Factor = 1 + providerAddBackoff.Jitter = 0 t.Cleanup(func() { providerAddBackoff = original }) attempts := 0 _, err := execWithProviderAddRetry( context.Background(), func(context.Context) (string, string, error) { attempts++ - if attempts < 3 { + if attempts < 6 { return "", providerServerFailure, errExitOne } return "", "", nil }, ) require.NoError(t, err) - assert.Equal(t, 3, attempts) + assert.Equal(t, 6, attempts) +} + +func TestExecWithProviderAddRetryBoundsAttemptDeadline(t *testing.T) { + started := time.Now() + _, err := execWithProviderAddRetry( + context.Background(), + func(ctx context.Context) (string, string, error) { + deadline, ok := ctx.Deadline() + require.True(t, ok) + assert.WithinDuration(t, started.Add(5*time.Minute), deadline, time.Second) + return "", "", nil + }, + ) + require.NoError(t, err) } func TestExecWithProviderAddRetrySkipsClientErrors(t *testing.T) { @@ -764,7 +784,9 @@ func TestExecWithProviderAddRetrySkipsClientErrors(t *testing.T) { func TestExecWithProviderAddRetryStopsAtBudget(t *testing.T) { original := providerAddBackoff - providerAddBackoff = wait.Backoff{Steps: 3, Duration: time.Millisecond, Factor: 1} + providerAddBackoff.Duration = time.Millisecond + providerAddBackoff.Factor = 1 + providerAddBackoff.Jitter = 0 t.Cleanup(func() { providerAddBackoff = original }) attempts := 0 _, err := execWithProviderAddRetry( @@ -774,8 +796,8 @@ func TestExecWithProviderAddRetryStopsAtBudget(t *testing.T) { return "", providerServerFailure, errExitOne }, ) - require.ErrorContains(t, err, "after 3 attempts") - assert.Equal(t, 3, attempts) + require.ErrorContains(t, err, fmt.Sprintf("after %d attempts", original.Steps)) + assert.Equal(t, original.Steps, attempts) } func TestSeedProviderDownloadCache(t *testing.T) { diff --git a/e2e/tests/up/microsandbox_session.go b/e2e/tests/up/microsandbox_session.go index db9b4b2365..73ba814d90 100644 --- a/e2e/tests/up/microsandbox_session.go +++ b/e2e/tests/up/microsandbox_session.go @@ -19,7 +19,7 @@ const ( microsandboxResourceBuiltin = provider.DeprecatedMicrosandboxSource osLinux = "linux" osDarwin = "darwin" - microsandboxExternalProvider = "github.com/devsy-org/devsy-provider-microsandbox@v1.1.1" + microsandboxExternalProvider = "github.com/devsy-org/devsy-provider-microsandbox@v1.1.2" microsandboxRootUser = "root" ) diff --git a/e2e/tests/up/provider_microsandbox_ephemeral.go b/e2e/tests/up/provider_microsandbox_ephemeral.go index a6b8b2c334..03ce5c4d8a 100644 --- a/e2e/tests/up/provider_microsandbox_ephemeral.go +++ b/e2e/tests/up/provider_microsandbox_ephemeral.go @@ -196,9 +196,15 @@ var _ = ginkgo.Describe("microsandbox ephemeral root parity", stdout, stderr, err := f.DevsyUpStreams(ctx, workspace, "--devcontainer", ".devcontainer.json", "--id-label", label) gomega.Expect(err).To(gomega.HaveOccurred()) - gomega.Expect(stdout + stderr).To(gomega.ContainSubstring( + diagnostic := gomega.ContainSubstring( "tmpfs root disk size (2048 MiB) must not exceed sandbox memory (1024 MiB)", - )) + ) + if provider.source == microsandboxExternalProvider { + diagnostic = gomega.MatchRegexp( + `tmpfs (root|\*\*\*) disk size \(2048 MiB\) must not exceed sandbox memory \(1024 MiB\)`, + ) + } + gomega.Expect(stdout + stderr).To(diagnostic) assertMicrosandboxEphemeralAbsent(ctx, label) configAfter, err := os.ReadFile( configPath, From 45652364aa55b74b23f70f850a92f2d3f2b0577f Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 9 Nov 2026 02:17:11 -0600 Subject: [PATCH 4/4] ci(e2e): test the latest microsandbox provider release --- e2e/framework/retry.go | 12 ++++-- e2e/framework/retry_test.go | 37 +++++++++++++++++-- .../up/provider_microsandbox_compatibility.go | 9 ++++- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/e2e/framework/retry.go b/e2e/framework/retry.go index 872d1a4e4c..b0ec0d196a 100644 --- a/e2e/framework/retry.go +++ b/e2e/framework/retry.go @@ -124,7 +124,9 @@ func isRetryableDockerError(stderr string) bool { // provider validation failures are never retried. func isRetryableProviderDownloadError(_ error, stderr string) bool { lower := strings.ToLower(stderr) - if providerClientErrorPattern.MatchString(stderr) || strings.Contains(lower, "checksum") { + if providerClientErrorPattern.MatchString(stderr) || + strings.Contains(lower, "checksum verification failed") || + strings.Contains(lower, "unexpected file checksum") { return false } if providerServerErrorPattern.MatchString(stderr) { @@ -216,15 +218,17 @@ func execWithDockerRetry(ctx context.Context, fn execFunc) (string, string, erro // execWithProviderAddRetry runs fn and retries when the provider manifest // download failed with a server or network error. func execWithProviderAddRetry(ctx context.Context, fn execFunc) (string, error) { - ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + retryCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() - res := runWithRetry(ctx, retryPolicy{ + res := runWithRetry(retryCtx, retryPolicy{ backoff: providerAddBackoff, retryable: isRetryableProviderDownloadError, label: "[retry] provider add:", kind: "retryable provider download error", - }, fn) + }, func(context.Context) (string, string, error) { + return fn(ctx) + }) if res.gaveUp { return res.stderr, fmt.Errorf("after %d attempts: %w", res.attempts, res.err) } diff --git a/e2e/framework/retry_test.go b/e2e/framework/retry_test.go index 1b1f6d258c..a99af6d2a1 100644 --- a/e2e/framework/retry_test.go +++ b/e2e/framework/retry_test.go @@ -708,6 +708,8 @@ func TestIsRetryableProviderDownloadError(t *testing.T) { retryable := []string{ "download: received status code 504 when trying to download https://github.com/o/r/provider.yaml", "download failed with status 502", + "download failed with status 503: checksum service temporarily unavailable", + "Get \"https://github.com/o/checksum-provider\": net/http: TLS handshake timeout", "Get \"https://github.com\": net/http: TLS handshake timeout", "read tcp: connection reset by peer", "download file: stalled: no progress within timeout", @@ -718,10 +720,12 @@ func TestIsRetryableProviderDownloadError(t *testing.T) { permanent := []string{ "download: received status code 404 when trying to download https://github.com/o/r/provider.yaml", "download failed with status 403", + "download failed with status 408", "received status code 404: unexpected EOF", "download failed with status 403 after status 502", "external runtime executable checksum verification failed", "checksum verification failed after connection reset by peer", + "unexpected file checksum abc != def after connection reset by peer", "provider microsandbox already exists", "validate: exec.command is required", } @@ -751,20 +755,45 @@ func TestExecWithProviderAddRetryRecoversFromServerErrors(t *testing.T) { assert.Equal(t, 6, attempts) } -func TestExecWithProviderAddRetryBoundsAttemptDeadline(t *testing.T) { - started := time.Now() +func TestExecWithProviderAddRetryPreservesAttemptDeadline(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + wantDeadline, ok := ctx.Deadline() + require.True(t, ok) _, err := execWithProviderAddRetry( - context.Background(), + ctx, func(ctx context.Context) (string, string, error) { deadline, ok := ctx.Deadline() require.True(t, ok) - assert.WithinDuration(t, started.Add(5*time.Minute), deadline, time.Second) + assert.Equal(t, wantDeadline, deadline) return "", "", nil }, ) require.NoError(t, err) } +func TestExecWithProviderAddRetryPropagatesActiveDeadline(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + attempts := 0 + _, err := execWithProviderAddRetry( + ctx, + func(ctx context.Context) (string, string, error) { + attempts++ + timer := time.NewTimer(time.Second) + defer timer.Stop() + select { + case <-ctx.Done(): + return "", "", ctx.Err() + case <-timer.C: + return "", "", errors.New("provider attempt lost caller deadline") + } + }, + ) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, attempts) +} + func TestExecWithProviderAddRetrySkipsClientErrors(t *testing.T) { original := providerAddBackoff providerAddBackoff = wait.Backoff{Steps: 4, Duration: time.Millisecond, Factor: 1} diff --git a/e2e/tests/up/provider_microsandbox_compatibility.go b/e2e/tests/up/provider_microsandbox_compatibility.go index 65a7069876..da71226b0c 100644 --- a/e2e/tests/up/provider_microsandbox_compatibility.go +++ b/e2e/tests/up/provider_microsandbox_compatibility.go @@ -53,7 +53,11 @@ var _ = ginkgo.Describe( } session := newMicrosandboxTestSession(ctx) fixture := newCompatibilityFixture(ctx, session, selection) - for _, fault := range []string{"old", "malformed", "failing"} { + faults := []string{"old", "malformed", "failing"} + if selection.runtime == providerpkg.ExternalDriver { + faults[0] = "old-external" + } + for _, fault := range faults { fixture.rejectVersionFault(ctx, fault) } fixture.recover(ctx) @@ -210,7 +214,7 @@ func (fixture *compatibilityFixture) rejectVersionFault(ctx context.Context, fau func assertCompatibilityDiagnostic(fault, output, minimumVersion string) { switch fault { - case "old": + case "old", "old-external": gomega.Expect(output). To(gomega.ContainSubstring("v" + minimumVersion + " or newer is required")) case "malformed": @@ -302,6 +306,7 @@ if [ "$#" -eq 1 ] && [ "$1" = "--version" ]; then if [ -n "$fault" ]; then case "$fault" in old) printf 'msb 0.7.1\n'; exit 0 ;; + old-external) printf 'msb 0.7.12\n'; exit 0 ;; malformed) printf 'msb not-a-version\n'; exit 0 ;; failing) printf '%s\n' 'injected version command failure' >&2; exit 42 ;; *) printf '%s\n' 'unknown test fault mode' >&2; exit 64 ;;