From 8a819f53f3f1e0e559bde28fa855fc16c0e62c9b Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 11 Oct 2026 04:48:22 -0600 Subject: [PATCH] fix(runtime): align MicroSandbox ephemeral root defaults --- .github/workflows/pr-ci.yml | 9 + e2e/tests/up/provider_microsandbox.go | 4 +- .../up/provider_microsandbox_ephemeral.go | 329 ++++++++++++++++++ pkg/driver/microsandbox/cliclient.go | 15 +- pkg/driver/microsandbox/cliclient_test.go | 4 +- providers/microsandbox/provider.yaml | 2 +- .../developing-providers/runtime-protocol.mdx | 12 +- 7 files changed, 360 insertions(+), 15 deletions(-) create mode 100644 e2e/tests/up/provider_microsandbox_ephemeral.go diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 46c228a85a..f57c08147f 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -687,6 +687,15 @@ jobs: test-timeout: 1200s job-timeout-minutes: 25 + - label: up-provider-microsandbox-ephemeral + runner: ubuntu-latest + free-disk-space: false + install-kind: false + requires-secret: false + install-microsandbox: true + test-timeout: 1200s + job-timeout-minutes: 25 + # Snapshot tests - label: snapshot diff --git a/e2e/tests/up/provider_microsandbox.go b/e2e/tests/up/provider_microsandbox.go index 86a2e7ff1c..2b671575db 100644 --- a/e2e/tests/up/provider_microsandbox.go +++ b/e2e/tests/up/provider_microsandbox.go @@ -23,7 +23,7 @@ import ( const ( osLinux = "linux" - microsandboxExternalProvider = "github.com/devsy-org/devsy-provider-microsandbox@v0.1.6" + microsandboxExternalProvider = "github.com/devsy-org/devsy-provider-microsandbox@v0.1.7" microsandboxRootUser = "root" ) @@ -213,7 +213,7 @@ var _ = ginkgo.Describe( ginkgo.Entry("built-in", "microsandbox", "microsandbox-builtin-parity", ginkgo.SpecTimeout(framework.TimeoutLong())), ginkgo.Entry( - "external v0.1.6", + "external v0.1.7", microsandboxExternalProvider, "microsandbox-external-parity", ginkgo.SpecTimeout(framework.TimeoutLong()), diff --git a/e2e/tests/up/provider_microsandbox_ephemeral.go b/e2e/tests/up/provider_microsandbox_ephemeral.go new file mode 100644 index 0000000000..15f287b6c9 --- /dev/null +++ b/e2e/tests/up/provider_microsandbox_ephemeral.go @@ -0,0 +1,329 @@ +package up + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +type microsandboxEphemeralCase struct { + memoryMiB uint64 + storageGiB uint64 + preserveMounts bool +} + +var _ = ginkgo.Describe("microsandbox ephemeral root parity", + ginkgo.Label("up-provider-microsandbox-ephemeral"), func() { + for _, provider := range []struct{ name, source string }{ + {"builtin-ephemeral", microsandboxResourceBuiltin}, + {"external-ephemeral", microsandboxExternalProvider}, + } { + ginkgo.Context(provider.name, func() { + var f *framework.Framework + var workspace, volume, ownedLabel string + var volumeOwned bool + + ginkgo.BeforeEach(func(ctx context.Context) { + skipIfNoMicrosandbox(ctx) + ginkgo.GinkgoT().Setenv("DEVSY_HOME", ginkgo.GinkgoT().TempDir()) + ginkgo.GinkgoT().Setenv("DEVSY_CONFIG", "") + initialDir, err := os.Getwd() + framework.ExpectNoError(err) + f = framework.NewDefaultFramework(filepath.Join(initialDir, "bin")) + workspace, err = framework.CreateTempDir() + framework.ExpectNoError(err) + volume, volumeOwned, ownedLabel = "", false, "" + ginkgo.DeferCleanup(framework.CleanupTempDir, initialDir, workspace) + framework.ExpectNoError( + f.DevsyProviderAdd(ctx, provider.source, "--name", provider.name), + ) + ginkgo.DeferCleanup(func(cleanupCtx context.Context) error { + cleanupCtx, cancel := context.WithTimeout( + context.WithoutCancel(cleanupCtx), + time.Minute, + ) + defer cancel() + return f.DevsyProviderDelete(cleanupCtx, provider.name) + }) + // A rejected creation can lose its Devsy record before runtime cleanup completes. + ginkgo.DeferCleanup(func(cleanupCtx context.Context) error { + if ownedLabel == "" { + return nil + } + cleanupCtx, cancel := context.WithTimeout( + context.WithoutCancel(cleanupCtx), + time.Minute, + ) + defer cancel() + out, err := microsandboxResourceCommand( + cleanupCtx, + "remove", + "--force", + "--label", + ownedLabel, + ) + if err != nil { + return fmt.Errorf( + "remove test-owned sandbox with label %s: %w: %s", + ownedLabel, + err, + out, + ) + } + return nil + }) + // Independent LIFO cleanup attempts stop the VM before removing our volume. + ginkgo.DeferCleanup(func(cleanupCtx context.Context) error { + if !volumeOwned { + return nil + } + cleanupCtx, cancel := context.WithTimeout( + context.WithoutCancel(cleanupCtx), + time.Minute, + ) + defer cancel() + out, err := microsandboxResourceCommand(cleanupCtx, "volume", "rm", volume) + if err != nil { + return fmt.Errorf( + "remove test-owned volume %s: %w: %s", + volume, + err, + out, + ) + } + return nil + }) + ginkgo.DeferCleanup(f.CleanupWorkspace, workspace) + framework.ExpectNoError( + os.WriteFile(filepath.Join(workspace, microsandboxStorageMarker), + []byte("host-preserved\n"), 0o600), + ) + }) + + ginkgo.DescribeTable( + "resets the tmpfs root without discarding the sandbox record", + func(ctx context.Context, testCase microsandboxEphemeralCase) { + options := []string{ + microsandboxResourceOption, + "MICROSANDBOX_EPHEMERAL=true", + microsandboxResourceOption, + fmt.Sprintf("MICROSANDBOX_MEMORY=%d", testCase.memoryMiB), + } + if testCase.storageGiB != 0 { + options = append(options, microsandboxResourceOption, + fmt.Sprintf("MICROSANDBOX_STORAGE=%d", testCase.storageGiB)) + } + framework.ExpectNoError(f.DevsyProviderUse(ctx, provider.name, options...)) + var mounts []string + if testCase.preserveMounts { + volume = "devsy-ephemeral-parity-" + filepath.Base(workspace) + out, err := microsandboxResourceCommand(ctx, "volume", "create", volume) + framework.ExpectNoError(err, string(out)) + volumeOwned = true + mounts = []string{ + "type=volume,source=" + volume + ",target=/ephemeral-volume", + } + } + writeMicrosandboxMountConfig(workspace, mounts) + framework.ExpectNoError( + f.DevsyUp(ctx, workspace, "--devcontainer", ".devcontainer.json"), + ) + sandbox := microsandboxResourceSandbox(ctx, f, workspace) + assertMicrosandboxEphemeralRoot(ctx, f, workspace) + assertMicrosandboxStorageWorkspace(ctx, f, workspace) + createdAt := microsandboxCreationTime(ctx, sandbox) + bootID := microsandboxStorageBootID(ctx, f, workspace) + writeMicrosandboxEphemeralRootMarker(ctx, f, workspace) + if testCase.preserveMounts { + out, err := f.DevsySSHOnce(ctx, workspace, + "printf 'volume-preserved\\n' > /ephemeral-volume/marker") + framework.ExpectNoError(err, out) + assertMicrosandboxEphemeralVolume(ctx, f, workspace) + } + + framework.ExpectNoError(f.DevsyWorkspaceStop(ctx, workspace)) + assertMicrosandboxEphemeralStopped(ctx, sandbox, createdAt) + framework.ExpectNoError(f.DevsyUp(ctx, workspace)) + gomega.Expect(microsandboxCreationTime(ctx, sandbox)). + To(gomega.Equal(createdAt)) + restartedBootID := microsandboxStorageBootID(ctx, f, workspace) + gomega.Expect(restartedBootID).NotTo(gomega.Equal(bootID)) + assertMicrosandboxEphemeralReset(ctx, f, workspace) + if !testCase.preserveMounts { + return + } + assertMicrosandboxEphemeralVolume(ctx, f, workspace) + writeMicrosandboxEphemeralRootMarker(ctx, f, workspace) + framework.ExpectNoError(f.DevsyUpRecreate(ctx, workspace)) + gomega.Expect(microsandboxCreationTime(ctx, sandbox)). + NotTo(gomega.Equal(createdAt)) + gomega.Expect(microsandboxStorageBootID(ctx, f, workspace)). + NotTo(gomega.Equal(restartedBootID)) + assertMicrosandboxEphemeralReset(ctx, f, workspace) + assertMicrosandboxEphemeralVolume(ctx, f, workspace) + }, + ginkgo.Entry( + "defaults to half of 2048 MiB and preserves bind and volume data through recreation", + microsandboxEphemeralCase{memoryMiB: 2048, preserveMounts: true}, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ), + ginkgo.Entry("explicit 1 GiB storage overrides half of 3072 MiB", + microsandboxEphemeralCase{memoryMiB: 3072, storageGiB: 1}, + ginkgo.SpecTimeout(framework.TimeoutLong())), + ) + + ginkgo.It( + "rejects a fresh tmpfs root larger than memory without creating a VM or deleting source", + func(ctx context.Context) { + framework.ExpectNoError(f.DevsyProviderUse(ctx, provider.name, + microsandboxResourceOption, "MICROSANDBOX_EPHEMERAL=true", + microsandboxResourceOption, "MICROSANDBOX_MEMORY=1024", + microsandboxResourceOption, "MICROSANDBOX_STORAGE=2")) + writeMicrosandboxMountConfig(workspace, nil) + configPath := filepath.Join(workspace, ".devcontainer.json") + configBefore, err := os.ReadFile( + configPath, + ) // #nosec G304 -- test-owned configuration + framework.ExpectNoError(err) + label := "devsy.ephemeral-parity=" + filepath.Base(workspace) + assertMicrosandboxEphemeralAbsent(ctx, label) + ownedLabel = label + stdout, stderr, err := f.DevsyUpStreams(ctx, workspace, + "--devcontainer", ".devcontainer.json", "--id-label", label) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(stdout + stderr).To(gomega.ContainSubstring( + "tmpfs root disk size (2048 MiB) must not exceed sandbox memory (1024 MiB)", + )) + assertMicrosandboxEphemeralAbsent(ctx, label) + configAfter, err := os.ReadFile( + configPath, + ) // #nosec G304 -- test-owned configuration + framework.ExpectNoError(err) + gomega.Expect(configAfter).To(gomega.Equal(configBefore)) + marker, err := os.ReadFile( + filepath.Join(workspace, microsandboxStorageMarker), + ) // #nosec G304 -- test-owned source + framework.ExpectNoError(err) + gomega.Expect(string(marker)).To(gomega.Equal("host-preserved\n")) + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + }) + } + }) + +func assertMicrosandboxEphemeralRoot( + ctx context.Context, + f *framework.Framework, + workspace string, +) { + sandbox := microsandboxResourceSandbox(ctx, f, workspace) + out, err := microsandboxResourceCommand( + ctx, + "inspect", + sandbox, + "--format", + microsandboxResourceJSON, + ) + framework.ExpectNoError(err, string(out)) + var inspection struct { + ActiveConfig *microsandboxStorageConfig `json:"active_config"` + } + framework.ExpectNoError(json.Unmarshal(out, &inspection), string(out)) + gomega.Expect(inspection.ActiveConfig).NotTo(gomega.BeNil()) + gomega.Expect(inspection.ActiveConfig.Image.OCI).NotTo(gomega.BeNil()) + rootDisk := inspection.ActiveConfig.Image.OCI.RootDisk + gomega.Expect(rootDisk).NotTo(gomega.BeNil()) + gomega.Expect(rootDisk.Kind).To(gomega.Equal("tmpfs")) + gomega.Expect(rootDisk.SizeMiB).To(gomega.Equal(uint64(1024))) + + // The guest root is overlayfs; measure its writable backing capacity, not its filesystem type. + capacity, err := f.DevsySSHOnce(ctx, workspace, "stat -f -c '%S %b' /") + framework.ExpectNoError(err) + var blockSize, blocks uint64 + n, err := fmt.Sscanf(strings.TrimSpace(capacity), "%d %d", &blockSize, &blocks) + framework.ExpectNoError(err) + gomega.Expect(n).To(gomega.Equal(2)) + gomega.Expect(blockSize).To(gomega.BeNumerically(">", 0)) + totalBytes := blockSize * blocks + const expectedBytes = uint64(1024 * 1024 * 1024) + gomega.Expect(totalBytes).To(gomega.BeNumerically(">=", expectedBytes*9/10)) + gomega.Expect(totalBytes).To(gomega.BeNumerically("<=", expectedBytes)) +} + +func writeMicrosandboxEphemeralRootMarker( + ctx context.Context, + f *framework.Framework, + workspace string, +) { + out, err := f.DevsySSHOnce( + ctx, + workspace, + "printf 'root-written\\n' > /root/ephemeral-parity-marker && cat /root/ephemeral-parity-marker", + ) + framework.ExpectNoError(err) + gomega.Expect(out).To(gomega.Equal("root-written\n")) +} + +func assertMicrosandboxEphemeralReset( + ctx context.Context, + f *framework.Framework, + workspace string, +) { + out, err := f.DevsySSHOnce(ctx, workspace, "test ! -e /root/ephemeral-parity-marker") + framework.ExpectNoError(err, out) + assertMicrosandboxEphemeralRoot(ctx, f, workspace) + assertMicrosandboxStorageWorkspace(ctx, f, workspace) +} + +func assertMicrosandboxEphemeralVolume( + ctx context.Context, + f *framework.Framework, + workspace string, +) { + out, err := f.DevsySSHOnce(ctx, workspace, "cat /ephemeral-volume/marker") + framework.ExpectNoError(err) + gomega.Expect(out).To(gomega.Equal("volume-preserved\n")) +} + +func assertMicrosandboxEphemeralStopped(ctx context.Context, sandbox, createdAt string) { + out, err := microsandboxResourceCommand( + ctx, + "inspect", + sandbox, + "--format", + microsandboxResourceJSON, + ) + framework.ExpectNoError(err, string(out)) + var inspection struct { + Status string `json:"status"` + CreatedAt string `json:"created_at"` + } + framework.ExpectNoError(json.Unmarshal(out, &inspection), string(out)) + gomega.Expect(inspection.Status).To(gomega.Equal("Stopped")) + gomega.Expect(inspection.CreatedAt).To(gomega.Equal(createdAt)) +} + +func assertMicrosandboxEphemeralAbsent(ctx context.Context, label string) { + out, err := microsandboxResourceCommand( + ctx, + "list", + "--label", + label, + "--format", + microsandboxResourceJSON, + ) + framework.ExpectNoError(err, string(out)) + var sandboxes []json.RawMessage + framework.ExpectNoError(json.Unmarshal(out, &sandboxes), string(out)) + gomega.Expect(sandboxes). + To(gomega.BeEmpty(), "the failed fresh creation must not leave a test-labeled sandbox") +} diff --git a/pkg/driver/microsandbox/cliclient.go b/pkg/driver/microsandbox/cliclient.go index 9624bcff30..4c0230b314 100644 --- a/pkg/driver/microsandbox/cliclient.go +++ b/pkg/driver/microsandbox/cliclient.go @@ -192,10 +192,9 @@ func (cliClient) ensureVolumes(ctx context.Context, mounts []volumeMount) error } const ( - msbCmdRun = "run" - msbFlagDetach = "--detach" - flagRootDisk = "--root-disk" - defaultEphemeralRootDiskGB = 8 + msbCmdRun = "run" + msbFlagDetach = "--detach" + flagRootDisk = "--root-disk" ) // runArgs builds a detached `msb run` invocation, matching microsandbox's own @@ -252,11 +251,11 @@ func resourceArgs(spec sandboxSpec) []string { args = append(args, "--max-cpus", strconv.Itoa(int(spec.MaxCPUs))) } if spec.Ephemeral { - size := spec.RootDiskGB - if size == 0 { - size = defaultEphemeralRootDiskGB + disk := "tmpfs" + if spec.RootDiskGB > 0 { + disk = fmt.Sprintf("tmpfs:%dG", spec.RootDiskGB) } - args = append(args, flagRootDisk, fmt.Sprintf("tmpfs:%dG", size)) + args = append(args, flagRootDisk, disk) } else if spec.RootDiskGB > 0 { args = append(args, flagRootDisk, fmt.Sprintf("%dG", spec.RootDiskGB)) } diff --git a/pkg/driver/microsandbox/cliclient_test.go b/pkg/driver/microsandbox/cliclient_test.go index 14f77b8e85..534a8c4fb6 100644 --- a/pkg/driver/microsandbox/cliclient_test.go +++ b/pkg/driver/microsandbox/cliclient_test.go @@ -188,10 +188,10 @@ func TestResourceArgsEphemeralUsesTmpfsRootDisk(t *testing.T) { } } -func TestResourceArgsEphemeralWithoutSizeUsesDefault(t *testing.T) { +func TestResourceArgsEphemeralWithoutSizeUsesRuntimeDefault(t *testing.T) { if got := resourceArgs(sandboxSpec{Ephemeral: true}); !slices.Equal( got, - []string{flagRootDisk, fmt.Sprintf("tmpfs:%dG", defaultEphemeralRootDiskGB)}, + []string{flagRootDisk, "tmpfs"}, ) { t.Errorf("resourceArgs = %v", got) } diff --git a/providers/microsandbox/provider.yaml b/providers/microsandbox/provider.yaml index 1e61e27e13..177fc99d10 100644 --- a/providers/microsandbox/provider.yaml +++ b/providers/microsandbox/provider.yaml @@ -34,7 +34,7 @@ options: default: "false" type: boolean MICROSANDBOX_EPHEMERAL: - description: "If true, boot from a tmpfs root disk so the microVM's disk state is discarded when it stops. Sized by MICROSANDBOX_STORAGE/hostRequirements.storage, or 8GiB if neither is set." + description: "If true, boot from a tmpfs root disk so the microVM's disk state is discarded when it stops. MICROSANDBOX_STORAGE/hostRequirements.storage sets its size; otherwise the runtime default applies." default: "false" type: boolean MICROSANDBOX_WORKSPACE_HOST_PERMISSIONS: diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx index 4930614a13..4e3569987e 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx @@ -47,7 +47,7 @@ For SDK usage and development commands, see the [SDK README](https://github.com/ ## MicroSandbox parity gate -The `up-provider-microsandbox` E2E label runs the same lifecycle and ownership scenario against the built-in provider and the external v0.1.6 release, each with isolated Devsy configuration. CI pins MicroSandbox v0.7.7 by checksum and requires access to KVM; unavailable virtualization fails this job instead of producing a passing skipped test. Each provider scenario has a ten-minute deadline and the CI job has a 25-minute deadline. +The `up-provider-microsandbox` E2E label runs the same lifecycle and ownership scenario against the built-in provider and the external v0.1.7 release, each with isolated Devsy configuration. CI pins MicroSandbox v0.7.7 by checksum and requires access to KVM; unavailable virtualization fails this job instead of producing a passing skipped test. Each provider scenario has a ten-minute deadline and the CI job has a 25-minute deadline. The shared scenario exercises agent delivery, SSH, a 1 MiB binary stdin/stdout round trip with separate stderr and a nonzero guest exit, root workload versus developer identity, bind-mount ownership and mode mirroring, stop/start, recreation, rejection of an identity change without recreation while preserving VM-local data, and deletion of the VM. @@ -89,4 +89,12 @@ Each scenario verifies that a VM-local file survives stop/start and disappears a DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-storage ``` -These scenarios do not establish complete parity. Ephemeral roots, egress denial, prebuilds, dockerless operation, logs, cancellation, and runtime compatibility failures still require coverage before replacing the built-in provider. Green lifecycle, image, mount, resource, and persistent-storage checks alone do not authorize that cutover. +The `up-provider-microsandbox-ephemeral` label runs three scenarios against each provider. An unsized ephemeral root delegates to the runtime default (half the initial memory on the pinned runtime); an explicit storage size overrides that default. The suite checks the active tmpfs backing and guest capacity, root-file loss after stop/start, retained sandbox identity, and persistent host workspace content. The default-size scenario also checks named-volume persistence and recreation. A fresh creation with a root larger than initial memory must fail with the runtime's size diagnostic and leave no test-labeled VM or source-file changes. + +Ephemeral roots use a RAM-backed upper layer beneath the guest's overlay filesystem. They do not enable the runtime's separate sandbox-record removal flag. These tests do not establish preservation of an existing VM when an invalid recreation is requested; that requires separate validation before teardown. Each scenario has a ten-minute deadline, with a 20-minute CI test deadline and a 25-minute job deadline. + +```sh +DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-ephemeral +``` + +These scenarios do not establish complete parity. Egress denial, prebuilds, dockerless operation, logs, cancellation, and runtime compatibility failures still require coverage before replacing the built-in provider. Green lifecycle, image, mount, resource, persistent-storage, and ephemeral-root checks alone do not authorize that cutover.