From ab0150fa1f617dd26b18fd249e50f02e0b36b6a8 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 11 Oct 2026 10:22:15 -0600 Subject: [PATCH 1/2] test(runtime): cover MicroSandbox Dockerless parity --- .github/workflows/pr-ci.yml | 9 + .../up/provider_microsandbox_dockerless.go | 500 ++++++++++++++++++ .../microsandbox-dockerless/Dockerfile | 9 + .../devsy-dockerless-parity-check | 7 + .../features/parity/devcontainer-feature.json | 8 + .../features/parity/install.sh | 4 + .../developing-providers/runtime-protocol.mdx | 10 +- 7 files changed, 546 insertions(+), 1 deletion(-) create mode 100644 e2e/tests/up/provider_microsandbox_dockerless.go create mode 100644 e2e/tests/up/testdata/microsandbox-dockerless/Dockerfile create mode 100755 e2e/tests/up/testdata/microsandbox-dockerless/devsy-dockerless-parity-check create mode 100644 e2e/tests/up/testdata/microsandbox-dockerless/features/parity/devcontainer-feature.json create mode 100755 e2e/tests/up/testdata/microsandbox-dockerless/features/parity/install.sh diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 523ef8b796..a9439b3ca3 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -714,6 +714,15 @@ jobs: test-timeout: 1200s job-timeout-minutes: 25 + - label: up-provider-microsandbox-dockerless + runner: ubuntu-latest + free-disk-space: false + install-kind: false + requires-secret: false + install-microsandbox: true + test-timeout: 1200s + job-timeout-minutes: 25 + # Snapshot tests - label: snapshot diff --git a/e2e/tests/up/provider_microsandbox_dockerless.go b/e2e/tests/up/provider_microsandbox_dockerless.go new file mode 100644 index 0000000000..87680302e5 --- /dev/null +++ b/e2e/tests/up/provider_microsandbox_dockerless.go @@ -0,0 +1,500 @@ +package up + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + devsyconfig "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/docker" + "github.com/devsy-org/devsy/pkg/encoding" + "github.com/devsy-org/devsy/pkg/envfile" + "github.com/devsy-org/devsy/pkg/flags/names" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +const ( + microsandboxDockerlessFixture = "tests/up/testdata/microsandbox-dockerless" + microsandboxDockerlessRoot = "/opt/devsy-dockerless-parity" + microsandboxDockerlessPath = microsandboxDockerlessRoot + + "/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + microsandboxDockerlessCommand = "devsy-dockerless-parity-check" + microsandboxDockerlessUser = "dlparity" + microsandboxDockerlessOff = "off" + microsandboxDockerlessDockerfile = "Dockerfile" + microsandboxDockerlessSentinel = "dockerless-host-sentinel" + microsandboxDockerlessRootMarker = "/root/dockerless-lifecycle-marker" +) + +type microsandboxDockerlessCase struct { + user string + policy string + refused bool +} + +type microsandboxDockerlessFixtureState struct { + f *framework.Framework + workspace string + uid string + label string + provider string + workspaceOwner string + sentinelOwner string + sentinelMode os.FileMode + configBefore []byte + dockerLog string +} + +var _ = ginkgo.Describe("microsandbox Dockerless parity", + ginkgo.Label("up-provider-microsandbox-dockerless"), func() { + for _, provider := range []struct{ name, source string }{ + {"builtin-dockerless", microsandboxResourceBuiltin}, + {"external-dockerless", microsandboxExternalProvider}, + } { + ginkgo.Context(provider.name, func() { + ginkgo.DescribeTable( + "runs without host Docker and enforces developer ownership", + func(ctx context.Context, testCase microsandboxDockerlessCase) { + fixture := newMicrosandboxDockerlessFixture( + ctx, + provider.name, + provider.source, + ) + fixture.configure(ctx, testCase) + if testCase.refused { + fixture.assertRefused(ctx) + return + } + fixture.up(ctx, true) + fixture.verify(ctx, testCase.user) + if testCase.user == microsandboxRootUser { + fixture.lifecycle(ctx) + } else { + fixture.verifyNonrootWrite(ctx) + } + }, + ginkgo.Entry( + "root Dockerfile and Feature retain image environment through lifecycle", + microsandboxDockerlessCase{user: microsandboxRootUser, policy: "strict"}, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ), + ginkgo.Entry( + "nonroot off/private runs with literal host permissions", + microsandboxDockerlessCase{ + user: microsandboxDockerlessUser, + policy: microsandboxDockerlessOff, + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ), + ginkgo.Entry( + "nonroot strict rejects fresh creation before VM or volume mutation", + microsandboxDockerlessCase{ + user: microsandboxDockerlessUser, + policy: "strict", + refused: true, + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ), + ginkgo.Entry( + "nonroot relaxed rejects fresh creation before VM or volume mutation", + microsandboxDockerlessCase{ + user: microsandboxDockerlessUser, + policy: "relaxed", + refused: true, + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ), + ) + }) + } + }) + +func newMicrosandboxDockerlessFixture( + ctx context.Context, providerName, providerSource string, +) *microsandboxDockerlessFixtureState { + if runtime.GOOS != osLinux { + ginkgo.Skip("Dockerless parity fixture currently targets the supported Linux KVM job") + } + skipIfNoMicrosandbox(ctx) + ginkgo.GinkgoT().Setenv("DEVSY_HOME", ginkgo.GinkgoT().TempDir()) + ginkgo.GinkgoT().Setenv("DEVSY_CONFIG", "") + initialDir, err := os.Getwd() + framework.ExpectNoError(err) + workspace, err := framework.CopyToTempDir(microsandboxDockerlessFixture) + framework.ExpectNoError(err) + ginkgo.DeferCleanup(framework.CleanupTempDir, initialDir, workspace) + fixture := µsandboxDockerlessFixtureState{ + f: framework.NewDefaultFramework(filepath.Join(initialDir, "bin")), + workspace: workspace, + uid: encoding.CreateNewUID("", filepath.Base(workspace)), + provider: providerName, + } + fixture.label = "devsy.dockerless-parity=" + fixture.uid + assertMicrosandboxEphemeralAbsent(ctx, fixture.label) + fixture.assertSandboxAbsent(ctx) + gomega.Expect(fixture.volumeExists(ctx)).To(gomega.BeFalse(), "never adopt an existing volume") + framework.ExpectNoError(fixture.f.DevsyProviderAdd(ctx, providerSource, "--name", providerName)) + fixture.registerCleanup() + fixture.dockerLog = denyMicrosandboxHostDocker(ctx) + return fixture +} + +func (fixture *microsandboxDockerlessFixtureState) registerCleanup() { + ginkgo.DeferCleanup(func() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + return fixture.f.DevsyProviderDelete(ctx, fixture.provider) + }) + ginkgo.DeferCleanup(func() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + volumes, err := microsandboxDockerlessVolumes(ctx) + if err != nil || !volumes[fixture.volume()] { + return err + } + out, err := microsandboxResourceCommand(ctx, "volume", "rm", fixture.volume()) + if err != nil { + return fmt.Errorf("remove owned Dockerless volume: %w: %s", err, out) + } + return nil + }) + ginkgo.DeferCleanup(func() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + out, err := microsandboxResourceCommand(ctx, "list", "--quiet") + if err != nil { + return err + } + for name := range strings.FieldsSeq(string(out)) { + if name != "devsy-"+fixture.uid { + continue + } + out, err = microsandboxResourceCommand(ctx, "remove", "--force", name) + if err != nil { + return fmt.Errorf("remove owned Dockerless sandbox: %w: %s", err, out) + } + } + return nil + }) + ginkgo.DeferCleanup(fixture.f.CleanupWorkspace, fixture.workspace) +} + +func (fixture *microsandboxDockerlessFixtureState) configure( + ctx context.Context, testCase microsandboxDockerlessCase, +) { + framework.ExpectNoError(fixture.f.DevsyProviderUse(ctx, fixture.provider, + microsandboxResourceOption, "MICROSANDBOX_WORKSPACE_STAT_VIRTUALIZATION="+testCase.policy, + microsandboxResourceOption, "MICROSANDBOX_WORKSPACE_HOST_PERMISSIONS=private")) + cfg := struct { + Build config.ConfigBuildOptions `json:"build"` + Features map[string]any `json:"features"` + ContainerUser string `json:"containerUser"` + RemoteUser string `json:"remoteUser"` + UpdateRemoteUserUID bool `json:"updateRemoteUserUID"` + UserEnvProbe config.UserEnvProbe `json:"userEnvProbe"` + RemoteEnv map[string]string `json:"remoteEnv"` + WaitFor string `json:"waitFor"` + PostStartCommand string `json:"postStartCommand"` + }{ + Build: config.ConfigBuildOptions{ + Dockerfile: microsandboxDockerlessDockerfile, + Context: ".", + }, + Features: map[string]any{"./features/parity": map[string]any{}}, + ContainerUser: testCase.user, + RemoteUser: testCase.user, + UserEnvProbe: config.NoneProbe, + RemoteEnv: map[string]string{"DOCKER_CONFIG": "${containerEnv:DOCKER_CONFIG}"}, + WaitFor: "postStartCommand", + PostStartCommand: microsandboxDockerlessCommand + " > /tmp/devsy-dockerless-post-start.env", + } + data, err := json.MarshalIndent(cfg, "", " ") + framework.ExpectNoError(err) + fixture.configBefore = data + framework.ExpectNoError( + os.WriteFile(filepath.Join(fixture.workspace, ".devcontainer.json"), data, 0o600), + ) + file, err := os.OpenFile( + filepath.Join(fixture.workspace, microsandboxDockerlessDockerfile), + os.O_APPEND|os.O_WRONLY, + 0, + ) + framework.ExpectNoError(err) + _, writeErr := fmt.Fprintf(file, "USER %s\n", testCase.user) + closeErr := file.Close() + framework.ExpectNoError(writeErr) + framework.ExpectNoError(closeErr) + // Off/private must be usable by a guest UID different from the host owner. + // These permissions apply only to this temporary fixture, never user paths. + // #nosec G302 -- Only the owned temporary source must be writable by the distinct guest UID. + framework.ExpectNoError(os.Chmod(fixture.workspace, 0o777)) + fixture.workspaceOwner = microsandboxHostOwner(ctx, fixture.workspace) + sentinel := filepath.Join(fixture.workspace, microsandboxDockerlessSentinel) + framework.ExpectNoError(os.WriteFile(sentinel, []byte("host-preserved\n"), 0o600)) + // #nosec G302 -- The owned temporary sentinel must be readable by the distinct guest UID. + framework.ExpectNoError(os.Chmod(sentinel, 0o644)) + fixture.sentinelOwner = microsandboxHostOwner(ctx, sentinel) + info, err := os.Stat(sentinel) + framework.ExpectNoError(err) + fixture.sentinelMode = info.Mode().Perm() + if testCase.user != microsandboxRootUser && testCase.policy == microsandboxDockerlessOff { + for _, owner := range []string{fixture.workspaceOwner, fixture.sentinelOwner} { + gomega.Expect(strings.SplitN(owner, ":", 2)[0]).NotTo(gomega.Equal("31001"), + "off-policy proof requires a host owner distinct from the guest UID") + } + } +} + +func denyMicrosandboxHostDocker(ctx context.Context) string { + dir := ginkgo.GinkgoT().TempDir() + logPath := filepath.Join(dir, "docker-attempts") + framework.ExpectNoError(os.WriteFile(logPath, nil, 0o600)) + script := "#!/bin/sh\nprintf '%s\\n' \"$*\" >> " + microsandboxDockerlessQuote(logPath) + + "\nprintf '%s\\n' 'Docker unavailable in Dockerless parity fixture' >&2\nexit 127\n" + // #nosec G306 -- This owned executable fixture denies every host Docker invocation. + framework.ExpectNoError(os.WriteFile(filepath.Join(dir, "docker"), []byte(script), 0o700)) + ginkgo.GinkgoT().Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + ginkgo.GinkgoT().Setenv("DOCKER_HOST", "unix://"+filepath.Join(dir, "absent-docker.sock")) + ginkgo.GinkgoT().Setenv("DOCKER_CONTEXT", "") + ginkgo.GinkgoT().Setenv("DOCKER_API_VERSION", "") + ginkgo.GinkgoT().Setenv("DOCKER_TLS_VERIFY", "") + ginkgo.GinkgoT().Setenv("DOCKER_CERT_PATH", "") + ginkgo.GinkgoT().Setenv("DOCKER_CONFIG", ginkgo.GinkgoT().TempDir()) + probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + command := exec.CommandContext(probeCtx, "docker", "version") + command.WaitDelay = 2 * time.Second + out, err := command.CombinedOutput() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(string(out)). + To(gomega.ContainSubstring("Docker unavailable in Dockerless parity fixture")) + framework.ExpectNoError(probeCtx.Err()) + assertMicrosandboxDockerAPIDenied(ctx) + framework.ExpectNoError(os.WriteFile(logPath, nil, 0o600)) + return logPath +} + +func assertMicrosandboxDockerAPIDenied(ctx context.Context) { + probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + client, err := docker.NewClient(probeCtx) + framework.ExpectNoError(err) + defer func() { framework.ExpectNoError(client.Close()) }() + gomega.Expect(client.DaemonHost()).To(gomega.Equal(os.Getenv("DOCKER_HOST"))) + _, err = client.Ping(probeCtx) + gomega.Expect(err). + To(gomega.HaveOccurred(), "the owned absent Docker socket must be unreachable") + framework.ExpectNoError(probeCtx.Err()) + gomega.Expect(errors.Is(err, context.Canceled)).To(gomega.BeFalse()) + gomega.Expect(errors.Is(err, context.DeadlineExceeded)).To(gomega.BeFalse()) +} + +func (fixture *microsandboxDockerlessFixtureState) up( + ctx context.Context, + mustBuild bool, + extra ...string, +) { + stdout, stderr, err := fixture.upOutput(ctx, extra...) + attempts, readErr := os.ReadFile(fixture.dockerLog) + framework.ExpectNoError(readErr) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), + "single up attempt: stdout=%s stderr=%s denied Docker calls=%s", stdout, stderr, attempts) + output := stdout + stderr + gomega.Expect(output).NotTo(gomega.ContainSubstring("failed to configure docker credentials")) + gomega.Expect(output).NotTo(gomega.ContainSubstring("docker credentials disabled")) + if mustBuild { + gomega.Expect(output).To(gomega.ContainSubstring( + "Wrote docker credentials helper to /.dockerless/.docker/docker-credentials-")) + gomega.Expect(output).To(gomega.ContainSubstring("starting dockerless build")) + gomega.Expect(output).To(gomega.ContainSubstring("dockerless build completed")) + } else { + gomega.Expect(output).NotTo(gomega.ContainSubstring("starting dockerless build")) + } +} + +func (fixture *microsandboxDockerlessFixtureState) upOutput( + ctx context.Context, extra ...string, +) (string, string, error) { + args := []string{ + cmdWorkspace, + "up", + names.Flag(names.Debug), + names.Flag(names.IDE), + string(devsyconfig.IDENone), + fixture.workspace, + "--uid", + fixture.uid, + "--id-label", + fixture.label, + "--devcontainer", + ".devcontainer.json", + "--force-dockerless", + } + return fixture.f.ExecCommandCapture(ctx, append(args, extra...)) +} + +func (fixture *microsandboxDockerlessFixtureState) verify(ctx context.Context, user string) { + identity := "0:0" + if user != microsandboxRootUser { + identity = "31001:31001" + } + expected := "command=" + microsandboxDockerlessRoot + "/bin/" + microsandboxDockerlessCommand + + "\nPATH=" + microsandboxDockerlessPath + "\nDOCKER_CONFIG=" + microsandboxDockerlessRoot + "/docker-config" + + "\nimage=from-image\nfeature=feature-parity\nidentity=" + identity + + "\ndockerfile-parity\nfeature-parity\n" + // A bare command and disabled userEnvProbe prevent a login shell from hiding image PATH loss. + gomega.Expect(fixture.ssh(ctx, microsandboxDockerlessCommand)).To(gomega.Equal(expected)) + gomega.Expect(fixture.ssh(ctx, "cat /tmp/devsy-dockerless-post-start.env")). + To(gomega.Equal(expected)) + fixture.assertSourcePreserved(ctx) + gomega.Expect(fixture.volumeExists(ctx)).To(gomega.BeTrue()) + info, err := fixture.f.FindWorkspace(ctx, fixture.workspace) + framework.ExpectNoError(err) + gomega.Expect(info.UID).To(gomega.Equal(fixture.uid)) + if user == microsandboxRootUser { + fixture.verifyImageEnvironment(ctx) + } +} + +func (fixture *microsandboxDockerlessFixtureState) verifyImageEnvironment(ctx context.Context) { + var image v1.ConfigFile + framework.ExpectNoError( + json.Unmarshal([]byte(fixture.ssh(ctx, "cat /.dockerless/image.json")), &image), + ) + gomega.Expect(image.Config.Env).To(gomega.ContainElement("PATH=" + microsandboxDockerlessPath)) + gomega.Expect(image.Config.Env). + To(gomega.ContainElement("DOCKER_CONFIG=" + microsandboxDockerlessRoot + "/docker-config")) + gomega.Expect(image.Config.Env).To(gomega.ContainElement("DEVSY_DOCKERLESS_IMAGE=from-image")) + var persisted envfile.EnvFile + framework.ExpectNoError( + json.Unmarshal([]byte(fixture.ssh(ctx, "cat /etc/envfile.json")), &persisted), + ) + gomega.Expect(persisted.Env["PATH"]).To(gomega.Equal(microsandboxDockerlessPath)) + gomega.Expect(persisted.Env["DOCKER_CONFIG"]). + To(gomega.Equal(microsandboxDockerlessRoot + "/docker-config")) + fixture.ssh( + ctx, + "for dir in /.dockerless/.docker/docker-credentials-*; do test ! -e \"$dir\" || exit 1; done", + ) +} + +func (fixture *microsandboxDockerlessFixtureState) lifecycle(ctx context.Context) { + sandbox := "devsy-" + fixture.uid + createdAt := microsandboxCreationTime(ctx, sandbox) + bootID := microsandboxStorageBootID(ctx, fixture.f, fixture.workspace) + fixture.ssh(ctx, "printf 'root-preserved\\n' > "+microsandboxDockerlessRootMarker) + fixture.up(ctx, false) + fixture.verify(ctx, microsandboxRootUser) + gomega.Expect(microsandboxCreationTime(ctx, sandbox)).To(gomega.Equal(createdAt)) + gomega.Expect(microsandboxStorageBootID(ctx, fixture.f, fixture.workspace)). + To(gomega.Equal(bootID)) + gomega.Expect(fixture.ssh(ctx, "cat "+microsandboxDockerlessRootMarker)). + To(gomega.Equal("root-preserved\n")) + framework.ExpectNoError(fixture.f.DevsyWorkspaceStop(ctx, fixture.workspace)) + assertMicrosandboxEphemeralStopped(ctx, sandbox, createdAt) + fixture.up(ctx, false) + fixture.verify(ctx, microsandboxRootUser) + gomega.Expect(microsandboxCreationTime(ctx, sandbox)).To(gomega.Equal(createdAt)) + restartedBootID := microsandboxStorageBootID(ctx, fixture.f, fixture.workspace) + gomega.Expect(restartedBootID).NotTo(gomega.Equal(bootID)) + gomega.Expect(fixture.ssh(ctx, "cat "+microsandboxDockerlessRootMarker)). + To(gomega.Equal("root-preserved\n")) + fixture.up(ctx, true, "--recreate") + fixture.verify(ctx, microsandboxRootUser) + gomega.Expect(microsandboxCreationTime(ctx, sandbox)).NotTo(gomega.Equal(createdAt)) + gomega.Expect(microsandboxStorageBootID(ctx, fixture.f, fixture.workspace)). + NotTo(gomega.Equal(restartedBootID)) + fixture.ssh(ctx, "test ! -e "+microsandboxDockerlessRootMarker) +} + +func (fixture *microsandboxDockerlessFixtureState) verifyNonrootWrite(ctx context.Context) { + path := filepath.Join("/workspaces", filepath.Base(fixture.workspace), "dockerless-guest-write") + fixture.ssh(ctx, "printf 'guest-write\\n' > "+microsandboxDockerlessQuote(path)) + data, err := os.ReadFile(filepath.Join(fixture.workspace, "dockerless-guest-write")) + framework.ExpectNoError(err) + gomega.Expect(string(data)).To(gomega.Equal("guest-write\n")) + fixture.assertSourcePreserved(ctx) +} + +func (fixture *microsandboxDockerlessFixtureState) assertRefused(ctx context.Context) { + stdout, stderr, err := fixture.upOutput(ctx) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(stdout + stderr).To(gomega.ContainSubstring( + "cannot resolve a non-root Dockerless workspace owner before VM creation")) + gomega.Expect(stdout + stderr).NotTo(gomega.ContainSubstring("starting dockerless build")) + assertMicrosandboxEphemeralAbsent(ctx, fixture.label) + fixture.assertSandboxAbsent(ctx) + gomega.Expect(fixture.volumeExists(ctx)).To(gomega.BeFalse()) + fixture.assertSourcePreserved(ctx) +} + +func (fixture *microsandboxDockerlessFixtureState) assertSourcePreserved(ctx context.Context) { + gomega.Expect(microsandboxHostOwner(ctx, fixture.workspace)). + To(gomega.Equal(fixture.workspaceOwner)) + dirInfo, err := os.Stat(fixture.workspace) + framework.ExpectNoError(err) + gomega.Expect(dirInfo.Mode().Perm()).To(gomega.Equal(os.FileMode(0o777))) + sentinel := filepath.Join(fixture.workspace, microsandboxDockerlessSentinel) + // #nosec G304 -- The path is the sentinel in this test-owned temporary workspace. + data, err := os.ReadFile(sentinel) + framework.ExpectNoError(err) + gomega.Expect(string(data)).To(gomega.Equal("host-preserved\n")) + gomega.Expect(microsandboxHostOwner(ctx, sentinel)).To(gomega.Equal(fixture.sentinelOwner)) + info, err := os.Stat(sentinel) + framework.ExpectNoError(err) + gomega.Expect(info.Mode().Perm()).To(gomega.Equal(fixture.sentinelMode)) + data, err = os.ReadFile(filepath.Join(fixture.workspace, ".devcontainer.json")) + framework.ExpectNoError(err) + gomega.Expect(data).To(gomega.Equal(fixture.configBefore)) +} + +func (fixture *microsandboxDockerlessFixtureState) ssh(ctx context.Context, command string) string { + commandCtx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + out, err := fixture.f.DevsySSHOnce(commandCtx, fixture.workspace, command) + framework.ExpectNoError(err, out) + return out +} + +func (fixture *microsandboxDockerlessFixtureState) volume() string { + return "dockerless-" + fixture.uid +} + +func (fixture *microsandboxDockerlessFixtureState) volumeExists(ctx context.Context) bool { + volumes, err := microsandboxDockerlessVolumes(ctx) + framework.ExpectNoError(err) + return volumes[fixture.volume()] +} + +func microsandboxDockerlessVolumes(ctx context.Context) (map[string]bool, error) { + out, err := microsandboxResourceCommand(ctx, "volume", "list", "--quiet") + if err != nil { + return nil, err + } + volumes := make(map[string]bool) + for name := range strings.FieldsSeq(string(out)) { + volumes[name] = true + } + return volumes, nil +} + +func microsandboxDockerlessQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" +} + +func (fixture *microsandboxDockerlessFixtureState) assertSandboxAbsent(ctx context.Context) { + out, err := microsandboxResourceCommand(ctx, "list", "--quiet") + framework.ExpectNoError(err) + gomega.Expect(strings.Fields(string(out))).NotTo(gomega.ContainElement("devsy-" + fixture.uid)) +} diff --git a/e2e/tests/up/testdata/microsandbox-dockerless/Dockerfile b/e2e/tests/up/testdata/microsandbox-dockerless/Dockerfile new file mode 100644 index 0000000000..13b5732362 --- /dev/null +++ b/e2e/tests/up/testdata/microsandbox-dockerless/Dockerfile @@ -0,0 +1,9 @@ +FROM ghcr.io/devsy-org/test-images/base:alpine +RUN addgroup -g 31001 dlparity && adduser -D -u 31001 -G dlparity dlparity \ + && mkdir -p /opt/devsy-dockerless-parity/bin /opt/devsy-dockerless-parity/docker-config \ + && printf '%s\n' dockerfile-parity > /usr/local/share/devsy-dockerless-marker +COPY devsy-dockerless-parity-check /opt/devsy-dockerless-parity/bin/devsy-dockerless-parity-check +RUN chmod 0755 /opt/devsy-dockerless-parity/bin/devsy-dockerless-parity-check +ENV PATH=/opt/devsy-dockerless-parity/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV DOCKER_CONFIG=/opt/devsy-dockerless-parity/docker-config +ENV DEVSY_DOCKERLESS_IMAGE=from-image diff --git a/e2e/tests/up/testdata/microsandbox-dockerless/devsy-dockerless-parity-check b/e2e/tests/up/testdata/microsandbox-dockerless/devsy-dockerless-parity-check new file mode 100755 index 0000000000..dcc6ab189d --- /dev/null +++ b/e2e/tests/up/testdata/microsandbox-dockerless/devsy-dockerless-parity-check @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +printf 'command=%s\nPATH=%s\nDOCKER_CONFIG=%s\nimage=%s\nfeature=%s\nidentity=%s:%s\n' \ + "$(command -v devsy-dockerless-parity-check)" "$PATH" "${DOCKER_CONFIG-}" \ + "${DEVSY_DOCKERLESS_IMAGE-}" "${DEVSY_FEATURE_PARITY-}" "$(id -u)" "$(id -g)" +cat /usr/local/share/devsy-dockerless-marker +cat /usr/local/share/devsy-feature-parity diff --git a/e2e/tests/up/testdata/microsandbox-dockerless/features/parity/devcontainer-feature.json b/e2e/tests/up/testdata/microsandbox-dockerless/features/parity/devcontainer-feature.json new file mode 100644 index 0000000000..eead25af16 --- /dev/null +++ b/e2e/tests/up/testdata/microsandbox-dockerless/features/parity/devcontainer-feature.json @@ -0,0 +1,8 @@ +{ + "id": "parity", + "version": "1.0.0", + "name": "MicroSandbox Dockerless parity marker", + "containerEnv": { + "DEVSY_FEATURE_PARITY": "feature-parity" + } +} diff --git a/e2e/tests/up/testdata/microsandbox-dockerless/features/parity/install.sh b/e2e/tests/up/testdata/microsandbox-dockerless/features/parity/install.sh new file mode 100755 index 0000000000..5c7bbb68a5 --- /dev/null +++ b/e2e/tests/up/testdata/microsandbox-dockerless/features/parity/install.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +mkdir -p /usr/local/share +printf '%s\n' feature-parity >/usr/local/share/devsy-feature-parity diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx index 738a288350..be936d0567 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx @@ -109,6 +109,14 @@ The `up-provider-microsandbox-prebuilds` label publishes a Dockerfile with a loc DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-prebuilds ``` -These scenarios do not establish complete parity. Dockerless operation, logs, cancellation, and runtime compatibility failures still require coverage before replacing the built-in provider. Green lifecycle, image, mount, resource, persistent-storage, ephemeral-root, TCP-egress, and prebuild checks alone do not authorize that cutover. Egress coverage does not establish UDP, DNS, TLS, ingress, or live-policy modification behavior. +The `up-provider-microsandbox-dockerless` label runs four scenarios against each provider with host Docker unavailable. It explicitly selects the existing Dockerless path; it does not test automatic fallback. Both the Docker CLI and Go client are directed to test-owned failures, while a real guest build must install a Dockerfile and local Feature. Image PATH, credentials-directory configuration, Feature environment and a bare-name executable are checked through setup hooks and SSH. The root scenario repeats up, stop/start and recreation, checking build reuse, VM identity, root-file persistence and preserved host data. A fixed non-root developer must succeed with `off` stat virtualization and private host permissions; fresh creation under `strict` or `relaxed` must fail before VM or build-volume creation, preserving host data and permissions. + +This suite requires Linux with KVM and public access to the base and Dockerless builder images. It does not require a usable host Docker daemon. Each scenario has a ten-minute deadline, with a 20-minute CI test deadline and a 25-minute job deadline. Cleanup removes only the fixture's workspace, VM, build volume and provider. + +```sh +DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-dockerless +``` + +These scenarios do not establish complete parity. Logs, cancellation, and runtime compatibility failures still require coverage before replacing the built-in provider. Green lifecycle, image, mount, resource, persistent-storage, ephemeral-root, TCP-egress, prebuild, and Dockerless checks alone do not authorize that cutover. Dockerless refusal checks cover fresh creation, not preservation after an invalid recreation request. Egress coverage does not establish UDP, DNS, TLS, ingress, or live-policy modification behavior. Prebuild invalidation coverage changes the Dockerfile; it does not independently establish hashing of local Feature script contents. From d8326004ebd0974bc6f5869a04d7c0370f14dda6 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 11 Oct 2026 11:31:16 -0600 Subject: [PATCH 2/2] fix(runtime): launch bootstrap SSH with literal arguments --- pkg/devcontainer/setup.go | 18 +- pkg/devcontainer/setup_test.go | 233 ++++++++++++++++++ .../developing-providers/runtime-protocol.mdx | 2 + 3 files changed, 248 insertions(+), 5 deletions(-) diff --git a/pkg/devcontainer/setup.go b/pkg/devcontainer/setup.go index 0beca0de45..f457cb682a 100644 --- a/pkg/devcontainer/setup.go +++ b/pkg/devcontainer/setup.go @@ -561,7 +561,8 @@ func (r *runner) executeSetup( ) } - sshTunnelCmd := r.buildSSHTunnelCommand() + sshTunnelArgv := r.buildSSHTunnelArgv() + sshTunnelCmd := shellescape.QuoteCommand(sshTunnelArgv) agentInjectFunc := func( cancelCtx context.Context, @@ -578,6 +579,7 @@ func (r *runner) executeSetup( Stderr: writer, }, agent.ExecStartupWatchdogOptions{}, + sshTunnelArgv..., ) } @@ -595,6 +597,7 @@ func (r *runner) execSetupSSHServer( ctx context.Context, req agent.ExecRequest, watchdogOpts agent.ExecStartupWatchdogOptions, + argv ...string, ) error { onStartupSilence := watchdogOpts.OnStartupSilence watchdogOpts.OnStartupSilence = func() { @@ -608,6 +611,11 @@ func (r *runner) execSetupSSHServer( return agent.ExecWithStartupWatchdog( ctx, func(ctx context.Context, req agent.ExecRequest) error { + if execDriver, ok := r.driver.(driver.ArgvExecDriver); ok && len(argv) > 0 { + return execDriver.CommandContainerArgv(ctx, r.id, argv, driver.Streams{ + Stdin: req.Stdin, Stdout: req.Stdout, Stderr: req.Stderr, + }) + } return r.driver.CommandDevContainer(ctx, &driver.CommandParams{ WorkspaceID: r.id, User: containerRootUser, @@ -623,9 +631,9 @@ func (r *runner) execSetupSSHServer( ) } -func (r *runner) buildSSHTunnelCommand() string { +func (r *runner) buildSSHTunnelArgv() []string { args := []string{ - shellescape.Quote(r.agentContainerPath()), + r.agentContainerPath(), "internal", "ssh-server", names.Flag(names.Stdio), } @@ -633,13 +641,13 @@ func (r *runner) buildSSHTunnelCommand() string { args = append( args, names.Flag(names.ReuseSSHAuthSock), - shellescape.Quote(r.workspaceConfig.CLIOptions.SSHAuthSockID), + r.workspaceConfig.CLIOptions.SSHAuthSockID, ) } if r.isDebugMode() { args = append(args, names.Flag(names.Debug)) } - return strings.Join(args, " ") + return args } func getRelativeDevContainerJson(origin, localWorkspaceFolder string) string { diff --git a/pkg/devcontainer/setup_test.go b/pkg/devcontainer/setup_test.go index 47da290990..72e3dec24d 100644 --- a/pkg/devcontainer/setup_test.go +++ b/pkg/devcontainer/setup_test.go @@ -10,17 +10,20 @@ import ( "testing" "time" + "al.essio.dev/pkg/shellescape" "github.com/devsy-org/devsy/pkg/agent" "github.com/devsy-org/devsy/pkg/agent/delivery" pkgconfig "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/docker" "github.com/devsy-org/devsy/pkg/driver" + "github.com/devsy-org/devsy/pkg/log" provider2 "github.com/devsy-org/devsy/pkg/provider" "github.com/devsy-org/devsy/pkg/types" "github.com/devsy-org/devsy/pkg/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" ) const ( @@ -541,3 +544,233 @@ func TestFallbackAgentDeliverySuccessfulPaths(t *testing.T) { }) } } + +type setupSSHArgvDriver struct { + setupSSHDriver + argv func(context.Context, string, []string, driver.Streams) error +} + +func (d *setupSSHArgvDriver) CommandContainerArgv( + ctx context.Context, workspaceID string, argv []string, streams driver.Streams, +) error { + return d.argv(ctx, workspaceID, argv, streams) +} + +func setupSSHArgvRunner(d driver.Driver) *runner { + r := newTestRunner(d) + r.workspaceConfig.Agent.Driver = provider2.KubernetesDriver + r.workspaceConfig.Agent.Kubernetes.AgentInstallPath = "/agent dir/devsy';$(touch never)" + r.workspaceConfig.Workspace = &provider2.Workspace{} + r.workspaceConfig.Workspace.IDE.Name = string(pkgconfig.IDECodeServer) + r.workspaceConfig.CLIOptions.SSHAuthSockID = "auth sock';$(touch never)" + return r +} + +func TestBuildSSHTunnelArgvPreservesLiteralArguments(t *testing.T) { + for _, debug := range []bool{false, true} { + level := zapcore.InfoLevel + if debug { + level = zapcore.DebugLevel + } + log.InitTestObserved(t, level) + r := setupSSHArgvRunner(&mockDriver{}) + want := []string{ + r.agentContainerPath(), "internal", "ssh-server", "--stdio", + "--reuse-ssh-auth-sock", r.workspaceConfig.CLIOptions.SSHAuthSockID, + } + if debug { + want = append(want, "--debug") + } + assert.Equal(t, want, r.buildSSHTunnelArgv()) + r.workspaceConfig.Workspace.IDE.Name = "none" + want = []string{r.agentContainerPath(), "internal", "ssh-server", "--stdio"} + if debug { + want = append(want, "--debug") + } + assert.Equal(t, want, r.buildSSHTunnelArgv()) + } +} + +func TestExecSetupSSHServerGeneratedArgv(t *testing.T) { + var calls, shellCalls int + var gotID string + var gotArgv []string + input := bytes.NewBufferString("stdin payload") + stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{} + payload := []byte{0, 255, 's', 'e', 'c', 'r', 'e', 't', '\n'} + d := &setupSSHArgvDriver{} + d.command = func(context.Context, *driver.CommandParams) error { + shellCalls++ + return errors.New("bootstrap shell is absent") + } + d.argv = func(_ context.Context, id string, argv []string, streams driver.Streams) error { + calls++ + gotID, gotArgv = id, append([]string(nil), argv...) + assert.Same(t, input, streams.Stdin) + if _, err := streams.Stdout.Write(payload); err != nil { + return err + } + _, err := streams.Stderr.Write(payload) + return err + } + r := setupSSHArgvRunner(d) + argv := r.buildSSHTunnelArgv() + err := r.execSetupSSHServer(context.Background(), agent.ExecRequest{ + Command: shellescape.QuoteCommand(argv), Stdin: input, Stdout: stdout, Stderr: stderr, + }, agent.ExecStartupWatchdogOptions{}, argv...) + require.NoError(t, err) + assert.Equal(t, 1, calls) + assert.Zero(t, shellCalls) + assert.Equal(t, r.id, gotID) + assert.Equal(t, argv, gotArgv) + assert.Equal(t, payload, stdout.Bytes()) + assert.Equal(t, payload, stderr.Bytes()) +} + +func TestExecSetupSSHServerArgvErrorDoesNotRetryShell(t *testing.T) { + var argvCalls, shellCalls int + wantErr := errors.New("agent exec failed") + d := &setupSSHArgvDriver{} + d.command = func(context.Context, *driver.CommandParams) error { shellCalls++; return nil } + d.argv = func(context.Context, string, []string, driver.Streams) error { + argvCalls++ + return wantErr + } + r := setupSSHArgvRunner(d) + argv := r.buildSSHTunnelArgv() + err := r.execSetupSSHServer(context.Background(), agent.ExecRequest{ + Command: shellescape.QuoteCommand(argv), + }, agent.ExecStartupWatchdogOptions{}, argv...) + assert.ErrorIs(t, err, wantErr) + assert.Equal(t, 1, argvCalls) + assert.Zero(t, shellCalls) +} + +func TestExecSetupSSHServerGeneratedArgvFallsBackForUnsupportedDriver(t *testing.T) { + var got *driver.CommandParams + wantErr := errors.New("existing command result") + d := &setupSSHDriver{command: func(_ context.Context, params *driver.CommandParams) error { + got = params + return wantErr + }} + r := setupSSHArgvRunner(d) + argv := r.buildSSHTunnelArgv() + command := shellescape.QuoteCommand(argv) + err := r.execSetupSSHServer(context.Background(), agent.ExecRequest{ + Command: command, + }, agent.ExecStartupWatchdogOptions{}, argv...) + assert.ErrorIs(t, err, wantErr) + require.NotNil(t, got) + assert.Equal(t, command, got.Command) + assert.Equal(t, r.id, got.WorkspaceID) + assert.Equal(t, containerRootUser, got.User) + assert.True(t, got.RawStdout) +} + +func TestExecSetupSSHServerArgvStartupSilenceClosesOwnedInput(t *testing.T) { + stdin, peer := net.Pipe() + defer func() { _ = stdin.Close(); _ = peer.Close() }() + done := make(chan struct{}) + d := &setupSSHArgvDriver{} + d.command = func(context.Context, *driver.CommandParams) error { + return errors.New("unexpected shell invocation") + } + d.argv = func(ctx context.Context, _ string, _ []string, streams driver.Streams) error { + defer close(done) + _, _ = io.Copy(io.Discard, streams.Stdin) + return ctx.Err() + } + r := newTestRunner(d) + err := r.execSetupSSHServer(context.Background(), agent.ExecRequest{Stdin: stdin}, + agent.ExecStartupWatchdogOptions{ + Timeout: 50 * time.Millisecond, TerminationWaitTimeout: time.Second, + }, "agent", "internal", "ssh-server", "--stdio") + var silenceErr *agent.ExecStartupSilenceError + assert.ErrorAs(t, err, &silenceErr) + select { + case <-done: + default: + t.Error("startup silence returned without joining the blocked input copy") + } + // Failure cleanup closes the owned input before the bounded join too. + _ = stdin.Close() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("argv command did not join after owned input closure") + } +} + +func TestExecSetupSSHServerArgvActivityRetainsCallerCancellation(t *testing.T) { + for _, output := range []string{"stdout", "stderr"} { + t.Run(output, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + done := make(chan struct{}) + d := &setupSSHArgvDriver{} + d.command = func(context.Context, *driver.CommandParams) error { + return errors.New("unexpected shell invocation") + } + d.argv = func(ctx context.Context, _ string, _ []string, streams driver.Streams) error { + defer close(done) + writer := streams.Stdout + if output == "stderr" { + writer = streams.Stderr + } + if _, err := io.WriteString(writer, "ready"); err != nil { + return err + } + <-ctx.Done() + return ctx.Err() + } + r := newTestRunner(d) + err := r.execSetupSSHServer( + ctx, + agent.ExecRequest{}, + agent.ExecStartupWatchdogOptions{ + Timeout: 50 * time.Millisecond, + }, + "agent", + "--stdio", + ) + assert.ErrorIs(t, err, context.DeadlineExceeded) + var silenceErr *agent.ExecStartupSilenceError + assert.NotErrorAs(t, err, &silenceErr) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("argv command did not join caller cancellation") + } + }) + } +} + +func TestBuildSetupCommandKeepsQuotedAgentPath(t *testing.T) { + r := setupSSHArgvRunner(&mockDriver{}) + command := r.buildSetupCommand("setup info", "workspace info") + assert.Contains( + t, + command, + shellescape.Quote(r.agentContainerPath())+" internal agent container setup ", + ) +} + +func TestExecSetupSSHServerWithoutGeneratedArgvKeepsShell(t *testing.T) { + var argvCalls, shellCalls int + command := "literal shell command; do not parse into argv" + d := &setupSSHArgvDriver{} + d.command = func(_ context.Context, params *driver.CommandParams) error { + shellCalls++ + assert.Equal(t, command, params.Command) + assert.Equal(t, containerRootUser, params.User) + assert.True(t, params.RawStdout) + return nil + } + d.argv = func(context.Context, string, []string, driver.Streams) error { argvCalls++; return nil } + r := newTestRunner(d) + require.NoError(t, r.execSetupSSHServer(context.Background(), agent.ExecRequest{ + Command: command, + }, agent.ExecStartupWatchdogOptions{})) + assert.Equal(t, 1, shellCalls) + assert.Zero(t, argvCalls) +} diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx index be936d0567..e2e470e2f7 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx @@ -109,6 +109,8 @@ The `up-provider-microsandbox-prebuilds` label publishes a Dockerfile with a loc DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-prebuilds ``` +The host starts its generated bootstrap SSH helper with literal arguments when the runtime supports argv execution, so setup can begin before the Dockerless filesystem provides `/bin/sh`. Runtimes without that capability retain command-string execution. + The `up-provider-microsandbox-dockerless` label runs four scenarios against each provider with host Docker unavailable. It explicitly selects the existing Dockerless path; it does not test automatic fallback. Both the Docker CLI and Go client are directed to test-owned failures, while a real guest build must install a Dockerfile and local Feature. Image PATH, credentials-directory configuration, Feature environment and a bare-name executable are checked through setup hooks and SSH. The root scenario repeats up, stop/start and recreation, checking build reuse, VM identity, root-file persistence and preserved host data. A fixed non-root developer must succeed with `off` stat virtualization and private host permissions; fresh creation under `strict` or `relaxed` must fail before VM or build-volume creation, preserving host data and permissions. This suite requires Linux with KVM and public access to the base and Dockerless builder images. It does not require a usable host Docker daemon. Each scenario has a ten-minute deadline, with a 20-minute CI test deadline and a 25-minute job deadline. Cleanup removes only the fixture's workspace, VM, build volume and provider.