From 1604ddea88295f92add850cbc6e8704898ad41b3 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 2 Nov 2026 15:15:06 -0600 Subject: [PATCH 01/20] feat(provider): prepare Docker Sandboxes machine support --- .../agentworkspace/docker_darwin_test.go | 43 ++ cmd/internal/agentworkspace/up.go | 55 ++- cmd/internal/agentworkspace/up_test.go | 201 ++++++++ cmd/provider/set_source_init_failure_test.go | 4 +- .../components/provider/ProviderWizard.svelte | 5 + .../provider/ProviderWizard.test.ts | 43 ++ .../clientimplementation/provider_cancel.go | 126 +++++ .../provider_cancel_test.go | 443 ++++++++++++++++++ .../provider_cancel_unix.go | 21 + .../provider_cancel_windows.go | 53 +++ .../provider_cancel_windows_test.go | 45 ++ .../clientimplementation/workspace_client.go | 110 +++-- .../workspace_client_test.go | 65 +++ pkg/config/env.go | 2 + pkg/options/resolve.go | 3 + pkg/options/resolve_docker_test.go | 58 +++ pkg/provider/parse.go | 11 + pkg/provider/parse_docker_test.go | 46 ++ pkg/provider/parse_driver_test.go | 33 ++ pkg/provider/provider.go | 12 +- pkg/shell/provider_exec.go | 78 +++ pkg/shell/provider_exec_test.go | 152 ++++++ pkg/shell/provider_exec_unix.go | 17 + pkg/shell/provider_exec_windows.go | 11 + pkg/shell/shell.go | 40 +- .../docs/developing-providers/agent.mdx | 18 + .../docs/developing-providers/quickstart.mdx | 9 +- .../managing-providers/docker-sandbox.mdx | 60 +++ .../managing-providers/manage-providers.mdx | 13 + .../content/docs/managing-providers/meta.json | 2 +- 30 files changed, 1707 insertions(+), 72 deletions(-) create mode 100644 pkg/client/clientimplementation/provider_cancel.go create mode 100644 pkg/client/clientimplementation/provider_cancel_test.go create mode 100644 pkg/client/clientimplementation/provider_cancel_unix.go create mode 100644 pkg/client/clientimplementation/provider_cancel_windows.go create mode 100644 pkg/client/clientimplementation/provider_cancel_windows_test.go create mode 100644 pkg/options/resolve_docker_test.go create mode 100644 pkg/provider/parse_docker_test.go create mode 100644 pkg/shell/provider_exec.go create mode 100644 pkg/shell/provider_exec_test.go create mode 100644 pkg/shell/provider_exec_unix.go create mode 100644 pkg/shell/provider_exec_windows.go create mode 100644 sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx diff --git a/cmd/internal/agentworkspace/docker_darwin_test.go b/cmd/internal/agentworkspace/docker_darwin_test.go index 7b21d4f345..9a67b65fcd 100644 --- a/cmd/internal/agentworkspace/docker_darwin_test.go +++ b/cmd/internal/agentworkspace/docker_darwin_test.go @@ -3,11 +3,16 @@ package agentworkspace import ( + "context" + "errors" "os" "path/filepath" "slices" "strings" "testing" + + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/types" ) func writeExecutable(t *testing.T, path string) { @@ -112,3 +117,41 @@ func TestFindDarwinDockerCLIRejectsInaccessibleCandidate(t *testing.T) { t.Fatalf("expected fallback to valid candidate %q, got %q (err: %v)", valid, path, err) } } + +func TestDarwinDockerDiscoveryPrecedesDisabledInstallation(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + existing := filepath.Join(t.TempDir(), "docker") + writeExecutable(t, existing) + info := &provider.AgentWorkspaceInfo{} + info.Agent.Docker.Install = types.StrBool("false") + calls := 0 + initializer := &workspaceInitializer{ + workspaceInfo: info, + findDarwinDocker: func() (string, error) { + calls++ + return findDarwinDockerCLIInPaths([]string{existing}) + }, + installDocker: func(context.Context) (string, error) { + t.Fatal("disabled installation called the installer") + return "", nil + }, + } + path, err := initializer.ensureDockerInstalled(context.Background()) + if err != nil || path != existing || calls != 1 { + t.Fatalf( + "Docker CLI discovery = %q, %v, calls %d; want %q, nil, 1", + path, + err, + calls, + existing, + ) + } + + initializer.findDarwinDocker = func() (string, error) { + return "", errors.New("Docker CLI absent") + } + path, err = initializer.ensureDockerInstalled(context.Background()) + if path != "" || err == nil || !strings.Contains(err.Error(), "agent.docker.install is false") { + t.Fatalf("missing Docker CLI = %q, %v; want disabled installation error", path, err) + } +} diff --git a/cmd/internal/agentworkspace/up.go b/cmd/internal/agentworkspace/up.go index bcd740b7df..27c3b2a7aa 100644 --- a/cmd/internal/agentworkspace/up.go +++ b/cmd/internal/agentworkspace/up.go @@ -385,6 +385,9 @@ type workspaceInitializer struct { logger tunnelserver.Logger dockerCredentialsDir string gitCredentialsHelper string + installDocker func(context.Context) (string, error) + findDarwinDocker func() (string, error) + configureDocker func(context.Context) error } type initWorkspaceParams struct { @@ -411,6 +414,10 @@ func initWorkspace( } func (w *workspaceInitializer) initialize(ctx context.Context) error { + if err := w.validateDockerManagement(); err != nil { + return err + } + if err := w.initializeTunnel(ctx); err != nil { return err } @@ -448,7 +455,11 @@ func (w *workspaceInitializer) tryConfigureDockerDaemon(ctx context.Context) { log.Debug("skipping configuring docker daemon") return } - if err := configureDockerDaemon(ctx); err != nil { + configure := w.configureDocker + if configure == nil { + configure = configureDockerDaemon + } + if err := configure(ctx); err != nil { log.Warn( "could not find docker daemon config file, if using the registry cache, " + "ensure the daemon is configured with containerd-snapshotter=true, " + @@ -522,17 +533,32 @@ func (w *workspaceInitializer) ensureDockerInstalled(ctx context.Context) (strin } if dockerCmd == docker && runtime.GOOS == "darwin" { - return findDarwinDockerCLI() + find := w.findDarwinDocker + if find == nil { + find = findDarwinDockerCLI + } + path, err := find() + if err == nil { + return path, nil + } + if !w.isDockerInstallDisabled() { + return "", err + } } if w.isDockerInstallDisabled() { - log.Debug( - "docker not found but installation was disabled, installing anyway as it is required", + return "", fmt.Errorf( + "docker CLI is unavailable and agent.docker.install is false (executable %q)", + dockerCmd, ) } log.Debug("attempting to install docker") - dockerPath, err := installDocker(ctx) + install := w.installDocker + if install == nil { + install = installDocker + } + dockerPath, err := install(ctx) log.Debugf("docker installation path=%q, err=%v", dockerPath, err) return dockerPath, err } @@ -579,6 +605,9 @@ func (w *workspaceInitializer) getDockerCommand() string { } func (w *workspaceInitializer) isDockerInstallDisabled() bool { + if w.workspaceInfo.Agent.Docker.Install == "" { + return false + } install, err := w.workspaceInfo.Agent.Docker.Install.Bool() return err == nil && !install } @@ -612,10 +641,26 @@ func (w *workspaceInitializer) waitForDocker(resultChan <-chan dockerInstallResu return nil } +func (w *workspaceInitializer) validateDockerManagement() error { + if _, err := w.workspaceInfo.Agent.Docker.Install.Bool(); err != nil { + return fmt.Errorf("agent.docker.install: %w", err) + } + if _, err := w.workspaceInfo.Agent.Docker.ConfigureDaemon.Bool(); err != nil { + return fmt.Errorf("agent.docker.configureDaemon: %w", err) + } + return nil +} + func (w *workspaceInitializer) shouldConfigureDockerDaemon() bool { if !w.workspaceInfo.Agent.IsDockerDriver() { return false } + if w.workspaceInfo.Agent.Docker.ConfigureDaemon != "" { + configure, err := w.workspaceInfo.Agent.Docker.ConfigureDaemon.Bool() + if err != nil || !configure { + return false + } + } local, err := w.workspaceInfo.Agent.Local.Bool() if err != nil { diff --git a/cmd/internal/agentworkspace/up_test.go b/cmd/internal/agentworkspace/up_test.go index 09a4aa5089..412e531c66 100644 --- a/cmd/internal/agentworkspace/up_test.go +++ b/cmd/internal/agentworkspace/up_test.go @@ -5,12 +5,15 @@ import ( "errors" "os" "path/filepath" + "runtime" + "strings" "testing" "github.com/devsy-org/devsy/pkg/agent/tunnel" "github.com/devsy-org/devsy/pkg/agent/tunnelserver" "github.com/devsy-org/devsy/pkg/provider" "github.com/devsy-org/devsy/pkg/status" + "github.com/devsy-org/devsy/pkg/types" "github.com/stretchr/testify/assert" ) @@ -242,3 +245,201 @@ func TestWaitForDockerReturnsDiscoveryError(t *testing.T) { t.Fatalf("waitForDocker() error = %v, want wrapped %v", err, discoveryErr) } } + +const ( + dockerPolicyDisabled = "false" + dockerPolicyEnabled = "true" + dockerPolicyInvalid = "invalid" + testInstalledDocker = "/installed/docker" +) + +type dockerInstallationCase struct { + name string + path string + policy string + existing bool + wantCalls int + wantPath string + wantError string +} + +func TestDockerInstallationPolicy(t *testing.T) { + cases := []dockerInstallationCase{ + { + name: "missing disabled", + policy: dockerPolicyDisabled, + wantError: `docker CLI is unavailable and agent.docker.install is false (executable "docker")`, + }, + {name: "missing unset", wantCalls: 1, wantPath: testInstalledDocker}, + { + name: "missing enabled", + policy: dockerPolicyEnabled, + wantCalls: 1, + wantPath: testInstalledDocker, + }, + {name: "existing disabled", policy: dockerPolicyDisabled, existing: true}, + { + name: "custom missing disabled", + path: "missing-custom", + policy: dockerPolicyDisabled, + wantError: "custom docker path", + }, + {name: "custom missing unset", path: "missing-custom", wantError: "custom docker path"}, + { + name: "existing custom disabled", + path: "custom-docker", + policy: dockerPolicyDisabled, + existing: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assertDockerInstallationPolicy(t, tc) + }) + } +} + +func assertDockerInstallationPolicy(t *testing.T, tc dockerInstallationCase) { + t.Helper() + if runtime.GOOS == "darwin" && tc.wantCalls > 0 { + tc.wantCalls = 0 + tc.wantPath = "" + tc.wantError = "not found" + } + bin := t.TempDir() + t.Setenv("PATH", bin) + if tc.existing { + writeDockerFixture(t, bin, tc.path) + } + info := &provider.AgentWorkspaceInfo{} + info.Agent.Docker.Path = tc.path + info.Agent.Docker.Install = types.StrBool(tc.policy) + calls := 0 + initializer := &workspaceInitializer{ + workspaceInfo: info, + findDarwinDocker: func() (string, error) { + return "", errors.New("not found") + }, + installDocker: func(context.Context) (string, error) { + calls++ + return testInstalledDocker, nil + }, + } + path, err := initializer.ensureDockerInstalled(context.Background()) + if tc.wantError == "" { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tc.wantError) + } + assert.Equal(t, tc.wantCalls, calls) + assert.Equal(t, tc.wantPath, path) +} + +func writeDockerFixture(t *testing.T, bin, configuredPath string) { + t.Helper() + name := configuredPath + if name == "" { + name = docker + } + path := filepath.Join(bin, name) + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + if err := os.Link(executable, path); err != nil { + t.Fatal(err) + } +} + +func TestDockerInstallDisabledTriState(t *testing.T) { + for _, tc := range []struct { + policy types.StrBool + disabled bool + }{ + {policy: "", disabled: false}, + {policy: dockerPolicyEnabled, disabled: false}, + {policy: dockerPolicyDisabled, disabled: true}, + } { + info := &provider.AgentWorkspaceInfo{} + info.Agent.Docker.Install = tc.policy + initializer := &workspaceInitializer{workspaceInfo: info} + if got := initializer.isDockerInstallDisabled(); got != tc.disabled { + t.Errorf("install %q disabled = %t, want %t", tc.policy, got, tc.disabled) + } + } +} + +func TestDockerManagementPolicy(t *testing.T) { + for _, driver := range []string{provider.DockerDriver, provider.KubernetesDriver} { + for _, local := range []string{dockerPolicyDisabled, dockerPolicyEnabled} { + for _, policy := range []string{"", dockerPolicyEnabled, dockerPolicyDisabled, dockerPolicyInvalid} { + t.Run(driver+"/"+local+"/"+policy, func(t *testing.T) { + assertDockerManagementPolicy(t, driver, local, policy) + }) + } + } + } +} + +func assertDockerManagementPolicy(t *testing.T, driver, local, policy string) { + t.Helper() + info := &provider.AgentWorkspaceInfo{} + info.Agent.Driver = driver + info.Agent.Local = types.StrBool(local) + info.Agent.Docker.ConfigureDaemon = types.StrBool(policy) + configureCalls := 0 + initializer := &workspaceInitializer{ + workspaceInfo: info, + configureDocker: func(context.Context) error { + configureCalls++ + return nil + }, + } + err := initializer.validateDockerManagement() + if policy == dockerPolicyInvalid { + assert.ErrorContains(t, err, "agent.docker.configureDaemon") + } else { + assert.NoError(t, err) + } + initializer.tryConfigureDockerDaemon(context.Background()) + want := 0 + if driver == provider.DockerDriver && local == dockerPolicyDisabled && + policy != dockerPolicyDisabled && policy != dockerPolicyInvalid { + want = 1 + } + assert.Equal(t, want, configureCalls) +} + +func TestInvalidDockerManagementFailsBeforeInitialization(t *testing.T) { + for _, field := range []string{"install", "configureDaemon"} { + t.Run(field, func(t *testing.T) { + info := &provider.AgentWorkspaceInfo{} + if field == "install" { + info.Agent.Docker.Install = dockerPolicyInvalid + } else { + info.Agent.Docker.ConfigureDaemon = dockerPolicyInvalid + } + calls := 0 + initializer := &workspaceInitializer{ + workspaceInfo: info, + installDocker: func(context.Context) (string, error) { + calls++ + return "", nil + }, + configureDocker: func(context.Context) error { + calls++ + return nil + }, + } + if err := initializer.initialize( + context.Background(), + ); err == nil || + !strings.Contains(err.Error(), field) { + t.Errorf("initialize() error = %v, want invalid %s error", err, field) + } + if calls != 0 { + t.Errorf("invalid policy triggered %d Docker management calls", calls) + } + }) + } +} diff --git a/cmd/provider/set_source_init_failure_test.go b/cmd/provider/set_source_init_failure_test.go index 65df1b6d77..c8050dd2a4 100644 --- a/cmd/provider/set_source_init_failure_test.go +++ b/cmd/provider/set_source_init_failure_test.go @@ -47,8 +47,7 @@ func TestSetSourceInitFailureKeepsCommittedReplacementAndWorkspace(t *testing.T) defer cancel() cmd := &SetSourceCmd{GlobalFlags: &flags.GlobalFlags{}, Use: true} err = cmd.Run(ctx, initial, []string{testProviderFoo, replacementSource}) - require.ErrorContains(t, err, "init:") - require.ErrorContains(t, err, "missing-init-executable") + require.Error(t, err) assertFailedSourceReplacement( t, @@ -115,7 +114,6 @@ func assertFailedSourceReplacement( require.NotNil(t, state) require.False(t, state.Initialized) require.True(t, state.InitAttempted) - require.Contains(t, state.InitError, "missing-init-executable") require.Equal(t, userOption.Value, state.Options["REGION"].Value) require.True(t, state.Options["REGION"].UserProvided) require.Equal(t, testProviderFoo, stored.Current().DefaultProvider) diff --git a/desktop/src/renderer/src/lib/components/provider/ProviderWizard.svelte b/desktop/src/renderer/src/lib/components/provider/ProviderWizard.svelte index b23505da7d..7a574d6ed5 100644 --- a/desktop/src/renderer/src/lib/components/provider/ProviderWizard.svelte +++ b/desktop/src/renderer/src/lib/components/provider/ProviderWizard.svelte @@ -56,6 +56,11 @@ const PRESETS: ProviderPreset[] = [ description: "Hardware-isolated microVMs", category: "Local", }, + { + name: "docker-sandbox", + description: "Devcontainers in Docker Sandboxes microVMs", + category: "Local", + }, { name: "lima", description: "Linux VMs via Lima (macOS)", diff --git a/desktop/src/renderer/src/lib/components/provider/ProviderWizard.test.ts b/desktop/src/renderer/src/lib/components/provider/ProviderWizard.test.ts index 37f77e5fa5..41fb6c83ef 100644 --- a/desktop/src/renderer/src/lib/components/provider/ProviderWizard.test.ts +++ b/desktop/src/renderer/src/lib/components/provider/ProviderWizard.test.ts @@ -312,6 +312,49 @@ describe("ProviderWizard", () => { }, ) + it.each([true, false])( + "configures the Docker Sandbox preset and preserves Reuse machine %s", + async (reuseMachine) => { + providerOptions.mockResolvedValue({ + SBX_CPUS: { name: "SBX_CPUS", default: "2" }, + }) + const page = renderWizard() + const preset = screen.getByRole("button", { + name: /docker-sandbox Devcontainers in Docker Sandboxes microVMs/, + }) + expect( + screen.getByRole("region", { name: "Local" }).contains(preset), + ).toBe(true) + await fireEvent.click(preset) + if (reuseMachine) + await fireEvent.click( + screen.getByRole("switch", { name: "Reuse machine" }), + ) + await fireEvent.click(screen.getByRole("button", { name: "Continue" })) + await fireEvent.input( + await screen.findByRole("textbox", { name: "SBX_CPUS" }), + { target: { value: "4" } }, + ) + await fireEvent.click(screen.getByRole("button", { name: "Continue" })) + await waitFor(() => { + expect(providerAdd).toHaveBeenCalledExactlyOnceWith( + "docker-sandbox", + undefined, + reuseMachine, + ) + expect(providerSetOptions).toHaveBeenCalledExactlyOnceWith( + "docker-sandbox", + { SBX_CPUS: "4" }, + reuseMachine, + ) + expect(providerInitStreaming).toHaveBeenCalledExactlyOnceWith( + "docker-sandbox", + ) + }) + page.unmount() + }, + ) + it("retries loading options without adding the provider a second time", async () => { providerOptions .mockRejectedValueOnce(new Error("temporary read failure")) diff --git a/pkg/client/clientimplementation/provider_cancel.go b/pkg/client/clientimplementation/provider_cancel.go new file mode 100644 index 0000000000..9647d37a54 --- /dev/null +++ b/pkg/client/clientimplementation/provider_cancel.go @@ -0,0 +1,126 @@ +package clientimplementation + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/devsy-org/devsy/pkg/config" +) + +const providerCancellationGrace = 10 * time.Second + +func runProviderWithCancellation(parent context.Context, opts RunCommandOptions) error { + return runProviderWithCancellationUsingPublisher(parent, opts, publishProviderCancellation) +} + +func runProviderWithCancellationUsingPublisher( + parent context.Context, + opts RunCommandOptions, + publish func(string) error, +) error { + if parent.Err() != nil { + return context.Cause(parent) + } + if len(opts.Command) == 0 { + return nil + } + + dir, err := createProviderInvocationDir() + if err != nil { + return fmt.Errorf("create provider cancellation directory: %w", err) + } + defer os.RemoveAll(dir) + + marker := filepath.Join(dir, "cancel") + environ := make([]string, 0, len(opts.Environ)+1) + for _, kv := range opts.Environ { + key, _, ok := strings.Cut(kv, "=") + if ok && providerReservedKey(key) { + continue + } + environ = append(environ, kv) + } + opts.Environ = append(environ, config.EnvProviderCancelFile+"="+marker) + + execution, cancel := context.WithCancelCause(context.WithoutCancel(parent)) + defer cancel(context.Canceled) + if parent.Err() != nil { + return context.Cause(parent) + } + completed := make(chan error, 1) + go func() { + completed <- runCommandExecution(execution, opts) + }() + + select { + case result := <-completed: + if parent.Err() != nil { + return context.Cause(parent) + } + return result + case <-parent.Done(): + } + + cause := context.Cause(parent) + if err := publish(marker); err != nil { + cancel(cause) + <-completed + return errors.Join( + cause, + fmt.Errorf("publish provider cancellation marker: %w", sanitizeMarkerError(err)), + ) + } + + timer := time.NewTimer(providerCancellationGrace) + defer timer.Stop() + select { + case <-completed: + return cause + case <-timer.C: + cancel(cause) + <-completed + return cause + } +} + +func providerReservedKey(key string) bool { + if runtime.GOOS == "windows" { + return strings.EqualFold(key, config.EnvProviderCancelFile) + } + return key == config.EnvProviderCancelFile +} + +func publishProviderCancellation(marker string) error { + f, err := os.OpenFile(marker, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err == nil { + return sanitizeMarkerError(f.Close()) + } + if !errors.Is(err, os.ErrExist) { + return fmt.Errorf("cannot create marker: %w", sanitizeMarkerError(err)) + } + info, statErr := os.Lstat(marker) + if statErr != nil { + return fmt.Errorf("cannot inspect existing marker: %w", sanitizeMarkerError(statErr)) + } + if !info.Mode().IsRegular() { + return errors.New("existing marker is not a regular file") + } + return nil +} + +func sanitizeMarkerError(err error) error { + if err == nil { + return nil + } + var pathErr *os.PathError + if errors.As(err, &pathErr) { + return pathErr.Err + } + return errors.New("marker filesystem operation failed") +} diff --git a/pkg/client/clientimplementation/provider_cancel_test.go b/pkg/client/clientimplementation/provider_cancel_test.go new file mode 100644 index 0000000000..5ec5872f43 --- /dev/null +++ b/pkg/client/clientimplementation/provider_cancel_test.go @@ -0,0 +1,443 @@ +package clientimplementation + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "path/filepath" + "runtime" + "strings" + "syscall" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/client" + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/types" + "github.com/stretchr/testify/require" + "mvdan.cc/sh/v3/interp" +) + +func TestProviderCancellationHelper(t *testing.T) { + if os.Getenv("DEVSY_CANCEL_HELPER") != "1" { + return + } + marker := os.Getenv(config.EnvProviderCancelFile) + switch os.Getenv("DEVSY_CANCEL_BEHAVIOR") { + case "acknowledge": + fmt.Fprint(os.Stdout, "ready") + for { + if _, err := os.Stat(marker); err == nil { + if path := os.Getenv("DEVSY_CANCEL_ACK_PATH"); path != "" { + _ = os.WriteFile(path, []byte("finished"), 0o600) + } + return + } + time.Sleep(20 * time.Millisecond) + } + case "ignore": + fmt.Fprint(os.Stdout, "ready") + time.Sleep(time.Minute) + case "transport-read", "transport-write": + ready := os.Getenv("DEVSY_CANCEL_READY_PATH") + _ = os.WriteFile(ready, []byte("ready"), 0o600) + go func() { + for { + if _, err := os.Stat(marker); err == nil { + _ = os.WriteFile(os.Getenv("DEVSY_CANCEL_ACK_PATH"), []byte("finished"), 0o600) + os.Exit(0) + } + time.Sleep(20 * time.Millisecond) + } + }() + if os.Getenv("DEVSY_CANCEL_BEHAVIOR") == "transport-read" { + var b [1]byte + _, _ = os.Stdin.Read(b[:]) + for { + time.Sleep(time.Second) + } + } else { + if runtime.GOOS != "windows" { + signal.Ignore(syscall.Signal(13)) + } + chunk := bytes.Repeat([]byte("x"), 1024) + for { + if _, err := os.Stdout.Write(chunk); err != nil { + for { + time.Sleep(time.Second) + } + } + } + } + case "status": + os.Exit(7) + case "status-127": + os.Exit(127) + case "bytes": + os.Stdout.Write([]byte{0, 0xff, 0x80, '\r', '\n'}) + os.Stderr.Write([]byte(os.Getenv("DEVSY_CANCEL_SECRET"))) + case "environment": + fmt.Fprint(os.Stdout, marker) + default: + os.Exit(125) + } + os.Exit(0) +} + +func cancelHelperOptions(behavior string) RunCommandOptions { + binary, err := os.Executable() + if err != nil { + panic(err) + } + return RunCommandOptions{ + Command: types.StrArray{binary, "-test.run=^TestProviderCancellationHelper$"}, + Environ: append( + os.Environ(), + "DEVSY_CANCEL_HELPER=1", + "DEVSY_CANCEL_BEHAVIOR="+behavior, + ), + Cancellation: provider.ProviderCancellationFile, + } +} + +func TestProviderCancellationCooperatesAndCleansUp(t *testing.T) { + ack := filepath.Join(t.TempDir(), "ack") + opts := cancelHelperOptions("acknowledge") + opts.Environ = append(opts.Environ, "DEVSY_CANCEL_ACK_PATH="+ack) + ctx, cancel := context.WithCancel(context.Background()) + ready := make(chan struct{}) + opts.Stdout = &readyWriter{ready: ready} + opts.RawStdout = true + finished := make(chan error, 1) + go func() { finished <- RunCommand(ctx, opts) }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("provider did not start") + } + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("provider did not acknowledge cancellation") + } + contents, err := os.ReadFile(ack) + require.NoError(t, err) + require.Equal(t, "finished", string(contents)) +} + +type readyWriter struct{ ready chan struct{} } + +func (w *readyWriter) Write(p []byte) (int, error) { + if bytes.Contains(p, []byte("ready")) { + select { + case <-w.ready: + default: + close(w.ready) + } + } + return len(p), nil +} + +func TestProviderCancellationForcedAfterProductionGrace(t *testing.T) { + opts := cancelHelperOptions("ignore") + ctx, cancel := context.WithCancel(context.Background()) + ready := make(chan struct{}) + opts.Stdout = &readyWriter{ready: ready} + opts.RawStdout = true + finished := make(chan error, 1) + go func() { finished <- RunCommand(ctx, opts) }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("provider did not start") + } + start := time.Now() + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + require.GreaterOrEqual(t, time.Since(start), providerCancellationGrace) + case <-time.After(providerCancellationGrace + 8*time.Second): + t.Fatal("owned provider process was not reaped") + } +} + +func TestProviderCancellationPrelaunchAndMarkerFailure(t *testing.T) { + ctx, cancel := context.WithCancelCause(context.Background()) + cause := errors.New("caller stopped") + cancel(cause) + require.ErrorIs(t, RunCommand(ctx, cancelHelperOptions("ignore")), cause) + + dir := t.TempDir() + marker := filepath.Join(dir, "cancel") + require.NoError(t, os.Mkdir(marker, 0o700)) + err := publishProviderCancellation(marker) + require.ErrorContains(t, err, "not a regular file") + require.NotContains(t, err.Error(), dir) + + require.NoError(t, os.Remove(marker)) + require.NoError(t, publishProviderCancellation(marker)) + require.NoError(t, publishProviderCancellation(marker)) + require.NoError(t, os.Remove(marker)) + if err := os.Symlink(filepath.Join(dir, "missing"), marker); err == nil { + require.ErrorContains(t, publishProviderCancellation(marker), "not a regular file") + } +} + +func TestProviderCancellationPublicationFailureReapsProvider(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + opts := cancelHelperOptions("ignore") + opts.RawStdout = true + ready := make(chan struct{}) + opts.Stdout = &readyWriter{ready: ready} + markerPath := make(chan string, 1) + finished := make(chan error, 1) + go func() { + finished <- runProviderWithCancellationUsingPublisher(ctx, opts, func(marker string) error { + markerPath <- marker + return &os.PathError{Op: "open", Path: marker, Err: os.ErrPermission} + }) + }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("provider did not start") + } + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + require.ErrorIs(t, err, os.ErrPermission) + marker := <-markerPath + require.NotContains(t, err.Error(), marker) + _, statErr := os.Stat(filepath.Dir(marker)) + require.ErrorIs(t, statErr, os.ErrNotExist) + case <-time.After(5 * time.Second): + t.Fatal("provider did not stop after publication failure") + } +} + +func TestProviderCancellationPrivateEnvironmentAndTypedExits(t *testing.T) { + dir, err := createProviderInvocationDir() + require.NoError(t, err) + defer os.RemoveAll(dir) + require.True(t, filepath.IsAbs(dir)) + if runtime.GOOS != "windows" { + info, err := os.Stat(dir) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o700), info.Mode().Perm()) + } + + opts := cancelHelperOptions("environment") + opts.Environ = append( + opts.Environ, + config.EnvProviderCancelFile+"=forged", + config.EnvProviderCancelFile+"=again", + ) + if runtime.GOOS == "windows" { + opts.Environ = append( + opts.Environ, + strings.ToLower(config.EnvProviderCancelFile)+"=windows-forged", + ) + } + var stdout bytes.Buffer + opts.Stdout = &stdout + require.NoError(t, RunCommand(context.Background(), opts)) + marker := stdout.String() + require.NotContains(t, marker, "forged") + require.True(t, filepath.IsAbs(marker)) + _, err = os.Stat(filepath.Dir(marker)) + require.ErrorIs(t, err, os.ErrNotExist) + + for _, status := range []int{7, 125, 127} { + opts = cancelHelperOptions("status") + if status == 125 { + opts.Environ = append(opts.Environ, "DEVSY_CANCEL_BEHAVIOR=other") + } + if status == 127 { + opts.Environ = append(opts.Environ, "DEVSY_CANCEL_BEHAVIOR=status-127") + } + err = RunCommand(context.Background(), opts) + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + require.Equal(t, status, exitErr.ExitCode()) + } + + if runtime.GOOS != "windows" { + binary, err := os.Executable() + require.NoError(t, err) + for _, status := range []int{7, 127} { + command := "command-that-does-not-exist-846307" + if status == 7 { + command = fmt.Sprintf( + "DEVSY_CANCEL_HELPER=1 DEVSY_CANCEL_BEHAVIOR=status '%s' -test.run=^TestProviderCancellationHelper$", + binary, + ) + } + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{ + command, + }, + Environ: os.Environ(), + Cancellation: provider.ProviderCancellationFile, + }) + var shellExit interp.ExitStatus + require.ErrorAs(t, err, &shellExit) + require.Equal(t, status, int(shellExit)) + } + } +} + +func TestProviderCancellationRawBytesAndStderrRedaction(t *testing.T) { + opts := cancelHelperOptions("bytes") + secret := "provider-cancel-secret-84392" + opts.Environ = append(opts.Environ, "DEVSY_CANCEL_SECRET="+secret) + opts.RawStdout = true + var stdout, stderr bytes.Buffer + opts.Stdout, opts.Stderr = &stdout, &stderr + require.NoError(t, RunCommand(context.Background(), opts)) + require.Equal(t, []byte{0, 0xff, 0x80, '\r', '\n'}, stdout.Bytes()) + require.NotContains(t, stderr.String(), secret) + require.Contains(t, stderr.String(), "***") +} + +func TestMachineCommandTransportCancellationJoinsCallback(t *testing.T) { + for _, behavior := range []string{"transport-read", "transport-write"} { + t.Run(behavior, func(t *testing.T) { + dir := t.TempDir() + ready, ack := filepath.Join(dir, "ready"), filepath.Join(dir, "ack") + t.Setenv("DEVSY_CANCEL_HELPER", "1") + t.Setenv("DEVSY_CANCEL_BEHAVIOR", behavior) + t.Setenv("DEVSY_CANCEL_READY_PATH", ready) + t.Setenv("DEVSY_CANCEL_ACK_PATH", ack) + machine := &provider.Machine{ID: "cancel-machine", Context: testLocksContext} + cfg := &provider.ProviderConfig{ + Name: "cancel-provider", + Exec: provider.ProviderCommands{ + Create: types.StrArray{ + "create", + }, + Command: cancelHelperOptions(behavior).Command, + Cancellation: provider.ProviderCancellationFile, + }, + } + devsy := &config.Config{ + DefaultContext: testLocksContext, + Contexts: map[string]*config.ContextConfig{testLocksContext: {}}, + } + mc, err := NewMachineClient(devsy, cfg, machine) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + conn, err := mc.OpenCommandTransport( + ctx, + client.CommandTransportOptions{Command: "opaque"}, + ) + require.NoError(t, err) + defer conn.Close() + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(ready); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("transport provider did not start") + } + time.Sleep(10 * time.Millisecond) + } + cancel() + completed := make(chan error, 1) + go func() { completed <- conn.Wait() }() + select { + case err := <-completed: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("transport callback did not finish") + } + contents, err := os.ReadFile(ack) + require.NoError(t, err) + require.Equal(t, "finished", string(contents)) + }) + } +} + +func TestWorkspaceCommandCancellationJoinsProviderCleanup(t *testing.T) { + for _, transportMode := range []bool{false, true} { + t.Run(fmt.Sprint("transport=", transportMode), func(t *testing.T) { + ack := filepath.Join(t.TempDir(), "ack") + t.Setenv("DEVSY_CANCEL_HELPER", "1") + t.Setenv("DEVSY_CANCEL_BEHAVIOR", "acknowledge") + t.Setenv("DEVSY_CANCEL_ACK_PATH", ack) + workspace := &provider.Workspace{ + ID: "cancel-workspace", + Context: testLocksContext, + Provider: provider.WorkspaceProviderConfig{ + Name: "cancel-provider", + }, + } + cfg := &provider.ProviderConfig{ + Name: "cancel-provider", + Exec: provider.ProviderCommands{ + Command: cancelHelperOptions("acknowledge").Command, + Cancellation: provider.ProviderCancellationFile, + }, + } + devsy := &config.Config{ + DefaultContext: testLocksContext, + Contexts: map[string]*config.ContextConfig{testLocksContext: {}}, + } + wc := &workspaceClient{ + providerWorkspace: workspace, + providerConfig: cfg, + devsyConfig: devsy, + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if transportMode { + conn, err := wc.OpenCommandTransport(ctx, client.CommandTransportOptions{Command: "opaque"}) + require.NoError(t, err) + defer conn.Close() + ready := make([]byte, 5) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + _, err = io.ReadFull(conn, ready) + require.NoError(t, err) + require.Equal(t, "ready", string(ready)) + cancel() + require.ErrorIs(t, conn.Wait(), context.Canceled) + } else { + ready := make(chan struct{}) + finished := make(chan error, 1) + go func() { + finished <- wc.Command(ctx, client.CommandOptions{ + Command: "opaque", + Stdout: &readyWriter{ready: ready}, + }) + }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("workspace provider did not start") + } + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("workspace provider did not finish") + } + } + contents, err := os.ReadFile(ack) + require.NoError(t, err) + require.Equal(t, "finished", string(contents)) + }) + } +} diff --git a/pkg/client/clientimplementation/provider_cancel_unix.go b/pkg/client/clientimplementation/provider_cancel_unix.go new file mode 100644 index 0000000000..d647d17cad --- /dev/null +++ b/pkg/client/clientimplementation/provider_cancel_unix.go @@ -0,0 +1,21 @@ +//go:build !windows + +package clientimplementation + +import ( + "os" + "path/filepath" +) + +func createProviderInvocationDir() (string, error) { + dir, err := os.MkdirTemp("", "devsy-provider-invocation-") + if err != nil { + return "", err + } + absolute, err := filepath.Abs(dir) + if err != nil { + _ = os.Remove(dir) + return "", err + } + return absolute, nil +} diff --git a/pkg/client/clientimplementation/provider_cancel_windows.go b/pkg/client/clientimplementation/provider_cancel_windows.go new file mode 100644 index 0000000000..c583781b8d --- /dev/null +++ b/pkg/client/clientimplementation/provider_cancel_windows.go @@ -0,0 +1,53 @@ +package clientimplementation + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "os" + "path/filepath" + "unsafe" + + "golang.org/x/sys/windows" +) + +func createProviderInvocationDir() (string, error) { + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + return "", err + } + sd, err := windows.SecurityDescriptorFromString( + "D:P(A;OICI;GA;;;" + user.User.Sid.String() + ")(A;OICI;GA;;;SY)", + ) + if err != nil { + return "", err + } + attributes := windows.SecurityAttributes{ + Length: uint32(unsafe.Sizeof(windows.SecurityAttributes{})), + SecurityDescriptor: sd, + } + for range 10 { + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return "", err + } + path, err := filepath.Abs( + filepath.Join(os.TempDir(), "devsy-provider-invocation-"+hex.EncodeToString(random[:])), + ) + if err != nil { + return "", err + } + name, err := windows.UTF16PtrFromString(path) + if err != nil { + return "", err + } + err = windows.CreateDirectory(name, &attributes) + if err == nil { + return path, nil + } + if !errors.Is(err, windows.ERROR_ALREADY_EXISTS) { + return "", err + } + } + return "", os.ErrExist +} diff --git a/pkg/client/clientimplementation/provider_cancel_windows_test.go b/pkg/client/clientimplementation/provider_cancel_windows_test.go new file mode 100644 index 0000000000..b76bd46e36 --- /dev/null +++ b/pkg/client/clientimplementation/provider_cancel_windows_test.go @@ -0,0 +1,45 @@ +package clientimplementation + +import ( + "os" + "strings" + "testing" + + "golang.org/x/sys/windows" +) + +func TestProviderCancellationDirectoryWindowsDACL(t *testing.T) { + dir, err := createProviderInvocationDir() + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + sd, err := windows.GetNamedSecurityInfo( + dir, + windows.SE_FILE_OBJECT, + windows.DACL_SECURITY_INFORMATION, + ) + if err != nil { + t.Fatal(err) + } + control, _, err := sd.Control() + if err != nil { + t.Fatal(err) + } + if control&windows.SE_DACL_PROTECTED == 0 { + t.Fatal("invocation directory inherits an unprotected DACL") + } + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + t.Fatal(err) + } + sddl := sd.String() + if !strings.Contains(sddl, user.User.Sid.String()) || + (!strings.Contains(sddl, ";;;SY)") && !strings.Contains(sddl, ";;;S-1-5-18)")) || + strings.Contains(sddl, ";;;WD)") { + t.Fatalf( + "invocation directory DACL does not restrict access to current user and SYSTEM: %s", + sddl, + ) + } +} diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index 54d4e44c01..2e50d78c06 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -245,12 +245,13 @@ func (s *workspaceClient) Command(ctx context.Context, opt client.CommandOptions } return RunCommand(ctx, RunCommandOptions{ - Command: s.providerConfig.Exec.Command, - Environ: environ, - Stdin: opt.Stdin, - Stdout: opt.Stdout, - Stderr: opt.Stderr, - RawStdout: opt.RawStdout, + Command: s.providerConfig.Exec.Command, + Environ: environ, + Stdin: opt.Stdin, + Stdout: opt.Stdout, + Stderr: opt.Stderr, + RawStdout: opt.RawStdout, + Cancellation: s.providerConfig.Exec.Cancellation, }) } @@ -269,7 +270,8 @@ func (s *workspaceClient) OpenCommandTransport( Command: s.providerConfig.Exec.Command, Environ: environ, Stdin: stdin, Stdout: stdout, Stderr: opt.Stderr, - RawStdout: true, + RawStdout: true, + Cancellation: s.providerConfig.Exec.Cancellation, }) }, transport.CallbackConnOptions{ @@ -810,25 +812,34 @@ func RunCommandWithBinaries(ctx context.Context, cfg WorkspaceCommandConfig) err } return RunCommand(ctx, RunCommandOptions{ - Command: cfg.Command, - Environ: environ, - Stdin: cfg.Stdin, - Stdout: cfg.Stdout, - Stderr: cfg.Stderr, - RawStdout: cfg.RawStdout, + Command: cfg.Command, + Environ: environ, + Stdin: cfg.Stdin, + Stdout: cfg.Stdout, + Stderr: cfg.Stderr, + RawStdout: cfg.RawStdout, + Cancellation: cfg.Config.Exec.Cancellation, }) } type RunCommandOptions struct { - Command types.StrArray - Environ []string - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer - RawStdout bool + Command types.StrArray + Environ []string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + RawStdout bool + Cancellation provider.ProviderCancellationMode } -func RunCommand(ctx context.Context, opts RunCommandOptions) (retErr error) { +func RunCommand(ctx context.Context, opts RunCommandOptions) error { + if opts.Cancellation == provider.ProviderCancellationFile { + return runProviderWithCancellation(ctx, opts) + } + return runCommandExecution(ctx, opts) +} + +func runCommandExecution(ctx context.Context, opts RunCommandOptions) (retErr error) { if len(opts.Command) == 0 { return nil } @@ -856,33 +867,47 @@ func RunCommand(ctx context.Context, opts RunCommandOptions) (retErr error) { }() if len(opts.Command) == 1 { - if err := shell.RunEmulatedShell(ctx, &shell.CommandRunner{ - Command: opts.Command[0], - Stdin: opts.Stdin, - Stdout: stdout, - Stderr: stderr, - Environ: opts.Environ, - }); err != nil { - return fmt.Errorf( - "run provider command %q (%s): %w", - opts.Command[0], - providerAction(opts.Environ), - err, - ) - } - return nil + return runProviderShellCommand(ctx, opts, stdout, stderr) + } + return runProviderArgvCommand(ctx, opts, stdout, stderr) +} + +func runProviderShellCommand( + ctx context.Context, + opts RunCommandOptions, + stdout, stderr io.Writer, +) error { + runner := &shell.CommandRunner{ + Command: opts.Command[0], + Stdin: opts.Stdin, + Stdout: stdout, + Stderr: stderr, + Environ: opts.Environ, + } + if opts.Cancellation == provider.ProviderCancellationFile { + runner.ExecHandler = shell.ProviderExecHandler() + } + if err := shell.RunEmulatedShell(ctx, runner); err != nil { + return fmt.Errorf("run provider %s: %w", providerAction(opts.Environ), err) } + return nil +} +func runProviderArgvCommand( + ctx context.Context, + opts RunCommandOptions, + stdout, stderr io.Writer, +) error { cmd := exec.CommandContext(ctx, opts.Command[0], opts.Command[1:]...) // #nosec G204 cmd.Stdin = opts.Stdin cmd.Stdout = stdout cmd.Stderr = stderr cmd.Env = opts.Environ + if opts.Cancellation == provider.ProviderCancellationFile { + cmd.WaitDelay = 2 * time.Second + } if err := cmd.Run(); err != nil { - return fmt.Errorf( - "run provider command %q (%s): %w", - strings.Join(opts.Command, " "), providerAction(opts.Environ), err, - ) + return fmt.Errorf("run provider %s: %w", providerAction(opts.Environ), err) } return nil } @@ -901,14 +926,13 @@ func flushCommandOutput(stdout, stderr *subprocess.StreamingRedactingWriter) err return errors.Join(stdoutErr, stderrErr) } -// providerAction extracts the provider action from the environment variables. func providerAction(environ []string) string { for _, kv := range environ { - if rest, ok := strings.CutPrefix(kv, provider.CommandEnv+"="); ok { - return rest + if strings.HasPrefix(kv, provider.CommandEnv+"=") { + return "command" } } - return "" + return "lifecycle" } func DeleteMachineFolder(context, machineID string) error { diff --git a/pkg/client/clientimplementation/workspace_client_test.go b/pkg/client/clientimplementation/workspace_client_test.go index 2c2d65e488..6915ca2077 100644 --- a/pkg/client/clientimplementation/workspace_client_test.go +++ b/pkg/client/clientimplementation/workspace_client_test.go @@ -6,6 +6,7 @@ import ( "errors" "io" "os" + "os/exec" "path/filepath" "runtime" "strings" @@ -18,6 +19,7 @@ import ( "github.com/devsy-org/devsy/pkg/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "mvdan.cc/sh/v3/interp" ) const ( @@ -141,6 +143,69 @@ func TestRunCommand(t *testing.T) { }) } +func TestMachineCommandErrorsRedactOpaquePayloadAndArguments(t *testing.T) { + requirePOSIXShell(t) + const payload = "private-command-payload-796412" + const argument = "private-configured-argument-796412" + for _, tc := range []struct { + name string + command types.StrArray + }{ + { + name: "emulated shell", + command: types.StrArray{ + `: '` + argument + `'; printf '%s' "$DEVSY_RUN_COMMAND_SECRET" >&2; exit 7`, + }, + }, + { + name: "argv", + command: types.StrArray{binSh, "-c", `printf '%s' "$DEVSY_RUN_COMMAND_SECRET" >&2; exit 7`, argument}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("DEVSY_RUN_COMMAND_SECRET", "credential-sentinel-61395") + cfg := &provider.ProviderConfig{ + Name: "redaction-test", + Exec: provider.ProviderCommands{ + Create: types.StrArray{"create"}, + Command: tc.command, + }, + } + devsy := &config.Config{ + DefaultContext: testLocksContext, + Contexts: map[string]*config.ContextConfig{testLocksContext: {}}, + } + mc, err := NewMachineClient( + devsy, + cfg, + &provider.Machine{ID: "machine", Context: testLocksContext}, + ) + require.NoError(t, err) + var stderr bytes.Buffer + err = mc.Command( + context.Background(), + client.CommandOptions{Command: payload, Stderr: &stderr}, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "run provider command:") + assert.NotContains(t, err.Error(), payload) + assert.NotContains(t, err.Error(), argument) + assert.NotContains(t, stderr.String(), payload) + assert.NotContains(t, stderr.String(), argument) + assert.Contains(t, stderr.String(), "***") + if tc.name == "emulated shell" { + var status interp.ExitStatus + require.ErrorAs(t, err, &status) + assert.Equal(t, 7, int(status)) + } else { + var status *exec.ExitError + require.ErrorAs(t, err, &status) + assert.Equal(t, 7, status.ExitCode()) + } + }) + } +} + func TestRunCommand_RawStdoutDoesNotHoldProtocolBytes(t *testing.T) { requireProtocolByteRoundTrip( t, diff --git a/pkg/config/env.go b/pkg/config/env.go index f3fa99eaa3..408552fa4b 100644 --- a/pkg/config/env.go +++ b/pkg/config/env.go @@ -13,6 +13,8 @@ const ( // EnvBinaryPath is set to the path of the Devsy binary. EnvBinaryPath = "DEVSY" + EnvProviderCancelFile = "DEVSY_PROVIDER_CANCEL_FILE" + // EnvHome overrides the default Devsy home directory. EnvHome = "DEVSY_HOME" diff --git a/pkg/options/resolve.go b/pkg/options/resolve.go index 6faf0ef92f..9716275965 100644 --- a/pkg/options/resolve.go +++ b/pkg/options/resolve.go @@ -344,6 +344,9 @@ func resolveAgentDockerConfig( agentConfig.Docker.Install = types.StrBool( resolver.ResolveDefaultValue(string(agentConfig.Docker.Install), options), ) + agentConfig.Docker.ConfigureDaemon = types.StrBool( + resolver.ResolveDefaultValue(string(agentConfig.Docker.ConfigureDaemon), options), + ) agentConfig.Docker.Env = resolver.ResolveDefaultValues(agentConfig.Docker.Env, options) } diff --git a/pkg/options/resolve_docker_test.go b/pkg/options/resolve_docker_test.go new file mode 100644 index 0000000000..20567c402d --- /dev/null +++ b/pkg/options/resolve_docker_test.go @@ -0,0 +1,58 @@ +package options + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/types" +) + +func TestResolveDockerDaemonPolicyIntoWorkspaceInfo(t *testing.T) { + manifest := &provider.ProviderConfig{ + Name: "docker-policy", + Agent: provider.ProviderAgentConfig{ + Driver: provider.DockerDriver, + Docker: provider.ProviderDockerDriverConfig{ + Install: "${INSTALL_DOCKER}", + ConfigureDaemon: "${CONFIGURE_DAEMON}", + }, + }, + } + devConfig := &config.Config{ + DefaultContext: config.DefaultContext, + Contexts: map[string]*config.ContextConfig{ + config.DefaultContext: {Providers: map[string]*config.ProviderConfig{ + manifest.Name: {Options: map[string]config.OptionValue{ + "INSTALL_DOCKER": {Value: "false"}, + "CONFIGURE_DAEMON": {Value: "false"}, + }}, + }}, + }, + } + resolved := ResolveAgentConfig(devConfig, manifest, nil, nil) + if resolved.Docker.ConfigureDaemon != types.StrBool("false") || + resolved.Docker.Install != types.StrBool("false") { + t.Fatalf("resolved Docker management = %+v, want disabled", resolved.Docker) + } + if manifest.Agent.Docker.ConfigureDaemon != "${CONFIGURE_DAEMON}" { + t.Fatalf("original provider policy was mutated: %q", manifest.Agent.Docker.ConfigureDaemon) + } + data, err := json.Marshal(&provider.AgentWorkspaceInfo{Agent: resolved}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"configureDaemon":"false"`) || + !strings.Contains(string(data), `"install":"false"`) { + t.Fatalf("agent workspace lost disabled Docker management policy: %s", data) + } + var decoded provider.AgentWorkspaceInfo + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatal(err) + } + if decoded.Agent.Docker.ConfigureDaemon != "false" || decoded.Agent.Docker.Install != "false" { + t.Fatalf("decoded agent workspace lost policy: %+v", decoded.Agent.Docker) + } +} diff --git a/pkg/provider/parse.go b/pkg/provider/parse.go index bda8b07fde..09fd88a2d8 100644 --- a/pkg/provider/parse.go +++ b/pkg/provider/parse.go @@ -281,6 +281,12 @@ func validateStandardProvider(config *ProviderConfig) error { if err := validateAgentDriver(config); err != nil { return err } + if value := config.Agent.Docker.ConfigureDaemon; value != "" && + !templatedValueRegex.MatchString(string(value)) { + if _, err := value.Bool(); err != nil { + return fmt.Errorf("agent.docker.configureDaemon: %w", err) + } + } if err := validateBinaries("agent.binaries", config.Agent.Binaries); err != nil { return err @@ -342,6 +348,11 @@ func validateCustomDriver(config *ProviderConfig) error { } func validateExecCommands(config *ProviderConfig) error { + if config.Exec.Cancellation != ProviderCancellationDefault && + config.Exec.Cancellation != ProviderCancellationFile { + return fmt.Errorf("exec.cancellation must be empty or file") + } + if len(config.Exec.Command) == 0 { return fmt.Errorf("exec.command is required") } diff --git a/pkg/provider/parse_docker_test.go b/pkg/provider/parse_docker_test.go new file mode 100644 index 0000000000..c52ab62c0c --- /dev/null +++ b/pkg/provider/parse_docker_test.go @@ -0,0 +1,46 @@ +package provider + +import ( + "strings" + "testing" +) + +func TestParseDockerConfigureDaemon(t *testing.T) { + for _, tc := range []struct { + name string + value string + wantErr bool + }{ + {name: "unset"}, + {name: "true", value: "true"}, + {name: "false", value: "false"}, + {name: "template", value: "${CONFIGURE_DAEMON}"}, + {name: "bare template", value: "$CONFIGURE_DAEMON"}, + {name: "invalid", value: "sometimes", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + manifest := "name: docker-policy\nagent:\n driver: docker\n" + if tc.value != "" { + manifest += " docker:\n configureDaemon: " + tc.value + "\n" + } + manifest += "exec:\n command: echo hi\n" + parsed, err := ParseProvider(strings.NewReader(manifest)) + if tc.wantErr { + if err == nil || !strings.Contains(err.Error(), "agent.docker.configureDaemon") { + t.Fatalf("ParseProvider() error = %v, want Docker daemon policy error", err) + } + return + } + if err != nil { + t.Fatal(err) + } + if string(parsed.Agent.Docker.ConfigureDaemon) != tc.value { + t.Errorf( + "ConfigureDaemon = %q, want %q", + parsed.Agent.Docker.ConfigureDaemon, + tc.value, + ) + } + }) + } +} diff --git a/pkg/provider/parse_driver_test.go b/pkg/provider/parse_driver_test.go index ee7c241eaa..3d11f33320 100644 --- a/pkg/provider/parse_driver_test.go +++ b/pkg/provider/parse_driver_test.go @@ -42,3 +42,36 @@ exec: }) } } + +func TestProviderCancellationMode(t *testing.T) { + for _, tc := range []struct { + name string + value string + valid bool + }{ + {name: "default", valid: true}, + {name: string(ProviderCancellationFile), value: string(ProviderCancellationFile), valid: true}, + {name: "unknown", value: "signal"}, + {name: "template", value: "${CANCELLATION}"}, + } { + t.Run(tc.name, func(t *testing.T) { + manifest := "name: example\nversion: v1.0.0\nexec:\n command: echo ok\n" + if tc.value != "" { + manifest += " cancellation: '" + tc.value + "'\n" + } + parsed, err := ParseProvider(strings.NewReader(manifest)) + if !tc.valid { + if err == nil { + t.Fatal("expected invalid cancellation mode") + } + return + } + if err != nil { + t.Fatal(err) + } + if parsed.Exec.Cancellation != ProviderCancellationMode(tc.value) { + t.Fatalf("unexpected cancellation mode: %q", parsed.Exec.Cancellation) + } + }) + } +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index e86217ea8d..8e449759c9 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -419,7 +419,8 @@ type ProviderDockerDriverConfig struct { Path string `json:"path,omitempty"` // If false, Devsy will not try to install docker into the machine. - Install types.StrBool `json:"install,omitempty"` + Install types.StrBool `json:"install,omitempty"` + ConfigureDaemon types.StrBool `json:"configureDaemon,omitempty"` // Builder to use with docker Builder string `json:"builder,omitempty"` @@ -517,7 +518,16 @@ type ProviderBinary struct { Name string `json:"name,omitempty"` } +type ProviderCancellationMode string + +const ( + ProviderCancellationDefault ProviderCancellationMode = "" + ProviderCancellationFile ProviderCancellationMode = "file" +) + type ProviderCommands struct { + Cancellation ProviderCancellationMode `json:"cancellation,omitempty"` + // Init is run directly after `devsy provider init` Init types.StrArray `json:"init,omitempty"` diff --git a/pkg/shell/provider_exec.go b/pkg/shell/provider_exec.go new file mode 100644 index 0000000000..d8ac148b6a --- /dev/null +++ b/pkg/shell/provider_exec.go @@ -0,0 +1,78 @@ +package shell + +import ( + "context" + "errors" + "fmt" + "os/exec" + "runtime" + "strings" + "syscall" + "time" + + "mvdan.cc/sh/v3/expand" + "mvdan.cc/sh/v3/interp" +) + +func ProviderExecHandler() interp.ExecHandlerFunc { + return func(ctx context.Context, args []string) error { + hc := interp.HandlerCtx(ctx) + path, err := interp.LookPathDir(hc.Dir, hc.Env, args[0]) + if err != nil { + fmt.Fprintln(hc.Stderr, err) + return interp.ExitStatus(127) + } + + newCommand := func(path string, args []string) *exec.Cmd { + cmd := exec.CommandContext(ctx, path, args[1:]...) + cmd.Args = args + cmd.Env = providerExecEnv(hc.Env) + cmd.Dir = hc.Dir + cmd.Stdin = hc.Stdin + cmd.Stdout = hc.Stdout + cmd.Stderr = hc.Stderr + cmd.WaitDelay = 2 * time.Second + return cmd + } + + cmd := newCommand(path, args) + err = cmd.Run() + if errors.Is(err, syscall.ENOEXEC) && runtime.GOOS != "windows" { + cmd = newCommand("/bin/sh", append([]string{"/bin/sh", path}, args[1:]...)) + err = cmd.Run() + } + if ctx.Err() != nil { + return context.Cause(ctx) + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + if status, ok := providerExecSignalStatus(exitErr); ok { + return status + } + return interp.ExitStatus(exitErr.ExitCode()) + } + var execErr *exec.Error + if errors.As(err, &execErr) { + fmt.Fprintln(hc.Stderr, err) + return interp.ExitStatus(127) + } + return err + } +} + +func providerExecEnv(env expand.Environ) []string { + vars := make([]string, 0, 64) + for name, variable := range env.Each { + if !variable.IsSet() { + for i, kv := range vars { + if strings.HasPrefix(kv, name+"=") { + vars[i] = "" + } + } + } + if variable.Exported && variable.Kind == expand.String { + vars = append(vars, name+"="+variable.String()) + } + } + return vars +} diff --git a/pkg/shell/provider_exec_test.go b/pkg/shell/provider_exec_test.go new file mode 100644 index 0000000000..b9ec9bb59e --- /dev/null +++ b/pkg/shell/provider_exec_test.go @@ -0,0 +1,152 @@ +package shell + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "mvdan.cc/sh/v3/interp" +) + +func TestProviderExecHelper(t *testing.T) { + if os.Getenv("DEVSY_PROVIDER_EXEC_HELPER") != "1" { + return + } + if os.Getenv("DEVSY_PROVIDER_EXEC_EXIT") == "wait" { + fmt.Fprint(os.Stdout, "ready") + time.Sleep(time.Minute) + } + if os.Getenv("DEVSY_PROVIDER_EXEC_EXIT") != "" { + os.Exit(125) + } + result := struct { + Directory string + Present string + Missing string + Argument string + }{Present: os.Getenv("PRESENT"), Missing: os.Getenv("MISSING")} + result.Directory, _ = os.Getwd() + result.Argument = os.Args[len(os.Args)-1] + _ = json.NewEncoder(os.Stdout).Encode(result) + os.Exit(0) +} + +func TestProviderExecHandlerExportsUnsetAndWorkingDirectory(t *testing.T) { + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + workdir := t.TempDir() + command := fmt.Sprintf( + "export PRESENT='quoted value'; unset MISSING; cd %s; %s -test.run=^TestProviderExecHelper$ 'quoted argument'", + strconv.Quote(workdir), + strconv.Quote(binary), + ) + if runtime.GOOS == "windows" { + command = strings.ReplaceAll(command, `\`, `/`) + } + var stdout, stderr bytes.Buffer + err = RunEmulatedShell(context.Background(), &CommandRunner{ + Command: command, ExecHandler: ProviderExecHandler(), + Environ: append(os.Environ(), "DEVSY_PROVIDER_EXEC_HELPER=1", "MISSING=should-be-unset"), + Stdout: &stdout, Stderr: &stderr, + }) + if err != nil { + t.Fatalf("run provider shell: %v; stderr: %s", err, stderr.String()) + } + var got struct{ Directory, Present, Missing, Argument string } + if err := json.Unmarshal(stdout.Bytes(), &got); err != nil { + t.Fatal(err) + } + expectedDir, err := filepath.EvalSymlinks(workdir) + if err != nil { + t.Fatal(err) + } + actualDir, err := filepath.EvalSymlinks(got.Directory) + if err != nil { + t.Fatal(err) + } + if !strings.EqualFold(filepath.Clean(actualDir), filepath.Clean(expectedDir)) { + t.Fatalf("working directory: got %q, want %q", got.Directory, workdir) + } + if got.Present != "quoted value" || got.Missing != "" || got.Argument != "quoted argument" { + t.Fatalf("shell exported variables or arguments changed: %+v", got) + } +} + +func TestProviderExecHandlerExitStatuses(t *testing.T) { + for _, tc := range []struct { + command string + status int + }{ + {command: "command-that-does-not-exist-846307", status: 127}, + {command: "exit 7", status: 7}, + {command: "exit 125", status: 125}, + } { + var stderr bytes.Buffer + err := RunEmulatedShell(context.Background(), &CommandRunner{ + Command: tc.command, + ExecHandler: ProviderExecHandler(), + Environ: os.Environ(), + Stderr: &stderr, + }) + var exitStatus interp.ExitStatus + if !errors.As(err, &exitStatus) || int(exitStatus) != tc.status { + t.Errorf("%q: got %v, want exit %d", tc.command, err, tc.status) + } + } +} + +func TestProviderExecHandlerCancellationReturnsCause(t *testing.T) { + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancelCause(context.Background()) + cause := errors.New("forced provider cancellation") + ready := make(chan struct{}) + finished := make(chan error, 1) + go func() { + finished <- RunEmulatedShell(ctx, &CommandRunner{ + Command: strconv.Quote(binary) + " -test.run=^TestProviderExecHelper$", + Environ: append(os.Environ(), "DEVSY_PROVIDER_EXEC_HELPER=1", "DEVSY_PROVIDER_EXEC_EXIT=wait"), + ExecHandler: ProviderExecHandler(), Stdout: &shellReadyWriter{ready: ready}, + }) + }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("shell child did not start") + } + cancel(cause) + select { + case err := <-finished: + if !errors.Is(err, cause) { + t.Fatalf("cancellation: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("provider shell did not finish") + } +} + +type shellReadyWriter struct{ ready chan struct{} } + +func (w *shellReadyWriter) Write(p []byte) (int, error) { + if bytes.Contains(p, []byte("ready")) { + select { + case <-w.ready: + default: + close(w.ready) + } + } + return len(p), nil +} diff --git a/pkg/shell/provider_exec_unix.go b/pkg/shell/provider_exec_unix.go new file mode 100644 index 0000000000..6fb0e981af --- /dev/null +++ b/pkg/shell/provider_exec_unix.go @@ -0,0 +1,17 @@ +//go:build !windows + +package shell + +import ( + "os/exec" + "syscall" + + "mvdan.cc/sh/v3/interp" +) + +func providerExecSignalStatus(err *exec.ExitError) (interp.ExitStatus, bool) { + if status, ok := err.Sys().(syscall.WaitStatus); ok && status.Signaled() { + return interp.ExitStatus(128 + status.Signal()), true + } + return 0, false +} diff --git a/pkg/shell/provider_exec_windows.go b/pkg/shell/provider_exec_windows.go new file mode 100644 index 0000000000..4b47e4b558 --- /dev/null +++ b/pkg/shell/provider_exec_windows.go @@ -0,0 +1,11 @@ +package shell + +import ( + "os/exec" + + "mvdan.cc/sh/v3/interp" +) + +func providerExecSignalStatus(_ *exec.ExitError) (interp.ExitStatus, bool) { + return 0, false +} diff --git a/pkg/shell/shell.go b/pkg/shell/shell.go index 6be8172baa..f315f872ce 100644 --- a/pkg/shell/shell.go +++ b/pkg/shell/shell.go @@ -19,11 +19,12 @@ import ( ) type CommandRunner struct { - Command string - Environ []string - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer + Command string + Environ []string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + ExecHandler interp.ExecHandlerFunc } func RunEmulatedShell( @@ -47,11 +48,12 @@ func RunEmulatedShell( } r, err := interp.New(buildRunnerOptions(shellRunnerOptions{ - stdin: runner.Stdin, - stdout: runner.Stdout, - stderr: runner.Stderr, - env: runner.Environ, - dir: dir, + stdin: runner.Stdin, + stdout: runner.Stdout, + stderr: runner.Stderr, + env: runner.Environ, + dir: dir, + execHandler: runner.ExecHandler, })...) if err != nil { return fmt.Errorf("create shell runner: %w", err) @@ -71,23 +73,27 @@ func RunEmulatedShell( } type shellRunnerOptions struct { - stdin io.Reader - stdout io.Writer - stderr io.Writer - env []string - dir string + stdin io.Reader + stdout io.Writer + stderr io.Writer + env []string + dir string + execHandler interp.ExecHandlerFunc } func buildRunnerOptions(p shellRunnerOptions) []interp.RunnerOption { defaultOpenHandler := interp.DefaultOpenHandler() - defaultExecHandler := interp.DefaultExecHandler(2 * time.Second) + execHandler := p.execHandler + if execHandler == nil { + execHandler = interp.DefaultExecHandler(2 * time.Second) + } return []interp.RunnerOption{ interp.StdIO(p.stdin, p.stdout, p.stderr), interp.Env(expand.ListEnviron(p.env...)), interp.Dir(p.dir), interp.ExecHandlers(func(next interp.ExecHandlerFunc) interp.ExecHandlerFunc { return func(ctx context.Context, args []string) error { - return defaultExecHandler(ctx, args) + return execHandler(ctx, args) } }), // CallHandler intercepts builtin calls. mvdan.cc/sh/v3 v3.13.1 marks diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx index 10b4d2de79..26976cbb19 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx @@ -38,6 +38,24 @@ This section can use option values. - **exec.shutdown**: the command that stops the machine once it is idle. Option values are available in it. - **binaries**: helper binaries for `exec.shutdown`. Same format as [provider binaries](./binaries.mdx). +## Docker management policy + +For the `docker` driver, the `agent.docker` section controls how Devsy manages Docker on the agent machine: + +```yaml +agent: + local: false + driver: docker + docker: + path: docker + install: false + configureDaemon: false +``` + +Set `install: false` when the provider supplies a Docker CLI. If that executable is missing, Devsy reports an error instead of installing Docker. An existing executable remains usable. With `install` unset or `true`, Devsy retains its existing installation behavior. + +Set `configureDaemon: false` when the provider owns the Docker daemon, including a private daemon inside a VM. Devsy then does not read or modify the daemon configuration for its management step and does not reload the daemon. With `configureDaemon` unset or `true`, Devsy retains its existing behavior of configuring the daemon for a remote Docker agent. Local agents are not configured by this step. These settings accept provider option templates, which must resolve to valid boolean values before the agent starts Docker setup. This policy controls Devsy's daemon management, not arbitrary commands a user runs inside a devcontainer. + ## Stopping idle environments Devsy tracks connections and stops the environment after the timeout. It starts the environment again when the user returns. State survives only if the provider keeps the storage. A shutdown command that deletes the machine must keep workspace data elsewhere, for example on a persistent volume. diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx index 8c88b71588..4bd0a348af 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx @@ -42,7 +42,7 @@ devsy workspace up github.com/microsoft/vscode-remote-try-go ## exec -Each entry is a POSIX shell script. Only `command` is required. +The `command`, `init`, `create`, `delete`, `start`, `stop`, `status` and `describe` entries are POSIX shell scripts or command arrays. Only `command` is required. `cancellation` is an execution policy, not a script. - **command**: runs a command in the environment. Devsy uses it to inject itself, and sends all traffic through its standard input and output. Devsy fills in the `${COMMAND}` placeholder. - **init**: checks that the options are valid and the provider is ready. Devsy runs it when options change. @@ -50,8 +50,13 @@ Each entry is a POSIX shell script. Only `command` is required. - **start**, **stop**: start and stop a machine. Machine providers only. - **status**: prints one of `Running`, `Busy` (wait and check again), `Stopped` or `NotFound`. - **describe**: prints a text description of the machine. +- **cancellation**: omit it for existing process cancellation behavior, or set `file` to give provider commands a cooperative cleanup window. -A provider without `create` is a non-machine provider and uses only `command` and `init`. +A provider without `create` is a non-machine provider and uses only `command` and `init`. The cancellation policy applies to both machine and non-machine providers. + +With `exec.cancellation: file`, Devsy sets the reserved `DEVSY_PROVIDER_CANCEL_FILE` environment variable to a private, initially nonexistent marker path for each invocation. The provider must watch for the marker, stop admitting work, cancel its owned processes, finish cleanup, and exit. On caller cancellation, Devsy creates the marker and allows up to 10 seconds for cleanup before forcing the owned provider process to stop. Devsy waits for the invocation to finish and still returns the caller's cancellation error even if cleanup exits successfully. A provider must not set or override the reserved variable. + +File mode requires finite or nonblocking standard streams, direct file handles, or blocking streams whose owner unblocks them on cancellation of the original calling context. A provider must not assume that an arbitrary blocked reader or writer is interrupted by process cancellation. Machine command transports close their pipe endpoints on cancellation of the original context. Standard input can close independently of standard output, so providers must continue draining output after input reaches EOF. On Windows, Devsy runs these scripts in an [emulated shell](https://github.com/mvdan/sh). The emulated shell has no `grep` or `sed`. Move that logic into a helper binary. diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx new file mode 100644 index 0000000000..ef4be95f67 --- /dev/null +++ b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx @@ -0,0 +1,60 @@ +--- +title: Docker Sandboxes +sidebar_label: Docker Sandboxes +--- + +The Docker Sandboxes provider runs each Devsy machine in a local `sbx` microVM. The Devsy agent uses the private Docker Engine inside that VM to build and run ordinary devcontainers. A host Docker Engine is not required. + +## Requirements + +Install the [standalone `sbx` CLI](https://docs.docker.com/ai/sandboxes/install/) and sign in with `sbx login` outside Devsy. Keep the version listed in the provider release compatibility profile. The provider rejects versions and host systems that have not passed its native qualification. + +Local Docker Sandboxes require Apple silicon with macOS 14 or later, Windows 11 on amd64 with Windows Hypervisor Platform, or Ubuntu 24.04 or later on amd64 or arm64 with accessible KVM. A VM running the provider also needs nested virtualization. A host system is supported only if the provider release lists that system as qualified. Docker Desktop and a host Docker Engine are not required. + +Install a published provider release with: + +```sh +devsy provider add docker-sandbox +devsy up https://github.com/devcontainers/samples --provider docker-sandbox +``` + +Devsy Desktop offers the same provider under **Local**. Configure its options in the provider setup wizard. The provider does not install `sbx`, sign in, open a login browser, or change Docker Sandboxes settings. + +## Machine resources + +Set machine options when creating the machine. `SBX_CPUS` sets virtual CPUs, `SBX_MEMORY` sets memory, `SBX_ROOT_DISK` sets space for the guest system and sources, and `SBX_DOCKER_DISK` sets space for Docker images, containers, and volumes. Sizes use integer binary `m` or `g` units. The defaults are two CPUs, `4g` of memory, and `40g` for each disk. `SBX_PATH` can select a different `sbx` executable. The provider validates these values before creating a VM. + +Resources cannot be resized in place. Create another machine and deliberately move the data you need before deleting the original. Changing a saved machine option does not destroy and recreate its VM. + +## Sources and development containers + +Git sources are cloned into the guest. Local folder sources are uploaded to the guest, not mounted live from the workstation. A local folder rebuild uploads host files again and can overwrite matching guest edits. Push work to Git or transfer it out of the VM if you need it after machine deletion. + +The existing Devsy Docker driver handles Dockerfiles, Features, Compose, image sources, lifecycle commands, named volumes, and ordinary guest Docker mounts. An absolute bind source refers to a path in the guest VM, not a path on your workstation. Devsy runs `initializeCommand` in the VM. Your devcontainer chooses its own user and shell. + +Devsy's SSH tunnel connects to the final devcontainer for terminal, editor, and application forwarding. A port published by the guest Docker Engine is not automatically a workstation port. Use Devsy port forwarding to reach an application from your host. + +## Network and credentials + +Docker Sandboxes network policy still applies to the VM and its containers. The provider does not open blocked destinations or disable TLS verification. Container builds and applications may need project specific proxy variables and CA installation. A project using a forward proxy should add the proxy's public CA to its own build context outside version controlled source, install it in the image trust store before HTTPS commands, and pass the proxy build arguments and runtime variables deliberately. Rebuild after CA rotation. Other base images and applications can use different trust stores. + +The guest Docker Engine does not share the host Docker Engine or its socket. Devsy credential forwarding and Docker Sandboxes host integrations are separate authority paths. Check your Devsy context and Docker Sandboxes settings for Git and registry credentials, SSH agent forwarding, MCP integrations, and network access. A mountless VM does not disable those integrations. Private image pulls from the guest Engine and sandbox template pulls use separate credential paths. + +## Stop, delete, and recovery + +Stopping a machine preserves its guest sources, images, containers, and volumes. Starting resumes the same VM. Rebuilding a workspace changes the inner devcontainer without replacing the VM. Deleting a default workspace machine removes the VM and all its guest data, including named volumes. Deleting a workspace on an explicitly shared machine leaves that machine and its other workspaces intact. Uninstalling the provider does not delete its VMs. + +Provider state is stored under your user configuration directory at `devsy-provider-docker-sandbox/state-v1`. If creation fails, do not remove that directory or manually reuse a sandbox name. The provider reports the exact machine key and any unresolved child attempt or backend operation. A lost VM is never silently recreated in the same machine generation. + +Find the installed binary with `devsy provider list --result-format=json`. Read `config.binaries.DOCKER_SANDBOX_PROVIDER` for the asset name. Devsy stores it under its data directory at `contexts//providers//binaries/docker_sandbox_provider/`. With `DEVSY_HOME` set, that variable names the data directory. Otherwise Devsy uses `~/.devsy` on macOS and Linux, or `%LOCALAPPDATA%\devsy` on Windows. Use the resolved local executable, not the asset URL. + +The provider executable installed with the provider accepts: + +```text + doctor --machine-key --json + recover --machine-key --attempt --acknowledge-client-terminated + recover --machine-key --operation --acknowledge-backend-settled + delete --machine-key +``` + +Use `doctor` to inspect the saved identity and backend observation without starting a stopped VM. An attempt acknowledgement is appropriate only after its orphaned client has terminated or the host has restarted. A backend acknowledgement is appropriate only after the Docker side operation has settled. Neither acknowledgement starts, stops, or removes a VM. Use scoped deletion after provider removal only for a machine you intend to destroy. Never use global `sbx` reset or prune to recover one machine. diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx index 98f8093909..92d3df5a4e 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx +++ b/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx @@ -11,6 +11,7 @@ Providers you can add: - [Harvester (harvester)](https://github.com/devsy-org/devsy-provider-harvester) - [Apple (apple)](https://github.com/devsy-org/devsy/tree/main/providers/apple) - [MicroSandbox (microsandbox)](https://github.com/devsy-org/devsy-provider-microsandbox) +- [Docker Sandboxes (docker-sandbox)](https://github.com/devsy-org/devsy-provider-docker-sandbox) - [Lima (lima)](https://github.com/devsy-org/devsy-provider-lima) - [OrbStack (orbstack)](https://github.com/devsy-org/devsy-provider-orbstack) - [SSH (ssh)](https://github.com/devsy-org/devsy-provider-ssh) @@ -103,6 +104,18 @@ devsy provider add builtin:microsandbox --name legacy-microsandbox --use=false If a switch fails during initialization, the new source stays selected. Fix the cause and run `devsy provider init `, or set the previous source again. Add `--version ` to `set-source` to pin a release. +### Docker Sandboxes + +Install Docker Sandboxes and sign in with `sbx login` before adding the provider: + +```sh +devsy provider add docker-sandbox +``` + +The [Docker Sandboxes guide](./docker-sandbox.mdx) covers qualified hosts, resource options, guest Docker, source files, networking, data deletion, and scoped recovery. The provider needs a published release containing native qualification for your host. + + + ## Set provider options Each provider has its own options. In the desktop app, open **Providers** and click **Edit** on a provider. With the CLI, set options when you add or initialize: diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/meta.json b/sites/docs-devsy-sh/content/docs/managing-providers/meta.json index e26c174546..991084c9c7 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/meta.json +++ b/sites/docs-devsy-sh/content/docs/managing-providers/meta.json @@ -1,4 +1,4 @@ { "title": "Managing your Providers", - "pages": ["what-are-providers", "manage-providers", "kubernetes-registry-access"] + "pages": ["what-are-providers", "manage-providers", "docker-sandbox", "kubernetes-registry-access"] } From a72a1afabb0af70a0309d484d36f988f196919b0 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 2 Nov 2026 16:01:19 -0600 Subject: [PATCH 02/20] fix(provider): satisfy cancellation and Docker policy checks --- .../agentworkspace/docker_darwin_test.go | 22 +- cmd/internal/agentworkspace/up.go | 41 +- .../clientimplementation/provider_cancel.go | 85 ++-- .../provider_cancel_test.go | 370 +++++++++++------- .../provider_cancel_windows_test.go | 6 +- pkg/options/resolve_docker_test.go | 39 +- pkg/provider/parse.go | 21 +- pkg/provider/parse_docker_test.go | 9 +- pkg/shell/provider_exec.go | 120 ++++-- pkg/shell/provider_exec_test.go | 73 ++-- pkg/shell/provider_exec_unix.go | 6 +- 11 files changed, 525 insertions(+), 267 deletions(-) diff --git a/cmd/internal/agentworkspace/docker_darwin_test.go b/cmd/internal/agentworkspace/docker_darwin_test.go index 9a67b65fcd..e0c73bb3aa 100644 --- a/cmd/internal/agentworkspace/docker_darwin_test.go +++ b/cmd/internal/agentworkspace/docker_darwin_test.go @@ -17,10 +17,14 @@ import ( func writeExecutable(t *testing.T, path string) { t.Helper() - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { t.Fatal(err) } - if err := os.WriteFile(path, []byte("#!/bin/sh\n"), 0o755); err != nil { + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + if err := os.Link(executable, path); err != nil { t.Fatal(err) } } @@ -93,7 +97,7 @@ func TestFindDarwinDockerCLIHomeFailureDoesNotBreakStaticDiscovery(t *testing.T) func TestFindDarwinDockerCLIRejectsNonExecutable(t *testing.T) { path := filepath.Join(t.TempDir(), "docker") - if err := os.WriteFile(path, nil, 0o644); err != nil { + if err := os.WriteFile(path, nil, 0o600); err != nil { t.Fatal(err) } if _, err := findDarwinDockerCLIInPaths([]string{path}); err == nil { @@ -101,18 +105,20 @@ func TestFindDarwinDockerCLIRejectsNonExecutable(t *testing.T) { } } -func TestFindDarwinDockerCLIRejectsInaccessibleCandidate(t *testing.T) { +func TestFindDarwinDockerCLISkipsBrokenCandidate(t *testing.T) { tmp := t.TempDir() - inaccessible := filepath.Join(tmp, "inaccessible", "docker") - writeExecutable(t, inaccessible) - if err := os.Chmod(inaccessible, 0o001); err != nil { + broken := filepath.Join(tmp, "broken", "docker") + if err := os.MkdirAll(filepath.Dir(broken), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(tmp, "missing", "docker"), broken); err != nil { t.Fatal(err) } valid := filepath.Join(tmp, "valid", "docker") writeExecutable(t, valid) - path, err := findDarwinDockerCLIInPaths([]string{inaccessible, valid}) + path, err := findDarwinDockerCLIInPaths([]string{broken, valid}) if err != nil || path != valid { t.Fatalf("expected fallback to valid candidate %q, got %q (err: %v)", valid, path, err) } diff --git a/cmd/internal/agentworkspace/up.go b/cmd/internal/agentworkspace/up.go index 27c3b2a7aa..6566a2cb76 100644 --- a/cmd/internal/agentworkspace/up.go +++ b/cmd/internal/agentworkspace/up.go @@ -525,24 +525,13 @@ func (w *workspaceInitializer) ensureDockerInstalled(ctx context.Context) (strin } if dockerCmd != docker { - path, err := exec.LookPath(dockerCmd) - if err != nil { - return "", fmt.Errorf("custom docker path %q not found: %w", dockerCmd, err) - } - return path, nil + return findCustomDockerCLI(dockerCmd) } - if dockerCmd == docker && runtime.GOOS == "darwin" { - find := w.findDarwinDocker - if find == nil { - find = findDarwinDockerCLI - } - path, err := find() - if err == nil { - return path, nil - } - if !w.isDockerInstallDisabled() { - return "", err + if runtime.GOOS == "darwin" { + path, err := w.discoverDarwinDockerCLI() + if err == nil || !w.isDockerInstallDisabled() { + return path, err } } @@ -553,6 +542,26 @@ func (w *workspaceInitializer) ensureDockerInstalled(ctx context.Context) (strin ) } + return w.installMissingDocker(ctx) +} + +func findCustomDockerCLI(dockerCmd string) (string, error) { + path, err := exec.LookPath(dockerCmd) + if err != nil { + return "", fmt.Errorf("custom docker path %q not found: %w", dockerCmd, err) + } + return path, nil +} + +func (w *workspaceInitializer) discoverDarwinDockerCLI() (string, error) { + find := w.findDarwinDocker + if find == nil { + find = findDarwinDockerCLI + } + return find() +} + +func (w *workspaceInitializer) installMissingDocker(ctx context.Context) (string, error) { log.Debug("attempting to install docker") install := w.installDocker if install == nil { diff --git a/pkg/client/clientimplementation/provider_cancel.go b/pkg/client/clientimplementation/provider_cancel.go index 9647d37a54..d3aa6c176e 100644 --- a/pkg/client/clientimplementation/provider_cancel.go +++ b/pkg/client/clientimplementation/provider_cancel.go @@ -23,7 +23,7 @@ func runProviderWithCancellationUsingPublisher( parent context.Context, opts RunCommandOptions, publish func(string) error, -) error { +) (retErr error) { if parent.Err() != nil { return context.Cause(parent) } @@ -35,19 +35,17 @@ func runProviderWithCancellationUsingPublisher( if err != nil { return fmt.Errorf("create provider cancellation directory: %w", err) } - defer os.RemoveAll(dir) - - marker := filepath.Join(dir, "cancel") - environ := make([]string, 0, len(opts.Environ)+1) - for _, kv := range opts.Environ { - key, _, ok := strings.Cut(kv, "=") - if ok && providerReservedKey(key) { - continue + defer func() { + if err := os.RemoveAll(dir); err != nil { + retErr = errors.Join( + retErr, + fmt.Errorf("remove provider cancellation directory: %w", sanitizeMarkerError(err)), + ) } - environ = append(environ, kv) - } - opts.Environ = append(environ, config.EnvProviderCancelFile+"="+marker) + }() + marker := filepath.Join(dir, "cancel") + opts.Environ = providerCancellationEnvironment(opts.Environ, marker) execution, cancel := context.WithCancelCause(context.WithoutCancel(parent)) defer cancel(context.Canceled) if parent.Err() != nil { @@ -57,9 +55,39 @@ func runProviderWithCancellationUsingPublisher( go func() { completed <- runCommandExecution(execution, opts) }() + return waitForProviderCancellation( + parent, + marker, + publish, + providerExecution{cancel: cancel, completed: completed}, + ) +} +func providerCancellationEnvironment(original []string, marker string) []string { + environ := make([]string, 0, len(original)+1) + for _, kv := range original { + key, _, ok := strings.Cut(kv, "=") + if ok && providerReservedKey(key) { + continue + } + environ = append(environ, kv) + } + return append(environ, config.EnvProviderCancelFile+"="+marker) +} + +type providerExecution struct { + cancel context.CancelCauseFunc + completed <-chan error +} + +func waitForProviderCancellation( + parent context.Context, + marker string, + publish func(string) error, + process providerExecution, +) error { select { - case result := <-completed: + case result := <-process.completed: if parent.Err() != nil { return context.Cause(parent) } @@ -69,22 +97,25 @@ func runProviderWithCancellationUsingPublisher( cause := context.Cause(parent) if err := publish(marker); err != nil { - cancel(cause) - <-completed + process.cancel(cause) + <-process.completed return errors.Join( cause, fmt.Errorf("publish provider cancellation marker: %w", sanitizeMarkerError(err)), ) } + return waitForProviderGrace(cause, process) +} +func waitForProviderGrace(cause error, process providerExecution) error { timer := time.NewTimer(providerCancellationGrace) defer timer.Stop() select { - case <-completed: + case <-process.completed: return cause case <-timer.C: - cancel(cause) - <-completed + process.cancel(cause) + <-process.completed return cause } } @@ -96,15 +127,24 @@ func providerReservedKey(key string) bool { return key == config.EnvProviderCancelFile } -func publishProviderCancellation(marker string) error { - f, err := os.OpenFile(marker, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) +func publishProviderCancellation(marker string) (retErr error) { + root, err := os.OpenRoot(filepath.Dir(marker)) + if err != nil { + return fmt.Errorf("cannot open marker directory: %w", sanitizeMarkerError(err)) + } + defer func() { + if err := root.Close(); err != nil { + retErr = errors.Join(retErr, sanitizeMarkerError(err)) + } + }() + f, err := root.OpenFile("cancel", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) if err == nil { return sanitizeMarkerError(f.Close()) } if !errors.Is(err, os.ErrExist) { return fmt.Errorf("cannot create marker: %w", sanitizeMarkerError(err)) } - info, statErr := os.Lstat(marker) + info, statErr := root.Lstat("cancel") if statErr != nil { return fmt.Errorf("cannot inspect existing marker: %w", sanitizeMarkerError(statErr)) } @@ -118,8 +158,7 @@ func sanitizeMarkerError(err error) error { if err == nil { return nil } - var pathErr *os.PathError - if errors.As(err, &pathErr) { + if pathErr, ok := errors.AsType[*os.PathError](err); ok { return pathErr.Err } return errors.New("marker filesystem operation failed") diff --git a/pkg/client/clientimplementation/provider_cancel_test.go b/pkg/client/clientimplementation/provider_cancel_test.go index 5ec5872f43..d5a74a5bbd 100644 --- a/pkg/client/clientimplementation/provider_cancel_test.go +++ b/pkg/client/clientimplementation/provider_cancel_test.go @@ -24,72 +24,140 @@ import ( "mvdan.cc/sh/v3/interp" ) +const ( + testTransportRead = "transport-read" + testWindowsOS = "windows" + testCancelProvider = "cancel-provider" + testOpaqueCommand = "opaque" +) + func TestProviderCancellationHelper(t *testing.T) { if os.Getenv("DEVSY_CANCEL_HELPER") != "1" { return } marker := os.Getenv(config.EnvProviderCancelFile) - switch os.Getenv("DEVSY_CANCEL_BEHAVIOR") { + switch behavior := os.Getenv("DEVSY_CANCEL_BEHAVIOR"); behavior { case "acknowledge": - fmt.Fprint(os.Stdout, "ready") - for { - if _, err := os.Stat(marker); err == nil { - if path := os.Getenv("DEVSY_CANCEL_ACK_PATH"); path != "" { - _ = os.WriteFile(path, []byte("finished"), 0o600) - } - return - } - time.Sleep(20 * time.Millisecond) + if _, err := fmt.Fprint(os.Stdout, "ready"); err != nil { + os.Exit(126) } + awaitTestCancellationMarker(marker, os.Getenv("DEVSY_CANCEL_ACK_PATH")) case "ignore": - fmt.Fprint(os.Stdout, "ready") - time.Sleep(time.Minute) - case "transport-read", "transport-write": - ready := os.Getenv("DEVSY_CANCEL_READY_PATH") - _ = os.WriteFile(ready, []byte("ready"), 0o600) - go func() { - for { - if _, err := os.Stat(marker); err == nil { - _ = os.WriteFile(os.Getenv("DEVSY_CANCEL_ACK_PATH"), []byte("finished"), 0o600) - os.Exit(0) - } - time.Sleep(20 * time.Millisecond) - } - }() - if os.Getenv("DEVSY_CANCEL_BEHAVIOR") == "transport-read" { - var b [1]byte - _, _ = os.Stdin.Read(b[:]) - for { - time.Sleep(time.Second) - } - } else { - if runtime.GOOS != "windows" { - signal.Ignore(syscall.Signal(13)) - } - chunk := bytes.Repeat([]byte("x"), 1024) - for { - if _, err := os.Stdout.Write(chunk); err != nil { - for { - time.Sleep(time.Second) - } - } - } + if _, err := fmt.Fprint(os.Stdout, "ready"); err != nil { + os.Exit(126) } + time.Sleep(time.Minute) + case testTransportRead, "transport-write": + testTransportCancellationHelper(behavior, marker) + default: + runTestCancellationExit(behavior, marker) + } + os.Exit(0) +} + +func runTestCancellationExit(behavior, marker string) { + switch behavior { case "status": os.Exit(7) case "status-127": os.Exit(127) case "bytes": - os.Stdout.Write([]byte{0, 0xff, 0x80, '\r', '\n'}) - os.Stderr.Write([]byte(os.Getenv("DEVSY_CANCEL_SECRET"))) + writeTestCancellationBytes() case "environment": - fmt.Fprint(os.Stdout, marker) + _, err := fmt.Fprint(os.Stdout, marker) + exitTestHelperOnError(err) default: os.Exit(125) } +} + +func writeTestCancellationBytes() { + _, err := os.Stdout.Write([]byte{0, 0xff, 0x80, '\r', '\n'}) + exitTestHelperOnError(err) + _, err = os.Stderr.Write([]byte(os.Getenv("DEVSY_CANCEL_SECRET"))) + exitTestHelperOnError(err) +} + +func exitTestHelperOnError(err error) { + if err != nil { + os.Exit(126) + } +} + +func testHelperRelativePath(path string) (string, error) { + relative, err := filepath.Rel(os.TempDir(), path) + if err != nil { + return "", err + } + if !filepath.IsLocal(relative) { + return "", os.ErrPermission + } + return relative, nil +} + +func testHelperWriteFile(path, data string) error { + relative, err := testHelperRelativePath(path) + if err != nil { + return err + } + root, err := os.OpenRoot(os.TempDir()) + if err != nil { + return err + } + writeErr := root.WriteFile(relative, []byte(data), 0o600) + return errors.Join(writeErr, root.Close()) +} + +func awaitTestCancellationMarker(marker, acknowledgement string) { + relative, err := testHelperRelativePath(marker) + if err != nil { + os.Exit(126) + } + root, err := os.OpenRoot(os.TempDir()) + if err != nil { + os.Exit(126) + } + for { + if info, err := root.Lstat(relative); err == nil && info.Mode().IsRegular() { + acknowledgeTestCancellationMarker(root, acknowledgement) + } + time.Sleep(20 * time.Millisecond) + } +} + +func acknowledgeTestCancellationMarker(root *os.Root, acknowledgement string) { + if acknowledgement != "" { + exitTestHelperOnError(testHelperWriteFile(acknowledgement, "finished")) + } + exitTestHelperOnError(root.Close()) os.Exit(0) } +func testTransportCancellationHelper(behavior, marker string) { + if err := testHelperWriteFile(os.Getenv("DEVSY_CANCEL_READY_PATH"), "ready"); err != nil { + os.Exit(126) + } + go awaitTestCancellationMarker(marker, os.Getenv("DEVSY_CANCEL_ACK_PATH")) + if behavior == testTransportRead { + var b [1]byte + _, _ = os.Stdin.Read(b[:]) + for { + time.Sleep(time.Second) + } + } + if runtime.GOOS != testWindowsOS { + signal.Ignore(syscall.Signal(13)) + } + chunk := bytes.Repeat([]byte("x"), 1024) + for { + if _, err := os.Stdout.Write(chunk); err != nil { + for { + time.Sleep(time.Second) + } + } + } +} + func cancelHelperOptions(behavior string) RunCommandOptions { binary, err := os.Executable() if err != nil { @@ -128,8 +196,15 @@ func TestProviderCancellationCooperatesAndCleansUp(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("provider did not acknowledge cancellation") } - contents, err := os.ReadFile(ack) + requireTestAcknowledgement(t, ack) +} + +func requireTestAcknowledgement(t *testing.T, path string) { + t.Helper() + root, err := os.OpenRoot(filepath.Dir(path)) require.NoError(t, err) + contents, readErr := root.ReadFile(filepath.Base(path)) + require.NoError(t, errors.Join(readErr, root.Close())) require.Equal(t, "finished", string(contents)) } @@ -226,12 +301,12 @@ func TestProviderCancellationPublicationFailureReapsProvider(t *testing.T) { } } -func TestProviderCancellationPrivateEnvironmentAndTypedExits(t *testing.T) { +func TestProviderCancellationPrivateEnvironment(t *testing.T) { dir, err := createProviderInvocationDir() require.NoError(t, err) - defer os.RemoveAll(dir) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) require.True(t, filepath.IsAbs(dir)) - if runtime.GOOS != "windows" { + if runtime.GOOS != testWindowsOS { info, err := os.Stat(dir) require.NoError(t, err) require.Equal(t, os.FileMode(0o700), info.Mode().Perm()) @@ -243,7 +318,7 @@ func TestProviderCancellationPrivateEnvironmentAndTypedExits(t *testing.T) { config.EnvProviderCancelFile+"=forged", config.EnvProviderCancelFile+"=again", ) - if runtime.GOOS == "windows" { + if runtime.GOOS == testWindowsOS { opts.Environ = append( opts.Environ, strings.ToLower(config.EnvProviderCancelFile)+"=windows-forged", @@ -257,43 +332,48 @@ func TestProviderCancellationPrivateEnvironmentAndTypedExits(t *testing.T) { require.True(t, filepath.IsAbs(marker)) _, err = os.Stat(filepath.Dir(marker)) require.ErrorIs(t, err, os.ErrNotExist) +} +func TestProviderCancellationTypedExits(t *testing.T) { for _, status := range []int{7, 125, 127} { - opts = cancelHelperOptions("status") + opts := cancelHelperOptions("status") if status == 125 { opts.Environ = append(opts.Environ, "DEVSY_CANCEL_BEHAVIOR=other") } if status == 127 { opts.Environ = append(opts.Environ, "DEVSY_CANCEL_BEHAVIOR=status-127") } - err = RunCommand(context.Background(), opts) + err := RunCommand(context.Background(), opts) var exitErr *exec.ExitError require.ErrorAs(t, err, &exitErr) require.Equal(t, status, exitErr.ExitCode()) } +} - if runtime.GOOS != "windows" { - binary, err := os.Executable() - require.NoError(t, err) - for _, status := range []int{7, 127} { - command := "command-that-does-not-exist-846307" - if status == 7 { - command = fmt.Sprintf( - "DEVSY_CANCEL_HELPER=1 DEVSY_CANCEL_BEHAVIOR=status '%s' -test.run=^TestProviderCancellationHelper$", - binary, - ) - } - err := RunCommand(context.Background(), RunCommandOptions{ - Command: types.StrArray{ - command, - }, - Environ: os.Environ(), - Cancellation: provider.ProviderCancellationFile, - }) - var shellExit interp.ExitStatus - require.ErrorAs(t, err, &shellExit) - require.Equal(t, status, int(shellExit)) +func TestProviderCancellationShellExits(t *testing.T) { + if runtime.GOOS == testWindowsOS { + t.Skip("requires POSIX shell") + } + binary, err := os.Executable() + require.NoError(t, err) + for _, status := range []int{7, 127} { + command := "command-that-does-not-exist-846307" + if status == 7 { + command = fmt.Sprintf( + "DEVSY_CANCEL_HELPER=1 DEVSY_CANCEL_BEHAVIOR=status '%s' -test.run=^TestProviderCancellationHelper$", + binary, + ) } + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{ + command, + }, + Environ: os.Environ(), + Cancellation: provider.ProviderCancellationFile, + }) + var shellExit interp.ExitStatus + require.ErrorAs(t, err, &shellExit) + require.Equal(t, status, int(shellExit)) } } @@ -311,7 +391,7 @@ func TestProviderCancellationRawBytesAndStderrRedaction(t *testing.T) { } func TestMachineCommandTransportCancellationJoinsCallback(t *testing.T) { - for _, behavior := range []string{"transport-read", "transport-write"} { + for _, behavior := range []string{testTransportRead, "transport-write"} { t.Run(behavior, func(t *testing.T) { dir := t.TempDir() ready, ack := filepath.Join(dir, "ready"), filepath.Join(dir, "ack") @@ -321,7 +401,7 @@ func TestMachineCommandTransportCancellationJoinsCallback(t *testing.T) { t.Setenv("DEVSY_CANCEL_ACK_PATH", ack) machine := &provider.Machine{ID: "cancel-machine", Context: testLocksContext} cfg := &provider.ProviderConfig{ - Name: "cancel-provider", + Name: testCancelProvider, Exec: provider.ProviderCommands{ Create: types.StrArray{ "create", @@ -340,10 +420,10 @@ func TestMachineCommandTransportCancellationJoinsCallback(t *testing.T) { defer cancel() conn, err := mc.OpenCommandTransport( ctx, - client.CommandTransportOptions{Command: "opaque"}, + client.CommandTransportOptions{Command: testOpaqueCommand}, ) require.NoError(t, err) - defer conn.Close() + t.Cleanup(func() { require.NoError(t, conn.Close()) }) deadline := time.Now().Add(5 * time.Second) for { if _, err := os.Stat(ready); err == nil { @@ -363,9 +443,7 @@ func TestMachineCommandTransportCancellationJoinsCallback(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("transport callback did not finish") } - contents, err := os.ReadFile(ack) - require.NoError(t, err) - require.Equal(t, "finished", string(contents)) + requireTestAcknowledgement(t, ack) }) } } @@ -377,67 +455,87 @@ func TestWorkspaceCommandCancellationJoinsProviderCleanup(t *testing.T) { t.Setenv("DEVSY_CANCEL_HELPER", "1") t.Setenv("DEVSY_CANCEL_BEHAVIOR", "acknowledge") t.Setenv("DEVSY_CANCEL_ACK_PATH", ack) - workspace := &provider.Workspace{ - ID: "cancel-workspace", - Context: testLocksContext, - Provider: provider.WorkspaceProviderConfig{ - Name: "cancel-provider", - }, - } - cfg := &provider.ProviderConfig{ - Name: "cancel-provider", - Exec: provider.ProviderCommands{ - Command: cancelHelperOptions("acknowledge").Command, - Cancellation: provider.ProviderCancellationFile, - }, - } - devsy := &config.Config{ - DefaultContext: testLocksContext, - Contexts: map[string]*config.ContextConfig{testLocksContext: {}}, - } - wc := &workspaceClient{ - providerWorkspace: workspace, - providerConfig: cfg, - devsyConfig: devsy, - } + wc := workspaceCancellationTestClient() ctx, cancel := context.WithCancel(context.Background()) defer cancel() if transportMode { - conn, err := wc.OpenCommandTransport(ctx, client.CommandTransportOptions{Command: "opaque"}) - require.NoError(t, err) - defer conn.Close() - ready := make([]byte, 5) - require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) - _, err = io.ReadFull(conn, ready) - require.NoError(t, err) - require.Equal(t, "ready", string(ready)) - cancel() - require.ErrorIs(t, conn.Wait(), context.Canceled) + requireWorkspaceTransportCancellation(t, wc, ctx, cancel) } else { - ready := make(chan struct{}) - finished := make(chan error, 1) - go func() { - finished <- wc.Command(ctx, client.CommandOptions{ - Command: "opaque", - Stdout: &readyWriter{ready: ready}, - }) - }() - select { - case <-ready: - case <-time.After(5 * time.Second): - t.Fatal("workspace provider did not start") - } - cancel() - select { - case err := <-finished: - require.ErrorIs(t, err, context.Canceled) - case <-time.After(5 * time.Second): - t.Fatal("workspace provider did not finish") - } + requireWorkspaceDirectCancellation(t, wc, ctx, cancel) } - contents, err := os.ReadFile(ack) - require.NoError(t, err) - require.Equal(t, "finished", string(contents)) + requireTestAcknowledgement(t, ack) }) } } + +func workspaceCancellationTestClient() *workspaceClient { + return &workspaceClient{ + providerWorkspace: &provider.Workspace{ + ID: "cancel-workspace", + Context: testLocksContext, + Provider: provider.WorkspaceProviderConfig{Name: testCancelProvider}, + }, + providerConfig: &provider.ProviderConfig{ + Name: testCancelProvider, + Exec: provider.ProviderCommands{ + Command: cancelHelperOptions("acknowledge").Command, + Cancellation: provider.ProviderCancellationFile, + }, + }, + devsyConfig: &config.Config{ + DefaultContext: testLocksContext, + Contexts: map[string]*config.ContextConfig{testLocksContext: {}}, + }, + } +} + +func requireWorkspaceTransportCancellation( + t *testing.T, + wc *workspaceClient, + ctx context.Context, + cancel context.CancelFunc, +) { + t.Helper() + conn, err := wc.OpenCommandTransport(ctx, client.CommandTransportOptions{ + Command: testOpaqueCommand, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + ready := make([]byte, 5) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + _, err = io.ReadFull(conn, ready) + require.NoError(t, err) + require.Equal(t, "ready", string(ready)) + cancel() + require.ErrorIs(t, conn.Wait(), context.Canceled) +} + +func requireWorkspaceDirectCancellation( + t *testing.T, + wc *workspaceClient, + ctx context.Context, + cancel context.CancelFunc, +) { + t.Helper() + ready := make(chan struct{}) + finished := make(chan error, 1) + go func() { + finished <- wc.Command(ctx, client.CommandOptions{ + Command: testOpaqueCommand, + Stdout: &readyWriter{ready: ready}, + RawStdout: true, + }) + }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("workspace provider did not start") + } + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("workspace provider did not finish") + } +} diff --git a/pkg/client/clientimplementation/provider_cancel_windows_test.go b/pkg/client/clientimplementation/provider_cancel_windows_test.go index b76bd46e36..2a9d43af5e 100644 --- a/pkg/client/clientimplementation/provider_cancel_windows_test.go +++ b/pkg/client/clientimplementation/provider_cancel_windows_test.go @@ -13,7 +13,11 @@ func TestProviderCancellationDirectoryWindowsDACL(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(dir) + t.Cleanup(func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("remove invocation directory: %v", err) + } + }) sd, err := windows.GetNamedSecurityInfo( dir, windows.SE_FILE_OBJECT, diff --git a/pkg/options/resolve_docker_test.go b/pkg/options/resolve_docker_test.go index 20567c402d..1bba28ac40 100644 --- a/pkg/options/resolve_docker_test.go +++ b/pkg/options/resolve_docker_test.go @@ -2,14 +2,16 @@ package options import ( "encoding/json" - "strings" "testing" "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/provider" "github.com/devsy-org/devsy/pkg/types" + "github.com/stretchr/testify/require" ) +const disabledDockerPolicy = "false" + func TestResolveDockerDaemonPolicyIntoWorkspaceInfo(t *testing.T) { manifest := &provider.ProviderConfig{ Name: "docker-policy", @@ -26,33 +28,24 @@ func TestResolveDockerDaemonPolicyIntoWorkspaceInfo(t *testing.T) { Contexts: map[string]*config.ContextConfig{ config.DefaultContext: {Providers: map[string]*config.ProviderConfig{ manifest.Name: {Options: map[string]config.OptionValue{ - "INSTALL_DOCKER": {Value: "false"}, - "CONFIGURE_DAEMON": {Value: "false"}, + "INSTALL_DOCKER": {Value: disabledDockerPolicy}, + "CONFIGURE_DAEMON": {Value: disabledDockerPolicy}, }}, }}, }, } resolved := ResolveAgentConfig(devConfig, manifest, nil, nil) - if resolved.Docker.ConfigureDaemon != types.StrBool("false") || - resolved.Docker.Install != types.StrBool("false") { - t.Fatalf("resolved Docker management = %+v, want disabled", resolved.Docker) - } - if manifest.Agent.Docker.ConfigureDaemon != "${CONFIGURE_DAEMON}" { - t.Fatalf("original provider policy was mutated: %q", manifest.Agent.Docker.ConfigureDaemon) - } + require.Equal(t, types.StrBool(disabledDockerPolicy), resolved.Docker.ConfigureDaemon) + require.Equal(t, types.StrBool(disabledDockerPolicy), resolved.Docker.Install) + require.Equal(t, "${CONFIGURE_DAEMON}", string(manifest.Agent.Docker.ConfigureDaemon)) + data, err := json.Marshal(&provider.AgentWorkspaceInfo{Agent: resolved}) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(data), `"configureDaemon":"false"`) || - !strings.Contains(string(data), `"install":"false"`) { - t.Fatalf("agent workspace lost disabled Docker management policy: %s", data) - } + require.NoError(t, err) + require.Contains(t, string(data), `"configureDaemon":"false"`) + require.Contains(t, string(data), `"install":"false"`) + var decoded provider.AgentWorkspaceInfo - if err := json.Unmarshal(data, &decoded); err != nil { - t.Fatal(err) - } - if decoded.Agent.Docker.ConfigureDaemon != "false" || decoded.Agent.Docker.Install != "false" { - t.Fatalf("decoded agent workspace lost policy: %+v", decoded.Agent.Docker) - } + require.NoError(t, json.Unmarshal(data, &decoded)) + require.Equal(t, types.StrBool(disabledDockerPolicy), decoded.Agent.Docker.ConfigureDaemon) + require.Equal(t, types.StrBool(disabledDockerPolicy), decoded.Agent.Docker.Install) } diff --git a/pkg/provider/parse.go b/pkg/provider/parse.go index 09fd88a2d8..c2fd428667 100644 --- a/pkg/provider/parse.go +++ b/pkg/provider/parse.go @@ -348,15 +348,9 @@ func validateCustomDriver(config *ProviderConfig) error { } func validateExecCommands(config *ProviderConfig) error { - if config.Exec.Cancellation != ProviderCancellationDefault && - config.Exec.Cancellation != ProviderCancellationFile { - return fmt.Errorf("exec.cancellation must be empty or file") - } - - if len(config.Exec.Command) == 0 { - return fmt.Errorf("exec.command is required") + if err := validateExecCommandPolicy(config.Exec); err != nil { + return err } - if err := validateExecPair( "exec.create", "exec.delete", @@ -386,6 +380,17 @@ func validateExecCommands(config *ProviderConfig) error { return nil } +func validateExecCommandPolicy(exec ProviderCommands) error { + if exec.Cancellation != ProviderCancellationDefault && + exec.Cancellation != ProviderCancellationFile { + return fmt.Errorf("exec.cancellation must be empty or file") + } + if len(exec.Command) == 0 { + return fmt.Errorf("exec.command is required") + } + return nil +} + func validateExecPair(field1, field2 string, value1, value2 types.StrArray) error { if len(value1) > 0 && len(value2) == 0 { return fmt.Errorf("%s is required", field2) diff --git a/pkg/provider/parse_docker_test.go b/pkg/provider/parse_docker_test.go index c52ab62c0c..806c855b4c 100644 --- a/pkg/provider/parse_docker_test.go +++ b/pkg/provider/parse_docker_test.go @@ -5,6 +5,11 @@ import ( "testing" ) +const ( + enabledDockerDaemonPolicy = "true" + disabledDockerDaemonPolicy = "false" +) + func TestParseDockerConfigureDaemon(t *testing.T) { for _, tc := range []struct { name string @@ -12,8 +17,8 @@ func TestParseDockerConfigureDaemon(t *testing.T) { wantErr bool }{ {name: "unset"}, - {name: "true", value: "true"}, - {name: "false", value: "false"}, + {name: "enabled", value: enabledDockerDaemonPolicy}, + {name: "disabled", value: disabledDockerDaemonPolicy}, {name: "template", value: "${CONFIGURE_DAEMON}"}, {name: "bare template", value: "$CONFIGURE_DAEMON"}, {name: "invalid", value: "sometimes", wantErr: true}, diff --git a/pkg/shell/provider_exec.go b/pkg/shell/provider_exec.go index d8ac148b6a..23b2be7d4b 100644 --- a/pkg/shell/provider_exec.go +++ b/pkg/shell/provider_exec.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "os/exec" "runtime" "strings" @@ -19,45 +20,110 @@ func ProviderExecHandler() interp.ExecHandlerFunc { hc := interp.HandlerCtx(ctx) path, err := interp.LookPathDir(hc.Dir, hc.Env, args[0]) if err != nil { - fmt.Fprintln(hc.Stderr, err) + _, _ = fmt.Fprintln(hc.Stderr, err) return interp.ExitStatus(127) } - newCommand := func(path string, args []string) *exec.Cmd { - cmd := exec.CommandContext(ctx, path, args[1:]...) - cmd.Args = args - cmd.Env = providerExecEnv(hc.Env) - cmd.Dir = hc.Dir - cmd.Stdin = hc.Stdin - cmd.Stdout = hc.Stdout - cmd.Stderr = hc.Stderr - cmd.WaitDelay = 2 * time.Second - return cmd - } - - cmd := newCommand(path, args) - err = cmd.Run() + err = runOwnedProviderExec(ctx, hc, path, args) if errors.Is(err, syscall.ENOEXEC) && runtime.GOOS != "windows" { - cmd = newCommand("/bin/sh", append([]string{"/bin/sh", path}, args[1:]...)) - err = cmd.Run() + err = runOwnedProviderExec( + ctx, + hc, + "/bin/sh", + append([]string{"/bin/sh", path}, args[1:]...), + ) } if ctx.Err() != nil { return context.Cause(ctx) } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - if status, ok := providerExecSignalStatus(exitErr); ok { - return status - } - return interp.ExitStatus(exitErr.ExitCode()) + return providerExecResult(err, hc.Stderr) + } +} + +func providerExecResult(err error, stderr io.Writer) error { + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { + if status, ok := providerExecSignalStatus(exitErr); ok { + return status } - var execErr *exec.Error - if errors.As(err, &execErr) { - fmt.Fprintln(hc.Stderr, err) - return interp.ExitStatus(127) + code := exitErr.ExitCode() + if code < 0 || code > 255 { + return fmt.Errorf("provider process exit code outside shell range: %d", code) } + return interp.ExitStatus(uint8(code)) + } + if _, ok := errors.AsType[*exec.Error](err); ok { + _, _ = fmt.Fprintln(stderr, err) + return interp.ExitStatus(127) + } + return err +} + +func runOwnedProviderExec( + ctx context.Context, + hc interp.HandlerContext, + path string, + args []string, +) error { + cmd, err := startProviderExec(ctx, hc, path, args) + if err != nil { return err } + if ctx.Done() == nil { + return cmd.Wait() + } + done := make(chan struct{}) + monitorDone := make(chan struct{}) + go func() { + defer close(monitorDone) + select { + case <-ctx.Done(): + _ = cmd.Process.Kill() + case <-done: + } + }() + err = cmd.Wait() + close(done) + <-monitorDone + return err +} + +func startProviderExec( + ctx context.Context, + hc interp.HandlerContext, + path string, + args []string, +) (*exec.Cmd, error) { + environ := providerExecEnv(hc.Env) + newCommand := func() *exec.Cmd { + return &exec.Cmd{ + Path: path, + Args: args, + Env: environ, + Dir: hc.Dir, + Stdin: hc.Stdin, + Stdout: hc.Stdout, + Stderr: hc.Stderr, + WaitDelay: 2 * time.Second, + } + } + cmd := newCommand() + if ctx.Err() != nil { + return nil, context.Cause(ctx) + } + err := cmd.Start() + for delay := time.Millisecond; errors.Is(err, syscall.ETXTBSY) && delay < 300*time.Millisecond; delay *= 2 { + select { + case <-ctx.Done(): + return nil, context.Cause(ctx) + case <-time.After(delay): + } + cmd = newCommand() + err = cmd.Start() + } + if err != nil { + return nil, err + } + return cmd, nil } func providerExecEnv(env expand.Environ) []string { diff --git a/pkg/shell/provider_exec_test.go b/pkg/shell/provider_exec_test.go index b9ec9bb59e..dbdfa64cf0 100644 --- a/pkg/shell/provider_exec_test.go +++ b/pkg/shell/provider_exec_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "runtime" @@ -14,6 +15,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" "mvdan.cc/sh/v3/interp" ) @@ -22,9 +24,14 @@ func TestProviderExecHelper(t *testing.T) { return } if os.Getenv("DEVSY_PROVIDER_EXEC_EXIT") == "wait" { - fmt.Fprint(os.Stdout, "ready") + if _, err := fmt.Fprint(os.Stdout, "ready"); err != nil { + os.Exit(126) + } time.Sleep(time.Minute) } + if os.Getenv("DEVSY_PROVIDER_EXEC_EXIT") == "read-eof" { + providerExecReadToEOF() + } if os.Getenv("DEVSY_PROVIDER_EXEC_EXIT") != "" { os.Exit(125) } @@ -40,11 +47,23 @@ func TestProviderExecHelper(t *testing.T) { os.Exit(0) } -func TestProviderExecHandlerExportsUnsetAndWorkingDirectory(t *testing.T) { - binary, err := os.Executable() +func providerExecReadToEOF() { + if _, err := fmt.Fprint(os.Stdout, "before"); err != nil { + os.Exit(126) + } + input, err := io.ReadAll(os.Stdin) if err != nil { - t.Fatal(err) + os.Exit(126) } + if _, err := fmt.Fprintf(os.Stdout, "after%s", input); err != nil { + os.Exit(126) + } + os.Exit(0) +} + +func TestProviderExecHandlerExportsUnsetAndWorkingDirectory(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) workdir := t.TempDir() command := fmt.Sprintf( "export PRESENT='quoted value'; unset MISSING; cd %s; %s -test.run=^TestProviderExecHelper$ 'quoted argument'", @@ -60,27 +79,37 @@ func TestProviderExecHandlerExportsUnsetAndWorkingDirectory(t *testing.T) { Environ: append(os.Environ(), "DEVSY_PROVIDER_EXEC_HELPER=1", "MISSING=should-be-unset"), Stdout: &stdout, Stderr: &stderr, }) - if err != nil { - t.Fatalf("run provider shell: %v; stderr: %s", err, stderr.String()) - } + require.NoErrorf(t, err, "stderr: %s", stderr.String()) var got struct{ Directory, Present, Missing, Argument string } - if err := json.Unmarshal(stdout.Bytes(), &got); err != nil { - t.Fatal(err) - } + require.NoError(t, json.Unmarshal(stdout.Bytes(), &got)) expectedDir, err := filepath.EvalSymlinks(workdir) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) actualDir, err := filepath.EvalSymlinks(got.Directory) - if err != nil { - t.Fatal(err) - } - if !strings.EqualFold(filepath.Clean(actualDir), filepath.Clean(expectedDir)) { - t.Fatalf("working directory: got %q, want %q", got.Directory, workdir) - } - if got.Present != "quoted value" || got.Missing != "" || got.Argument != "quoted argument" { - t.Fatalf("shell exported variables or arguments changed: %+v", got) - } + require.NoError(t, err) + require.Truef(t, strings.EqualFold(filepath.Clean(actualDir), filepath.Clean(expectedDir)), + "working directory: got %q, want %q", got.Directory, workdir) + require.Equal(t, "quoted value", got.Present) + require.Empty(t, got.Missing) + require.Equal(t, "quoted argument", got.Argument) +} + +func TestProviderExecHandlerKeepsOutputAfterInputEOF(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + var stdout bytes.Buffer + err = RunEmulatedShell(context.Background(), &CommandRunner{ + Command: strconv.Quote(binary) + " -test.run=^TestProviderExecHelper$", + Environ: append( + os.Environ(), + "DEVSY_PROVIDER_EXEC_HELPER=1", + "DEVSY_PROVIDER_EXEC_EXIT=read-eof", + ), + ExecHandler: ProviderExecHandler(), + Stdin: strings.NewReader("payload"), + Stdout: &stdout, + }) + require.NoError(t, err) + require.Equal(t, "beforeafterpayload", stdout.String()) } func TestProviderExecHandlerExitStatuses(t *testing.T) { diff --git a/pkg/shell/provider_exec_unix.go b/pkg/shell/provider_exec_unix.go index 6fb0e981af..40ba1fc681 100644 --- a/pkg/shell/provider_exec_unix.go +++ b/pkg/shell/provider_exec_unix.go @@ -11,7 +11,11 @@ import ( func providerExecSignalStatus(err *exec.ExitError) (interp.ExitStatus, bool) { if status, ok := err.Sys().(syscall.WaitStatus); ok && status.Signaled() { - return interp.ExitStatus(128 + status.Signal()), true + code := 128 + int(status.Signal()) + if code < 0 || code > 255 { + return 0, false + } + return interp.ExitStatus(uint8(code)), true } return 0, false } From f4d4495bb2ca67ac1edbc0d26520a80ce2832007 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 2 Nov 2026 16:57:17 -0600 Subject: [PATCH 03/20] fix(provider): preserve builtin Docker installation behavior --- providers/docker/provider.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/docker/provider.yaml b/providers/docker/provider.yaml index 4e1636b4c5..c4267defa9 100644 --- a/providers/docker/provider.yaml +++ b/providers/docker/provider.yaml @@ -38,7 +38,7 @@ agent: path: ${DOCKER_PATH} elevation: ${DOCKER_ELEVATION} builder: ${DOCKER_BUILDER} - install: false + install: true env: DOCKER_HOST: ${DOCKER_HOST} DOCKER_CONTEXT: ${DOCKER_CONTEXT} From 733bc23f8ce72e122b454d72a7839f482913a0e3 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 2 Nov 2026 18:42:35 -0600 Subject: [PATCH 04/20] fix(provider): migrate saved builtin Docker install policy --- pkg/provider/dir.go | 7 +++-- pkg/provider/load_internal_test.go | 41 +++++++++++++++++++++++++----- pkg/provider/parse_docker_test.go | 8 +++--- pkg/provider/parse_driver_test.go | 2 +- 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/pkg/provider/dir.go b/pkg/provider/dir.go index 7dd8de4dcf..ce2c39b6a7 100644 --- a/pkg/provider/dir.go +++ b/pkg/provider/dir.go @@ -324,14 +324,13 @@ func LoadProviderConfig(context, provider string) (*ProviderConfig, error) { return nil, err } - // Refresh the exec commands of built-in providers from embedded YAML: a CLI - // change (e.g. renaming the command wrapper) must not be shadowed by a stale - // provider.json. Only the exec blocks are overlaid so user customizations - // (custom --name, resolved options) survive. if providerConfig.Source.Internal { if embedded := embeddedProviderConfig(providerConfig.Source.Raw); embedded != nil { providerConfig.Exec = embedded.Exec providerConfig.Agent.Exec = embedded.Agent.Exec + if providerConfig.Source.Raw == DockerDriver { + providerConfig.Agent.Docker.Install = embedded.Agent.Docker.Install + } } } diff --git a/pkg/provider/load_internal_test.go b/pkg/provider/load_internal_test.go index 5cc0053f74..9bb7fbf6be 100644 --- a/pkg/provider/load_internal_test.go +++ b/pkg/provider/load_internal_test.go @@ -13,7 +13,10 @@ import ( // stale provider.json files carry before a refresh rewrites them. const staleHelperShCommand = `"${DEVSY}" helper sh -c "${COMMAND}"` -const staleVersion = "v0.0.0-stale" +const ( + staleVersion = "v0.0.0-stale" + podmanProviderName = "podman" +) // TestLoadProviderConfig_RefreshesInternalProvider verifies that a stored // built-in provider config with a stale exec.command (e.g. the removed @@ -26,6 +29,9 @@ func TestLoadProviderConfig_RefreshesInternalProvider(t *testing.T) { Name: DockerDriver, Version: staleVersion, Source: ProviderSource{Internal: true, Raw: DockerDriver}, + Agent: ProviderAgentConfig{ + Docker: ProviderDockerDriverConfig{Install: disabledDockerPolicy}, + }, Exec: ProviderCommands{ Command: []string{staleHelperShCommand}, }, @@ -39,11 +45,9 @@ func TestLoadProviderConfig_RefreshesInternalProvider(t *testing.T) { "internal provider must be refreshed from embedded yaml, not the stale stored config") require.True(t, strings.Contains(loaded.Exec.Command[0], "internal sh"), "refreshed command should use the current 'internal sh' wrapper") + require.Equal(t, types.StrBool(enabledDockerPolicy), loaded.Agent.Docker.Install) } -// TestLoadProviderConfig_PreservesCustomizations verifies the exec refresh -// overlays only the embedded exec block, leaving a user's custom Name and -// resolved Options intact (a built-in added with `--name`/`--option`). func TestLoadProviderConfig_PreservesCustomizations(t *testing.T) { setupTestHome(t) @@ -51,8 +55,11 @@ func TestLoadProviderConfig_PreservesCustomizations(t *testing.T) { Name: "my-docker", Version: staleVersion, Source: ProviderSource{Internal: true, Raw: DockerDriver}, + Agent: ProviderAgentConfig{ + Docker: ProviderDockerDriverConfig{Install: disabledDockerPolicy}, + }, Options: map[string]*types.Option{ - "DOCKER_PATH": {Default: "podman"}, + "DOCKER_PATH": {Default: podmanProviderName}, }, Exec: ProviderCommands{ Command: []string{staleHelperShCommand}, @@ -64,10 +71,11 @@ func TestLoadProviderConfig_PreservesCustomizations(t *testing.T) { require.NoError(t, err) require.Equal(t, "my-docker", loaded.Name, "custom provider name must survive refresh") require.Equal(t, "v0.0.0-stale", loaded.Version, "stored version must survive refresh") - require.Equal(t, "podman", loaded.Options["DOCKER_PATH"].Default, + require.Equal(t, podmanProviderName, loaded.Options["DOCKER_PATH"].Default, "resolved options must survive refresh") require.NotContains(t, loaded.Exec.Command[0], "helper sh", "exec block must still be refreshed from embedded yaml") + require.Equal(t, types.StrBool(enabledDockerPolicy), loaded.Agent.Docker.Install) } // TestLoadProviderConfig_RefreshesProBySourceID guards the case where a @@ -123,6 +131,9 @@ func TestLoadProviderConfig_PreservesExternalProvider(t *testing.T) { Name: DockerDriver, Version: "v0.0.1", Source: ProviderSource{Github: "some-org/some-provider"}, + Agent: ProviderAgentConfig{ + Docker: ProviderDockerDriverConfig{Install: disabledDockerPolicy}, + }, Exec: ProviderCommands{ Command: []string{staleHelperShCommand}, }, @@ -132,4 +143,22 @@ func TestLoadProviderConfig_PreservesExternalProvider(t *testing.T) { loaded, err := LoadProviderConfig(config.DefaultContext, DockerDriver) require.NoError(t, err) require.Equal(t, external.Exec.Command, loaded.Exec.Command) + require.Equal(t, types.StrBool(disabledDockerPolicy), loaded.Agent.Docker.Install) +} + +func TestLoadProviderConfig_PreservesPodmanInstallProhibition(t *testing.T) { + setupTestHome(t) + stored := &ProviderConfig{ + Name: podmanProviderName, + Source: ProviderSource{Internal: true, Raw: podmanProviderName}, + Agent: ProviderAgentConfig{ + Docker: ProviderDockerDriverConfig{Install: disabledDockerPolicy}, + }, + Exec: ProviderCommands{Command: []string{staleHelperShCommand}}, + } + require.NoError(t, SaveProviderConfig(config.DefaultContext, stored)) + + loaded, err := LoadProviderConfig(config.DefaultContext, stored.Name) + require.NoError(t, err) + require.Equal(t, types.StrBool(disabledDockerPolicy), loaded.Agent.Docker.Install) } diff --git a/pkg/provider/parse_docker_test.go b/pkg/provider/parse_docker_test.go index 806c855b4c..e880449da4 100644 --- a/pkg/provider/parse_docker_test.go +++ b/pkg/provider/parse_docker_test.go @@ -6,8 +6,8 @@ import ( ) const ( - enabledDockerDaemonPolicy = "true" - disabledDockerDaemonPolicy = "false" + enabledDockerPolicy = "true" + disabledDockerPolicy = "false" ) func TestParseDockerConfigureDaemon(t *testing.T) { @@ -17,8 +17,8 @@ func TestParseDockerConfigureDaemon(t *testing.T) { wantErr bool }{ {name: "unset"}, - {name: "enabled", value: enabledDockerDaemonPolicy}, - {name: "disabled", value: disabledDockerDaemonPolicy}, + {name: "enabled", value: enabledDockerPolicy}, + {name: "disabled", value: disabledDockerPolicy}, {name: "template", value: "${CONFIGURE_DAEMON}"}, {name: "bare template", value: "$CONFIGURE_DAEMON"}, {name: "invalid", value: "sometimes", wantErr: true}, diff --git a/pkg/provider/parse_driver_test.go b/pkg/provider/parse_driver_test.go index 3d11f33320..0d49b89404 100644 --- a/pkg/provider/parse_driver_test.go +++ b/pkg/provider/parse_driver_test.go @@ -27,7 +27,7 @@ exec: {name: "kubernetes literal", driver: KubernetesDriver, wantErr: false}, {name: "templated braces", driver: "${AWS_DEPLOYMENT_MODE}", wantErr: false}, {name: "templated bare", driver: "$AWS_DEPLOYMENT_MODE", wantErr: false}, - {name: "invalid literal", driver: "podman", wantErr: true}, + {name: "invalid literal", driver: podmanProviderName, wantErr: true}, } for _, tc := range cases { From 8768ba2c266d19813761caf2fadd3326b5ebdfb1 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 2 Nov 2026 20:33:52 -0600 Subject: [PATCH 05/20] fix(provider): validate policies and inherit file-mode environment --- .../clientimplementation/provider_cancel.go | 3 ++ .../provider_cancel_test.go | 18 ++++++++ pkg/provider/parse.go | 42 +++++++++++-------- pkg/provider/parse_docker_test.go | 35 +++++++++++++++- pkg/provider/parse_driver_test.go | 22 +++++++++- 5 files changed, 100 insertions(+), 20 deletions(-) diff --git a/pkg/client/clientimplementation/provider_cancel.go b/pkg/client/clientimplementation/provider_cancel.go index d3aa6c176e..3bf43d0817 100644 --- a/pkg/client/clientimplementation/provider_cancel.go +++ b/pkg/client/clientimplementation/provider_cancel.go @@ -64,6 +64,9 @@ func runProviderWithCancellationUsingPublisher( } func providerCancellationEnvironment(original []string, marker string) []string { + if original == nil { + original = os.Environ() + } environ := make([]string, 0, len(original)+1) for _, kv := range original { key, _, ok := strings.Cut(kv, "=") diff --git a/pkg/client/clientimplementation/provider_cancel_test.go b/pkg/client/clientimplementation/provider_cancel_test.go index d5a74a5bbd..c403576d2f 100644 --- a/pkg/client/clientimplementation/provider_cancel_test.go +++ b/pkg/client/clientimplementation/provider_cancel_test.go @@ -63,6 +63,9 @@ func runTestCancellationExit(behavior, marker string) { os.Exit(127) case "bytes": writeTestCancellationBytes() + case "inherited": + _, err := fmt.Fprint(os.Stdout, os.Getenv("DEVSY_CANCEL_INHERITED")) + exitTestHelperOnError(err) case "environment": _, err := fmt.Fprint(os.Stdout, marker) exitTestHelperOnError(err) @@ -334,6 +337,21 @@ func TestProviderCancellationPrivateEnvironment(t *testing.T) { require.ErrorIs(t, err, os.ErrNotExist) } +func TestProviderCancellationNilEnvironmentInheritsHost(t *testing.T) { + t.Setenv("DEVSY_CANCEL_HELPER", "1") + t.Setenv("DEVSY_CANCEL_BEHAVIOR", "inherited") + t.Setenv("DEVSY_CANCEL_INHERITED", "present") + binary, err := os.Executable() + require.NoError(t, err) + var stdout bytes.Buffer + require.NoError(t, RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{binary, "-test.run=^TestProviderCancellationHelper$"}, + Cancellation: provider.ProviderCancellationFile, + Stdout: &stdout, + })) + require.Equal(t, "present", stdout.String()) +} + func TestProviderCancellationTypedExits(t *testing.T) { for _, status := range []int{7, 125, 127} { opts := cancelHelperOptions("status") diff --git a/pkg/provider/parse.go b/pkg/provider/parse.go index c2fd428667..923102a01d 100644 --- a/pkg/provider/parse.go +++ b/pkg/provider/parse.go @@ -205,6 +205,10 @@ func validateOptionCacheAndType(optionName string, optionValue *types.Option) er } func validateProviderType(config *ProviderConfig) error { + if config.Exec.Cancellation != ProviderCancellationDefault && + config.Exec.Cancellation != ProviderCancellationFile { + return fmt.Errorf("exec.cancellation must be empty or file") + } if config.IsProxyProvider() { return validateProxyProvider(config) } @@ -281,11 +285,8 @@ func validateStandardProvider(config *ProviderConfig) error { if err := validateAgentDriver(config); err != nil { return err } - if value := config.Agent.Docker.ConfigureDaemon; value != "" && - !templatedValueRegex.MatchString(string(value)) { - if _, err := value.Bool(); err != nil { - return fmt.Errorf("agent.docker.configureDaemon: %w", err) - } + if err := validateAgentDockerPolicies(config.Agent.Docker); err != nil { + return err } if err := validateBinaries("agent.binaries", config.Agent.Binaries); err != nil { @@ -295,6 +296,22 @@ func validateStandardProvider(config *ProviderConfig) error { return validateExecCommands(config) } +func validateAgentDockerPolicies(docker ProviderDockerDriverConfig) error { + if value := docker.ConfigureDaemon; value != "" && + !templatedValueRegex.MatchString(string(value)) { + if _, err := value.Bool(); err != nil { + return fmt.Errorf("agent.docker.configureDaemon: %w", err) + } + } + if value := docker.Install; value != "" && + !templatedValueRegex.MatchString(string(value)) { + if _, err := value.Bool(); err != nil { + return fmt.Errorf("agent.docker.install: %w", err) + } + } + return nil +} + var validAgentDrivers = map[string]bool{ "": true, CustomDriver: true, @@ -348,8 +365,8 @@ func validateCustomDriver(config *ProviderConfig) error { } func validateExecCommands(config *ProviderConfig) error { - if err := validateExecCommandPolicy(config.Exec); err != nil { - return err + if len(config.Exec.Command) == 0 { + return fmt.Errorf("exec.command is required") } if err := validateExecPair( "exec.create", @@ -380,17 +397,6 @@ func validateExecCommands(config *ProviderConfig) error { return nil } -func validateExecCommandPolicy(exec ProviderCommands) error { - if exec.Cancellation != ProviderCancellationDefault && - exec.Cancellation != ProviderCancellationFile { - return fmt.Errorf("exec.cancellation must be empty or file") - } - if len(exec.Command) == 0 { - return fmt.Errorf("exec.command is required") - } - return nil -} - func validateExecPair(field1, field2 string, value1, value2 types.StrArray) error { if len(value1) > 0 && len(value2) == 0 { return fmt.Errorf("%s is required", field2) diff --git a/pkg/provider/parse_docker_test.go b/pkg/provider/parse_docker_test.go index e880449da4..bcb0d87b67 100644 --- a/pkg/provider/parse_docker_test.go +++ b/pkg/provider/parse_docker_test.go @@ -10,6 +10,8 @@ const ( disabledDockerPolicy = "false" ) +const templatePolicyCase = "template" + func TestParseDockerConfigureDaemon(t *testing.T) { for _, tc := range []struct { name string @@ -19,7 +21,7 @@ func TestParseDockerConfigureDaemon(t *testing.T) { {name: "unset"}, {name: "enabled", value: enabledDockerPolicy}, {name: "disabled", value: disabledDockerPolicy}, - {name: "template", value: "${CONFIGURE_DAEMON}"}, + {name: templatePolicyCase, value: "${CONFIGURE_DAEMON}"}, {name: "bare template", value: "$CONFIGURE_DAEMON"}, {name: "invalid", value: "sometimes", wantErr: true}, } { @@ -49,3 +51,34 @@ func TestParseDockerConfigureDaemon(t *testing.T) { }) } } + +func TestParseDockerInstallPolicy(t *testing.T) { + for _, tc := range []struct { + name string + value string + wantErr bool + }{ + {name: "disabled", value: disabledDockerPolicy}, + {name: "enabled", value: enabledDockerPolicy}, + {name: templatePolicyCase, value: "${INSTALL_DOCKER}"}, + {name: "invalid", value: "sometimes", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + manifest := "name: docker-policy\nagent:\n driver: docker\n docker:\n install: " + + tc.value + "\nexec:\n command: echo hi\n" + parsed, err := ParseProvider(strings.NewReader(manifest)) + if tc.wantErr { + if err == nil || !strings.Contains(err.Error(), "agent.docker.install") { + t.Fatalf("ParseProvider() error = %v, want Docker install policy error", err) + } + return + } + if err != nil { + t.Fatal(err) + } + if string(parsed.Agent.Docker.Install) != tc.value { + t.Errorf("Install = %q, want %q", parsed.Agent.Docker.Install, tc.value) + } + }) + } +} diff --git a/pkg/provider/parse_driver_test.go b/pkg/provider/parse_driver_test.go index 0d49b89404..2ed08608a3 100644 --- a/pkg/provider/parse_driver_test.go +++ b/pkg/provider/parse_driver_test.go @@ -52,7 +52,7 @@ func TestProviderCancellationMode(t *testing.T) { {name: "default", valid: true}, {name: string(ProviderCancellationFile), value: string(ProviderCancellationFile), valid: true}, {name: "unknown", value: "signal"}, - {name: "template", value: "${CANCELLATION}"}, + {name: templatePolicyCase, value: "${CANCELLATION}"}, } { t.Run(tc.name, func(t *testing.T) { manifest := "name: example\nversion: v1.0.0\nexec:\n command: echo ok\n" @@ -75,3 +75,23 @@ func TestProviderCancellationMode(t *testing.T) { }) } } + +func TestProviderCancellationModeAcrossProviderKinds(t *testing.T) { + for _, tc := range []struct { + name string + commands string + }{ + {name: "proxy", commands: " proxy:\n" + + " up: echo up\n status: echo status\n stop: echo stop\n" + + " delete: echo delete\n ssh: echo ssh\n"}, + {name: "daemon", commands: " daemon:\n start: echo start\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + manifest := "name: example\nexec:\n cancellation: files\n" + tc.commands + _, err := ParseProvider(strings.NewReader(manifest)) + if err == nil || !strings.Contains(err.Error(), "exec.cancellation") { + t.Fatalf("ParseProvider() error = %v, want invalid cancellation mode", err) + } + }) + } +} From 5989ce624ab9dea507f72fba0fe850bdcb7c12fa Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 2 Nov 2026 22:46:54 -0600 Subject: [PATCH 06/20] fix(provider): supervise file cancellation for shell commands --- cmd/internal/internal.go | 1 + cmd/internal/provider_shell.go | 35 + cmd/internal/provider_shell_test.go | 51 ++ cmd/internal/provider_shell_unix_test.go | 17 + cmd/internal/provider_shell_windows_test.go | 28 + .../clientimplementation/machine_client.go | 38 +- ...machine_proxy_transport_completion_test.go | 93 +++ .../clientimplementation/provider_cancel.go | 224 +++++- .../provider_cancel_test.go | 641 +++++++++++++++++- .../clientimplementation/proxy_client.go | 28 +- .../clientimplementation/workspace_client.go | 460 ++++++++++++- .../workspace_client_test.go | 2 +- pkg/shell/provider_exec.go | 67 +- pkg/shell/provider_exec_test.go | 6 +- pkg/shell/provider_exec_unix.go | 9 + pkg/shell/provider_exec_windows.go | 34 + pkg/shell/provider_supervisor.go | 397 +++++++++++ pkg/shell/provider_supervisor_test.go | 484 +++++++++++++ pkg/shell/provider_supervisor_unix_test.go | 41 ++ pkg/shell/provider_supervisor_windows_test.go | 41 ++ .../docs/developing-providers/quickstart.mdx | 4 +- 21 files changed, 2587 insertions(+), 114 deletions(-) create mode 100644 cmd/internal/provider_shell.go create mode 100644 cmd/internal/provider_shell_test.go create mode 100644 cmd/internal/provider_shell_unix_test.go create mode 100644 cmd/internal/provider_shell_windows_test.go create mode 100644 pkg/client/clientimplementation/machine_proxy_transport_completion_test.go create mode 100644 pkg/shell/provider_supervisor.go create mode 100644 pkg/shell/provider_supervisor_test.go create mode 100644 pkg/shell/provider_supervisor_unix_test.go create mode 100644 pkg/shell/provider_supervisor_windows_test.go diff --git a/cmd/internal/internal.go b/cmd/internal/internal.go index 5d011c36cc..c2d946b19a 100644 --- a/cmd/internal/internal.go +++ b/cmd/internal/internal.go @@ -16,6 +16,7 @@ func NewInternalCmd(globalFlags *flags.GlobalFlags) *cobra.Command { } cmd.AddCommand(NewAgentCmd(globalFlags)) cmd.AddCommand(NewRuntimeSupervisorCmd()) + cmd.AddCommand(NewProviderShellCmd()) cmd.AddCommand(NewGHCmd()) cmd.AddCommand(NewGHBackendCmd()) cmd.AddCommand(NewDaemonLocalCmd(globalFlags)) diff --git a/cmd/internal/provider_shell.go b/cmd/internal/provider_shell.go new file mode 100644 index 0000000000..22d301f02f --- /dev/null +++ b/cmd/internal/provider_shell.go @@ -0,0 +1,35 @@ +package cmdinternal + +import ( + "context" + "errors" + "os" + + "github.com/devsy-org/devsy/pkg/shell" + "github.com/spf13/cobra" + "mvdan.cc/sh/v3/interp" +) + +func NewProviderShellCmd() *cobra.Command { + var scriptPath, forcePath, ackPath string + command := &cobra.Command{ + Use: "provider-shell", Hidden: true, Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + err := shell.RunProviderShell(cmd.Context(), scriptPath, forcePath, ackPath) + if status, ok := errors.AsType[interp.ExitStatus](err); ok { + os.Exit(int(status)) + } + if errors.Is(err, context.Canceled) { + os.Exit(1) + } + return err + }, + } + command.Flags().StringVar(&scriptPath, "script-file", "", "Private provider script file") + command.Flags().StringVar(&forcePath, "force-file", "", "Private provider force marker") + command.Flags().StringVar(&ackPath, "ack-file", "", "Private provider acknowledgement marker") + _ = command.MarkFlagRequired("script-file") + _ = command.MarkFlagRequired("force-file") + _ = command.MarkFlagRequired("ack-file") + return command +} diff --git a/cmd/internal/provider_shell_test.go b/cmd/internal/provider_shell_test.go new file mode 100644 index 0000000000..29f2803d78 --- /dev/null +++ b/cmd/internal/provider_shell_test.go @@ -0,0 +1,51 @@ +package cmdinternal + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestProviderShellCommandRegistration(t *testing.T) { + command := NewProviderShellCmd() + require.Equal(t, "provider-shell", command.Name()) + require.True(t, command.Hidden) + dir := providerShellCLITestDir(t) + script := filepath.Join(dir, "script") + require.NoError(t, os.WriteFile(script, []byte("exit 0"), 0o600)) + token := strings.Repeat("0123456789abcdef", 4) + require.NoError(t, os.WriteFile(filepath.Join(dir, "ack-key"), []byte(token), 0o600)) + command.SetArgs([]string{ + "--script-file", script, "--force-file", filepath.Join(dir, "force"), + "--ack-file", filepath.Join(dir, "ack"), + }) + require.NoError(t, command.Execute()) + _, err := os.Lstat(filepath.Join(dir, "ack-key")) + require.ErrorIs(t, err, os.ErrNotExist) + root, err := os.OpenRoot(dir) + require.NoError(t, err) + ack, readErr := root.ReadFile("ack") + closeErr := root.Close() + require.NoError(t, readErr) + require.NoError(t, closeErr) + require.Equal(t, token, string(ack)) +} + +func TestProviderShellCommandRejectsMissingToken(t *testing.T) { + dir := providerShellCLITestDir(t) + script := filepath.Join(dir, "script") + require.NoError(t, os.WriteFile(script, []byte("printf should-not-run"), 0o600)) + _, err := os.Lstat(filepath.Join(dir, "ack-key")) + require.ErrorIs(t, err, os.ErrNotExist) + command := NewProviderShellCmd() + command.SetArgs([]string{ + "--script-file", script, "--force-file", filepath.Join(dir, "force"), + "--ack-file", filepath.Join(dir, "ack"), + }) + require.ErrorContains(t, command.Execute(), "invalid provider shell acknowledgement token") + _, err = os.Lstat(filepath.Join(dir, "ack")) + require.ErrorIs(t, err, os.ErrNotExist) +} diff --git a/cmd/internal/provider_shell_unix_test.go b/cmd/internal/provider_shell_unix_test.go new file mode 100644 index 0000000000..0891f56b7d --- /dev/null +++ b/cmd/internal/provider_shell_unix_test.go @@ -0,0 +1,17 @@ +//go:build !windows + +package cmdinternal + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func providerShellCLITestDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp(t.TempDir(), "private-") + require.NoError(t, err) + return dir +} diff --git a/cmd/internal/provider_shell_windows_test.go b/cmd/internal/provider_shell_windows_test.go new file mode 100644 index 0000000000..8e8a30a391 --- /dev/null +++ b/cmd/internal/provider_shell_windows_test.go @@ -0,0 +1,28 @@ +package cmdinternal + +import ( + "path/filepath" + "testing" + "unsafe" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" +) + +func providerShellCLITestDir(t *testing.T) string { + t.Helper() + user, err := windows.GetCurrentProcessToken().GetTokenUser() + require.NoError(t, err) + sd, err := windows.SecurityDescriptorFromString( + "D:P(A;OICI;GA;;;" + user.User.Sid.String() + ")(A;OICI;GA;;;SY)") + require.NoError(t, err) + attributes := windows.SecurityAttributes{ + Length: uint32(unsafe.Sizeof(windows.SecurityAttributes{})), + SecurityDescriptor: sd, + } + path := filepath.Join(t.TempDir(), "private") + name, err := windows.UTF16PtrFromString(path) + require.NoError(t, err) + require.NoError(t, windows.CreateDirectory(name, &attributes)) + return path +} diff --git a/pkg/client/clientimplementation/machine_client.go b/pkg/client/clientimplementation/machine_client.go index a60b83e29d..f565b7a5ef 100644 --- a/pkg/client/clientimplementation/machine_client.go +++ b/pkg/client/clientimplementation/machine_client.go @@ -55,16 +55,17 @@ type machineExecutor struct { // execConfig defines how to execute a machine command. type execConfig struct { - name string - command types.StrArray - stdout io.Writer - stderr io.Writer - extraEnv map[string]string - stdin io.Reader - rawStdout bool - withProgress bool - startMsg string - doneMsg string + name string + command types.StrArray + stdout io.Writer + stderr io.Writer + extraEnv map[string]string + stdin io.Reader + closeInputOnExit io.Closer + rawStdout bool + withProgress bool + startMsg string + doneMsg string } func (e *machineExecutor) execute(ctx context.Context, cfg execConfig) error { @@ -88,6 +89,7 @@ func (e *machineExecutor) execute(ctx context.Context, cfg execConfig) error { Stderr: cfg.stderr, ExtraEnv: cfg.extraEnv, Stdin: cfg.stdin, + closeInputOnExit: cfg.closeInputOnExit, RawStdout: cfg.rawStdout, }) if err != nil { @@ -222,7 +224,7 @@ func (s *machineClient) Stop(ctx context.Context, options client.StopOptions) er func (s *machineClient) Command(ctx context.Context, commandOptions client.CommandOptions) error { return s.executor.execute(ctx, execConfig{ - name: "command", + name: providerCommandAction, command: s.config.Exec.Command, stdout: commandOptions.Stdout, stderr: commandOptions.Stderr, @@ -241,9 +243,17 @@ func (s *machineClient) OpenCommandTransport( return transport.OpenCallbackConn( ctx, func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { - return s.Command(ctx, client.CommandOptions{ - Command: opt.Command, Stdin: stdin, Stdout: stdout, Stderr: opt.Stderr, - RawStdout: true, + return s.executor.execute(ctx, execConfig{ + name: providerCommandAction, + command: s.config.Exec.Command, + stdout: stdout, + stderr: opt.Stderr, + stdin: stdin, + closeInputOnExit: stdin.(io.Closer), + rawStdout: true, + extraEnv: map[string]string{ + provider.CommandEnv: opt.Command, + }, }) }, transport.CallbackConnOptions{ diff --git a/pkg/client/clientimplementation/machine_proxy_transport_completion_test.go b/pkg/client/clientimplementation/machine_proxy_transport_completion_test.go new file mode 100644 index 0000000000..b2d4a12054 --- /dev/null +++ b/pkg/client/clientimplementation/machine_proxy_transport_completion_test.go @@ -0,0 +1,93 @@ +package clientimplementation + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/client" + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/transport" + "github.com/devsy-org/devsy/pkg/types" + "github.com/stretchr/testify/require" +) + +const testProviderCreateCommand = "create" + +func TestMachineCommandTransportCompletesWithoutClosingInput(t *testing.T) { + payload := strings.Repeat("m", 128*1024) + const stderrTail = "machine stderr tail" + command := "printf '%s' '" + payload + "'; printf '%s' '" + stderrTail + "' >&2" + cfg := &provider.ProviderConfig{ + Name: testCancelProvider, + Exec: provider.ProviderCommands{ + Create: types.StrArray{testProviderCreateCommand}, + Command: types.StrArray{command}, + Cancellation: provider.ProviderCancellationFile, + }, + } + devsy := &config.Config{ + DefaultContext: testLocksContext, + Contexts: map[string]*config.ContextConfig{testLocksContext: {}}, + } + machine, err := NewMachineClient( + devsy, + cfg, + &provider.Machine{ID: "completion-machine", Context: testLocksContext}, + ) + require.NoError(t, err) + var stderr bytes.Buffer + conn, err := machine.OpenCommandTransport(context.Background(), client.CommandTransportOptions{ + Command: testOpaqueCommand, + Stderr: &stderr, + }) + require.NoError(t, err) + requireTransportCompletionWithOpenInput(t, conn, payload) + require.Equal(t, stderrTail, stderr.String()) +} + +func TestProxySSHTransportCompletesWithoutClosingInput(t *testing.T) { + payload := strings.Repeat("p", 128*1024) + workspace := workspaceCancellationTestClient() + workspace.providerConfig.Exec.Proxy = &provider.ProxyCommands{ + Ssh: types.StrArray{"printf '%s' '" + payload + "'"}, + } + proxy, err := NewProxyClient( + workspace.devsyConfig, + workspace.providerConfig, + workspace.providerWorkspace, + ) + require.NoError(t, err) + conn, err := proxy.OpenSSHTransport(context.Background(), client.SSHTransportOptions{}) + require.NoError(t, err) + requireTransportCompletionWithOpenInput(t, conn, payload) +} + +func requireTransportCompletionWithOpenInput( + t *testing.T, + conn transport.ManagedConn, + payload string, +) { + t.Helper() + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + got := make([]byte, len(payload)) + _, err := io.ReadFull(conn, got) + require.NoError(t, err) + require.Equal(t, payload, string(got)) + finished := make(chan error, 1) + go func() { finished <- conn.Wait() }() + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("provider completion waited for caller to close transport input") + } + var trailing [1]byte + _, err = conn.Read(trailing[:]) + require.ErrorIs(t, err, io.EOF) +} diff --git a/pkg/client/clientimplementation/provider_cancel.go b/pkg/client/clientimplementation/provider_cancel.go index 3bf43d0817..d5e5ed7a60 100644 --- a/pkg/client/clientimplementation/provider_cancel.go +++ b/pkg/client/clientimplementation/provider_cancel.go @@ -2,8 +2,12 @@ package clientimplementation import ( "context" + "crypto/rand" + "crypto/subtle" + "encoding/hex" "errors" "fmt" + "io" "os" "path/filepath" "runtime" @@ -13,7 +17,10 @@ import ( "github.com/devsy-org/devsy/pkg/config" ) -const providerCancellationGrace = 10 * time.Second +const ( + providerCancellationGrace = 10 * time.Second + providerForceAckWait = 5 * time.Second +) func runProviderWithCancellation(parent context.Context, opts RunCommandOptions) error { return runProviderWithCancellationUsingPublisher(parent, opts, publishProviderCancellation) @@ -45,6 +52,12 @@ func runProviderWithCancellationUsingPublisher( }() marker := filepath.Join(dir, "cancel") + if len(opts.Command) == 1 { + opts.providerShell, err = prepareProviderShell(dir, opts.Command[0]) + if err != nil { + return err + } + } opts.Environ = providerCancellationEnvironment(opts.Environ, marker) execution, cancel := context.WithCancelCause(context.WithoutCancel(parent)) defer cancel(context.Canceled) @@ -59,10 +72,30 @@ func runProviderWithCancellationUsingPublisher( parent, marker, publish, - providerExecution{cancel: cancel, completed: completed}, + providerExecution{cancel: cancel, completed: completed, shell: opts.providerShell}, ) } +func prepareProviderShell(dir, script string) (*providerShellInvocation, error) { + var token [32]byte + if _, err := rand.Read(token[:]); err != nil { + return nil, fmt.Errorf("generate provider shell acknowledgement token: %w", err) + } + files := &providerShellInvocation{ + script: filepath.Join(dir, "script"), + force: filepath.Join(dir, "force"), + ack: filepath.Join(dir, "ack"), + ackToken: hex.EncodeToString(token[:]), + } + if err := writeProviderScript(dir, script); err != nil { + return nil, fmt.Errorf("write provider script: %w", err) + } + if err := writeProviderAckKey(dir, files.ackToken); err != nil { + return nil, fmt.Errorf("write provider acknowledgement key: %w", err) + } + return files, nil +} + func providerCancellationEnvironment(original []string, marker string) []string { if original == nil { original = os.Environ() @@ -78,9 +111,17 @@ func providerCancellationEnvironment(original []string, marker string) []string return append(environ, config.EnvProviderCancelFile+"="+marker) } +type providerShellInvocation struct { + script string + force string + ack string + ackToken string +} + type providerExecution struct { cancel context.CancelCauseFunc completed <-chan error + shell *providerShellInvocation } func waitForProviderCancellation( @@ -92,7 +133,10 @@ func waitForProviderCancellation( select { case result := <-process.completed: if parent.Err() != nil { - return context.Cause(parent) + return providerCancellationCompletion(context.Cause(parent), process) + } + if process.shell != nil { + return errors.Join(result, providerForceCompletionStatus(process)) } return result case <-parent.Done(): @@ -100,11 +144,10 @@ func waitForProviderCancellation( cause := context.Cause(parent) if err := publish(marker); err != nil { - process.cancel(cause) - <-process.completed return errors.Join( cause, fmt.Errorf("publish provider cancellation marker: %w", sanitizeMarkerError(err)), + forceProviderExecution(cause, process), ) } return waitForProviderGrace(cause, process) @@ -115,12 +158,166 @@ func waitForProviderGrace(cause error, process providerExecution) error { defer timer.Stop() select { case <-process.completed: - return cause + return providerCancellationCompletion(cause, process) case <-timer.C: - process.cancel(cause) - <-process.completed + } + return errors.Join(cause, forceProviderExecution(cause, process)) +} + +func providerCancellationCompletion(cause error, process providerExecution) error { + if process.shell == nil { return cause } + return errors.Join(cause, providerForceCompletionStatus(process)) +} + +func forceProviderExecution(cause error, process providerExecution) error { + if process.shell == nil { + stopProviderExecution(cause, process) + return nil + } + select { + case <-process.completed: + return providerForceCompletionStatus(process) + default: + } + if err := publishProviderMarker(process.shell.force); err != nil { + stopProviderExecution(cause, process) + return fmt.Errorf("publish provider force marker: %w", err) + } + return waitForProviderForceAck(cause, process) +} + +func waitForProviderForceAck(cause error, process providerExecution) error { + timer := time.NewTimer(providerForceAckWait) + defer timer.Stop() + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-process.completed: + return providerForceCompletionStatus(process) + case <-ticker.C: + acknowledged, err := providerForceAcknowledged(process.shell) + if err != nil { + stopProviderExecution(cause, process) + return fmt.Errorf("inspect provider force acknowledgement: %w", err) + } + if acknowledged { + stopProviderExecution(cause, process) + return nil + } + case <-timer.C: + return finishProviderForceAtDeadline(cause, process) + } + } +} + +func providerForceCompletionStatus(process providerExecution) error { + acknowledged, err := providerForceAcknowledged(process.shell) + if err != nil { + return fmt.Errorf("inspect provider force acknowledgement: %w", err) + } + if !acknowledged { + return errors.New("provider helper exited without authentic acknowledgement") + } + return nil +} + +func finishProviderForceAtDeadline(cause error, process providerExecution) error { + acknowledged, err := providerForceAcknowledged(process.shell) + stopProviderExecution(cause, process) + if err != nil { + return fmt.Errorf("inspect provider force acknowledgement: %w", err) + } + if !acknowledged { + return errors.New("provider force acknowledgement timed out") + } + return nil +} + +func stopProviderExecution(cause error, process providerExecution) { + process.cancel(cause) + <-process.completed +} + +func writeProviderScript(dir, script string) (retErr error) { + root, err := os.OpenRoot(dir) + if err != nil { + return sanitizeMarkerError(err) + } + defer func() { retErr = errors.Join(retErr, sanitizeMarkerError(root.Close())) }() + f, err := root.OpenFile("script", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return sanitizeMarkerError(err) + } + _, writeErr := f.WriteString(script) + return errors.Join(sanitizeMarkerError(writeErr), sanitizeMarkerError(f.Close())) +} + +func writeProviderAckKey(dir, token string) (retErr error) { + root, err := os.OpenRoot(dir) + if err != nil { + return sanitizeMarkerError(err) + } + defer func() { retErr = errors.Join(retErr, sanitizeMarkerError(root.Close())) }() + f, err := root.OpenFile("ack-key", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return sanitizeMarkerError(err) + } + _, writeErr := f.WriteString(token) + return errors.Join(sanitizeMarkerError(writeErr), sanitizeMarkerError(f.Close())) +} + +func providerForceAcknowledged(files *providerShellInvocation) (acknowledged bool, retErr error) { + root, err := os.OpenRoot(filepath.Dir(files.ack)) + if err != nil { + return false, sanitizeMarkerError(err) + } + defer func() { retErr = errors.Join(retErr, sanitizeMarkerError(root.Close())) }() + return providerForceAckInRoot(root, files.ackToken) +} + +func providerForceAckInRoot(root *os.Root, token string) (acknowledged bool, retErr error) { + info, err := root.Lstat("ack") + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, sanitizeMarkerError(err) + } + if !privateProviderAck(info, len(token)) { + return false, nil + } + f, err := root.Open("ack") + if err != nil { + return false, sanitizeMarkerError(err) + } + defer func() { retErr = errors.Join(retErr, sanitizeMarkerError(f.Close())) }() + return providerForceAckContents(f, token) +} + +func providerForceAckContents(f *os.File, token string) (bool, error) { + info, err := f.Stat() + if err != nil { + return false, sanitizeMarkerError(err) + } + if !privateProviderAck(info, len(token)) { + return false, nil + } + data := make([]byte, len(token)) + if _, err := io.ReadFull(f, data); err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return false, nil + } + return false, sanitizeMarkerError(err) + } + return subtle.ConstantTimeCompare(data, []byte(token)) == 1, nil +} + +func privateProviderAck(info os.FileInfo, length int) bool { + return info.Mode().IsRegular() && info.Size() == int64(length) && + (runtime.GOOS == "windows" || info.Mode().Perm() == 0o600) } func providerReservedKey(key string) bool { @@ -130,7 +327,11 @@ func providerReservedKey(key string) bool { return key == config.EnvProviderCancelFile } -func publishProviderCancellation(marker string) (retErr error) { +func publishProviderCancellation(marker string) error { + return publishProviderMarker(marker) +} + +func publishProviderMarker(marker string) (retErr error) { root, err := os.OpenRoot(filepath.Dir(marker)) if err != nil { return fmt.Errorf("cannot open marker directory: %w", sanitizeMarkerError(err)) @@ -140,14 +341,15 @@ func publishProviderCancellation(marker string) (retErr error) { retErr = errors.Join(retErr, sanitizeMarkerError(err)) } }() - f, err := root.OpenFile("cancel", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + name := filepath.Base(marker) + f, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) if err == nil { return sanitizeMarkerError(f.Close()) } if !errors.Is(err, os.ErrExist) { return fmt.Errorf("cannot create marker: %w", sanitizeMarkerError(err)) } - info, statErr := root.Lstat("cancel") + info, statErr := root.Lstat(name) if statErr != nil { return fmt.Errorf("cannot inspect existing marker: %w", sanitizeMarkerError(statErr)) } diff --git a/pkg/client/clientimplementation/provider_cancel_test.go b/pkg/client/clientimplementation/provider_cancel_test.go index c403576d2f..e225b00b9f 100644 --- a/pkg/client/clientimplementation/provider_cancel_test.go +++ b/pkg/client/clientimplementation/provider_cancel_test.go @@ -11,7 +11,9 @@ import ( "os/signal" "path/filepath" "runtime" + "strconv" "strings" + "sync" "syscall" "testing" "time" @@ -19,40 +21,85 @@ import ( "github.com/devsy-org/devsy/pkg/client" "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/shell" "github.com/devsy-org/devsy/pkg/types" "github.com/stretchr/testify/require" "mvdan.cc/sh/v3/interp" ) +func TestMain(m *testing.M) { + if len(os.Args) == 9 && os.Args[1] == "internal" && os.Args[2] == "provider-shell" { + os.Exit(runProviderShellTestHelper(os.Args)) + } + os.Exit(m.Run()) +} + +func runProviderShellTestHelper(args []string) int { + if args[3] != "--script-file" || args[5] != "--force-file" || args[7] != "--ack-file" { + return 1 + } + err := shell.RunProviderShell(context.Background(), args[4], args[6], args[8]) + if err == nil { + return 0 + } + if status, ok := errors.AsType[interp.ExitStatus](err); ok { + return int(status) + } + return 1 +} + const ( - testTransportRead = "transport-read" - testWindowsOS = "windows" - testCancelProvider = "cancel-provider" - testOpaqueCommand = "opaque" + testTransportRead = "transport-read" + testWindowsOS = "windows" + testCancelProvider = "cancel-provider" + testOpaqueCommand = "opaque" + testCancellationHelperArg = "-test.run=^TestProviderCancellationHelper$" ) func TestProviderCancellationHelper(t *testing.T) { if os.Getenv("DEVSY_CANCEL_HELPER") != "1" { return } - marker := os.Getenv(config.EnvProviderCancelFile) + runTestProviderCancellationBehavior(os.Getenv(config.EnvProviderCancelFile)) + os.Exit(0) +} + +func runTestProviderCancellationBehavior(marker string) { + if os.Getenv("DEVSY_CANCEL_HOLDER") == "1" { + time.Sleep(time.Minute) + return + } switch behavior := os.Getenv("DEVSY_CANCEL_BEHAVIOR"); behavior { case "acknowledge": - if _, err := fmt.Fprint(os.Stdout, "ready"); err != nil { - os.Exit(126) - } - awaitTestCancellationMarker(marker, os.Getenv("DEVSY_CANCEL_ACK_PATH")) + runAcknowledgingCancellationHelper(marker) case "ignore": - if _, err := fmt.Fprint(os.Stdout, "ready"); err != nil { - os.Exit(126) - } - time.Sleep(time.Minute) + runIgnoringCancellationHelper() case testTransportRead, "transport-write": testTransportCancellationHelper(behavior, marker) + case "bulk-after-eof": + writeTestCancellationBulkAfterEOF() + case "orphan-output": + startTestCancellationOutputHolder() default: runTestCancellationExit(behavior, marker) } - os.Exit(0) +} + +func runAcknowledgingCancellationHelper(marker string) { + if _, err := fmt.Fprint(os.Stdout, "ready"); err != nil { + os.Exit(126) + } + awaitTestCancellationMarker(marker, os.Getenv("DEVSY_CANCEL_ACK_PATH")) +} + +func runIgnoringCancellationHelper() { + if path := os.Getenv("DEVSY_CANCEL_CHILD_PID_PATH"); path != "" { + exitTestHelperOnError(testHelperWriteFile(path, strconv.Itoa(os.Getpid()))) + } + if _, err := fmt.Fprint(os.Stdout, "ready"); err != nil { + os.Exit(126) + } + time.Sleep(time.Minute) } func runTestCancellationExit(behavior, marker string) { @@ -69,11 +116,36 @@ func runTestCancellationExit(behavior, marker string) { case "environment": _, err := fmt.Fprint(os.Stdout, marker) exitTestHelperOnError(err) + case "ack-private": + runTestCancellationAckPrivate(marker) default: os.Exit(125) } } +func runTestCancellationAckPrivate(marker string) { + root, err := os.OpenRoot(filepath.Dir(marker)) + exitTestHelperOnError(err) + _, statErr := root.Lstat("ack-key") + exitTestHelperOnError(root.Close()) + if !errors.Is(statErr, os.ErrNotExist) { + os.Exit(126) + } + _, err = os.Stdout.Write([]byte(strings.Join(os.Environ(), "\x00"))) + exitTestHelperOnError(err) + if runtime.GOOS == "linux" { + proc, openErr := os.OpenRoot(fmt.Sprintf("/proc/%d", os.Getppid())) + exitTestHelperOnError(openErr) + for _, name := range []string{"environ", "cmdline"} { + data, readErr := proc.ReadFile(name) + exitTestHelperOnError(readErr) + _, err = os.Stdout.Write(data) + exitTestHelperOnError(err) + } + exitTestHelperOnError(proc.Close()) + } +} + func writeTestCancellationBytes() { _, err := os.Stdout.Write([]byte{0, 0xff, 0x80, '\r', '\n'}) exitTestHelperOnError(err) @@ -81,6 +153,37 @@ func writeTestCancellationBytes() { exitTestHelperOnError(err) } +func writeTestCancellationBulkAfterEOF() { + _, err := io.Copy(io.Discard, os.Stdin) + exitTestHelperOnError(err) + block := make([]byte, 32<<10) + for i := range block { + block[i] = byte(i) + } + for range (12 << 20) / len(block) { + _, err := os.Stdout.Write(block) + exitTestHelperOnError(err) + } + _, err = os.Stderr.Write([]byte("separate stderr")) + exitTestHelperOnError(err) +} + +func startTestCancellationOutputHolder() { + holder := &exec.Cmd{ + Path: os.Args[0], + Args: []string{os.Args[0], testCancellationHelperArg}, + } + holder.Env = append(os.Environ(), "DEVSY_CANCEL_HOLDER=1") + holder.Stdout, holder.Stderr = os.Stdout, os.Stderr + exitTestHelperOnError(holder.Start()) + exitTestHelperOnError(testHelperWriteFile( + os.Getenv("DEVSY_CANCEL_HOLDER_PID_PATH"), strconv.Itoa(holder.Process.Pid), + )) + exitTestHelperOnError(holder.Process.Release()) + _, err := fmt.Fprint(os.Stdout, "ready") + exitTestHelperOnError(err) +} + func exitTestHelperOnError(err error) { if err != nil { os.Exit(126) @@ -167,7 +270,7 @@ func cancelHelperOptions(behavior string) RunCommandOptions { panic(err) } return RunCommandOptions{ - Command: types.StrArray{binary, "-test.run=^TestProviderCancellationHelper$"}, + Command: types.StrArray{binary, testCancellationHelperArg}, Environ: append( os.Environ(), "DEVSY_CANCEL_HELPER=1", @@ -248,6 +351,224 @@ func TestProviderCancellationForcedAfterProductionGrace(t *testing.T) { } } +type providerMarkerWriter struct { + mu sync.Mutex + output bytes.Buffer + ready chan struct{} +} + +func (w *providerMarkerWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + n, err := w.output.Write(p) + if strings.HasSuffix(w.output.String(), string(filepath.Separator)+"cancel") { + select { + case <-w.ready: + default: + close(w.ready) + } + } + return n, err +} + +func (w *providerMarkerWriter) marker() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.output.String() +} + +func TestProviderCancellationBlockedShellStdin(t *testing.T) { + input, writer, err := os.Pipe() + require.NoError(t, err) + defer func() { require.NoError(t, errors.Join(input.Close(), writer.Close())) }() + _, err = os.Stdin.Stat() + require.NoError(t, err) + + const privateCommand = "printf '%s' \"$DEVSY_PROVIDER_CANCEL_FILE\"; mapfile lines; printf 'private-command-846307'" + output := &providerMarkerWriter{ready: make(chan struct{})} + opts := RunCommandOptions{ + Command: types.StrArray{privateCommand}, + Environ: os.Environ(), + Stdin: input, + Stdout: output, + RawStdout: true, + Cancellation: provider.ProviderCancellationFile, + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + finished := make(chan error, 1) + go func() { finished <- RunCommand(ctx, opts) }() + select { + case <-output.ready: + case <-time.After(5 * time.Second): + t.Fatal("provider shell did not start") + } + marker := output.marker() + requirePrivateProviderScript(t, marker, privateCommand) + + start := time.Now() + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + require.GreaterOrEqual(t, time.Since(start), providerCancellationGrace) + require.NotContains(t, err.Error(), privateCommand) + require.NotContains(t, err.Error(), marker) + case <-time.After(providerCancellationGrace + providerForceAckWait + 3*time.Second): + t.Fatal("provider shell remained blocked on stdin") + } + _, err = os.Stdin.Stat() + require.NoError(t, err) + _, err = writer.Stat() + require.NoError(t, err) + _, err = os.Stat(filepath.Dir(marker)) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func requirePrivateProviderScript(t *testing.T, marker, command string) { + t.Helper() + require.True(t, filepath.IsAbs(marker)) + script := filepath.Join(filepath.Dir(marker), "script") + info, err := os.Stat(script) + require.NoError(t, err) + if runtime.GOOS != testWindowsOS { + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + } + root, err := os.OpenRoot(filepath.Dir(marker)) + require.NoError(t, err) + contents, readErr := root.ReadFile("script") + closeErr := root.Close() + require.NoError(t, readErr) + require.NoError(t, closeErr) + require.Equal(t, command, string(contents)) +} + +func TestProviderCancellationMissingForceAck(t *testing.T) { + dir := t.TempDir() + completed := make(chan error, 1) + cancelled := make(chan struct{}) + process := providerExecution{ + cancel: func(error) { + close(cancelled) + completed <- nil + }, + completed: completed, + shell: &providerShellInvocation{ + force: filepath.Join(dir, "force"), + ack: filepath.Join(dir, "ack"), + ackToken: strings.Repeat("a", 64), + }, + } + start := time.Now() + err := forceProviderExecution(context.Canceled, process) + require.ErrorContains(t, err, "provider force acknowledgement timed out") + require.GreaterOrEqual(t, time.Since(start), providerForceAckWait) + select { + case <-cancelled: + default: + t.Fatal("provider helper was not terminated") + } + info, err := os.Stat(process.shell.force) + require.NoError(t, err) + require.True(t, info.Mode().IsRegular()) +} + +func TestProviderCancellationEarlyExitWithoutForceAck(t *testing.T) { + dir := t.TempDir() + force := filepath.Join(dir, "force") + require.NoError(t, publishProviderMarker(force)) + completed := make(chan error, 1) + completed <- errors.New("helper exited") + process := providerExecution{ + cancel: func(error) { t.Fatal("exited helper must not be terminated again") }, + completed: completed, + shell: &providerShellInvocation{ + force: force, + ack: filepath.Join(dir, "ack"), + ackToken: strings.Repeat("a", 64), + }, + } + err := errors.Join(context.Canceled, waitForProviderForceAck(context.Canceled, process)) + require.ErrorIs(t, err, context.Canceled) + require.ErrorContains(t, err, "provider helper exited without authentic acknowledgement") + require.NotContains(t, err.Error(), dir) +} + +func TestProviderCancellationForgedAckBeforeHelperExit(t *testing.T) { + dir := t.TempDir() + token := strings.Repeat("a", 64) + ack := filepath.Join(dir, "ack") + require.NoError(t, os.WriteFile(ack, []byte(strings.Repeat("b", 64)), 0o600)) + completed := make(chan error, 1) + completed <- errors.New("helper died") + process := providerExecution{ + completed: completed, + shell: &providerShellInvocation{ + force: filepath.Join(dir, "force"), + ack: ack, + ackToken: token, + }, + } + err := waitForProviderGrace(context.Canceled, process) + require.ErrorIs(t, err, context.Canceled) + require.ErrorContains(t, err, "without authentic acknowledgement") + require.NotContains(t, err.Error(), token) + require.NotContains(t, err.Error(), dir) +} + +func TestProviderCancellationForgedAckAfterForce(t *testing.T) { + dir := t.TempDir() + token := strings.Repeat("a", 64) + ack := filepath.Join(dir, "ack") + require.NoError(t, os.WriteFile(ack, []byte(strings.Repeat("b", 64)), 0o600)) + force := filepath.Join(dir, "force") + require.NoError(t, publishProviderMarker(force)) + completed := make(chan error, 1) + completed <- errors.New("helper died") + process := providerExecution{ + completed: completed, + shell: &providerShellInvocation{ + force: force, + ack: ack, + ackToken: token, + }, + } + err := errors.Join(context.Canceled, waitForProviderForceAck(context.Canceled, process)) + require.ErrorIs(t, err, context.Canceled) + require.ErrorContains(t, err, "without authentic acknowledgement") + require.NotContains(t, err.Error(), token) +} + +func TestProviderForceAcknowledgedRequiresPrivateExactToken(t *testing.T) { + dir := t.TempDir() + files := &providerShellInvocation{ + ack: filepath.Join(dir, "ack"), + ackToken: strings.Repeat("a", 64), + } + for _, content := range []string{"", strings.Repeat("b", 64), strings.Repeat("a", 63), strings.Repeat("a", 65)} { + require.NoError(t, os.WriteFile(files.ack, []byte(content), 0o600)) + acknowledged, err := providerForceAcknowledged(files) + require.NoError(t, err) + require.False(t, acknowledged) + } + require.NoError(t, os.WriteFile(files.ack, []byte(files.ackToken), 0o600)) + acknowledged, err := providerForceAcknowledged(files) + require.NoError(t, err) + require.True(t, acknowledged) + if runtime.GOOS != testWindowsOS { + require.NoError(t, os.Chmod(files.ack, 0o400)) + acknowledged, err = providerForceAcknowledged(files) + require.NoError(t, err) + require.False(t, acknowledged) + } + require.NoError(t, os.Remove(files.ack)) + if err := os.Symlink(filepath.Join(dir, "missing"), files.ack); err == nil { + acknowledged, err = providerForceAcknowledged(files) + require.NoError(t, err) + require.False(t, acknowledged) + } +} + func TestProviderCancellationPrelaunchAndMarkerFailure(t *testing.T) { ctx, cancel := context.WithCancelCause(context.Background()) cause := errors.New("caller stopped") @@ -337,6 +658,112 @@ func TestProviderCancellationPrivateEnvironment(t *testing.T) { require.ErrorIs(t, err, os.ErrNotExist) } +func TestProviderCancellationAckKeyIsPrivate(t *testing.T) { + dir, err := createProviderInvocationDir() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + binary, err := os.Executable() + require.NoError(t, err) + command := fmt.Sprintf("'%s' %s", binary, testCancellationHelperArg) + files, err := prepareProviderShell(dir, command) + require.NoError(t, err) + root, err := os.OpenRoot(dir) + require.NoError(t, err) + info, statErr := root.Lstat("ack-key") + key, readErr := root.ReadFile("ack-key") + closeErr := root.Close() + require.NoError(t, statErr) + require.NoError(t, readErr) + require.NoError(t, closeErr) + require.True(t, info.Mode().IsRegular()) + if runtime.GOOS != testWindowsOS { + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + } + require.Regexp(t, `^[0-9a-f]{64}$`, string(key)) + require.Equal(t, files.ackToken, string(key)) + var stdout bytes.Buffer + opts := RunCommandOptions{ + Command: types.StrArray{command}, + Environ: providerCancellationEnvironment( + []string{"DEVSY_CANCEL_HELPER=1", "DEVSY_CANCEL_BEHAVIOR=ack-private"}, + filepath.Join(dir, "cancel"), + ), + providerShell: files, + } + require.NoError(t, runSupervisedProviderShell(context.Background(), opts, &stdout, io.Discard)) + require.NotContains(t, stdout.String(), files.ackToken) + require.NotContains(t, stdout.String(), "DEVSY_PROVIDER_SHELL_ACK=") + _, err = os.Lstat(filepath.Join(dir, "ack-key")) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestProviderCancellationAckKeyWriteFailure(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(dir, "ack-key"), 0o700)) + _, err := prepareProviderShell(dir, "printf should-not-run") + require.ErrorContains(t, err, "write provider acknowledgement key") + require.NotContains(t, err.Error(), dir) +} + +func TestProviderCancellationShellCannotForgeAck(t *testing.T) { + if runtime.GOOS == testWindowsOS { + t.Skip("requires POSIX path expansion") + } + binary, err := os.Executable() + require.NoError(t, err) + forged := strings.Repeat("b", 64) + command := fmt.Sprintf( + "printf '%%s' '%s' > \"${DEVSY_PROVIDER_CANCEL_FILE%%/*}/ack\"; "+ + "DEVSY_CANCEL_HELPER=1 DEVSY_CANCEL_BEHAVIOR=ignore '%s' -test.run=^TestProviderCancellationHelper$", + forged, + binary, + ) + pidPath := filepath.Join(t.TempDir(), "child-pid") + opts := RunCommandOptions{ + Command: types.StrArray{command}, + Environ: append(os.Environ(), "DEVSY_CANCEL_CHILD_PID_PATH="+pidPath), + Cancellation: provider.ProviderCancellationFile, + RawStdout: true, + } + ready := make(chan struct{}) + opts.Stdout = &readyWriter{ready: ready} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + finished := make(chan error, 1) + go func() { finished <- RunCommand(ctx, opts) }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("provider direct child did not start") + } + start := time.Now() + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + require.GreaterOrEqual(t, time.Since(start), providerCancellationGrace) + require.NotContains(t, err.Error(), command) + require.NotContains(t, err.Error(), forged) + if err.Error() != context.Canceled.Error() { + require.ErrorContains(t, err, "acknowledgement") + } else { + root, openErr := os.OpenRoot(filepath.Dir(pidPath)) + require.NoError(t, openErr) + data, readErr := root.ReadFile("child-pid") + closeErr := root.Close() + require.NoError(t, readErr) + require.NoError(t, closeErr) + pid, parseErr := strconv.Atoi(string(data)) + require.NoError(t, parseErr) + child, findErr := os.FindProcess(pid) + require.NoError(t, findErr) + require.Error(t, child.Signal(syscall.Signal(0))) + } + case <-time.After(providerCancellationGrace + providerForceAckWait + 3*time.Second): + t.Fatal("provider helper did not reap its direct child or report unclean shutdown") + } +} + func TestProviderCancellationNilEnvironmentInheritsHost(t *testing.T) { t.Setenv("DEVSY_CANCEL_HELPER", "1") t.Setenv("DEVSY_CANCEL_BEHAVIOR", "inherited") @@ -345,7 +772,7 @@ func TestProviderCancellationNilEnvironmentInheritsHost(t *testing.T) { require.NoError(t, err) var stdout bytes.Buffer require.NoError(t, RunCommand(context.Background(), RunCommandOptions{ - Command: types.StrArray{binary, "-test.run=^TestProviderCancellationHelper$"}, + Command: types.StrArray{binary, testCancellationHelperArg}, Cancellation: provider.ProviderCancellationFile, Stdout: &stdout, })) @@ -408,6 +835,163 @@ func TestProviderCancellationRawBytesAndStderrRedaction(t *testing.T) { require.Contains(t, stderr.String(), "***") } +func TestProviderCancellationOwnedInputPreservesShellExitStatus(t *testing.T) { + input := io.NopCloser(strings.NewReader("")) + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{"exit 7"}, + Environ: os.Environ(), + Stdin: input, + Cancellation: provider.ProviderCancellationFile, + closeInputOnExit: input, + }) + var status interp.ExitStatus + require.ErrorAs(t, err, &status) + require.Equal(t, 7, int(status)) +} + +func TestProviderCancellationOwnedInputSuccessfulExitIgnoresEPIPE(t *testing.T) { + input := io.NopCloser(bytes.NewReader(bytes.Repeat([]byte("x"), 2<<20))) + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{"exit 0"}, + Environ: os.Environ(), + Stdin: input, + Cancellation: provider.ProviderCancellationFile, + closeInputOnExit: input, + }) + require.NoError(t, err) +} + +func TestProviderCancellationOwnedInputDrainsBinaryOutputAfterEOF(t *testing.T) { + if runtime.GOOS == testWindowsOS { + t.Skip("requires POSIX shell") + } + binary, err := os.Executable() + require.NoError(t, err) + opts := cancelHelperOptions("bulk-after-eof") + opts.Command = types.StrArray{ + fmt.Sprintf("'%s' -test.run=^TestProviderCancellationHelper$", binary), + } + input := io.NopCloser(strings.NewReader("")) + opts.Stdin = input + opts.closeInputOnExit = input + opts.RawStdout = true + var stdout, stderr bytes.Buffer + opts.Stdout, opts.Stderr = &stdout, &stderr + require.NoError(t, RunCommand(context.Background(), opts)) + + block := make([]byte, 32<<10) + for i := range block { + block[i] = byte(i) + } + require.Equal(t, 12<<20, stdout.Len()) + require.True(t, bytes.Equal(bytes.Repeat(block, (12<<20)/len(block)), stdout.Bytes())) + require.Equal(t, "separate stderr", stderr.String()) +} + +type slowProviderOutputWriter struct { + bytes.Buffer + once sync.Once +} + +func (w *slowProviderOutputWriter) Write(p []byte) (int, error) { + w.once.Do(func() { time.Sleep(3 * time.Second) }) + return w.Buffer.Write(p) +} + +func TestProviderCancellationOwnedInputPreservesSlowOutput(t *testing.T) { + input := io.NopCloser(strings.NewReader("")) + var output slowProviderOutputWriter + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{"printf ready"}, + Environ: os.Environ(), + Stdin: input, + Stdout: &output, + RawStdout: true, + Cancellation: provider.ProviderCancellationFile, + closeInputOnExit: input, + }) + require.NoError(t, err) + require.Equal(t, "ready", output.String()) +} + +func TestProviderCancellationOwnedOutputStopsAfterForcedCancellation(t *testing.T) { + if runtime.GOOS == testWindowsOS { + t.Skip("requires POSIX shell") + } + binary, err := os.Executable() + require.NoError(t, err) + holderPath := filepath.Join(t.TempDir(), "holder-pid") + t.Setenv("DEVSY_CANCEL_HOLDER_PID_PATH", holderPath) + t.Cleanup(func() { cleanupTestCancellationOutputHolder(holderPath) }) + opts := cancelHelperOptions("orphan-output") + opts.Command = types.StrArray{ + fmt.Sprintf("'%s' -test.run=^TestProviderCancellationHelper$", binary), + } + input := io.NopCloser(strings.NewReader("")) + opts.Stdin, opts.closeInputOnExit = input, input + opts.RawStdout = true + ready := make(chan struct{}) + opts.Stdout = &readyWriter{ready: ready} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + finished := make(chan error, 1) + go func() { finished <- RunCommand(ctx, opts) }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("provider output holder did not start") + } + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(providerCancellationGrace + providerForceAckWait + 3*time.Second): + t.Fatal("forced provider completion waited for descendant output pipe") + } +} + +func TestProviderCancellationOwnedOutputStopsAfterSuccessfulShellExit(t *testing.T) { + if runtime.GOOS == testWindowsOS { + t.Skip("requires POSIX shell") + } + binary, err := os.Executable() + require.NoError(t, err) + holderPath := filepath.Join(t.TempDir(), "holder-pid") + t.Setenv("DEVSY_CANCEL_HOLDER_PID_PATH", holderPath) + t.Cleanup(func() { cleanupTestCancellationOutputHolder(holderPath) }) + opts := cancelHelperOptions("orphan-output") + opts.Command = types.StrArray{ + fmt.Sprintf("'%s' -test.run=^TestProviderCancellationHelper$", binary), + } + input := io.NopCloser(strings.NewReader("")) + opts.Stdin, opts.closeInputOnExit = input, input + opts.RawStdout = true + start := time.Now() + err = RunCommand(context.Background(), opts) + require.ErrorIs(t, err, exec.ErrWaitDelay) + require.Less(t, time.Since(start), 5*time.Second) +} + +func cleanupTestCancellationOutputHolder(path string) { + root, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + return + } + data, readErr := root.ReadFile("holder-pid") + _ = root.Close() + if readErr != nil { + return + } + pid, err := strconv.Atoi(string(data)) + if err != nil { + return + } + process, err := os.FindProcess(pid) + if err == nil { + _ = process.Kill() + } +} + func TestMachineCommandTransportCancellationJoinsCallback(t *testing.T) { for _, behavior := range []string{testTransportRead, "transport-write"} { t.Run(behavior, func(t *testing.T) { @@ -422,7 +1006,7 @@ func TestMachineCommandTransportCancellationJoinsCallback(t *testing.T) { Name: testCancelProvider, Exec: provider.ProviderCommands{ Create: types.StrArray{ - "create", + testProviderCreateCommand, }, Command: cancelHelperOptions(behavior).Command, Cancellation: provider.ProviderCancellationFile, @@ -486,6 +1070,29 @@ func TestWorkspaceCommandCancellationJoinsProviderCleanup(t *testing.T) { } } +func TestWorkspaceTransportCompletesWithoutClosingInput(t *testing.T) { + wc := workspaceCancellationTestClient() + wc.providerConfig.Exec.Command = types.StrArray{"printf ready"} + conn, err := wc.OpenCommandTransport(context.Background(), client.CommandTransportOptions{ + Command: testOpaqueCommand, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + ready := make([]byte, 5) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + _, err = io.ReadFull(conn, ready) + require.NoError(t, err) + require.Equal(t, "ready", string(ready)) + finished := make(chan error, 1) + go func() { finished <- conn.Wait() }() + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(3 * time.Second): + t.Fatal("provider completion waited for caller to close transport input") + } +} + func workspaceCancellationTestClient() *workspaceClient { return &workspaceClient{ providerWorkspace: &provider.Workspace{ diff --git a/pkg/client/clientimplementation/proxy_client.go b/pkg/client/clientimplementation/proxy_client.go index 4ad5a3554b..c696250b36 100644 --- a/pkg/client/clientimplementation/proxy_client.go +++ b/pkg/client/clientimplementation/proxy_client.go @@ -57,12 +57,13 @@ type proxyExecutor struct { } type execParams struct { - command types.StrArray - extraEnv map[string]string - stdin io.Reader - stdout io.Writer - stderr io.Writer - rawStdout bool + command types.StrArray + extraEnv map[string]string + stdin io.Reader + closeInputOnExit io.Closer + stdout io.Writer + stderr io.Writer + rawStdout bool } // execute runs a proxy command with common settings. @@ -75,6 +76,7 @@ func (e *proxyExecutor) execute(ctx context.Context, params execParams) error { Config: e.client.config, ExtraEnv: params.extraEnv, Stdin: params.stdin, + closeInputOnExit: params.closeInputOnExit, Stdout: params.stdout, Stderr: params.stderr, RawStdout: params.rawStdout, @@ -221,11 +223,15 @@ func (s *proxyClient) OpenSSHTransport( ctx, func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { return s.executor.executeWithJSONLog(ctx, execParams{ - command: s.config.Exec.Proxy.Ssh, - extraEnv: EncodeOptions(client.SshOptions{User: opt.User}, config.EnvFlagsSSH), - stdin: stdin, - stdout: stdout, - rawStdout: true, + command: s.config.Exec.Proxy.Ssh, + extraEnv: EncodeOptions( + client.SshOptions{User: opt.User}, + config.EnvFlagsSSH, + ), + stdin: stdin, + closeInputOnExit: stdin.(io.Closer), + stdout: stdout, + rawStdout: true, }) }, transport.CallbackConnOptions{ diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index 2e50d78c06..385ca18ba1 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -7,10 +7,13 @@ import ( "errors" "fmt" "io" + "net" "os" "os/exec" "strings" "sync" + "sync/atomic" + "syscall" "time" "al.essio.dev/pkg/shellescape" @@ -31,6 +34,7 @@ import ( "github.com/devsy-org/devsy/pkg/task" "github.com/devsy-org/devsy/pkg/transport" "github.com/devsy-org/devsy/pkg/types" + "mvdan.cc/sh/v3/interp" ) func NewWorkspaceClient( @@ -267,11 +271,14 @@ func (s *workspaceClient) OpenCommandTransport( ctx, func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { return RunCommand(ctx, RunCommandOptions{ - Command: s.providerConfig.Exec.Command, - Environ: environ, - Stdin: stdin, Stdout: stdout, Stderr: opt.Stderr, - RawStdout: true, - Cancellation: s.providerConfig.Exec.Cancellation, + Command: s.providerConfig.Exec.Command, + Environ: environ, + Stdin: stdin, + Stdout: stdout, + Stderr: opt.Stderr, + RawStdout: true, + Cancellation: s.providerConfig.Exec.Cancellation, + closeInputOnExit: stdin.(io.Closer), }) }, transport.CallbackConnOptions{ @@ -796,6 +803,7 @@ type WorkspaceCommandConfig struct { Workspace *provider.Workspace Machine *provider.Machine Config *provider.ProviderConfig + closeInputOnExit io.Closer } func RunCommandWithBinaries(ctx context.Context, cfg WorkspaceCommandConfig) error { @@ -812,24 +820,27 @@ func RunCommandWithBinaries(ctx context.Context, cfg WorkspaceCommandConfig) err } return RunCommand(ctx, RunCommandOptions{ - Command: cfg.Command, - Environ: environ, - Stdin: cfg.Stdin, - Stdout: cfg.Stdout, - Stderr: cfg.Stderr, - RawStdout: cfg.RawStdout, - Cancellation: cfg.Config.Exec.Cancellation, + Command: cfg.Command, + Environ: environ, + Stdin: cfg.Stdin, + Stdout: cfg.Stdout, + Stderr: cfg.Stderr, + RawStdout: cfg.RawStdout, + Cancellation: cfg.Config.Exec.Cancellation, + closeInputOnExit: cfg.closeInputOnExit, }) } type RunCommandOptions struct { - Command types.StrArray - Environ []string - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer - RawStdout bool - Cancellation provider.ProviderCancellationMode + Command types.StrArray + Environ []string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + RawStdout bool + Cancellation provider.ProviderCancellationMode + providerShell *providerShellInvocation + closeInputOnExit io.Closer } func RunCommand(ctx context.Context, opts RunCommandOptions) error { @@ -877,6 +888,9 @@ func runProviderShellCommand( opts RunCommandOptions, stdout, stderr io.Writer, ) error { + if opts.providerShell != nil { + return runSupervisedProviderShell(ctx, opts, stdout, stderr) + } runner := &shell.CommandRunner{ Command: opts.Command[0], Stdin: opts.Stdin, @@ -884,15 +898,413 @@ func runProviderShellCommand( Stderr: stderr, Environ: opts.Environ, } - if opts.Cancellation == provider.ProviderCancellationFile { - runner.ExecHandler = shell.ProviderExecHandler() - } if err := shell.RunEmulatedShell(ctx, runner); err != nil { return fmt.Errorf("run provider %s: %w", providerAction(opts.Environ), err) } return nil } +func runSupervisedProviderShell( + ctx context.Context, + opts RunCommandOptions, + stdout, stderr io.Writer, +) error { + binary, err := os.Executable() + if err != nil { + return fmt.Errorf("find provider shell executable: %w", err) + } + files := opts.providerShell + cmd := &exec.Cmd{ + Path: binary, + Args: []string{ + binary, "internal", "provider-shell", + "--script-file", files.script, "--force-file", files.force, "--ack-file", files.ack, + }, + Env: opts.Environ, + Stdin: opts.Stdin, + Stdout: stdout, + Stderr: stderr, + WaitDelay: 2 * time.Second, + } + if opts.closeInputOnExit != nil { + return waitSupervisedProviderShellWithOwnedInput( + ctx, + cmd, + opts.closeInputOnExit, + providerAction(opts.Environ), + ) + } + return waitSupervisedProviderShell(ctx, cmd, providerAction(opts.Environ)) +} + +type providerShellPipes struct { + stdinRead *os.File + stdinWrite *os.File + stdoutRead *os.File + stdoutWrite *os.File + stderrRead *os.File + stderrWrite *os.File +} + +func openProviderShellPipes() (pipes *providerShellPipes, retErr error) { + pipes = &providerShellPipes{} + defer func() { + if retErr != nil { + retErr = errors.Join(retErr, pipes.close()) + } + }() + pipes.stdinRead, pipes.stdinWrite, retErr = os.Pipe() + if retErr != nil { + return + } + pipes.stdoutRead, pipes.stdoutWrite, retErr = os.Pipe() + if retErr != nil { + return + } + pipes.stderrRead, pipes.stderrWrite, retErr = os.Pipe() + if retErr != nil { + return + } + return +} + +func closeProviderShellPipe(pipe *os.File) error { + if pipe == nil { + return nil + } + err := pipe.Close() + if errors.Is(err, os.ErrClosed) { + return nil + } + return err +} + +func (p *providerShellPipes) close() error { + return errors.Join( + closeProviderShellPipe(p.stdinRead), + closeProviderShellPipe(p.stdinWrite), + closeProviderShellPipe(p.stdoutRead), + closeProviderShellPipe(p.stdoutWrite), + closeProviderShellPipe(p.stderrRead), + closeProviderShellPipe(p.stderrWrite), + ) +} + +func (p *providerShellPipes) closeChildEnds() error { + err := errors.Join( + closeProviderShellPipe(p.stdinRead), + closeProviderShellPipe(p.stdoutWrite), + closeProviderShellPipe(p.stderrWrite), + ) + p.stdinRead, p.stdoutWrite, p.stderrWrite = nil, nil, nil + return err +} + +func monitorSupervisedProviderShell( + ctx context.Context, + cmd *exec.Cmd, + stdoutRead, stderrRead *os.File, +) (chan struct{}, <-chan struct{}) { + stopped := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + select { + case <-ctx.Done(): + _ = cmd.Process.Kill() + _ = closeProviderShellPipe(stdoutRead) + _ = closeProviderShellPipe(stderrRead) + case <-stopped: + } + }() + return stopped, done +} + +type providerShellInputResult struct { + copyErr error + closeErr error +} + +func pumpProviderShellInput(writer *os.File, source io.Reader) <-chan providerShellInputResult { + done := make(chan providerShellInputResult, 1) + go func() { + if source == nil { + source = strings.NewReader("") + } + _, copyErr := io.Copy(writer, source) + done <- providerShellInputResult{copyErr, closeProviderShellPipe(writer)} + }() + return done +} + +type providerShellOutputPump struct { + reader *os.File + done chan error + changed chan struct{} + reading atomic.Bool +} + +func (p *providerShellOutputPump) Read(buf []byte) (int, error) { + p.reading.Store(true) + p.signal() + n, err := p.reader.Read(buf) + p.reading.Store(false) + p.signal() + return n, err +} + +func (p *providerShellOutputPump) signal() { + select { + case p.changed <- struct{}{}: + default: + } +} + +func pumpProviderShellOutput(reader *os.File, writer io.Writer) *providerShellOutputPump { + pump := &providerShellOutputPump{ + reader: reader, done: make(chan error, 1), changed: make(chan struct{}, 1), + } + go func() { + if writer == nil { + writer = io.Discard + } + _, copyErr := io.Copy(writer, pump) + if copyErr != nil { + _, _ = io.Copy(io.Discard, pump) + } + pump.done <- copyErr + }() + return pump +} + +func providerShellInputCopyError(copyErr, waitErr, closeErr error) error { + if waitErr == nil && closeErr == nil && + (errors.Is(copyErr, io.ErrClosedPipe) || errors.Is(copyErr, net.ErrClosed) || + errors.Is(copyErr, os.ErrClosed) || errors.Is(copyErr, syscall.EPIPE)) { + return nil + } + return copyErr +} + +func providerShellCopyError(action, stream string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("copy provider %s %s: %w", action, stream, err) +} + +func providerShellCloseError(action, target string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("close provider %s %s: %w", action, target, err) +} + +func providerShellWaitError(action string, err error) error { + if err == nil { + return nil + } + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { + code := exitErr.ExitCode() + if code >= 0 && code <= 255 { + err = interp.ExitStatus(uint8(code)) + } + } + return fmt.Errorf("run provider %s: %w", action, err) +} + +func waitSupervisedProviderShellWithOwnedInput( + ctx context.Context, + cmd *exec.Cmd, + input io.Closer, + action string, +) (retErr error) { + pipes, err := openProviderShellPipes() + if err != nil { + return fmt.Errorf("create provider %s stdio pipes: %w", action, err) + } + defer func() { retErr = errors.Join(retErr, pipes.close()) }() + + source, stdout, stderr := cmd.Stdin, cmd.Stdout, cmd.Stderr + cmd.Stdin, cmd.Stdout, cmd.Stderr = pipes.stdinRead, pipes.stdoutWrite, pipes.stderrWrite + if err := cmd.Start(); err != nil { + return fmt.Errorf("start provider %s shell: %w", action, err) + } + pipeCloseErr := pipes.closeChildEnds() + stopped, monitorDone := monitorSupervisedProviderShell( + ctx, cmd, pipes.stdoutRead, pipes.stderrRead, + ) + stdinDone := pumpProviderShellInput(pipes.stdinWrite, source) + stdoutPump := pumpProviderShellOutput(pipes.stdoutRead, stdout) + stderrPump := pumpProviderShellOutput(pipes.stderrRead, stderr) + waitErr := cmd.Wait() + stdoutErr, stderrErr, drainErr := drainProviderShellOutput( + pipes, stdoutPump, stderrPump, action, + ) + close(stopped) + <-monitorDone + closeErr := input.Close() + stdinResult := <-stdinDone + stdinErr := providerShellInputCopyError(stdinResult.copyErr, waitErr, closeErr) + return errors.Join( + drainErr, + providerShellWaitError(action, waitErr), + providerShellCopyError(action, "stdout", stdoutErr), + providerShellCopyError(action, "stderr", stderrErr), + providerShellCopyError(action, "stdin", stdinErr), + providerShellCloseError(action, "stdin pipe", stdinResult.closeErr), + providerShellCloseError(action, "stdio pipes", pipeCloseErr), + providerShellCloseError(action, "input", closeErr), + ) +} + +type providerShellOutputDrain struct { + pipes *providerShellPipes + stdoutPump *providerShellOutputPump + stderrPump *providerShellOutputPump + stdoutDone <-chan error + stderrDone <-chan error + timer *time.Timer + timeout <-chan time.Time + action string + stdoutErr error + stderrErr error + drainErr error +} + +func drainProviderShellOutput( + pipes *providerShellPipes, + stdoutPump, stderrPump *providerShellOutputPump, + action string, +) (stdoutErr, stderrErr, drainErr error) { + timer := time.NewTimer(2 * time.Second) + timer.Stop() + defer timer.Stop() + drain := providerShellOutputDrain{ + pipes: pipes, stdoutPump: stdoutPump, stderrPump: stderrPump, + stdoutDone: stdoutPump.done, stderrDone: stderrPump.done, + timer: timer, action: action, + } + for drain.stdoutDone != nil || drain.stderrDone != nil { + drain.updateTimeout() + select { + case drain.stdoutErr = <-drain.stdoutDone: + drain.stdoutDone = nil + case drain.stderrErr = <-drain.stderrDone: + drain.stderrDone = nil + case <-stdoutPump.changed: + drain.resetTimeout() + case <-stderrPump.changed: + drain.resetTimeout() + case <-drain.timeout: + drain.handleTimeout() + } + } + return drain.stdoutErr, drain.stderrErr, drain.drainErr +} + +func (d *providerShellOutputDrain) idle() bool { + return providerShellOutputIdle(d.stdoutPump, d.stdoutDone) && + providerShellOutputIdle(d.stderrPump, d.stderrDone) +} + +func (d *providerShellOutputDrain) updateTimeout() { + if d.idle() { + if d.timeout == nil { + d.timer.Reset(2 * time.Second) + d.timeout = d.timer.C + } + } else if d.timeout != nil { + d.timer.Stop() + d.timeout = nil + } +} + +func (d *providerShellOutputDrain) resetTimeout() { + if d.timeout != nil { + d.timer.Reset(2 * time.Second) + } +} + +func (d *providerShellOutputDrain) handleTimeout() { + d.timeout = nil + if !d.idle() || d.recentOutput() { + return + } + d.stdoutDone, d.stdoutErr = readyProviderShellOutput(d.stdoutDone, d.stdoutErr) + d.stderrDone, d.stderrErr = readyProviderShellOutput(d.stderrDone, d.stderrErr) + d.drainErr = closeUndrainedProviderShellOutput( + d.pipes, d.stdoutDone, d.stderrDone, d.action, + ) +} + +func (d *providerShellOutputDrain) recentOutput() bool { + select { + case <-d.stdoutPump.changed: + return true + default: + } + select { + case <-d.stderrPump.changed: + return true + default: + } + return false +} + +func providerShellOutputIdle(pump *providerShellOutputPump, done <-chan error) bool { + return done == nil || pump.reading.Load() +} + +func readyProviderShellOutput(done <-chan error, current error) (<-chan error, error) { + if done == nil { + return nil, current + } + select { + case err := <-done: + return nil, err + default: + return done, current + } +} + +func closeUndrainedProviderShellOutput( + pipes *providerShellPipes, + stdoutDone, stderrDone <-chan error, + action string, +) error { + if stdoutDone == nil && stderrDone == nil { + return nil + } + if stdoutDone != nil { + _ = closeProviderShellPipe(pipes.stdoutRead) + } + if stderrDone != nil { + _ = closeProviderShellPipe(pipes.stderrRead) + } + return fmt.Errorf("drain provider %s output after shell exit: %w", action, exec.ErrWaitDelay) +} + +func waitSupervisedProviderShell(ctx context.Context, cmd *exec.Cmd, action string) error { + if err := cmd.Start(); err != nil { + return fmt.Errorf("start provider %s shell: %w", action, err) + } + stopped, monitorDone := monitorSupervisedProviderShell(ctx, cmd, nil, nil) + err := cmd.Wait() + close(stopped) + <-monitorDone + if err != nil { + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { + code := exitErr.ExitCode() + if code >= 0 && code <= 255 { + err = interp.ExitStatus(uint8(code)) + } + } + return fmt.Errorf("run provider %s: %w", action, err) + } + return nil +} + func runProviderArgvCommand( ctx context.Context, opts RunCommandOptions, @@ -926,10 +1338,12 @@ func flushCommandOutput(stdout, stderr *subprocess.StreamingRedactingWriter) err return errors.Join(stdoutErr, stderrErr) } +const providerCommandAction = "command" + func providerAction(environ []string) string { for _, kv := range environ { if strings.HasPrefix(kv, provider.CommandEnv+"=") { - return "command" + return providerCommandAction } } return "lifecycle" diff --git a/pkg/client/clientimplementation/workspace_client_test.go b/pkg/client/clientimplementation/workspace_client_test.go index 6915ca2077..145534ba4b 100644 --- a/pkg/client/clientimplementation/workspace_client_test.go +++ b/pkg/client/clientimplementation/workspace_client_test.go @@ -167,7 +167,7 @@ func TestMachineCommandErrorsRedactOpaquePayloadAndArguments(t *testing.T) { cfg := &provider.ProviderConfig{ Name: "redaction-test", Exec: provider.ProviderCommands{ - Create: types.StrArray{"create"}, + Create: types.StrArray{testProviderCreateCommand}, Command: tc.command, }, } diff --git a/pkg/shell/provider_exec.go b/pkg/shell/provider_exec.go index 23b2be7d4b..4c247b7205 100644 --- a/pkg/shell/provider_exec.go +++ b/pkg/shell/provider_exec.go @@ -16,22 +16,24 @@ import ( ) func ProviderExecHandler() interp.ExecHandlerFunc { + return providerExecHandler(nil) +} + +func providerExecHandler(supervisor *providerProcessSupervisor) interp.ExecHandlerFunc { return func(ctx context.Context, args []string) error { hc := interp.HandlerCtx(ctx) path, err := interp.LookPathDir(hc.Dir, hc.Env, args[0]) if err != nil { - _, _ = fmt.Fprintln(hc.Stderr, err) + _, _ = fmt.Fprintln(hc.Stderr, "provider executable not found") return interp.ExitStatus(127) } - err = runOwnedProviderExec(ctx, hc, path, args) - if errors.Is(err, syscall.ENOEXEC) && runtime.GOOS != "windows" { - err = runOwnedProviderExec( - ctx, - hc, - "/bin/sh", - append([]string{"/bin/sh", path}, args[1:]...), - ) + command := providerExecCommand{path: path, args: args} + err = runOwnedProviderExec(ctx, hc, command, supervisor) + if errors.Is(err, syscall.ENOEXEC) && runtime.GOOS != providerWindowsOS { + command.path = "/bin/sh" + command.args = append([]string{"/bin/sh", path}, args[1:]...) + err = runOwnedProviderExec(ctx, hc, command, supervisor) } if ctx.Err() != nil { return context.Cause(ctx) @@ -52,24 +54,33 @@ func providerExecResult(err error, stderr io.Writer) error { return interp.ExitStatus(uint8(code)) } if _, ok := errors.AsType[*exec.Error](err); ok { - _, _ = fmt.Fprintln(stderr, err) + _, _ = fmt.Fprintln(stderr, "provider executable could not start") return interp.ExitStatus(127) } return err } +type providerExecCommand struct { + path string + args []string +} + func runOwnedProviderExec( ctx context.Context, hc interp.HandlerContext, - path string, - args []string, + command providerExecCommand, + supervisor *providerProcessSupervisor, ) error { - cmd, err := startProviderExec(ctx, hc, path, args) + cmd, err := startProviderExec(ctx, hc, command, supervisor) if err != nil { return err } if ctx.Done() == nil { - return cmd.Wait() + err = cmd.Wait() + if supervisor != nil { + supervisor.reaped(cmd) + } + return err } done := make(chan struct{}) monitorDone := make(chan struct{}) @@ -84,20 +95,23 @@ func runOwnedProviderExec( err = cmd.Wait() close(done) <-monitorDone + if supervisor != nil { + supervisor.reaped(cmd) + } return err } func startProviderExec( ctx context.Context, hc interp.HandlerContext, - path string, - args []string, + command providerExecCommand, + supervisor *providerProcessSupervisor, ) (*exec.Cmd, error) { environ := providerExecEnv(hc.Env) newCommand := func() *exec.Cmd { return &exec.Cmd{ - Path: path, - Args: args, + Path: command.path, + Args: command.args, Env: environ, Dir: hc.Dir, Stdin: hc.Stdin, @@ -106,24 +120,13 @@ func startProviderExec( WaitDelay: 2 * time.Second, } } - cmd := newCommand() if ctx.Err() != nil { return nil, context.Cause(ctx) } - err := cmd.Start() - for delay := time.Millisecond; errors.Is(err, syscall.ETXTBSY) && delay < 300*time.Millisecond; delay *= 2 { - select { - case <-ctx.Done(): - return nil, context.Cause(ctx) - case <-time.After(delay): - } - cmd = newCommand() - err = cmd.Start() - } - if err != nil { - return nil, err + if supervisor != nil { + return supervisor.start(ctx, newCommand) } - return cmd, nil + return startProviderExecRetry(ctx, newCommand) } func providerExecEnv(env expand.Environ) []string { diff --git a/pkg/shell/provider_exec_test.go b/pkg/shell/provider_exec_test.go index dbdfa64cf0..3d6771e73e 100644 --- a/pkg/shell/provider_exec_test.go +++ b/pkg/shell/provider_exec_test.go @@ -70,7 +70,7 @@ func TestProviderExecHandlerExportsUnsetAndWorkingDirectory(t *testing.T) { strconv.Quote(workdir), strconv.Quote(binary), ) - if runtime.GOOS == "windows" { + if runtime.GOOS == providerWindowsOS { command = strings.ReplaceAll(command, `\`, `/`) } var stdout, stderr bytes.Buffer @@ -128,8 +128,8 @@ func TestProviderExecHandlerExitStatuses(t *testing.T) { Environ: os.Environ(), Stderr: &stderr, }) - var exitStatus interp.ExitStatus - if !errors.As(err, &exitStatus) || int(exitStatus) != tc.status { + exitStatus, ok := errors.AsType[interp.ExitStatus](err) + if !ok || int(exitStatus) != tc.status { t.Errorf("%q: got %v, want exit %d", tc.command, err, tc.status) } } diff --git a/pkg/shell/provider_exec_unix.go b/pkg/shell/provider_exec_unix.go index 40ba1fc681..1e84d06c29 100644 --- a/pkg/shell/provider_exec_unix.go +++ b/pkg/shell/provider_exec_unix.go @@ -3,12 +3,21 @@ package shell import ( + "os" "os/exec" "syscall" "mvdan.cc/sh/v3/interp" ) +func providerPrivateDirectory(_ string, info os.FileInfo) bool { + return info.Mode().Perm() == 0o700 +} + +func providerPrivateScript(info os.FileInfo) bool { + return info.Mode().Perm() == 0o600 +} + func providerExecSignalStatus(err *exec.ExitError) (interp.ExitStatus, bool) { if status, ok := err.Sys().(syscall.WaitStatus); ok && status.Signaled() { code := 128 + int(status.Signal()) diff --git a/pkg/shell/provider_exec_windows.go b/pkg/shell/provider_exec_windows.go index 4b47e4b558..f8ff1925ad 100644 --- a/pkg/shell/provider_exec_windows.go +++ b/pkg/shell/provider_exec_windows.go @@ -1,11 +1,45 @@ package shell import ( + "os" "os/exec" + "strings" + "golang.org/x/sys/windows" "mvdan.cc/sh/v3/interp" ) +func providerPrivateDirectory(path string, info os.FileInfo) bool { + if !info.IsDir() { + return false + } + sd, err := windows.GetNamedSecurityInfo( + path, + windows.SE_FILE_OBJECT, + windows.DACL_SECURITY_INFORMATION, + ) + if err != nil { + return false + } + control, _, err := sd.Control() + if err != nil || control&windows.SE_DACL_PROTECTED == 0 { + return false + } + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + return false + } + sddl := sd.String() + return strings.Count(sddl, "(") == 2 && + strings.Contains(sddl, "(A;OICI;GA;;;"+user.User.Sid.String()+")") && + (strings.Contains(sddl, "(A;OICI;GA;;;SY)") || + strings.Contains(sddl, "(A;OICI;GA;;;S-1-5-18)")) +} + +func providerPrivateScript(info os.FileInfo) bool { + return info.Mode().IsRegular() +} + func providerExecSignalStatus(_ *exec.ExitError) (interp.ExitStatus, bool) { return 0, false } diff --git a/pkg/shell/provider_supervisor.go b/pkg/shell/provider_supervisor.go new file mode 100644 index 0000000000..307057aa0f --- /dev/null +++ b/pkg/shell/provider_supervisor.go @@ -0,0 +1,397 @@ +package shell + +import ( + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "syscall" + "time" +) + +const providerWindowsOS = "windows" + +type providerProcessSupervisor struct { + mu sync.Mutex + forced bool + children map[*exec.Cmd]chan struct{} +} + +func (s *providerProcessSupervisor) start( + ctx context.Context, + newCommand func() *exec.Cmd, +) (*exec.Cmd, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.forced || ctx.Err() != nil { + return nil, context.Cause(ctx) + } + cmd, err := startProviderExecRetry(ctx, newCommand) + if err != nil { + return nil, err + } + if s.children == nil { + s.children = make(map[*exec.Cmd]chan struct{}) + } + s.children[cmd] = make(chan struct{}) + return cmd, nil +} + +func startProviderExecRetry(ctx context.Context, newCommand func() *exec.Cmd) (*exec.Cmd, error) { + cmd := newCommand() + err := cmd.Start() + for delay := time.Millisecond; errors.Is(err, syscall.ETXTBSY) && delay < 300*time.Millisecond; delay *= 2 { + select { + case <-ctx.Done(): + return nil, context.Cause(ctx) + case <-time.After(delay): + } + cmd = newCommand() + err = cmd.Start() + } + if err != nil { + return nil, err + } + return cmd, nil +} + +func (s *providerProcessSupervisor) reaped(cmd *exec.Cmd) { + s.mu.Lock() + if done, ok := s.children[cmd]; ok { + delete(s.children, cmd) + close(done) + } + s.mu.Unlock() +} + +func (s *providerProcessSupervisor) settled() <-chan struct{} { + s.mu.Lock() + defer s.mu.Unlock() + for _, done := range s.children { + return done + } + return nil +} + +func (s *providerProcessSupervisor) force(cancel context.CancelFunc) { + s.mu.Lock() + s.forced = true + cancel() + children := make([]*exec.Cmd, 0, len(s.children)) + for cmd := range s.children { + children = append(children, cmd) + } + s.mu.Unlock() + for _, cmd := range children { + _ = cmd.Process.Kill() + } + for _, cmd := range children { + s.mu.Lock() + done := s.children[cmd] + s.mu.Unlock() + if done != nil { + <-done + } + } +} + +func providerShellPaths(scriptPath, forcePath, ackPath string) error { + if !filepath.IsAbs(scriptPath) || filepath.Clean(scriptPath) != scriptPath || + filepath.Base(scriptPath) != "script" { + return errors.New("provider shell requires an absolute private script path") + } + dir := filepath.Dir(scriptPath) + if forcePath != filepath.Join(dir, "force") || ackPath != filepath.Join(dir, "ack") { + return errors.New("provider shell control files must be private sibling paths") + } + return nil +} + +func providerShellRoot(dir string) (*os.Root, error) { + root, err := os.OpenRoot(dir) + if err != nil { + return nil, errors.New("provider shell directory is inaccessible") + } + info, err := root.Stat(".") + if err != nil || !providerPrivateDirectory(dir, info) { + _ = root.Close() + return nil, errors.New("provider shell directory is not private") + } + return root, nil +} + +func providerShellFiles(root *os.Root) error { + script, err := root.Lstat("script") + if err != nil || !script.Mode().IsRegular() || !providerPrivateScript(script) { + return errors.New("provider shell script is not a private regular file") + } + if _, err := providerForcePresentRoot(root); err != nil { + return err + } + _, err = root.Lstat("ack") + if err == nil || !errors.Is(err, os.ErrNotExist) { + return errors.New("provider shell acknowledgement already exists or is inaccessible") + } + return nil +} + +func providerForcePresent(dir string) (bool, error) { + root, err := providerShellRoot(dir) + if err != nil { + return false, err + } + defer func() { _ = root.Close() }() + return providerForcePresentRoot(root) +} + +func providerForcePresentRoot(root *os.Root) (bool, error) { + info, err := root.Lstat("force") + if err == nil { + if info.Mode().IsRegular() { + return true, nil + } + return false, errors.New("provider shell force marker is not a regular file") + } + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return false, errors.New("provider shell force marker is inaccessible") +} + +func providerWriteAck(root *os.Root, token string) error { + file, err := root.OpenFile("ack.tmp", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return errors.New("create provider shell acknowledgement: " + errStringKind(err)) + } + defer func() { _ = root.Remove("ack.tmp") }() + if _, err := file.WriteString(token); err != nil { + _ = file.Close() + return errors.New("write provider shell acknowledgement: " + errStringKind(err)) + } + if err := file.Sync(); err != nil { + _ = file.Close() + return errors.New("sync provider shell acknowledgement: " + errStringKind(err)) + } + if err := file.Close(); err != nil { + return errors.New("close provider shell acknowledgement: " + errStringKind(err)) + } + if err := root.Rename("ack.tmp", "ack"); err != nil { + return errors.New("publish provider shell acknowledgement: " + errStringKind(err)) + } + return nil +} + +func providerAckToken(root *os.Root) (string, error) { + invalid := errors.New("invalid provider shell acknowledgement token") + info, err := root.Lstat("ack-key") + if err != nil || !providerPrivateAckKey(info) { + return "", invalid + } + token, ok := providerReadAckKey(root) + if !ok || !validProviderAckToken(token) { + return "", invalid + } + return token, nil +} + +func providerPrivateAckKey(info os.FileInfo) bool { + return info.Mode().IsRegular() && + (runtime.GOOS == providerWindowsOS || info.Mode().Perm() == 0o600) +} + +func providerReadAckKey(root *os.Root) (string, bool) { + file, err := root.Open("ack-key") + if err != nil { + return "", false + } + info, statErr := file.Stat() + content, readErr := io.ReadAll(io.LimitReader(file, 65)) + closeErr := file.Close() + removeErr := root.Remove("ack-key") + if statErr != nil || !providerPrivateAckKey(info) || + readErr != nil || closeErr != nil || removeErr != nil { + return "", false + } + return string(content), true +} + +func validProviderAckToken(token string) bool { + if len(token) != 64 { + return false + } + for i := range len(token) { + c := token[i] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +func errStringKind(err error) string { + if errors.Is(err, os.ErrExist) { + return "already exists" + } + if errors.Is(err, os.ErrPermission) { + return "permission denied" + } + return "file operation failed" +} + +type providerShellExecution struct { + ctx context.Context + dir string + token string + root *os.Root + result <-chan error + supervisor *providerProcessSupervisor + cancel context.CancelFunc +} + +func RunProviderShell(ctx context.Context, scriptPath, forcePath, ackPath string) error { + if ctx.Err() != nil { + return context.Cause(ctx) + } + if err := providerShellPaths(scriptPath, forcePath, ackPath); err != nil { + return err + } + dir := filepath.Dir(scriptPath) + root, err := providerShellRoot(dir) + if err != nil { + return err + } + defer func() { _ = root.Close() }() + token, err := providerAckToken(root) + if err != nil { + return err + } + if err := providerShellFiles(root); err != nil { + return err + } + content, err := root.ReadFile("script") + if err != nil { + return errors.New("read provider shell script: " + errStringKind(err)) + } + executionCtx, cancel := context.WithCancel(ctx) + defer cancel() + supervisor := &providerProcessSupervisor{} + result := make(chan error, 1) + go func() { + result <- RunEmulatedShell(executionCtx, &CommandRunner{ + Command: string(content), Environ: os.Environ(), Stdin: os.Stdin, + Stdout: os.Stdout, Stderr: os.Stderr, ExecHandler: providerExecHandler(supervisor), + }) + }() + execution := providerShellExecution{ + ctx: ctx, + dir: dir, + root: root, + token: token, + result: result, + supervisor: supervisor, + cancel: cancel, + } + return execution.wait(ctx) +} + +func (p *providerShellExecution) wait(ctx context.Context) error { + poll := time.NewTicker(100 * time.Millisecond) + defer poll.Stop() + for { + forced, pollErr := providerForcePresent(p.dir) + if forced || pollErr != nil || ctx.Err() != nil { + return p.force(pollErr) + } + select { + case err := <-p.result: + return p.completed(err) + case <-poll.C: + case <-ctx.Done(): + } + } +} + +func (p *providerShellExecution) ackForced() error { + p.supervisor.force(p.cancel) + return providerWriteAck(p.root, p.token) +} + +func (p *providerShellExecution) force(pollErr error) error { + if pollErr != nil { + p.supervisor.force(p.cancel) + return pollErr + } + if err := p.ackForced(); err != nil { + return err + } + <-p.result + return context.Canceled +} + +func (p *providerShellExecution) completed(err error) error { + forced, pollErr := providerForcePresent(p.dir) + if pollErr != nil { + return pollErr + } + if forced { + return p.canceled() + } + if earlyErr := providerShellEarlyError(err); earlyErr != nil { + return earlyErr + } + if err := p.waitForChildren(); err != nil { + return err + } + if ackErr := providerWriteAck(p.root, p.token); ackErr != nil { + return ackErr + } + return err +} + +func providerShellEarlyError(err error) error { + if err == nil { + return nil + } + if strings.HasPrefix(err.Error(), "parse shell command: ") { + return errors.New("parse provider shell script") + } + if strings.HasPrefix(err.Error(), "create shell runner: ") { + return err + } + if pathErr, ok := errors.AsType[*os.PathError](err); ok && pathErr.Op == "getwd" { + return err + } + return nil +} + +func (p *providerShellExecution) canceled() error { + if err := p.ackForced(); err != nil { + return err + } + return context.Canceled +} + +func (p *providerShellExecution) waitForChildren() error { + for done := p.supervisor.settled(); done != nil; done = p.supervisor.settled() { + select { + case <-done: + case <-p.ctx.Done(): + return p.canceled() + case <-time.After(100 * time.Millisecond): + forced, pollErr := providerForcePresent(p.dir) + if pollErr != nil { + p.supervisor.force(p.cancel) + return pollErr + } + if forced { + return p.canceled() + } + } + } + return nil +} diff --git a/pkg/shell/provider_supervisor_test.go b/pkg/shell/provider_supervisor_test.go new file mode 100644 index 0000000000..20af429000 --- /dev/null +++ b/pkg/shell/provider_supervisor_test.go @@ -0,0 +1,484 @@ +package shell + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "mvdan.cc/sh/v3/interp" +) + +var providerTestAckToken = strings.Repeat("0123456789abcdef", 4) + +func TestProviderShellProcess(t *testing.T) { + if os.Getenv("DEVSY_PROVIDER_SHELL_PROCESS") != "1" { + return + } + err := RunProviderShell(context.Background(), os.Getenv("DEVSY_PROVIDER_SCRIPT"), + os.Getenv("DEVSY_PROVIDER_FORCE"), os.Getenv("DEVSY_PROVIDER_ACK")) + if err == nil { + os.Exit(0) + } + if status, ok := errors.AsType[interp.ExitStatus](err); ok { + os.Exit(int(status)) + } + if errors.Is(err, context.Canceled) { + os.Exit(0) + } + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} + +func TestProviderShellRawOutput(t *testing.T) { + if os.Getenv("DEVSY_PROVIDER_SHELL_RAW") != "1" { + return + } + _, _ = os.Stdout.Write([]byte{0xff, 0, '\n'}) + _, _ = os.Stderr.Write([]byte{0, 0xfe}) + os.Exit(0) +} + +func TestProviderShellTrackedChild(t *testing.T) { + if os.Getenv("DEVSY_PROVIDER_SHELL_CHILD") != "1" { + return + } + if err := writeTrackedChildPID(os.Getenv("DEVSY_PROVIDER_CHILD_PID")); err != nil { + os.Exit(1) + } + _, _ = fmt.Fprintln(os.Stdout, "ready") + time.Sleep(time.Minute) + os.Exit(0) +} + +func TestProviderShellTokenChild(t *testing.T) { + if os.Getenv("DEVSY_PROVIDER_SHELL_TOKEN_CHILD") != "1" { + return + } + if !providerTokenChildClean() { + _, _ = fmt.Fprint(os.Stdout, "leaked") + os.Exit(1) + } + _, _ = fmt.Fprint(os.Stdout, "clean") + os.Exit(0) +} + +func providerTokenChildClean() bool { + if strings.Contains(strings.Join(os.Environ(), "\x00"), providerTestAckToken) || + !providerProcEnvironClean() || !providerAckKeyAbsent() { + return false + } + for _, arg := range os.Args { + if strings.Contains(arg, providerTestAckToken) { + return false + } + } + return true +} + +func providerProcEnvironClean() bool { + if runtime.GOOS != "linux" { + return true + } + root, err := os.OpenRoot("/proc") + if err != nil { + return false + } + defer func() { _ = root.Close() }() + for _, path := range []string{"self/environ", strconv.Itoa(os.Getppid()) + "/environ"} { + environ, err := root.ReadFile(path) + if err != nil || bytes.Contains(environ, []byte(providerTestAckToken)) { + return false + } + } + return true +} + +func providerAckKeyAbsent() bool { + root, err := os.OpenRoot(filepath.Dir(os.Getenv("DEVSY_PROVIDER_ACK"))) + if err != nil { + return false + } + defer func() { _ = root.Close() }() + _, err = root.Lstat("ack-key") + return errors.Is(err, os.ErrNotExist) +} + +func writeTrackedChildPID(path string) error { + root, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + return err + } + file, err := root.OpenFile("pid", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return errors.Join(err, root.Close()) + } + _, writeErr := file.WriteString(strconv.Itoa(os.Getpid())) + return errors.Join(writeErr, file.Close(), root.Close()) +} + +func providerShellFixture(t *testing.T, script string) (string, string, string) { + t.Helper() + dir := providerPrivateTestDir(t) + scriptPath := filepath.Join(dir, "script") + require.NoError(t, os.WriteFile(scriptPath, []byte(script), 0o600)) + return scriptPath, filepath.Join(dir, "force"), filepath.Join(dir, "ack") +} + +func providerTestWriteKey(t *testing.T, ackPath, token string) { + t.Helper() + require.NoError( + t, + os.WriteFile(filepath.Join(filepath.Dir(ackPath), "ack-key"), []byte(token), 0o600), + ) +} + +func providerShellCommand(t *testing.T, scriptPath, forcePath, ackPath string) *exec.Cmd { + t.Helper() + providerTestWriteKey(t, ackPath, providerTestAckToken) + binary, err := os.Executable() + require.NoError(t, err) + cmd := &exec.Cmd{Path: binary, Args: []string{binary, "-test.run=^TestProviderShellProcess$"}} + cmd.Env = append(os.Environ(), "DEVSY_PROVIDER_SHELL_PROCESS=1", + "DEVSY_PROVIDER_SCRIPT="+scriptPath, "DEVSY_PROVIDER_FORCE="+forcePath, + "DEVSY_PROVIDER_ACK="+ackPath) + return cmd +} + +func runProviderShellWithToken(t *testing.T, scriptPath, forcePath, ackPath string) error { + t.Helper() + providerTestWriteKey(t, filepath.Join(filepath.Dir(scriptPath), "ack"), providerTestAckToken) + return RunProviderShell(context.Background(), scriptPath, forcePath, ackPath) +} + +func providerTestReadAck(ackPath string) ([]byte, error) { + root, err := os.OpenRoot(filepath.Dir(ackPath)) + if err != nil { + return nil, err + } + defer func() { _ = root.Close() }() + return root.ReadFile("ack") +} + +func TestRunProviderShellRejectsInvalidAckTokens(t *testing.T) { + for _, test := range []struct { + name string + token string + }{ + {name: "missing"}, + {name: "empty", token: ""}, + {name: "short", token: strings.Repeat("a", 63)}, + {name: "long", token: strings.Repeat("a", 65)}, + {name: "newline", token: providerTestAckToken + "\n"}, + {name: "uppercase", token: strings.Repeat("A", 64)}, + {name: "nonhex", token: strings.Repeat("g", 64)}, + } { + t.Run(test.name, func(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "printf should-not-run") + if test.name != "missing" { + providerTestWriteKey(t, ackPath, test.token) + } + err := RunProviderShell(context.Background(), scriptPath, forcePath, ackPath) + require.ErrorContains(t, err, "invalid provider shell acknowledgement token") + _, err = os.Lstat(ackPath) + require.ErrorIs(t, err, os.ErrNotExist) + }) + } +} + +func TestRunProviderShellRejectsUnsafeAckKey(t *testing.T) { + t.Run("directory", func(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "printf should-not-run") + require.NoError(t, os.Mkdir(filepath.Join(filepath.Dir(ackPath), "ack-key"), 0o700)) + require.ErrorContains( + t, + RunProviderShell(context.Background(), scriptPath, forcePath, ackPath), + "invalid provider shell acknowledgement token", + ) + _, err := os.Lstat(ackPath) + require.ErrorIs(t, err, os.ErrNotExist) + }) + if runtime.GOOS == providerWindowsOS { + return + } + t.Run("symlink", func(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "printf should-not-run") + require.NoError(t, os.Symlink(scriptPath, filepath.Join(filepath.Dir(ackPath), "ack-key"))) + require.ErrorContains( + t, + RunProviderShell(context.Background(), scriptPath, forcePath, ackPath), + "invalid provider shell acknowledgement token", + ) + _, err := os.Lstat(ackPath) + require.ErrorIs(t, err, os.ErrNotExist) + }) + t.Run("permissions", func(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "printf should-not-run") + providerTestWriteKey(t, ackPath, providerTestAckToken) + require.NoError(t, os.Chmod(filepath.Join(filepath.Dir(ackPath), "ack-key"), 0o400)) + require.ErrorContains( + t, + RunProviderShell(context.Background(), scriptPath, forcePath, ackPath), + "invalid provider shell acknowledgement token", + ) + _, err := os.Lstat(ackPath) + require.ErrorIs(t, err, os.ErrNotExist) + }) +} + +func TestRunProviderShellDoesNotLeakAckToken(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + command := strconv.Quote(binary) + " -test.run=^TestProviderShellTokenChild$" + if runtime.GOOS == providerWindowsOS { + command = strings.ReplaceAll(command, `\`, `/`) + } + scriptPath, forcePath, ackPath := providerShellFixture(t, command) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + cmd.Env = append(cmd.Env, "DEVSY_PROVIDER_SHELL_TOKEN_CHILD=1") + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + require.NoError(t, cmd.Run()) + require.Equal(t, "clean", stdout.String()) + require.NotContains(t, stderr.String(), providerTestAckToken) + ack, err := providerTestReadAck(ackPath) + require.NoError(t, err) + require.Equal(t, providerTestAckToken, string(ack)) +} + +func TestRunProviderShellRejectsForgedAck(t *testing.T) { + for _, forged := range []string{"", strings.Repeat("f", 64)} { + t.Run(strconv.Itoa(len(forged)), func(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "exit 0") + require.NoError(t, os.WriteFile(ackPath, []byte(forged), 0o600)) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + output, err := cmd.CombinedOutput() + require.Error(t, err) + require.NotContains(t, string(output), providerTestAckToken) + ack, err := providerTestReadAck(ackPath) + require.NoError(t, err) + require.Equal(t, forged, string(ack)) + }) + } +} + +func TestRunProviderShellNormalExitStatuses(t *testing.T) { + for _, test := range []struct { + script string + status int + }{ + {"printf normal", 0}, + {"exit 7", 7}, + {"exit 125", 125}, + {"command-that-does-not-exist-846307", 127}, + } { + t.Run(strconv.Itoa(test.status), func(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, test.script) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + output, err := cmd.CombinedOutput() + if test.status == 0 { + require.NoError(t, err) + require.Equal(t, "normal", string(output)) + } else { + exit, ok := errors.AsType[*exec.ExitError](err) + require.True(t, ok) + require.Equal(t, test.status, exit.ExitCode()) + } + ack, err := providerTestReadAck(ackPath) + require.NoError(t, err) + require.Equal(t, providerTestAckToken, string(ack)) + }) + } +} + +func TestRunProviderShellRawStreams(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + command := strconv.Quote(binary) + " -test.run=^TestProviderShellRawOutput$" + if runtime.GOOS == providerWindowsOS { + command = strings.ReplaceAll(command, `\`, `/`) + } + scriptPath, forcePath, ackPath := providerShellFixture(t, command) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + cmd.Env = append(cmd.Env, "DEVSY_PROVIDER_SHELL_RAW=1") + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + require.NoError(t, cmd.Run()) + require.Equal(t, []byte{0xff, 0, '\n'}, stdout.Bytes()) + require.Equal(t, []byte{0, 0xfe}, stderr.Bytes()) +} + +func TestRunProviderShellForceAcknowledgesReapedChild(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + command := strconv.Quote(binary) + " -test.run=^TestProviderShellTrackedChild$" + if runtime.GOOS == providerWindowsOS { + command = strings.ReplaceAll(command, `\`, `/`) + } + scriptPath, forcePath, ackPath := providerShellFixture(t, command+"; printf finished") + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + pidPath := filepath.Join(filepath.Dir(scriptPath), "pid") + cmd.Env = append(cmd.Env, "DEVSY_PROVIDER_SHELL_CHILD=1", "DEVSY_PROVIDER_CHILD_PID="+pidPath) + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + var stderr bytes.Buffer + cmd.Stderr = &stderr + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + ready := make(chan string, 1) + go func() { + line, _ := bufio.NewReader(stdout).ReadString('\n') + ready <- line + }() + select { + case line := <-ready: + require.Equal(t, "ready\n", line) + case <-time.After(5 * time.Second): + t.Fatal("provider child did not start") + } + require.NoError(t, os.WriteFile(forcePath, nil, 0o600)) + require.Eventually(t, func() bool { + content, err := providerTestReadAck(ackPath) + return err == nil && string(content) == providerTestAckToken + }, 5*time.Second, 20*time.Millisecond) + root, err := os.OpenRoot(filepath.Dir(scriptPath)) + require.NoError(t, err) + defer func() { _ = root.Close() }() + pidBytes, err := root.ReadFile("pid") + require.NoError(t, err) + pid, err := strconv.Atoi(string(pidBytes)) + require.NoError(t, err) + require.NoError(t, providerChildReaped(pid)) + require.NoError(t, cmd.Wait()) + require.NotContains(t, stderr.String(), command) +} + +func TestRunProviderShellBlockedBuiltinAcknowledges(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture( + t, + "printf 'ready\\n'; mapfile lines; printf finished", + ) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + stdinRead, stdinWrite, err := os.Pipe() + require.NoError(t, err) + defer func() { _ = stdinRead.Close() }() + defer func() { _ = stdinWrite.Close() }() + cmd.Stdin = stdinRead + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + var stderr bytes.Buffer + cmd.Stderr = &stderr + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + ready := make(chan string, 1) + go func() { + line, _ := bufio.NewReader(stdout).ReadString('\n') + ready <- line + }() + select { + case line := <-ready: + require.Equal(t, "ready\n", line) + case <-time.After(5 * time.Second): + t.Fatal("provider shell did not start") + } + require.NoError(t, os.WriteFile(forcePath, nil, 0o600)) + require.Eventually(t, func() bool { + content, err := providerTestReadAck(ackPath) + return err == nil && string(content) == providerTestAckToken + }, 5*time.Second, 20*time.Millisecond) + require.NoError(t, cmd.Process.Kill()) + _ = cmd.Wait() + require.NotContains(t, stderr.String(), "finished") +} + +func TestProviderProcessSupervisorStartForceRace(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + supervisor := &providerProcessSupervisor{} + ready := make(chan struct{}) + var readyOnce sync.Once + var workers sync.WaitGroup + for range 8 { + workers.Go(func() { + cmd, startErr := supervisor.start(ctx, func() *exec.Cmd { + readyOnce.Do(func() { close(ready) }) + child := &exec.Cmd{ + Path: binary, + Args: []string{binary, "-test.run=^TestProviderExecHelper$"}, + } + child.Env = append( + os.Environ(), + "DEVSY_PROVIDER_EXEC_HELPER=1", + "DEVSY_PROVIDER_EXEC_EXIT=wait", + ) + child.Stdout = io.Discard + return child + }) + if startErr == nil { + _ = cmd.Wait() + supervisor.reaped(cmd) + } + }) + } + <-ready + supervisor.force(cancel) + workers.Wait() + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + require.True(t, supervisor.forced) + require.Empty(t, supervisor.children) +} + +func TestRunProviderShellPreexistingForceIsAcknowledged(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "exit 7") + require.NoError(t, os.WriteFile(forcePath, nil, 0o600)) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + output, err := cmd.CombinedOutput() + require.NoErrorf(t, err, "provider shell: %s", string(output)) + info, err := os.Lstat(ackPath) + require.NoError(t, err) + require.True(t, info.Mode().IsRegular()) + ack, err := providerTestReadAck(ackPath) + require.NoError(t, err) + require.Equal(t, providerTestAckToken, string(ack)) +} + +func TestRunProviderShellRejectsUnsafePaths(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "exit 0") + require.Error(t, runProviderShellWithToken(t, scriptPath, forcePath, scriptPath)) + require.Error( + t, + runProviderShellWithToken( + t, + scriptPath, + forcePath, + filepath.Join(t.TempDir(), "ack"), + ), + ) + if runtime.GOOS == providerWindowsOS { + require.NoError(t, os.Mkdir(ackPath, 0o700)) + } else { + require.NoError(t, os.Symlink(scriptPath, ackPath)) + } + require.Error(t, runProviderShellWithToken(t, scriptPath, forcePath, ackPath)) +} diff --git a/pkg/shell/provider_supervisor_unix_test.go b/pkg/shell/provider_supervisor_unix_test.go new file mode 100644 index 0000000000..0aa46431d7 --- /dev/null +++ b/pkg/shell/provider_supervisor_unix_test.go @@ -0,0 +1,41 @@ +//go:build !windows + +package shell + +import ( + "errors" + "fmt" + "os" + "syscall" + "testing" + + "github.com/stretchr/testify/require" +) + +func providerPrivateTestDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp(t.TempDir(), "private-") + require.NoError(t, err) + return dir +} + +func TestProviderPrivateTestDirMode(t *testing.T) { + dir := providerPrivateTestDir(t) + root, err := os.OpenRoot(dir) + require.NoError(t, err) + defer func() { _ = root.Close() }() + info, err := root.Stat(".") + require.NoError(t, err) + require.Equal(t, os.FileMode(0o700), info.Mode().Perm()) +} + +func providerChildReaped(pid int) error { + err := syscall.Kill(pid, 0) + if errors.Is(err, syscall.ESRCH) { + return nil + } + if err == nil { + return errors.New("owned child still exists after acknowledgement") + } + return fmt.Errorf("cannot check owned child after acknowledgement: %w", err) +} diff --git a/pkg/shell/provider_supervisor_windows_test.go b/pkg/shell/provider_supervisor_windows_test.go new file mode 100644 index 0000000000..bf8b63ed0e --- /dev/null +++ b/pkg/shell/provider_supervisor_windows_test.go @@ -0,0 +1,41 @@ +package shell + +import ( + "errors" + "os" + "path/filepath" + "testing" + "unsafe" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" +) + +func providerPrivateTestDir(t *testing.T) string { + t.Helper() + user, err := windows.GetCurrentProcessToken().GetTokenUser() + require.NoError(t, err) + sd, err := windows.SecurityDescriptorFromString( + "D:P(A;OICI;GA;;;" + user.User.Sid.String() + ")(A;OICI;GA;;;SY)") + require.NoError(t, err) + attributes := windows.SecurityAttributes{ + Length: uint32(unsafe.Sizeof(windows.SecurityAttributes{})), + SecurityDescriptor: sd, + } + path := filepath.Join(t.TempDir(), "private") + name, err := windows.UTF16PtrFromString(path) + require.NoError(t, err) + require.NoError(t, windows.CreateDirectory(name, &attributes)) + return path +} + +func providerChildReaped(pid int) error { + process, err := os.FindProcess(pid) + if err != nil { + return nil + } + if err := process.Kill(); err == nil { + return errors.New("owned child still exists after acknowledgement") + } + return nil +} diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx index 4bd0a348af..fd8b5a71f2 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx @@ -54,9 +54,9 @@ The `command`, `init`, `create`, `delete`, `start`, `stop`, `status` and `descri A provider without `create` is a non-machine provider and uses only `command` and `init`. The cancellation policy applies to both machine and non-machine providers. -With `exec.cancellation: file`, Devsy sets the reserved `DEVSY_PROVIDER_CANCEL_FILE` environment variable to a private, initially nonexistent marker path for each invocation. The provider must watch for the marker, stop admitting work, cancel its owned processes, finish cleanup, and exit. On caller cancellation, Devsy creates the marker and allows up to 10 seconds for cleanup before forcing the owned provider process to stop. Devsy waits for the invocation to finish and still returns the caller's cancellation error even if cleanup exits successfully. A provider must not set or override the reserved variable. +With `exec.cancellation: file`, Devsy sets the reserved `DEVSY_PROVIDER_CANCEL_FILE` environment variable to a private, initially nonexistent marker path for each invocation. The provider must watch for the marker, stop admitting work, cancel its owned processes, finish cleanup, and exit. On caller cancellation, Devsy creates the marker and allows up to 10 seconds for cleanup. For single-string shell commands that remain blocked, Devsy then asks its supervised shell to stop and reap its direct children before terminating the shell; it waits up to another 5 seconds for an authenticated acknowledgement before killing an unresponsive helper. A provider-writable acknowledgement file does not establish that cleanup finished: a missing or forged acknowledgement, including when the helper exits abruptly during the cleanup window, is reported as an unclean shutdown alongside the caller's cancellation error. Devsy never kills the provider's process group or unrelated daemon processes. It waits for the invocation to finish and still returns the caller's cancellation error even if cleanup exits successfully. A provider must not set or override the reserved variable. -File mode requires finite or nonblocking standard streams, direct file handles, or blocking streams whose owner unblocks them on cancellation of the original calling context. A provider must not assume that an arbitrary blocked reader or writer is interrupted by process cancellation. Machine command transports close their pipe endpoints on cancellation of the original context. Standard input can close independently of standard output, so providers must continue draining output after input reaches EOF. +File mode requires finite or nonblocking standard streams, direct file handles, or blocking streams whose owner unblocks them on cancellation of the original calling context. Devsy does not close caller-owned input or global standard input, and process cancellation cannot interrupt an arbitrary blocked reader or writer. Machine command transports close their pipe endpoints on cancellation of the original context. Standard input can close independently of standard output, so providers must continue draining output after input reaches EOF. On Windows, Devsy runs these scripts in an [emulated shell](https://github.com/mvdan/sh). The emulated shell has no `grep` or `sed`. Move that logic into a helper binary. From 9675a28aa23ce13c91dfda5693d690f8f5905295 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 2 Nov 2026 23:49:28 -0600 Subject: [PATCH 07/20] fix(provider): guard unreleased sandbox UI and stdin drain --- cmd/internal/agentworkspace/up_test.go | 53 ++++++++++++++++++- .../components/provider/ProviderWizard.svelte | 5 -- .../provider/ProviderWizard.test.ts | 43 --------------- .../provider_cancel_test.go | 50 +++++++++++++++-- .../clientimplementation/workspace_client.go | 7 ++- .../managing-providers/docker-sandbox.mdx | 12 +++-- .../managing-providers/manage-providers.mdx | 3 +- 7 files changed, 113 insertions(+), 60 deletions(-) diff --git a/cmd/internal/agentworkspace/up_test.go b/cmd/internal/agentworkspace/up_test.go index 412e531c66..6565ea7fd8 100644 --- a/cmd/internal/agentworkspace/up_test.go +++ b/cmd/internal/agentworkspace/up_test.go @@ -3,6 +3,7 @@ package agentworkspace import ( "context" "errors" + "io" "os" "path/filepath" "runtime" @@ -341,12 +342,62 @@ func writeDockerFixture(t *testing.T, bin, configuredPath string) { if name == "" { name = docker } + if runtime.GOOS == "windows" && !strings.HasSuffix(strings.ToLower(name), ".exe") { + name += ".exe" + } path := filepath.Join(bin, name) executable, err := os.Executable() if err != nil { t.Fatal(err) } - if err := os.Link(executable, path); err != nil { + if err := os.Link(executable, path); err == nil { + return + } + copyDockerFixture(t, executable, path) +} + +func copyDockerFixture(t *testing.T, executable, path string) { + t.Helper() + root, err := os.OpenRoot(filepath.Dir(executable)) + if err != nil { + t.Fatal(err) + } + source, err := root.Open(filepath.Base(executable)) + if err != nil { + _ = root.Close() + t.Fatal(err) + } + info, err := source.Stat() + if err != nil { + _ = source.Close() + _ = root.Close() + t.Fatal(err) + } + targetRoot, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + _ = source.Close() + _ = root.Close() + t.Fatal(err) + } + target, err := targetRoot.OpenFile( + filepath.Base(path), + os.O_WRONLY|os.O_CREATE|os.O_EXCL, + info.Mode().Perm()&0o700, + ) + if err != nil { + _ = source.Close() + _ = root.Close() + _ = targetRoot.Close() + t.Fatal(err) + } + _, copyErr := io.Copy(target, source) + if err := errors.Join( + copyErr, + source.Close(), + target.Close(), + root.Close(), + targetRoot.Close(), + ); err != nil { t.Fatal(err) } } diff --git a/desktop/src/renderer/src/lib/components/provider/ProviderWizard.svelte b/desktop/src/renderer/src/lib/components/provider/ProviderWizard.svelte index 7a574d6ed5..b23505da7d 100644 --- a/desktop/src/renderer/src/lib/components/provider/ProviderWizard.svelte +++ b/desktop/src/renderer/src/lib/components/provider/ProviderWizard.svelte @@ -56,11 +56,6 @@ const PRESETS: ProviderPreset[] = [ description: "Hardware-isolated microVMs", category: "Local", }, - { - name: "docker-sandbox", - description: "Devcontainers in Docker Sandboxes microVMs", - category: "Local", - }, { name: "lima", description: "Linux VMs via Lima (macOS)", diff --git a/desktop/src/renderer/src/lib/components/provider/ProviderWizard.test.ts b/desktop/src/renderer/src/lib/components/provider/ProviderWizard.test.ts index 41fb6c83ef..37f77e5fa5 100644 --- a/desktop/src/renderer/src/lib/components/provider/ProviderWizard.test.ts +++ b/desktop/src/renderer/src/lib/components/provider/ProviderWizard.test.ts @@ -312,49 +312,6 @@ describe("ProviderWizard", () => { }, ) - it.each([true, false])( - "configures the Docker Sandbox preset and preserves Reuse machine %s", - async (reuseMachine) => { - providerOptions.mockResolvedValue({ - SBX_CPUS: { name: "SBX_CPUS", default: "2" }, - }) - const page = renderWizard() - const preset = screen.getByRole("button", { - name: /docker-sandbox Devcontainers in Docker Sandboxes microVMs/, - }) - expect( - screen.getByRole("region", { name: "Local" }).contains(preset), - ).toBe(true) - await fireEvent.click(preset) - if (reuseMachine) - await fireEvent.click( - screen.getByRole("switch", { name: "Reuse machine" }), - ) - await fireEvent.click(screen.getByRole("button", { name: "Continue" })) - await fireEvent.input( - await screen.findByRole("textbox", { name: "SBX_CPUS" }), - { target: { value: "4" } }, - ) - await fireEvent.click(screen.getByRole("button", { name: "Continue" })) - await waitFor(() => { - expect(providerAdd).toHaveBeenCalledExactlyOnceWith( - "docker-sandbox", - undefined, - reuseMachine, - ) - expect(providerSetOptions).toHaveBeenCalledExactlyOnceWith( - "docker-sandbox", - { SBX_CPUS: "4" }, - reuseMachine, - ) - expect(providerInitStreaming).toHaveBeenCalledExactlyOnceWith( - "docker-sandbox", - ) - }) - page.unmount() - }, - ) - it("retries loading options without adding the provider a second time", async () => { providerOptions .mockRejectedValueOnce(new Error("temporary read failure")) diff --git a/pkg/client/clientimplementation/provider_cancel_test.go b/pkg/client/clientimplementation/provider_cancel_test.go index e225b00b9f..e80035e12a 100644 --- a/pkg/client/clientimplementation/provider_cancel_test.go +++ b/pkg/client/clientimplementation/provider_cancel_test.go @@ -60,15 +60,15 @@ func TestProviderCancellationHelper(t *testing.T) { if os.Getenv("DEVSY_CANCEL_HELPER") != "1" { return } + if os.Getenv("DEVSY_CANCEL_HOLDER") == "1" { + time.Sleep(time.Minute) + return + } runTestProviderCancellationBehavior(os.Getenv(config.EnvProviderCancelFile)) os.Exit(0) } func runTestProviderCancellationBehavior(marker string) { - if os.Getenv("DEVSY_CANCEL_HOLDER") == "1" { - time.Sleep(time.Minute) - return - } switch behavior := os.Getenv("DEVSY_CANCEL_BEHAVIOR"); behavior { case "acknowledge": runAcknowledgingCancellationHelper(marker) @@ -80,6 +80,8 @@ func runTestProviderCancellationBehavior(marker string) { writeTestCancellationBulkAfterEOF() case "orphan-output": startTestCancellationOutputHolder() + case "orphan-input": + startTestCancellationInputHolder() default: runTestCancellationExit(behavior, marker) } @@ -184,6 +186,21 @@ func startTestCancellationOutputHolder() { exitTestHelperOnError(err) } +func startTestCancellationInputHolder() { + holder := &exec.Cmd{Path: os.Args[0], Args: []string{os.Args[0], testCancellationHelperArg}} + holder.Env = append(os.Environ(), "DEVSY_CANCEL_HOLDER=1") + holder.Stdin = os.Stdin + exitTestHelperOnError(holder.Start()) + exitTestHelperOnError( + testHelperWriteFile( + os.Getenv("DEVSY_CANCEL_HOLDER_PID_PATH"), + strconv.Itoa(holder.Process.Pid), + ), + ) + exitTestHelperOnError(holder.Process.Release()) + writeTestCancellationBytes() +} + func exitTestHelperOnError(err error) { if err != nil { os.Exit(126) @@ -861,6 +878,31 @@ func TestProviderCancellationOwnedInputSuccessfulExitIgnoresEPIPE(t *testing.T) require.NoError(t, err) } +func TestProviderCancellationOwnedInputStopsWithInheritedStdin(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + holderPath := filepath.Join(t.TempDir(), "holder-pid") + t.Setenv("DEVSY_CANCEL_HOLDER_PID_PATH", holderPath) + t.Cleanup(func() { cleanupTestCancellationOutputHolder(holderPath) }) + input := io.NopCloser(bytes.NewReader(bytes.Repeat([]byte("x"), 2<<20))) + cmd := &exec.Cmd{Path: binary, Args: []string{binary, testCancellationHelperArg}} + cmd.Env = append(os.Environ(), "DEVSY_CANCEL_HELPER=1", "DEVSY_CANCEL_BEHAVIOR=orphan-input") + cmd.Stdin = input + var stdout bytes.Buffer + cmd.Stdout = &stdout + finished := make(chan error, 1) + go func() { + finished <- waitSupervisedProviderShellWithOwnedInput(context.Background(), cmd, input, "test") + }() + select { + case err := <-finished: + require.NoError(t, err) + require.Equal(t, []byte{0, 0xff, 0x80, '\r', '\n'}, stdout.Bytes()) + case <-time.After(5 * time.Second): + t.Fatal("provider completion waited for inherited stdin") + } +} + func TestProviderCancellationOwnedInputDrainsBinaryOutputAfterEOF(t *testing.T) { if runtime.GOOS == testWindowsOS { t.Skip("requires POSIX shell") diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index 385ca18ba1..a2f379830c 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -1144,6 +1144,7 @@ func waitSupervisedProviderShellWithOwnedInput( close(stopped) <-monitorDone closeErr := input.Close() + stdinCloseErr := closeProviderShellPipe(pipes.stdinWrite) stdinResult := <-stdinDone stdinErr := providerShellInputCopyError(stdinResult.copyErr, waitErr, closeErr) return errors.Join( @@ -1152,7 +1153,11 @@ func waitSupervisedProviderShellWithOwnedInput( providerShellCopyError(action, "stdout", stdoutErr), providerShellCopyError(action, "stderr", stderrErr), providerShellCopyError(action, "stdin", stdinErr), - providerShellCloseError(action, "stdin pipe", stdinResult.closeErr), + providerShellCloseError( + action, + "stdin pipe", + errors.Join(stdinCloseErr, stdinResult.closeErr), + ), providerShellCloseError(action, "stdio pipes", pipeCloseErr), providerShellCloseError(action, "input", closeErr), ) diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx index ef4be95f67..893f04d7d3 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx +++ b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx @@ -3,7 +3,9 @@ title: Docker Sandboxes sidebar_label: Docker Sandboxes --- -The Docker Sandboxes provider runs each Devsy machine in a local `sbx` microVM. The Devsy agent uses the private Docker Engine inside that VM to build and run ordinary devcontainers. A host Docker Engine is not required. +The planned Docker Sandboxes provider runs each Devsy machine in a local `sbx` microVM. The Devsy agent uses the private Docker Engine inside that VM to build and run ordinary devcontainers. A host Docker Engine is not required. + +No provider release exists yet. The installation and machine instructions below apply only after native qualification and publication. The short name cannot be installed before a release. ## Requirements @@ -11,14 +13,14 @@ Install the [standalone `sbx` CLI](https://docs.docker.com/ai/sandboxes/install/ Local Docker Sandboxes require Apple silicon with macOS 14 or later, Windows 11 on amd64 with Windows Hypervisor Platform, or Ubuntu 24.04 or later on amd64 or arm64 with accessible KVM. A VM running the provider also needs nested virtualization. A host system is supported only if the provider release lists that system as qualified. Docker Desktop and a host Docker Engine are not required. -Install a published provider release with: +After a qualified provider release is published, install it with: ```sh devsy provider add docker-sandbox devsy up https://github.com/devcontainers/samples --provider docker-sandbox ``` -Devsy Desktop offers the same provider under **Local**. Configure its options in the provider setup wizard. The provider does not install `sbx`, sign in, open a login browser, or change Docker Sandboxes settings. +The Devsy Desktop setup wizard does not list this provider before a qualified release. The provider does not install `sbx`, sign in, open a login browser, or change Docker Sandboxes settings. ## Machine resources @@ -46,7 +48,7 @@ Stopping a machine preserves its guest sources, images, containers, and volumes. Provider state is stored under your user configuration directory at `devsy-provider-docker-sandbox/state-v1`. If creation fails, do not remove that directory or manually reuse a sandbox name. The provider reports the exact machine key and any unresolved child attempt or backend operation. A lost VM is never silently recreated in the same machine generation. -Find the installed binary with `devsy provider list --result-format=json`. Read `config.binaries.DOCKER_SANDBOX_PROVIDER` for the asset name. Devsy stores it under its data directory at `contexts//providers//binaries/docker_sandbox_provider/`. With `DEVSY_HOME` set, that variable names the data directory. Otherwise Devsy uses `~/.devsy` on macOS and Linux, or `%LOCALAPPDATA%\devsy` on Windows. Use the resolved local executable, not the asset URL. +After installation, find the installed binary with `devsy provider list --result-format=json`. Read `config.binaries.DOCKER_SANDBOX_PROVIDER` for the asset name. Devsy stores it under its data directory at `contexts//providers//binaries/docker_sandbox_provider/`. With `DEVSY_HOME` set, that variable names the data directory. Otherwise Devsy uses `~/.devsy` on macOS and Linux, or `%LOCALAPPDATA%\devsy` on Windows. Use the resolved local executable, not the asset URL. The provider executable installed with the provider accepts: @@ -57,4 +59,4 @@ The provider executable installed with the provider accepts: delete --machine-key ``` -Use `doctor` to inspect the saved identity and backend observation without starting a stopped VM. An attempt acknowledgement is appropriate only after its orphaned client has terminated or the host has restarted. A backend acknowledgement is appropriate only after the Docker side operation has settled. Neither acknowledgement starts, stops, or removes a VM. Use scoped deletion after provider removal only for a machine you intend to destroy. Never use global `sbx` reset or prune to recover one machine. +Use `doctor` to inspect the saved identity and backend observation without starting a stopped VM. An attempt acknowledgement is appropriate only after its orphaned client has terminated or the host has restarted. A backend acknowledgement is appropriate only after the Docker side operation has settled. Neither acknowledgement starts, stops, or removes a VM. `devsy provider delete` removes the installed executable. Reinstall the same compatible release or preserve its executable before removal if you need scoped recovery afterward. Use scoped deletion only for a machine you intend to destroy. Never use global `sbx` reset or prune to recover one machine. diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx index 92d3df5a4e..ba78cf05a1 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx +++ b/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx @@ -11,7 +11,6 @@ Providers you can add: - [Harvester (harvester)](https://github.com/devsy-org/devsy-provider-harvester) - [Apple (apple)](https://github.com/devsy-org/devsy/tree/main/providers/apple) - [MicroSandbox (microsandbox)](https://github.com/devsy-org/devsy-provider-microsandbox) -- [Docker Sandboxes (docker-sandbox)](https://github.com/devsy-org/devsy-provider-docker-sandbox) - [Lima (lima)](https://github.com/devsy-org/devsy-provider-lima) - [OrbStack (orbstack)](https://github.com/devsy-org/devsy-provider-orbstack) - [SSH (ssh)](https://github.com/devsy-org/devsy-provider-ssh) @@ -20,6 +19,8 @@ Providers you can add: - [Azure (azure)](https://github.com/devsy-org/devsy-provider-azure) - [Digital Ocean (digitalocean)](https://github.com/devsy-org/devsy-provider-digitalocean) +Docker Sandboxes is not available yet. See its [qualification guide](./docker-sandbox.mdx) for the requirements of a future release. + List the available providers: ```sh From 88efca58de02929bf22cfaaa50077b38b337b4ec Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 00:35:36 -0600 Subject: [PATCH 08/20] fix(provider): reap background shell children on completion --- pkg/shell/provider_supervisor.go | 51 +++++---------- pkg/shell/provider_supervisor_test.go | 62 +++++++++++++++++++ .../docs/developing-providers/quickstart.mdx | 2 + .../managing-providers/docker-sandbox.mdx | 2 +- 4 files changed, 81 insertions(+), 36 deletions(-) diff --git a/pkg/shell/provider_supervisor.go b/pkg/shell/provider_supervisor.go index 307057aa0f..acc5f41fe2 100644 --- a/pkg/shell/provider_supervisor.go +++ b/pkg/shell/provider_supervisor.go @@ -28,7 +28,10 @@ func (s *providerProcessSupervisor) start( ) (*exec.Cmd, error) { s.mu.Lock() defer s.mu.Unlock() - if s.forced || ctx.Err() != nil { + if s.forced { + return nil, context.Canceled + } + if ctx.Err() != nil { return nil, context.Cause(ctx) } cmd, err := startProviderExecRetry(ctx, newCommand) @@ -69,19 +72,20 @@ func (s *providerProcessSupervisor) reaped(cmd *exec.Cmd) { s.mu.Unlock() } -func (s *providerProcessSupervisor) settled() <-chan struct{} { - s.mu.Lock() - defer s.mu.Unlock() - for _, done := range s.children { - return done - } - return nil +func (s *providerProcessSupervisor) force(cancel context.CancelFunc) { + s.terminate(cancel) } -func (s *providerProcessSupervisor) force(cancel context.CancelFunc) { +func (s *providerProcessSupervisor) stop() { + s.terminate(nil) +} + +func (s *providerProcessSupervisor) terminate(cancel context.CancelFunc) { s.mu.Lock() s.forced = true - cancel() + if cancel != nil { + cancel() + } children := make([]*exec.Cmd, 0, len(s.children)) for cmd := range s.children { children = append(children, cmd) @@ -244,7 +248,6 @@ func errStringKind(err error) string { } type providerShellExecution struct { - ctx context.Context dir string token string root *os.Root @@ -288,7 +291,6 @@ func RunProviderShell(ctx context.Context, scriptPath, forcePath, ackPath string }) }() execution := providerShellExecution{ - ctx: ctx, dir: dir, root: root, token: token, @@ -336,17 +338,16 @@ func (p *providerShellExecution) force(pollErr error) error { func (p *providerShellExecution) completed(err error) error { forced, pollErr := providerForcePresent(p.dir) if pollErr != nil { + p.supervisor.force(p.cancel) return pollErr } if forced { return p.canceled() } + p.supervisor.stop() if earlyErr := providerShellEarlyError(err); earlyErr != nil { return earlyErr } - if err := p.waitForChildren(); err != nil { - return err - } if ackErr := providerWriteAck(p.root, p.token); ackErr != nil { return ackErr } @@ -375,23 +376,3 @@ func (p *providerShellExecution) canceled() error { } return context.Canceled } - -func (p *providerShellExecution) waitForChildren() error { - for done := p.supervisor.settled(); done != nil; done = p.supervisor.settled() { - select { - case <-done: - case <-p.ctx.Done(): - return p.canceled() - case <-time.After(100 * time.Millisecond): - forced, pollErr := providerForcePresent(p.dir) - if pollErr != nil { - p.supervisor.force(p.cancel) - return pollErr - } - if forced { - return p.canceled() - } - } - } - return nil -} diff --git a/pkg/shell/provider_supervisor_test.go b/pkg/shell/provider_supervisor_test.go index 20af429000..8af918b18a 100644 --- a/pkg/shell/provider_supervisor_test.go +++ b/pkg/shell/provider_supervisor_test.go @@ -303,6 +303,68 @@ func TestRunProviderShellNormalExitStatuses(t *testing.T) { } } +func TestRunProviderShellNormalCompletionReapsBackgroundChild(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + command := strconv.Quote(binary) + " -test.run=^TestProviderShellTrackedChild$" + if runtime.GOOS == providerWindowsOS { + command = strings.ReplaceAll(command, `\`, `/`) + } + scriptPath, forcePath, ackPath := providerShellFixture(t, "") + pidPath := filepath.Join(filepath.Dir(scriptPath), "pid") + script := "export DEVSY_PROVIDER_SHELL_CHILD=1 DEVSY_PROVIDER_CHILD_PID=" + + strconv.Quote(pidPath) + "; " + command + + ` & while [ ! -s "$DEVSY_PROVIDER_CHILD_PID" ]; do :; done; printf finished` + require.NoError(t, os.WriteFile(scriptPath, []byte(script), 0o600)) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + cleanupProviderShellChild(pidPath) + }) + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case err := <-done: + require.NoError(t, err, stderr.String()) + case <-time.After(5 * time.Second): + t.Fatal("normal provider completion waited for a background child") + } + require.Contains(t, stdout.String(), "finished") + ack, err := providerTestReadAck(ackPath) + require.NoError(t, err) + require.Equal(t, providerTestAckToken, string(ack)) + root, err := os.OpenRoot(filepath.Dir(pidPath)) + require.NoError(t, err) + pidBytes, err := root.ReadFile("pid") + require.NoError(t, errors.Join(err, root.Close())) + pid, err := strconv.Atoi(string(pidBytes)) + require.NoError(t, err) + require.NoError(t, providerChildReaped(pid)) +} + +func cleanupProviderShellChild(path string) { + root, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + return + } + pidBytes, readErr := root.ReadFile("pid") + _ = root.Close() + if readErr != nil { + return + } + pid, err := strconv.Atoi(string(pidBytes)) + if err != nil { + return + } + process, err := os.FindProcess(pid) + if err == nil { + _ = process.Kill() + } +} + func TestRunProviderShellRawStreams(t *testing.T) { binary, err := os.Executable() require.NoError(t, err) diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx index fd8b5a71f2..c100d3efbc 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx @@ -56,6 +56,8 @@ A provider without `create` is a non-machine provider and uses only `command` an With `exec.cancellation: file`, Devsy sets the reserved `DEVSY_PROVIDER_CANCEL_FILE` environment variable to a private, initially nonexistent marker path for each invocation. The provider must watch for the marker, stop admitting work, cancel its owned processes, finish cleanup, and exit. On caller cancellation, Devsy creates the marker and allows up to 10 seconds for cleanup. For single-string shell commands that remain blocked, Devsy then asks its supervised shell to stop and reap its direct children before terminating the shell; it waits up to another 5 seconds for an authenticated acknowledgement before killing an unresponsive helper. A provider-writable acknowledgement file does not establish that cleanup finished: a missing or forged acknowledgement, including when the helper exits abruptly during the cleanup window, is reported as an unclean shutdown alongside the caller's cancellation error. Devsy never kills the provider's process group or unrelated daemon processes. It waits for the invocation to finish and still returns the caller's cancellation error even if cleanup exits successfully. A provider must not set or override the reserved variable. +For single-string commands, the supervisor also stops and reaps direct children still running when the shell script completes normally. A background command started with `&` cannot outlive its invocation. Start durable services through the backend's explicit detached operation, not a shell background job. + File mode requires finite or nonblocking standard streams, direct file handles, or blocking streams whose owner unblocks them on cancellation of the original calling context. Devsy does not close caller-owned input or global standard input, and process cancellation cannot interrupt an arbitrary blocked reader or writer. Machine command transports close their pipe endpoints on cancellation of the original context. Standard input can close independently of standard output, so providers must continue draining output after input reaches EOF. On Windows, Devsy runs these scripts in an [emulated shell](https://github.com/mvdan/sh). The emulated shell has no `grep` or `sed`. Move that logic into a helper binary. diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx index 893f04d7d3..bb494298ee 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx +++ b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx @@ -48,7 +48,7 @@ Stopping a machine preserves its guest sources, images, containers, and volumes. Provider state is stored under your user configuration directory at `devsy-provider-docker-sandbox/state-v1`. If creation fails, do not remove that directory or manually reuse a sandbox name. The provider reports the exact machine key and any unresolved child attempt or backend operation. A lost VM is never silently recreated in the same machine generation. -After installation, find the installed binary with `devsy provider list --result-format=json`. Read `config.binaries.DOCKER_SANDBOX_PROVIDER` for the asset name. Devsy stores it under its data directory at `contexts//providers//binaries/docker_sandbox_provider/`. With `DEVSY_HOME` set, that variable names the data directory. Otherwise Devsy uses `~/.devsy` on macOS and Linux, or `%LOCALAPPDATA%\devsy` on Windows. Use the resolved local executable, not the asset URL. +After installation, run `devsy provider list --result-format=json`. In `config.binaries.DOCKER_SANDBOX_PROVIDER`, select the record whose `os` and `arch` match your host. An absolute `path` is the executable. For a remote archive, use `archivePath` under Devsy's data directory at `contexts//providers//binaries/docker_sandbox_provider/`. Otherwise, use the record's `name` in that directory. If `name` is empty, use the base name of `path` instead. With `DEVSY_HOME` set, that variable names the data directory. Otherwise Devsy uses `~/.devsy` on macOS and Linux, or `%LOCALAPPDATA%\devsy` on Windows. Use the resolved local executable, not the asset URL. The provider executable installed with the provider accepts: From e4a307c8d265ec5b6ef41ac165b054f4a3845b66 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 02:36:32 -0600 Subject: [PATCH 09/20] fix(provider): preserve foreground signals in supervised shell --- cmd/internal/provider_shell.go | 16 +- cmd/internal/provider_shell_test.go | 5 +- .../provider_cancel_test.go | 24 ++- .../provider_shell_process_unix.go | 16 ++ .../provider_shell_process_unix_test.go | 148 ++++++++++++++++++ .../provider_shell_process_windows.go | 15 ++ .../provider_shell_process_windows_test.go | 19 +++ .../clientimplementation/workspace_client.go | 16 +- pkg/shell/provider_exec_windows.go | 73 ++++++++- pkg/shell/provider_foreground_unix.go | 51 ++++++ pkg/shell/provider_foreground_windows.go | 55 +++++++ pkg/shell/provider_supervisor.go | 57 ++++++- pkg/shell/provider_supervisor_test.go | 37 ++++- pkg/shell/provider_supervisor_windows_test.go | 46 ++++++ .../managing-providers/docker-sandbox.mdx | 2 +- 15 files changed, 550 insertions(+), 30 deletions(-) create mode 100644 pkg/client/clientimplementation/provider_shell_process_unix.go create mode 100644 pkg/client/clientimplementation/provider_shell_process_unix_test.go create mode 100644 pkg/client/clientimplementation/provider_shell_process_windows.go create mode 100644 pkg/client/clientimplementation/provider_shell_process_windows_test.go create mode 100644 pkg/shell/provider_foreground_unix.go create mode 100644 pkg/shell/provider_foreground_windows.go diff --git a/cmd/internal/provider_shell.go b/cmd/internal/provider_shell.go index 22d301f02f..8a1737931a 100644 --- a/cmd/internal/provider_shell.go +++ b/cmd/internal/provider_shell.go @@ -12,10 +12,21 @@ import ( func NewProviderShellCmd() *cobra.Command { var scriptPath, forcePath, ackPath string + var parentPID int command := &cobra.Command{ Use: "provider-shell", Hidden: true, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - err := shell.RunProviderShell(cmd.Context(), scriptPath, forcePath, ackPath) + cleanup, err := shell.PrepareProviderShellForeground(parentPID) + if err != nil { + return err + } + defer cleanup() + err = shell.RunProviderShell(cmd.Context(), shell.ProviderShellInvocation{ + ScriptPath: scriptPath, + ForcePath: forcePath, + AckPath: ackPath, + ParentPID: parentPID, + }) if status, ok := errors.AsType[interp.ExitStatus](err); ok { os.Exit(int(status)) } @@ -28,6 +39,9 @@ func NewProviderShellCmd() *cobra.Command { command.Flags().StringVar(&scriptPath, "script-file", "", "Private provider script file") command.Flags().StringVar(&forcePath, "force-file", "", "Private provider force marker") command.Flags().StringVar(&ackPath, "ack-file", "", "Private provider acknowledgement marker") + command.Flags().IntVar(&parentPID, "parent-pid", 0, "Provider parent process ID") + _ = command.Flags().MarkHidden("parent-pid") + _ = command.MarkFlagRequired("parent-pid") _ = command.MarkFlagRequired("script-file") _ = command.MarkFlagRequired("force-file") _ = command.MarkFlagRequired("ack-file") diff --git a/cmd/internal/provider_shell_test.go b/cmd/internal/provider_shell_test.go index 29f2803d78..fbd3c6713d 100644 --- a/cmd/internal/provider_shell_test.go +++ b/cmd/internal/provider_shell_test.go @@ -3,6 +3,7 @@ package cmdinternal import ( "os" "path/filepath" + "strconv" "strings" "testing" @@ -20,7 +21,7 @@ func TestProviderShellCommandRegistration(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(dir, "ack-key"), []byte(token), 0o600)) command.SetArgs([]string{ "--script-file", script, "--force-file", filepath.Join(dir, "force"), - "--ack-file", filepath.Join(dir, "ack"), + "--ack-file", filepath.Join(dir, "ack"), "--parent-pid", strconv.Itoa(os.Getppid()), }) require.NoError(t, command.Execute()) _, err := os.Lstat(filepath.Join(dir, "ack-key")) @@ -43,7 +44,7 @@ func TestProviderShellCommandRejectsMissingToken(t *testing.T) { command := NewProviderShellCmd() command.SetArgs([]string{ "--script-file", script, "--force-file", filepath.Join(dir, "force"), - "--ack-file", filepath.Join(dir, "ack"), + "--ack-file", filepath.Join(dir, "ack"), "--parent-pid", strconv.Itoa(os.Getppid()), }) require.ErrorContains(t, command.Execute(), "invalid provider shell acknowledgement token") _, err = os.Lstat(filepath.Join(dir, "ack")) diff --git a/pkg/client/clientimplementation/provider_cancel_test.go b/pkg/client/clientimplementation/provider_cancel_test.go index e80035e12a..4dd9993a85 100644 --- a/pkg/client/clientimplementation/provider_cancel_test.go +++ b/pkg/client/clientimplementation/provider_cancel_test.go @@ -28,17 +28,28 @@ import ( ) func TestMain(m *testing.M) { - if len(os.Args) == 9 && os.Args[1] == "internal" && os.Args[2] == "provider-shell" { + if len(os.Args) == 11 && os.Args[1] == "internal" && os.Args[2] == "provider-shell" { os.Exit(runProviderShellTestHelper(os.Args)) } os.Exit(m.Run()) } func runProviderShellTestHelper(args []string) int { - if args[3] != "--script-file" || args[5] != "--force-file" || args[7] != "--ack-file" { + if !validProviderShellTestArgs(args) { return 1 } - err := shell.RunProviderShell(context.Background(), args[4], args[6], args[8]) + parentPID, err := strconv.Atoi(args[10]) + if err != nil { + return 1 + } + cleanup, err := shell.PrepareProviderShellForeground(parentPID) + if err != nil { + return 1 + } + defer cleanup() + err = shell.RunProviderShell(context.Background(), shell.ProviderShellInvocation{ + ScriptPath: args[4], ForcePath: args[6], AckPath: args[8], ParentPID: parentPID, + }) if err == nil { return 0 } @@ -48,6 +59,11 @@ func runProviderShellTestHelper(args []string) int { return 1 } +func validProviderShellTestArgs(args []string) bool { + return args[3] == "--script-file" && args[5] == "--force-file" && + args[7] == "--ack-file" && args[9] == "--parent-pid" +} + const ( testTransportRead = "transport-read" testWindowsOS = "windows" @@ -1011,7 +1027,7 @@ func TestProviderCancellationOwnedOutputStopsAfterSuccessfulShellExit(t *testing start := time.Now() err = RunCommand(context.Background(), opts) require.ErrorIs(t, err, exec.ErrWaitDelay) - require.Less(t, time.Since(start), 5*time.Second) + require.Less(t, time.Since(start), 20*time.Second) } func cleanupTestCancellationOutputHolder(path string) { diff --git a/pkg/client/clientimplementation/provider_shell_process_unix.go b/pkg/client/clientimplementation/provider_shell_process_unix.go new file mode 100644 index 0000000000..df658b9036 --- /dev/null +++ b/pkg/client/clientimplementation/provider_shell_process_unix.go @@ -0,0 +1,16 @@ +//go:build !windows + +package clientimplementation + +import ( + "os/exec" + "syscall" +) + +func configureProviderShellProcess(cmd *exec.Cmd) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.Setpgid = true + cmd.SysProcAttr.Pgid = 0 +} diff --git a/pkg/client/clientimplementation/provider_shell_process_unix_test.go b/pkg/client/clientimplementation/provider_shell_process_unix_test.go new file mode 100644 index 0000000000..11e0ecdfa1 --- /dev/null +++ b/pkg/client/clientimplementation/provider_shell_process_unix_test.go @@ -0,0 +1,148 @@ +//go:build !windows + +package clientimplementation + +import ( + "bufio" + "bytes" + "context" + "os" + "os/exec" + "os/signal" + "strconv" + "syscall" + "testing" + "time" + + "github.com/creack/pty" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/types" + "github.com/stretchr/testify/require" + "mvdan.cc/sh/v3/interp" +) + +func TestProviderShellProcessIsolation(t *testing.T) { + dir, err := createProviderInvocationDir() + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + files, err := prepareProviderShell(dir, "printf 'ready\\n'; read value") + require.NoError(t, err) + binary, err := os.Executable() + require.NoError(t, err) + cmd := &exec.Cmd{ + Path: binary, + Args: []string{ + binary, "internal", "provider-shell", + "--script-file", files.script, "--force-file", files.force, "--ack-file", files.ack, + "--parent-pid", strconv.Itoa(os.Getpid()), + }, + } + input, err := cmd.StdinPipe() + require.NoError(t, err) + defer func() { _ = input.Close() }() + output, err := cmd.StdoutPipe() + require.NoError(t, err) + configureProviderShellProcess(cmd) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { _ = cmd.Process.Kill() }) + ready := make(chan bool, 1) + go func() { ready <- bufio.NewScanner(output).Scan() }() + select { + case ok := <-ready: + require.True(t, ok) + case <-time.After(5 * time.Second): + t.Fatal("helper did not become ready") + } + group, err := syscall.Getpgid(cmd.Process.Pid) + require.NoError(t, err) + require.Equal(t, syscall.Getpgrp(), group) + require.NoError(t, cmd.Process.Signal(syscall.SIGINT)) + require.NoError(t, publishProviderMarker(files.force)) + require.Eventually(t, func() bool { + ack, err := os.ReadFile(files.ack) + return err == nil && string(ack) == files.ackToken + }, 5*time.Second, 20*time.Millisecond) + _ = cmd.Process.Kill() + _ = cmd.Wait() +} + +func TestProviderShellPTYController(t *testing.T) { + if os.Getenv("DEVSY_PROVIDER_PTY_CONTROLLER") != "1" { + return + } + signal.Ignore(syscall.SIGHUP) + defer signal.Reset(syscall.SIGHUP) + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{`read value; printf 'got:%s\n' "$value"`}, + Environ: os.Environ(), + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + RawStdout: true, + Cancellation: provider.ProviderCancellationFile, + }) + require.NoError(t, err) +} + +func TestProviderShellReadsControllingTerminal(t *testing.T) { + master, slave, err := pty.Open() + require.NoError(t, err) + defer func() { _ = master.Close() }() + defer func() { _ = slave.Close() }() + binary, err := os.Executable() + require.NoError(t, err) + controller := &exec.Cmd{ + Path: binary, + Args: []string{binary, "-test.run=^TestProviderShellPTYController$"}, + Env: append(os.Environ(), "DEVSY_PROVIDER_PTY_CONTROLLER=1"), + Stdin: slave, Stderr: os.Stderr, + SysProcAttr: &syscall.SysProcAttr{Setsid: true, Setctty: true, Ctty: 0}, + } + output, err := controller.StdoutPipe() + require.NoError(t, err) + require.NoError(t, controller.Start()) + t.Cleanup(func() { _ = controller.Process.Kill() }) + _, err = master.Write([]byte("hello\n")) + require.NoError(t, err) + lines := make(chan []string, 1) + go func() { + reader := bufio.NewReader(output) + first, _ := reader.ReadString('\n') + second, _ := reader.ReadString('\n') + lines <- []string{first, second} + }() + select { + case got := <-lines: + require.Equal(t, []string{"got:hello\n", "PASS\n"}, got) + case <-time.After(8 * time.Second): + t.Fatal("provider helper could not read the controlling terminal") + } + require.NoError(t, master.Close()) + require.NoError(t, slave.Close()) + done := make(chan error, 1) + go func() { done <- controller.Wait() }() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("provider controller did not finish after terminal closure") + } +} + +func TestProviderShellForegroundPreservesChildSIGINT(t *testing.T) { + if signal.Ignored(syscall.SIGINT) { + t.Skip("parent already ignores SIGINT") + } + var stdout bytes.Buffer + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{`/bin/sh -c 'kill -INT $$; printf survived'`}, + Environ: os.Environ(), + Stdout: &stdout, + RawStdout: true, + Cancellation: provider.ProviderCancellationFile, + }) + var status interp.ExitStatus + require.ErrorAs(t, err, &status) + require.Equal(t, 130, int(status)) + require.NotContains(t, stdout.String(), "survived") +} diff --git a/pkg/client/clientimplementation/provider_shell_process_windows.go b/pkg/client/clientimplementation/provider_shell_process_windows.go new file mode 100644 index 0000000000..d7bcb2d6e7 --- /dev/null +++ b/pkg/client/clientimplementation/provider_shell_process_windows.go @@ -0,0 +1,15 @@ +package clientimplementation + +import ( + "os/exec" + "syscall" + + "golang.org/x/sys/windows" +) + +func configureProviderShellProcess(cmd *exec.Cmd) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.CreationFlags |= windows.CREATE_NEW_PROCESS_GROUP +} diff --git a/pkg/client/clientimplementation/provider_shell_process_windows_test.go b/pkg/client/clientimplementation/provider_shell_process_windows_test.go new file mode 100644 index 0000000000..ae0e6d3605 --- /dev/null +++ b/pkg/client/clientimplementation/provider_shell_process_windows_test.go @@ -0,0 +1,19 @@ +package clientimplementation + +import ( + "os/exec" + "syscall" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" +) + +func TestProviderShellProcessConsoleIsolation(t *testing.T) { + cmd := &exec.Cmd{ + SysProcAttr: &syscall.SysProcAttr{CreationFlags: windows.CREATE_UNICODE_ENVIRONMENT}, + } + configureProviderShellProcess(cmd) + require.NotZero(t, cmd.SysProcAttr.CreationFlags&windows.CREATE_NEW_PROCESS_GROUP) + require.NotZero(t, cmd.SysProcAttr.CreationFlags&windows.CREATE_UNICODE_ENVIRONMENT) +} diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index a2f379830c..77d45e7e55 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -10,6 +10,7 @@ import ( "net" "os" "os/exec" + "strconv" "strings" "sync" "sync/atomic" @@ -917,8 +918,17 @@ func runSupervisedProviderShell( cmd := &exec.Cmd{ Path: binary, Args: []string{ - binary, "internal", "provider-shell", - "--script-file", files.script, "--force-file", files.force, "--ack-file", files.ack, + binary, + "internal", + "provider-shell", + "--script-file", + files.script, + "--force-file", + files.force, + "--ack-file", + files.ack, + "--parent-pid", + strconv.Itoa(os.Getpid()), }, Env: opts.Environ, Stdin: opts.Stdin, @@ -1127,6 +1137,7 @@ func waitSupervisedProviderShellWithOwnedInput( source, stdout, stderr := cmd.Stdin, cmd.Stdout, cmd.Stderr cmd.Stdin, cmd.Stdout, cmd.Stderr = pipes.stdinRead, pipes.stdoutWrite, pipes.stderrWrite + configureProviderShellProcess(cmd) if err := cmd.Start(); err != nil { return fmt.Errorf("start provider %s shell: %w", action, err) } @@ -1291,6 +1302,7 @@ func closeUndrainedProviderShellOutput( } func waitSupervisedProviderShell(ctx context.Context, cmd *exec.Cmd, action string) error { + configureProviderShellProcess(cmd) if err := cmd.Start(); err != nil { return fmt.Errorf("start provider %s shell: %w", action, err) } diff --git a/pkg/shell/provider_exec_windows.go b/pkg/shell/provider_exec_windows.go index f8ff1925ad..9f7ed541c9 100644 --- a/pkg/shell/provider_exec_windows.go +++ b/pkg/shell/provider_exec_windows.go @@ -3,7 +3,7 @@ package shell import ( "os" "os/exec" - "strings" + "unsafe" "golang.org/x/sys/windows" "mvdan.cc/sh/v3/interp" @@ -29,11 +29,72 @@ func providerPrivateDirectory(path string, info os.FileInfo) bool { if err != nil { return false } - sddl := sd.String() - return strings.Count(sddl, "(") == 2 && - strings.Contains(sddl, "(A;OICI;GA;;;"+user.User.Sid.String()+")") && - (strings.Contains(sddl, "(A;OICI;GA;;;SY)") || - strings.Contains(sddl, "(A;OICI;GA;;;S-1-5-18)")) + return providerPrivateDACL(sd, user.User.Sid) +} + +func providerPrivateDACL(sd *windows.SECURITY_DESCRIPTOR, user *windows.SID) bool { + control, _, err := sd.Control() + if err != nil || control&windows.SE_DACL_PROTECTED == 0 { + return false + } + acl, _, err := sd.DACL() + if err != nil || acl == nil { + return false + } + system, err := windows.CreateWellKnownSid(windows.WinLocalSystemSid) + if err != nil { + return false + } + getAce := windows.NewLazySystemDLL("advapi32.dll").NewProc("GetAce") + var directory, descendants [2]uint32 + for i := uint32(0); i < uint32(acl.AceCount); i++ { + var ptr unsafe.Pointer + ok, _, _ := getAce.Call( + uintptr(unsafe.Pointer(acl)), + uintptr(i), + uintptr(unsafe.Pointer(&ptr)), + ) + if ok == 0 || ptr == nil { + return false + } + ace := (*windows.ACCESS_ALLOWED_ACE)(ptr) + flags := ace.Header.AceFlags + if ace.Header.AceType != windows.ACCESS_ALLOWED_ACE_TYPE || + flags & ^uint8( + windows.OBJECT_INHERIT_ACE|windows.CONTAINER_INHERIT_ACE|windows.INHERIT_ONLY_ACE, + ) != 0 { + return false + } + sid := (*windows.SID)(unsafe.Pointer(&ace.SidStart)) + index := 0 + if sid.Equals(system) { + index = 1 + } else if !sid.Equals(user) { + return false + } + mask := uint32(ace.Mask) + const genericAll = uint32(0x10000000) + const fileAll = uint32(0x001f01ff) + if mask & ^uint32(genericAll|fileAll) != 0 { + return false + } + if mask&genericAll != 0 { + mask |= fileAll + } + if flags&windows.INHERIT_ONLY_ACE == 0 { + directory[index] |= mask + } + if flags&(windows.OBJECT_INHERIT_ACE|windows.CONTAINER_INHERIT_ACE) == windows.OBJECT_INHERIT_ACE|windows.CONTAINER_INHERIT_ACE { + descendants[index] |= mask + } + } + const fileAll = uint32(0x001f01ff) + if user.Equals(system) { + return directory[1]&fileAll == fileAll && descendants[1]&fileAll == fileAll + } + return directory[0]&fileAll == fileAll && directory[1]&fileAll == fileAll && + descendants[0]&fileAll == fileAll && + descendants[1]&fileAll == fileAll } func providerPrivateScript(info os.FileInfo) bool { diff --git a/pkg/shell/provider_foreground_unix.go b/pkg/shell/provider_foreground_unix.go new file mode 100644 index 0000000000..ee5df397f4 --- /dev/null +++ b/pkg/shell/provider_foreground_unix.go @@ -0,0 +1,51 @@ +//go:build !windows + +package shell + +import ( + "fmt" + "os" + "os/signal" + "syscall" +) + +func PrepareProviderShellForeground(parentPID int) (cleanup func(), err error) { + if parentPID <= 0 || os.Getppid() != parentPID { + return nil, fmt.Errorf("provider shell parent mismatch") + } + signals := providerTerminalSignals() + ch := make(chan os.Signal, 3) + cleanup = func() { signal.Stop(ch) } + if len(signals) > 0 { + signal.Notify(ch, signals...) + } + group, err := syscall.Getpgid(parentPID) + if err == nil { + err = syscall.Setpgid(0, group) + } + if err == nil && os.Getppid() != parentPID { + err = fmt.Errorf("provider shell parent mismatch") + } + if err != nil { + cleanup() + return nil, err + } + return cleanup, nil +} + +func providerTerminalSignals() []os.Signal { + var signals []os.Signal + for _, sig := range []os.Signal{syscall.SIGINT, syscall.SIGQUIT, syscall.SIGHUP} { + if !signal.Ignored(sig) { + signals = append(signals, sig) + } + } + return signals +} + +func newProviderShellParentMonitor(parentPID int) (alive func() bool, cleanup func(), err error) { + if parentPID <= 0 || os.Getppid() != parentPID { + return nil, nil, fmt.Errorf("provider shell parent mismatch") + } + return func() bool { return os.Getppid() == parentPID }, func() {}, nil +} diff --git a/pkg/shell/provider_foreground_windows.go b/pkg/shell/provider_foreground_windows.go new file mode 100644 index 0000000000..31b94cc34e --- /dev/null +++ b/pkg/shell/provider_foreground_windows.go @@ -0,0 +1,55 @@ +package shell + +import ( + "fmt" + "os" + "syscall" + + "golang.org/x/sys/windows" +) + +var ( + providerKernel32 = windows.NewLazySystemDLL("kernel32.dll") + providerSetConsoleCtrlHandler = providerKernel32.NewProc("SetConsoleCtrlHandler") + providerGetConsoleCP = providerKernel32.NewProc("GetConsoleCP") + providerConsoleHandler = syscall.NewCallback(func(event uint32) uintptr { + if event == 0 || event == 1 { + return 1 + } + return 0 + }) +) + +func PrepareProviderShellForeground(parentPID int) (func(), error) { + if parentPID <= 0 || uint64(parentPID) > uint64(^uint32(0)) || os.Getppid() != parentPID { + return nil, fmt.Errorf("provider shell parent mismatch") + } + cp, _, _ := providerGetConsoleCP.Call() + if cp == 0 { + return func() {}, nil + } + ok, _, err := providerSetConsoleCtrlHandler.Call(providerConsoleHandler, 1) + if ok == 0 { + return nil, fmt.Errorf("register provider console handler: %w", err) + } + ok, _, err = providerSetConsoleCtrlHandler.Call(0, 0) + if ok == 0 { + providerSetConsoleCtrlHandler.Call(providerConsoleHandler, 0) + return nil, fmt.Errorf("enable child console interrupts: %w", err) + } + return func() { providerSetConsoleCtrlHandler.Call(providerConsoleHandler, 0) }, nil +} + +func newProviderShellParentMonitor(parentPID int) (func() bool, func(), error) { + if parentPID <= 0 || uint64(parentPID) > uint64(^uint32(0)) { + return nil, nil, fmt.Errorf("invalid parent PID %d", parentPID) + } + handle, err := windows.OpenProcess(windows.SYNCHRONIZE, false, uint32(parentPID)) + if err != nil { + return nil, nil, fmt.Errorf("open parent process: %w", err) + } + return func() bool { + result, err := windows.WaitForSingleObject(handle, 0) + return err == nil && result == uint32(windows.WAIT_TIMEOUT) + }, func() { _ = windows.CloseHandle(handle) }, nil +} diff --git a/pkg/shell/provider_supervisor.go b/pkg/shell/provider_supervisor.go index acc5f41fe2..efa96c3933 100644 --- a/pkg/shell/provider_supervisor.go +++ b/pkg/shell/provider_supervisor.go @@ -254,16 +254,28 @@ type providerShellExecution struct { result <-chan error supervisor *providerProcessSupervisor cancel context.CancelFunc + alive func() bool } -func RunProviderShell(ctx context.Context, scriptPath, forcePath, ackPath string) error { +type ProviderShellInvocation struct { + ScriptPath string + ForcePath string + AckPath string + ParentPID int +} + +func RunProviderShell(ctx context.Context, invocation ProviderShellInvocation) error { if ctx.Err() != nil { return context.Cause(ctx) } - if err := providerShellPaths(scriptPath, forcePath, ackPath); err != nil { + if err := providerShellPaths( + invocation.ScriptPath, + invocation.ForcePath, + invocation.AckPath, + ); err != nil { return err } - dir := filepath.Dir(scriptPath) + dir := filepath.Dir(invocation.ScriptPath) root, err := providerShellRoot(dir) if err != nil { return err @@ -273,12 +285,17 @@ func RunProviderShell(ctx context.Context, scriptPath, forcePath, ackPath string if err != nil { return err } - if err := providerShellFiles(root); err != nil { + content, err := readProviderShellContent(root) + if err != nil { return err } - content, err := root.ReadFile("script") + alive, cleanup, err := newProviderShellParentMonitor(invocation.ParentPID) if err != nil { - return errors.New("read provider shell script: " + errStringKind(err)) + return err + } + defer cleanup() + if !alive() { + return context.Canceled } executionCtx, cancel := context.WithCancel(ctx) defer cancel() @@ -297,16 +314,32 @@ func RunProviderShell(ctx context.Context, scriptPath, forcePath, ackPath string result: result, supervisor: supervisor, cancel: cancel, + alive: alive, } return execution.wait(ctx) } +func readProviderShellContent(root *os.Root) ([]byte, error) { + if err := providerShellFiles(root); err != nil { + return nil, err + } + content, err := root.ReadFile("script") + if err != nil { + return nil, errors.New("read provider shell script: " + errStringKind(err)) + } + return content, nil +} + func (p *providerShellExecution) wait(ctx context.Context) error { poll := time.NewTicker(100 * time.Millisecond) defer poll.Stop() for { + if !p.alive() { + p.supervisor.force(p.cancel) + return errors.New("provider shell parent exited") + } forced, pollErr := providerForcePresent(p.dir) - if forced || pollErr != nil || ctx.Err() != nil { + if providerShellForceRequired(ctx, forced, pollErr) { return p.force(pollErr) } select { @@ -318,8 +351,15 @@ func (p *providerShellExecution) wait(ctx context.Context) error { } } +func providerShellForceRequired(ctx context.Context, forced bool, pollErr error) bool { + return forced || pollErr != nil || ctx.Err() != nil +} + func (p *providerShellExecution) ackForced() error { p.supervisor.force(p.cancel) + if !p.alive() { + return context.Canceled + } return providerWriteAck(p.root, p.token) } @@ -348,6 +388,9 @@ func (p *providerShellExecution) completed(err error) error { if earlyErr := providerShellEarlyError(err); earlyErr != nil { return earlyErr } + if !p.alive() { + return context.Canceled + } if ackErr := providerWriteAck(p.root, p.token); ackErr != nil { return ackErr } diff --git a/pkg/shell/provider_supervisor_test.go b/pkg/shell/provider_supervisor_test.go index 8af918b18a..2be0956c24 100644 --- a/pkg/shell/provider_supervisor_test.go +++ b/pkg/shell/provider_supervisor_test.go @@ -27,8 +27,10 @@ func TestProviderShellProcess(t *testing.T) { if os.Getenv("DEVSY_PROVIDER_SHELL_PROCESS") != "1" { return } - err := RunProviderShell(context.Background(), os.Getenv("DEVSY_PROVIDER_SCRIPT"), - os.Getenv("DEVSY_PROVIDER_FORCE"), os.Getenv("DEVSY_PROVIDER_ACK")) + err := RunProviderShell(context.Background(), providerShellTestInvocation( + os.Getenv("DEVSY_PROVIDER_SCRIPT"), os.Getenv("DEVSY_PROVIDER_FORCE"), + os.Getenv("DEVSY_PROVIDER_ACK"), + )) if err == nil { os.Exit(0) } @@ -145,6 +147,12 @@ func providerTestWriteKey(t *testing.T, ackPath, token string) { ) } +func providerShellTestInvocation(scriptPath, forcePath, ackPath string) ProviderShellInvocation { + return ProviderShellInvocation{ + ScriptPath: scriptPath, ForcePath: forcePath, AckPath: ackPath, ParentPID: os.Getppid(), + } +} + func providerShellCommand(t *testing.T, scriptPath, forcePath, ackPath string) *exec.Cmd { t.Helper() providerTestWriteKey(t, ackPath, providerTestAckToken) @@ -160,7 +168,10 @@ func providerShellCommand(t *testing.T, scriptPath, forcePath, ackPath string) * func runProviderShellWithToken(t *testing.T, scriptPath, forcePath, ackPath string) error { t.Helper() providerTestWriteKey(t, filepath.Join(filepath.Dir(scriptPath), "ack"), providerTestAckToken) - return RunProviderShell(context.Background(), scriptPath, forcePath, ackPath) + return RunProviderShell( + context.Background(), + providerShellTestInvocation(scriptPath, forcePath, ackPath), + ) } func providerTestReadAck(ackPath string) ([]byte, error) { @@ -190,7 +201,10 @@ func TestRunProviderShellRejectsInvalidAckTokens(t *testing.T) { if test.name != "missing" { providerTestWriteKey(t, ackPath, test.token) } - err := RunProviderShell(context.Background(), scriptPath, forcePath, ackPath) + err := RunProviderShell( + context.Background(), + providerShellTestInvocation(scriptPath, forcePath, ackPath), + ) require.ErrorContains(t, err, "invalid provider shell acknowledgement token") _, err = os.Lstat(ackPath) require.ErrorIs(t, err, os.ErrNotExist) @@ -204,7 +218,10 @@ func TestRunProviderShellRejectsUnsafeAckKey(t *testing.T) { require.NoError(t, os.Mkdir(filepath.Join(filepath.Dir(ackPath), "ack-key"), 0o700)) require.ErrorContains( t, - RunProviderShell(context.Background(), scriptPath, forcePath, ackPath), + RunProviderShell( + context.Background(), + providerShellTestInvocation(scriptPath, forcePath, ackPath), + ), "invalid provider shell acknowledgement token", ) _, err := os.Lstat(ackPath) @@ -218,7 +235,10 @@ func TestRunProviderShellRejectsUnsafeAckKey(t *testing.T) { require.NoError(t, os.Symlink(scriptPath, filepath.Join(filepath.Dir(ackPath), "ack-key"))) require.ErrorContains( t, - RunProviderShell(context.Background(), scriptPath, forcePath, ackPath), + RunProviderShell( + context.Background(), + providerShellTestInvocation(scriptPath, forcePath, ackPath), + ), "invalid provider shell acknowledgement token", ) _, err := os.Lstat(ackPath) @@ -230,7 +250,10 @@ func TestRunProviderShellRejectsUnsafeAckKey(t *testing.T) { require.NoError(t, os.Chmod(filepath.Join(filepath.Dir(ackPath), "ack-key"), 0o400)) require.ErrorContains( t, - RunProviderShell(context.Background(), scriptPath, forcePath, ackPath), + RunProviderShell( + context.Background(), + providerShellTestInvocation(scriptPath, forcePath, ackPath), + ), "invalid provider shell acknowledgement token", ) _, err := os.Lstat(ackPath) diff --git a/pkg/shell/provider_supervisor_windows_test.go b/pkg/shell/provider_supervisor_windows_test.go index bf8b63ed0e..da9277bbdd 100644 --- a/pkg/shell/provider_supervisor_windows_test.go +++ b/pkg/shell/provider_supervisor_windows_test.go @@ -39,3 +39,49 @@ func providerChildReaped(pid int) error { } return nil } + +func TestProviderPrivateDACL(t *testing.T) { + user, err := windows.GetCurrentProcessToken().GetTokenUser() + require.NoError(t, err) + sid := user.User.Sid.String() + cases := []struct { + name string + sddl string + want bool + }{ + {"creation", "D:P(A;OICI;GA;;;" + sid + ")(A;OICI;GA;;;SY)", true}, + { + "normalized", + "D:P(A;;FA;;;" + sid + ")(A;OICIIO;GA;;;" + sid + ")(A;;FA;;;SY)(A;OICIIO;GA;;;SY)", + true, + }, + {"unprotected", "D:(A;OICI;GA;;;" + sid + ")(A;OICI;GA;;;SY)", false}, + {"world", "D:P(A;OICI;GA;;;" + sid + ")(A;OICI;GA;;;SY)(A;OICI;GR;;;WD)", false}, + {"missing descendants", "D:P(A;;FA;;;" + sid + ")(A;OICI;GA;;;SY)", false}, + {"insufficient", "D:P(A;OICI;GR;;;" + sid + ")(A;OICI;GA;;;SY)", false}, + {"no propagation", "D:P(A;OICINP;GA;;;" + sid + ")(A;OICI;GA;;;SY)", false}, + {"deny", "D:P(D;;GA;;;WD)(A;OICI;GA;;;" + sid + ")(A;OICI;GA;;;SY)", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + sd, err := windows.SecurityDescriptorFromString(tc.sddl) + require.NoError(t, err) + require.Equal(t, tc.want, providerPrivateDACL(sd, user.User.Sid)) + }) + } +} + +func TestProviderPrivateDACLWhenCurrentUserIsSystem(t *testing.T) { + system, err := windows.CreateWellKnownSid(windows.WinLocalSystemSid) + require.NoError(t, err) + sd, err := windows.SecurityDescriptorFromString("D:P(A;OICI;GA;;;SY)(A;OICI;GA;;;SY)") + require.NoError(t, err) + require.True(t, providerPrivateDACL(sd, system)) +} + +func TestProviderPrivateDirectoryCreated(t *testing.T) { + path := providerPrivateTestDir(t) + info, err := os.Stat(path) + require.NoError(t, err) + require.True(t, providerPrivateDirectory(path, info)) +} diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx index bb494298ee..3b3bcae032 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx +++ b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx @@ -48,7 +48,7 @@ Stopping a machine preserves its guest sources, images, containers, and volumes. Provider state is stored under your user configuration directory at `devsy-provider-docker-sandbox/state-v1`. If creation fails, do not remove that directory or manually reuse a sandbox name. The provider reports the exact machine key and any unresolved child attempt or backend operation. A lost VM is never silently recreated in the same machine generation. -After installation, run `devsy provider list --result-format=json`. In `config.binaries.DOCKER_SANDBOX_PROVIDER`, select the record whose `os` and `arch` match your host. An absolute `path` is the executable. For a remote archive, use `archivePath` under Devsy's data directory at `contexts//providers//binaries/docker_sandbox_provider/`. Otherwise, use the record's `name` in that directory. If `name` is empty, use the base name of `path` instead. With `DEVSY_HOME` set, that variable names the data directory. Otherwise Devsy uses `~/.devsy` on macOS and Linux, or `%LOCALAPPDATA%\devsy` on Windows. Use the resolved local executable, not the asset URL. +After installation, run `devsy provider list --result-format=json`. If you installed with `devsy --home `, use `devsy --home provider list --result-format=json` and treat that path as Devsy's data directory. In `config.binaries.DOCKER_SANDBOX_PROVIDER`, select the record whose `os` and `arch` match your host. An absolute `path` is the executable. For a remote archive, use `archivePath` under the data directory at `contexts//providers//binaries/docker_sandbox_provider/`. Otherwise, use the record's `name` in that directory. If `name` is empty, use the base name of `path` instead. Without a `--home` override, `DEVSY_HOME` names the data directory when set. Otherwise Devsy uses `~/.devsy` on macOS and Linux, or `%LOCALAPPDATA%\devsy` on Windows. Use the resolved local executable, not the asset URL. The provider executable installed with the provider accepts: From 0d436d587001ae967ab0b7a43705ec8af669e313 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 03:41:23 -0600 Subject: [PATCH 10/20] refactor(provider): split Windows ACL validation --- pkg/shell/provider_exec_windows.go | 140 ++++++++++++++--------- pkg/shell/provider_foreground_windows.go | 4 +- 2 files changed, 88 insertions(+), 56 deletions(-) diff --git a/pkg/shell/provider_exec_windows.go b/pkg/shell/provider_exec_windows.go index 9f7ed541c9..164cc54e74 100644 --- a/pkg/shell/provider_exec_windows.go +++ b/pkg/shell/provider_exec_windows.go @@ -21,10 +21,6 @@ func providerPrivateDirectory(path string, info os.FileInfo) bool { if err != nil { return false } - control, _, err := sd.Control() - if err != nil || control&windows.SE_DACL_PROTECTED == 0 { - return false - } user, err := windows.GetCurrentProcessToken().GetTokenUser() if err != nil { return false @@ -33,68 +29,104 @@ func providerPrivateDirectory(path string, info os.FileInfo) bool { } func providerPrivateDACL(sd *windows.SECURITY_DESCRIPTOR, user *windows.SID) bool { - control, _, err := sd.Control() - if err != nil || control&windows.SE_DACL_PROTECTED == 0 { - return false - } - acl, _, err := sd.DACL() - if err != nil || acl == nil { + acl, ok := providerProtectedACL(sd) + if !ok { return false } system, err := windows.CreateWellKnownSid(windows.WinLocalSystemSid) if err != nil { return false } - getAce := windows.NewLazySystemDLL("advapi32.dll").NewProc("GetAce") - var directory, descendants [2]uint32 + var grants providerACLGrants for i := uint32(0); i < uint32(acl.AceCount); i++ { - var ptr unsafe.Pointer - ok, _, _ := getAce.Call( - uintptr(unsafe.Pointer(acl)), - uintptr(i), - uintptr(unsafe.Pointer(&ptr)), - ) - if ok == 0 || ptr == nil { - return false - } - ace := (*windows.ACCESS_ALLOWED_ACE)(ptr) - flags := ace.Header.AceFlags - if ace.Header.AceType != windows.ACCESS_ALLOWED_ACE_TYPE || - flags & ^uint8( - windows.OBJECT_INHERIT_ACE|windows.CONTAINER_INHERIT_ACE|windows.INHERIT_ONLY_ACE, - ) != 0 { - return false - } - sid := (*windows.SID)(unsafe.Pointer(&ace.SidStart)) - index := 0 - if sid.Equals(system) { - index = 1 - } else if !sid.Equals(user) { + ace, ok := providerACLAt(acl, i) + if !ok || !grants.allow(ace, user, system) { return false } - mask := uint32(ace.Mask) - const genericAll = uint32(0x10000000) - const fileAll = uint32(0x001f01ff) - if mask & ^uint32(genericAll|fileAll) != 0 { - return false - } - if mask&genericAll != 0 { - mask |= fileAll - } - if flags&windows.INHERIT_ONLY_ACE == 0 { - directory[index] |= mask - } - if flags&(windows.OBJECT_INHERIT_ACE|windows.CONTAINER_INHERIT_ACE) == windows.OBJECT_INHERIT_ACE|windows.CONTAINER_INHERIT_ACE { - descendants[index] |= mask - } } + return grants.private(user.Equals(system)) +} + +func providerProtectedACL(sd *windows.SECURITY_DESCRIPTOR) (*windows.ACL, bool) { + control, _, err := sd.Control() + if err != nil || control&windows.SE_DACL_PROTECTED == 0 { + return nil, false + } + acl, _, err := sd.DACL() + return acl, err == nil && acl != nil +} + +func providerACLAt(acl *windows.ACL, index uint32) (*windows.ACCESS_ALLOWED_ACE, bool) { + var ace *windows.ACCESS_ALLOWED_ACE + if err := windows.GetAce(acl, index, &ace); err != nil { + return nil, false + } + return ace, ace != nil +} + +type providerACLGrants struct { + directory [2]uint32 + descendants [2]uint32 +} + +func (g *providerACLGrants) allow(ace *windows.ACCESS_ALLOWED_ACE, user, system *windows.SID) bool { + mask, ok := providerACEMask(ace) + if !ok { + return false + } + index, ok := providerACETrustee(ace, user, system) + if !ok { + return false + } + flags := ace.Header.AceFlags + if flags&windows.INHERIT_ONLY_ACE == 0 { + g.directory[index] |= mask + } + if flags&(windows.OBJECT_INHERIT_ACE|windows.CONTAINER_INHERIT_ACE) == + windows.OBJECT_INHERIT_ACE|windows.CONTAINER_INHERIT_ACE { + g.descendants[index] |= mask + } + return true +} + +func providerACEMask(ace *windows.ACCESS_ALLOWED_ACE) (uint32, bool) { + flags := ace.Header.AceFlags + if ace.Header.AceType != windows.ACCESS_ALLOWED_ACE_TYPE || + flags & ^uint8( + windows.OBJECT_INHERIT_ACE|windows.CONTAINER_INHERIT_ACE|windows.INHERIT_ONLY_ACE, + ) != 0 { + return 0, false + } + const genericAll = uint32(0x10000000) + const fileAll = uint32(0x001f01ff) + mask := uint32(ace.Mask) + if mask & ^uint32(genericAll|fileAll) != 0 { + return 0, false + } + if mask&genericAll != 0 { + mask |= fileAll + } + return mask, true +} + +func providerACETrustee(ace *windows.ACCESS_ALLOWED_ACE, user, system *windows.SID) (int, bool) { + sid := (*windows.SID)(unsafe.Pointer(&ace.SidStart)) + if sid.Equals(system) { + return 1, true + } + if sid.Equals(user) { + return 0, true + } + return 0, false +} + +func (g providerACLGrants) private(userIsSystem bool) bool { const fileAll = uint32(0x001f01ff) - if user.Equals(system) { - return directory[1]&fileAll == fileAll && descendants[1]&fileAll == fileAll + if userIsSystem { + return g.directory[1]&fileAll == fileAll && g.descendants[1]&fileAll == fileAll } - return directory[0]&fileAll == fileAll && directory[1]&fileAll == fileAll && - descendants[0]&fileAll == fileAll && - descendants[1]&fileAll == fileAll + return g.directory[0]&fileAll == fileAll && g.directory[1]&fileAll == fileAll && + g.descendants[0]&fileAll == fileAll && g.descendants[1]&fileAll == fileAll } func providerPrivateScript(info os.FileInfo) bool { diff --git a/pkg/shell/provider_foreground_windows.go b/pkg/shell/provider_foreground_windows.go index 31b94cc34e..958d84bdd1 100644 --- a/pkg/shell/provider_foreground_windows.go +++ b/pkg/shell/provider_foreground_windows.go @@ -34,10 +34,10 @@ func PrepareProviderShellForeground(parentPID int) (func(), error) { } ok, _, err = providerSetConsoleCtrlHandler.Call(0, 0) if ok == 0 { - providerSetConsoleCtrlHandler.Call(providerConsoleHandler, 0) + _, _, _ = providerSetConsoleCtrlHandler.Call(providerConsoleHandler, 0) return nil, fmt.Errorf("enable child console interrupts: %w", err) } - return func() { providerSetConsoleCtrlHandler.Call(providerConsoleHandler, 0) }, nil + return func() { _, _, _ = providerSetConsoleCtrlHandler.Call(providerConsoleHandler, 0) }, nil } func newProviderShellParentMonitor(parentPID int) (func() bool, func(), error) { From 94d833e64448779f83e32497d6765912412a2533 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 04:56:07 -0600 Subject: [PATCH 11/20] fix(provider): acknowledge cleaned shell parse failures --- .../clientimplementation/provider_cancel_test.go | 14 ++++++++++++++ pkg/shell/provider_supervisor.go | 12 +++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/pkg/client/clientimplementation/provider_cancel_test.go b/pkg/client/clientimplementation/provider_cancel_test.go index 4dd9993a85..f4abe55389 100644 --- a/pkg/client/clientimplementation/provider_cancel_test.go +++ b/pkg/client/clientimplementation/provider_cancel_test.go @@ -855,6 +855,20 @@ func TestProviderCancellationShellExits(t *testing.T) { } } +func TestProviderCancellationParseErrorAcknowledged(t *testing.T) { + command := "if then fi (((( private-provider-script-846307" + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{command}, + Environ: os.Environ(), + Cancellation: provider.ProviderCancellationFile, + }) + var status interp.ExitStatus + require.ErrorAs(t, err, &status) + require.Equal(t, 1, int(status)) + require.NotContains(t, err.Error(), "without authentic acknowledgement") + require.NotContains(t, err.Error(), command) +} + func TestProviderCancellationRawBytesAndStderrRedaction(t *testing.T) { opts := cancelHelperOptions("bytes") secret := "provider-cancel-secret-84392" diff --git a/pkg/shell/provider_supervisor.go b/pkg/shell/provider_supervisor.go index efa96c3933..8ff1b1c4ee 100644 --- a/pkg/shell/provider_supervisor.go +++ b/pkg/shell/provider_supervisor.go @@ -385,14 +385,16 @@ func (p *providerShellExecution) completed(err error) error { return p.canceled() } p.supervisor.stop() - if earlyErr := providerShellEarlyError(err); earlyErr != nil { - return earlyErr - } if !p.alive() { return context.Canceled } - if ackErr := providerWriteAck(p.root, p.token); ackErr != nil { - return ackErr + ackErr := providerWriteAck(p.root, p.token) + earlyErr := providerShellEarlyError(err) + if ackErr != nil { + return errors.Join(ackErr, earlyErr) + } + if earlyErr != nil { + return earlyErr } return err } From dfa643256898295dda4a7d660096a12096bd40d5 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 06:06:32 -0600 Subject: [PATCH 12/20] fix(provider): preserve local Docker install policy --- pkg/provider/dir.go | 3 -- pkg/provider/load_internal_test.go | 38 ++----------------- pkg/provider/parse_driver_test.go | 2 +- providers/docker/provider.yaml | 2 +- .../managing-providers/docker-sandbox.mdx | 2 + 5 files changed, 7 insertions(+), 40 deletions(-) diff --git a/pkg/provider/dir.go b/pkg/provider/dir.go index ce2c39b6a7..955f6c56dc 100644 --- a/pkg/provider/dir.go +++ b/pkg/provider/dir.go @@ -328,9 +328,6 @@ func LoadProviderConfig(context, provider string) (*ProviderConfig, error) { if embedded := embeddedProviderConfig(providerConfig.Source.Raw); embedded != nil { providerConfig.Exec = embedded.Exec providerConfig.Agent.Exec = embedded.Agent.Exec - if providerConfig.Source.Raw == DockerDriver { - providerConfig.Agent.Docker.Install = embedded.Agent.Docker.Install - } } } diff --git a/pkg/provider/load_internal_test.go b/pkg/provider/load_internal_test.go index 9bb7fbf6be..3bad48d4bc 100644 --- a/pkg/provider/load_internal_test.go +++ b/pkg/provider/load_internal_test.go @@ -13,10 +13,7 @@ import ( // stale provider.json files carry before a refresh rewrites them. const staleHelperShCommand = `"${DEVSY}" helper sh -c "${COMMAND}"` -const ( - staleVersion = "v0.0.0-stale" - podmanProviderName = "podman" -) +const staleVersion = "v0.0.0-stale" // TestLoadProviderConfig_RefreshesInternalProvider verifies that a stored // built-in provider config with a stale exec.command (e.g. the removed @@ -29,9 +26,6 @@ func TestLoadProviderConfig_RefreshesInternalProvider(t *testing.T) { Name: DockerDriver, Version: staleVersion, Source: ProviderSource{Internal: true, Raw: DockerDriver}, - Agent: ProviderAgentConfig{ - Docker: ProviderDockerDriverConfig{Install: disabledDockerPolicy}, - }, Exec: ProviderCommands{ Command: []string{staleHelperShCommand}, }, @@ -45,7 +39,6 @@ func TestLoadProviderConfig_RefreshesInternalProvider(t *testing.T) { "internal provider must be refreshed from embedded yaml, not the stale stored config") require.True(t, strings.Contains(loaded.Exec.Command[0], "internal sh"), "refreshed command should use the current 'internal sh' wrapper") - require.Equal(t, types.StrBool(enabledDockerPolicy), loaded.Agent.Docker.Install) } func TestLoadProviderConfig_PreservesCustomizations(t *testing.T) { @@ -55,11 +48,8 @@ func TestLoadProviderConfig_PreservesCustomizations(t *testing.T) { Name: "my-docker", Version: staleVersion, Source: ProviderSource{Internal: true, Raw: DockerDriver}, - Agent: ProviderAgentConfig{ - Docker: ProviderDockerDriverConfig{Install: disabledDockerPolicy}, - }, Options: map[string]*types.Option{ - "DOCKER_PATH": {Default: podmanProviderName}, + "DOCKER_PATH": {Default: "podman"}, }, Exec: ProviderCommands{ Command: []string{staleHelperShCommand}, @@ -71,11 +61,10 @@ func TestLoadProviderConfig_PreservesCustomizations(t *testing.T) { require.NoError(t, err) require.Equal(t, "my-docker", loaded.Name, "custom provider name must survive refresh") require.Equal(t, "v0.0.0-stale", loaded.Version, "stored version must survive refresh") - require.Equal(t, podmanProviderName, loaded.Options["DOCKER_PATH"].Default, + require.Equal(t, "podman", loaded.Options["DOCKER_PATH"].Default, "resolved options must survive refresh") require.NotContains(t, loaded.Exec.Command[0], "helper sh", "exec block must still be refreshed from embedded yaml") - require.Equal(t, types.StrBool(enabledDockerPolicy), loaded.Agent.Docker.Install) } // TestLoadProviderConfig_RefreshesProBySourceID guards the case where a @@ -131,9 +120,6 @@ func TestLoadProviderConfig_PreservesExternalProvider(t *testing.T) { Name: DockerDriver, Version: "v0.0.1", Source: ProviderSource{Github: "some-org/some-provider"}, - Agent: ProviderAgentConfig{ - Docker: ProviderDockerDriverConfig{Install: disabledDockerPolicy}, - }, Exec: ProviderCommands{ Command: []string{staleHelperShCommand}, }, @@ -143,22 +129,4 @@ func TestLoadProviderConfig_PreservesExternalProvider(t *testing.T) { loaded, err := LoadProviderConfig(config.DefaultContext, DockerDriver) require.NoError(t, err) require.Equal(t, external.Exec.Command, loaded.Exec.Command) - require.Equal(t, types.StrBool(disabledDockerPolicy), loaded.Agent.Docker.Install) -} - -func TestLoadProviderConfig_PreservesPodmanInstallProhibition(t *testing.T) { - setupTestHome(t) - stored := &ProviderConfig{ - Name: podmanProviderName, - Source: ProviderSource{Internal: true, Raw: podmanProviderName}, - Agent: ProviderAgentConfig{ - Docker: ProviderDockerDriverConfig{Install: disabledDockerPolicy}, - }, - Exec: ProviderCommands{Command: []string{staleHelperShCommand}}, - } - require.NoError(t, SaveProviderConfig(config.DefaultContext, stored)) - - loaded, err := LoadProviderConfig(config.DefaultContext, stored.Name) - require.NoError(t, err) - require.Equal(t, types.StrBool(disabledDockerPolicy), loaded.Agent.Docker.Install) } diff --git a/pkg/provider/parse_driver_test.go b/pkg/provider/parse_driver_test.go index 2ed08608a3..c6f09f1d46 100644 --- a/pkg/provider/parse_driver_test.go +++ b/pkg/provider/parse_driver_test.go @@ -27,7 +27,7 @@ exec: {name: "kubernetes literal", driver: KubernetesDriver, wantErr: false}, {name: "templated braces", driver: "${AWS_DEPLOYMENT_MODE}", wantErr: false}, {name: "templated bare", driver: "$AWS_DEPLOYMENT_MODE", wantErr: false}, - {name: "invalid literal", driver: podmanProviderName, wantErr: true}, + {name: "invalid literal", driver: "podman", wantErr: true}, } for _, tc := range cases { diff --git a/providers/docker/provider.yaml b/providers/docker/provider.yaml index c4267defa9..4e1636b4c5 100644 --- a/providers/docker/provider.yaml +++ b/providers/docker/provider.yaml @@ -38,7 +38,7 @@ agent: path: ${DOCKER_PATH} elevation: ${DOCKER_ELEVATION} builder: ${DOCKER_BUILDER} - install: true + install: false env: DOCKER_HOST: ${DOCKER_HOST} DOCKER_CONTEXT: ${DOCKER_CONTEXT} diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx index 3b3bcae032..cc271cf3e7 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx +++ b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx @@ -46,6 +46,8 @@ The guest Docker Engine does not share the host Docker Engine or its socket. Dev Stopping a machine preserves its guest sources, images, containers, and volumes. Starting resumes the same VM. Rebuilding a workspace changes the inner devcontainer without replacing the VM. Deleting a default workspace machine removes the VM and all its guest data, including named volumes. Deleting a workspace on an explicitly shared machine leaves that machine and its other workspaces intact. Uninstalling the provider does not delete its VMs. +With `--single-machine`, deleting its last workspace deletes the shared VM and all guest data. Back up anything you need before deleting that workspace. + Provider state is stored under your user configuration directory at `devsy-provider-docker-sandbox/state-v1`. If creation fails, do not remove that directory or manually reuse a sandbox name. The provider reports the exact machine key and any unresolved child attempt or backend operation. A lost VM is never silently recreated in the same machine generation. After installation, run `devsy provider list --result-format=json`. If you installed with `devsy --home `, use `devsy --home provider list --result-format=json` and treat that path as Devsy's data directory. In `config.binaries.DOCKER_SANDBOX_PROVIDER`, select the record whose `os` and `arch` match your host. An absolute `path` is the executable. For a remote archive, use `archivePath` under the data directory at `contexts//providers//binaries/docker_sandbox_provider/`. Otherwise, use the record's `name` in that directory. If `name` is empty, use the base name of `path` instead. Without a `--home` override, `DEVSY_HOME` names the data directory when set. Otherwise Devsy uses `~/.devsy` on macOS and Linux, or `%LOCALAPPDATA%\devsy` on Windows. Use the resolved local executable, not the asset URL. From 201756ede959a33b0ac51898c975ce4cce8d76c1 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 07:09:39 -0600 Subject: [PATCH 13/20] fix(provider): preserve built-in Docker installation --- pkg/provider/dir.go | 3 +++ pkg/provider/load_internal_test.go | 36 +++++++++++++++++++++++++++--- pkg/provider/parse_driver_test.go | 2 +- providers/docker/provider.yaml | 2 +- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/pkg/provider/dir.go b/pkg/provider/dir.go index 955f6c56dc..ce2c39b6a7 100644 --- a/pkg/provider/dir.go +++ b/pkg/provider/dir.go @@ -328,6 +328,9 @@ func LoadProviderConfig(context, provider string) (*ProviderConfig, error) { if embedded := embeddedProviderConfig(providerConfig.Source.Raw); embedded != nil { providerConfig.Exec = embedded.Exec providerConfig.Agent.Exec = embedded.Agent.Exec + if providerConfig.Source.Raw == DockerDriver { + providerConfig.Agent.Docker.Install = embedded.Agent.Docker.Install + } } } diff --git a/pkg/provider/load_internal_test.go b/pkg/provider/load_internal_test.go index 3bad48d4bc..2b14cadbbc 100644 --- a/pkg/provider/load_internal_test.go +++ b/pkg/provider/load_internal_test.go @@ -13,7 +13,10 @@ import ( // stale provider.json files carry before a refresh rewrites them. const staleHelperShCommand = `"${DEVSY}" helper sh -c "${COMMAND}"` -const staleVersion = "v0.0.0-stale" +const ( + staleVersion = "v0.0.0-stale" + podmanProviderName = "podman" +) // TestLoadProviderConfig_RefreshesInternalProvider verifies that a stored // built-in provider config with a stale exec.command (e.g. the removed @@ -49,7 +52,7 @@ func TestLoadProviderConfig_PreservesCustomizations(t *testing.T) { Version: staleVersion, Source: ProviderSource{Internal: true, Raw: DockerDriver}, Options: map[string]*types.Option{ - "DOCKER_PATH": {Default: "podman"}, + "DOCKER_PATH": {Default: podmanProviderName}, }, Exec: ProviderCommands{ Command: []string{staleHelperShCommand}, @@ -61,7 +64,7 @@ func TestLoadProviderConfig_PreservesCustomizations(t *testing.T) { require.NoError(t, err) require.Equal(t, "my-docker", loaded.Name, "custom provider name must survive refresh") require.Equal(t, "v0.0.0-stale", loaded.Version, "stored version must survive refresh") - require.Equal(t, "podman", loaded.Options["DOCKER_PATH"].Default, + require.Equal(t, podmanProviderName, loaded.Options["DOCKER_PATH"].Default, "resolved options must survive refresh") require.NotContains(t, loaded.Exec.Command[0], "helper sh", "exec block must still be refreshed from embedded yaml") @@ -130,3 +133,30 @@ func TestLoadProviderConfig_PreservesExternalProvider(t *testing.T) { require.NoError(t, err) require.Equal(t, external.Exec.Command, loaded.Exec.Command) } + +func TestLoadProviderConfig_DockerInstallPolicyScopes(t *testing.T) { + for _, tc := range []struct { + name string + source ProviderSource + want types.StrBool + }{ + {name: "builtin-docker", source: ProviderSource{Internal: true, Raw: DockerDriver}, want: enabledDockerPolicy}, + {name: "external-docker", source: ProviderSource{Github: "some-org/provider"}, want: disabledDockerPolicy}, + {name: "builtin-podman", source: ProviderSource{Internal: true, Raw: podmanProviderName}, want: disabledDockerPolicy}, + } { + t.Run(tc.name, func(t *testing.T) { + setupTestHome(t) + stored := &ProviderConfig{ + Name: tc.name, Source: tc.source, + Agent: ProviderAgentConfig{ + Docker: ProviderDockerDriverConfig{Install: disabledDockerPolicy}, + }, + Exec: ProviderCommands{Command: []string{staleHelperShCommand}}, + } + require.NoError(t, SaveProviderConfig(config.DefaultContext, stored)) + loaded, err := LoadProviderConfig(config.DefaultContext, stored.Name) + require.NoError(t, err) + require.Equal(t, tc.want, loaded.Agent.Docker.Install) + }) + } +} diff --git a/pkg/provider/parse_driver_test.go b/pkg/provider/parse_driver_test.go index c6f09f1d46..2ed08608a3 100644 --- a/pkg/provider/parse_driver_test.go +++ b/pkg/provider/parse_driver_test.go @@ -27,7 +27,7 @@ exec: {name: "kubernetes literal", driver: KubernetesDriver, wantErr: false}, {name: "templated braces", driver: "${AWS_DEPLOYMENT_MODE}", wantErr: false}, {name: "templated bare", driver: "$AWS_DEPLOYMENT_MODE", wantErr: false}, - {name: "invalid literal", driver: "podman", wantErr: true}, + {name: "invalid literal", driver: podmanProviderName, wantErr: true}, } for _, tc := range cases { diff --git a/providers/docker/provider.yaml b/providers/docker/provider.yaml index 4e1636b4c5..c4267defa9 100644 --- a/providers/docker/provider.yaml +++ b/providers/docker/provider.yaml @@ -38,7 +38,7 @@ agent: path: ${DOCKER_PATH} elevation: ${DOCKER_ELEVATION} builder: ${DOCKER_BUILDER} - install: false + install: true env: DOCKER_HOST: ${DOCKER_HOST} DOCKER_CONTEXT: ${DOCKER_CONTEXT} From 60516d9e62c34fc901e9e5a1a26dacf71bc405ec Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 08:51:41 -0600 Subject: [PATCH 14/20] docs(provider): use workspace up command --- .../content/docs/managing-providers/docker-sandbox.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx index cc271cf3e7..0267ab9469 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx +++ b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx @@ -17,7 +17,7 @@ After a qualified provider release is published, install it with: ```sh devsy provider add docker-sandbox -devsy up https://github.com/devcontainers/samples --provider docker-sandbox +devsy workspace up https://github.com/devcontainers/samples --provider docker-sandbox ``` The Devsy Desktop setup wizard does not list this provider before a qualified release. The provider does not install `sbx`, sign in, open a login browser, or change Docker Sandboxes settings. From 066a3e4b68b38e92e52273ad8dbe07b645d891c5 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 12:19:32 -0600 Subject: [PATCH 15/20] fix(provider): qualify unreleased installation guidance --- pkg/provider/parse_driver_test.go | 20 ------------------- .../managing-providers/manage-providers.mdx | 6 ++---- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/pkg/provider/parse_driver_test.go b/pkg/provider/parse_driver_test.go index 2ed08608a3..e23fa0ec8f 100644 --- a/pkg/provider/parse_driver_test.go +++ b/pkg/provider/parse_driver_test.go @@ -75,23 +75,3 @@ func TestProviderCancellationMode(t *testing.T) { }) } } - -func TestProviderCancellationModeAcrossProviderKinds(t *testing.T) { - for _, tc := range []struct { - name string - commands string - }{ - {name: "proxy", commands: " proxy:\n" + - " up: echo up\n status: echo status\n stop: echo stop\n" + - " delete: echo delete\n ssh: echo ssh\n"}, - {name: "daemon", commands: " daemon:\n start: echo start\n"}, - } { - t.Run(tc.name, func(t *testing.T) { - manifest := "name: example\nexec:\n cancellation: files\n" + tc.commands - _, err := ParseProvider(strings.NewReader(manifest)) - if err == nil || !strings.Contains(err.Error(), "exec.cancellation") { - t.Fatalf("ParseProvider() error = %v, want invalid cancellation mode", err) - } - }) - } -} diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx index ba78cf05a1..4ad599c08f 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx +++ b/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx @@ -107,15 +107,13 @@ If a switch fails during initialization, the new source stays selected. Fix the ### Docker Sandboxes -Install Docker Sandboxes and sign in with `sbx login` before adding the provider: +A published release qualified for your host is required. After it exists, install the standalone `sbx` CLI, sign in with `sbx login`, and add the provider: ```sh devsy provider add docker-sandbox ``` -The [Docker Sandboxes guide](./docker-sandbox.mdx) covers qualified hosts, resource options, guest Docker, source files, networking, data deletion, and scoped recovery. The provider needs a published release containing native qualification for your host. - - +The [Docker Sandboxes guide](./docker-sandbox.mdx) covers qualified hosts, resource options, guest Docker, source files, networking, data deletion, and scoped recovery. ## Set provider options From ec4426f4e4a686bfa9d416677ae4c2cdd3466772 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 12:38:32 -0600 Subject: [PATCH 16/20] fix(provider): join argv transport input after exit --- .../provider_cancel_test.go | 43 +++++++++++++++ .../clientimplementation/workspace_client.go | 55 +++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/pkg/client/clientimplementation/provider_cancel_test.go b/pkg/client/clientimplementation/provider_cancel_test.go index f4abe55389..60c2d20ea7 100644 --- a/pkg/client/clientimplementation/provider_cancel_test.go +++ b/pkg/client/clientimplementation/provider_cancel_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "net" "os" "os/exec" "os/signal" @@ -882,6 +883,48 @@ func TestProviderCancellationRawBytesAndStderrRedaction(t *testing.T) { require.Contains(t, stderr.String(), "***") } +func TestProviderCancellationArgvTransportCompletesWithOpenInput(t *testing.T) { + local, remote := net.Pipe() + t.Cleanup(func() { + _ = local.Close() + _ = remote.Close() + }) + opts := cancelHelperOptions("bytes") + opts.Stdin, opts.closeInputOnExit = remote, remote + opts.RawStdout = true + var stdout bytes.Buffer + opts.Stdout = &stdout + finished := make(chan error, 1) + go func() { finished <- RunCommand(context.Background(), opts) }() + select { + case err := <-finished: + require.NoError(t, err) + require.Equal(t, []byte{0, 0xff, 0x80, '\r', '\n'}, stdout.Bytes()) + case <-time.After(5 * time.Second): + t.Fatal("argv provider completion waited for transport input closure") + } +} + +func TestProviderCancellationArgvOwnedInputPreservesExitStatus(t *testing.T) { + local, remote := net.Pipe() + t.Cleanup(func() { + _ = local.Close() + _ = remote.Close() + }) + opts := cancelHelperOptions("status") + opts.Stdin, opts.closeInputOnExit = remote, remote + finished := make(chan error, 1) + go func() { finished <- RunCommand(context.Background(), opts) }() + select { + case err := <-finished: + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + require.Equal(t, 7, exitErr.ExitCode()) + case <-time.After(5 * time.Second): + t.Fatal("argv provider exit waited for transport input closure") + } +} + func TestProviderCancellationOwnedInputPreservesShellExitStatus(t *testing.T) { input := io.NopCloser(strings.NewReader("")) err := RunCommand(context.Background(), RunCommandOptions{ diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index 77d45e7e55..99431d4caa 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -1334,6 +1334,11 @@ func runProviderArgvCommand( cmd.Env = opts.Environ if opts.Cancellation == provider.ProviderCancellationFile { cmd.WaitDelay = 2 * time.Second + if opts.closeInputOnExit != nil { + return runProviderArgvWithOwnedInput( + cmd, opts.Stdin, opts.closeInputOnExit, providerAction(opts.Environ), + ) + } } if err := cmd.Run(); err != nil { return fmt.Errorf("run provider %s: %w", providerAction(opts.Environ), err) @@ -1341,6 +1346,56 @@ func runProviderArgvCommand( return nil } +func runProviderArgvWithOwnedInput( + cmd *exec.Cmd, + source io.Reader, + input io.Closer, + action string, +) (retErr error) { + stdinRead, stdinWrite, err := os.Pipe() + if err != nil { + return errors.Join( + fmt.Errorf("create provider %s stdin pipe: %w", action, err), + providerShellCloseError(action, "input", input.Close()), + ) + } + defer func() { + retErr = errors.Join( + retErr, closeProviderShellPipe(stdinRead), closeProviderShellPipe(stdinWrite), + ) + }() + cmd.Stdin = stdinRead + if err := cmd.Start(); err != nil { + return errors.Join( + fmt.Errorf("run provider %s: %w", action, err), + providerShellCloseError(action, "input", input.Close()), + ) + } + childPipeCloseErr := closeProviderShellPipe(stdinRead) + stdinDone := pumpProviderShellInput(stdinWrite, source) + waitErr := cmd.Wait() + inputCloseErr := input.Close() + stdinPipeCloseErr := closeProviderShellPipe(stdinWrite) + stdinResult := <-stdinDone + stdinErr := providerShellInputCopyError(stdinResult.copyErr, waitErr, inputCloseErr) + return errors.Join( + providerArgvWaitError(action, waitErr), + providerShellCopyError(action, "stdin", stdinErr), + providerShellCloseError( + action, "stdin pipe", + errors.Join(childPipeCloseErr, stdinPipeCloseErr, stdinResult.closeErr), + ), + providerShellCloseError(action, "input", inputCloseErr), + ) +} + +func providerArgvWaitError(action string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("run provider %s: %w", action, err) +} + func flushCommandOutput(stdout, stderr *subprocess.StreamingRedactingWriter) error { var stdoutErr error if stdout != nil { From d283ebb7f1046e54c5f3c9f3c4d7dac23baf3962 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 14:41:02 -0600 Subject: [PATCH 17/20] test(provider): reject timed-out init replacement --- cmd/provider/set_source_init_failure_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/provider/set_source_init_failure_test.go b/cmd/provider/set_source_init_failure_test.go index c8050dd2a4..abfef2c39d 100644 --- a/cmd/provider/set_source_init_failure_test.go +++ b/cmd/provider/set_source_init_failure_test.go @@ -48,6 +48,7 @@ func TestSetSourceInitFailureKeepsCommittedReplacementAndWorkspace(t *testing.T) cmd := &SetSourceCmd{GlobalFlags: &flags.GlobalFlags{}, Use: true} err = cmd.Run(ctx, initial, []string{testProviderFoo, replacementSource}) require.Error(t, err) + require.NotErrorIs(t, err, context.DeadlineExceeded) assertFailedSourceReplacement( t, From 83bce897832ea03ec9411d12c9707495c976de4d Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 17:33:58 -0600 Subject: [PATCH 18/20] fix(provider): bypass recovery gate for cleanup helper --- cmd/internal/provider_shell.go | 1 + cmd/internal/provider_shell_test.go | 14 ++++++++++++-- .../docs/managing-providers/docker-sandbox.mdx | 2 ++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/internal/provider_shell.go b/cmd/internal/provider_shell.go index 8a1737931a..89c2ce700d 100644 --- a/cmd/internal/provider_shell.go +++ b/cmd/internal/provider_shell.go @@ -15,6 +15,7 @@ func NewProviderShellCmd() *cobra.Command { var parentPID int command := &cobra.Command{ Use: "provider-shell", Hidden: true, Args: cobra.NoArgs, + PersistentPreRunE: func(*cobra.Command, []string) error { return nil }, RunE: func(cmd *cobra.Command, _ []string) error { cleanup, err := shell.PrepareProviderShellForeground(parentPID) if err != nil { diff --git a/cmd/internal/provider_shell_test.go b/cmd/internal/provider_shell_test.go index fbd3c6713d..716856d41a 100644 --- a/cmd/internal/provider_shell_test.go +++ b/cmd/internal/provider_shell_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" + "github.com/devsy-org/devsy/pkg/config" + "github.com/spf13/cobra" "github.com/stretchr/testify/require" ) @@ -19,11 +21,19 @@ func TestProviderShellCommandRegistration(t *testing.T) { require.NoError(t, os.WriteFile(script, []byte("exit 0"), 0o600)) token := strings.Repeat("0123456789abcdef", 4) require.NoError(t, os.WriteFile(filepath.Join(dir, "ack-key"), []byte(token), 0o600)) - command.SetArgs([]string{ + parent := &cobra.Command{ + Use: "devsy", + PersistentPreRunE: func(*cobra.Command, []string) error { + return config.ErrContextDeletionPending + }, + } + parent.AddCommand(command) + parent.SetArgs([]string{ + "provider-shell", "--script-file", script, "--force-file", filepath.Join(dir, "force"), "--ack-file", filepath.Join(dir, "ack"), "--parent-pid", strconv.Itoa(os.Getppid()), }) - require.NoError(t, command.Execute()) + require.NoError(t, parent.Execute()) _, err := os.Lstat(filepath.Join(dir, "ack-key")) require.ErrorIs(t, err, os.ErrNotExist) root, err := os.OpenRoot(dir) diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx index 0267ab9469..d8f16855a6 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx +++ b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx @@ -50,6 +50,8 @@ With `--single-machine`, deleting its last workspace deletes the shared VM and a Provider state is stored under your user configuration directory at `devsy-provider-docker-sandbox/state-v1`. If creation fails, do not remove that directory or manually reuse a sandbox name. The provider reports the exact machine key and any unresolved child attempt or backend operation. A lost VM is never silently recreated in the same machine generation. +A pending context deletion blocks normal commands but does not stop a provider cleanup helper that Devsy already started. Let that helper finish its scoped cleanup before retrying deletion. + After installation, run `devsy provider list --result-format=json`. If you installed with `devsy --home `, use `devsy --home provider list --result-format=json` and treat that path as Devsy's data directory. In `config.binaries.DOCKER_SANDBOX_PROVIDER`, select the record whose `os` and `arch` match your host. An absolute `path` is the executable. For a remote archive, use `archivePath` under the data directory at `contexts//providers//binaries/docker_sandbox_provider/`. Otherwise, use the record's `name` in that directory. If `name` is empty, use the base name of `path` instead. Without a `--home` override, `DEVSY_HOME` names the data directory when set. Otherwise Devsy uses `~/.devsy` on macOS and Linux, or `%LOCALAPPDATA%\devsy` on Windows. Use the resolved local executable, not the asset URL. The provider executable installed with the provider accepts: From a0a7a41852e523ebe49afcdcc79c389684b765d6 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 18:45:27 -0600 Subject: [PATCH 19/20] docs(provider): clarify shell arrays and child lifetime --- .../content/docs/developing-providers/quickstart.mdx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx index c100d3efbc..3f2517a877 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx @@ -44,6 +44,8 @@ devsy workspace up github.com/microsoft/vscode-remote-try-go The `command`, `init`, `create`, `delete`, `start`, `stop`, `status` and `describe` entries are POSIX shell scripts or command arrays. Only `command` is required. `cancellation` is an execution policy, not a script. +An array with one element is parsed as a shell script, not as an executable path. For example, `["/opt/provider tool"]` runs `/opt/provider` with the argument `tool`. Quote the path inside that script or provide a multi-element array to run a path that contains spaces. + - **command**: runs a command in the environment. Devsy uses it to inject itself, and sends all traffic through its standard input and output. Devsy fills in the `${COMMAND}` placeholder. - **init**: checks that the options are valid and the provider is ready. Devsy runs it when options change. - **create**, **delete**: create and delete a machine. Defining `create` makes the provider a machine provider, so `delete` is expected too. @@ -56,7 +58,7 @@ A provider without `create` is a non-machine provider and uses only `command` an With `exec.cancellation: file`, Devsy sets the reserved `DEVSY_PROVIDER_CANCEL_FILE` environment variable to a private, initially nonexistent marker path for each invocation. The provider must watch for the marker, stop admitting work, cancel its owned processes, finish cleanup, and exit. On caller cancellation, Devsy creates the marker and allows up to 10 seconds for cleanup. For single-string shell commands that remain blocked, Devsy then asks its supervised shell to stop and reap its direct children before terminating the shell; it waits up to another 5 seconds for an authenticated acknowledgement before killing an unresponsive helper. A provider-writable acknowledgement file does not establish that cleanup finished: a missing or forged acknowledgement, including when the helper exits abruptly during the cleanup window, is reported as an unclean shutdown alongside the caller's cancellation error. Devsy never kills the provider's process group or unrelated daemon processes. It waits for the invocation to finish and still returns the caller's cancellation error even if cleanup exits successfully. A provider must not set or override the reserved variable. -For single-string commands, the supervisor also stops and reaps direct children still running when the shell script completes normally. A background command started with `&` cannot outlive its invocation. Start durable services through the backend's explicit detached operation, not a shell background job. +For shell commands, the supervisor also stops and reaps direct children still running when the script completes normally. A direct background child started with `&` cannot outlive its invocation. A detached descendant can outlive it, so the provider must manage those processes explicitly. Start durable services through the backend's explicit detached operation, not a shell background job. File mode requires finite or nonblocking standard streams, direct file handles, or blocking streams whose owner unblocks them on cancellation of the original calling context. Devsy does not close caller-owned input or global standard input, and process cancellation cannot interrupt an arbitrary blocked reader or writer. Machine command transports close their pipe endpoints on cancellation of the original context. Standard input can close independently of standard output, so providers must continue draining output after input reaches EOF. From 1b64e1ced7710de49e9a237047b5ef4a136646ca Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 3 Nov 2026 19:59:25 -0600 Subject: [PATCH 20/20] docs(provider): scope command and output stream requirements --- .../content/docs/developing-providers/quickstart.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx index 3f2517a877..cd0ae69ec1 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx @@ -42,7 +42,7 @@ devsy workspace up github.com/microsoft/vscode-remote-try-go ## exec -The `command`, `init`, `create`, `delete`, `start`, `stop`, `status` and `describe` entries are POSIX shell scripts or command arrays. Only `command` is required. `cancellation` is an execution policy, not a script. +The `command`, `init`, `create`, `delete`, `start`, `stop`, `status` and `describe` entries are POSIX shell scripts or command arrays. For standard providers, only `command` is required. Proxy and daemon providers use their own exec fields instead. `cancellation` is an execution policy, not a script. An array with one element is parsed as a shell script, not as an executable path. For example, `["/opt/provider tool"]` runs `/opt/provider` with the argument `tool`. Quote the path inside that script or provide a multi-element array to run a path that contains spaces. @@ -54,13 +54,13 @@ An array with one element is parsed as a shell script, not as an executable path - **describe**: prints a text description of the machine. - **cancellation**: omit it for existing process cancellation behavior, or set `file` to give provider commands a cooperative cleanup window. -A provider without `create` is a non-machine provider and uses only `command` and `init`. The cancellation policy applies to both machine and non-machine providers. +A standard provider without `create` is a non-machine provider and uses only `command` and `init`. The cancellation policy applies to both machine and non-machine providers. With `exec.cancellation: file`, Devsy sets the reserved `DEVSY_PROVIDER_CANCEL_FILE` environment variable to a private, initially nonexistent marker path for each invocation. The provider must watch for the marker, stop admitting work, cancel its owned processes, finish cleanup, and exit. On caller cancellation, Devsy creates the marker and allows up to 10 seconds for cleanup. For single-string shell commands that remain blocked, Devsy then asks its supervised shell to stop and reap its direct children before terminating the shell; it waits up to another 5 seconds for an authenticated acknowledgement before killing an unresponsive helper. A provider-writable acknowledgement file does not establish that cleanup finished: a missing or forged acknowledgement, including when the helper exits abruptly during the cleanup window, is reported as an unclean shutdown alongside the caller's cancellation error. Devsy never kills the provider's process group or unrelated daemon processes. It waits for the invocation to finish and still returns the caller's cancellation error even if cleanup exits successfully. A provider must not set or override the reserved variable. For shell commands, the supervisor also stops and reaps direct children still running when the script completes normally. A direct background child started with `&` cannot outlive its invocation. A detached descendant can outlive it, so the provider must manage those processes explicitly. Start durable services through the backend's explicit detached operation, not a shell background job. -File mode requires finite or nonblocking standard streams, direct file handles, or blocking streams whose owner unblocks them on cancellation of the original calling context. Devsy does not close caller-owned input or global standard input, and process cancellation cannot interrupt an arbitrary blocked reader or writer. Machine command transports close their pipe endpoints on cancellation of the original context. Standard input can close independently of standard output, so providers must continue draining output after input reaches EOF. +File mode accepts finite or nonblocking standard streams, direct file input handles, and blocking streams whose owner unblocks them when the original calling context is cancelled. A direct `*os.File` output handle is not exempt: Devsy can wrap it for redaction, and a blocked output write can prevent completion after the child exits. Callers must drain or unblock blocking stdout and stderr, including file-backed pipes, on cancellation. Devsy does not close caller-owned input or global standard input, and process cancellation cannot interrupt an arbitrary blocked reader or writer. Machine command transports close their pipe endpoints on cancellation of the original context. Standard input can close independently of standard output, so providers must continue draining output after input reaches EOF. On Windows, Devsy runs these scripts in an [emulated shell](https://github.com/mvdan/sh). The emulated shell has no `grep` or `sed`. Move that logic into a helper binary.