diff --git a/cmd/internal/agentworkspace/docker_darwin_test.go b/cmd/internal/agentworkspace/docker_darwin_test.go index 7b21d4f345..e0c73bb3aa 100644 --- a/cmd/internal/agentworkspace/docker_darwin_test.go +++ b/cmd/internal/agentworkspace/docker_darwin_test.go @@ -3,19 +3,28 @@ package agentworkspace import ( + "context" + "errors" "os" "path/filepath" "slices" "strings" "testing" + + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/types" ) func writeExecutable(t *testing.T, path string) { t.Helper() - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + executable, err := os.Executable() + if err != nil { t.Fatal(err) } - if err := os.WriteFile(path, []byte("#!/bin/sh\n"), 0o755); err != nil { + if err := os.Link(executable, path); err != nil { t.Fatal(err) } } @@ -88,7 +97,7 @@ func TestFindDarwinDockerCLIHomeFailureDoesNotBreakStaticDiscovery(t *testing.T) func TestFindDarwinDockerCLIRejectsNonExecutable(t *testing.T) { path := filepath.Join(t.TempDir(), "docker") - if err := os.WriteFile(path, nil, 0o644); err != nil { + if err := os.WriteFile(path, nil, 0o600); err != nil { t.Fatal(err) } if _, err := findDarwinDockerCLIInPaths([]string{path}); err == nil { @@ -96,19 +105,59 @@ func TestFindDarwinDockerCLIRejectsNonExecutable(t *testing.T) { } } -func TestFindDarwinDockerCLIRejectsInaccessibleCandidate(t *testing.T) { +func TestFindDarwinDockerCLISkipsBrokenCandidate(t *testing.T) { tmp := t.TempDir() - inaccessible := filepath.Join(tmp, "inaccessible", "docker") - writeExecutable(t, inaccessible) - if err := os.Chmod(inaccessible, 0o001); err != nil { + broken := filepath.Join(tmp, "broken", "docker") + if err := os.MkdirAll(filepath.Dir(broken), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(tmp, "missing", "docker"), broken); err != nil { t.Fatal(err) } valid := filepath.Join(tmp, "valid", "docker") writeExecutable(t, valid) - path, err := findDarwinDockerCLIInPaths([]string{inaccessible, valid}) + path, err := findDarwinDockerCLIInPaths([]string{broken, valid}) if err != nil || path != valid { t.Fatalf("expected fallback to valid candidate %q, got %q (err: %v)", valid, path, err) } } + +func TestDarwinDockerDiscoveryPrecedesDisabledInstallation(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + existing := filepath.Join(t.TempDir(), "docker") + writeExecutable(t, existing) + info := &provider.AgentWorkspaceInfo{} + info.Agent.Docker.Install = types.StrBool("false") + calls := 0 + initializer := &workspaceInitializer{ + workspaceInfo: info, + findDarwinDocker: func() (string, error) { + calls++ + return findDarwinDockerCLIInPaths([]string{existing}) + }, + installDocker: func(context.Context) (string, error) { + t.Fatal("disabled installation called the installer") + return "", nil + }, + } + path, err := initializer.ensureDockerInstalled(context.Background()) + if err != nil || path != existing || calls != 1 { + t.Fatalf( + "Docker CLI discovery = %q, %v, calls %d; want %q, nil, 1", + path, + err, + calls, + existing, + ) + } + + initializer.findDarwinDocker = func() (string, error) { + return "", errors.New("Docker CLI absent") + } + path, err = initializer.ensureDockerInstalled(context.Background()) + if path != "" || err == nil || !strings.Contains(err.Error(), "agent.docker.install is false") { + t.Fatalf("missing Docker CLI = %q, %v; want disabled installation error", path, err) + } +} diff --git a/cmd/internal/agentworkspace/up.go b/cmd/internal/agentworkspace/up.go index bcd740b7df..6566a2cb76 100644 --- a/cmd/internal/agentworkspace/up.go +++ b/cmd/internal/agentworkspace/up.go @@ -385,6 +385,9 @@ type workspaceInitializer struct { logger tunnelserver.Logger dockerCredentialsDir string gitCredentialsHelper string + installDocker func(context.Context) (string, error) + findDarwinDocker func() (string, error) + configureDocker func(context.Context) error } type initWorkspaceParams struct { @@ -411,6 +414,10 @@ func initWorkspace( } func (w *workspaceInitializer) initialize(ctx context.Context) error { + if err := w.validateDockerManagement(); err != nil { + return err + } + if err := w.initializeTunnel(ctx); err != nil { return err } @@ -448,7 +455,11 @@ func (w *workspaceInitializer) tryConfigureDockerDaemon(ctx context.Context) { log.Debug("skipping configuring docker daemon") return } - if err := configureDockerDaemon(ctx); err != nil { + configure := w.configureDocker + if configure == nil { + configure = configureDockerDaemon + } + if err := configure(ctx); err != nil { log.Warn( "could not find docker daemon config file, if using the registry cache, " + "ensure the daemon is configured with containerd-snapshotter=true, " + @@ -514,25 +525,49 @@ func (w *workspaceInitializer) ensureDockerInstalled(ctx context.Context) (strin } if dockerCmd != docker { - path, err := exec.LookPath(dockerCmd) - if err != nil { - return "", fmt.Errorf("custom docker path %q not found: %w", dockerCmd, err) - } - return path, nil + return findCustomDockerCLI(dockerCmd) } - if dockerCmd == docker && runtime.GOOS == "darwin" { - return findDarwinDockerCLI() + if runtime.GOOS == "darwin" { + path, err := w.discoverDarwinDockerCLI() + if err == nil || !w.isDockerInstallDisabled() { + return path, err + } } if w.isDockerInstallDisabled() { - log.Debug( - "docker not found but installation was disabled, installing anyway as it is required", + return "", fmt.Errorf( + "docker CLI is unavailable and agent.docker.install is false (executable %q)", + dockerCmd, ) } + return w.installMissingDocker(ctx) +} + +func findCustomDockerCLI(dockerCmd string) (string, error) { + path, err := exec.LookPath(dockerCmd) + if err != nil { + return "", fmt.Errorf("custom docker path %q not found: %w", dockerCmd, err) + } + return path, nil +} + +func (w *workspaceInitializer) discoverDarwinDockerCLI() (string, error) { + find := w.findDarwinDocker + if find == nil { + find = findDarwinDockerCLI + } + return find() +} + +func (w *workspaceInitializer) installMissingDocker(ctx context.Context) (string, error) { log.Debug("attempting to install docker") - dockerPath, err := installDocker(ctx) + install := w.installDocker + if install == nil { + install = installDocker + } + dockerPath, err := install(ctx) log.Debugf("docker installation path=%q, err=%v", dockerPath, err) return dockerPath, err } @@ -579,6 +614,9 @@ func (w *workspaceInitializer) getDockerCommand() string { } func (w *workspaceInitializer) isDockerInstallDisabled() bool { + if w.workspaceInfo.Agent.Docker.Install == "" { + return false + } install, err := w.workspaceInfo.Agent.Docker.Install.Bool() return err == nil && !install } @@ -612,10 +650,26 @@ func (w *workspaceInitializer) waitForDocker(resultChan <-chan dockerInstallResu return nil } +func (w *workspaceInitializer) validateDockerManagement() error { + if _, err := w.workspaceInfo.Agent.Docker.Install.Bool(); err != nil { + return fmt.Errorf("agent.docker.install: %w", err) + } + if _, err := w.workspaceInfo.Agent.Docker.ConfigureDaemon.Bool(); err != nil { + return fmt.Errorf("agent.docker.configureDaemon: %w", err) + } + return nil +} + func (w *workspaceInitializer) shouldConfigureDockerDaemon() bool { if !w.workspaceInfo.Agent.IsDockerDriver() { return false } + if w.workspaceInfo.Agent.Docker.ConfigureDaemon != "" { + configure, err := w.workspaceInfo.Agent.Docker.ConfigureDaemon.Bool() + if err != nil || !configure { + return false + } + } local, err := w.workspaceInfo.Agent.Local.Bool() if err != nil { diff --git a/cmd/internal/agentworkspace/up_test.go b/cmd/internal/agentworkspace/up_test.go index 09a4aa5089..6565ea7fd8 100644 --- a/cmd/internal/agentworkspace/up_test.go +++ b/cmd/internal/agentworkspace/up_test.go @@ -3,14 +3,18 @@ package agentworkspace import ( "context" "errors" + "io" "os" "path/filepath" + "runtime" + "strings" "testing" "github.com/devsy-org/devsy/pkg/agent/tunnel" "github.com/devsy-org/devsy/pkg/agent/tunnelserver" "github.com/devsy-org/devsy/pkg/provider" "github.com/devsy-org/devsy/pkg/status" + "github.com/devsy-org/devsy/pkg/types" "github.com/stretchr/testify/assert" ) @@ -242,3 +246,251 @@ func TestWaitForDockerReturnsDiscoveryError(t *testing.T) { t.Fatalf("waitForDocker() error = %v, want wrapped %v", err, discoveryErr) } } + +const ( + dockerPolicyDisabled = "false" + dockerPolicyEnabled = "true" + dockerPolicyInvalid = "invalid" + testInstalledDocker = "/installed/docker" +) + +type dockerInstallationCase struct { + name string + path string + policy string + existing bool + wantCalls int + wantPath string + wantError string +} + +func TestDockerInstallationPolicy(t *testing.T) { + cases := []dockerInstallationCase{ + { + name: "missing disabled", + policy: dockerPolicyDisabled, + wantError: `docker CLI is unavailable and agent.docker.install is false (executable "docker")`, + }, + {name: "missing unset", wantCalls: 1, wantPath: testInstalledDocker}, + { + name: "missing enabled", + policy: dockerPolicyEnabled, + wantCalls: 1, + wantPath: testInstalledDocker, + }, + {name: "existing disabled", policy: dockerPolicyDisabled, existing: true}, + { + name: "custom missing disabled", + path: "missing-custom", + policy: dockerPolicyDisabled, + wantError: "custom docker path", + }, + {name: "custom missing unset", path: "missing-custom", wantError: "custom docker path"}, + { + name: "existing custom disabled", + path: "custom-docker", + policy: dockerPolicyDisabled, + existing: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assertDockerInstallationPolicy(t, tc) + }) + } +} + +func assertDockerInstallationPolicy(t *testing.T, tc dockerInstallationCase) { + t.Helper() + if runtime.GOOS == "darwin" && tc.wantCalls > 0 { + tc.wantCalls = 0 + tc.wantPath = "" + tc.wantError = "not found" + } + bin := t.TempDir() + t.Setenv("PATH", bin) + if tc.existing { + writeDockerFixture(t, bin, tc.path) + } + info := &provider.AgentWorkspaceInfo{} + info.Agent.Docker.Path = tc.path + info.Agent.Docker.Install = types.StrBool(tc.policy) + calls := 0 + initializer := &workspaceInitializer{ + workspaceInfo: info, + findDarwinDocker: func() (string, error) { + return "", errors.New("not found") + }, + installDocker: func(context.Context) (string, error) { + calls++ + return testInstalledDocker, nil + }, + } + path, err := initializer.ensureDockerInstalled(context.Background()) + if tc.wantError == "" { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tc.wantError) + } + assert.Equal(t, tc.wantCalls, calls) + assert.Equal(t, tc.wantPath, path) +} + +func writeDockerFixture(t *testing.T, bin, configuredPath string) { + t.Helper() + name := configuredPath + if name == "" { + name = docker + } + if runtime.GOOS == "windows" && !strings.HasSuffix(strings.ToLower(name), ".exe") { + name += ".exe" + } + path := filepath.Join(bin, name) + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + if err := os.Link(executable, path); err == nil { + return + } + copyDockerFixture(t, executable, path) +} + +func copyDockerFixture(t *testing.T, executable, path string) { + t.Helper() + root, err := os.OpenRoot(filepath.Dir(executable)) + if err != nil { + t.Fatal(err) + } + source, err := root.Open(filepath.Base(executable)) + if err != nil { + _ = root.Close() + t.Fatal(err) + } + info, err := source.Stat() + if err != nil { + _ = source.Close() + _ = root.Close() + t.Fatal(err) + } + targetRoot, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + _ = source.Close() + _ = root.Close() + t.Fatal(err) + } + target, err := targetRoot.OpenFile( + filepath.Base(path), + os.O_WRONLY|os.O_CREATE|os.O_EXCL, + info.Mode().Perm()&0o700, + ) + if err != nil { + _ = source.Close() + _ = root.Close() + _ = targetRoot.Close() + t.Fatal(err) + } + _, copyErr := io.Copy(target, source) + if err := errors.Join( + copyErr, + source.Close(), + target.Close(), + root.Close(), + targetRoot.Close(), + ); err != nil { + t.Fatal(err) + } +} + +func TestDockerInstallDisabledTriState(t *testing.T) { + for _, tc := range []struct { + policy types.StrBool + disabled bool + }{ + {policy: "", disabled: false}, + {policy: dockerPolicyEnabled, disabled: false}, + {policy: dockerPolicyDisabled, disabled: true}, + } { + info := &provider.AgentWorkspaceInfo{} + info.Agent.Docker.Install = tc.policy + initializer := &workspaceInitializer{workspaceInfo: info} + if got := initializer.isDockerInstallDisabled(); got != tc.disabled { + t.Errorf("install %q disabled = %t, want %t", tc.policy, got, tc.disabled) + } + } +} + +func TestDockerManagementPolicy(t *testing.T) { + for _, driver := range []string{provider.DockerDriver, provider.KubernetesDriver} { + for _, local := range []string{dockerPolicyDisabled, dockerPolicyEnabled} { + for _, policy := range []string{"", dockerPolicyEnabled, dockerPolicyDisabled, dockerPolicyInvalid} { + t.Run(driver+"/"+local+"/"+policy, func(t *testing.T) { + assertDockerManagementPolicy(t, driver, local, policy) + }) + } + } + } +} + +func assertDockerManagementPolicy(t *testing.T, driver, local, policy string) { + t.Helper() + info := &provider.AgentWorkspaceInfo{} + info.Agent.Driver = driver + info.Agent.Local = types.StrBool(local) + info.Agent.Docker.ConfigureDaemon = types.StrBool(policy) + configureCalls := 0 + initializer := &workspaceInitializer{ + workspaceInfo: info, + configureDocker: func(context.Context) error { + configureCalls++ + return nil + }, + } + err := initializer.validateDockerManagement() + if policy == dockerPolicyInvalid { + assert.ErrorContains(t, err, "agent.docker.configureDaemon") + } else { + assert.NoError(t, err) + } + initializer.tryConfigureDockerDaemon(context.Background()) + want := 0 + if driver == provider.DockerDriver && local == dockerPolicyDisabled && + policy != dockerPolicyDisabled && policy != dockerPolicyInvalid { + want = 1 + } + assert.Equal(t, want, configureCalls) +} + +func TestInvalidDockerManagementFailsBeforeInitialization(t *testing.T) { + for _, field := range []string{"install", "configureDaemon"} { + t.Run(field, func(t *testing.T) { + info := &provider.AgentWorkspaceInfo{} + if field == "install" { + info.Agent.Docker.Install = dockerPolicyInvalid + } else { + info.Agent.Docker.ConfigureDaemon = dockerPolicyInvalid + } + calls := 0 + initializer := &workspaceInitializer{ + workspaceInfo: info, + installDocker: func(context.Context) (string, error) { + calls++ + return "", nil + }, + configureDocker: func(context.Context) error { + calls++ + return nil + }, + } + if err := initializer.initialize( + context.Background(), + ); err == nil || + !strings.Contains(err.Error(), field) { + t.Errorf("initialize() error = %v, want invalid %s error", err, field) + } + if calls != 0 { + t.Errorf("invalid policy triggered %d Docker management calls", calls) + } + }) + } +} diff --git a/cmd/internal/internal.go b/cmd/internal/internal.go index 5d011c36cc..c2d946b19a 100644 --- a/cmd/internal/internal.go +++ b/cmd/internal/internal.go @@ -16,6 +16,7 @@ func NewInternalCmd(globalFlags *flags.GlobalFlags) *cobra.Command { } cmd.AddCommand(NewAgentCmd(globalFlags)) cmd.AddCommand(NewRuntimeSupervisorCmd()) + cmd.AddCommand(NewProviderShellCmd()) cmd.AddCommand(NewGHCmd()) cmd.AddCommand(NewGHBackendCmd()) cmd.AddCommand(NewDaemonLocalCmd(globalFlags)) diff --git a/cmd/internal/provider_shell.go b/cmd/internal/provider_shell.go new file mode 100644 index 0000000000..89c2ce700d --- /dev/null +++ b/cmd/internal/provider_shell.go @@ -0,0 +1,50 @@ +package cmdinternal + +import ( + "context" + "errors" + "os" + + "github.com/devsy-org/devsy/pkg/shell" + "github.com/spf13/cobra" + "mvdan.cc/sh/v3/interp" +) + +func NewProviderShellCmd() *cobra.Command { + var scriptPath, forcePath, ackPath string + var parentPID int + command := &cobra.Command{ + Use: "provider-shell", Hidden: true, Args: cobra.NoArgs, + PersistentPreRunE: func(*cobra.Command, []string) error { return nil }, + RunE: func(cmd *cobra.Command, _ []string) error { + cleanup, err := shell.PrepareProviderShellForeground(parentPID) + if err != nil { + return err + } + defer cleanup() + err = shell.RunProviderShell(cmd.Context(), shell.ProviderShellInvocation{ + ScriptPath: scriptPath, + ForcePath: forcePath, + AckPath: ackPath, + ParentPID: parentPID, + }) + if status, ok := errors.AsType[interp.ExitStatus](err); ok { + os.Exit(int(status)) + } + if errors.Is(err, context.Canceled) { + os.Exit(1) + } + return err + }, + } + command.Flags().StringVar(&scriptPath, "script-file", "", "Private provider script file") + command.Flags().StringVar(&forcePath, "force-file", "", "Private provider force marker") + command.Flags().StringVar(&ackPath, "ack-file", "", "Private provider acknowledgement marker") + command.Flags().IntVar(&parentPID, "parent-pid", 0, "Provider parent process ID") + _ = command.Flags().MarkHidden("parent-pid") + _ = command.MarkFlagRequired("parent-pid") + _ = command.MarkFlagRequired("script-file") + _ = command.MarkFlagRequired("force-file") + _ = command.MarkFlagRequired("ack-file") + return command +} diff --git a/cmd/internal/provider_shell_test.go b/cmd/internal/provider_shell_test.go new file mode 100644 index 0000000000..716856d41a --- /dev/null +++ b/cmd/internal/provider_shell_test.go @@ -0,0 +1,62 @@ +package cmdinternal + +import ( + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/devsy-org/devsy/pkg/config" + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" +) + +func TestProviderShellCommandRegistration(t *testing.T) { + command := NewProviderShellCmd() + require.Equal(t, "provider-shell", command.Name()) + require.True(t, command.Hidden) + dir := providerShellCLITestDir(t) + script := filepath.Join(dir, "script") + require.NoError(t, os.WriteFile(script, []byte("exit 0"), 0o600)) + token := strings.Repeat("0123456789abcdef", 4) + require.NoError(t, os.WriteFile(filepath.Join(dir, "ack-key"), []byte(token), 0o600)) + parent := &cobra.Command{ + Use: "devsy", + PersistentPreRunE: func(*cobra.Command, []string) error { + return config.ErrContextDeletionPending + }, + } + parent.AddCommand(command) + parent.SetArgs([]string{ + "provider-shell", + "--script-file", script, "--force-file", filepath.Join(dir, "force"), + "--ack-file", filepath.Join(dir, "ack"), "--parent-pid", strconv.Itoa(os.Getppid()), + }) + require.NoError(t, parent.Execute()) + _, err := os.Lstat(filepath.Join(dir, "ack-key")) + require.ErrorIs(t, err, os.ErrNotExist) + root, err := os.OpenRoot(dir) + require.NoError(t, err) + ack, readErr := root.ReadFile("ack") + closeErr := root.Close() + require.NoError(t, readErr) + require.NoError(t, closeErr) + require.Equal(t, token, string(ack)) +} + +func TestProviderShellCommandRejectsMissingToken(t *testing.T) { + dir := providerShellCLITestDir(t) + script := filepath.Join(dir, "script") + require.NoError(t, os.WriteFile(script, []byte("printf should-not-run"), 0o600)) + _, err := os.Lstat(filepath.Join(dir, "ack-key")) + require.ErrorIs(t, err, os.ErrNotExist) + command := NewProviderShellCmd() + command.SetArgs([]string{ + "--script-file", script, "--force-file", filepath.Join(dir, "force"), + "--ack-file", filepath.Join(dir, "ack"), "--parent-pid", strconv.Itoa(os.Getppid()), + }) + require.ErrorContains(t, command.Execute(), "invalid provider shell acknowledgement token") + _, err = os.Lstat(filepath.Join(dir, "ack")) + require.ErrorIs(t, err, os.ErrNotExist) +} diff --git a/cmd/internal/provider_shell_unix_test.go b/cmd/internal/provider_shell_unix_test.go new file mode 100644 index 0000000000..0891f56b7d --- /dev/null +++ b/cmd/internal/provider_shell_unix_test.go @@ -0,0 +1,17 @@ +//go:build !windows + +package cmdinternal + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func providerShellCLITestDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp(t.TempDir(), "private-") + require.NoError(t, err) + return dir +} diff --git a/cmd/internal/provider_shell_windows_test.go b/cmd/internal/provider_shell_windows_test.go new file mode 100644 index 0000000000..8e8a30a391 --- /dev/null +++ b/cmd/internal/provider_shell_windows_test.go @@ -0,0 +1,28 @@ +package cmdinternal + +import ( + "path/filepath" + "testing" + "unsafe" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" +) + +func providerShellCLITestDir(t *testing.T) string { + t.Helper() + user, err := windows.GetCurrentProcessToken().GetTokenUser() + require.NoError(t, err) + sd, err := windows.SecurityDescriptorFromString( + "D:P(A;OICI;GA;;;" + user.User.Sid.String() + ")(A;OICI;GA;;;SY)") + require.NoError(t, err) + attributes := windows.SecurityAttributes{ + Length: uint32(unsafe.Sizeof(windows.SecurityAttributes{})), + SecurityDescriptor: sd, + } + path := filepath.Join(t.TempDir(), "private") + name, err := windows.UTF16PtrFromString(path) + require.NoError(t, err) + require.NoError(t, windows.CreateDirectory(name, &attributes)) + return path +} diff --git a/cmd/provider/set_source_init_failure_test.go b/cmd/provider/set_source_init_failure_test.go index 65df1b6d77..abfef2c39d 100644 --- a/cmd/provider/set_source_init_failure_test.go +++ b/cmd/provider/set_source_init_failure_test.go @@ -47,8 +47,8 @@ func TestSetSourceInitFailureKeepsCommittedReplacementAndWorkspace(t *testing.T) defer cancel() cmd := &SetSourceCmd{GlobalFlags: &flags.GlobalFlags{}, Use: true} err = cmd.Run(ctx, initial, []string{testProviderFoo, replacementSource}) - require.ErrorContains(t, err, "init:") - require.ErrorContains(t, err, "missing-init-executable") + require.Error(t, err) + require.NotErrorIs(t, err, context.DeadlineExceeded) assertFailedSourceReplacement( t, @@ -115,7 +115,6 @@ func assertFailedSourceReplacement( require.NotNil(t, state) require.False(t, state.Initialized) require.True(t, state.InitAttempted) - require.Contains(t, state.InitError, "missing-init-executable") require.Equal(t, userOption.Value, state.Options["REGION"].Value) require.True(t, state.Options["REGION"].UserProvided) require.Equal(t, testProviderFoo, stored.Current().DefaultProvider) diff --git a/pkg/client/clientimplementation/machine_client.go b/pkg/client/clientimplementation/machine_client.go index a60b83e29d..f565b7a5ef 100644 --- a/pkg/client/clientimplementation/machine_client.go +++ b/pkg/client/clientimplementation/machine_client.go @@ -55,16 +55,17 @@ type machineExecutor struct { // execConfig defines how to execute a machine command. type execConfig struct { - name string - command types.StrArray - stdout io.Writer - stderr io.Writer - extraEnv map[string]string - stdin io.Reader - rawStdout bool - withProgress bool - startMsg string - doneMsg string + name string + command types.StrArray + stdout io.Writer + stderr io.Writer + extraEnv map[string]string + stdin io.Reader + closeInputOnExit io.Closer + rawStdout bool + withProgress bool + startMsg string + doneMsg string } func (e *machineExecutor) execute(ctx context.Context, cfg execConfig) error { @@ -88,6 +89,7 @@ func (e *machineExecutor) execute(ctx context.Context, cfg execConfig) error { Stderr: cfg.stderr, ExtraEnv: cfg.extraEnv, Stdin: cfg.stdin, + closeInputOnExit: cfg.closeInputOnExit, RawStdout: cfg.rawStdout, }) if err != nil { @@ -222,7 +224,7 @@ func (s *machineClient) Stop(ctx context.Context, options client.StopOptions) er func (s *machineClient) Command(ctx context.Context, commandOptions client.CommandOptions) error { return s.executor.execute(ctx, execConfig{ - name: "command", + name: providerCommandAction, command: s.config.Exec.Command, stdout: commandOptions.Stdout, stderr: commandOptions.Stderr, @@ -241,9 +243,17 @@ func (s *machineClient) OpenCommandTransport( return transport.OpenCallbackConn( ctx, func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { - return s.Command(ctx, client.CommandOptions{ - Command: opt.Command, Stdin: stdin, Stdout: stdout, Stderr: opt.Stderr, - RawStdout: true, + return s.executor.execute(ctx, execConfig{ + name: providerCommandAction, + command: s.config.Exec.Command, + stdout: stdout, + stderr: opt.Stderr, + stdin: stdin, + closeInputOnExit: stdin.(io.Closer), + rawStdout: true, + extraEnv: map[string]string{ + provider.CommandEnv: opt.Command, + }, }) }, transport.CallbackConnOptions{ diff --git a/pkg/client/clientimplementation/machine_proxy_transport_completion_test.go b/pkg/client/clientimplementation/machine_proxy_transport_completion_test.go new file mode 100644 index 0000000000..b2d4a12054 --- /dev/null +++ b/pkg/client/clientimplementation/machine_proxy_transport_completion_test.go @@ -0,0 +1,93 @@ +package clientimplementation + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/client" + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/transport" + "github.com/devsy-org/devsy/pkg/types" + "github.com/stretchr/testify/require" +) + +const testProviderCreateCommand = "create" + +func TestMachineCommandTransportCompletesWithoutClosingInput(t *testing.T) { + payload := strings.Repeat("m", 128*1024) + const stderrTail = "machine stderr tail" + command := "printf '%s' '" + payload + "'; printf '%s' '" + stderrTail + "' >&2" + cfg := &provider.ProviderConfig{ + Name: testCancelProvider, + Exec: provider.ProviderCommands{ + Create: types.StrArray{testProviderCreateCommand}, + Command: types.StrArray{command}, + Cancellation: provider.ProviderCancellationFile, + }, + } + devsy := &config.Config{ + DefaultContext: testLocksContext, + Contexts: map[string]*config.ContextConfig{testLocksContext: {}}, + } + machine, err := NewMachineClient( + devsy, + cfg, + &provider.Machine{ID: "completion-machine", Context: testLocksContext}, + ) + require.NoError(t, err) + var stderr bytes.Buffer + conn, err := machine.OpenCommandTransport(context.Background(), client.CommandTransportOptions{ + Command: testOpaqueCommand, + Stderr: &stderr, + }) + require.NoError(t, err) + requireTransportCompletionWithOpenInput(t, conn, payload) + require.Equal(t, stderrTail, stderr.String()) +} + +func TestProxySSHTransportCompletesWithoutClosingInput(t *testing.T) { + payload := strings.Repeat("p", 128*1024) + workspace := workspaceCancellationTestClient() + workspace.providerConfig.Exec.Proxy = &provider.ProxyCommands{ + Ssh: types.StrArray{"printf '%s' '" + payload + "'"}, + } + proxy, err := NewProxyClient( + workspace.devsyConfig, + workspace.providerConfig, + workspace.providerWorkspace, + ) + require.NoError(t, err) + conn, err := proxy.OpenSSHTransport(context.Background(), client.SSHTransportOptions{}) + require.NoError(t, err) + requireTransportCompletionWithOpenInput(t, conn, payload) +} + +func requireTransportCompletionWithOpenInput( + t *testing.T, + conn transport.ManagedConn, + payload string, +) { + t.Helper() + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + got := make([]byte, len(payload)) + _, err := io.ReadFull(conn, got) + require.NoError(t, err) + require.Equal(t, payload, string(got)) + finished := make(chan error, 1) + go func() { finished <- conn.Wait() }() + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("provider completion waited for caller to close transport input") + } + var trailing [1]byte + _, err = conn.Read(trailing[:]) + require.ErrorIs(t, err, io.EOF) +} diff --git a/pkg/client/clientimplementation/provider_cancel.go b/pkg/client/clientimplementation/provider_cancel.go new file mode 100644 index 0000000000..d5e5ed7a60 --- /dev/null +++ b/pkg/client/clientimplementation/provider_cancel.go @@ -0,0 +1,370 @@ +package clientimplementation + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/devsy-org/devsy/pkg/config" +) + +const ( + providerCancellationGrace = 10 * time.Second + providerForceAckWait = 5 * time.Second +) + +func runProviderWithCancellation(parent context.Context, opts RunCommandOptions) error { + return runProviderWithCancellationUsingPublisher(parent, opts, publishProviderCancellation) +} + +func runProviderWithCancellationUsingPublisher( + parent context.Context, + opts RunCommandOptions, + publish func(string) error, +) (retErr error) { + if parent.Err() != nil { + return context.Cause(parent) + } + if len(opts.Command) == 0 { + return nil + } + + dir, err := createProviderInvocationDir() + if err != nil { + return fmt.Errorf("create provider cancellation directory: %w", err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + retErr = errors.Join( + retErr, + fmt.Errorf("remove provider cancellation directory: %w", sanitizeMarkerError(err)), + ) + } + }() + + marker := filepath.Join(dir, "cancel") + if len(opts.Command) == 1 { + opts.providerShell, err = prepareProviderShell(dir, opts.Command[0]) + if err != nil { + return err + } + } + opts.Environ = providerCancellationEnvironment(opts.Environ, marker) + execution, cancel := context.WithCancelCause(context.WithoutCancel(parent)) + defer cancel(context.Canceled) + if parent.Err() != nil { + return context.Cause(parent) + } + completed := make(chan error, 1) + go func() { + completed <- runCommandExecution(execution, opts) + }() + return waitForProviderCancellation( + parent, + marker, + publish, + providerExecution{cancel: cancel, completed: completed, shell: opts.providerShell}, + ) +} + +func prepareProviderShell(dir, script string) (*providerShellInvocation, error) { + var token [32]byte + if _, err := rand.Read(token[:]); err != nil { + return nil, fmt.Errorf("generate provider shell acknowledgement token: %w", err) + } + files := &providerShellInvocation{ + script: filepath.Join(dir, "script"), + force: filepath.Join(dir, "force"), + ack: filepath.Join(dir, "ack"), + ackToken: hex.EncodeToString(token[:]), + } + if err := writeProviderScript(dir, script); err != nil { + return nil, fmt.Errorf("write provider script: %w", err) + } + if err := writeProviderAckKey(dir, files.ackToken); err != nil { + return nil, fmt.Errorf("write provider acknowledgement key: %w", err) + } + return files, nil +} + +func providerCancellationEnvironment(original []string, marker string) []string { + if original == nil { + original = os.Environ() + } + environ := make([]string, 0, len(original)+1) + for _, kv := range original { + key, _, ok := strings.Cut(kv, "=") + if ok && providerReservedKey(key) { + continue + } + environ = append(environ, kv) + } + return append(environ, config.EnvProviderCancelFile+"="+marker) +} + +type providerShellInvocation struct { + script string + force string + ack string + ackToken string +} + +type providerExecution struct { + cancel context.CancelCauseFunc + completed <-chan error + shell *providerShellInvocation +} + +func waitForProviderCancellation( + parent context.Context, + marker string, + publish func(string) error, + process providerExecution, +) error { + select { + case result := <-process.completed: + if parent.Err() != nil { + return providerCancellationCompletion(context.Cause(parent), process) + } + if process.shell != nil { + return errors.Join(result, providerForceCompletionStatus(process)) + } + return result + case <-parent.Done(): + } + + cause := context.Cause(parent) + if err := publish(marker); err != nil { + return errors.Join( + cause, + fmt.Errorf("publish provider cancellation marker: %w", sanitizeMarkerError(err)), + forceProviderExecution(cause, process), + ) + } + return waitForProviderGrace(cause, process) +} + +func waitForProviderGrace(cause error, process providerExecution) error { + timer := time.NewTimer(providerCancellationGrace) + defer timer.Stop() + select { + case <-process.completed: + return providerCancellationCompletion(cause, process) + case <-timer.C: + } + return errors.Join(cause, forceProviderExecution(cause, process)) +} + +func providerCancellationCompletion(cause error, process providerExecution) error { + if process.shell == nil { + return cause + } + return errors.Join(cause, providerForceCompletionStatus(process)) +} + +func forceProviderExecution(cause error, process providerExecution) error { + if process.shell == nil { + stopProviderExecution(cause, process) + return nil + } + select { + case <-process.completed: + return providerForceCompletionStatus(process) + default: + } + if err := publishProviderMarker(process.shell.force); err != nil { + stopProviderExecution(cause, process) + return fmt.Errorf("publish provider force marker: %w", err) + } + return waitForProviderForceAck(cause, process) +} + +func waitForProviderForceAck(cause error, process providerExecution) error { + timer := time.NewTimer(providerForceAckWait) + defer timer.Stop() + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-process.completed: + return providerForceCompletionStatus(process) + case <-ticker.C: + acknowledged, err := providerForceAcknowledged(process.shell) + if err != nil { + stopProviderExecution(cause, process) + return fmt.Errorf("inspect provider force acknowledgement: %w", err) + } + if acknowledged { + stopProviderExecution(cause, process) + return nil + } + case <-timer.C: + return finishProviderForceAtDeadline(cause, process) + } + } +} + +func providerForceCompletionStatus(process providerExecution) error { + acknowledged, err := providerForceAcknowledged(process.shell) + if err != nil { + return fmt.Errorf("inspect provider force acknowledgement: %w", err) + } + if !acknowledged { + return errors.New("provider helper exited without authentic acknowledgement") + } + return nil +} + +func finishProviderForceAtDeadline(cause error, process providerExecution) error { + acknowledged, err := providerForceAcknowledged(process.shell) + stopProviderExecution(cause, process) + if err != nil { + return fmt.Errorf("inspect provider force acknowledgement: %w", err) + } + if !acknowledged { + return errors.New("provider force acknowledgement timed out") + } + return nil +} + +func stopProviderExecution(cause error, process providerExecution) { + process.cancel(cause) + <-process.completed +} + +func writeProviderScript(dir, script string) (retErr error) { + root, err := os.OpenRoot(dir) + if err != nil { + return sanitizeMarkerError(err) + } + defer func() { retErr = errors.Join(retErr, sanitizeMarkerError(root.Close())) }() + f, err := root.OpenFile("script", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return sanitizeMarkerError(err) + } + _, writeErr := f.WriteString(script) + return errors.Join(sanitizeMarkerError(writeErr), sanitizeMarkerError(f.Close())) +} + +func writeProviderAckKey(dir, token string) (retErr error) { + root, err := os.OpenRoot(dir) + if err != nil { + return sanitizeMarkerError(err) + } + defer func() { retErr = errors.Join(retErr, sanitizeMarkerError(root.Close())) }() + f, err := root.OpenFile("ack-key", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return sanitizeMarkerError(err) + } + _, writeErr := f.WriteString(token) + return errors.Join(sanitizeMarkerError(writeErr), sanitizeMarkerError(f.Close())) +} + +func providerForceAcknowledged(files *providerShellInvocation) (acknowledged bool, retErr error) { + root, err := os.OpenRoot(filepath.Dir(files.ack)) + if err != nil { + return false, sanitizeMarkerError(err) + } + defer func() { retErr = errors.Join(retErr, sanitizeMarkerError(root.Close())) }() + return providerForceAckInRoot(root, files.ackToken) +} + +func providerForceAckInRoot(root *os.Root, token string) (acknowledged bool, retErr error) { + info, err := root.Lstat("ack") + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, sanitizeMarkerError(err) + } + if !privateProviderAck(info, len(token)) { + return false, nil + } + f, err := root.Open("ack") + if err != nil { + return false, sanitizeMarkerError(err) + } + defer func() { retErr = errors.Join(retErr, sanitizeMarkerError(f.Close())) }() + return providerForceAckContents(f, token) +} + +func providerForceAckContents(f *os.File, token string) (bool, error) { + info, err := f.Stat() + if err != nil { + return false, sanitizeMarkerError(err) + } + if !privateProviderAck(info, len(token)) { + return false, nil + } + data := make([]byte, len(token)) + if _, err := io.ReadFull(f, data); err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return false, nil + } + return false, sanitizeMarkerError(err) + } + return subtle.ConstantTimeCompare(data, []byte(token)) == 1, nil +} + +func privateProviderAck(info os.FileInfo, length int) bool { + return info.Mode().IsRegular() && info.Size() == int64(length) && + (runtime.GOOS == "windows" || info.Mode().Perm() == 0o600) +} + +func providerReservedKey(key string) bool { + if runtime.GOOS == "windows" { + return strings.EqualFold(key, config.EnvProviderCancelFile) + } + return key == config.EnvProviderCancelFile +} + +func publishProviderCancellation(marker string) error { + return publishProviderMarker(marker) +} + +func publishProviderMarker(marker string) (retErr error) { + root, err := os.OpenRoot(filepath.Dir(marker)) + if err != nil { + return fmt.Errorf("cannot open marker directory: %w", sanitizeMarkerError(err)) + } + defer func() { + if err := root.Close(); err != nil { + retErr = errors.Join(retErr, sanitizeMarkerError(err)) + } + }() + name := filepath.Base(marker) + f, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err == nil { + return sanitizeMarkerError(f.Close()) + } + if !errors.Is(err, os.ErrExist) { + return fmt.Errorf("cannot create marker: %w", sanitizeMarkerError(err)) + } + info, statErr := root.Lstat(name) + if statErr != nil { + return fmt.Errorf("cannot inspect existing marker: %w", sanitizeMarkerError(statErr)) + } + if !info.Mode().IsRegular() { + return errors.New("existing marker is not a regular file") + } + return nil +} + +func sanitizeMarkerError(err error) error { + if err == nil { + return nil + } + if pathErr, ok := errors.AsType[*os.PathError](err); ok { + return pathErr.Err + } + return errors.New("marker filesystem operation failed") +} diff --git a/pkg/client/clientimplementation/provider_cancel_test.go b/pkg/client/clientimplementation/provider_cancel_test.go new file mode 100644 index 0000000000..60c2d20ea7 --- /dev/null +++ b/pkg/client/clientimplementation/provider_cancel_test.go @@ -0,0 +1,1281 @@ +package clientimplementation + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "os/signal" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/client" + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/shell" + "github.com/devsy-org/devsy/pkg/types" + "github.com/stretchr/testify/require" + "mvdan.cc/sh/v3/interp" +) + +func TestMain(m *testing.M) { + if len(os.Args) == 11 && os.Args[1] == "internal" && os.Args[2] == "provider-shell" { + os.Exit(runProviderShellTestHelper(os.Args)) + } + os.Exit(m.Run()) +} + +func runProviderShellTestHelper(args []string) int { + if !validProviderShellTestArgs(args) { + return 1 + } + parentPID, err := strconv.Atoi(args[10]) + if err != nil { + return 1 + } + cleanup, err := shell.PrepareProviderShellForeground(parentPID) + if err != nil { + return 1 + } + defer cleanup() + err = shell.RunProviderShell(context.Background(), shell.ProviderShellInvocation{ + ScriptPath: args[4], ForcePath: args[6], AckPath: args[8], ParentPID: parentPID, + }) + if err == nil { + return 0 + } + if status, ok := errors.AsType[interp.ExitStatus](err); ok { + return int(status) + } + return 1 +} + +func validProviderShellTestArgs(args []string) bool { + return args[3] == "--script-file" && args[5] == "--force-file" && + args[7] == "--ack-file" && args[9] == "--parent-pid" +} + +const ( + testTransportRead = "transport-read" + testWindowsOS = "windows" + testCancelProvider = "cancel-provider" + testOpaqueCommand = "opaque" + testCancellationHelperArg = "-test.run=^TestProviderCancellationHelper$" +) + +func TestProviderCancellationHelper(t *testing.T) { + if os.Getenv("DEVSY_CANCEL_HELPER") != "1" { + return + } + if os.Getenv("DEVSY_CANCEL_HOLDER") == "1" { + time.Sleep(time.Minute) + return + } + runTestProviderCancellationBehavior(os.Getenv(config.EnvProviderCancelFile)) + os.Exit(0) +} + +func runTestProviderCancellationBehavior(marker string) { + switch behavior := os.Getenv("DEVSY_CANCEL_BEHAVIOR"); behavior { + case "acknowledge": + runAcknowledgingCancellationHelper(marker) + case "ignore": + runIgnoringCancellationHelper() + case testTransportRead, "transport-write": + testTransportCancellationHelper(behavior, marker) + case "bulk-after-eof": + writeTestCancellationBulkAfterEOF() + case "orphan-output": + startTestCancellationOutputHolder() + case "orphan-input": + startTestCancellationInputHolder() + default: + runTestCancellationExit(behavior, marker) + } +} + +func runAcknowledgingCancellationHelper(marker string) { + if _, err := fmt.Fprint(os.Stdout, "ready"); err != nil { + os.Exit(126) + } + awaitTestCancellationMarker(marker, os.Getenv("DEVSY_CANCEL_ACK_PATH")) +} + +func runIgnoringCancellationHelper() { + if path := os.Getenv("DEVSY_CANCEL_CHILD_PID_PATH"); path != "" { + exitTestHelperOnError(testHelperWriteFile(path, strconv.Itoa(os.Getpid()))) + } + if _, err := fmt.Fprint(os.Stdout, "ready"); err != nil { + os.Exit(126) + } + time.Sleep(time.Minute) +} + +func runTestCancellationExit(behavior, marker string) { + switch behavior { + case "status": + os.Exit(7) + case "status-127": + os.Exit(127) + case "bytes": + writeTestCancellationBytes() + case "inherited": + _, err := fmt.Fprint(os.Stdout, os.Getenv("DEVSY_CANCEL_INHERITED")) + exitTestHelperOnError(err) + case "environment": + _, err := fmt.Fprint(os.Stdout, marker) + exitTestHelperOnError(err) + case "ack-private": + runTestCancellationAckPrivate(marker) + default: + os.Exit(125) + } +} + +func runTestCancellationAckPrivate(marker string) { + root, err := os.OpenRoot(filepath.Dir(marker)) + exitTestHelperOnError(err) + _, statErr := root.Lstat("ack-key") + exitTestHelperOnError(root.Close()) + if !errors.Is(statErr, os.ErrNotExist) { + os.Exit(126) + } + _, err = os.Stdout.Write([]byte(strings.Join(os.Environ(), "\x00"))) + exitTestHelperOnError(err) + if runtime.GOOS == "linux" { + proc, openErr := os.OpenRoot(fmt.Sprintf("/proc/%d", os.Getppid())) + exitTestHelperOnError(openErr) + for _, name := range []string{"environ", "cmdline"} { + data, readErr := proc.ReadFile(name) + exitTestHelperOnError(readErr) + _, err = os.Stdout.Write(data) + exitTestHelperOnError(err) + } + exitTestHelperOnError(proc.Close()) + } +} + +func writeTestCancellationBytes() { + _, err := os.Stdout.Write([]byte{0, 0xff, 0x80, '\r', '\n'}) + exitTestHelperOnError(err) + _, err = os.Stderr.Write([]byte(os.Getenv("DEVSY_CANCEL_SECRET"))) + exitTestHelperOnError(err) +} + +func writeTestCancellationBulkAfterEOF() { + _, err := io.Copy(io.Discard, os.Stdin) + exitTestHelperOnError(err) + block := make([]byte, 32<<10) + for i := range block { + block[i] = byte(i) + } + for range (12 << 20) / len(block) { + _, err := os.Stdout.Write(block) + exitTestHelperOnError(err) + } + _, err = os.Stderr.Write([]byte("separate stderr")) + exitTestHelperOnError(err) +} + +func startTestCancellationOutputHolder() { + holder := &exec.Cmd{ + Path: os.Args[0], + Args: []string{os.Args[0], testCancellationHelperArg}, + } + holder.Env = append(os.Environ(), "DEVSY_CANCEL_HOLDER=1") + holder.Stdout, holder.Stderr = os.Stdout, os.Stderr + exitTestHelperOnError(holder.Start()) + exitTestHelperOnError(testHelperWriteFile( + os.Getenv("DEVSY_CANCEL_HOLDER_PID_PATH"), strconv.Itoa(holder.Process.Pid), + )) + exitTestHelperOnError(holder.Process.Release()) + _, err := fmt.Fprint(os.Stdout, "ready") + exitTestHelperOnError(err) +} + +func startTestCancellationInputHolder() { + holder := &exec.Cmd{Path: os.Args[0], Args: []string{os.Args[0], testCancellationHelperArg}} + holder.Env = append(os.Environ(), "DEVSY_CANCEL_HOLDER=1") + holder.Stdin = os.Stdin + exitTestHelperOnError(holder.Start()) + exitTestHelperOnError( + testHelperWriteFile( + os.Getenv("DEVSY_CANCEL_HOLDER_PID_PATH"), + strconv.Itoa(holder.Process.Pid), + ), + ) + exitTestHelperOnError(holder.Process.Release()) + writeTestCancellationBytes() +} + +func exitTestHelperOnError(err error) { + if err != nil { + os.Exit(126) + } +} + +func testHelperRelativePath(path string) (string, error) { + relative, err := filepath.Rel(os.TempDir(), path) + if err != nil { + return "", err + } + if !filepath.IsLocal(relative) { + return "", os.ErrPermission + } + return relative, nil +} + +func testHelperWriteFile(path, data string) error { + relative, err := testHelperRelativePath(path) + if err != nil { + return err + } + root, err := os.OpenRoot(os.TempDir()) + if err != nil { + return err + } + writeErr := root.WriteFile(relative, []byte(data), 0o600) + return errors.Join(writeErr, root.Close()) +} + +func awaitTestCancellationMarker(marker, acknowledgement string) { + relative, err := testHelperRelativePath(marker) + if err != nil { + os.Exit(126) + } + root, err := os.OpenRoot(os.TempDir()) + if err != nil { + os.Exit(126) + } + for { + if info, err := root.Lstat(relative); err == nil && info.Mode().IsRegular() { + acknowledgeTestCancellationMarker(root, acknowledgement) + } + time.Sleep(20 * time.Millisecond) + } +} + +func acknowledgeTestCancellationMarker(root *os.Root, acknowledgement string) { + if acknowledgement != "" { + exitTestHelperOnError(testHelperWriteFile(acknowledgement, "finished")) + } + exitTestHelperOnError(root.Close()) + os.Exit(0) +} + +func testTransportCancellationHelper(behavior, marker string) { + if err := testHelperWriteFile(os.Getenv("DEVSY_CANCEL_READY_PATH"), "ready"); err != nil { + os.Exit(126) + } + go awaitTestCancellationMarker(marker, os.Getenv("DEVSY_CANCEL_ACK_PATH")) + if behavior == testTransportRead { + var b [1]byte + _, _ = os.Stdin.Read(b[:]) + for { + time.Sleep(time.Second) + } + } + if runtime.GOOS != testWindowsOS { + signal.Ignore(syscall.Signal(13)) + } + chunk := bytes.Repeat([]byte("x"), 1024) + for { + if _, err := os.Stdout.Write(chunk); err != nil { + for { + time.Sleep(time.Second) + } + } + } +} + +func cancelHelperOptions(behavior string) RunCommandOptions { + binary, err := os.Executable() + if err != nil { + panic(err) + } + return RunCommandOptions{ + Command: types.StrArray{binary, testCancellationHelperArg}, + Environ: append( + os.Environ(), + "DEVSY_CANCEL_HELPER=1", + "DEVSY_CANCEL_BEHAVIOR="+behavior, + ), + Cancellation: provider.ProviderCancellationFile, + } +} + +func TestProviderCancellationCooperatesAndCleansUp(t *testing.T) { + ack := filepath.Join(t.TempDir(), "ack") + opts := cancelHelperOptions("acknowledge") + opts.Environ = append(opts.Environ, "DEVSY_CANCEL_ACK_PATH="+ack) + ctx, cancel := context.WithCancel(context.Background()) + ready := make(chan struct{}) + opts.Stdout = &readyWriter{ready: ready} + opts.RawStdout = true + finished := make(chan error, 1) + go func() { finished <- RunCommand(ctx, opts) }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("provider did not start") + } + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("provider did not acknowledge cancellation") + } + requireTestAcknowledgement(t, ack) +} + +func requireTestAcknowledgement(t *testing.T, path string) { + t.Helper() + root, err := os.OpenRoot(filepath.Dir(path)) + require.NoError(t, err) + contents, readErr := root.ReadFile(filepath.Base(path)) + require.NoError(t, errors.Join(readErr, root.Close())) + require.Equal(t, "finished", string(contents)) +} + +type readyWriter struct{ ready chan struct{} } + +func (w *readyWriter) Write(p []byte) (int, error) { + if bytes.Contains(p, []byte("ready")) { + select { + case <-w.ready: + default: + close(w.ready) + } + } + return len(p), nil +} + +func TestProviderCancellationForcedAfterProductionGrace(t *testing.T) { + opts := cancelHelperOptions("ignore") + ctx, cancel := context.WithCancel(context.Background()) + ready := make(chan struct{}) + opts.Stdout = &readyWriter{ready: ready} + opts.RawStdout = true + finished := make(chan error, 1) + go func() { finished <- RunCommand(ctx, opts) }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("provider did not start") + } + start := time.Now() + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + require.GreaterOrEqual(t, time.Since(start), providerCancellationGrace) + case <-time.After(providerCancellationGrace + 8*time.Second): + t.Fatal("owned provider process was not reaped") + } +} + +type providerMarkerWriter struct { + mu sync.Mutex + output bytes.Buffer + ready chan struct{} +} + +func (w *providerMarkerWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + n, err := w.output.Write(p) + if strings.HasSuffix(w.output.String(), string(filepath.Separator)+"cancel") { + select { + case <-w.ready: + default: + close(w.ready) + } + } + return n, err +} + +func (w *providerMarkerWriter) marker() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.output.String() +} + +func TestProviderCancellationBlockedShellStdin(t *testing.T) { + input, writer, err := os.Pipe() + require.NoError(t, err) + defer func() { require.NoError(t, errors.Join(input.Close(), writer.Close())) }() + _, err = os.Stdin.Stat() + require.NoError(t, err) + + const privateCommand = "printf '%s' \"$DEVSY_PROVIDER_CANCEL_FILE\"; mapfile lines; printf 'private-command-846307'" + output := &providerMarkerWriter{ready: make(chan struct{})} + opts := RunCommandOptions{ + Command: types.StrArray{privateCommand}, + Environ: os.Environ(), + Stdin: input, + Stdout: output, + RawStdout: true, + Cancellation: provider.ProviderCancellationFile, + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + finished := make(chan error, 1) + go func() { finished <- RunCommand(ctx, opts) }() + select { + case <-output.ready: + case <-time.After(5 * time.Second): + t.Fatal("provider shell did not start") + } + marker := output.marker() + requirePrivateProviderScript(t, marker, privateCommand) + + start := time.Now() + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + require.GreaterOrEqual(t, time.Since(start), providerCancellationGrace) + require.NotContains(t, err.Error(), privateCommand) + require.NotContains(t, err.Error(), marker) + case <-time.After(providerCancellationGrace + providerForceAckWait + 3*time.Second): + t.Fatal("provider shell remained blocked on stdin") + } + _, err = os.Stdin.Stat() + require.NoError(t, err) + _, err = writer.Stat() + require.NoError(t, err) + _, err = os.Stat(filepath.Dir(marker)) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func requirePrivateProviderScript(t *testing.T, marker, command string) { + t.Helper() + require.True(t, filepath.IsAbs(marker)) + script := filepath.Join(filepath.Dir(marker), "script") + info, err := os.Stat(script) + require.NoError(t, err) + if runtime.GOOS != testWindowsOS { + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + } + root, err := os.OpenRoot(filepath.Dir(marker)) + require.NoError(t, err) + contents, readErr := root.ReadFile("script") + closeErr := root.Close() + require.NoError(t, readErr) + require.NoError(t, closeErr) + require.Equal(t, command, string(contents)) +} + +func TestProviderCancellationMissingForceAck(t *testing.T) { + dir := t.TempDir() + completed := make(chan error, 1) + cancelled := make(chan struct{}) + process := providerExecution{ + cancel: func(error) { + close(cancelled) + completed <- nil + }, + completed: completed, + shell: &providerShellInvocation{ + force: filepath.Join(dir, "force"), + ack: filepath.Join(dir, "ack"), + ackToken: strings.Repeat("a", 64), + }, + } + start := time.Now() + err := forceProviderExecution(context.Canceled, process) + require.ErrorContains(t, err, "provider force acknowledgement timed out") + require.GreaterOrEqual(t, time.Since(start), providerForceAckWait) + select { + case <-cancelled: + default: + t.Fatal("provider helper was not terminated") + } + info, err := os.Stat(process.shell.force) + require.NoError(t, err) + require.True(t, info.Mode().IsRegular()) +} + +func TestProviderCancellationEarlyExitWithoutForceAck(t *testing.T) { + dir := t.TempDir() + force := filepath.Join(dir, "force") + require.NoError(t, publishProviderMarker(force)) + completed := make(chan error, 1) + completed <- errors.New("helper exited") + process := providerExecution{ + cancel: func(error) { t.Fatal("exited helper must not be terminated again") }, + completed: completed, + shell: &providerShellInvocation{ + force: force, + ack: filepath.Join(dir, "ack"), + ackToken: strings.Repeat("a", 64), + }, + } + err := errors.Join(context.Canceled, waitForProviderForceAck(context.Canceled, process)) + require.ErrorIs(t, err, context.Canceled) + require.ErrorContains(t, err, "provider helper exited without authentic acknowledgement") + require.NotContains(t, err.Error(), dir) +} + +func TestProviderCancellationForgedAckBeforeHelperExit(t *testing.T) { + dir := t.TempDir() + token := strings.Repeat("a", 64) + ack := filepath.Join(dir, "ack") + require.NoError(t, os.WriteFile(ack, []byte(strings.Repeat("b", 64)), 0o600)) + completed := make(chan error, 1) + completed <- errors.New("helper died") + process := providerExecution{ + completed: completed, + shell: &providerShellInvocation{ + force: filepath.Join(dir, "force"), + ack: ack, + ackToken: token, + }, + } + err := waitForProviderGrace(context.Canceled, process) + require.ErrorIs(t, err, context.Canceled) + require.ErrorContains(t, err, "without authentic acknowledgement") + require.NotContains(t, err.Error(), token) + require.NotContains(t, err.Error(), dir) +} + +func TestProviderCancellationForgedAckAfterForce(t *testing.T) { + dir := t.TempDir() + token := strings.Repeat("a", 64) + ack := filepath.Join(dir, "ack") + require.NoError(t, os.WriteFile(ack, []byte(strings.Repeat("b", 64)), 0o600)) + force := filepath.Join(dir, "force") + require.NoError(t, publishProviderMarker(force)) + completed := make(chan error, 1) + completed <- errors.New("helper died") + process := providerExecution{ + completed: completed, + shell: &providerShellInvocation{ + force: force, + ack: ack, + ackToken: token, + }, + } + err := errors.Join(context.Canceled, waitForProviderForceAck(context.Canceled, process)) + require.ErrorIs(t, err, context.Canceled) + require.ErrorContains(t, err, "without authentic acknowledgement") + require.NotContains(t, err.Error(), token) +} + +func TestProviderForceAcknowledgedRequiresPrivateExactToken(t *testing.T) { + dir := t.TempDir() + files := &providerShellInvocation{ + ack: filepath.Join(dir, "ack"), + ackToken: strings.Repeat("a", 64), + } + for _, content := range []string{"", strings.Repeat("b", 64), strings.Repeat("a", 63), strings.Repeat("a", 65)} { + require.NoError(t, os.WriteFile(files.ack, []byte(content), 0o600)) + acknowledged, err := providerForceAcknowledged(files) + require.NoError(t, err) + require.False(t, acknowledged) + } + require.NoError(t, os.WriteFile(files.ack, []byte(files.ackToken), 0o600)) + acknowledged, err := providerForceAcknowledged(files) + require.NoError(t, err) + require.True(t, acknowledged) + if runtime.GOOS != testWindowsOS { + require.NoError(t, os.Chmod(files.ack, 0o400)) + acknowledged, err = providerForceAcknowledged(files) + require.NoError(t, err) + require.False(t, acknowledged) + } + require.NoError(t, os.Remove(files.ack)) + if err := os.Symlink(filepath.Join(dir, "missing"), files.ack); err == nil { + acknowledged, err = providerForceAcknowledged(files) + require.NoError(t, err) + require.False(t, acknowledged) + } +} + +func TestProviderCancellationPrelaunchAndMarkerFailure(t *testing.T) { + ctx, cancel := context.WithCancelCause(context.Background()) + cause := errors.New("caller stopped") + cancel(cause) + require.ErrorIs(t, RunCommand(ctx, cancelHelperOptions("ignore")), cause) + + dir := t.TempDir() + marker := filepath.Join(dir, "cancel") + require.NoError(t, os.Mkdir(marker, 0o700)) + err := publishProviderCancellation(marker) + require.ErrorContains(t, err, "not a regular file") + require.NotContains(t, err.Error(), dir) + + require.NoError(t, os.Remove(marker)) + require.NoError(t, publishProviderCancellation(marker)) + require.NoError(t, publishProviderCancellation(marker)) + require.NoError(t, os.Remove(marker)) + if err := os.Symlink(filepath.Join(dir, "missing"), marker); err == nil { + require.ErrorContains(t, publishProviderCancellation(marker), "not a regular file") + } +} + +func TestProviderCancellationPublicationFailureReapsProvider(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + opts := cancelHelperOptions("ignore") + opts.RawStdout = true + ready := make(chan struct{}) + opts.Stdout = &readyWriter{ready: ready} + markerPath := make(chan string, 1) + finished := make(chan error, 1) + go func() { + finished <- runProviderWithCancellationUsingPublisher(ctx, opts, func(marker string) error { + markerPath <- marker + return &os.PathError{Op: "open", Path: marker, Err: os.ErrPermission} + }) + }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("provider did not start") + } + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + require.ErrorIs(t, err, os.ErrPermission) + marker := <-markerPath + require.NotContains(t, err.Error(), marker) + _, statErr := os.Stat(filepath.Dir(marker)) + require.ErrorIs(t, statErr, os.ErrNotExist) + case <-time.After(5 * time.Second): + t.Fatal("provider did not stop after publication failure") + } +} + +func TestProviderCancellationPrivateEnvironment(t *testing.T) { + dir, err := createProviderInvocationDir() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + require.True(t, filepath.IsAbs(dir)) + if runtime.GOOS != testWindowsOS { + info, err := os.Stat(dir) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o700), info.Mode().Perm()) + } + + opts := cancelHelperOptions("environment") + opts.Environ = append( + opts.Environ, + config.EnvProviderCancelFile+"=forged", + config.EnvProviderCancelFile+"=again", + ) + if runtime.GOOS == testWindowsOS { + opts.Environ = append( + opts.Environ, + strings.ToLower(config.EnvProviderCancelFile)+"=windows-forged", + ) + } + var stdout bytes.Buffer + opts.Stdout = &stdout + require.NoError(t, RunCommand(context.Background(), opts)) + marker := stdout.String() + require.NotContains(t, marker, "forged") + require.True(t, filepath.IsAbs(marker)) + _, err = os.Stat(filepath.Dir(marker)) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestProviderCancellationAckKeyIsPrivate(t *testing.T) { + dir, err := createProviderInvocationDir() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + binary, err := os.Executable() + require.NoError(t, err) + command := fmt.Sprintf("'%s' %s", binary, testCancellationHelperArg) + files, err := prepareProviderShell(dir, command) + require.NoError(t, err) + root, err := os.OpenRoot(dir) + require.NoError(t, err) + info, statErr := root.Lstat("ack-key") + key, readErr := root.ReadFile("ack-key") + closeErr := root.Close() + require.NoError(t, statErr) + require.NoError(t, readErr) + require.NoError(t, closeErr) + require.True(t, info.Mode().IsRegular()) + if runtime.GOOS != testWindowsOS { + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + } + require.Regexp(t, `^[0-9a-f]{64}$`, string(key)) + require.Equal(t, files.ackToken, string(key)) + var stdout bytes.Buffer + opts := RunCommandOptions{ + Command: types.StrArray{command}, + Environ: providerCancellationEnvironment( + []string{"DEVSY_CANCEL_HELPER=1", "DEVSY_CANCEL_BEHAVIOR=ack-private"}, + filepath.Join(dir, "cancel"), + ), + providerShell: files, + } + require.NoError(t, runSupervisedProviderShell(context.Background(), opts, &stdout, io.Discard)) + require.NotContains(t, stdout.String(), files.ackToken) + require.NotContains(t, stdout.String(), "DEVSY_PROVIDER_SHELL_ACK=") + _, err = os.Lstat(filepath.Join(dir, "ack-key")) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestProviderCancellationAckKeyWriteFailure(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(dir, "ack-key"), 0o700)) + _, err := prepareProviderShell(dir, "printf should-not-run") + require.ErrorContains(t, err, "write provider acknowledgement key") + require.NotContains(t, err.Error(), dir) +} + +func TestProviderCancellationShellCannotForgeAck(t *testing.T) { + if runtime.GOOS == testWindowsOS { + t.Skip("requires POSIX path expansion") + } + binary, err := os.Executable() + require.NoError(t, err) + forged := strings.Repeat("b", 64) + command := fmt.Sprintf( + "printf '%%s' '%s' > \"${DEVSY_PROVIDER_CANCEL_FILE%%/*}/ack\"; "+ + "DEVSY_CANCEL_HELPER=1 DEVSY_CANCEL_BEHAVIOR=ignore '%s' -test.run=^TestProviderCancellationHelper$", + forged, + binary, + ) + pidPath := filepath.Join(t.TempDir(), "child-pid") + opts := RunCommandOptions{ + Command: types.StrArray{command}, + Environ: append(os.Environ(), "DEVSY_CANCEL_CHILD_PID_PATH="+pidPath), + Cancellation: provider.ProviderCancellationFile, + RawStdout: true, + } + ready := make(chan struct{}) + opts.Stdout = &readyWriter{ready: ready} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + finished := make(chan error, 1) + go func() { finished <- RunCommand(ctx, opts) }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("provider direct child did not start") + } + start := time.Now() + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + require.GreaterOrEqual(t, time.Since(start), providerCancellationGrace) + require.NotContains(t, err.Error(), command) + require.NotContains(t, err.Error(), forged) + if err.Error() != context.Canceled.Error() { + require.ErrorContains(t, err, "acknowledgement") + } else { + root, openErr := os.OpenRoot(filepath.Dir(pidPath)) + require.NoError(t, openErr) + data, readErr := root.ReadFile("child-pid") + closeErr := root.Close() + require.NoError(t, readErr) + require.NoError(t, closeErr) + pid, parseErr := strconv.Atoi(string(data)) + require.NoError(t, parseErr) + child, findErr := os.FindProcess(pid) + require.NoError(t, findErr) + require.Error(t, child.Signal(syscall.Signal(0))) + } + case <-time.After(providerCancellationGrace + providerForceAckWait + 3*time.Second): + t.Fatal("provider helper did not reap its direct child or report unclean shutdown") + } +} + +func TestProviderCancellationNilEnvironmentInheritsHost(t *testing.T) { + t.Setenv("DEVSY_CANCEL_HELPER", "1") + t.Setenv("DEVSY_CANCEL_BEHAVIOR", "inherited") + t.Setenv("DEVSY_CANCEL_INHERITED", "present") + binary, err := os.Executable() + require.NoError(t, err) + var stdout bytes.Buffer + require.NoError(t, RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{binary, testCancellationHelperArg}, + Cancellation: provider.ProviderCancellationFile, + Stdout: &stdout, + })) + require.Equal(t, "present", stdout.String()) +} + +func TestProviderCancellationTypedExits(t *testing.T) { + for _, status := range []int{7, 125, 127} { + opts := cancelHelperOptions("status") + if status == 125 { + opts.Environ = append(opts.Environ, "DEVSY_CANCEL_BEHAVIOR=other") + } + if status == 127 { + opts.Environ = append(opts.Environ, "DEVSY_CANCEL_BEHAVIOR=status-127") + } + err := RunCommand(context.Background(), opts) + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + require.Equal(t, status, exitErr.ExitCode()) + } +} + +func TestProviderCancellationShellExits(t *testing.T) { + if runtime.GOOS == testWindowsOS { + t.Skip("requires POSIX shell") + } + binary, err := os.Executable() + require.NoError(t, err) + for _, status := range []int{7, 127} { + command := "command-that-does-not-exist-846307" + if status == 7 { + command = fmt.Sprintf( + "DEVSY_CANCEL_HELPER=1 DEVSY_CANCEL_BEHAVIOR=status '%s' -test.run=^TestProviderCancellationHelper$", + binary, + ) + } + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{ + command, + }, + Environ: os.Environ(), + Cancellation: provider.ProviderCancellationFile, + }) + var shellExit interp.ExitStatus + require.ErrorAs(t, err, &shellExit) + require.Equal(t, status, int(shellExit)) + } +} + +func TestProviderCancellationParseErrorAcknowledged(t *testing.T) { + command := "if then fi (((( private-provider-script-846307" + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{command}, + Environ: os.Environ(), + Cancellation: provider.ProviderCancellationFile, + }) + var status interp.ExitStatus + require.ErrorAs(t, err, &status) + require.Equal(t, 1, int(status)) + require.NotContains(t, err.Error(), "without authentic acknowledgement") + require.NotContains(t, err.Error(), command) +} + +func TestProviderCancellationRawBytesAndStderrRedaction(t *testing.T) { + opts := cancelHelperOptions("bytes") + secret := "provider-cancel-secret-84392" + opts.Environ = append(opts.Environ, "DEVSY_CANCEL_SECRET="+secret) + opts.RawStdout = true + var stdout, stderr bytes.Buffer + opts.Stdout, opts.Stderr = &stdout, &stderr + require.NoError(t, RunCommand(context.Background(), opts)) + require.Equal(t, []byte{0, 0xff, 0x80, '\r', '\n'}, stdout.Bytes()) + require.NotContains(t, stderr.String(), secret) + require.Contains(t, stderr.String(), "***") +} + +func TestProviderCancellationArgvTransportCompletesWithOpenInput(t *testing.T) { + local, remote := net.Pipe() + t.Cleanup(func() { + _ = local.Close() + _ = remote.Close() + }) + opts := cancelHelperOptions("bytes") + opts.Stdin, opts.closeInputOnExit = remote, remote + opts.RawStdout = true + var stdout bytes.Buffer + opts.Stdout = &stdout + finished := make(chan error, 1) + go func() { finished <- RunCommand(context.Background(), opts) }() + select { + case err := <-finished: + require.NoError(t, err) + require.Equal(t, []byte{0, 0xff, 0x80, '\r', '\n'}, stdout.Bytes()) + case <-time.After(5 * time.Second): + t.Fatal("argv provider completion waited for transport input closure") + } +} + +func TestProviderCancellationArgvOwnedInputPreservesExitStatus(t *testing.T) { + local, remote := net.Pipe() + t.Cleanup(func() { + _ = local.Close() + _ = remote.Close() + }) + opts := cancelHelperOptions("status") + opts.Stdin, opts.closeInputOnExit = remote, remote + finished := make(chan error, 1) + go func() { finished <- RunCommand(context.Background(), opts) }() + select { + case err := <-finished: + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + require.Equal(t, 7, exitErr.ExitCode()) + case <-time.After(5 * time.Second): + t.Fatal("argv provider exit waited for transport input closure") + } +} + +func TestProviderCancellationOwnedInputPreservesShellExitStatus(t *testing.T) { + input := io.NopCloser(strings.NewReader("")) + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{"exit 7"}, + Environ: os.Environ(), + Stdin: input, + Cancellation: provider.ProviderCancellationFile, + closeInputOnExit: input, + }) + var status interp.ExitStatus + require.ErrorAs(t, err, &status) + require.Equal(t, 7, int(status)) +} + +func TestProviderCancellationOwnedInputSuccessfulExitIgnoresEPIPE(t *testing.T) { + input := io.NopCloser(bytes.NewReader(bytes.Repeat([]byte("x"), 2<<20))) + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{"exit 0"}, + Environ: os.Environ(), + Stdin: input, + Cancellation: provider.ProviderCancellationFile, + closeInputOnExit: input, + }) + require.NoError(t, err) +} + +func TestProviderCancellationOwnedInputStopsWithInheritedStdin(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + holderPath := filepath.Join(t.TempDir(), "holder-pid") + t.Setenv("DEVSY_CANCEL_HOLDER_PID_PATH", holderPath) + t.Cleanup(func() { cleanupTestCancellationOutputHolder(holderPath) }) + input := io.NopCloser(bytes.NewReader(bytes.Repeat([]byte("x"), 2<<20))) + cmd := &exec.Cmd{Path: binary, Args: []string{binary, testCancellationHelperArg}} + cmd.Env = append(os.Environ(), "DEVSY_CANCEL_HELPER=1", "DEVSY_CANCEL_BEHAVIOR=orphan-input") + cmd.Stdin = input + var stdout bytes.Buffer + cmd.Stdout = &stdout + finished := make(chan error, 1) + go func() { + finished <- waitSupervisedProviderShellWithOwnedInput(context.Background(), cmd, input, "test") + }() + select { + case err := <-finished: + require.NoError(t, err) + require.Equal(t, []byte{0, 0xff, 0x80, '\r', '\n'}, stdout.Bytes()) + case <-time.After(5 * time.Second): + t.Fatal("provider completion waited for inherited stdin") + } +} + +func TestProviderCancellationOwnedInputDrainsBinaryOutputAfterEOF(t *testing.T) { + if runtime.GOOS == testWindowsOS { + t.Skip("requires POSIX shell") + } + binary, err := os.Executable() + require.NoError(t, err) + opts := cancelHelperOptions("bulk-after-eof") + opts.Command = types.StrArray{ + fmt.Sprintf("'%s' -test.run=^TestProviderCancellationHelper$", binary), + } + input := io.NopCloser(strings.NewReader("")) + opts.Stdin = input + opts.closeInputOnExit = input + opts.RawStdout = true + var stdout, stderr bytes.Buffer + opts.Stdout, opts.Stderr = &stdout, &stderr + require.NoError(t, RunCommand(context.Background(), opts)) + + block := make([]byte, 32<<10) + for i := range block { + block[i] = byte(i) + } + require.Equal(t, 12<<20, stdout.Len()) + require.True(t, bytes.Equal(bytes.Repeat(block, (12<<20)/len(block)), stdout.Bytes())) + require.Equal(t, "separate stderr", stderr.String()) +} + +type slowProviderOutputWriter struct { + bytes.Buffer + once sync.Once +} + +func (w *slowProviderOutputWriter) Write(p []byte) (int, error) { + w.once.Do(func() { time.Sleep(3 * time.Second) }) + return w.Buffer.Write(p) +} + +func TestProviderCancellationOwnedInputPreservesSlowOutput(t *testing.T) { + input := io.NopCloser(strings.NewReader("")) + var output slowProviderOutputWriter + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{"printf ready"}, + Environ: os.Environ(), + Stdin: input, + Stdout: &output, + RawStdout: true, + Cancellation: provider.ProviderCancellationFile, + closeInputOnExit: input, + }) + require.NoError(t, err) + require.Equal(t, "ready", output.String()) +} + +func TestProviderCancellationOwnedOutputStopsAfterForcedCancellation(t *testing.T) { + if runtime.GOOS == testWindowsOS { + t.Skip("requires POSIX shell") + } + binary, err := os.Executable() + require.NoError(t, err) + holderPath := filepath.Join(t.TempDir(), "holder-pid") + t.Setenv("DEVSY_CANCEL_HOLDER_PID_PATH", holderPath) + t.Cleanup(func() { cleanupTestCancellationOutputHolder(holderPath) }) + opts := cancelHelperOptions("orphan-output") + opts.Command = types.StrArray{ + fmt.Sprintf("'%s' -test.run=^TestProviderCancellationHelper$", binary), + } + input := io.NopCloser(strings.NewReader("")) + opts.Stdin, opts.closeInputOnExit = input, input + opts.RawStdout = true + ready := make(chan struct{}) + opts.Stdout = &readyWriter{ready: ready} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + finished := make(chan error, 1) + go func() { finished <- RunCommand(ctx, opts) }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("provider output holder did not start") + } + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(providerCancellationGrace + providerForceAckWait + 3*time.Second): + t.Fatal("forced provider completion waited for descendant output pipe") + } +} + +func TestProviderCancellationOwnedOutputStopsAfterSuccessfulShellExit(t *testing.T) { + if runtime.GOOS == testWindowsOS { + t.Skip("requires POSIX shell") + } + binary, err := os.Executable() + require.NoError(t, err) + holderPath := filepath.Join(t.TempDir(), "holder-pid") + t.Setenv("DEVSY_CANCEL_HOLDER_PID_PATH", holderPath) + t.Cleanup(func() { cleanupTestCancellationOutputHolder(holderPath) }) + opts := cancelHelperOptions("orphan-output") + opts.Command = types.StrArray{ + fmt.Sprintf("'%s' -test.run=^TestProviderCancellationHelper$", binary), + } + input := io.NopCloser(strings.NewReader("")) + opts.Stdin, opts.closeInputOnExit = input, input + opts.RawStdout = true + start := time.Now() + err = RunCommand(context.Background(), opts) + require.ErrorIs(t, err, exec.ErrWaitDelay) + require.Less(t, time.Since(start), 20*time.Second) +} + +func cleanupTestCancellationOutputHolder(path string) { + root, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + return + } + data, readErr := root.ReadFile("holder-pid") + _ = root.Close() + if readErr != nil { + return + } + pid, err := strconv.Atoi(string(data)) + if err != nil { + return + } + process, err := os.FindProcess(pid) + if err == nil { + _ = process.Kill() + } +} + +func TestMachineCommandTransportCancellationJoinsCallback(t *testing.T) { + for _, behavior := range []string{testTransportRead, "transport-write"} { + t.Run(behavior, func(t *testing.T) { + dir := t.TempDir() + ready, ack := filepath.Join(dir, "ready"), filepath.Join(dir, "ack") + t.Setenv("DEVSY_CANCEL_HELPER", "1") + t.Setenv("DEVSY_CANCEL_BEHAVIOR", behavior) + t.Setenv("DEVSY_CANCEL_READY_PATH", ready) + t.Setenv("DEVSY_CANCEL_ACK_PATH", ack) + machine := &provider.Machine{ID: "cancel-machine", Context: testLocksContext} + cfg := &provider.ProviderConfig{ + Name: testCancelProvider, + Exec: provider.ProviderCommands{ + Create: types.StrArray{ + testProviderCreateCommand, + }, + Command: cancelHelperOptions(behavior).Command, + Cancellation: provider.ProviderCancellationFile, + }, + } + devsy := &config.Config{ + DefaultContext: testLocksContext, + Contexts: map[string]*config.ContextConfig{testLocksContext: {}}, + } + mc, err := NewMachineClient(devsy, cfg, machine) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + conn, err := mc.OpenCommandTransport( + ctx, + client.CommandTransportOptions{Command: testOpaqueCommand}, + ) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(ready); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("transport provider did not start") + } + time.Sleep(10 * time.Millisecond) + } + cancel() + completed := make(chan error, 1) + go func() { completed <- conn.Wait() }() + select { + case err := <-completed: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("transport callback did not finish") + } + requireTestAcknowledgement(t, ack) + }) + } +} + +func TestWorkspaceCommandCancellationJoinsProviderCleanup(t *testing.T) { + for _, transportMode := range []bool{false, true} { + t.Run(fmt.Sprint("transport=", transportMode), func(t *testing.T) { + ack := filepath.Join(t.TempDir(), "ack") + t.Setenv("DEVSY_CANCEL_HELPER", "1") + t.Setenv("DEVSY_CANCEL_BEHAVIOR", "acknowledge") + t.Setenv("DEVSY_CANCEL_ACK_PATH", ack) + wc := workspaceCancellationTestClient() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if transportMode { + requireWorkspaceTransportCancellation(t, wc, ctx, cancel) + } else { + requireWorkspaceDirectCancellation(t, wc, ctx, cancel) + } + requireTestAcknowledgement(t, ack) + }) + } +} + +func TestWorkspaceTransportCompletesWithoutClosingInput(t *testing.T) { + wc := workspaceCancellationTestClient() + wc.providerConfig.Exec.Command = types.StrArray{"printf ready"} + conn, err := wc.OpenCommandTransport(context.Background(), client.CommandTransportOptions{ + Command: testOpaqueCommand, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + ready := make([]byte, 5) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + _, err = io.ReadFull(conn, ready) + require.NoError(t, err) + require.Equal(t, "ready", string(ready)) + finished := make(chan error, 1) + go func() { finished <- conn.Wait() }() + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(3 * time.Second): + t.Fatal("provider completion waited for caller to close transport input") + } +} + +func workspaceCancellationTestClient() *workspaceClient { + return &workspaceClient{ + providerWorkspace: &provider.Workspace{ + ID: "cancel-workspace", + Context: testLocksContext, + Provider: provider.WorkspaceProviderConfig{Name: testCancelProvider}, + }, + providerConfig: &provider.ProviderConfig{ + Name: testCancelProvider, + Exec: provider.ProviderCommands{ + Command: cancelHelperOptions("acknowledge").Command, + Cancellation: provider.ProviderCancellationFile, + }, + }, + devsyConfig: &config.Config{ + DefaultContext: testLocksContext, + Contexts: map[string]*config.ContextConfig{testLocksContext: {}}, + }, + } +} + +func requireWorkspaceTransportCancellation( + t *testing.T, + wc *workspaceClient, + ctx context.Context, + cancel context.CancelFunc, +) { + t.Helper() + conn, err := wc.OpenCommandTransport(ctx, client.CommandTransportOptions{ + Command: testOpaqueCommand, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + ready := make([]byte, 5) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + _, err = io.ReadFull(conn, ready) + require.NoError(t, err) + require.Equal(t, "ready", string(ready)) + cancel() + require.ErrorIs(t, conn.Wait(), context.Canceled) +} + +func requireWorkspaceDirectCancellation( + t *testing.T, + wc *workspaceClient, + ctx context.Context, + cancel context.CancelFunc, +) { + t.Helper() + ready := make(chan struct{}) + finished := make(chan error, 1) + go func() { + finished <- wc.Command(ctx, client.CommandOptions{ + Command: testOpaqueCommand, + Stdout: &readyWriter{ready: ready}, + RawStdout: true, + }) + }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("workspace provider did not start") + } + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("workspace provider did not finish") + } +} diff --git a/pkg/client/clientimplementation/provider_cancel_unix.go b/pkg/client/clientimplementation/provider_cancel_unix.go new file mode 100644 index 0000000000..d647d17cad --- /dev/null +++ b/pkg/client/clientimplementation/provider_cancel_unix.go @@ -0,0 +1,21 @@ +//go:build !windows + +package clientimplementation + +import ( + "os" + "path/filepath" +) + +func createProviderInvocationDir() (string, error) { + dir, err := os.MkdirTemp("", "devsy-provider-invocation-") + if err != nil { + return "", err + } + absolute, err := filepath.Abs(dir) + if err != nil { + _ = os.Remove(dir) + return "", err + } + return absolute, nil +} diff --git a/pkg/client/clientimplementation/provider_cancel_windows.go b/pkg/client/clientimplementation/provider_cancel_windows.go new file mode 100644 index 0000000000..c583781b8d --- /dev/null +++ b/pkg/client/clientimplementation/provider_cancel_windows.go @@ -0,0 +1,53 @@ +package clientimplementation + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "os" + "path/filepath" + "unsafe" + + "golang.org/x/sys/windows" +) + +func createProviderInvocationDir() (string, error) { + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + return "", err + } + sd, err := windows.SecurityDescriptorFromString( + "D:P(A;OICI;GA;;;" + user.User.Sid.String() + ")(A;OICI;GA;;;SY)", + ) + if err != nil { + return "", err + } + attributes := windows.SecurityAttributes{ + Length: uint32(unsafe.Sizeof(windows.SecurityAttributes{})), + SecurityDescriptor: sd, + } + for range 10 { + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return "", err + } + path, err := filepath.Abs( + filepath.Join(os.TempDir(), "devsy-provider-invocation-"+hex.EncodeToString(random[:])), + ) + if err != nil { + return "", err + } + name, err := windows.UTF16PtrFromString(path) + if err != nil { + return "", err + } + err = windows.CreateDirectory(name, &attributes) + if err == nil { + return path, nil + } + if !errors.Is(err, windows.ERROR_ALREADY_EXISTS) { + return "", err + } + } + return "", os.ErrExist +} diff --git a/pkg/client/clientimplementation/provider_cancel_windows_test.go b/pkg/client/clientimplementation/provider_cancel_windows_test.go new file mode 100644 index 0000000000..2a9d43af5e --- /dev/null +++ b/pkg/client/clientimplementation/provider_cancel_windows_test.go @@ -0,0 +1,49 @@ +package clientimplementation + +import ( + "os" + "strings" + "testing" + + "golang.org/x/sys/windows" +) + +func TestProviderCancellationDirectoryWindowsDACL(t *testing.T) { + dir, err := createProviderInvocationDir() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("remove invocation directory: %v", err) + } + }) + sd, err := windows.GetNamedSecurityInfo( + dir, + windows.SE_FILE_OBJECT, + windows.DACL_SECURITY_INFORMATION, + ) + if err != nil { + t.Fatal(err) + } + control, _, err := sd.Control() + if err != nil { + t.Fatal(err) + } + if control&windows.SE_DACL_PROTECTED == 0 { + t.Fatal("invocation directory inherits an unprotected DACL") + } + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + t.Fatal(err) + } + sddl := sd.String() + if !strings.Contains(sddl, user.User.Sid.String()) || + (!strings.Contains(sddl, ";;;SY)") && !strings.Contains(sddl, ";;;S-1-5-18)")) || + strings.Contains(sddl, ";;;WD)") { + t.Fatalf( + "invocation directory DACL does not restrict access to current user and SYSTEM: %s", + sddl, + ) + } +} diff --git a/pkg/client/clientimplementation/provider_shell_process_unix.go b/pkg/client/clientimplementation/provider_shell_process_unix.go new file mode 100644 index 0000000000..df658b9036 --- /dev/null +++ b/pkg/client/clientimplementation/provider_shell_process_unix.go @@ -0,0 +1,16 @@ +//go:build !windows + +package clientimplementation + +import ( + "os/exec" + "syscall" +) + +func configureProviderShellProcess(cmd *exec.Cmd) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.Setpgid = true + cmd.SysProcAttr.Pgid = 0 +} diff --git a/pkg/client/clientimplementation/provider_shell_process_unix_test.go b/pkg/client/clientimplementation/provider_shell_process_unix_test.go new file mode 100644 index 0000000000..11e0ecdfa1 --- /dev/null +++ b/pkg/client/clientimplementation/provider_shell_process_unix_test.go @@ -0,0 +1,148 @@ +//go:build !windows + +package clientimplementation + +import ( + "bufio" + "bytes" + "context" + "os" + "os/exec" + "os/signal" + "strconv" + "syscall" + "testing" + "time" + + "github.com/creack/pty" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/types" + "github.com/stretchr/testify/require" + "mvdan.cc/sh/v3/interp" +) + +func TestProviderShellProcessIsolation(t *testing.T) { + dir, err := createProviderInvocationDir() + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + files, err := prepareProviderShell(dir, "printf 'ready\\n'; read value") + require.NoError(t, err) + binary, err := os.Executable() + require.NoError(t, err) + cmd := &exec.Cmd{ + Path: binary, + Args: []string{ + binary, "internal", "provider-shell", + "--script-file", files.script, "--force-file", files.force, "--ack-file", files.ack, + "--parent-pid", strconv.Itoa(os.Getpid()), + }, + } + input, err := cmd.StdinPipe() + require.NoError(t, err) + defer func() { _ = input.Close() }() + output, err := cmd.StdoutPipe() + require.NoError(t, err) + configureProviderShellProcess(cmd) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { _ = cmd.Process.Kill() }) + ready := make(chan bool, 1) + go func() { ready <- bufio.NewScanner(output).Scan() }() + select { + case ok := <-ready: + require.True(t, ok) + case <-time.After(5 * time.Second): + t.Fatal("helper did not become ready") + } + group, err := syscall.Getpgid(cmd.Process.Pid) + require.NoError(t, err) + require.Equal(t, syscall.Getpgrp(), group) + require.NoError(t, cmd.Process.Signal(syscall.SIGINT)) + require.NoError(t, publishProviderMarker(files.force)) + require.Eventually(t, func() bool { + ack, err := os.ReadFile(files.ack) + return err == nil && string(ack) == files.ackToken + }, 5*time.Second, 20*time.Millisecond) + _ = cmd.Process.Kill() + _ = cmd.Wait() +} + +func TestProviderShellPTYController(t *testing.T) { + if os.Getenv("DEVSY_PROVIDER_PTY_CONTROLLER") != "1" { + return + } + signal.Ignore(syscall.SIGHUP) + defer signal.Reset(syscall.SIGHUP) + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{`read value; printf 'got:%s\n' "$value"`}, + Environ: os.Environ(), + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + RawStdout: true, + Cancellation: provider.ProviderCancellationFile, + }) + require.NoError(t, err) +} + +func TestProviderShellReadsControllingTerminal(t *testing.T) { + master, slave, err := pty.Open() + require.NoError(t, err) + defer func() { _ = master.Close() }() + defer func() { _ = slave.Close() }() + binary, err := os.Executable() + require.NoError(t, err) + controller := &exec.Cmd{ + Path: binary, + Args: []string{binary, "-test.run=^TestProviderShellPTYController$"}, + Env: append(os.Environ(), "DEVSY_PROVIDER_PTY_CONTROLLER=1"), + Stdin: slave, Stderr: os.Stderr, + SysProcAttr: &syscall.SysProcAttr{Setsid: true, Setctty: true, Ctty: 0}, + } + output, err := controller.StdoutPipe() + require.NoError(t, err) + require.NoError(t, controller.Start()) + t.Cleanup(func() { _ = controller.Process.Kill() }) + _, err = master.Write([]byte("hello\n")) + require.NoError(t, err) + lines := make(chan []string, 1) + go func() { + reader := bufio.NewReader(output) + first, _ := reader.ReadString('\n') + second, _ := reader.ReadString('\n') + lines <- []string{first, second} + }() + select { + case got := <-lines: + require.Equal(t, []string{"got:hello\n", "PASS\n"}, got) + case <-time.After(8 * time.Second): + t.Fatal("provider helper could not read the controlling terminal") + } + require.NoError(t, master.Close()) + require.NoError(t, slave.Close()) + done := make(chan error, 1) + go func() { done <- controller.Wait() }() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("provider controller did not finish after terminal closure") + } +} + +func TestProviderShellForegroundPreservesChildSIGINT(t *testing.T) { + if signal.Ignored(syscall.SIGINT) { + t.Skip("parent already ignores SIGINT") + } + var stdout bytes.Buffer + err := RunCommand(context.Background(), RunCommandOptions{ + Command: types.StrArray{`/bin/sh -c 'kill -INT $$; printf survived'`}, + Environ: os.Environ(), + Stdout: &stdout, + RawStdout: true, + Cancellation: provider.ProviderCancellationFile, + }) + var status interp.ExitStatus + require.ErrorAs(t, err, &status) + require.Equal(t, 130, int(status)) + require.NotContains(t, stdout.String(), "survived") +} diff --git a/pkg/client/clientimplementation/provider_shell_process_windows.go b/pkg/client/clientimplementation/provider_shell_process_windows.go new file mode 100644 index 0000000000..d7bcb2d6e7 --- /dev/null +++ b/pkg/client/clientimplementation/provider_shell_process_windows.go @@ -0,0 +1,15 @@ +package clientimplementation + +import ( + "os/exec" + "syscall" + + "golang.org/x/sys/windows" +) + +func configureProviderShellProcess(cmd *exec.Cmd) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.CreationFlags |= windows.CREATE_NEW_PROCESS_GROUP +} diff --git a/pkg/client/clientimplementation/provider_shell_process_windows_test.go b/pkg/client/clientimplementation/provider_shell_process_windows_test.go new file mode 100644 index 0000000000..ae0e6d3605 --- /dev/null +++ b/pkg/client/clientimplementation/provider_shell_process_windows_test.go @@ -0,0 +1,19 @@ +package clientimplementation + +import ( + "os/exec" + "syscall" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" +) + +func TestProviderShellProcessConsoleIsolation(t *testing.T) { + cmd := &exec.Cmd{ + SysProcAttr: &syscall.SysProcAttr{CreationFlags: windows.CREATE_UNICODE_ENVIRONMENT}, + } + configureProviderShellProcess(cmd) + require.NotZero(t, cmd.SysProcAttr.CreationFlags&windows.CREATE_NEW_PROCESS_GROUP) + require.NotZero(t, cmd.SysProcAttr.CreationFlags&windows.CREATE_UNICODE_ENVIRONMENT) +} diff --git a/pkg/client/clientimplementation/proxy_client.go b/pkg/client/clientimplementation/proxy_client.go index 4ad5a3554b..c696250b36 100644 --- a/pkg/client/clientimplementation/proxy_client.go +++ b/pkg/client/clientimplementation/proxy_client.go @@ -57,12 +57,13 @@ type proxyExecutor struct { } type execParams struct { - command types.StrArray - extraEnv map[string]string - stdin io.Reader - stdout io.Writer - stderr io.Writer - rawStdout bool + command types.StrArray + extraEnv map[string]string + stdin io.Reader + closeInputOnExit io.Closer + stdout io.Writer + stderr io.Writer + rawStdout bool } // execute runs a proxy command with common settings. @@ -75,6 +76,7 @@ func (e *proxyExecutor) execute(ctx context.Context, params execParams) error { Config: e.client.config, ExtraEnv: params.extraEnv, Stdin: params.stdin, + closeInputOnExit: params.closeInputOnExit, Stdout: params.stdout, Stderr: params.stderr, RawStdout: params.rawStdout, @@ -221,11 +223,15 @@ func (s *proxyClient) OpenSSHTransport( ctx, func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { return s.executor.executeWithJSONLog(ctx, execParams{ - command: s.config.Exec.Proxy.Ssh, - extraEnv: EncodeOptions(client.SshOptions{User: opt.User}, config.EnvFlagsSSH), - stdin: stdin, - stdout: stdout, - rawStdout: true, + command: s.config.Exec.Proxy.Ssh, + extraEnv: EncodeOptions( + client.SshOptions{User: opt.User}, + config.EnvFlagsSSH, + ), + stdin: stdin, + closeInputOnExit: stdin.(io.Closer), + stdout: stdout, + rawStdout: true, }) }, transport.CallbackConnOptions{ diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index 54d4e44c01..99431d4caa 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -7,10 +7,14 @@ import ( "errors" "fmt" "io" + "net" "os" "os/exec" + "strconv" "strings" "sync" + "sync/atomic" + "syscall" "time" "al.essio.dev/pkg/shellescape" @@ -31,6 +35,7 @@ import ( "github.com/devsy-org/devsy/pkg/task" "github.com/devsy-org/devsy/pkg/transport" "github.com/devsy-org/devsy/pkg/types" + "mvdan.cc/sh/v3/interp" ) func NewWorkspaceClient( @@ -245,12 +250,13 @@ func (s *workspaceClient) Command(ctx context.Context, opt client.CommandOptions } return RunCommand(ctx, RunCommandOptions{ - Command: s.providerConfig.Exec.Command, - Environ: environ, - Stdin: opt.Stdin, - Stdout: opt.Stdout, - Stderr: opt.Stderr, - RawStdout: opt.RawStdout, + Command: s.providerConfig.Exec.Command, + Environ: environ, + Stdin: opt.Stdin, + Stdout: opt.Stdout, + Stderr: opt.Stderr, + RawStdout: opt.RawStdout, + Cancellation: s.providerConfig.Exec.Cancellation, }) } @@ -266,10 +272,14 @@ func (s *workspaceClient) OpenCommandTransport( ctx, func(ctx context.Context, stdin io.Reader, stdout io.Writer) error { return RunCommand(ctx, RunCommandOptions{ - Command: s.providerConfig.Exec.Command, - Environ: environ, - Stdin: stdin, Stdout: stdout, Stderr: opt.Stderr, - RawStdout: true, + Command: s.providerConfig.Exec.Command, + Environ: environ, + Stdin: stdin, + Stdout: stdout, + Stderr: opt.Stderr, + RawStdout: true, + Cancellation: s.providerConfig.Exec.Cancellation, + closeInputOnExit: stdin.(io.Closer), }) }, transport.CallbackConnOptions{ @@ -794,6 +804,7 @@ type WorkspaceCommandConfig struct { Workspace *provider.Workspace Machine *provider.Machine Config *provider.ProviderConfig + closeInputOnExit io.Closer } func RunCommandWithBinaries(ctx context.Context, cfg WorkspaceCommandConfig) error { @@ -810,25 +821,37 @@ func RunCommandWithBinaries(ctx context.Context, cfg WorkspaceCommandConfig) err } return RunCommand(ctx, RunCommandOptions{ - Command: cfg.Command, - Environ: environ, - Stdin: cfg.Stdin, - Stdout: cfg.Stdout, - Stderr: cfg.Stderr, - RawStdout: cfg.RawStdout, + Command: cfg.Command, + Environ: environ, + Stdin: cfg.Stdin, + Stdout: cfg.Stdout, + Stderr: cfg.Stderr, + RawStdout: cfg.RawStdout, + Cancellation: cfg.Config.Exec.Cancellation, + closeInputOnExit: cfg.closeInputOnExit, }) } type RunCommandOptions struct { - Command types.StrArray - Environ []string - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer - RawStdout bool + Command types.StrArray + Environ []string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + RawStdout bool + Cancellation provider.ProviderCancellationMode + providerShell *providerShellInvocation + closeInputOnExit io.Closer } -func RunCommand(ctx context.Context, opts RunCommandOptions) (retErr error) { +func RunCommand(ctx context.Context, opts RunCommandOptions) error { + if opts.Cancellation == provider.ProviderCancellationFile { + return runProviderWithCancellation(ctx, opts) + } + return runCommandExecution(ctx, opts) +} + +func runCommandExecution(ctx context.Context, opts RunCommandOptions) (retErr error) { if len(opts.Command) == 0 { return nil } @@ -856,37 +879,523 @@ func RunCommand(ctx context.Context, opts RunCommandOptions) (retErr error) { }() if len(opts.Command) == 1 { - if err := shell.RunEmulatedShell(ctx, &shell.CommandRunner{ - Command: opts.Command[0], - Stdin: opts.Stdin, - Stdout: stdout, - Stderr: stderr, - Environ: opts.Environ, - }); err != nil { - return fmt.Errorf( - "run provider command %q (%s): %w", - opts.Command[0], - providerAction(opts.Environ), - err, - ) + return runProviderShellCommand(ctx, opts, stdout, stderr) + } + return runProviderArgvCommand(ctx, opts, stdout, stderr) +} + +func runProviderShellCommand( + ctx context.Context, + opts RunCommandOptions, + stdout, stderr io.Writer, +) error { + if opts.providerShell != nil { + return runSupervisedProviderShell(ctx, opts, stdout, stderr) + } + runner := &shell.CommandRunner{ + Command: opts.Command[0], + Stdin: opts.Stdin, + Stdout: stdout, + Stderr: stderr, + Environ: opts.Environ, + } + if err := shell.RunEmulatedShell(ctx, runner); err != nil { + return fmt.Errorf("run provider %s: %w", providerAction(opts.Environ), err) + } + return nil +} + +func runSupervisedProviderShell( + ctx context.Context, + opts RunCommandOptions, + stdout, stderr io.Writer, +) error { + binary, err := os.Executable() + if err != nil { + return fmt.Errorf("find provider shell executable: %w", err) + } + files := opts.providerShell + cmd := &exec.Cmd{ + Path: binary, + Args: []string{ + binary, + "internal", + "provider-shell", + "--script-file", + files.script, + "--force-file", + files.force, + "--ack-file", + files.ack, + "--parent-pid", + strconv.Itoa(os.Getpid()), + }, + Env: opts.Environ, + Stdin: opts.Stdin, + Stdout: stdout, + Stderr: stderr, + WaitDelay: 2 * time.Second, + } + if opts.closeInputOnExit != nil { + return waitSupervisedProviderShellWithOwnedInput( + ctx, + cmd, + opts.closeInputOnExit, + providerAction(opts.Environ), + ) + } + return waitSupervisedProviderShell(ctx, cmd, providerAction(opts.Environ)) +} + +type providerShellPipes struct { + stdinRead *os.File + stdinWrite *os.File + stdoutRead *os.File + stdoutWrite *os.File + stderrRead *os.File + stderrWrite *os.File +} + +func openProviderShellPipes() (pipes *providerShellPipes, retErr error) { + pipes = &providerShellPipes{} + defer func() { + if retErr != nil { + retErr = errors.Join(retErr, pipes.close()) } + }() + pipes.stdinRead, pipes.stdinWrite, retErr = os.Pipe() + if retErr != nil { + return + } + pipes.stdoutRead, pipes.stdoutWrite, retErr = os.Pipe() + if retErr != nil { + return + } + pipes.stderrRead, pipes.stderrWrite, retErr = os.Pipe() + if retErr != nil { + return + } + return +} + +func closeProviderShellPipe(pipe *os.File) error { + if pipe == nil { + return nil + } + err := pipe.Close() + if errors.Is(err, os.ErrClosed) { return nil } + return err +} + +func (p *providerShellPipes) close() error { + return errors.Join( + closeProviderShellPipe(p.stdinRead), + closeProviderShellPipe(p.stdinWrite), + closeProviderShellPipe(p.stdoutRead), + closeProviderShellPipe(p.stdoutWrite), + closeProviderShellPipe(p.stderrRead), + closeProviderShellPipe(p.stderrWrite), + ) +} + +func (p *providerShellPipes) closeChildEnds() error { + err := errors.Join( + closeProviderShellPipe(p.stdinRead), + closeProviderShellPipe(p.stdoutWrite), + closeProviderShellPipe(p.stderrWrite), + ) + p.stdinRead, p.stdoutWrite, p.stderrWrite = nil, nil, nil + return err +} +func monitorSupervisedProviderShell( + ctx context.Context, + cmd *exec.Cmd, + stdoutRead, stderrRead *os.File, +) (chan struct{}, <-chan struct{}) { + stopped := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + select { + case <-ctx.Done(): + _ = cmd.Process.Kill() + _ = closeProviderShellPipe(stdoutRead) + _ = closeProviderShellPipe(stderrRead) + case <-stopped: + } + }() + return stopped, done +} + +type providerShellInputResult struct { + copyErr error + closeErr error +} + +func pumpProviderShellInput(writer *os.File, source io.Reader) <-chan providerShellInputResult { + done := make(chan providerShellInputResult, 1) + go func() { + if source == nil { + source = strings.NewReader("") + } + _, copyErr := io.Copy(writer, source) + done <- providerShellInputResult{copyErr, closeProviderShellPipe(writer)} + }() + return done +} + +type providerShellOutputPump struct { + reader *os.File + done chan error + changed chan struct{} + reading atomic.Bool +} + +func (p *providerShellOutputPump) Read(buf []byte) (int, error) { + p.reading.Store(true) + p.signal() + n, err := p.reader.Read(buf) + p.reading.Store(false) + p.signal() + return n, err +} + +func (p *providerShellOutputPump) signal() { + select { + case p.changed <- struct{}{}: + default: + } +} + +func pumpProviderShellOutput(reader *os.File, writer io.Writer) *providerShellOutputPump { + pump := &providerShellOutputPump{ + reader: reader, done: make(chan error, 1), changed: make(chan struct{}, 1), + } + go func() { + if writer == nil { + writer = io.Discard + } + _, copyErr := io.Copy(writer, pump) + if copyErr != nil { + _, _ = io.Copy(io.Discard, pump) + } + pump.done <- copyErr + }() + return pump +} + +func providerShellInputCopyError(copyErr, waitErr, closeErr error) error { + if waitErr == nil && closeErr == nil && + (errors.Is(copyErr, io.ErrClosedPipe) || errors.Is(copyErr, net.ErrClosed) || + errors.Is(copyErr, os.ErrClosed) || errors.Is(copyErr, syscall.EPIPE)) { + return nil + } + return copyErr +} + +func providerShellCopyError(action, stream string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("copy provider %s %s: %w", action, stream, err) +} + +func providerShellCloseError(action, target string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("close provider %s %s: %w", action, target, err) +} + +func providerShellWaitError(action string, err error) error { + if err == nil { + return nil + } + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { + code := exitErr.ExitCode() + if code >= 0 && code <= 255 { + err = interp.ExitStatus(uint8(code)) + } + } + return fmt.Errorf("run provider %s: %w", action, err) +} + +func waitSupervisedProviderShellWithOwnedInput( + ctx context.Context, + cmd *exec.Cmd, + input io.Closer, + action string, +) (retErr error) { + pipes, err := openProviderShellPipes() + if err != nil { + return fmt.Errorf("create provider %s stdio pipes: %w", action, err) + } + defer func() { retErr = errors.Join(retErr, pipes.close()) }() + + source, stdout, stderr := cmd.Stdin, cmd.Stdout, cmd.Stderr + cmd.Stdin, cmd.Stdout, cmd.Stderr = pipes.stdinRead, pipes.stdoutWrite, pipes.stderrWrite + configureProviderShellProcess(cmd) + if err := cmd.Start(); err != nil { + return fmt.Errorf("start provider %s shell: %w", action, err) + } + pipeCloseErr := pipes.closeChildEnds() + stopped, monitorDone := monitorSupervisedProviderShell( + ctx, cmd, pipes.stdoutRead, pipes.stderrRead, + ) + stdinDone := pumpProviderShellInput(pipes.stdinWrite, source) + stdoutPump := pumpProviderShellOutput(pipes.stdoutRead, stdout) + stderrPump := pumpProviderShellOutput(pipes.stderrRead, stderr) + waitErr := cmd.Wait() + stdoutErr, stderrErr, drainErr := drainProviderShellOutput( + pipes, stdoutPump, stderrPump, action, + ) + close(stopped) + <-monitorDone + closeErr := input.Close() + stdinCloseErr := closeProviderShellPipe(pipes.stdinWrite) + stdinResult := <-stdinDone + stdinErr := providerShellInputCopyError(stdinResult.copyErr, waitErr, closeErr) + return errors.Join( + drainErr, + providerShellWaitError(action, waitErr), + providerShellCopyError(action, "stdout", stdoutErr), + providerShellCopyError(action, "stderr", stderrErr), + providerShellCopyError(action, "stdin", stdinErr), + providerShellCloseError( + action, + "stdin pipe", + errors.Join(stdinCloseErr, stdinResult.closeErr), + ), + providerShellCloseError(action, "stdio pipes", pipeCloseErr), + providerShellCloseError(action, "input", closeErr), + ) +} + +type providerShellOutputDrain struct { + pipes *providerShellPipes + stdoutPump *providerShellOutputPump + stderrPump *providerShellOutputPump + stdoutDone <-chan error + stderrDone <-chan error + timer *time.Timer + timeout <-chan time.Time + action string + stdoutErr error + stderrErr error + drainErr error +} + +func drainProviderShellOutput( + pipes *providerShellPipes, + stdoutPump, stderrPump *providerShellOutputPump, + action string, +) (stdoutErr, stderrErr, drainErr error) { + timer := time.NewTimer(2 * time.Second) + timer.Stop() + defer timer.Stop() + drain := providerShellOutputDrain{ + pipes: pipes, stdoutPump: stdoutPump, stderrPump: stderrPump, + stdoutDone: stdoutPump.done, stderrDone: stderrPump.done, + timer: timer, action: action, + } + for drain.stdoutDone != nil || drain.stderrDone != nil { + drain.updateTimeout() + select { + case drain.stdoutErr = <-drain.stdoutDone: + drain.stdoutDone = nil + case drain.stderrErr = <-drain.stderrDone: + drain.stderrDone = nil + case <-stdoutPump.changed: + drain.resetTimeout() + case <-stderrPump.changed: + drain.resetTimeout() + case <-drain.timeout: + drain.handleTimeout() + } + } + return drain.stdoutErr, drain.stderrErr, drain.drainErr +} + +func (d *providerShellOutputDrain) idle() bool { + return providerShellOutputIdle(d.stdoutPump, d.stdoutDone) && + providerShellOutputIdle(d.stderrPump, d.stderrDone) +} + +func (d *providerShellOutputDrain) updateTimeout() { + if d.idle() { + if d.timeout == nil { + d.timer.Reset(2 * time.Second) + d.timeout = d.timer.C + } + } else if d.timeout != nil { + d.timer.Stop() + d.timeout = nil + } +} + +func (d *providerShellOutputDrain) resetTimeout() { + if d.timeout != nil { + d.timer.Reset(2 * time.Second) + } +} + +func (d *providerShellOutputDrain) handleTimeout() { + d.timeout = nil + if !d.idle() || d.recentOutput() { + return + } + d.stdoutDone, d.stdoutErr = readyProviderShellOutput(d.stdoutDone, d.stdoutErr) + d.stderrDone, d.stderrErr = readyProviderShellOutput(d.stderrDone, d.stderrErr) + d.drainErr = closeUndrainedProviderShellOutput( + d.pipes, d.stdoutDone, d.stderrDone, d.action, + ) +} + +func (d *providerShellOutputDrain) recentOutput() bool { + select { + case <-d.stdoutPump.changed: + return true + default: + } + select { + case <-d.stderrPump.changed: + return true + default: + } + return false +} + +func providerShellOutputIdle(pump *providerShellOutputPump, done <-chan error) bool { + return done == nil || pump.reading.Load() +} + +func readyProviderShellOutput(done <-chan error, current error) (<-chan error, error) { + if done == nil { + return nil, current + } + select { + case err := <-done: + return nil, err + default: + return done, current + } +} + +func closeUndrainedProviderShellOutput( + pipes *providerShellPipes, + stdoutDone, stderrDone <-chan error, + action string, +) error { + if stdoutDone == nil && stderrDone == nil { + return nil + } + if stdoutDone != nil { + _ = closeProviderShellPipe(pipes.stdoutRead) + } + if stderrDone != nil { + _ = closeProviderShellPipe(pipes.stderrRead) + } + return fmt.Errorf("drain provider %s output after shell exit: %w", action, exec.ErrWaitDelay) +} + +func waitSupervisedProviderShell(ctx context.Context, cmd *exec.Cmd, action string) error { + configureProviderShellProcess(cmd) + if err := cmd.Start(); err != nil { + return fmt.Errorf("start provider %s shell: %w", action, err) + } + stopped, monitorDone := monitorSupervisedProviderShell(ctx, cmd, nil, nil) + err := cmd.Wait() + close(stopped) + <-monitorDone + if err != nil { + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { + code := exitErr.ExitCode() + if code >= 0 && code <= 255 { + err = interp.ExitStatus(uint8(code)) + } + } + return fmt.Errorf("run provider %s: %w", action, err) + } + return nil +} + +func runProviderArgvCommand( + ctx context.Context, + opts RunCommandOptions, + stdout, stderr io.Writer, +) error { cmd := exec.CommandContext(ctx, opts.Command[0], opts.Command[1:]...) // #nosec G204 cmd.Stdin = opts.Stdin cmd.Stdout = stdout cmd.Stderr = stderr cmd.Env = opts.Environ + if opts.Cancellation == provider.ProviderCancellationFile { + cmd.WaitDelay = 2 * time.Second + if opts.closeInputOnExit != nil { + return runProviderArgvWithOwnedInput( + cmd, opts.Stdin, opts.closeInputOnExit, providerAction(opts.Environ), + ) + } + } if err := cmd.Run(); err != nil { - return fmt.Errorf( - "run provider command %q (%s): %w", - strings.Join(opts.Command, " "), providerAction(opts.Environ), err, - ) + return fmt.Errorf("run provider %s: %w", providerAction(opts.Environ), err) } return nil } +func runProviderArgvWithOwnedInput( + cmd *exec.Cmd, + source io.Reader, + input io.Closer, + action string, +) (retErr error) { + stdinRead, stdinWrite, err := os.Pipe() + if err != nil { + return errors.Join( + fmt.Errorf("create provider %s stdin pipe: %w", action, err), + providerShellCloseError(action, "input", input.Close()), + ) + } + defer func() { + retErr = errors.Join( + retErr, closeProviderShellPipe(stdinRead), closeProviderShellPipe(stdinWrite), + ) + }() + cmd.Stdin = stdinRead + if err := cmd.Start(); err != nil { + return errors.Join( + fmt.Errorf("run provider %s: %w", action, err), + providerShellCloseError(action, "input", input.Close()), + ) + } + childPipeCloseErr := closeProviderShellPipe(stdinRead) + stdinDone := pumpProviderShellInput(stdinWrite, source) + waitErr := cmd.Wait() + inputCloseErr := input.Close() + stdinPipeCloseErr := closeProviderShellPipe(stdinWrite) + stdinResult := <-stdinDone + stdinErr := providerShellInputCopyError(stdinResult.copyErr, waitErr, inputCloseErr) + return errors.Join( + providerArgvWaitError(action, waitErr), + providerShellCopyError(action, "stdin", stdinErr), + providerShellCloseError( + action, "stdin pipe", + errors.Join(childPipeCloseErr, stdinPipeCloseErr, stdinResult.closeErr), + ), + providerShellCloseError(action, "input", inputCloseErr), + ) +} + +func providerArgvWaitError(action string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("run provider %s: %w", action, err) +} + func flushCommandOutput(stdout, stderr *subprocess.StreamingRedactingWriter) error { var stdoutErr error if stdout != nil { @@ -901,14 +1410,15 @@ func flushCommandOutput(stdout, stderr *subprocess.StreamingRedactingWriter) err return errors.Join(stdoutErr, stderrErr) } -// providerAction extracts the provider action from the environment variables. +const providerCommandAction = "command" + func providerAction(environ []string) string { for _, kv := range environ { - if rest, ok := strings.CutPrefix(kv, provider.CommandEnv+"="); ok { - return rest + if strings.HasPrefix(kv, provider.CommandEnv+"=") { + return providerCommandAction } } - return "" + return "lifecycle" } func DeleteMachineFolder(context, machineID string) error { diff --git a/pkg/client/clientimplementation/workspace_client_test.go b/pkg/client/clientimplementation/workspace_client_test.go index 2c2d65e488..145534ba4b 100644 --- a/pkg/client/clientimplementation/workspace_client_test.go +++ b/pkg/client/clientimplementation/workspace_client_test.go @@ -6,6 +6,7 @@ import ( "errors" "io" "os" + "os/exec" "path/filepath" "runtime" "strings" @@ -18,6 +19,7 @@ import ( "github.com/devsy-org/devsy/pkg/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "mvdan.cc/sh/v3/interp" ) const ( @@ -141,6 +143,69 @@ func TestRunCommand(t *testing.T) { }) } +func TestMachineCommandErrorsRedactOpaquePayloadAndArguments(t *testing.T) { + requirePOSIXShell(t) + const payload = "private-command-payload-796412" + const argument = "private-configured-argument-796412" + for _, tc := range []struct { + name string + command types.StrArray + }{ + { + name: "emulated shell", + command: types.StrArray{ + `: '` + argument + `'; printf '%s' "$DEVSY_RUN_COMMAND_SECRET" >&2; exit 7`, + }, + }, + { + name: "argv", + command: types.StrArray{binSh, "-c", `printf '%s' "$DEVSY_RUN_COMMAND_SECRET" >&2; exit 7`, argument}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("DEVSY_RUN_COMMAND_SECRET", "credential-sentinel-61395") + cfg := &provider.ProviderConfig{ + Name: "redaction-test", + Exec: provider.ProviderCommands{ + Create: types.StrArray{testProviderCreateCommand}, + Command: tc.command, + }, + } + devsy := &config.Config{ + DefaultContext: testLocksContext, + Contexts: map[string]*config.ContextConfig{testLocksContext: {}}, + } + mc, err := NewMachineClient( + devsy, + cfg, + &provider.Machine{ID: "machine", Context: testLocksContext}, + ) + require.NoError(t, err) + var stderr bytes.Buffer + err = mc.Command( + context.Background(), + client.CommandOptions{Command: payload, Stderr: &stderr}, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "run provider command:") + assert.NotContains(t, err.Error(), payload) + assert.NotContains(t, err.Error(), argument) + assert.NotContains(t, stderr.String(), payload) + assert.NotContains(t, stderr.String(), argument) + assert.Contains(t, stderr.String(), "***") + if tc.name == "emulated shell" { + var status interp.ExitStatus + require.ErrorAs(t, err, &status) + assert.Equal(t, 7, int(status)) + } else { + var status *exec.ExitError + require.ErrorAs(t, err, &status) + assert.Equal(t, 7, status.ExitCode()) + } + }) + } +} + func TestRunCommand_RawStdoutDoesNotHoldProtocolBytes(t *testing.T) { requireProtocolByteRoundTrip( t, diff --git a/pkg/config/env.go b/pkg/config/env.go index f3fa99eaa3..408552fa4b 100644 --- a/pkg/config/env.go +++ b/pkg/config/env.go @@ -13,6 +13,8 @@ const ( // EnvBinaryPath is set to the path of the Devsy binary. EnvBinaryPath = "DEVSY" + EnvProviderCancelFile = "DEVSY_PROVIDER_CANCEL_FILE" + // EnvHome overrides the default Devsy home directory. EnvHome = "DEVSY_HOME" diff --git a/pkg/options/resolve.go b/pkg/options/resolve.go index 6faf0ef92f..9716275965 100644 --- a/pkg/options/resolve.go +++ b/pkg/options/resolve.go @@ -344,6 +344,9 @@ func resolveAgentDockerConfig( agentConfig.Docker.Install = types.StrBool( resolver.ResolveDefaultValue(string(agentConfig.Docker.Install), options), ) + agentConfig.Docker.ConfigureDaemon = types.StrBool( + resolver.ResolveDefaultValue(string(agentConfig.Docker.ConfigureDaemon), options), + ) agentConfig.Docker.Env = resolver.ResolveDefaultValues(agentConfig.Docker.Env, options) } diff --git a/pkg/options/resolve_docker_test.go b/pkg/options/resolve_docker_test.go new file mode 100644 index 0000000000..1bba28ac40 --- /dev/null +++ b/pkg/options/resolve_docker_test.go @@ -0,0 +1,51 @@ +package options + +import ( + "encoding/json" + "testing" + + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/types" + "github.com/stretchr/testify/require" +) + +const disabledDockerPolicy = "false" + +func TestResolveDockerDaemonPolicyIntoWorkspaceInfo(t *testing.T) { + manifest := &provider.ProviderConfig{ + Name: "docker-policy", + Agent: provider.ProviderAgentConfig{ + Driver: provider.DockerDriver, + Docker: provider.ProviderDockerDriverConfig{ + Install: "${INSTALL_DOCKER}", + ConfigureDaemon: "${CONFIGURE_DAEMON}", + }, + }, + } + devConfig := &config.Config{ + DefaultContext: config.DefaultContext, + Contexts: map[string]*config.ContextConfig{ + config.DefaultContext: {Providers: map[string]*config.ProviderConfig{ + manifest.Name: {Options: map[string]config.OptionValue{ + "INSTALL_DOCKER": {Value: disabledDockerPolicy}, + "CONFIGURE_DAEMON": {Value: disabledDockerPolicy}, + }}, + }}, + }, + } + resolved := ResolveAgentConfig(devConfig, manifest, nil, nil) + require.Equal(t, types.StrBool(disabledDockerPolicy), resolved.Docker.ConfigureDaemon) + require.Equal(t, types.StrBool(disabledDockerPolicy), resolved.Docker.Install) + require.Equal(t, "${CONFIGURE_DAEMON}", string(manifest.Agent.Docker.ConfigureDaemon)) + + data, err := json.Marshal(&provider.AgentWorkspaceInfo{Agent: resolved}) + require.NoError(t, err) + require.Contains(t, string(data), `"configureDaemon":"false"`) + require.Contains(t, string(data), `"install":"false"`) + + var decoded provider.AgentWorkspaceInfo + require.NoError(t, json.Unmarshal(data, &decoded)) + require.Equal(t, types.StrBool(disabledDockerPolicy), decoded.Agent.Docker.ConfigureDaemon) + require.Equal(t, types.StrBool(disabledDockerPolicy), decoded.Agent.Docker.Install) +} diff --git a/pkg/provider/dir.go b/pkg/provider/dir.go index 7dd8de4dcf..ce2c39b6a7 100644 --- a/pkg/provider/dir.go +++ b/pkg/provider/dir.go @@ -324,14 +324,13 @@ func LoadProviderConfig(context, provider string) (*ProviderConfig, error) { return nil, err } - // Refresh the exec commands of built-in providers from embedded YAML: a CLI - // change (e.g. renaming the command wrapper) must not be shadowed by a stale - // provider.json. Only the exec blocks are overlaid so user customizations - // (custom --name, resolved options) survive. if providerConfig.Source.Internal { if embedded := embeddedProviderConfig(providerConfig.Source.Raw); embedded != nil { providerConfig.Exec = embedded.Exec providerConfig.Agent.Exec = embedded.Agent.Exec + if providerConfig.Source.Raw == DockerDriver { + providerConfig.Agent.Docker.Install = embedded.Agent.Docker.Install + } } } diff --git a/pkg/provider/load_internal_test.go b/pkg/provider/load_internal_test.go index 5cc0053f74..2b14cadbbc 100644 --- a/pkg/provider/load_internal_test.go +++ b/pkg/provider/load_internal_test.go @@ -13,7 +13,10 @@ import ( // stale provider.json files carry before a refresh rewrites them. const staleHelperShCommand = `"${DEVSY}" helper sh -c "${COMMAND}"` -const staleVersion = "v0.0.0-stale" +const ( + staleVersion = "v0.0.0-stale" + podmanProviderName = "podman" +) // TestLoadProviderConfig_RefreshesInternalProvider verifies that a stored // built-in provider config with a stale exec.command (e.g. the removed @@ -41,9 +44,6 @@ func TestLoadProviderConfig_RefreshesInternalProvider(t *testing.T) { "refreshed command should use the current 'internal sh' wrapper") } -// TestLoadProviderConfig_PreservesCustomizations verifies the exec refresh -// overlays only the embedded exec block, leaving a user's custom Name and -// resolved Options intact (a built-in added with `--name`/`--option`). func TestLoadProviderConfig_PreservesCustomizations(t *testing.T) { setupTestHome(t) @@ -52,7 +52,7 @@ func TestLoadProviderConfig_PreservesCustomizations(t *testing.T) { Version: staleVersion, Source: ProviderSource{Internal: true, Raw: DockerDriver}, Options: map[string]*types.Option{ - "DOCKER_PATH": {Default: "podman"}, + "DOCKER_PATH": {Default: podmanProviderName}, }, Exec: ProviderCommands{ Command: []string{staleHelperShCommand}, @@ -64,7 +64,7 @@ func TestLoadProviderConfig_PreservesCustomizations(t *testing.T) { require.NoError(t, err) require.Equal(t, "my-docker", loaded.Name, "custom provider name must survive refresh") require.Equal(t, "v0.0.0-stale", loaded.Version, "stored version must survive refresh") - require.Equal(t, "podman", loaded.Options["DOCKER_PATH"].Default, + require.Equal(t, podmanProviderName, loaded.Options["DOCKER_PATH"].Default, "resolved options must survive refresh") require.NotContains(t, loaded.Exec.Command[0], "helper sh", "exec block must still be refreshed from embedded yaml") @@ -133,3 +133,30 @@ func TestLoadProviderConfig_PreservesExternalProvider(t *testing.T) { require.NoError(t, err) require.Equal(t, external.Exec.Command, loaded.Exec.Command) } + +func TestLoadProviderConfig_DockerInstallPolicyScopes(t *testing.T) { + for _, tc := range []struct { + name string + source ProviderSource + want types.StrBool + }{ + {name: "builtin-docker", source: ProviderSource{Internal: true, Raw: DockerDriver}, want: enabledDockerPolicy}, + {name: "external-docker", source: ProviderSource{Github: "some-org/provider"}, want: disabledDockerPolicy}, + {name: "builtin-podman", source: ProviderSource{Internal: true, Raw: podmanProviderName}, want: disabledDockerPolicy}, + } { + t.Run(tc.name, func(t *testing.T) { + setupTestHome(t) + stored := &ProviderConfig{ + Name: tc.name, Source: tc.source, + Agent: ProviderAgentConfig{ + Docker: ProviderDockerDriverConfig{Install: disabledDockerPolicy}, + }, + Exec: ProviderCommands{Command: []string{staleHelperShCommand}}, + } + require.NoError(t, SaveProviderConfig(config.DefaultContext, stored)) + loaded, err := LoadProviderConfig(config.DefaultContext, stored.Name) + require.NoError(t, err) + require.Equal(t, tc.want, loaded.Agent.Docker.Install) + }) + } +} diff --git a/pkg/provider/parse.go b/pkg/provider/parse.go index bda8b07fde..923102a01d 100644 --- a/pkg/provider/parse.go +++ b/pkg/provider/parse.go @@ -205,6 +205,10 @@ func validateOptionCacheAndType(optionName string, optionValue *types.Option) er } func validateProviderType(config *ProviderConfig) error { + if config.Exec.Cancellation != ProviderCancellationDefault && + config.Exec.Cancellation != ProviderCancellationFile { + return fmt.Errorf("exec.cancellation must be empty or file") + } if config.IsProxyProvider() { return validateProxyProvider(config) } @@ -281,6 +285,9 @@ func validateStandardProvider(config *ProviderConfig) error { if err := validateAgentDriver(config); err != nil { return err } + if err := validateAgentDockerPolicies(config.Agent.Docker); err != nil { + return err + } if err := validateBinaries("agent.binaries", config.Agent.Binaries); err != nil { return err @@ -289,6 +296,22 @@ func validateStandardProvider(config *ProviderConfig) error { return validateExecCommands(config) } +func validateAgentDockerPolicies(docker ProviderDockerDriverConfig) error { + if value := docker.ConfigureDaemon; value != "" && + !templatedValueRegex.MatchString(string(value)) { + if _, err := value.Bool(); err != nil { + return fmt.Errorf("agent.docker.configureDaemon: %w", err) + } + } + if value := docker.Install; value != "" && + !templatedValueRegex.MatchString(string(value)) { + if _, err := value.Bool(); err != nil { + return fmt.Errorf("agent.docker.install: %w", err) + } + } + return nil +} + var validAgentDrivers = map[string]bool{ "": true, CustomDriver: true, @@ -345,7 +368,6 @@ func validateExecCommands(config *ProviderConfig) error { if len(config.Exec.Command) == 0 { return fmt.Errorf("exec.command is required") } - if err := validateExecPair( "exec.create", "exec.delete", diff --git a/pkg/provider/parse_docker_test.go b/pkg/provider/parse_docker_test.go new file mode 100644 index 0000000000..bcb0d87b67 --- /dev/null +++ b/pkg/provider/parse_docker_test.go @@ -0,0 +1,84 @@ +package provider + +import ( + "strings" + "testing" +) + +const ( + enabledDockerPolicy = "true" + disabledDockerPolicy = "false" +) + +const templatePolicyCase = "template" + +func TestParseDockerConfigureDaemon(t *testing.T) { + for _, tc := range []struct { + name string + value string + wantErr bool + }{ + {name: "unset"}, + {name: "enabled", value: enabledDockerPolicy}, + {name: "disabled", value: disabledDockerPolicy}, + {name: templatePolicyCase, value: "${CONFIGURE_DAEMON}"}, + {name: "bare template", value: "$CONFIGURE_DAEMON"}, + {name: "invalid", value: "sometimes", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + manifest := "name: docker-policy\nagent:\n driver: docker\n" + if tc.value != "" { + manifest += " docker:\n configureDaemon: " + tc.value + "\n" + } + manifest += "exec:\n command: echo hi\n" + parsed, err := ParseProvider(strings.NewReader(manifest)) + if tc.wantErr { + if err == nil || !strings.Contains(err.Error(), "agent.docker.configureDaemon") { + t.Fatalf("ParseProvider() error = %v, want Docker daemon policy error", err) + } + return + } + if err != nil { + t.Fatal(err) + } + if string(parsed.Agent.Docker.ConfigureDaemon) != tc.value { + t.Errorf( + "ConfigureDaemon = %q, want %q", + parsed.Agent.Docker.ConfigureDaemon, + tc.value, + ) + } + }) + } +} + +func TestParseDockerInstallPolicy(t *testing.T) { + for _, tc := range []struct { + name string + value string + wantErr bool + }{ + {name: "disabled", value: disabledDockerPolicy}, + {name: "enabled", value: enabledDockerPolicy}, + {name: templatePolicyCase, value: "${INSTALL_DOCKER}"}, + {name: "invalid", value: "sometimes", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + manifest := "name: docker-policy\nagent:\n driver: docker\n docker:\n install: " + + tc.value + "\nexec:\n command: echo hi\n" + parsed, err := ParseProvider(strings.NewReader(manifest)) + if tc.wantErr { + if err == nil || !strings.Contains(err.Error(), "agent.docker.install") { + t.Fatalf("ParseProvider() error = %v, want Docker install policy error", err) + } + return + } + if err != nil { + t.Fatal(err) + } + if string(parsed.Agent.Docker.Install) != tc.value { + t.Errorf("Install = %q, want %q", parsed.Agent.Docker.Install, tc.value) + } + }) + } +} diff --git a/pkg/provider/parse_driver_test.go b/pkg/provider/parse_driver_test.go index ee7c241eaa..e23fa0ec8f 100644 --- a/pkg/provider/parse_driver_test.go +++ b/pkg/provider/parse_driver_test.go @@ -27,7 +27,7 @@ exec: {name: "kubernetes literal", driver: KubernetesDriver, wantErr: false}, {name: "templated braces", driver: "${AWS_DEPLOYMENT_MODE}", wantErr: false}, {name: "templated bare", driver: "$AWS_DEPLOYMENT_MODE", wantErr: false}, - {name: "invalid literal", driver: "podman", wantErr: true}, + {name: "invalid literal", driver: podmanProviderName, wantErr: true}, } for _, tc := range cases { @@ -42,3 +42,36 @@ exec: }) } } + +func TestProviderCancellationMode(t *testing.T) { + for _, tc := range []struct { + name string + value string + valid bool + }{ + {name: "default", valid: true}, + {name: string(ProviderCancellationFile), value: string(ProviderCancellationFile), valid: true}, + {name: "unknown", value: "signal"}, + {name: templatePolicyCase, value: "${CANCELLATION}"}, + } { + t.Run(tc.name, func(t *testing.T) { + manifest := "name: example\nversion: v1.0.0\nexec:\n command: echo ok\n" + if tc.value != "" { + manifest += " cancellation: '" + tc.value + "'\n" + } + parsed, err := ParseProvider(strings.NewReader(manifest)) + if !tc.valid { + if err == nil { + t.Fatal("expected invalid cancellation mode") + } + return + } + if err != nil { + t.Fatal(err) + } + if parsed.Exec.Cancellation != ProviderCancellationMode(tc.value) { + t.Fatalf("unexpected cancellation mode: %q", parsed.Exec.Cancellation) + } + }) + } +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index e86217ea8d..8e449759c9 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -419,7 +419,8 @@ type ProviderDockerDriverConfig struct { Path string `json:"path,omitempty"` // If false, Devsy will not try to install docker into the machine. - Install types.StrBool `json:"install,omitempty"` + Install types.StrBool `json:"install,omitempty"` + ConfigureDaemon types.StrBool `json:"configureDaemon,omitempty"` // Builder to use with docker Builder string `json:"builder,omitempty"` @@ -517,7 +518,16 @@ type ProviderBinary struct { Name string `json:"name,omitempty"` } +type ProviderCancellationMode string + +const ( + ProviderCancellationDefault ProviderCancellationMode = "" + ProviderCancellationFile ProviderCancellationMode = "file" +) + type ProviderCommands struct { + Cancellation ProviderCancellationMode `json:"cancellation,omitempty"` + // Init is run directly after `devsy provider init` Init types.StrArray `json:"init,omitempty"` diff --git a/pkg/shell/provider_exec.go b/pkg/shell/provider_exec.go new file mode 100644 index 0000000000..4c247b7205 --- /dev/null +++ b/pkg/shell/provider_exec.go @@ -0,0 +1,147 @@ +package shell + +import ( + "context" + "errors" + "fmt" + "io" + "os/exec" + "runtime" + "strings" + "syscall" + "time" + + "mvdan.cc/sh/v3/expand" + "mvdan.cc/sh/v3/interp" +) + +func ProviderExecHandler() interp.ExecHandlerFunc { + return providerExecHandler(nil) +} + +func providerExecHandler(supervisor *providerProcessSupervisor) interp.ExecHandlerFunc { + return func(ctx context.Context, args []string) error { + hc := interp.HandlerCtx(ctx) + path, err := interp.LookPathDir(hc.Dir, hc.Env, args[0]) + if err != nil { + _, _ = fmt.Fprintln(hc.Stderr, "provider executable not found") + return interp.ExitStatus(127) + } + + command := providerExecCommand{path: path, args: args} + err = runOwnedProviderExec(ctx, hc, command, supervisor) + if errors.Is(err, syscall.ENOEXEC) && runtime.GOOS != providerWindowsOS { + command.path = "/bin/sh" + command.args = append([]string{"/bin/sh", path}, args[1:]...) + err = runOwnedProviderExec(ctx, hc, command, supervisor) + } + if ctx.Err() != nil { + return context.Cause(ctx) + } + return providerExecResult(err, hc.Stderr) + } +} + +func providerExecResult(err error, stderr io.Writer) error { + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { + if status, ok := providerExecSignalStatus(exitErr); ok { + return status + } + code := exitErr.ExitCode() + if code < 0 || code > 255 { + return fmt.Errorf("provider process exit code outside shell range: %d", code) + } + return interp.ExitStatus(uint8(code)) + } + if _, ok := errors.AsType[*exec.Error](err); ok { + _, _ = fmt.Fprintln(stderr, "provider executable could not start") + return interp.ExitStatus(127) + } + return err +} + +type providerExecCommand struct { + path string + args []string +} + +func runOwnedProviderExec( + ctx context.Context, + hc interp.HandlerContext, + command providerExecCommand, + supervisor *providerProcessSupervisor, +) error { + cmd, err := startProviderExec(ctx, hc, command, supervisor) + if err != nil { + return err + } + if ctx.Done() == nil { + err = cmd.Wait() + if supervisor != nil { + supervisor.reaped(cmd) + } + return err + } + done := make(chan struct{}) + monitorDone := make(chan struct{}) + go func() { + defer close(monitorDone) + select { + case <-ctx.Done(): + _ = cmd.Process.Kill() + case <-done: + } + }() + err = cmd.Wait() + close(done) + <-monitorDone + if supervisor != nil { + supervisor.reaped(cmd) + } + return err +} + +func startProviderExec( + ctx context.Context, + hc interp.HandlerContext, + command providerExecCommand, + supervisor *providerProcessSupervisor, +) (*exec.Cmd, error) { + environ := providerExecEnv(hc.Env) + newCommand := func() *exec.Cmd { + return &exec.Cmd{ + Path: command.path, + Args: command.args, + Env: environ, + Dir: hc.Dir, + Stdin: hc.Stdin, + Stdout: hc.Stdout, + Stderr: hc.Stderr, + WaitDelay: 2 * time.Second, + } + } + if ctx.Err() != nil { + return nil, context.Cause(ctx) + } + if supervisor != nil { + return supervisor.start(ctx, newCommand) + } + return startProviderExecRetry(ctx, newCommand) +} + +func providerExecEnv(env expand.Environ) []string { + vars := make([]string, 0, 64) + for name, variable := range env.Each { + if !variable.IsSet() { + for i, kv := range vars { + if strings.HasPrefix(kv, name+"=") { + vars[i] = "" + } + } + } + if variable.Exported && variable.Kind == expand.String { + vars = append(vars, name+"="+variable.String()) + } + } + return vars +} diff --git a/pkg/shell/provider_exec_test.go b/pkg/shell/provider_exec_test.go new file mode 100644 index 0000000000..3d6771e73e --- /dev/null +++ b/pkg/shell/provider_exec_test.go @@ -0,0 +1,181 @@ +package shell + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "mvdan.cc/sh/v3/interp" +) + +func TestProviderExecHelper(t *testing.T) { + if os.Getenv("DEVSY_PROVIDER_EXEC_HELPER") != "1" { + return + } + if os.Getenv("DEVSY_PROVIDER_EXEC_EXIT") == "wait" { + if _, err := fmt.Fprint(os.Stdout, "ready"); err != nil { + os.Exit(126) + } + time.Sleep(time.Minute) + } + if os.Getenv("DEVSY_PROVIDER_EXEC_EXIT") == "read-eof" { + providerExecReadToEOF() + } + if os.Getenv("DEVSY_PROVIDER_EXEC_EXIT") != "" { + os.Exit(125) + } + result := struct { + Directory string + Present string + Missing string + Argument string + }{Present: os.Getenv("PRESENT"), Missing: os.Getenv("MISSING")} + result.Directory, _ = os.Getwd() + result.Argument = os.Args[len(os.Args)-1] + _ = json.NewEncoder(os.Stdout).Encode(result) + os.Exit(0) +} + +func providerExecReadToEOF() { + if _, err := fmt.Fprint(os.Stdout, "before"); err != nil { + os.Exit(126) + } + input, err := io.ReadAll(os.Stdin) + if err != nil { + os.Exit(126) + } + if _, err := fmt.Fprintf(os.Stdout, "after%s", input); err != nil { + os.Exit(126) + } + os.Exit(0) +} + +func TestProviderExecHandlerExportsUnsetAndWorkingDirectory(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + workdir := t.TempDir() + command := fmt.Sprintf( + "export PRESENT='quoted value'; unset MISSING; cd %s; %s -test.run=^TestProviderExecHelper$ 'quoted argument'", + strconv.Quote(workdir), + strconv.Quote(binary), + ) + if runtime.GOOS == providerWindowsOS { + command = strings.ReplaceAll(command, `\`, `/`) + } + var stdout, stderr bytes.Buffer + err = RunEmulatedShell(context.Background(), &CommandRunner{ + Command: command, ExecHandler: ProviderExecHandler(), + Environ: append(os.Environ(), "DEVSY_PROVIDER_EXEC_HELPER=1", "MISSING=should-be-unset"), + Stdout: &stdout, Stderr: &stderr, + }) + require.NoErrorf(t, err, "stderr: %s", stderr.String()) + var got struct{ Directory, Present, Missing, Argument string } + require.NoError(t, json.Unmarshal(stdout.Bytes(), &got)) + expectedDir, err := filepath.EvalSymlinks(workdir) + require.NoError(t, err) + actualDir, err := filepath.EvalSymlinks(got.Directory) + require.NoError(t, err) + require.Truef(t, strings.EqualFold(filepath.Clean(actualDir), filepath.Clean(expectedDir)), + "working directory: got %q, want %q", got.Directory, workdir) + require.Equal(t, "quoted value", got.Present) + require.Empty(t, got.Missing) + require.Equal(t, "quoted argument", got.Argument) +} + +func TestProviderExecHandlerKeepsOutputAfterInputEOF(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + var stdout bytes.Buffer + err = RunEmulatedShell(context.Background(), &CommandRunner{ + Command: strconv.Quote(binary) + " -test.run=^TestProviderExecHelper$", + Environ: append( + os.Environ(), + "DEVSY_PROVIDER_EXEC_HELPER=1", + "DEVSY_PROVIDER_EXEC_EXIT=read-eof", + ), + ExecHandler: ProviderExecHandler(), + Stdin: strings.NewReader("payload"), + Stdout: &stdout, + }) + require.NoError(t, err) + require.Equal(t, "beforeafterpayload", stdout.String()) +} + +func TestProviderExecHandlerExitStatuses(t *testing.T) { + for _, tc := range []struct { + command string + status int + }{ + {command: "command-that-does-not-exist-846307", status: 127}, + {command: "exit 7", status: 7}, + {command: "exit 125", status: 125}, + } { + var stderr bytes.Buffer + err := RunEmulatedShell(context.Background(), &CommandRunner{ + Command: tc.command, + ExecHandler: ProviderExecHandler(), + Environ: os.Environ(), + Stderr: &stderr, + }) + exitStatus, ok := errors.AsType[interp.ExitStatus](err) + if !ok || int(exitStatus) != tc.status { + t.Errorf("%q: got %v, want exit %d", tc.command, err, tc.status) + } + } +} + +func TestProviderExecHandlerCancellationReturnsCause(t *testing.T) { + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancelCause(context.Background()) + cause := errors.New("forced provider cancellation") + ready := make(chan struct{}) + finished := make(chan error, 1) + go func() { + finished <- RunEmulatedShell(ctx, &CommandRunner{ + Command: strconv.Quote(binary) + " -test.run=^TestProviderExecHelper$", + Environ: append(os.Environ(), "DEVSY_PROVIDER_EXEC_HELPER=1", "DEVSY_PROVIDER_EXEC_EXIT=wait"), + ExecHandler: ProviderExecHandler(), Stdout: &shellReadyWriter{ready: ready}, + }) + }() + select { + case <-ready: + case <-time.After(5 * time.Second): + t.Fatal("shell child did not start") + } + cancel(cause) + select { + case err := <-finished: + if !errors.Is(err, cause) { + t.Fatalf("cancellation: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("provider shell did not finish") + } +} + +type shellReadyWriter struct{ ready chan struct{} } + +func (w *shellReadyWriter) Write(p []byte) (int, error) { + if bytes.Contains(p, []byte("ready")) { + select { + case <-w.ready: + default: + close(w.ready) + } + } + return len(p), nil +} diff --git a/pkg/shell/provider_exec_unix.go b/pkg/shell/provider_exec_unix.go new file mode 100644 index 0000000000..1e84d06c29 --- /dev/null +++ b/pkg/shell/provider_exec_unix.go @@ -0,0 +1,30 @@ +//go:build !windows + +package shell + +import ( + "os" + "os/exec" + "syscall" + + "mvdan.cc/sh/v3/interp" +) + +func providerPrivateDirectory(_ string, info os.FileInfo) bool { + return info.Mode().Perm() == 0o700 +} + +func providerPrivateScript(info os.FileInfo) bool { + return info.Mode().Perm() == 0o600 +} + +func providerExecSignalStatus(err *exec.ExitError) (interp.ExitStatus, bool) { + if status, ok := err.Sys().(syscall.WaitStatus); ok && status.Signaled() { + code := 128 + int(status.Signal()) + if code < 0 || code > 255 { + return 0, false + } + return interp.ExitStatus(uint8(code)), true + } + return 0, false +} diff --git a/pkg/shell/provider_exec_windows.go b/pkg/shell/provider_exec_windows.go new file mode 100644 index 0000000000..164cc54e74 --- /dev/null +++ b/pkg/shell/provider_exec_windows.go @@ -0,0 +1,138 @@ +package shell + +import ( + "os" + "os/exec" + "unsafe" + + "golang.org/x/sys/windows" + "mvdan.cc/sh/v3/interp" +) + +func providerPrivateDirectory(path string, info os.FileInfo) bool { + if !info.IsDir() { + return false + } + sd, err := windows.GetNamedSecurityInfo( + path, + windows.SE_FILE_OBJECT, + windows.DACL_SECURITY_INFORMATION, + ) + if err != nil { + return false + } + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + return false + } + return providerPrivateDACL(sd, user.User.Sid) +} + +func providerPrivateDACL(sd *windows.SECURITY_DESCRIPTOR, user *windows.SID) bool { + acl, ok := providerProtectedACL(sd) + if !ok { + return false + } + system, err := windows.CreateWellKnownSid(windows.WinLocalSystemSid) + if err != nil { + return false + } + var grants providerACLGrants + for i := uint32(0); i < uint32(acl.AceCount); i++ { + ace, ok := providerACLAt(acl, i) + if !ok || !grants.allow(ace, user, system) { + return false + } + } + return grants.private(user.Equals(system)) +} + +func providerProtectedACL(sd *windows.SECURITY_DESCRIPTOR) (*windows.ACL, bool) { + control, _, err := sd.Control() + if err != nil || control&windows.SE_DACL_PROTECTED == 0 { + return nil, false + } + acl, _, err := sd.DACL() + return acl, err == nil && acl != nil +} + +func providerACLAt(acl *windows.ACL, index uint32) (*windows.ACCESS_ALLOWED_ACE, bool) { + var ace *windows.ACCESS_ALLOWED_ACE + if err := windows.GetAce(acl, index, &ace); err != nil { + return nil, false + } + return ace, ace != nil +} + +type providerACLGrants struct { + directory [2]uint32 + descendants [2]uint32 +} + +func (g *providerACLGrants) allow(ace *windows.ACCESS_ALLOWED_ACE, user, system *windows.SID) bool { + mask, ok := providerACEMask(ace) + if !ok { + return false + } + index, ok := providerACETrustee(ace, user, system) + if !ok { + return false + } + flags := ace.Header.AceFlags + if flags&windows.INHERIT_ONLY_ACE == 0 { + g.directory[index] |= mask + } + if flags&(windows.OBJECT_INHERIT_ACE|windows.CONTAINER_INHERIT_ACE) == + windows.OBJECT_INHERIT_ACE|windows.CONTAINER_INHERIT_ACE { + g.descendants[index] |= mask + } + return true +} + +func providerACEMask(ace *windows.ACCESS_ALLOWED_ACE) (uint32, bool) { + flags := ace.Header.AceFlags + if ace.Header.AceType != windows.ACCESS_ALLOWED_ACE_TYPE || + flags & ^uint8( + windows.OBJECT_INHERIT_ACE|windows.CONTAINER_INHERIT_ACE|windows.INHERIT_ONLY_ACE, + ) != 0 { + return 0, false + } + const genericAll = uint32(0x10000000) + const fileAll = uint32(0x001f01ff) + mask := uint32(ace.Mask) + if mask & ^uint32(genericAll|fileAll) != 0 { + return 0, false + } + if mask&genericAll != 0 { + mask |= fileAll + } + return mask, true +} + +func providerACETrustee(ace *windows.ACCESS_ALLOWED_ACE, user, system *windows.SID) (int, bool) { + sid := (*windows.SID)(unsafe.Pointer(&ace.SidStart)) + if sid.Equals(system) { + return 1, true + } + if sid.Equals(user) { + return 0, true + } + return 0, false +} + +func (g providerACLGrants) private(userIsSystem bool) bool { + const fileAll = uint32(0x001f01ff) + if userIsSystem { + return g.directory[1]&fileAll == fileAll && g.descendants[1]&fileAll == fileAll + } + return g.directory[0]&fileAll == fileAll && g.directory[1]&fileAll == fileAll && + g.descendants[0]&fileAll == fileAll && g.descendants[1]&fileAll == fileAll +} + +func providerPrivateScript(info os.FileInfo) bool { + return info.Mode().IsRegular() +} + +func providerExecSignalStatus(_ *exec.ExitError) (interp.ExitStatus, bool) { + return 0, false +} diff --git a/pkg/shell/provider_foreground_unix.go b/pkg/shell/provider_foreground_unix.go new file mode 100644 index 0000000000..ee5df397f4 --- /dev/null +++ b/pkg/shell/provider_foreground_unix.go @@ -0,0 +1,51 @@ +//go:build !windows + +package shell + +import ( + "fmt" + "os" + "os/signal" + "syscall" +) + +func PrepareProviderShellForeground(parentPID int) (cleanup func(), err error) { + if parentPID <= 0 || os.Getppid() != parentPID { + return nil, fmt.Errorf("provider shell parent mismatch") + } + signals := providerTerminalSignals() + ch := make(chan os.Signal, 3) + cleanup = func() { signal.Stop(ch) } + if len(signals) > 0 { + signal.Notify(ch, signals...) + } + group, err := syscall.Getpgid(parentPID) + if err == nil { + err = syscall.Setpgid(0, group) + } + if err == nil && os.Getppid() != parentPID { + err = fmt.Errorf("provider shell parent mismatch") + } + if err != nil { + cleanup() + return nil, err + } + return cleanup, nil +} + +func providerTerminalSignals() []os.Signal { + var signals []os.Signal + for _, sig := range []os.Signal{syscall.SIGINT, syscall.SIGQUIT, syscall.SIGHUP} { + if !signal.Ignored(sig) { + signals = append(signals, sig) + } + } + return signals +} + +func newProviderShellParentMonitor(parentPID int) (alive func() bool, cleanup func(), err error) { + if parentPID <= 0 || os.Getppid() != parentPID { + return nil, nil, fmt.Errorf("provider shell parent mismatch") + } + return func() bool { return os.Getppid() == parentPID }, func() {}, nil +} diff --git a/pkg/shell/provider_foreground_windows.go b/pkg/shell/provider_foreground_windows.go new file mode 100644 index 0000000000..958d84bdd1 --- /dev/null +++ b/pkg/shell/provider_foreground_windows.go @@ -0,0 +1,55 @@ +package shell + +import ( + "fmt" + "os" + "syscall" + + "golang.org/x/sys/windows" +) + +var ( + providerKernel32 = windows.NewLazySystemDLL("kernel32.dll") + providerSetConsoleCtrlHandler = providerKernel32.NewProc("SetConsoleCtrlHandler") + providerGetConsoleCP = providerKernel32.NewProc("GetConsoleCP") + providerConsoleHandler = syscall.NewCallback(func(event uint32) uintptr { + if event == 0 || event == 1 { + return 1 + } + return 0 + }) +) + +func PrepareProviderShellForeground(parentPID int) (func(), error) { + if parentPID <= 0 || uint64(parentPID) > uint64(^uint32(0)) || os.Getppid() != parentPID { + return nil, fmt.Errorf("provider shell parent mismatch") + } + cp, _, _ := providerGetConsoleCP.Call() + if cp == 0 { + return func() {}, nil + } + ok, _, err := providerSetConsoleCtrlHandler.Call(providerConsoleHandler, 1) + if ok == 0 { + return nil, fmt.Errorf("register provider console handler: %w", err) + } + ok, _, err = providerSetConsoleCtrlHandler.Call(0, 0) + if ok == 0 { + _, _, _ = providerSetConsoleCtrlHandler.Call(providerConsoleHandler, 0) + return nil, fmt.Errorf("enable child console interrupts: %w", err) + } + return func() { _, _, _ = providerSetConsoleCtrlHandler.Call(providerConsoleHandler, 0) }, nil +} + +func newProviderShellParentMonitor(parentPID int) (func() bool, func(), error) { + if parentPID <= 0 || uint64(parentPID) > uint64(^uint32(0)) { + return nil, nil, fmt.Errorf("invalid parent PID %d", parentPID) + } + handle, err := windows.OpenProcess(windows.SYNCHRONIZE, false, uint32(parentPID)) + if err != nil { + return nil, nil, fmt.Errorf("open parent process: %w", err) + } + return func() bool { + result, err := windows.WaitForSingleObject(handle, 0) + return err == nil && result == uint32(windows.WAIT_TIMEOUT) + }, func() { _ = windows.CloseHandle(handle) }, nil +} diff --git a/pkg/shell/provider_supervisor.go b/pkg/shell/provider_supervisor.go new file mode 100644 index 0000000000..8ff1b1c4ee --- /dev/null +++ b/pkg/shell/provider_supervisor.go @@ -0,0 +1,423 @@ +package shell + +import ( + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "syscall" + "time" +) + +const providerWindowsOS = "windows" + +type providerProcessSupervisor struct { + mu sync.Mutex + forced bool + children map[*exec.Cmd]chan struct{} +} + +func (s *providerProcessSupervisor) start( + ctx context.Context, + newCommand func() *exec.Cmd, +) (*exec.Cmd, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.forced { + return nil, context.Canceled + } + if ctx.Err() != nil { + return nil, context.Cause(ctx) + } + cmd, err := startProviderExecRetry(ctx, newCommand) + if err != nil { + return nil, err + } + if s.children == nil { + s.children = make(map[*exec.Cmd]chan struct{}) + } + s.children[cmd] = make(chan struct{}) + return cmd, nil +} + +func startProviderExecRetry(ctx context.Context, newCommand func() *exec.Cmd) (*exec.Cmd, error) { + cmd := newCommand() + err := cmd.Start() + for delay := time.Millisecond; errors.Is(err, syscall.ETXTBSY) && delay < 300*time.Millisecond; delay *= 2 { + select { + case <-ctx.Done(): + return nil, context.Cause(ctx) + case <-time.After(delay): + } + cmd = newCommand() + err = cmd.Start() + } + if err != nil { + return nil, err + } + return cmd, nil +} + +func (s *providerProcessSupervisor) reaped(cmd *exec.Cmd) { + s.mu.Lock() + if done, ok := s.children[cmd]; ok { + delete(s.children, cmd) + close(done) + } + s.mu.Unlock() +} + +func (s *providerProcessSupervisor) force(cancel context.CancelFunc) { + s.terminate(cancel) +} + +func (s *providerProcessSupervisor) stop() { + s.terminate(nil) +} + +func (s *providerProcessSupervisor) terminate(cancel context.CancelFunc) { + s.mu.Lock() + s.forced = true + if cancel != nil { + cancel() + } + children := make([]*exec.Cmd, 0, len(s.children)) + for cmd := range s.children { + children = append(children, cmd) + } + s.mu.Unlock() + for _, cmd := range children { + _ = cmd.Process.Kill() + } + for _, cmd := range children { + s.mu.Lock() + done := s.children[cmd] + s.mu.Unlock() + if done != nil { + <-done + } + } +} + +func providerShellPaths(scriptPath, forcePath, ackPath string) error { + if !filepath.IsAbs(scriptPath) || filepath.Clean(scriptPath) != scriptPath || + filepath.Base(scriptPath) != "script" { + return errors.New("provider shell requires an absolute private script path") + } + dir := filepath.Dir(scriptPath) + if forcePath != filepath.Join(dir, "force") || ackPath != filepath.Join(dir, "ack") { + return errors.New("provider shell control files must be private sibling paths") + } + return nil +} + +func providerShellRoot(dir string) (*os.Root, error) { + root, err := os.OpenRoot(dir) + if err != nil { + return nil, errors.New("provider shell directory is inaccessible") + } + info, err := root.Stat(".") + if err != nil || !providerPrivateDirectory(dir, info) { + _ = root.Close() + return nil, errors.New("provider shell directory is not private") + } + return root, nil +} + +func providerShellFiles(root *os.Root) error { + script, err := root.Lstat("script") + if err != nil || !script.Mode().IsRegular() || !providerPrivateScript(script) { + return errors.New("provider shell script is not a private regular file") + } + if _, err := providerForcePresentRoot(root); err != nil { + return err + } + _, err = root.Lstat("ack") + if err == nil || !errors.Is(err, os.ErrNotExist) { + return errors.New("provider shell acknowledgement already exists or is inaccessible") + } + return nil +} + +func providerForcePresent(dir string) (bool, error) { + root, err := providerShellRoot(dir) + if err != nil { + return false, err + } + defer func() { _ = root.Close() }() + return providerForcePresentRoot(root) +} + +func providerForcePresentRoot(root *os.Root) (bool, error) { + info, err := root.Lstat("force") + if err == nil { + if info.Mode().IsRegular() { + return true, nil + } + return false, errors.New("provider shell force marker is not a regular file") + } + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return false, errors.New("provider shell force marker is inaccessible") +} + +func providerWriteAck(root *os.Root, token string) error { + file, err := root.OpenFile("ack.tmp", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return errors.New("create provider shell acknowledgement: " + errStringKind(err)) + } + defer func() { _ = root.Remove("ack.tmp") }() + if _, err := file.WriteString(token); err != nil { + _ = file.Close() + return errors.New("write provider shell acknowledgement: " + errStringKind(err)) + } + if err := file.Sync(); err != nil { + _ = file.Close() + return errors.New("sync provider shell acknowledgement: " + errStringKind(err)) + } + if err := file.Close(); err != nil { + return errors.New("close provider shell acknowledgement: " + errStringKind(err)) + } + if err := root.Rename("ack.tmp", "ack"); err != nil { + return errors.New("publish provider shell acknowledgement: " + errStringKind(err)) + } + return nil +} + +func providerAckToken(root *os.Root) (string, error) { + invalid := errors.New("invalid provider shell acknowledgement token") + info, err := root.Lstat("ack-key") + if err != nil || !providerPrivateAckKey(info) { + return "", invalid + } + token, ok := providerReadAckKey(root) + if !ok || !validProviderAckToken(token) { + return "", invalid + } + return token, nil +} + +func providerPrivateAckKey(info os.FileInfo) bool { + return info.Mode().IsRegular() && + (runtime.GOOS == providerWindowsOS || info.Mode().Perm() == 0o600) +} + +func providerReadAckKey(root *os.Root) (string, bool) { + file, err := root.Open("ack-key") + if err != nil { + return "", false + } + info, statErr := file.Stat() + content, readErr := io.ReadAll(io.LimitReader(file, 65)) + closeErr := file.Close() + removeErr := root.Remove("ack-key") + if statErr != nil || !providerPrivateAckKey(info) || + readErr != nil || closeErr != nil || removeErr != nil { + return "", false + } + return string(content), true +} + +func validProviderAckToken(token string) bool { + if len(token) != 64 { + return false + } + for i := range len(token) { + c := token[i] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +func errStringKind(err error) string { + if errors.Is(err, os.ErrExist) { + return "already exists" + } + if errors.Is(err, os.ErrPermission) { + return "permission denied" + } + return "file operation failed" +} + +type providerShellExecution struct { + dir string + token string + root *os.Root + result <-chan error + supervisor *providerProcessSupervisor + cancel context.CancelFunc + alive func() bool +} + +type ProviderShellInvocation struct { + ScriptPath string + ForcePath string + AckPath string + ParentPID int +} + +func RunProviderShell(ctx context.Context, invocation ProviderShellInvocation) error { + if ctx.Err() != nil { + return context.Cause(ctx) + } + if err := providerShellPaths( + invocation.ScriptPath, + invocation.ForcePath, + invocation.AckPath, + ); err != nil { + return err + } + dir := filepath.Dir(invocation.ScriptPath) + root, err := providerShellRoot(dir) + if err != nil { + return err + } + defer func() { _ = root.Close() }() + token, err := providerAckToken(root) + if err != nil { + return err + } + content, err := readProviderShellContent(root) + if err != nil { + return err + } + alive, cleanup, err := newProviderShellParentMonitor(invocation.ParentPID) + if err != nil { + return err + } + defer cleanup() + if !alive() { + return context.Canceled + } + executionCtx, cancel := context.WithCancel(ctx) + defer cancel() + supervisor := &providerProcessSupervisor{} + result := make(chan error, 1) + go func() { + result <- RunEmulatedShell(executionCtx, &CommandRunner{ + Command: string(content), Environ: os.Environ(), Stdin: os.Stdin, + Stdout: os.Stdout, Stderr: os.Stderr, ExecHandler: providerExecHandler(supervisor), + }) + }() + execution := providerShellExecution{ + dir: dir, + root: root, + token: token, + result: result, + supervisor: supervisor, + cancel: cancel, + alive: alive, + } + return execution.wait(ctx) +} + +func readProviderShellContent(root *os.Root) ([]byte, error) { + if err := providerShellFiles(root); err != nil { + return nil, err + } + content, err := root.ReadFile("script") + if err != nil { + return nil, errors.New("read provider shell script: " + errStringKind(err)) + } + return content, nil +} + +func (p *providerShellExecution) wait(ctx context.Context) error { + poll := time.NewTicker(100 * time.Millisecond) + defer poll.Stop() + for { + if !p.alive() { + p.supervisor.force(p.cancel) + return errors.New("provider shell parent exited") + } + forced, pollErr := providerForcePresent(p.dir) + if providerShellForceRequired(ctx, forced, pollErr) { + return p.force(pollErr) + } + select { + case err := <-p.result: + return p.completed(err) + case <-poll.C: + case <-ctx.Done(): + } + } +} + +func providerShellForceRequired(ctx context.Context, forced bool, pollErr error) bool { + return forced || pollErr != nil || ctx.Err() != nil +} + +func (p *providerShellExecution) ackForced() error { + p.supervisor.force(p.cancel) + if !p.alive() { + return context.Canceled + } + return providerWriteAck(p.root, p.token) +} + +func (p *providerShellExecution) force(pollErr error) error { + if pollErr != nil { + p.supervisor.force(p.cancel) + return pollErr + } + if err := p.ackForced(); err != nil { + return err + } + <-p.result + return context.Canceled +} + +func (p *providerShellExecution) completed(err error) error { + forced, pollErr := providerForcePresent(p.dir) + if pollErr != nil { + p.supervisor.force(p.cancel) + return pollErr + } + if forced { + return p.canceled() + } + p.supervisor.stop() + if !p.alive() { + return context.Canceled + } + ackErr := providerWriteAck(p.root, p.token) + earlyErr := providerShellEarlyError(err) + if ackErr != nil { + return errors.Join(ackErr, earlyErr) + } + if earlyErr != nil { + return earlyErr + } + return err +} + +func providerShellEarlyError(err error) error { + if err == nil { + return nil + } + if strings.HasPrefix(err.Error(), "parse shell command: ") { + return errors.New("parse provider shell script") + } + if strings.HasPrefix(err.Error(), "create shell runner: ") { + return err + } + if pathErr, ok := errors.AsType[*os.PathError](err); ok && pathErr.Op == "getwd" { + return err + } + return nil +} + +func (p *providerShellExecution) canceled() error { + if err := p.ackForced(); err != nil { + return err + } + return context.Canceled +} diff --git a/pkg/shell/provider_supervisor_test.go b/pkg/shell/provider_supervisor_test.go new file mode 100644 index 0000000000..2be0956c24 --- /dev/null +++ b/pkg/shell/provider_supervisor_test.go @@ -0,0 +1,569 @@ +package shell + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "mvdan.cc/sh/v3/interp" +) + +var providerTestAckToken = strings.Repeat("0123456789abcdef", 4) + +func TestProviderShellProcess(t *testing.T) { + if os.Getenv("DEVSY_PROVIDER_SHELL_PROCESS") != "1" { + return + } + err := RunProviderShell(context.Background(), providerShellTestInvocation( + os.Getenv("DEVSY_PROVIDER_SCRIPT"), os.Getenv("DEVSY_PROVIDER_FORCE"), + os.Getenv("DEVSY_PROVIDER_ACK"), + )) + if err == nil { + os.Exit(0) + } + if status, ok := errors.AsType[interp.ExitStatus](err); ok { + os.Exit(int(status)) + } + if errors.Is(err, context.Canceled) { + os.Exit(0) + } + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} + +func TestProviderShellRawOutput(t *testing.T) { + if os.Getenv("DEVSY_PROVIDER_SHELL_RAW") != "1" { + return + } + _, _ = os.Stdout.Write([]byte{0xff, 0, '\n'}) + _, _ = os.Stderr.Write([]byte{0, 0xfe}) + os.Exit(0) +} + +func TestProviderShellTrackedChild(t *testing.T) { + if os.Getenv("DEVSY_PROVIDER_SHELL_CHILD") != "1" { + return + } + if err := writeTrackedChildPID(os.Getenv("DEVSY_PROVIDER_CHILD_PID")); err != nil { + os.Exit(1) + } + _, _ = fmt.Fprintln(os.Stdout, "ready") + time.Sleep(time.Minute) + os.Exit(0) +} + +func TestProviderShellTokenChild(t *testing.T) { + if os.Getenv("DEVSY_PROVIDER_SHELL_TOKEN_CHILD") != "1" { + return + } + if !providerTokenChildClean() { + _, _ = fmt.Fprint(os.Stdout, "leaked") + os.Exit(1) + } + _, _ = fmt.Fprint(os.Stdout, "clean") + os.Exit(0) +} + +func providerTokenChildClean() bool { + if strings.Contains(strings.Join(os.Environ(), "\x00"), providerTestAckToken) || + !providerProcEnvironClean() || !providerAckKeyAbsent() { + return false + } + for _, arg := range os.Args { + if strings.Contains(arg, providerTestAckToken) { + return false + } + } + return true +} + +func providerProcEnvironClean() bool { + if runtime.GOOS != "linux" { + return true + } + root, err := os.OpenRoot("/proc") + if err != nil { + return false + } + defer func() { _ = root.Close() }() + for _, path := range []string{"self/environ", strconv.Itoa(os.Getppid()) + "/environ"} { + environ, err := root.ReadFile(path) + if err != nil || bytes.Contains(environ, []byte(providerTestAckToken)) { + return false + } + } + return true +} + +func providerAckKeyAbsent() bool { + root, err := os.OpenRoot(filepath.Dir(os.Getenv("DEVSY_PROVIDER_ACK"))) + if err != nil { + return false + } + defer func() { _ = root.Close() }() + _, err = root.Lstat("ack-key") + return errors.Is(err, os.ErrNotExist) +} + +func writeTrackedChildPID(path string) error { + root, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + return err + } + file, err := root.OpenFile("pid", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return errors.Join(err, root.Close()) + } + _, writeErr := file.WriteString(strconv.Itoa(os.Getpid())) + return errors.Join(writeErr, file.Close(), root.Close()) +} + +func providerShellFixture(t *testing.T, script string) (string, string, string) { + t.Helper() + dir := providerPrivateTestDir(t) + scriptPath := filepath.Join(dir, "script") + require.NoError(t, os.WriteFile(scriptPath, []byte(script), 0o600)) + return scriptPath, filepath.Join(dir, "force"), filepath.Join(dir, "ack") +} + +func providerTestWriteKey(t *testing.T, ackPath, token string) { + t.Helper() + require.NoError( + t, + os.WriteFile(filepath.Join(filepath.Dir(ackPath), "ack-key"), []byte(token), 0o600), + ) +} + +func providerShellTestInvocation(scriptPath, forcePath, ackPath string) ProviderShellInvocation { + return ProviderShellInvocation{ + ScriptPath: scriptPath, ForcePath: forcePath, AckPath: ackPath, ParentPID: os.Getppid(), + } +} + +func providerShellCommand(t *testing.T, scriptPath, forcePath, ackPath string) *exec.Cmd { + t.Helper() + providerTestWriteKey(t, ackPath, providerTestAckToken) + binary, err := os.Executable() + require.NoError(t, err) + cmd := &exec.Cmd{Path: binary, Args: []string{binary, "-test.run=^TestProviderShellProcess$"}} + cmd.Env = append(os.Environ(), "DEVSY_PROVIDER_SHELL_PROCESS=1", + "DEVSY_PROVIDER_SCRIPT="+scriptPath, "DEVSY_PROVIDER_FORCE="+forcePath, + "DEVSY_PROVIDER_ACK="+ackPath) + return cmd +} + +func runProviderShellWithToken(t *testing.T, scriptPath, forcePath, ackPath string) error { + t.Helper() + providerTestWriteKey(t, filepath.Join(filepath.Dir(scriptPath), "ack"), providerTestAckToken) + return RunProviderShell( + context.Background(), + providerShellTestInvocation(scriptPath, forcePath, ackPath), + ) +} + +func providerTestReadAck(ackPath string) ([]byte, error) { + root, err := os.OpenRoot(filepath.Dir(ackPath)) + if err != nil { + return nil, err + } + defer func() { _ = root.Close() }() + return root.ReadFile("ack") +} + +func TestRunProviderShellRejectsInvalidAckTokens(t *testing.T) { + for _, test := range []struct { + name string + token string + }{ + {name: "missing"}, + {name: "empty", token: ""}, + {name: "short", token: strings.Repeat("a", 63)}, + {name: "long", token: strings.Repeat("a", 65)}, + {name: "newline", token: providerTestAckToken + "\n"}, + {name: "uppercase", token: strings.Repeat("A", 64)}, + {name: "nonhex", token: strings.Repeat("g", 64)}, + } { + t.Run(test.name, func(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "printf should-not-run") + if test.name != "missing" { + providerTestWriteKey(t, ackPath, test.token) + } + err := RunProviderShell( + context.Background(), + providerShellTestInvocation(scriptPath, forcePath, ackPath), + ) + require.ErrorContains(t, err, "invalid provider shell acknowledgement token") + _, err = os.Lstat(ackPath) + require.ErrorIs(t, err, os.ErrNotExist) + }) + } +} + +func TestRunProviderShellRejectsUnsafeAckKey(t *testing.T) { + t.Run("directory", func(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "printf should-not-run") + require.NoError(t, os.Mkdir(filepath.Join(filepath.Dir(ackPath), "ack-key"), 0o700)) + require.ErrorContains( + t, + RunProviderShell( + context.Background(), + providerShellTestInvocation(scriptPath, forcePath, ackPath), + ), + "invalid provider shell acknowledgement token", + ) + _, err := os.Lstat(ackPath) + require.ErrorIs(t, err, os.ErrNotExist) + }) + if runtime.GOOS == providerWindowsOS { + return + } + t.Run("symlink", func(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "printf should-not-run") + require.NoError(t, os.Symlink(scriptPath, filepath.Join(filepath.Dir(ackPath), "ack-key"))) + require.ErrorContains( + t, + RunProviderShell( + context.Background(), + providerShellTestInvocation(scriptPath, forcePath, ackPath), + ), + "invalid provider shell acknowledgement token", + ) + _, err := os.Lstat(ackPath) + require.ErrorIs(t, err, os.ErrNotExist) + }) + t.Run("permissions", func(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "printf should-not-run") + providerTestWriteKey(t, ackPath, providerTestAckToken) + require.NoError(t, os.Chmod(filepath.Join(filepath.Dir(ackPath), "ack-key"), 0o400)) + require.ErrorContains( + t, + RunProviderShell( + context.Background(), + providerShellTestInvocation(scriptPath, forcePath, ackPath), + ), + "invalid provider shell acknowledgement token", + ) + _, err := os.Lstat(ackPath) + require.ErrorIs(t, err, os.ErrNotExist) + }) +} + +func TestRunProviderShellDoesNotLeakAckToken(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + command := strconv.Quote(binary) + " -test.run=^TestProviderShellTokenChild$" + if runtime.GOOS == providerWindowsOS { + command = strings.ReplaceAll(command, `\`, `/`) + } + scriptPath, forcePath, ackPath := providerShellFixture(t, command) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + cmd.Env = append(cmd.Env, "DEVSY_PROVIDER_SHELL_TOKEN_CHILD=1") + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + require.NoError(t, cmd.Run()) + require.Equal(t, "clean", stdout.String()) + require.NotContains(t, stderr.String(), providerTestAckToken) + ack, err := providerTestReadAck(ackPath) + require.NoError(t, err) + require.Equal(t, providerTestAckToken, string(ack)) +} + +func TestRunProviderShellRejectsForgedAck(t *testing.T) { + for _, forged := range []string{"", strings.Repeat("f", 64)} { + t.Run(strconv.Itoa(len(forged)), func(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "exit 0") + require.NoError(t, os.WriteFile(ackPath, []byte(forged), 0o600)) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + output, err := cmd.CombinedOutput() + require.Error(t, err) + require.NotContains(t, string(output), providerTestAckToken) + ack, err := providerTestReadAck(ackPath) + require.NoError(t, err) + require.Equal(t, forged, string(ack)) + }) + } +} + +func TestRunProviderShellNormalExitStatuses(t *testing.T) { + for _, test := range []struct { + script string + status int + }{ + {"printf normal", 0}, + {"exit 7", 7}, + {"exit 125", 125}, + {"command-that-does-not-exist-846307", 127}, + } { + t.Run(strconv.Itoa(test.status), func(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, test.script) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + output, err := cmd.CombinedOutput() + if test.status == 0 { + require.NoError(t, err) + require.Equal(t, "normal", string(output)) + } else { + exit, ok := errors.AsType[*exec.ExitError](err) + require.True(t, ok) + require.Equal(t, test.status, exit.ExitCode()) + } + ack, err := providerTestReadAck(ackPath) + require.NoError(t, err) + require.Equal(t, providerTestAckToken, string(ack)) + }) + } +} + +func TestRunProviderShellNormalCompletionReapsBackgroundChild(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + command := strconv.Quote(binary) + " -test.run=^TestProviderShellTrackedChild$" + if runtime.GOOS == providerWindowsOS { + command = strings.ReplaceAll(command, `\`, `/`) + } + scriptPath, forcePath, ackPath := providerShellFixture(t, "") + pidPath := filepath.Join(filepath.Dir(scriptPath), "pid") + script := "export DEVSY_PROVIDER_SHELL_CHILD=1 DEVSY_PROVIDER_CHILD_PID=" + + strconv.Quote(pidPath) + "; " + command + + ` & while [ ! -s "$DEVSY_PROVIDER_CHILD_PID" ]; do :; done; printf finished` + require.NoError(t, os.WriteFile(scriptPath, []byte(script), 0o600)) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + cleanupProviderShellChild(pidPath) + }) + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case err := <-done: + require.NoError(t, err, stderr.String()) + case <-time.After(5 * time.Second): + t.Fatal("normal provider completion waited for a background child") + } + require.Contains(t, stdout.String(), "finished") + ack, err := providerTestReadAck(ackPath) + require.NoError(t, err) + require.Equal(t, providerTestAckToken, string(ack)) + root, err := os.OpenRoot(filepath.Dir(pidPath)) + require.NoError(t, err) + pidBytes, err := root.ReadFile("pid") + require.NoError(t, errors.Join(err, root.Close())) + pid, err := strconv.Atoi(string(pidBytes)) + require.NoError(t, err) + require.NoError(t, providerChildReaped(pid)) +} + +func cleanupProviderShellChild(path string) { + root, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + return + } + pidBytes, readErr := root.ReadFile("pid") + _ = root.Close() + if readErr != nil { + return + } + pid, err := strconv.Atoi(string(pidBytes)) + if err != nil { + return + } + process, err := os.FindProcess(pid) + if err == nil { + _ = process.Kill() + } +} + +func TestRunProviderShellRawStreams(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + command := strconv.Quote(binary) + " -test.run=^TestProviderShellRawOutput$" + if runtime.GOOS == providerWindowsOS { + command = strings.ReplaceAll(command, `\`, `/`) + } + scriptPath, forcePath, ackPath := providerShellFixture(t, command) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + cmd.Env = append(cmd.Env, "DEVSY_PROVIDER_SHELL_RAW=1") + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + require.NoError(t, cmd.Run()) + require.Equal(t, []byte{0xff, 0, '\n'}, stdout.Bytes()) + require.Equal(t, []byte{0, 0xfe}, stderr.Bytes()) +} + +func TestRunProviderShellForceAcknowledgesReapedChild(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + command := strconv.Quote(binary) + " -test.run=^TestProviderShellTrackedChild$" + if runtime.GOOS == providerWindowsOS { + command = strings.ReplaceAll(command, `\`, `/`) + } + scriptPath, forcePath, ackPath := providerShellFixture(t, command+"; printf finished") + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + pidPath := filepath.Join(filepath.Dir(scriptPath), "pid") + cmd.Env = append(cmd.Env, "DEVSY_PROVIDER_SHELL_CHILD=1", "DEVSY_PROVIDER_CHILD_PID="+pidPath) + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + var stderr bytes.Buffer + cmd.Stderr = &stderr + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + ready := make(chan string, 1) + go func() { + line, _ := bufio.NewReader(stdout).ReadString('\n') + ready <- line + }() + select { + case line := <-ready: + require.Equal(t, "ready\n", line) + case <-time.After(5 * time.Second): + t.Fatal("provider child did not start") + } + require.NoError(t, os.WriteFile(forcePath, nil, 0o600)) + require.Eventually(t, func() bool { + content, err := providerTestReadAck(ackPath) + return err == nil && string(content) == providerTestAckToken + }, 5*time.Second, 20*time.Millisecond) + root, err := os.OpenRoot(filepath.Dir(scriptPath)) + require.NoError(t, err) + defer func() { _ = root.Close() }() + pidBytes, err := root.ReadFile("pid") + require.NoError(t, err) + pid, err := strconv.Atoi(string(pidBytes)) + require.NoError(t, err) + require.NoError(t, providerChildReaped(pid)) + require.NoError(t, cmd.Wait()) + require.NotContains(t, stderr.String(), command) +} + +func TestRunProviderShellBlockedBuiltinAcknowledges(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture( + t, + "printf 'ready\\n'; mapfile lines; printf finished", + ) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + stdinRead, stdinWrite, err := os.Pipe() + require.NoError(t, err) + defer func() { _ = stdinRead.Close() }() + defer func() { _ = stdinWrite.Close() }() + cmd.Stdin = stdinRead + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + var stderr bytes.Buffer + cmd.Stderr = &stderr + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + ready := make(chan string, 1) + go func() { + line, _ := bufio.NewReader(stdout).ReadString('\n') + ready <- line + }() + select { + case line := <-ready: + require.Equal(t, "ready\n", line) + case <-time.After(5 * time.Second): + t.Fatal("provider shell did not start") + } + require.NoError(t, os.WriteFile(forcePath, nil, 0o600)) + require.Eventually(t, func() bool { + content, err := providerTestReadAck(ackPath) + return err == nil && string(content) == providerTestAckToken + }, 5*time.Second, 20*time.Millisecond) + require.NoError(t, cmd.Process.Kill()) + _ = cmd.Wait() + require.NotContains(t, stderr.String(), "finished") +} + +func TestProviderProcessSupervisorStartForceRace(t *testing.T) { + binary, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + supervisor := &providerProcessSupervisor{} + ready := make(chan struct{}) + var readyOnce sync.Once + var workers sync.WaitGroup + for range 8 { + workers.Go(func() { + cmd, startErr := supervisor.start(ctx, func() *exec.Cmd { + readyOnce.Do(func() { close(ready) }) + child := &exec.Cmd{ + Path: binary, + Args: []string{binary, "-test.run=^TestProviderExecHelper$"}, + } + child.Env = append( + os.Environ(), + "DEVSY_PROVIDER_EXEC_HELPER=1", + "DEVSY_PROVIDER_EXEC_EXIT=wait", + ) + child.Stdout = io.Discard + return child + }) + if startErr == nil { + _ = cmd.Wait() + supervisor.reaped(cmd) + } + }) + } + <-ready + supervisor.force(cancel) + workers.Wait() + supervisor.mu.Lock() + defer supervisor.mu.Unlock() + require.True(t, supervisor.forced) + require.Empty(t, supervisor.children) +} + +func TestRunProviderShellPreexistingForceIsAcknowledged(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "exit 7") + require.NoError(t, os.WriteFile(forcePath, nil, 0o600)) + cmd := providerShellCommand(t, scriptPath, forcePath, ackPath) + output, err := cmd.CombinedOutput() + require.NoErrorf(t, err, "provider shell: %s", string(output)) + info, err := os.Lstat(ackPath) + require.NoError(t, err) + require.True(t, info.Mode().IsRegular()) + ack, err := providerTestReadAck(ackPath) + require.NoError(t, err) + require.Equal(t, providerTestAckToken, string(ack)) +} + +func TestRunProviderShellRejectsUnsafePaths(t *testing.T) { + scriptPath, forcePath, ackPath := providerShellFixture(t, "exit 0") + require.Error(t, runProviderShellWithToken(t, scriptPath, forcePath, scriptPath)) + require.Error( + t, + runProviderShellWithToken( + t, + scriptPath, + forcePath, + filepath.Join(t.TempDir(), "ack"), + ), + ) + if runtime.GOOS == providerWindowsOS { + require.NoError(t, os.Mkdir(ackPath, 0o700)) + } else { + require.NoError(t, os.Symlink(scriptPath, ackPath)) + } + require.Error(t, runProviderShellWithToken(t, scriptPath, forcePath, ackPath)) +} diff --git a/pkg/shell/provider_supervisor_unix_test.go b/pkg/shell/provider_supervisor_unix_test.go new file mode 100644 index 0000000000..0aa46431d7 --- /dev/null +++ b/pkg/shell/provider_supervisor_unix_test.go @@ -0,0 +1,41 @@ +//go:build !windows + +package shell + +import ( + "errors" + "fmt" + "os" + "syscall" + "testing" + + "github.com/stretchr/testify/require" +) + +func providerPrivateTestDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp(t.TempDir(), "private-") + require.NoError(t, err) + return dir +} + +func TestProviderPrivateTestDirMode(t *testing.T) { + dir := providerPrivateTestDir(t) + root, err := os.OpenRoot(dir) + require.NoError(t, err) + defer func() { _ = root.Close() }() + info, err := root.Stat(".") + require.NoError(t, err) + require.Equal(t, os.FileMode(0o700), info.Mode().Perm()) +} + +func providerChildReaped(pid int) error { + err := syscall.Kill(pid, 0) + if errors.Is(err, syscall.ESRCH) { + return nil + } + if err == nil { + return errors.New("owned child still exists after acknowledgement") + } + return fmt.Errorf("cannot check owned child after acknowledgement: %w", err) +} diff --git a/pkg/shell/provider_supervisor_windows_test.go b/pkg/shell/provider_supervisor_windows_test.go new file mode 100644 index 0000000000..da9277bbdd --- /dev/null +++ b/pkg/shell/provider_supervisor_windows_test.go @@ -0,0 +1,87 @@ +package shell + +import ( + "errors" + "os" + "path/filepath" + "testing" + "unsafe" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" +) + +func providerPrivateTestDir(t *testing.T) string { + t.Helper() + user, err := windows.GetCurrentProcessToken().GetTokenUser() + require.NoError(t, err) + sd, err := windows.SecurityDescriptorFromString( + "D:P(A;OICI;GA;;;" + user.User.Sid.String() + ")(A;OICI;GA;;;SY)") + require.NoError(t, err) + attributes := windows.SecurityAttributes{ + Length: uint32(unsafe.Sizeof(windows.SecurityAttributes{})), + SecurityDescriptor: sd, + } + path := filepath.Join(t.TempDir(), "private") + name, err := windows.UTF16PtrFromString(path) + require.NoError(t, err) + require.NoError(t, windows.CreateDirectory(name, &attributes)) + return path +} + +func providerChildReaped(pid int) error { + process, err := os.FindProcess(pid) + if err != nil { + return nil + } + if err := process.Kill(); err == nil { + return errors.New("owned child still exists after acknowledgement") + } + return nil +} + +func TestProviderPrivateDACL(t *testing.T) { + user, err := windows.GetCurrentProcessToken().GetTokenUser() + require.NoError(t, err) + sid := user.User.Sid.String() + cases := []struct { + name string + sddl string + want bool + }{ + {"creation", "D:P(A;OICI;GA;;;" + sid + ")(A;OICI;GA;;;SY)", true}, + { + "normalized", + "D:P(A;;FA;;;" + sid + ")(A;OICIIO;GA;;;" + sid + ")(A;;FA;;;SY)(A;OICIIO;GA;;;SY)", + true, + }, + {"unprotected", "D:(A;OICI;GA;;;" + sid + ")(A;OICI;GA;;;SY)", false}, + {"world", "D:P(A;OICI;GA;;;" + sid + ")(A;OICI;GA;;;SY)(A;OICI;GR;;;WD)", false}, + {"missing descendants", "D:P(A;;FA;;;" + sid + ")(A;OICI;GA;;;SY)", false}, + {"insufficient", "D:P(A;OICI;GR;;;" + sid + ")(A;OICI;GA;;;SY)", false}, + {"no propagation", "D:P(A;OICINP;GA;;;" + sid + ")(A;OICI;GA;;;SY)", false}, + {"deny", "D:P(D;;GA;;;WD)(A;OICI;GA;;;" + sid + ")(A;OICI;GA;;;SY)", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + sd, err := windows.SecurityDescriptorFromString(tc.sddl) + require.NoError(t, err) + require.Equal(t, tc.want, providerPrivateDACL(sd, user.User.Sid)) + }) + } +} + +func TestProviderPrivateDACLWhenCurrentUserIsSystem(t *testing.T) { + system, err := windows.CreateWellKnownSid(windows.WinLocalSystemSid) + require.NoError(t, err) + sd, err := windows.SecurityDescriptorFromString("D:P(A;OICI;GA;;;SY)(A;OICI;GA;;;SY)") + require.NoError(t, err) + require.True(t, providerPrivateDACL(sd, system)) +} + +func TestProviderPrivateDirectoryCreated(t *testing.T) { + path := providerPrivateTestDir(t) + info, err := os.Stat(path) + require.NoError(t, err) + require.True(t, providerPrivateDirectory(path, info)) +} diff --git a/pkg/shell/shell.go b/pkg/shell/shell.go index 6be8172baa..f315f872ce 100644 --- a/pkg/shell/shell.go +++ b/pkg/shell/shell.go @@ -19,11 +19,12 @@ import ( ) type CommandRunner struct { - Command string - Environ []string - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer + Command string + Environ []string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + ExecHandler interp.ExecHandlerFunc } func RunEmulatedShell( @@ -47,11 +48,12 @@ func RunEmulatedShell( } r, err := interp.New(buildRunnerOptions(shellRunnerOptions{ - stdin: runner.Stdin, - stdout: runner.Stdout, - stderr: runner.Stderr, - env: runner.Environ, - dir: dir, + stdin: runner.Stdin, + stdout: runner.Stdout, + stderr: runner.Stderr, + env: runner.Environ, + dir: dir, + execHandler: runner.ExecHandler, })...) if err != nil { return fmt.Errorf("create shell runner: %w", err) @@ -71,23 +73,27 @@ func RunEmulatedShell( } type shellRunnerOptions struct { - stdin io.Reader - stdout io.Writer - stderr io.Writer - env []string - dir string + stdin io.Reader + stdout io.Writer + stderr io.Writer + env []string + dir string + execHandler interp.ExecHandlerFunc } func buildRunnerOptions(p shellRunnerOptions) []interp.RunnerOption { defaultOpenHandler := interp.DefaultOpenHandler() - defaultExecHandler := interp.DefaultExecHandler(2 * time.Second) + execHandler := p.execHandler + if execHandler == nil { + execHandler = interp.DefaultExecHandler(2 * time.Second) + } return []interp.RunnerOption{ interp.StdIO(p.stdin, p.stdout, p.stderr), interp.Env(expand.ListEnviron(p.env...)), interp.Dir(p.dir), interp.ExecHandlers(func(next interp.ExecHandlerFunc) interp.ExecHandlerFunc { return func(ctx context.Context, args []string) error { - return defaultExecHandler(ctx, args) + return execHandler(ctx, args) } }), // CallHandler intercepts builtin calls. mvdan.cc/sh/v3 v3.13.1 marks diff --git a/providers/docker/provider.yaml b/providers/docker/provider.yaml index 4e1636b4c5..c4267defa9 100644 --- a/providers/docker/provider.yaml +++ b/providers/docker/provider.yaml @@ -38,7 +38,7 @@ agent: path: ${DOCKER_PATH} elevation: ${DOCKER_ELEVATION} builder: ${DOCKER_BUILDER} - install: false + install: true env: DOCKER_HOST: ${DOCKER_HOST} DOCKER_CONTEXT: ${DOCKER_CONTEXT} diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx index 10b4d2de79..26976cbb19 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/agent.mdx @@ -38,6 +38,24 @@ This section can use option values. - **exec.shutdown**: the command that stops the machine once it is idle. Option values are available in it. - **binaries**: helper binaries for `exec.shutdown`. Same format as [provider binaries](./binaries.mdx). +## Docker management policy + +For the `docker` driver, the `agent.docker` section controls how Devsy manages Docker on the agent machine: + +```yaml +agent: + local: false + driver: docker + docker: + path: docker + install: false + configureDaemon: false +``` + +Set `install: false` when the provider supplies a Docker CLI. If that executable is missing, Devsy reports an error instead of installing Docker. An existing executable remains usable. With `install` unset or `true`, Devsy retains its existing installation behavior. + +Set `configureDaemon: false` when the provider owns the Docker daemon, including a private daemon inside a VM. Devsy then does not read or modify the daemon configuration for its management step and does not reload the daemon. With `configureDaemon` unset or `true`, Devsy retains its existing behavior of configuring the daemon for a remote Docker agent. Local agents are not configured by this step. These settings accept provider option templates, which must resolve to valid boolean values before the agent starts Docker setup. This policy controls Devsy's daemon management, not arbitrary commands a user runs inside a devcontainer. + ## Stopping idle environments Devsy tracks connections and stops the environment after the timeout. It starts the environment again when the user returns. State survives only if the provider keeps the storage. A shutdown command that deletes the machine must keep workspace data elsewhere, for example on a persistent volume. diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx index 8c88b71588..cd0ae69ec1 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/quickstart.mdx @@ -42,7 +42,9 @@ devsy workspace up github.com/microsoft/vscode-remote-try-go ## exec -Each entry is a POSIX shell script. Only `command` is required. +The `command`, `init`, `create`, `delete`, `start`, `stop`, `status` and `describe` entries are POSIX shell scripts or command arrays. For standard providers, only `command` is required. Proxy and daemon providers use their own exec fields instead. `cancellation` is an execution policy, not a script. + +An array with one element is parsed as a shell script, not as an executable path. For example, `["/opt/provider tool"]` runs `/opt/provider` with the argument `tool`. Quote the path inside that script or provide a multi-element array to run a path that contains spaces. - **command**: runs a command in the environment. Devsy uses it to inject itself, and sends all traffic through its standard input and output. Devsy fills in the `${COMMAND}` placeholder. - **init**: checks that the options are valid and the provider is ready. Devsy runs it when options change. @@ -50,8 +52,15 @@ Each entry is a POSIX shell script. Only `command` is required. - **start**, **stop**: start and stop a machine. Machine providers only. - **status**: prints one of `Running`, `Busy` (wait and check again), `Stopped` or `NotFound`. - **describe**: prints a text description of the machine. +- **cancellation**: omit it for existing process cancellation behavior, or set `file` to give provider commands a cooperative cleanup window. + +A standard provider without `create` is a non-machine provider and uses only `command` and `init`. The cancellation policy applies to both machine and non-machine providers. + +With `exec.cancellation: file`, Devsy sets the reserved `DEVSY_PROVIDER_CANCEL_FILE` environment variable to a private, initially nonexistent marker path for each invocation. The provider must watch for the marker, stop admitting work, cancel its owned processes, finish cleanup, and exit. On caller cancellation, Devsy creates the marker and allows up to 10 seconds for cleanup. For single-string shell commands that remain blocked, Devsy then asks its supervised shell to stop and reap its direct children before terminating the shell; it waits up to another 5 seconds for an authenticated acknowledgement before killing an unresponsive helper. A provider-writable acknowledgement file does not establish that cleanup finished: a missing or forged acknowledgement, including when the helper exits abruptly during the cleanup window, is reported as an unclean shutdown alongside the caller's cancellation error. Devsy never kills the provider's process group or unrelated daemon processes. It waits for the invocation to finish and still returns the caller's cancellation error even if cleanup exits successfully. A provider must not set or override the reserved variable. + +For shell commands, the supervisor also stops and reaps direct children still running when the script completes normally. A direct background child started with `&` cannot outlive its invocation. A detached descendant can outlive it, so the provider must manage those processes explicitly. Start durable services through the backend's explicit detached operation, not a shell background job. -A provider without `create` is a non-machine provider and uses only `command` and `init`. +File mode accepts finite or nonblocking standard streams, direct file input handles, and blocking streams whose owner unblocks them when the original calling context is cancelled. A direct `*os.File` output handle is not exempt: Devsy can wrap it for redaction, and a blocked output write can prevent completion after the child exits. Callers must drain or unblock blocking stdout and stderr, including file-backed pipes, on cancellation. Devsy does not close caller-owned input or global standard input, and process cancellation cannot interrupt an arbitrary blocked reader or writer. Machine command transports close their pipe endpoints on cancellation of the original context. Standard input can close independently of standard output, so providers must continue draining output after input reaches EOF. On Windows, Devsy runs these scripts in an [emulated shell](https://github.com/mvdan/sh). The emulated shell has no `grep` or `sed`. Move that logic into a helper binary. diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx new file mode 100644 index 0000000000..d8f16855a6 --- /dev/null +++ b/sites/docs-devsy-sh/content/docs/managing-providers/docker-sandbox.mdx @@ -0,0 +1,66 @@ +--- +title: Docker Sandboxes +sidebar_label: Docker Sandboxes +--- + +The planned Docker Sandboxes provider runs each Devsy machine in a local `sbx` microVM. The Devsy agent uses the private Docker Engine inside that VM to build and run ordinary devcontainers. A host Docker Engine is not required. + +No provider release exists yet. The installation and machine instructions below apply only after native qualification and publication. The short name cannot be installed before a release. + +## Requirements + +Install the [standalone `sbx` CLI](https://docs.docker.com/ai/sandboxes/install/) and sign in with `sbx login` outside Devsy. Keep the version listed in the provider release compatibility profile. The provider rejects versions and host systems that have not passed its native qualification. + +Local Docker Sandboxes require Apple silicon with macOS 14 or later, Windows 11 on amd64 with Windows Hypervisor Platform, or Ubuntu 24.04 or later on amd64 or arm64 with accessible KVM. A VM running the provider also needs nested virtualization. A host system is supported only if the provider release lists that system as qualified. Docker Desktop and a host Docker Engine are not required. + +After a qualified provider release is published, install it with: + +```sh +devsy provider add docker-sandbox +devsy workspace up https://github.com/devcontainers/samples --provider docker-sandbox +``` + +The Devsy Desktop setup wizard does not list this provider before a qualified release. The provider does not install `sbx`, sign in, open a login browser, or change Docker Sandboxes settings. + +## Machine resources + +Set machine options when creating the machine. `SBX_CPUS` sets virtual CPUs, `SBX_MEMORY` sets memory, `SBX_ROOT_DISK` sets space for the guest system and sources, and `SBX_DOCKER_DISK` sets space for Docker images, containers, and volumes. Sizes use integer binary `m` or `g` units. The defaults are two CPUs, `4g` of memory, and `40g` for each disk. `SBX_PATH` can select a different `sbx` executable. The provider validates these values before creating a VM. + +Resources cannot be resized in place. Create another machine and deliberately move the data you need before deleting the original. Changing a saved machine option does not destroy and recreate its VM. + +## Sources and development containers + +Git sources are cloned into the guest. Local folder sources are uploaded to the guest, not mounted live from the workstation. A local folder rebuild uploads host files again and can overwrite matching guest edits. Push work to Git or transfer it out of the VM if you need it after machine deletion. + +The existing Devsy Docker driver handles Dockerfiles, Features, Compose, image sources, lifecycle commands, named volumes, and ordinary guest Docker mounts. An absolute bind source refers to a path in the guest VM, not a path on your workstation. Devsy runs `initializeCommand` in the VM. Your devcontainer chooses its own user and shell. + +Devsy's SSH tunnel connects to the final devcontainer for terminal, editor, and application forwarding. A port published by the guest Docker Engine is not automatically a workstation port. Use Devsy port forwarding to reach an application from your host. + +## Network and credentials + +Docker Sandboxes network policy still applies to the VM and its containers. The provider does not open blocked destinations or disable TLS verification. Container builds and applications may need project specific proxy variables and CA installation. A project using a forward proxy should add the proxy's public CA to its own build context outside version controlled source, install it in the image trust store before HTTPS commands, and pass the proxy build arguments and runtime variables deliberately. Rebuild after CA rotation. Other base images and applications can use different trust stores. + +The guest Docker Engine does not share the host Docker Engine or its socket. Devsy credential forwarding and Docker Sandboxes host integrations are separate authority paths. Check your Devsy context and Docker Sandboxes settings for Git and registry credentials, SSH agent forwarding, MCP integrations, and network access. A mountless VM does not disable those integrations. Private image pulls from the guest Engine and sandbox template pulls use separate credential paths. + +## Stop, delete, and recovery + +Stopping a machine preserves its guest sources, images, containers, and volumes. Starting resumes the same VM. Rebuilding a workspace changes the inner devcontainer without replacing the VM. Deleting a default workspace machine removes the VM and all its guest data, including named volumes. Deleting a workspace on an explicitly shared machine leaves that machine and its other workspaces intact. Uninstalling the provider does not delete its VMs. + +With `--single-machine`, deleting its last workspace deletes the shared VM and all guest data. Back up anything you need before deleting that workspace. + +Provider state is stored under your user configuration directory at `devsy-provider-docker-sandbox/state-v1`. If creation fails, do not remove that directory or manually reuse a sandbox name. The provider reports the exact machine key and any unresolved child attempt or backend operation. A lost VM is never silently recreated in the same machine generation. + +A pending context deletion blocks normal commands but does not stop a provider cleanup helper that Devsy already started. Let that helper finish its scoped cleanup before retrying deletion. + +After installation, run `devsy provider list --result-format=json`. If you installed with `devsy --home `, use `devsy --home provider list --result-format=json` and treat that path as Devsy's data directory. In `config.binaries.DOCKER_SANDBOX_PROVIDER`, select the record whose `os` and `arch` match your host. An absolute `path` is the executable. For a remote archive, use `archivePath` under the data directory at `contexts//providers//binaries/docker_sandbox_provider/`. Otherwise, use the record's `name` in that directory. If `name` is empty, use the base name of `path` instead. Without a `--home` override, `DEVSY_HOME` names the data directory when set. Otherwise Devsy uses `~/.devsy` on macOS and Linux, or `%LOCALAPPDATA%\devsy` on Windows. Use the resolved local executable, not the asset URL. + +The provider executable installed with the provider accepts: + +```text + doctor --machine-key --json + recover --machine-key --attempt --acknowledge-client-terminated + recover --machine-key --operation --acknowledge-backend-settled + delete --machine-key +``` + +Use `doctor` to inspect the saved identity and backend observation without starting a stopped VM. An attempt acknowledgement is appropriate only after its orphaned client has terminated or the host has restarted. A backend acknowledgement is appropriate only after the Docker side operation has settled. Neither acknowledgement starts, stops, or removes a VM. `devsy provider delete` removes the installed executable. Reinstall the same compatible release or preserve its executable before removal if you need scoped recovery afterward. Use scoped deletion only for a machine you intend to destroy. Never use global `sbx` reset or prune to recover one machine. diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx b/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx index 98f8093909..4ad599c08f 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx +++ b/sites/docs-devsy-sh/content/docs/managing-providers/manage-providers.mdx @@ -19,6 +19,8 @@ Providers you can add: - [Azure (azure)](https://github.com/devsy-org/devsy-provider-azure) - [Digital Ocean (digitalocean)](https://github.com/devsy-org/devsy-provider-digitalocean) +Docker Sandboxes is not available yet. See its [qualification guide](./docker-sandbox.mdx) for the requirements of a future release. + List the available providers: ```sh @@ -103,6 +105,16 @@ devsy provider add builtin:microsandbox --name legacy-microsandbox --use=false If a switch fails during initialization, the new source stays selected. Fix the cause and run `devsy provider init `, or set the previous source again. Add `--version ` to `set-source` to pin a release. +### Docker Sandboxes + +A published release qualified for your host is required. After it exists, install the standalone `sbx` CLI, sign in with `sbx login`, and add the provider: + +```sh +devsy provider add docker-sandbox +``` + +The [Docker Sandboxes guide](./docker-sandbox.mdx) covers qualified hosts, resource options, guest Docker, source files, networking, data deletion, and scoped recovery. + ## Set provider options Each provider has its own options. In the desktop app, open **Providers** and click **Edit** on a provider. With the CLI, set options when you add or initialize: diff --git a/sites/docs-devsy-sh/content/docs/managing-providers/meta.json b/sites/docs-devsy-sh/content/docs/managing-providers/meta.json index e26c174546..991084c9c7 100644 --- a/sites/docs-devsy-sh/content/docs/managing-providers/meta.json +++ b/sites/docs-devsy-sh/content/docs/managing-providers/meta.json @@ -1,4 +1,4 @@ { "title": "Managing your Providers", - "pages": ["what-are-providers", "manage-providers", "kubernetes-registry-access"] + "pages": ["what-are-providers", "manage-providers", "docker-sandbox", "kubernetes-registry-access"] }