From f51de4ab0fab3aa02d7d1cfc349c5233a7b93ff2 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 11 Oct 2026 12:35:14 -0600 Subject: [PATCH 1/2] test(runtime): preserve workspaces after lost create replies --- .../lost_reply_resolution_test.go | 373 +++++++++++++++ .../external/internal/testfixture/main.go | 205 +++++++- pkg/driver/external/lost_reply_test.go | 437 ++++++++++++++++++ 3 files changed, 1006 insertions(+), 9 deletions(-) create mode 100644 pkg/devcontainer/lost_reply_resolution_test.go create mode 100644 pkg/driver/external/lost_reply_test.go diff --git a/pkg/devcontainer/lost_reply_resolution_test.go b/pkg/devcontainer/lost_reply_resolution_test.go new file mode 100644 index 0000000000..3225428a39 --- /dev/null +++ b/pkg/devcontainer/lost_reply_resolution_test.go @@ -0,0 +1,373 @@ +package devcontainer + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/devsy-org/devsy-runtime-sdk/runtimev1" + "github.com/devsy-org/devsy-runtime-sdk/supervisor" + "github.com/devsy-org/devsy/pkg/agent" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/driver" + "github.com/devsy-org/devsy/pkg/encoding" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/shirou/gopsutil/v4/process" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// This matches the SDK delegate of cmd/internal/runtime_supervisor.go. Importing +// cmd here would cycle through devcontainer; Layer A separately covers Cobra. +func TestMain(m *testing.M) { + if len(os.Args) > 2 && + os.Args[1] == resolutionInternalCommand && + os.Args[2] == "runtime-supervisor" { + supervisor.Main(os.Args[3:]) + return + } + os.Exit(m.Run()) +} + +const ( + resolutionInternalCommand = "internal" + resolutionDisabledSetting = "none" + resolutionReplyNonceLabel = "fixture.lost-reply-nonce" + resolutionWorkspaceMount = "/workspaces/lost-reply" +) + +type resolutionReplyFixture struct { + info *provider.AgentWorkspaceInfo + state, record, sentinel, nonce string + recordBytes []byte +} + +type resolutionPersistedReply struct { + Workspace, Nonce, Mode string + Container *runtimev1.ContainerDetails +} + +type resolutionReplyOperation struct { + Operation, Workspace, Nonce string +} + +type resolutionReplyProcess struct { + PID int32 + Started int64 + Kind string +} + +func TestResolveContainerReusesPersistedCreateAfterLostReply(t *testing.T) { + f := newResolutionReplyFixture(t) + marker := f.createWithLostReply(t) + f.requireProcessesGone(t) + persisted, err := agent.ParseAgentWorkspaceInfo(f.record) + require.NoError(t, err) + require.Equal(t, f.info.Workspace.UID, persisted.Workspace.UID) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + publicRunner, err := NewRunner(ctx, "", "", persisted) + require.NoError(t, err) + fresh, ok := publicRunner.(*runner) + require.True(t, ok) + found, err := fresh.findExistingDevContainer(ctx) + require.NoError(t, err) + require.NotNil(t, found) + f.assertAcknowledgedContainer(t, marker, found) + params := unchangedLostReplyResolveParams() + resolved, err := fresh.resolveContainer(ctx, params, found) + require.NoError(t, err) + require.NotNil(t, resolved) + require.NotNil(t, resolved.details) + require.NotNil(t, resolved.mergedConfig) + require.False(t, params.options.Recreate) + assert.Equal(t, resolutionWorkspaceMount, params.substitutionContext.ContainerWorkspaceFolder, + "existing-container branch must adopt its actual persisted bind destination") + assert.Equal(t, containerRootUser, resolved.mergedConfig.RemoteUser) + assert.Equal(t, found, resolved.details) + again, err := fresh.findExistingDevContainer(ctx) + require.NoError(t, err) + assert.Equal(t, found, again) + f.assertOperationCounts(t) + data, err := os.ReadFile(f.record) + require.NoError(t, err) + assert.Equal(t, f.recordBytes, data) + data, err = os.ReadFile(f.sentinel) + require.NoError(t, err) + assert.Equal(t, f.nonce, string(data)) + f.requireProcessesGone(t) +} + +func unchangedLostReplyResolveParams() *resolveParams { + current := &config.DevContainerConfig{} + current.Image = "fixture" + current.RemoteUser = containerRootUser + current.ContainerUser = containerRootUser + return &resolveParams{ + parsedConfig: &config.SubstitutedConfig{Config: current, Raw: current}, + substitutionContext: &config.SubstitutionContext{ + ContainerWorkspaceFolder: "/computed-before-discovery", + }, + } +} + +func newResolutionReplyFixture(t *testing.T) *resolutionReplyFixture { + t.Helper() + f := &resolutionReplyFixture{state: t.TempDir(), nonce: encoding.CreateNewUID("", "reply")} + origin, source := t.TempDir(), t.TempDir() + name, checksum := buildResolutionRuntime(t, origin) + f.info = &provider.AgentWorkspaceInfo{ + Origin: origin, ContentFolder: source, + Workspace: &provider.Workspace{ + ID: "lost-resolution", UID: encoding.CreateNewUID("default", "resolve"), + Source: provider.WorkspaceSource{LocalFolder: source}, + }, + Agent: provider.ProviderAgentConfig{ + Driver: provider.ExternalDriver, + Docker: provider.ProviderDockerDriverConfig{ + Runtime: "docker", + Elevation: resolutionDisabledSetting, + }, + External: provider.ProviderExternalDriverConfig{ + Binary: "RUNTIME", + ImageBackend: resolutionDisabledSetting, + Args: []string{ + "--state-dir", + f.state, + "--mode", + "persist-run-then-exit", + "--reply-nonce", + f.nonce, + }, + }, + Binaries: map[string][]*provider.ProviderBinary{"RUNTIME": {{ + OS: runtime.GOOS, Arch: runtime.GOARCH, Path: name, Checksum: checksum, + }}}, + }, + } + require.Equal(t, f.info.Workspace.UID, GetRunnerIDFromWorkspace(f.info.Workspace)) + f.record = filepath.Join(origin, provider.WorkspaceConfigFile) + recordBytes, err := json.Marshal(f.info) + f.recordBytes = recordBytes + require.NoError(t, err) + require.NoError(t, os.WriteFile(f.record, f.recordBytes, 0o600)) + f.sentinel = filepath.Join(source, "owned-user-data") + require.NoError(t, os.WriteFile(f.sentinel, []byte(f.nonce), 0o600)) + t.Cleanup(func() { f.cleanup(t) }) + return f +} + +func buildResolutionRuntime(t *testing.T, origin string) (string, string) { + t.Helper() + directory := filepath.Join(origin, "binaries", "runtime") + require.NoError(t, os.MkdirAll(directory, 0o700)) + name := "resolution runtime" + if runtime.GOOS == goosWindows { + name += ".exe" + } + binary := filepath.Join(directory, name) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + // #nosec G204 -- Builds the existing trusted fixture command into this test's private origin. + build := exec.CommandContext( + ctx, + "go", + "build", + "-o", + binary, + "../driver/external/internal/testfixture", + ) + output, err := build.CombinedOutput() + require.NoError(t, err, string(output)) + // #nosec G304 -- Binary is built from the trusted fixture into this test-owned origin. + bytes, err := os.ReadFile(binary) + require.NoError(t, err) + checksum := sha256.Sum256(bytes) + return name, hex.EncodeToString(checksum[:]) +} + +func (f *resolutionReplyFixture) createWithLostReply(t *testing.T) resolutionPersistedReply { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + first, err := NewRunner(ctx, "", "", f.info) + require.NoError(t, err) + initial, ok := first.(*runner) + require.True(t, ok) + imageRunner, ok := initial.driver.(driver.ImageRunner) + require.True(t, ok) + err = imageRunner.RunImageDevContainer(ctx, &driver.RunImageDevContainerParams{ + WorkspaceID: f.info.Workspace.UID, + Options: &driver.RunOptions{ + Image: "fixture", + User: containerRootUser, + RemoteUser: containerRootUser, + Labels: []string{resolutionReplyNonceLabel + "=" + f.nonce}, + WorkspaceMount: &config.Mount{ + Type: driver.MountTypeBind, + Source: f.info.ContentFolder, + Target: resolutionWorkspaceMount, + }, + }, + }) + require.Error(t, err) + require.NoError(t, ctx.Err(), "intentional exit must fail before caller deadline") + require.NotErrorIs(t, err, context.Canceled) + data, err := os.ReadFile(filepath.Join(f.state, "reply-persisted.json")) + require.NoError(t, err) + var marker resolutionPersistedReply + require.NoError(t, json.Unmarshal(data, &marker)) + require.Equal(t, f.nonce, marker.Nonce) + require.Equal(t, f.info.Workspace.UID, marker.Workspace) + require.Equal(t, "persist-run-then-exit", marker.Mode) + require.NotNil(t, marker.Container) + require.NotNil(t, marker.Container.State) + require.NotNil(t, marker.Container.Config) + require.Equal(t, f.info.Workspace.UID, marker.Container.Id) + require.NotEmpty(t, marker.Container.CreatedAt) + require.Equal(t, "running", marker.Container.State.Status) + return marker +} + +func (f *resolutionReplyFixture) assertAcknowledgedContainer( + t *testing.T, + marker resolutionPersistedReply, + found *config.ContainerDetails, +) { + t.Helper() + assert.Equal(t, marker.Container.Id, found.ID) + assert.Equal(t, marker.Container.CreatedAt, found.Created) + assert.Equal(t, config.ContainerStatusRunning, found.State.Status) + assert.Equal(t, marker.Container.State.StartedAt, found.State.StartedAt) + assert.Equal(t, marker.Container.Config.Labels, found.Config.Labels) + assert.Equal(t, f.nonce, found.Config.Labels[resolutionReplyNonceLabel]) + assert.Equal(t, containerRootUser, found.Config.User) + require.Len(t, marker.Container.Mounts, 1) + require.Len(t, found.Mounts, 1) + assert.Equal(t, marker.Container.Mounts[0].Source, found.Mounts[0].Source) + assert.Equal(t, marker.Container.Mounts[0].Destination, found.Mounts[0].Destination) + assert.Equal(t, marker.Container.Mounts[0].Type, found.Mounts[0].Type) +} + +func (f *resolutionReplyFixture) assertOperationCounts(t *testing.T) { + t.Helper() + records, err := resolutionReplyRecords[resolutionReplyOperation]( + filepath.Join(f.state, "reply-operations.jsonl"), + ) + require.NoError(t, err) + counts := map[string]int{} + for _, record := range records { + require.Equal(t, f.nonce, record.Nonce) + require.Equal(t, f.info.Workspace.UID, record.Workspace) + counts[record.Operation]++ + } + require.Equal(t, map[string]int{"attempt": 1, "created": 1}, counts, + "ordinary resolver must neither retry creation nor stop/delete existing state") +} + +func resolutionReplyRecords[T any](path string) ([]T, error) { + // #nosec G304 -- Callers pass only files written by the owned subprocess fixture. + file, err := os.Open(path) + if err != nil { + return nil, err + } + var records []T + scan := bufio.NewScanner(file) + for scan.Scan() { + var record T + if err := json.Unmarshal(scan.Bytes(), &record); err != nil { + return nil, errors.Join(err, file.Close()) + } + records = append(records, record) + } + return records, errors.Join(scan.Err(), file.Close()) +} + +func resolutionReplyProcessAlive(identity resolutionReplyProcess) (*process.Process, bool, error) { + if identity.PID <= 0 || identity.Started <= 0 { + return nil, false, fmt.Errorf("invalid owned process identity: %+v", identity) + } + p, err := process.NewProcess(identity.PID) + if errors.Is(err, process.ErrorProcessNotRunning) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + started, err := p.CreateTime() + if err != nil { + exists, existErr := process.PidExists(identity.PID) + if existErr == nil && !exists { + return nil, false, nil + } + return nil, false, errors.Join(err, existErr) + } + return p, started == identity.Started, nil +} + +func (f *resolutionReplyFixture) requireProcessesGone(t *testing.T) { + t.Helper() + records, err := resolutionReplyRecords[resolutionReplyProcess]( + filepath.Join(f.state, "reply-processes.jsonl"), + ) + require.NoError(t, err) + require.NotEmpty(t, records) + require.EventuallyWithT(t, func(c *assert.CollectT) { + for _, identity := range records { + _, alive, err := resolutionReplyProcessAlive(identity) + assert.NoError(c, err) + assert.False( + c, + alive, + "owned %s pid %d/start %d must exit", + identity.Kind, + identity.PID, + identity.Started, + ) + } + }, 10*time.Second, 20*time.Millisecond) + t.Logf("all %d recorded plugin/supervisor identities exited", len(records)) +} + +func (f *resolutionReplyFixture) terminateOwnedProcesses(t *testing.T) { + t.Helper() + records, err := resolutionReplyRecords[resolutionReplyProcess]( + filepath.Join(f.state, "reply-processes.jsonl"), + ) + if errors.Is(err, os.ErrNotExist) { + return + } + if !assert.NoError(t, err) { + return + } + for _, identity := range records { + p, alive, err := resolutionReplyProcessAlive(identity) + if !assert.NoError(t, err) || !alive { + continue + } + // PID and birth time are checked immediately before failure-only cleanup. + assert.NoError(t, p.Kill()) + } +} + +func (f *resolutionReplyFixture) cleanup(t *testing.T) { + f.terminateOwnedProcesses(t) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + fresh, err := NewRunner(ctx, "", "", f.info) + if assert.NoError(t, err) { + assert.NoError(t, fresh.Delete(ctx, DeleteOptions{})) + } + f.terminateOwnedProcesses(t) + f.requireProcessesGone(t) +} diff --git a/pkg/driver/external/internal/testfixture/main.go b/pkg/driver/external/internal/testfixture/main.go index c020889662..84c0723c86 100644 --- a/pkg/driver/external/internal/testfixture/main.go +++ b/pkg/driver/external/internal/testfixture/main.go @@ -2,6 +2,8 @@ package main import ( "context" + "encoding/json" + "errors" "flag" "fmt" "os" @@ -15,6 +17,7 @@ import ( "github.com/devsy-org/devsy-runtime-sdk/runtimev1" "github.com/devsy-org/devsy-runtime-sdk/server" "github.com/devsy-org/devsy-runtime-sdk/supervisor" + "github.com/shirou/gopsutil/v4/process" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -33,35 +36,70 @@ func main() { } mode := flag.String("mode", fake.Normal, "fixture mode") state := flag.String("state-dir", "", "state directory") + nonce := flag.String("reply-nonce", "", "owned lost-reply test nonce") delay := flag.Duration("delay", 0, "handshake delay") flag.Parse() recordPID(*state, "plugin") - serve(*mode, *state, *delay) + serve(*mode, *state, *delay, *nonce) } -func serve(mode, state string, delay time.Duration) { +func serve(mode, state string, delay time.Duration, nonce string) { if mode == fake.CrashBeforeHandshake { os.Exit(22) } if delay > 0 { time.Sleep(delay) } - fixtureMode := mode - if fixtureMode == "blocked" || fixtureMode == "environment" || fixtureMode == "block-info" || - fixtureMode == "env-error" || fixtureMode == "workspace-identity" || - strings.HasPrefix(fixtureMode, "stream-") { - fixtureMode = fake.Normal + if isLostReplyMode(mode) { + if err := recordLostReplyProcesses(state, nonce); err != nil { + panic(err) + } } + fixtureMode := underlyingFixtureMode(mode) runtime, err := fake.New(fake.Config{StateDir: state, Mode: fixtureMode}) if err != nil { panic(err) } - server.Serve(&fixture{Driver: runtime, mode: mode, directory: state}) + server.Serve(&fixture{Driver: runtime, mode: mode, directory: state, replyNonce: nonce}) +} + +func underlyingFixtureMode(mode string) string { + switch mode { + case "blocked", + "environment", + "block-info", + "env-error", + "workspace-identity", + persistRunThenExit, + persistRunThenHold: + return fake.Normal + default: + if strings.HasPrefix(mode, "stream-") { + return fake.Normal + } + return mode + } +} + +func recordLostReplyProcesses(directory, nonce string) error { + if nonce == "" { + return errors.New("lost reply fixture requires an owned nonce") + } + for kind, pid := range map[string]int{"plugin": os.Getpid(), "supervisor": os.Getppid()} { + identity, err := lostReplyProcessIdentity(pid, kind) + if err != nil { + return err + } + if err := appendLostReplyRecord(directory, "reply-processes.jsonl", identity); err != nil { + return err + } + } + return nil } type fixture struct { *fake.Driver - mode, directory string + mode, directory, replyNonce string } func (f *fixture) Info( @@ -123,6 +161,9 @@ func (f *fixture) RunImage( ctx context.Context, request *runtimev1.RunImageRequest, ) (*runtimev1.RunImageResponse, error) { + if isLostReplyMode(f.mode) { + return f.runImageWithLostReply(ctx, request) + } if f.mode == "workspace-identity" { request.Labels = append(request.Labels, "fixture.remote-user="+request.GetRemoteUser(), @@ -167,3 +208,149 @@ func recordPID(directory, kind string) { panic(err) } } + +const ( + persistRunThenExit = "persist-run-then-exit" + persistRunThenHold = "persist-run-then-hold" + lostReplyExitCode = 37 +) + +type replyProcessIdentity struct { + PID int32 `json:"pid"` + Started int64 `json:"started"` + Kind string `json:"kind"` +} + +type replyOperationRecord struct { + Operation string `json:"operation"` + Workspace string `json:"workspace"` + Nonce string `json:"nonce"` +} + +type persistedReplyMarker struct { + Workspace string `json:"workspace"` + Nonce string `json:"nonce"` + Mode string `json:"mode"` + Container *runtimev1.ContainerDetails `json:"container"` +} + +func isLostReplyMode(mode string) bool { + return mode == persistRunThenExit || mode == persistRunThenHold +} + +func lostReplyProcessIdentity(pid int, kind string) (replyProcessIdentity, error) { + // #nosec G115 -- OS process identifiers fit the platform's signed PID representation. + observed, err := process.NewProcess(int32(pid)) + if err != nil { + return replyProcessIdentity{}, err + } + started, err := observed.CreateTime() + if err != nil { + return replyProcessIdentity{}, err + } + return replyProcessIdentity{PID: observed.Pid, Started: started, Kind: kind}, nil +} + +func appendLostReplyRecord(directory, name string, value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + // #nosec G304 G703 -- Directory is owned by the test; name is a fixture literal. + file, err := os.OpenFile( + filepath.Join(directory, name), + os.O_APPEND|os.O_CREATE|os.O_WRONLY, + 0o600, + ) + if err != nil { + return err + } + _, writeErr := file.Write(append(data, '\n')) + return errors.Join(writeErr, file.Close()) +} + +func (f *fixture) Stop( + ctx context.Context, + request *runtimev1.StopRequest, +) (*runtimev1.StopResponse, error) { + if isLostReplyMode(f.mode) { + if err := f.recordReplyOperation("stop", request.GetWorkspaceId()); err != nil { + return nil, err + } + } + return f.Driver.Stop(ctx, request) +} + +func (f *fixture) Delete( + ctx context.Context, + request *runtimev1.DeleteRequest, +) (*runtimev1.DeleteResponse, error) { + if isLostReplyMode(f.mode) { + if err := f.recordReplyOperation("delete", request.GetWorkspaceId()); err != nil { + return nil, err + } + } + return f.Driver.Delete(ctx, request) +} + +func (f *fixture) recordReplyOperation(operation, workspace string) error { + return appendLostReplyRecord(f.directory, "reply-operations.jsonl", replyOperationRecord{ + Operation: operation, Workspace: workspace, Nonce: f.replyNonce, + }) +} + +func (f *fixture) publishReplyMarker(value persistedReplyMarker) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + file, err := os.CreateTemp(f.directory, ".reply-marker-*") + if err != nil { + return err + } + defer func() { _ = os.Remove(file.Name()) }() + _, writeErr := file.Write(data) + if err := errors.Join(writeErr, file.Close()); err != nil { + return err + } + return os.Rename(file.Name(), filepath.Join(f.directory, "reply-persisted.json")) +} + +func (f *fixture) runImageWithLostReply( + ctx context.Context, + request *runtimev1.RunImageRequest, +) (*runtimev1.RunImageResponse, error) { + if err := f.recordReplyOperation("attempt", request.GetWorkspaceId()); err != nil { + return nil, err + } + if _, err := f.Driver.RunImage(ctx, request); err != nil { + return nil, err + } + if err := f.recordReplyOperation("created", request.GetWorkspaceId()); err != nil { + return nil, err + } + if err := f.publishCreatedReply(ctx, request.GetWorkspaceId()); err != nil { + return nil, err + } + if f.mode == persistRunThenExit { + os.Exit(lostReplyExitCode) + } + <-ctx.Done() + return nil, ctx.Err() +} + +func (f *fixture) publishCreatedReply(ctx context.Context, workspace string) error { + found, err := f.Find(ctx, &runtimev1.FindRequest{WorkspaceId: workspace}) + if err != nil { + return err + } + if !found.GetFound() || found.GetContainer() == nil { + return errors.New("created fixture record missing") + } + if err := f.publishReplyMarker(persistedReplyMarker{ + Workspace: workspace, Nonce: f.replyNonce, Mode: f.mode, Container: found.GetContainer(), + }); err != nil { + return err + } + return nil +} diff --git a/pkg/driver/external/lost_reply_test.go b/pkg/driver/external/lost_reply_test.go new file mode 100644 index 0000000000..67330950fb --- /dev/null +++ b/pkg/driver/external/lost_reply_test.go @@ -0,0 +1,437 @@ +package external_test + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/devsy-org/devsy-runtime-sdk/runtimev1" + "github.com/devsy-org/devsy/pkg/agent" + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/devcontainer" + containerconfig "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/driver" + "github.com/devsy-org/devsy/pkg/driver/drivercreate" + "github.com/devsy-org/devsy/pkg/encoding" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/shirou/gopsutil/v4/process" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + lostReplyRuntimeKey = "RUNTIME" + lostReplyLabel = "fixture.lost-reply-nonce" + lostReplyUser = "root" + lostReplyContext = "default" + lostReplyExitMode = "persist-run-then-exit" + lostReplyHoldMode = "persist-run-then-hold" +) + +type lostReplyFixture struct { + state, agentRecord, workspaceRecord, source, nonce string + workspace *provider.AgentWorkspaceInfo + snapshots map[string][]byte +} + +type lostReplyMarker struct { + Workspace string `json:"workspace"` + Nonce string `json:"nonce"` + Mode string `json:"mode"` + Container *runtimev1.ContainerDetails `json:"container"` +} + +type lostReplyOperation struct { + Operation string `json:"operation"` + Workspace string `json:"workspace"` + Nonce string `json:"nonce"` +} + +type lostReplyProcess struct { + PID int32 `json:"pid"` + Started int64 `json:"started"` + Kind string `json:"kind"` +} + +// TestMain in production_test.go supplies the real CLI supervisor dispatch. +// These tests deliberately stop at public NewRunner/Find/Command, not Runner.Up. +func TestLostRunImageReplyFreshRunner(t *testing.T) { + binary := buildLostReplyFixture(t) + for _, mode := range []string{lostReplyExitMode, lostReplyHoldMode} { + t.Run(mode, func(t *testing.T) { + f := newLostReplyFixture(t, binary, mode) + setupCtx, setupCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer setupCancel() + bundle, err := drivercreate.New(setupCtx, f.workspace) + require.NoError(t, err) + imageRunner, ok := bundle.Runtime.(driver.ImageRunner) + require.True(t, ok) + marker := exerciseLostReply(t, f, imageRunner, mode) + requireLostReplyProcessesGone(t, f.state) + // Only persisted metadata crosses this boundary; no original Host is reused. + f.assertFreshRunner(t, marker) + f.assertPreservedFiles(t) + f.assertOperations(t) + requireLostReplyProcessesGone(t, f.state) + }) + } +} + +func buildLostReplyFixture(t *testing.T) string { + t.Helper() + name := "runtime space é" + if runtime.GOOS == "windows" { + name += ".exe" + } + binary := filepath.Join(t.TempDir(), name) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + // #nosec G204 -- Builds only the checked-in trusted fixture into a test-owned directory. + build := exec.CommandContext(ctx, "go", "build", "-o", binary, "./internal/testfixture") + output, err := build.CombinedOutput() + require.NoError(t, err, string(output)) + return binary +} + +func newLostReplyFixture(t *testing.T, binary, mode string) *lostReplyFixture { + t.Helper() + t.Setenv(config.EnvHome, t.TempDir()) + config.ResetPathManager() + t.Cleanup(config.ResetPathManager) + f := &lostReplyFixture{state: t.TempDir(), source: t.TempDir(), snapshots: map[string][]byte{}} + f.nonce = encoding.CreateNewUID("", "reply") + origin := t.TempDir() + name, checksum := prepareLostReplyBinary(t, origin, binary) + ssh := filepath.Join(t.TempDir(), "config") + require.NoError(t, os.WriteFile(ssh, []byte("# owned SSH sentinel "+f.nonce+"\n"), 0o600)) + workspace := &provider.Workspace{ + ID: "lost-reply-workspace", + UID: encoding.CreateNewUID(lostReplyContext, "lost-reply"), + Context: lostReplyContext, + Provider: provider.WorkspaceProviderConfig{Name: "lost-reply-fixture"}, + Source: provider.WorkspaceSource{LocalFolder: f.source}, + SSHConfigPath: ssh, + } + require.NoError(t, provider.SaveWorkspaceConfig(workspace)) + hostDirectory, err := provider.GetWorkspaceDir(workspace.Context, workspace.ID) + require.NoError(t, err) + f.workspaceRecord = filepath.Join(hostDirectory, provider.WorkspaceConfigFile) + f.workspace = &provider.AgentWorkspaceInfo{ + Origin: origin, Workspace: workspace, ContentFolder: f.source, + Agent: provider.ProviderAgentConfig{ + Driver: provider.ExternalDriver, + Docker: provider.ProviderDockerDriverConfig{Runtime: "docker", Elevation: "none"}, + External: provider.ProviderExternalDriverConfig{ + Binary: lostReplyRuntimeKey, ImageBackend: noImageBackend, + Args: []string{"--state-dir", f.state, "--mode", mode, "--reply-nonce", f.nonce}, + }, + Binaries: map[string][]*provider.ProviderBinary{lostReplyRuntimeKey: {{ + OS: runtime.GOOS, Arch: runtime.GOARCH, Path: name, Checksum: checksum, + }}}, + }, + } + f.agentRecord = filepath.Join(origin, provider.WorkspaceConfigFile) + raw, err := json.Marshal(f.workspace) + require.NoError(t, err) + require.NoError(t, os.WriteFile(f.agentRecord, raw, 0o600)) + sentinel := filepath.Join(f.source, "user-data") + require.NoError(t, os.WriteFile(sentinel, []byte(f.nonce), 0o600)) + for _, path := range []string{f.agentRecord, f.workspaceRecord, sentinel, ssh} { + // #nosec G304 -- Snapshots include only this fixture's records, source and SSH sentinel. + f.snapshots[path], err = os.ReadFile(path) + require.NoError(t, err) + } + // Registered after owned directories and before any transport operation. + t.Cleanup(func() { f.cleanup(t) }) + return f +} + +func prepareLostReplyBinary(t *testing.T, origin, binary string) (string, string) { + t.Helper() + directory := filepath.Join(origin, "binaries", "runtime") + require.NoError(t, os.MkdirAll(directory, 0o700)) + // #nosec G304 -- The binary is built by this package's test fixture. + data, err := os.ReadFile(binary) + require.NoError(t, err) + name := filepath.Base(binary) + // #nosec G306 G703 -- Trusted fixture bytes are copied under this test-owned origin and executable basename. + require.NoError(t, os.WriteFile(filepath.Join(directory, name), data, 0o700)) + checksum := sha256.Sum256(data) + return name, hex.EncodeToString(checksum[:]) +} + +func exerciseLostReply( + t *testing.T, + f *lostReplyFixture, + imageRunner driver.ImageRunner, + mode string, +) lostReplyMarker { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + result := make(chan error, 1) + joined := false + t.Cleanup(func() { + cancel() + if !joined { + joinLostReplyCall(t, f.state, result) + } + }) + go func() { + result <- imageRunner.RunImageDevContainer(ctx, &driver.RunImageDevContainerParams{ + WorkspaceID: devcontainer.GetRunnerIDFromWorkspace(f.workspace.Workspace), + Options: &driver.RunOptions{ + Image: "fixture", User: lostReplyUser, RemoteUser: lostReplyUser, Labels: []string{lostReplyLabel + "=" + f.nonce}, + WorkspaceMount: &containerconfig.Mount{Type: driver.MountTypeBind, Source: f.source, Target: "/workspace"}, + }, + }) + }() + marker := waitForLostReplyMarker(t, f, mode) + if mode == lostReplyHoldMode { + cancel() + } + select { + case err := <-result: + joined = true + require.Error(t, err) + if mode == lostReplyHoldMode { + require.ErrorIs(t, err, context.Canceled) + } else { + require.NoError(t, ctx.Err(), "crash must fail before caller deadline") + require.NotErrorIs(t, err, context.Canceled) + require.NotContains(t, err.Error(), "already exists") + } + case <-time.After(10 * time.Second): + t.Fatal("lost-reply call did not return within cleanup bound") + } + cancel() + return marker +} + +func joinLostReplyCall(t *testing.T, state string, result <-chan error) { + t.Helper() + select { + case <-result: + return + case <-time.After(10 * time.Second): + terminateLostReplyProcesses(t, state) + } + select { + case <-result: + case <-time.After(10 * time.Second): + t.Error("RunImage caller did not join after owned cleanup") + } +} + +func waitForLostReplyMarker(t *testing.T, f *lostReplyFixture, mode string) lostReplyMarker { + t.Helper() + var marker lostReplyMarker + require.EventuallyWithT(t, func(c *assert.CollectT) { + data, err := os.ReadFile(filepath.Join(f.state, "reply-persisted.json")) + if !assert.NoError(c, err) { + return + } + assert.NoError(c, json.Unmarshal(data, &marker)) + }, 10*time.Second, 10*time.Millisecond, "must observe a complete persisted effect before triggering cancellation") + require.Equal(t, f.nonce, marker.Nonce) + require.Equal(t, mode, marker.Mode) + require.Equal(t, f.workspace.Workspace.UID, marker.Workspace) + require.NotNil(t, marker.Container) + require.NotNil(t, marker.Container.State) + require.NotNil(t, marker.Container.Config) + require.Equal(t, marker.Workspace, marker.Container.Id) + require.NotEmpty(t, marker.Container.CreatedAt) + require.Equal(t, "running", marker.Container.State.Status) + require.Equal(t, f.nonce, marker.Container.Config.Labels[lostReplyLabel]) + return marker +} + +func (f *lostReplyFixture) assertFreshRunner(t *testing.T, marker lostReplyMarker) { + t.Helper() + persisted, err := agent.ParseAgentWorkspaceInfo(f.agentRecord) + require.NoError(t, err) + require.Equal(t, f.workspace.Workspace.UID, persisted.Workspace.UID) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + fresh, err := devcontainer.NewRunner(ctx, "", "", persisted) + require.NoError(t, err) + found, err := fresh.Find(ctx) + require.NoError(t, err) + require.NotNil(t, found) + assert.Equal(t, marker.Container.Id, found.ID) + assert.Equal(t, marker.Container.CreatedAt, found.Created) + assert.Equal(t, containerconfig.ContainerStatusRunning, found.State.Status) + assert.Equal(t, marker.Container.State.StartedAt, found.State.StartedAt) + assert.Equal(t, marker.Container.Config.Labels, found.Config.Labels) + assert.Equal(t, marker.Container.Config.User, found.Config.User) + require.Len(t, marker.Container.Mounts, 1) + require.Len(t, found.Mounts, 1) + assert.Equal(t, marker.Container.Mounts[0].Source, found.Mounts[0].Source) + assert.Equal(t, marker.Container.Mounts[0].Destination, found.Mounts[0].Destination) + assert.Equal(t, marker.Container.Mounts[0].Type, found.Mounts[0].Type) + var stdout, stderr bytes.Buffer + require.NoError(t, fresh.Command(ctx, devcontainer.CommandParams{ + Command: "cat", Stdin: bytes.NewBufferString(f.nonce), Stdout: &stdout, Stderr: &stderr, + })) + assert.Equal(t, f.nonce, stdout.String(), "fixture echo proves transport usability only") + again, err := fresh.Find(ctx) + require.NoError(t, err) + assert.Equal(t, found, again, "a transport command must not replace the persisted container") +} + +func (f *lostReplyFixture) assertPreservedFiles(t *testing.T) { + t.Helper() + for path, want := range f.snapshots { + // #nosec G304 -- Snapshot paths belong to this test's isolated workspace and SSH configuration. + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, want, got, path) + } + loaded, err := provider.LoadWorkspaceConfig(lostReplyContext, f.workspace.Workspace.ID) + require.NoError(t, err) + assert.Equal(t, f.workspace.Workspace.UID, loaded.UID) + assert.Equal(t, f.workspace.Workspace.Source, loaded.Source) + assert.Equal(t, f.workspace.Workspace.Provider, loaded.Provider) +} + +func (f *lostReplyFixture) assertOperations(t *testing.T) { + t.Helper() + records, err := readLostReplyRecords[lostReplyOperation]( + filepath.Join(f.state, "reply-operations.jsonl"), + ) + require.NoError(t, err) + counts := map[string]int{} + for _, record := range records { + require.Equal(t, f.nonce, record.Nonce) + require.Equal(t, f.workspace.Workspace.UID, record.Workspace) + counts[record.Operation]++ + } + assert.Equal( + t, + map[string]int{"attempt": 1, "created": 1}, + counts, + "before cleanup: exactly one create attempt and effect, no stop/delete or hidden duplicate attempt", + ) +} + +func readLostReplyRecords[T any](path string) ([]T, error) { + // #nosec G304 -- Callers supply operation logs under this test's owned fixture directory. + file, err := os.Open(path) + if err != nil { + return nil, err + } + var records []T + scanner := bufio.NewScanner(file) + for scanner.Scan() { + var record T + if err := json.Unmarshal(scanner.Bytes(), &record); err != nil { + return nil, errors.Join(err, file.Close()) + } + records = append(records, record) + } + return records, errors.Join(scanner.Err(), file.Close()) +} + +func lostReplyProcessAlive(identity lostReplyProcess) (bool, error) { + if identity.PID <= 0 || identity.Started <= 0 { + return false, fmt.Errorf("invalid owned process identity: %+v", identity) + } + p, err := process.NewProcess(identity.PID) + if errors.Is(err, process.ErrorProcessNotRunning) { + return false, nil + } + if err != nil { + return false, err + } + started, err := p.CreateTime() + if err != nil { + exists, existErr := process.PidExists(identity.PID) + if existErr == nil && !exists { + return false, nil + } + return false, errors.Join(err, existErr) + } + return started == identity.Started, nil +} + +func requireLostReplyProcessesGone(t *testing.T, directory string) { + t.Helper() + records, err := readLostReplyRecords[lostReplyProcess]( + filepath.Join(directory, "reply-processes.jsonl"), + ) + require.NoError(t, err) + require.NotEmpty(t, records) + require.EventuallyWithT(t, func(c *assert.CollectT) { + for _, identity := range records { + alive, err := lostReplyProcessAlive(identity) + assert.NoError(c, err) + assert.False( + c, + alive, + "owned %s pid %d/start %d must exit", + identity.Kind, + identity.PID, + identity.Started, + ) + } + }, 10*time.Second, 20*time.Millisecond) +} + +func terminateLostReplyProcesses(t *testing.T, directory string) { + t.Helper() + records, err := readLostReplyRecords[lostReplyProcess]( + filepath.Join(directory, "reply-processes.jsonl"), + ) + if errors.Is(err, os.ErrNotExist) { + return + } + if !assert.NoError(t, err) { + return + } + for _, identity := range records { + terminateLostReplyProcess(t, identity) + } +} + +func terminateLostReplyProcess(t *testing.T, identity lostReplyProcess) { + t.Helper() + alive, err := lostReplyProcessAlive(identity) + if !assert.NoError(t, err) || !alive { + return + } + p, err := process.NewProcess(identity.PID) + if !assert.NoError(t, err) { + return + } + // Recheck immediately before killing; never intentionally kill a reused PID. + started, err := p.CreateTime() + if assert.NoError(t, err) && started == identity.Started { + assert.NoError(t, p.Kill()) + } +} + +func (f *lostReplyFixture) cleanup(t *testing.T) { + // The first-call join cleanup runs before this callback (LIFO). + terminateLostReplyProcesses(t, f.state) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + persisted, err := agent.ParseAgentWorkspaceInfo(f.agentRecord) + if assert.NoError(t, err) { + bundle, err := drivercreate.New(ctx, persisted) + if assert.NoError(t, err) { + assert.NoError(t, bundle.Runtime.DeleteDevContainer(ctx, f.workspace.Workspace.UID)) + } + } + terminateLostReplyProcesses(t, f.state) + requireLostReplyProcessesGone(t, f.state) +} From 96c6452957db71a733989048c79df140a5da16b8 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 11 Oct 2026 12:49:55 -0600 Subject: [PATCH 2/2] test(runtime): refresh process identity before failure cleanup --- pkg/devcontainer/lost_reply_resolution_test.go | 7 +++++-- pkg/driver/external/lost_reply_test.go | 7 ++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/devcontainer/lost_reply_resolution_test.go b/pkg/devcontainer/lost_reply_resolution_test.go index 3225428a39..8d82cbcb82 100644 --- a/pkg/devcontainer/lost_reply_resolution_test.go +++ b/pkg/devcontainer/lost_reply_resolution_test.go @@ -355,8 +355,11 @@ func (f *resolutionReplyFixture) terminateOwnedProcesses(t *testing.T) { if !assert.NoError(t, err) || !alive { continue } - // PID and birth time are checked immediately before failure-only cleanup. - assert.NoError(t, p.Kill()) + // IsRunning compares the cached birth time with a freshly observed process. + same, err := p.IsRunning() + if assert.NoError(t, err) && same { + assert.NoError(t, p.Kill()) + } } } diff --git a/pkg/driver/external/lost_reply_test.go b/pkg/driver/external/lost_reply_test.go index 67330950fb..761cbc09d0 100644 --- a/pkg/driver/external/lost_reply_test.go +++ b/pkg/driver/external/lost_reply_test.go @@ -415,7 +415,12 @@ func terminateLostReplyProcess(t *testing.T, identity lostReplyProcess) { } // Recheck immediately before killing; never intentionally kill a reused PID. started, err := p.CreateTime() - if assert.NoError(t, err) && started == identity.Started { + if !assert.NoError(t, err) || started != identity.Started { + return + } + // IsRunning refreshes identity; CreateTime on the same Process is cached. + same, err := p.IsRunning() + if assert.NoError(t, err) && same { assert.NoError(t, p.Kill()) } }