diff --git a/hack/podman_ci/setup_windows.go b/hack/podman_ci/setup_windows.go index cae015b509..5a029bf022 100644 --- a/hack/podman_ci/setup_windows.go +++ b/hack/podman_ci/setup_windows.go @@ -17,18 +17,20 @@ func runtimeGOOS() string { return goosWindows } func platformDefaultPodmanPath() string { return `C:\Program Files\RedHat\Podman\podman.exe` } const ( - commandWSL = "wsl.exe" - commandMachine = "machine" - machineInit = "init" - machineInspect = "inspect" - machineStart = "start" - windowsInitTimeout = 150 * time.Second - windowsStartTimeout = 90 * time.Second - windowsReadinessTimeout = 45 * time.Second - windowsProbeTimeout = 7 * time.Second - windowsDiagnosticReserve = 25 * time.Second - windowsMinimumRecoveryTime = 60 * time.Second - windowsHardDeadlinePadding = 10 * time.Second + commandWSL = "wsl.exe" + commandMachine = "machine" + machineInit = "init" + machineInspect = "inspect" + machineStart = "start" + windowsInitTimeout = 150 * time.Second + windowsStartTimeout = 90 * time.Second + windowsReadinessTimeout = 45 * time.Second + windowsProbeTimeout = 7 * time.Second + windowsDiagnosticReserve = 25 * time.Second + windowsMinimumRecoveryTime = 60 * time.Second + windowsHardDeadlinePadding = 10 * time.Second + windowsHardDeadlineDiagnosticsGrace = 2 * time.Second + maxTransientPullRetries = 2 ) func platformSetup(ctx context.Context, deps Dependencies, cfg SetupConfig) error { @@ -43,7 +45,17 @@ func setupWindows(ctx context.Context, deps Dependencies, cfg SetupConfig) error return err } state := &windowsSetup{ctx: ctx, deps: deps, budget: budget, cfg: cfg} - return state.run() + runErr := state.run() + if runErr != nil { + // Per-command timeouts deliberately leave descendants (podman, + // gvproxy, win-sshproxy, wsl helpers) running so recovery and + // readiness polling can observe a still-booting machine. Once setup + // gives up for good, nothing else reaps them, so sweep them here to + // guarantee the step can exit promptly instead of leaving processes + // that may outlive our own bootstrap budget. + terminateStrayProcessesHook(deps.Log) + } + return runErr } type windowsSetup struct { @@ -119,19 +131,11 @@ func (s *windowsSetup) initializeMachine(attempt int) error { if attempt == 1 { reserve += windowsMinimumRecoveryTime } - initTimeout := s.budget.Bound(windowsInitTimeout, reserve) - if initTimeout <= 0 { - return fmt.Errorf( - "PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT attempt=%d output=%s", - attempt, - "PODMAN_WINDOWS_BOOTSTRAP_BUDGET_EXHAUSTED: insufficient initialization budget", - ) - } args := []string{commandMachine, machineInit} - result := s.runCommand( - CommandSpec{Path: s.cfg.PodmanPath, Args: args, Timeout: initTimeout}, - reserve, - ) + result, err := s.runMachineInit(attempt, args, reserve) + if err != nil { + return err + } if result.TimedOut { return fmt.Errorf( "PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT attempt=%d %s", @@ -152,6 +156,69 @@ func (s *windowsSetup) initializeMachine(attempt int) error { ) } +// runMachineInit runs `podman machine init`, retrying a bounded number of +// times when the failure looks like a transient registry pull error (e.g. a +// truncated read on the machine-os image blob) instead of immediately +// burning a full attempt/recovery cycle. +func (s *windowsSetup) runMachineInit( + attempt int, + args []string, + reserve time.Duration, +) (CommandResult, error) { + for pullAttempt := 1; ; pullAttempt++ { + initTimeout := s.budget.Bound(windowsInitTimeout, reserve) + if initTimeout <= 0 { + return CommandResult{}, fmt.Errorf( + "PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT attempt=%d output=%s", + attempt, + "PODMAN_WINDOWS_BOOTSTRAP_BUDGET_EXHAUSTED: insufficient initialization budget", + ) + } + result := s.runCommand( + CommandSpec{Path: s.cfg.PodmanPath, Args: args, Timeout: initTimeout}, + reserve, + ) + if s.shouldStopInitRetry(result, pullAttempt, reserve) { + return result, nil + } + s.deps.Log.Printf( + "[podman-windows] machine_init_transient_pull_retry attempt=%d pull_attempt=%d", + attempt, + pullAttempt, + ) + } +} + +func (s *windowsSetup) shouldStopInitRetry( + result CommandResult, + pullAttempt int, + reserve time.Duration, +) bool { + return !isTransientPullError(result.Output()) || + pullAttempt > maxTransientPullRetries || + s.budget.Remaining(reserve) < windowsMinimumRecoveryTime +} + +// isTransientPullError reports whether a failed `podman machine init` +// output looks like a transient registry/network error pulling the +// machine-os image, rather than a durable configuration or environment +// failure. Observed in CI: "writing blob: happened during read: unexpected +// EOF" from a truncated image layer read. +func isTransientPullError(output string) bool { + markers := []string{ + "happened during read: unexpected EOF", + "connection reset by peer", + "TLS handshake timeout", + "i/o timeout", + } + for _, marker := range markers { + if strings.Contains(output, marker) { + return true + } + } + return false +} + func (s *windowsSetup) verifyExistingMachine(attempt int) error { args := []string{commandMachine, machineInspect} result := s.runCommand( @@ -407,6 +474,22 @@ func armWatchdog(timeout time.Duration, log *Logger) watchdog { defer timer.Stop() select { case <-timer.C: + // Diagnostics/cleanup write to the same stdout the stalled setup + // path may already be blocked on; never let them stall the forced + // exit itself, so bound the attempt and terminate unconditionally. + done := make(chan struct{}) + go func() { + defer close(done) + log.Printf( + "[podman-windows] hard_deadline_exceeded timeout_ms=%d", + timeout.Milliseconds(), + ) + terminateStrayProcessesHook(log) + }() + select { + case <-done: + case <-time.After(windowsHardDeadlineDiagnosticsGrace): + } _ = windows.TerminateProcess(windows.CurrentProcess(), 124) os.Exit(124) case <-watch.stop: diff --git a/hack/podman_ci/setup_windows_test.go b/hack/podman_ci/setup_windows_test.go index 6b3e9aeb89..d17db2c98f 100644 --- a/hack/podman_ci/setup_windows_test.go +++ b/hack/podman_ci/setup_windows_test.go @@ -100,6 +100,7 @@ func windowsDependencies(r *windowsFakeRunner, clock *fakeClock) Dependencies { func windowsConfig(t *testing.T) SetupConfig { t.Helper() + stubStrayProcessSweep(t) podman := filepath.Join(t.TempDir(), "podman.exe") if err := os.WriteFile(podman, []byte(""), 0o700); err != nil { t.Fatal(err) @@ -107,6 +108,18 @@ func windowsConfig(t *testing.T) SetupConfig { return SetupConfig{Mode: ModeRootful, PodmanPath: podman, BootstrapTimeout: 270 * time.Second} } +// stubStrayProcessSweep replaces the real system-wide process sweep with a +// no-op for the duration of the test: fake-runner tests exercise setup +// failure paths that call terminateStrayProcessesHook, and letting that +// reach the real OS-level kill could terminate unrelated podman/wsl +// processes on a developer's machine. +func stubStrayProcessSweep(t *testing.T) { + t.Helper() + original := terminateStrayProcessesHook + terminateStrayProcessesHook = func(*Logger) {} + t.Cleanup(func() { terminateStrayProcessesHook = original }) +} + func TestWindowsHealthyBootstrapAndExistingMachine(t *testing.T) { clock := &fakeClock{now: time.Unix(1, 0)} runner := &windowsFakeRunner{clock: clock, results: map[string][]CommandResult{ @@ -225,6 +238,47 @@ func TestWindowsPartialInitRecoveryRetriesWithFreshBudget(t *testing.T) { } } +func TestWindowsMachineInitRetriesTransientPullError(t *testing.T) { + clock := &fakeClock{now: time.Unix(1, 0)} + runner := &windowsFakeRunner{clock: clock, results: map[string][]CommandResult{ + testMachineInit: { + failureResult("Error: writing blob: happened during read: unexpected EOF"), + successResult(""), + }, + }, durations: map[string]time.Duration{}} + if err := setupWindows( + context.Background(), + windowsDependencies(runner, clock), + windowsConfig(t), + ); err != nil { + t.Fatal(err) + } + if countWindowsCalls(runner, testMachineInit) != 2 { + t.Fatalf("transient pull error did not retry machine init: %+v", runner.calls) + } + if countWindowsCalls(runner, "machine stop") != 0 { + t.Fatalf("transient pull retry should not trigger full recovery: %+v", runner.calls) + } +} + +func TestWindowsMachineInitTransientPullRetryIsBounded(t *testing.T) { + clock := &fakeClock{now: time.Unix(1, 0)} + transientFailure := failureResult("Error: writing blob: happened during read: unexpected EOF") + runner := &windowsFakeRunner{clock: clock, results: map[string][]CommandResult{ + testMachineInit: { + transientFailure, transientFailure, transientFailure, + transientFailure, transientFailure, transientFailure, + }, + }, durations: map[string]time.Duration{}} + err := setupWindows(context.Background(), windowsDependencies(runner, clock), windowsConfig(t)) + if err == nil || !strings.Contains(err.Error(), "unexpected EOF") { + t.Fatalf("expected terminal transient-pull failure, got %v", err) + } + if countWindowsCalls(runner, testMachineInit) != (maxTransientPullRetries+1)*2 { + t.Fatalf("expected bounded retries across both attempts: %+v", runner.calls) + } +} + func TestWindowsUnexpectedCleanupFailurePreventsRetry(t *testing.T) { failedCommands := []string{ "machine stop", "machine rm -f", "--shutdown", testWSLList, @@ -322,6 +376,11 @@ func TestWindowsWatchdogTerminatesAndCanBeDisarmed(t *testing.T) { } func runWatchdogChild(mode string) { + // This process is a fresh re-exec of the test binary, so package vars + // are back to their defaults; neutralize the real system-wide sweep so + // the watchdog firing in these tests cannot kill unrelated podman/wsl + // processes on the host. + terminateStrayProcessesHook = func(*Logger) {} if mode == "blocked-stderr" { reader, stderr, err := os.Pipe() if err != nil { @@ -362,6 +421,7 @@ func verifyWatchdogExpires(t *testing.T) { t.Fatalf("blocked watchdog exit=%v output=%s", err, output) } if !strings.Contains(string(output), "hard_deadline_armed") || + !strings.Contains(string(output), "hard_deadline_exceeded") || strings.Contains(string(output), "PODMAN_WINDOWS_BOOTSTRAP_HARD_TIMEOUT") { t.Fatalf("watchdog markers missing: %s", output) } diff --git a/hack/podman_ci/windows_processes.go b/hack/podman_ci/windows_processes.go index c30891bb2e..42848903b6 100644 --- a/hack/podman_ci/windows_processes.go +++ b/hack/podman_ci/windows_processes.go @@ -46,3 +46,49 @@ func isInterestingWindowsProcess(name string) bool { return false } } + +// terminateStrayProcessesHook is the seam setupWindows and armWatchdog call +// through; tests override it to avoid sweeping real podman/wsl/gvproxy/ +// win-sshproxy processes that may be running on a developer's machine for +// unrelated work. +var terminateStrayProcessesHook = terminateStrayWindowsProcesses + +// terminateStrayWindowsProcesses forcefully terminates any Podman/WSL helper +// processes still running system-wide (podman, wsl, gvproxy, win-sshproxy). +// It is called when devsy-ci-podman.exe is about to exit after a terminal +// failure (including the hard watchdog deadline) so that no descendant +// process is left behind holding handles or keeping a VM alive past our own +// bootstrap budget: per-command timeouts deliberately leave descendants +// running to let recovery/readiness polling observe a still-booting +// machine, but once setup gives up for good there is no further attempt to +// reap them. +func terminateStrayWindowsProcesses(log *Logger) { + terminateWindowsProcesses(matchingWindowsProcesses(), log) +} + +// terminateWindowsProcesses force-kills exactly the given processes. It is +// split out from terminateStrayWindowsProcesses so tests can exercise the +// real OpenProcess/TerminateProcess kill mechanics against a single +// known-owned helper process without sweeping the whole host. +func terminateWindowsProcesses(processes []processInfo, log *Logger) { + for _, process := range processes { + handle, err := windows.OpenProcess(windows.PROCESS_TERMINATE, false, process.PID) + if err != nil { + log.Printf( + "[podman-windows] stray_process_open_failed pid=%d name=%s err=%v", + process.PID, + process.Name, + err, + ) + continue + } + killErr := windows.TerminateProcess(handle, 124) + _ = windows.CloseHandle(handle) + log.Printf( + "[podman-windows] stray_process_terminated pid=%d name=%s err=%v", + process.PID, + process.Name, + killErr, + ) + } +} diff --git a/hack/podman_ci/windows_processes_test.go b/hack/podman_ci/windows_processes_test.go new file mode 100644 index 0000000000..cd8efeb278 --- /dev/null +++ b/hack/podman_ci/windows_processes_test.go @@ -0,0 +1,97 @@ +//go:build windows + +package main + +import ( + "io" + "os" + "os/exec" + "path/filepath" + "testing" + "time" +) + +// TestTerminateStrayWindowsProcessesKillsMatchingByName verifies that a +// process matching one of the "interesting" Podman/WSL helper names (here, +// win-sshproxy.exe) is observed by matchingWindowsProcesses and forcefully +// terminated by terminateWindowsProcesses. It targets only its own spawned +// helper (not the system-wide terminateStrayWindowsProcesses sweep) so it +// cannot kill unrelated podman/wsl processes a developer may have running. +func TestTerminateStrayWindowsProcessesKillsMatchingByName(t *testing.T) { + exePath := copySelfAs(t, "win-sshproxy.exe") + + cmd := exec.Command(exePath, "-test.run=^TestStrayProcessHelperFixture$") + cmd.Env = append(os.Environ(), "DEVSY_PODMAN_STRAY_HELPER=1") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + }) + + helper, ok := waitForInterestingProcess(uint32(cmd.Process.Pid), 5*time.Second) + if !ok { + t.Fatal("helper process was not observed as an interesting Windows process") + } + + terminateWindowsProcesses([]processInfo{helper}, &Logger{Out: io.Discard, Err: io.Discard}) + + if !waitForProcessExit(cmd, 5*time.Second) { + t.Fatal("stray process survived the termination sweep") + } +} + +func waitForInterestingProcess(pid uint32, timeout time.Duration) (processInfo, bool) { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if process, ok := findInterestingProcess(pid); ok { + return process, true + } + time.Sleep(50 * time.Millisecond) + } + return processInfo{}, false +} + +func findInterestingProcess(pid uint32) (processInfo, bool) { + for _, process := range matchingWindowsProcesses() { + if process.PID == pid { + return process, true + } + } + return processInfo{}, false +} + +func waitForProcessExit(cmd *exec.Cmd, timeout time.Duration) bool { + waitErr := make(chan error, 1) + go func() { waitErr <- cmd.Wait() }() + select { + case <-waitErr: + return true + case <-time.After(timeout): + return false + } +} + +// TestStrayProcessHelperFixture is re-executed (copied to an interesting +// binary name) by TestTerminateStrayWindowsProcessesKillsMatchingByName; it +// otherwise no-ops under `go test`. +func TestStrayProcessHelperFixture(_ *testing.T) { + if os.Getenv("DEVSY_PODMAN_STRAY_HELPER") != "1" { + return + } + time.Sleep(30 * time.Second) +} + +func copySelfAs(t *testing.T, name string) string { + t.Helper() + data, err := os.ReadFile(os.Args[0]) + if err != nil { + t.Fatal(err) + } + dest := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(dest, data, 0o700); err != nil { + t.Fatal(err) + } + return dest +}