Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 108 additions & 25 deletions hack/podman_ci/setup_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,20 @@ func runtimeGOOS() string { return goosWindows }
func platformDefaultPodmanPath() string { return `C:\Program Files\RedHat\Podman\podman.exe` }

const (
commandWSL = "wsl.exe"
commandMachine = "machine"
machineInit = "init"
machineInspect = "inspect"
machineStart = "start"
windowsInitTimeout = 150 * time.Second
windowsStartTimeout = 90 * time.Second
windowsReadinessTimeout = 45 * time.Second
windowsProbeTimeout = 7 * time.Second
windowsDiagnosticReserve = 25 * time.Second
windowsMinimumRecoveryTime = 60 * time.Second
windowsHardDeadlinePadding = 10 * time.Second
commandWSL = "wsl.exe"
commandMachine = "machine"
machineInit = "init"
machineInspect = "inspect"
machineStart = "start"
windowsInitTimeout = 150 * time.Second
windowsStartTimeout = 90 * time.Second
windowsReadinessTimeout = 45 * time.Second
windowsProbeTimeout = 7 * time.Second
windowsDiagnosticReserve = 25 * time.Second
windowsMinimumRecoveryTime = 60 * time.Second
windowsHardDeadlinePadding = 10 * time.Second
windowsHardDeadlineDiagnosticsGrace = 2 * time.Second
maxTransientPullRetries = 2
)

func platformSetup(ctx context.Context, deps Dependencies, cfg SetupConfig) error {
Expand All @@ -43,7 +45,17 @@ func setupWindows(ctx context.Context, deps Dependencies, cfg SetupConfig) error
return err
}
state := &windowsSetup{ctx: ctx, deps: deps, budget: budget, cfg: cfg}
return state.run()
runErr := state.run()
if runErr != nil {
// Per-command timeouts deliberately leave descendants (podman,
// gvproxy, win-sshproxy, wsl helpers) running so recovery and
// readiness polling can observe a still-booting machine. Once setup
// gives up for good, nothing else reaps them, so sweep them here to
// guarantee the step can exit promptly instead of leaving processes
// that may outlive our own bootstrap budget.
terminateStrayProcessesHook(deps.Log)
}
return runErr
}

type windowsSetup struct {
Expand Down Expand Up @@ -119,19 +131,11 @@ func (s *windowsSetup) initializeMachine(attempt int) error {
if attempt == 1 {
reserve += windowsMinimumRecoveryTime
}
initTimeout := s.budget.Bound(windowsInitTimeout, reserve)
if initTimeout <= 0 {
return fmt.Errorf(
"PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT attempt=%d output=%s",
attempt,
"PODMAN_WINDOWS_BOOTSTRAP_BUDGET_EXHAUSTED: insufficient initialization budget",
)
}
args := []string{commandMachine, machineInit}
result := s.runCommand(
CommandSpec{Path: s.cfg.PodmanPath, Args: args, Timeout: initTimeout},
reserve,
)
result, err := s.runMachineInit(attempt, args, reserve)
if err != nil {
return err
}
if result.TimedOut {
return fmt.Errorf(
"PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT attempt=%d %s",
Expand All @@ -152,6 +156,69 @@ func (s *windowsSetup) initializeMachine(attempt int) error {
)
}

// runMachineInit runs `podman machine init`, retrying a bounded number of
// times when the failure looks like a transient registry pull error (e.g. a
// truncated read on the machine-os image blob) instead of immediately
// burning a full attempt/recovery cycle.
func (s *windowsSetup) runMachineInit(
attempt int,
args []string,
reserve time.Duration,
) (CommandResult, error) {
for pullAttempt := 1; ; pullAttempt++ {
initTimeout := s.budget.Bound(windowsInitTimeout, reserve)
if initTimeout <= 0 {
return CommandResult{}, fmt.Errorf(
"PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT attempt=%d output=%s",
attempt,
"PODMAN_WINDOWS_BOOTSTRAP_BUDGET_EXHAUSTED: insufficient initialization budget",
)
}
result := s.runCommand(
CommandSpec{Path: s.cfg.PodmanPath, Args: args, Timeout: initTimeout},
reserve,
)
if s.shouldStopInitRetry(result, pullAttempt, reserve) {
return result, nil
}
s.deps.Log.Printf(
"[podman-windows] machine_init_transient_pull_retry attempt=%d pull_attempt=%d",
attempt,
pullAttempt,
)
}
}

func (s *windowsSetup) shouldStopInitRetry(
result CommandResult,
pullAttempt int,
reserve time.Duration,
) bool {
return !isTransientPullError(result.Output()) ||
pullAttempt > maxTransientPullRetries ||
s.budget.Remaining(reserve) < windowsMinimumRecoveryTime
}

// isTransientPullError reports whether a failed `podman machine init`
// output looks like a transient registry/network error pulling the
// machine-os image, rather than a durable configuration or environment
// failure. Observed in CI: "writing blob: happened during read: unexpected
// EOF" from a truncated image layer read.
func isTransientPullError(output string) bool {
markers := []string{
"happened during read: unexpected EOF",
"connection reset by peer",
"TLS handshake timeout",
"i/o timeout",
}
for _, marker := range markers {
if strings.Contains(output, marker) {
return true
}
}
return false
}

func (s *windowsSetup) verifyExistingMachine(attempt int) error {
args := []string{commandMachine, machineInspect}
result := s.runCommand(
Expand Down Expand Up @@ -407,6 +474,22 @@ func armWatchdog(timeout time.Duration, log *Logger) watchdog {
defer timer.Stop()
select {
case <-timer.C:
// Diagnostics/cleanup write to the same stdout the stalled setup
// path may already be blocked on; never let them stall the forced
// exit itself, so bound the attempt and terminate unconditionally.
done := make(chan struct{})
go func() {
defer close(done)
log.Printf(
"[podman-windows] hard_deadline_exceeded timeout_ms=%d",
timeout.Milliseconds(),
)
terminateStrayProcessesHook(log)
}()
select {
case <-done:
case <-time.After(windowsHardDeadlineDiagnosticsGrace):
}
_ = windows.TerminateProcess(windows.CurrentProcess(), 124)
os.Exit(124)
case <-watch.stop:
Expand Down
60 changes: 60 additions & 0 deletions hack/podman_ci/setup_windows_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,13 +100,26 @@ func windowsDependencies(r *windowsFakeRunner, clock *fakeClock) Dependencies {

func windowsConfig(t *testing.T) SetupConfig {
t.Helper()
stubStrayProcessSweep(t)
podman := filepath.Join(t.TempDir(), "podman.exe")
if err := os.WriteFile(podman, []byte(""), 0o700); err != nil {
t.Fatal(err)
}
return SetupConfig{Mode: ModeRootful, PodmanPath: podman, BootstrapTimeout: 270 * time.Second}
}

// stubStrayProcessSweep replaces the real system-wide process sweep with a
// no-op for the duration of the test: fake-runner tests exercise setup
// failure paths that call terminateStrayProcessesHook, and letting that
// reach the real OS-level kill could terminate unrelated podman/wsl
// processes on a developer's machine.
func stubStrayProcessSweep(t *testing.T) {
t.Helper()
original := terminateStrayProcessesHook
terminateStrayProcessesHook = func(*Logger) {}
t.Cleanup(func() { terminateStrayProcessesHook = original })
}

func TestWindowsHealthyBootstrapAndExistingMachine(t *testing.T) {
clock := &fakeClock{now: time.Unix(1, 0)}
runner := &windowsFakeRunner{clock: clock, results: map[string][]CommandResult{
Expand Down Expand Up @@ -225,6 +238,47 @@ func TestWindowsPartialInitRecoveryRetriesWithFreshBudget(t *testing.T) {
}
}

func TestWindowsMachineInitRetriesTransientPullError(t *testing.T) {
clock := &fakeClock{now: time.Unix(1, 0)}
runner := &windowsFakeRunner{clock: clock, results: map[string][]CommandResult{
testMachineInit: {
failureResult("Error: writing blob: happened during read: unexpected EOF"),
successResult(""),
},
}, durations: map[string]time.Duration{}}
if err := setupWindows(
context.Background(),
windowsDependencies(runner, clock),
windowsConfig(t),
); err != nil {
t.Fatal(err)
}
if countWindowsCalls(runner, testMachineInit) != 2 {
t.Fatalf("transient pull error did not retry machine init: %+v", runner.calls)
}
if countWindowsCalls(runner, "machine stop") != 0 {
t.Fatalf("transient pull retry should not trigger full recovery: %+v", runner.calls)
}
}

func TestWindowsMachineInitTransientPullRetryIsBounded(t *testing.T) {
clock := &fakeClock{now: time.Unix(1, 0)}
transientFailure := failureResult("Error: writing blob: happened during read: unexpected EOF")
runner := &windowsFakeRunner{clock: clock, results: map[string][]CommandResult{
testMachineInit: {
transientFailure, transientFailure, transientFailure,
transientFailure, transientFailure, transientFailure,
},
}, durations: map[string]time.Duration{}}
err := setupWindows(context.Background(), windowsDependencies(runner, clock), windowsConfig(t))
if err == nil || !strings.Contains(err.Error(), "unexpected EOF") {
t.Fatalf("expected terminal transient-pull failure, got %v", err)
}
if countWindowsCalls(runner, testMachineInit) != (maxTransientPullRetries+1)*2 {
t.Fatalf("expected bounded retries across both attempts: %+v", runner.calls)
}
}

func TestWindowsUnexpectedCleanupFailurePreventsRetry(t *testing.T) {
failedCommands := []string{
"machine stop", "machine rm -f", "--shutdown", testWSLList,
Expand Down Expand Up @@ -322,6 +376,11 @@ func TestWindowsWatchdogTerminatesAndCanBeDisarmed(t *testing.T) {
}

func runWatchdogChild(mode string) {
// This process is a fresh re-exec of the test binary, so package vars
// are back to their defaults; neutralize the real system-wide sweep so
// the watchdog firing in these tests cannot kill unrelated podman/wsl
// processes on the host.
terminateStrayProcessesHook = func(*Logger) {}
if mode == "blocked-stderr" {
reader, stderr, err := os.Pipe()
if err != nil {
Expand Down Expand Up @@ -362,6 +421,7 @@ func verifyWatchdogExpires(t *testing.T) {
t.Fatalf("blocked watchdog exit=%v output=%s", err, output)
}
if !strings.Contains(string(output), "hard_deadline_armed") ||
!strings.Contains(string(output), "hard_deadline_exceeded") ||
strings.Contains(string(output), "PODMAN_WINDOWS_BOOTSTRAP_HARD_TIMEOUT") {
t.Fatalf("watchdog markers missing: %s", output)
}
Expand Down
46 changes: 46 additions & 0 deletions hack/podman_ci/windows_processes.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,49 @@ func isInterestingWindowsProcess(name string) bool {
return false
}
}

// terminateStrayProcessesHook is the seam setupWindows and armWatchdog call
// through; tests override it to avoid sweeping real podman/wsl/gvproxy/
// win-sshproxy processes that may be running on a developer's machine for
// unrelated work.
var terminateStrayProcessesHook = terminateStrayWindowsProcesses

// terminateStrayWindowsProcesses forcefully terminates any Podman/WSL helper
// processes still running system-wide (podman, wsl, gvproxy, win-sshproxy).
// It is called when devsy-ci-podman.exe is about to exit after a terminal
// failure (including the hard watchdog deadline) so that no descendant
// process is left behind holding handles or keeping a VM alive past our own
// bootstrap budget: per-command timeouts deliberately leave descendants
// running to let recovery/readiness polling observe a still-booting
// machine, but once setup gives up for good there is no further attempt to
// reap them.
func terminateStrayWindowsProcesses(log *Logger) {
terminateWindowsProcesses(matchingWindowsProcesses(), log)
}

// terminateWindowsProcesses force-kills exactly the given processes. It is
// split out from terminateStrayWindowsProcesses so tests can exercise the
// real OpenProcess/TerminateProcess kill mechanics against a single
// known-owned helper process without sweeping the whole host.
func terminateWindowsProcesses(processes []processInfo, log *Logger) {
for _, process := range processes {
handle, err := windows.OpenProcess(windows.PROCESS_TERMINATE, false, process.PID)
if err != nil {
log.Printf(
"[podman-windows] stray_process_open_failed pid=%d name=%s err=%v",
process.PID,
process.Name,
err,
)
continue
}
killErr := windows.TerminateProcess(handle, 124)
_ = windows.CloseHandle(handle)
log.Printf(
"[podman-windows] stray_process_terminated pid=%d name=%s err=%v",
process.PID,
process.Name,
killErr,
)
}
}
Loading
Loading