diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 6223d8d588..b9b2e9a7b9 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -32,6 +32,7 @@ jobs: - 'e2e/**' - '.github/workflows/pr-ci.yml' - 'hack/licenses/**' + - 'hack/ci/setup-podman-windows*.ps1' - 'THIRD_PARTY_LICENSES.md' precommit: diff --git a/hack/ci/setup-podman-windows.Tests.ps1 b/hack/ci/setup-podman-windows.Tests.ps1 index 3c579253d5..f41293c9cf 100644 --- a/hack/ci/setup-podman-windows.Tests.ps1 +++ b/hack/ci/setup-podman-windows.Tests.ps1 @@ -11,6 +11,7 @@ function Assert-True($condition, $label) { } $realRunner = ${function:Invoke-BoundedCommand} +$realStopper = ${function:Stop-TimedOutProcess} $completed = & $realRunner $pwshPath @('-NoProfile', '-Command', 'Write-Output short-command-complete') ([TimeSpan]::FromSeconds(4)) Assert-True (-not $completed.TimedOut -and $completed.ExitCode -eq 0 -and $completed.Output -match 'short-command-complete') 'short command did not complete inside its timeout' @@ -24,31 +25,36 @@ $null = New-Item -ItemType Directory -Path $descendantTemp $parentScript = Join-Path $descendantTemp 'parent.ps1' $childPidFile = Join-Path $descendantTemp 'child.pid' @' -$child = Start-Process -FilePath (Join-Path $PSHOME 'pwsh.exe') -ArgumentList @('-NoProfile', '-Command', 'Start-Sleep -Seconds 30') -PassThru -$child.Id | Set-Content -Path $args[0] +$pwshPath = if ($IsWindows) { Join-Path $PSHOME 'pwsh.exe' } else { Join-Path $PSHOME 'pwsh' } +$child = Start-Process -FilePath $pwshPath -ArgumentList @('-NoProfile', '-Command', 'Start-Sleep -Seconds 30') -PassThru +$child.Id | Set-Content -Path "$($args[0]).tmp" +Move-Item -Path "$($args[0]).tmp" -Destination $args[0] Start-Sleep -Seconds 30 '@ | Set-Content -Path $parentScript -$descendantResult = $null +$parentProcess = $null $childPid = $null try { - $started = Get-Date - $descendantResult = & $realRunner $pwshPath @('-NoProfile', '-File', $parentScript, $childPidFile) ([TimeSpan]::FromMilliseconds(1000)) - $elapsedSeconds = ((Get-Date) - $started).TotalSeconds - Assert-True ($descendantResult.TimedOut -and $elapsedSeconds -lt 5) 'parent with a sleeping descendant exceeded the timeout ceiling' - Assert-True ($null -eq (Get-Process -Id $descendantResult.ProcessId -ErrorAction SilentlyContinue)) 'timed-out parent process remained alive' - $deadline = (Get-Date).AddSeconds(2) + $parentProcess = Start-Process -FilePath $pwshPath -ArgumentList @('-NoProfile', '-File', "`"$parentScript`"", "`"$childPidFile`"") -PassThru + $deadline = (Get-Date).AddSeconds(15) while (-not (Test-Path $childPidFile) -and (Get-Date) -lt $deadline) { Start-Sleep -Milliseconds 50 } - Assert-True (Test-Path $childPidFile) 'descendant did not record its process id' + Assert-True (Test-Path $childPidFile) 'descendant readiness timed out before termination test' $childPid = [int](Get-Content $childPidFile -Raw) - Assert-True ($null -ne (Get-Process -Id $childPid -ErrorAction SilentlyContinue)) 'timeout handler synchronously killed the descendant' + Assert-True ($null -ne (Get-Process -Id $childPid -ErrorAction SilentlyContinue)) 'ready descendant exited before termination test' + Assert-True (-not $parentProcess.WaitForExit(500)) 'parent exited before timeout handler was exercised' + $started = Get-Date + $stopped = & $realStopper $parentProcess ([TimeSpan]::FromSeconds(1)) + Assert-True ($stopped -and ((Get-Date) - $started).TotalSeconds -lt 3) 'direct parent termination exceeded its bounded grace' + Assert-True ($null -eq (Get-Process -Id $parentProcess.Id -ErrorAction SilentlyContinue)) 'directly stopped parent process remained alive' + Assert-True ($null -ne (Get-Process -Id $childPid -ErrorAction SilentlyContinue)) 'direct process stopper synchronously killed its descendant' } finally { + if ($parentProcess -and -not $parentProcess.HasExited) { Stop-Process -Id $parentProcess.Id -Force -ErrorAction SilentlyContinue } + if (-not $childPid -and (Test-Path $childPidFile)) { $childPid = [int](Get-Content $childPidFile -Raw) } if ($childPid -and (Get-Process -Id $childPid -ErrorAction SilentlyContinue)) { Stop-Process -Id $childPid -Force -ErrorAction SilentlyContinue } Remove-Item $descendantTemp -Recurse -Force -ErrorAction SilentlyContinue } -$realStopper = ${function:Stop-TimedOutProcess} function Stop-TimedOutProcess { param([Diagnostics.Process]$Process, [TimeSpan]$Grace) throw 'simulated termination failure' @@ -66,7 +72,11 @@ function Invoke-BoundedCommand { $command = $Arguments -join ' ' $script:calls.Add($command) $script:timeouts.Add($Timeout.TotalMilliseconds) - if ($script:durations.ContainsKey($command)) { + if ($script:durationQueues.ContainsKey($command) -and $script:durationQueues[$command].Count -gt 0) { + $duration = $script:durationQueues[$command].Dequeue() + $script:elapsedMilliseconds += [Math]::Min($duration, $Timeout.TotalMilliseconds) + if ($duration -gt $Timeout.TotalMilliseconds) { return @{ TimedOut = $true; ExitCode = $null; Output = 'timed out' } } + } elseif ($script:durations.ContainsKey($command)) { $duration = $script:durations[$command] $script:elapsedMilliseconds += [Math]::Min($duration, $Timeout.TotalMilliseconds) if ($duration -gt $Timeout.TotalMilliseconds) { @@ -99,6 +109,7 @@ function Set-Scenario { $script:calls = [Collections.Generic.List[string]]::new() $script:timeouts = [Collections.Generic.List[double]]::new() $script:durations = @{} + $script:durationQueues = @{} $script:overruns = @{} $script:results = @{} $script:elapsedMilliseconds = 0 @@ -140,6 +151,7 @@ Invoke-PodmanBootstrap (New-TestBudget) Assert-Equal ($calls | Where-Object { $_ -eq 'machine start' }).Count 2 'recovery starts' Assert-Equal ($calls | Where-Object { $_ -eq 'machine rm -f' }).Count 1 'recovery reset' Assert-Equal ($calls | Where-Object { $_ -eq '--unregister podman-machine-default' }).Count 1 'exact stale distribution cleanup' +Assert-True ($script:elapsedMilliseconds -le 270000) 'recovery scenario exceeded the shared bootstrap budget' Set-Scenario Add-Result 'machine start' $failed @@ -160,35 +172,52 @@ Assert-Equal $timeouts[0] 10000 'command did not preserve terminal diagnostic bu Assert-True (($calls | Where-Object { $_ -eq 'version' }).Count -eq 1) 'exhausted-budget diagnostics missing' Set-Scenario +$script:elapsedMilliseconds = 100001 $script:durations['machine init'] = 100000 +$script:durations['version'] = 1000 try { - Invoke-PodmanBootstrap (New-TestBudget 75000) - throw 'expected near-deadline failure' + Invoke-PodmanBootstrap (New-TestBudget) + throw 'expected insufficient init budget failure' } catch { - Assert-True ($_.Exception.Message -match '^PODMAN_WINDOWS_RECOVERY_FAILED: PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT') 'near-deadline marker missing' + Assert-True ($_.Exception.Message -match '^PODMAN_WINDOWS_RECOVERY_FAILED: PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT') 'insufficient init marker missing' } -Assert-Equal $timeouts[1] 50000 'near-deadline command was not capped by remaining budget' -Assert-True (($calls | Where-Object { $_ -eq 'version' }).Count -eq 1) 'terminal diagnostics did not run' +Assert-Equal (($calls | Where-Object { $_ -eq 'machine init' }).Count) 1 'init timeout was not capped to available first-attempt budget' +Assert-Equal $timeouts[$calls.IndexOf('machine init')] 84999 'first init did not reserve diagnostics and recovery time' +Assert-True (($calls | Where-Object { $_ -eq 'version' }).Count -ge 1) 'terminal diagnostics did not run' Assert-True (($calls | Where-Object { $_ -eq 'machine stop' }).Count -eq 0) 'recovery ran without enough time' +Set-Scenario +try { + Invoke-PodmanBootstrap (New-TestBudget 84999) + throw 'expected insufficient first init budget failure' +} catch { + Assert-True ($_.Exception.Message -match '^PODMAN_WINDOWS_RECOVERY_FAILED: PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT') 'no-init budget marker missing' +} +Assert-Equal (($calls | Where-Object { $_ -eq 'machine init' }).Count) 0 'init launched without any time after required reserves' + Set-Scenario Add-Result 'machine start' $failed -$script:elapsedMilliseconds = 155000 -$script:durations['machine start'] = 100000 +$script:durations['machine init'] = 150000 +$script:durations['machine set --rootful'] = 20000 +$script:durations['machine start'] = 45000 try { Invoke-PodmanBootstrap (New-TestBudget) throw 'expected insufficient retry budget failure' } catch { Assert-True ($_.Exception.Message -match '^PODMAN_WINDOWS_RECOVERY_FAILED:') 'insufficient retry marker missing' } +Assert-Equal ($calls | Where-Object { $_ -eq 'machine init' }).Count 1 'slow init was not allowed to complete' +Assert-Equal ($timeouts[1]) 150000 'first init cap changed' Assert-Equal ($calls | Where-Object { $_ -eq 'machine start' }).Count 1 'insufficient-budget retry' Assert-True (($calls | Where-Object { $_ -eq 'version' }).Count -eq 1) 'insufficient-budget diagnostics missing' $versionIndex = $calls.IndexOf('version') -Assert-Equal $timeouts[$versionIndex] 3000 'terminal diagnostics did not use the reserved budget' +Assert-True ($timeouts[$versionIndex] -le 3000) 'terminal diagnostics exceeded the remaining diagnostic budget' Set-Scenario Add-Result 'machine start' $failed -$script:elapsedMilliseconds = 170000 +$script:durations['machine init'] = 70000 +$script:durations['machine set --rootful'] = 20000 +$script:durations['machine start'] = 50000 $script:durations['machine stop'] = 20000 $script:durations['machine rm -f'] = 20000 $script:durations['--shutdown'] = 20000 @@ -202,6 +231,71 @@ try { Assert-Equal ($calls | Where-Object { $_ -eq 'machine start' }).Count 1 'second attempt began after cleanup consumed its budget' Assert-Equal ($calls | Where-Object { $_ -eq 'version' }).Count 2 'post-cleanup terminal diagnostics missing' +Set-Scenario +$script:durations['machine init'] = 110000 +$script:durations['machine set --rootful'] = 20000 +$script:durations['machine start'] = 20000 +Invoke-PodmanBootstrap (New-TestBudget) +Assert-Equal ($calls | Where-Object { $_ -eq 'machine init' }).Count 1 'slow successful init count' +Assert-True ($timeouts[1] -ge 110000) 'slow successful init was not allowed to exceed 90 seconds' + +Assert-True (-not (Test-MachineAbsentResult @{ TimedOut = $true; ExitCode = $null; Output = 'podman-machine-default: VM does not exist' })) 'timed-out machine removal was treated as confirmed absence' +Assert-True (-not (Test-MachineAbsentResult @{ TimedOut = $false; ExitCode = 1; Output = 'another-machine: VM does not exist' })) 'unrelated machine absence was accepted' +Assert-True (-not (Test-MachineAbsentResult @{ TimedOut = $false; ExitCode = 1; Output = "podman-machine-default: VM does not exist`npipe busy" })) 'mixed cleanup failure was swallowed as machine absence' +Assert-True (Test-MachineAbsentResult @{ TimedOut = $false; ExitCode = 125; Output = 'Error: podman-machine-default: VM does not exist' }) 'exact machine absence was not recognized' + +Set-Scenario +$script:durationQueues['machine init'] = [Collections.Generic.Queue[double]]::new() +$script:durationQueues['machine init'].Enqueue(151000) +$script:durationQueues['machine init'].Enqueue(20000) +$absent = @{ TimedOut = $false; ExitCode = 125; Output = 'Error: podman-machine-default: VM does not exist' } +Add-Result 'machine inspect' $absent +Add-Result 'machine stop' $absent +Add-Result 'machine rm -f' $absent +$script:distributionList = "Ubuntu`npodman-machine-default`npodman-machine-default-other" +Invoke-PodmanBootstrap (New-TestBudget) +Assert-Equal ($calls | Where-Object { $_ -eq 'machine init' }).Count 2 'retry did not perform a fresh init after timed-out init cleanup' +Assert-Equal ($calls | Where-Object { $_ -eq '--unregister podman-machine-default' }).Count 1 'timed-out init recovery did not unregister the exact stale distribution' +Assert-Equal $timeouts[$calls.IndexOf('machine init', 2)] 95000 'second init did not receive its fresh remaining budget' +Assert-Equal ($calls | Where-Object { $_ -like '--unregister*' }).Count 1 'partial-init recovery unregistered an unrelated distribution' +$cleanupIndex = $calls.IndexOf('machine stop') +Assert-Equal (($calls.GetRange($cleanupIndex, 9)) -join ',') 'machine stop,machine rm -f,--shutdown,--list --quiet,--unregister podman-machine-default,machine init,machine set --rootful,machine start,info' 'partial-init recovery sequence' +Assert-True ($script:elapsedMilliseconds -le 270000) 'partial-init recovery exceeded the shared bootstrap budget' + +foreach ($cleanupCommand in @('machine stop', 'machine rm -f', '--shutdown', '--list --quiet', '--unregister podman-machine-default')) { + Set-Scenario + Add-Result 'machine init' @{ TimedOut = $true; ExitCode = $null; Output = 'timed out' } + Add-Result $cleanupCommand $failed + $script:distributionList = 'podman-machine-default' + try { + Invoke-PodmanBootstrap (New-TestBudget) + throw "expected unexpected cleanup failure for $cleanupCommand" + } catch { + Assert-True ($_.Exception.Message -match '^PODMAN_WINDOWS_RECOVERY_FAILED: recovery cleanup command failed') "unexpected cleanup failure was accepted for $cleanupCommand" + } + Assert-Equal ($calls | Where-Object { $_ -eq 'machine init' }).Count 1 "retry began after failed $cleanupCommand" +} + +Set-Scenario +Add-Result 'machine start' $failed +$script:durations['machine init'] = 150000 +$script:durations['machine set --rootful'] = 20000 +$script:durations['machine start'] = 5000 +$script:durations['machine stop'] = 10000 +$script:durations['machine rm -f'] = 10000 +$script:durations['--shutdown'] = 10000 +$script:durations['--list --quiet'] = 10000 +$script:distributionList = "Ubuntu`npodman-machine-default" +try { + Invoke-PodmanBootstrap (New-TestBudget) + throw 'expected cleanup budget exhaustion' +} catch { + Assert-True ($_.Exception.Message -match '^PODMAN_WINDOWS_RECOVERY_FAILED:') 'cleanup budget failure marker missing' +} +Assert-Equal ($calls | Where-Object { $_ -eq 'machine stop' }).Count 1 'cleanup-budget scenario did not enter recovery' +Assert-Equal ($calls | Where-Object { $_ -eq 'machine init' }).Count 1 'second init began after cleanup consumed the shared budget' +Assert-True ($script:elapsedMilliseconds -le 270000) 'cleanup-budget scenario exceeded the shared bootstrap budget' + Set-Scenario Add-Result 'machine start' $failed Add-Result 'machine start' $failed diff --git a/hack/ci/setup-podman-windows.ps1 b/hack/ci/setup-podman-windows.ps1 index 1a43236683..14d5c47cda 100644 --- a/hack/ci/setup-podman-windows.ps1 +++ b/hack/ci/setup-podman-windows.ps1 @@ -5,7 +5,7 @@ param( ) $ErrorActionPreference = 'Stop' -$initTimeout = [TimeSpan]::FromSeconds(90) +$initTimeout = [TimeSpan]::FromSeconds(150) $startTimeout = [TimeSpan]::FromSeconds(90) $readinessTimeout = [TimeSpan]::FromSeconds(45) $probeTimeout = [TimeSpan]::FromSeconds(7) @@ -149,12 +149,19 @@ function Write-MachineDiagnostics { function Start-MachineAttempt { param([int]$Attempt, [hashtable]$Budget) + $initReserve = $diagnosticReserve + if ($Attempt -eq 1) { $initReserve += $minimumRecoveryTime } + $initRemaining = Get-BudgetRemaining $Budget $initReserve + if ($initRemaining.TotalMilliseconds -le 0) { + return "PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT attempt=$Attempt output=PODMAN_WINDOWS_BOOTSTRAP_BUDGET_EXHAUSTED: insufficient initialization budget" + } + $effectiveInitTimeout = if ($initRemaining -lt $initTimeout) { $initRemaining } else { $initTimeout } foreach ($entry in @( - @{ Args = @('machine', 'init'); Timeout = $initTimeout }, - @{ Args = @('machine', 'set', '--rootful'); Timeout = [TimeSpan]::FromSeconds(20) } + @{ Args = @('machine', 'init'); Timeout = $effectiveInitTimeout; Reserve = $initReserve }, + @{ Args = @('machine', 'set', '--rootful'); Timeout = [TimeSpan]::FromSeconds(20); Reserve = $diagnosticReserve } )) { $machineArgs = $entry.Args - $result = Invoke-BudgetedCommand $Budget $PodmanPath $machineArgs $entry.Timeout $diagnosticReserve + $result = Invoke-BudgetedCommand $Budget $PodmanPath $machineArgs $entry.Timeout $entry.Reserve if ($machineArgs[1] -eq 'init' -and $result.TimedOut) { return "PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT attempt=$Attempt output=$($result.Output)" } @@ -192,6 +199,12 @@ function Start-MachineAttempt { return "PODMAN_WINDOWS_READINESS_TIMEOUT attempt=$Attempt last=$($result.Output)" } +function Test-MachineAbsentResult { + param([hashtable]$Result) + + return (-not $Result.TimedOut -and $Result.ExitCode -ne 0 -and $Result.Output -match '^\s*(?:Error:\s*)?podman-machine-default: VM does not exist\s*$') +} + function Remove-StalePodmanDistribution { param([hashtable]$Budget) @@ -243,7 +256,8 @@ function Invoke-PodmanBootstrap { )) { $result = Invoke-BudgetedCommand $Budget $entry.Path $entry.Args ([TimeSpan]::FromSeconds(10)) $diagnosticReserve Write-Host "[podman-windows] reset $($entry.Path) $($entry.Args -join ' '): exit=$($result.ExitCode) timeout=$($result.TimedOut) $($result.Output)" - if ($result.TimedOut -or $result.ExitCode -ne 0) { $recoveryFailed = $true } + $machineAbsent = $entry.Path -eq $PodmanPath -and (Test-MachineAbsentResult $result) + if (($result.TimedOut -or $result.ExitCode -ne 0) -and -not $machineAbsent) { $recoveryFailed = $true } } if (-not (Remove-StalePodmanDistribution $Budget)) { $recoveryFailed = $true } if ($recoveryFailed) {