Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/pr-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ jobs:
- 'e2e/**'
- '.github/workflows/pr-ci.yml'
- 'hack/licenses/**'
- 'hack/ci/setup-podman-windows*.ps1'
- 'THIRD_PARTY_LICENSES.md'

precommit:
Expand Down
138 changes: 116 additions & 22 deletions hack/ci/setup-podman-windows.Tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ function Assert-True($condition, $label) {
}

$realRunner = ${function:Invoke-BoundedCommand}
$realStopper = ${function:Stop-TimedOutProcess}
$completed = & $realRunner $pwshPath @('-NoProfile', '-Command', 'Write-Output short-command-complete') ([TimeSpan]::FromSeconds(4))
Assert-True (-not $completed.TimedOut -and $completed.ExitCode -eq 0 -and $completed.Output -match 'short-command-complete') 'short command did not complete inside its timeout'

Expand All @@ -24,31 +25,36 @@ $null = New-Item -ItemType Directory -Path $descendantTemp
$parentScript = Join-Path $descendantTemp 'parent.ps1'
$childPidFile = Join-Path $descendantTemp 'child.pid'
@'
$child = Start-Process -FilePath (Join-Path $PSHOME 'pwsh.exe') -ArgumentList @('-NoProfile', '-Command', 'Start-Sleep -Seconds 30') -PassThru
$child.Id | Set-Content -Path $args[0]
$pwshPath = if ($IsWindows) { Join-Path $PSHOME 'pwsh.exe' } else { Join-Path $PSHOME 'pwsh' }
$child = Start-Process -FilePath $pwshPath -ArgumentList @('-NoProfile', '-Command', 'Start-Sleep -Seconds 30') -PassThru
$child.Id | Set-Content -Path "$($args[0]).tmp"
Move-Item -Path "$($args[0]).tmp" -Destination $args[0]
Start-Sleep -Seconds 30
'@ | Set-Content -Path $parentScript
$descendantResult = $null
$parentProcess = $null
$childPid = $null
try {
$started = Get-Date
$descendantResult = & $realRunner $pwshPath @('-NoProfile', '-File', $parentScript, $childPidFile) ([TimeSpan]::FromMilliseconds(1000))
$elapsedSeconds = ((Get-Date) - $started).TotalSeconds
Assert-True ($descendantResult.TimedOut -and $elapsedSeconds -lt 5) 'parent with a sleeping descendant exceeded the timeout ceiling'
Assert-True ($null -eq (Get-Process -Id $descendantResult.ProcessId -ErrorAction SilentlyContinue)) 'timed-out parent process remained alive'
$deadline = (Get-Date).AddSeconds(2)
$parentProcess = Start-Process -FilePath $pwshPath -ArgumentList @('-NoProfile', '-File', "`"$parentScript`"", "`"$childPidFile`"") -PassThru
$deadline = (Get-Date).AddSeconds(15)
while (-not (Test-Path $childPidFile) -and (Get-Date) -lt $deadline) { Start-Sleep -Milliseconds 50 }
Assert-True (Test-Path $childPidFile) 'descendant did not record its process id'
Assert-True (Test-Path $childPidFile) 'descendant readiness timed out before termination test'
$childPid = [int](Get-Content $childPidFile -Raw)
Assert-True ($null -ne (Get-Process -Id $childPid -ErrorAction SilentlyContinue)) 'timeout handler synchronously killed the descendant'
Assert-True ($null -ne (Get-Process -Id $childPid -ErrorAction SilentlyContinue)) 'ready descendant exited before termination test'
Assert-True (-not $parentProcess.WaitForExit(500)) 'parent exited before timeout handler was exercised'
$started = Get-Date
$stopped = & $realStopper $parentProcess ([TimeSpan]::FromSeconds(1))
Assert-True ($stopped -and ((Get-Date) - $started).TotalSeconds -lt 3) 'direct parent termination exceeded its bounded grace'
Assert-True ($null -eq (Get-Process -Id $parentProcess.Id -ErrorAction SilentlyContinue)) 'directly stopped parent process remained alive'
Assert-True ($null -ne (Get-Process -Id $childPid -ErrorAction SilentlyContinue)) 'direct process stopper synchronously killed its descendant'
} finally {
if ($parentProcess -and -not $parentProcess.HasExited) { Stop-Process -Id $parentProcess.Id -Force -ErrorAction SilentlyContinue }
if (-not $childPid -and (Test-Path $childPidFile)) { $childPid = [int](Get-Content $childPidFile -Raw) }
if ($childPid -and (Get-Process -Id $childPid -ErrorAction SilentlyContinue)) {
Stop-Process -Id $childPid -Force -ErrorAction SilentlyContinue
}
Remove-Item $descendantTemp -Recurse -Force -ErrorAction SilentlyContinue
}

$realStopper = ${function:Stop-TimedOutProcess}
function Stop-TimedOutProcess {
param([Diagnostics.Process]$Process, [TimeSpan]$Grace)
throw 'simulated termination failure'
Expand All @@ -66,7 +72,11 @@ function Invoke-BoundedCommand {
$command = $Arguments -join ' '
$script:calls.Add($command)
$script:timeouts.Add($Timeout.TotalMilliseconds)
if ($script:durations.ContainsKey($command)) {
if ($script:durationQueues.ContainsKey($command) -and $script:durationQueues[$command].Count -gt 0) {
$duration = $script:durationQueues[$command].Dequeue()
$script:elapsedMilliseconds += [Math]::Min($duration, $Timeout.TotalMilliseconds)
if ($duration -gt $Timeout.TotalMilliseconds) { return @{ TimedOut = $true; ExitCode = $null; Output = 'timed out' } }
} elseif ($script:durations.ContainsKey($command)) {
$duration = $script:durations[$command]
$script:elapsedMilliseconds += [Math]::Min($duration, $Timeout.TotalMilliseconds)
if ($duration -gt $Timeout.TotalMilliseconds) {
Expand Down Expand Up @@ -99,6 +109,7 @@ function Set-Scenario {
$script:calls = [Collections.Generic.List[string]]::new()
$script:timeouts = [Collections.Generic.List[double]]::new()
$script:durations = @{}
$script:durationQueues = @{}
$script:overruns = @{}
$script:results = @{}
$script:elapsedMilliseconds = 0
Expand Down Expand Up @@ -140,6 +151,7 @@ Invoke-PodmanBootstrap (New-TestBudget)
Assert-Equal ($calls | Where-Object { $_ -eq 'machine start' }).Count 2 'recovery starts'
Assert-Equal ($calls | Where-Object { $_ -eq 'machine rm -f' }).Count 1 'recovery reset'
Assert-Equal ($calls | Where-Object { $_ -eq '--unregister podman-machine-default' }).Count 1 'exact stale distribution cleanup'
Assert-True ($script:elapsedMilliseconds -le 270000) 'recovery scenario exceeded the shared bootstrap budget'

Set-Scenario
Add-Result 'machine start' $failed
Expand All @@ -160,35 +172,52 @@ Assert-Equal $timeouts[0] 10000 'command did not preserve terminal diagnostic bu
Assert-True (($calls | Where-Object { $_ -eq 'version' }).Count -eq 1) 'exhausted-budget diagnostics missing'

Set-Scenario
$script:elapsedMilliseconds = 100001
$script:durations['machine init'] = 100000
$script:durations['version'] = 1000
try {
Invoke-PodmanBootstrap (New-TestBudget 75000)
throw 'expected near-deadline failure'
Invoke-PodmanBootstrap (New-TestBudget)
throw 'expected insufficient init budget failure'
} catch {
Assert-True ($_.Exception.Message -match '^PODMAN_WINDOWS_RECOVERY_FAILED: PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT') 'near-deadline marker missing'
Assert-True ($_.Exception.Message -match '^PODMAN_WINDOWS_RECOVERY_FAILED: PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT') 'insufficient init marker missing'
}
Assert-Equal $timeouts[1] 50000 'near-deadline command was not capped by remaining budget'
Assert-True (($calls | Where-Object { $_ -eq 'version' }).Count -eq 1) 'terminal diagnostics did not run'
Assert-Equal (($calls | Where-Object { $_ -eq 'machine init' }).Count) 1 'init timeout was not capped to available first-attempt budget'
Assert-Equal $timeouts[$calls.IndexOf('machine init')] 84999 'first init did not reserve diagnostics and recovery time'
Assert-True (($calls | Where-Object { $_ -eq 'version' }).Count -ge 1) 'terminal diagnostics did not run'
Assert-True (($calls | Where-Object { $_ -eq 'machine stop' }).Count -eq 0) 'recovery ran without enough time'

Set-Scenario
try {
Invoke-PodmanBootstrap (New-TestBudget 84999)
throw 'expected insufficient first init budget failure'
} catch {
Assert-True ($_.Exception.Message -match '^PODMAN_WINDOWS_RECOVERY_FAILED: PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT') 'no-init budget marker missing'
}
Assert-Equal (($calls | Where-Object { $_ -eq 'machine init' }).Count) 0 'init launched without any time after required reserves'

Set-Scenario
Add-Result 'machine start' $failed
$script:elapsedMilliseconds = 155000
$script:durations['machine start'] = 100000
$script:durations['machine init'] = 150000
$script:durations['machine set --rootful'] = 20000
$script:durations['machine start'] = 45000
try {
Invoke-PodmanBootstrap (New-TestBudget)
throw 'expected insufficient retry budget failure'
} catch {
Assert-True ($_.Exception.Message -match '^PODMAN_WINDOWS_RECOVERY_FAILED:') 'insufficient retry marker missing'
}
Assert-Equal ($calls | Where-Object { $_ -eq 'machine init' }).Count 1 'slow init was not allowed to complete'
Assert-Equal ($timeouts[1]) 150000 'first init cap changed'
Assert-Equal ($calls | Where-Object { $_ -eq 'machine start' }).Count 1 'insufficient-budget retry'
Assert-True (($calls | Where-Object { $_ -eq 'version' }).Count -eq 1) 'insufficient-budget diagnostics missing'
$versionIndex = $calls.IndexOf('version')
Assert-Equal $timeouts[$versionIndex] 3000 'terminal diagnostics did not use the reserved budget'
Assert-True ($timeouts[$versionIndex] -le 3000) 'terminal diagnostics exceeded the remaining diagnostic budget'

Set-Scenario
Add-Result 'machine start' $failed
$script:elapsedMilliseconds = 170000
$script:durations['machine init'] = 70000
$script:durations['machine set --rootful'] = 20000
$script:durations['machine start'] = 50000
$script:durations['machine stop'] = 20000
$script:durations['machine rm -f'] = 20000
$script:durations['--shutdown'] = 20000
Expand All @@ -202,6 +231,71 @@ try {
Assert-Equal ($calls | Where-Object { $_ -eq 'machine start' }).Count 1 'second attempt began after cleanup consumed its budget'
Assert-Equal ($calls | Where-Object { $_ -eq 'version' }).Count 2 'post-cleanup terminal diagnostics missing'

Set-Scenario
$script:durations['machine init'] = 110000
$script:durations['machine set --rootful'] = 20000
$script:durations['machine start'] = 20000
Invoke-PodmanBootstrap (New-TestBudget)
Assert-Equal ($calls | Where-Object { $_ -eq 'machine init' }).Count 1 'slow successful init count'
Assert-True ($timeouts[1] -ge 110000) 'slow successful init was not allowed to exceed 90 seconds'

Assert-True (-not (Test-MachineAbsentResult @{ TimedOut = $true; ExitCode = $null; Output = 'podman-machine-default: VM does not exist' })) 'timed-out machine removal was treated as confirmed absence'
Assert-True (-not (Test-MachineAbsentResult @{ TimedOut = $false; ExitCode = 1; Output = 'another-machine: VM does not exist' })) 'unrelated machine absence was accepted'
Assert-True (-not (Test-MachineAbsentResult @{ TimedOut = $false; ExitCode = 1; Output = "podman-machine-default: VM does not exist`npipe busy" })) 'mixed cleanup failure was swallowed as machine absence'
Assert-True (Test-MachineAbsentResult @{ TimedOut = $false; ExitCode = 125; Output = 'Error: podman-machine-default: VM does not exist' }) 'exact machine absence was not recognized'

Set-Scenario
$script:durationQueues['machine init'] = [Collections.Generic.Queue[double]]::new()
$script:durationQueues['machine init'].Enqueue(151000)
$script:durationQueues['machine init'].Enqueue(20000)
$absent = @{ TimedOut = $false; ExitCode = 125; Output = 'Error: podman-machine-default: VM does not exist' }
Add-Result 'machine inspect' $absent
Add-Result 'machine stop' $absent
Add-Result 'machine rm -f' $absent
$script:distributionList = "Ubuntu`npodman-machine-default`npodman-machine-default-other"
Invoke-PodmanBootstrap (New-TestBudget)
Assert-Equal ($calls | Where-Object { $_ -eq 'machine init' }).Count 2 'retry did not perform a fresh init after timed-out init cleanup'
Assert-Equal ($calls | Where-Object { $_ -eq '--unregister podman-machine-default' }).Count 1 'timed-out init recovery did not unregister the exact stale distribution'
Assert-Equal $timeouts[$calls.IndexOf('machine init', 2)] 95000 'second init did not receive its fresh remaining budget'
Assert-Equal ($calls | Where-Object { $_ -like '--unregister*' }).Count 1 'partial-init recovery unregistered an unrelated distribution'
$cleanupIndex = $calls.IndexOf('machine stop')
Assert-Equal (($calls.GetRange($cleanupIndex, 9)) -join ',') 'machine stop,machine rm -f,--shutdown,--list --quiet,--unregister podman-machine-default,machine init,machine set --rootful,machine start,info' 'partial-init recovery sequence'
Assert-True ($script:elapsedMilliseconds -le 270000) 'partial-init recovery exceeded the shared bootstrap budget'

foreach ($cleanupCommand in @('machine stop', 'machine rm -f', '--shutdown', '--list --quiet', '--unregister podman-machine-default')) {
Set-Scenario
Add-Result 'machine init' @{ TimedOut = $true; ExitCode = $null; Output = 'timed out' }
Add-Result $cleanupCommand $failed
$script:distributionList = 'podman-machine-default'
try {
Invoke-PodmanBootstrap (New-TestBudget)
throw "expected unexpected cleanup failure for $cleanupCommand"
} catch {
Assert-True ($_.Exception.Message -match '^PODMAN_WINDOWS_RECOVERY_FAILED: recovery cleanup command failed') "unexpected cleanup failure was accepted for $cleanupCommand"
}
Assert-Equal ($calls | Where-Object { $_ -eq 'machine init' }).Count 1 "retry began after failed $cleanupCommand"
}

Set-Scenario
Add-Result 'machine start' $failed
$script:durations['machine init'] = 150000
$script:durations['machine set --rootful'] = 20000
$script:durations['machine start'] = 5000
$script:durations['machine stop'] = 10000
$script:durations['machine rm -f'] = 10000
$script:durations['--shutdown'] = 10000
$script:durations['--list --quiet'] = 10000
$script:distributionList = "Ubuntu`npodman-machine-default"
try {
Invoke-PodmanBootstrap (New-TestBudget)
throw 'expected cleanup budget exhaustion'
} catch {
Assert-True ($_.Exception.Message -match '^PODMAN_WINDOWS_RECOVERY_FAILED:') 'cleanup budget failure marker missing'
}
Assert-Equal ($calls | Where-Object { $_ -eq 'machine stop' }).Count 1 'cleanup-budget scenario did not enter recovery'
Assert-Equal ($calls | Where-Object { $_ -eq 'machine init' }).Count 1 'second init began after cleanup consumed the shared budget'
Assert-True ($script:elapsedMilliseconds -le 270000) 'cleanup-budget scenario exceeded the shared bootstrap budget'

Set-Scenario
Add-Result 'machine start' $failed
Add-Result 'machine start' $failed
Expand Down
24 changes: 19 additions & 5 deletions hack/ci/setup-podman-windows.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ param(
)

$ErrorActionPreference = 'Stop'
$initTimeout = [TimeSpan]::FromSeconds(90)
$initTimeout = [TimeSpan]::FromSeconds(150)
$startTimeout = [TimeSpan]::FromSeconds(90)
$readinessTimeout = [TimeSpan]::FromSeconds(45)
$probeTimeout = [TimeSpan]::FromSeconds(7)
Expand Down Expand Up @@ -149,12 +149,19 @@ function Write-MachineDiagnostics {
function Start-MachineAttempt {
param([int]$Attempt, [hashtable]$Budget)

$initReserve = $diagnosticReserve
if ($Attempt -eq 1) { $initReserve += $minimumRecoveryTime }
$initRemaining = Get-BudgetRemaining $Budget $initReserve
if ($initRemaining.TotalMilliseconds -le 0) {
return "PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT attempt=$Attempt output=PODMAN_WINDOWS_BOOTSTRAP_BUDGET_EXHAUSTED: insufficient initialization budget"
}
$effectiveInitTimeout = if ($initRemaining -lt $initTimeout) { $initRemaining } else { $initTimeout }
foreach ($entry in @(
@{ Args = @('machine', 'init'); Timeout = $initTimeout },
@{ Args = @('machine', 'set', '--rootful'); Timeout = [TimeSpan]::FromSeconds(20) }
@{ Args = @('machine', 'init'); Timeout = $effectiveInitTimeout; Reserve = $initReserve },
@{ Args = @('machine', 'set', '--rootful'); Timeout = [TimeSpan]::FromSeconds(20); Reserve = $diagnosticReserve }
)) {
$machineArgs = $entry.Args
$result = Invoke-BudgetedCommand $Budget $PodmanPath $machineArgs $entry.Timeout $diagnosticReserve
$result = Invoke-BudgetedCommand $Budget $PodmanPath $machineArgs $entry.Timeout $entry.Reserve
if ($machineArgs[1] -eq 'init' -and $result.TimedOut) {
return "PODMAN_WINDOWS_MACHINE_INIT_TIMEOUT attempt=$Attempt output=$($result.Output)"
}
Expand Down Expand Up @@ -192,6 +199,12 @@ function Start-MachineAttempt {
return "PODMAN_WINDOWS_READINESS_TIMEOUT attempt=$Attempt last=$($result.Output)"
}

function Test-MachineAbsentResult {
param([hashtable]$Result)

return (-not $Result.TimedOut -and $Result.ExitCode -ne 0 -and $Result.Output -match '^\s*(?:Error:\s*)?podman-machine-default: VM does not exist\s*$')
}

function Remove-StalePodmanDistribution {
param([hashtable]$Budget)

Expand Down Expand Up @@ -243,7 +256,8 @@ function Invoke-PodmanBootstrap {
)) {
$result = Invoke-BudgetedCommand $Budget $entry.Path $entry.Args ([TimeSpan]::FromSeconds(10)) $diagnosticReserve
Write-Host "[podman-windows] reset $($entry.Path) $($entry.Args -join ' '): exit=$($result.ExitCode) timeout=$($result.TimedOut) $($result.Output)"
if ($result.TimedOut -or $result.ExitCode -ne 0) { $recoveryFailed = $true }
$machineAbsent = $entry.Path -eq $PodmanPath -and (Test-MachineAbsentResult $result)
if (($result.TimedOut -or $result.ExitCode -ne 0) -and -not $machineAbsent) { $recoveryFailed = $true }
}
if (-not (Remove-StalePodmanDistribution $Budget)) { $recoveryFailed = $true }
if ($recoveryFailed) {
Expand Down
Loading