diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index 39bfebe..9ce29d1 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -1049,7 +1049,7 @@ Test-Assert ([PkState]::SourceCount -eq 0) "and nothing is left registered" # Every batch result has every documented field, so a strict-mode caller # can read any of them without a "property cannot be found" error. Reset-Harness -$fields = @('Command', 'ExitCode', 'Stdout', 'Stderr', 'StreamTimedOut', 'StreamError', 'StartError', 'Aborted') +$fields = @('Command', 'ExitCode', 'Stdout', 'Stderr', 'StreamTimedOut', 'StreamError', 'StartError', 'Aborted', 'TimedOut') $batch = Invoke-PkProcessBatch -MaxConcurrent 1 -Commands @( @{ FilePath = $cmdExe; Arguments = '/c exit 0' }, @{ FilePath = 'C:\definitely\not\here.exe' }) @@ -1065,6 +1065,34 @@ Test-Assert ($batch.Aborted -and $batch.Results.Count -eq 2 -and @($batch.Result Test-Assert (@($missing).Count -eq 0) ("and their results carry every field too (missing: " + (@($missing) -join ', ') + ")") Reset-PkAbortState +# -TimeoutSeconds kills a program that overruns; the rest run normally. +Reset-Harness +$completed = [System.Collections.Generic.List[object]]::new() +$sw = [System.Diagnostics.Stopwatch]::StartNew() +$batch = Invoke-PkProcessBatch -MaxConcurrent 2 -TimeoutSeconds 2 -OnComplete { param($c, $r) $completed.Add($r) }.GetNewClosure() -Commands @( + @{ FilePath = $pingExe; Arguments = '-n 30 127.0.0.1'; Name = 'slow' }, + @{ FilePath = $cmdExe; Arguments = '/c exit 0'; Name = 'fast' }) +$sw.Stop() +$slow = @($batch.Results | Where-Object { $_.Command['Name'] -eq 'slow' }) +$fast = @($batch.Results | Where-Object { $_.Command['Name'] -eq 'fast' }) +Test-Assert ($slow.Count -eq 1 -and $slow[0].TimedOut -and $slow[0].ExitCode -ne 0 -and $sw.Elapsed.TotalSeconds -lt 15) ("batch: -TimeoutSeconds kills an overrunning program and marks it TimedOut (" + [int]$sw.Elapsed.TotalSeconds + "s)") +Test-Assert ($fast.Count -eq 1 -and -not $fast[0].TimedOut -and $fast[0].ExitCode -eq 0 -and -not $batch.Aborted) "and the others finish normally, not TimedOut" +Test-Assert ($completed.Count -eq 2) "a timed-out program still goes through -OnComplete" +# A program whose own child holds the output open: the whole tree is killed, +# so the output closes at once instead of after the 15-second stream wait. +$sw = [System.Diagnostics.Stopwatch]::StartNew() +$batch = Invoke-PkProcessBatch -MaxConcurrent 1 -TimeoutSeconds 2 -Commands @(@{ FilePath = $cmdExe; Arguments = ('/c ""' + $pingExe + '" -n 30 127.0.0.1"') }) +$sw.Stop() +$r = @($batch.Results)[0] +Test-Assert ($r.TimedOut -and -not $r.StreamTimedOut -and $sw.Elapsed.TotalSeconds -lt 10) ("batch: a timeout kills the program's children too (" + [int]$sw.Elapsed.TotalSeconds + "s, StreamTimedOut=" + $r.StreamTimedOut + ")") +$j = Start-PkProcess -FilePath $cmdExe -Arguments '/c exit 0' +$null = Wait-PkProcess -Process $j.Proc -TimeoutMs 5000 +Test-Assert (-not (Stop-PkProcessTree -Process $j.Proc)) "Stop-PkProcessTree reports `$false for a program that already exited" +$j.Proc.Dispose() +$threw = $false +try { $null = Invoke-PkProcessBatch -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) -TimeoutSeconds -1 } catch { $threw = ($_.ToString() -like '*-TimeoutSeconds*') } +Test-Assert $threw "a negative -TimeoutSeconds is refused" + # An aborted group still gets -GroupCompleteCallback (the README warns about # this - a caller zipping a folder there must check for the abort first). Reset-Harness diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index b4aa176..efcad3e 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -559,6 +559,32 @@ function Start-PkProcess { } } +function Stop-PkProcessTree { + # Kills a process and every process it started (taskkill /T /F), so + # none of them is left holding its output open. Falls back to killing + # just the process. Returns $true if this call killed it, $false if it + # had already exited on its own. + param([Parameter(Mandatory)][System.Diagnostics.Process]$Process) + if ($Process.HasExited) { return $false } + $killed = $false + $psi = [System.Diagnostics.ProcessStartInfo]::new((Join-Path $env:SystemRoot 'System32\taskkill.exe'), ('/T /F /PID ' + $Process.Id)) + $psi.UseShellExecute = $false + $psi.CreateNoWindow = $true + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + try { + $taskkill = [System.Diagnostics.Process]::Start($psi) + try { + if ($taskkill.WaitForExit(10000)) { $killed = ($taskkill.ExitCode -eq 0) } + } finally { $taskkill.Dispose() } + } catch { } # taskkill missing or blocked: the fallback below still kills the process itself + if (-not $killed -and -not $Process.HasExited) { + try { $Process.Kill(); $killed = $true } catch { } # it exited between the check and the kill + } + if ($killed) { $null = $Process.WaitForExit(5000) } + return $killed +} + function Wait-PkProcess { # Waits on the raw process handle. Returns $true if it exited in time. # (Process.WaitForExit(int) also waits on the redirected streams and can @@ -1148,8 +1174,10 @@ function Invoke-PkProcessBatch { # Each command is @{ FilePath = ...; Arguments = ...; WorkingDirectory = ... (optional); }. # $OnComplete { param($command, $result) } fires as each one finishes # (not for one that couldn't start or was stopped by an abort). + # $TimeoutSeconds > 0 kills a program still running after that long; its + # result has TimedOut = $true (0 = no limit). # Returns @{ Results; Aborted; ForceKilledCount }; each result is - # @{ Command; ExitCode; Stdout; Stderr; StreamTimedOut; StreamError; StartError; Aborted }, + # @{ Command; ExitCode; Stdout; Stderr; StreamTimedOut; StreamError; StartError; Aborted; TimedOut }, # every field always present - keep the three places that build one in sync. # $Commands may be $null: a foreach that matches nothing yields $null, not # an empty array, and an empty batch should just return no results. @@ -1160,9 +1188,12 @@ function Invoke-PkProcessBatch { [hashtable[]]$Commands, [int]$MaxConcurrent = 4, [scriptblock]$OnComplete = $null, - [int]$AbortTimeoutSeconds = 60 + [int]$AbortTimeoutSeconds = 60, + [int]$TimeoutSeconds = 0 ) + if ($TimeoutSeconds -lt 0) { throw "Invoke-PkProcessBatch: -TimeoutSeconds must be 0 (no limit) or more." } + $results = [System.Collections.Generic.List[hashtable]]::new() $state = @{ Given = $false } @@ -1187,17 +1218,18 @@ function Invoke-PkProcessBatch { try { $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments $job['Arguments'] -WorkingDirectory $job['WorkingDirectory'] } catch { - $results.Add(@{ Command = $job['Command']; ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; StartError = [string]$_; Aborted = $false }) + $results.Add(@{ Command = $job['Command']; ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; StartError = [string]$_; Aborted = $false; TimedOut = $false }) throw } $job['Proc'] = $started.Proc; $job['StdoutTask'] = $started.StdoutTask; $job['StderrTask'] = $started.StderrTask + $job['Stopwatch'] = $started.Stopwatch; $job['TimedOut'] = $false return $job }.GetNewClosure() $jobFinalize = { param($job, $pendingQueue) $r = Complete-PkProcess -Job $job - $r.Command = $job['Command']; $r.StartError = $null; $r.Aborted = $false + $r.Command = $job['Command']; $r.StartError = $null; $r.Aborted = $false; $r.TimedOut = [bool]$job['TimedOut'] $results.Add($r) if ($null -ne $OnComplete) { $null = & $OnComplete $job['Command'] $r } return @{ Success = ($r.ExitCode -eq 0) } @@ -1205,12 +1237,28 @@ function Invoke-PkProcessBatch { $jobAbortCleanup = { param($job) - $results.Add(@{ Command = $job['Command']; ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; StartError = $null; Aborted = $true }) + $results.Add(@{ Command = $job['Command']; ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; StartError = $null; Aborted = $true; TimedOut = $false }) }.GetNewClosure() - $dispatch = Invoke-PkProcessDispatch -GroupProvider $groupProvider -JobBuilder $jobBuilder -JobStarter $jobStarter ` - -JobFinalize $jobFinalize -JobAbortCleanup $jobAbortCleanup -MaxConcurrent $MaxConcurrent ` - -AbortTimeoutSeconds $AbortTimeoutSeconds -MaxQueueAhead 1 + $dispatchParams = @{ + GroupProvider = $groupProvider; JobBuilder = $jobBuilder; JobStarter = $jobStarter + JobFinalize = $jobFinalize; JobAbortCleanup = $jobAbortCleanup; MaxConcurrent = $MaxConcurrent + AbortTimeoutSeconds = $AbortTimeoutSeconds; MaxQueueAhead = 1 + } + if ($TimeoutSeconds -gt 0) { + # Kill an overrunning program; the next poll sees it exited and it + # goes through $jobFinalize like any other, flagged TimedOut. + $dispatchParams['JobPollCompleted'] = { + param($job) + if (-not $job['Proc'].HasExited -and $job['Stopwatch'].Elapsed.TotalSeconds -ge $TimeoutSeconds) { + # Only flagged if this call did the killing: a program that + # exited on its own right at the deadline didn't time out. + if (Stop-PkProcessTree -Process $job['Proc']) { $job['TimedOut'] = $true } + } + return $job['Proc'].HasExited + }.GetNewClosure() + } + $dispatch = Invoke-PkProcessDispatch @dispatchParams return @{ Results = $results; Aborted = $dispatch.Aborted; ForceKilledCount = $dispatch.ForceKilledCount } } @@ -1221,7 +1269,7 @@ function Invoke-PkProcessBatch { function Get-PkWorkerSafeFunctionNames { # Kit functions every worker runspace gets automatically. - return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', + return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', 'Stop-PkProcessTree', 'Start-PkProcess', 'Wait-PkProcess', 'Complete-PkProcess', 'Invoke-PkProcessWithTimeout', 'Invoke-PkProcessDispatch', 'Invoke-PkProcessDispatchCore', 'Invoke-PkProcessBatch', 'Register-PkCancellationSource', 'Unregister-PkCancellationSource') diff --git a/README.md b/README.md index 8452e2d..7a02913 100644 --- a/README.md +++ b/README.md @@ -119,18 +119,21 @@ Long parameter lists in these examples use splatting (`@batchParams`) instead of Each command is a hashtable with `FilePath`, `Arguments` and optionally `WorkingDirectory`, plus any keys of your own. `Arguments` is passed to the program as one string, exactly as written, so put quotes around paths that may contain spaces (as above). Programs run hidden, with their output captured. -`Invoke-PkProcessBatch` returns `@{ Results; Aborted; ForceKilledCount }`. Each result is `@{ Command; ExitCode; Stdout; Stderr; StreamTimedOut; StreamError; StartError; Aborted }`, with every field always present: +`Invoke-PkProcessBatch` returns `@{ Results; Aborted; ForceKilledCount }`. Each result is `@{ Command; ExitCode; Stdout; Stderr; StreamTimedOut; StreamError; StartError; Aborted; TimedOut }`, with every field always present: - `Command` is your original command hashtable, so extra keys like `Source` above come back with it. - `ExitCode` is whatever the program returned. The kit doesn't treat a non-zero code as a failure; that's for you to decide. - `StartError` is set when the program couldn't be started (not found, say). - `Aborted` is `$true` for a command an abort killed or kept from starting. +- `TimedOut` is `$true` for a program `-TimeoutSeconds` killed. - `StreamTimedOut` / `StreamError` mean the output may be incomplete. The output read gives up 15 seconds after the program exits, which only happens when a program it started is still holding the output open. `Results` is in the order commands finished, not the order you gave them. Use `Command` to match them up. `-OnComplete` runs as each command finishes, but only for commands that actually ran. Check `Results` for the ones that couldn't start or were aborted. +`-TimeoutSeconds 600` kills any program still running after 10 minutes. Its result has `TimedOut = $true`, still goes through `-OnComplete`, and the rest of the batch carries on. The default, 0, means no limit. Any programs it started are killed with it. + `Invoke-PkRun { ... }` is what makes Ctrl+C graceful. Whatever the block outputs is passed through, so `$batch` above gets the batch result. It also stops the PC idle-sleeping until the run ends (`-KeepAwake $false` to opt out); the display can still turn off. Without `Invoke-PkRun` the engines still work, but Ctrl+C kills the script the normal way. See "Abort" below for what it means for your own code inside the braces. ## Run PowerShell code in parallel @@ -238,6 +241,7 @@ Closures: build every callback that uses your local variables with `{ ... }.GetN | `Start-PkProcess -FilePath -Arguments` | Start one hidden, with output captured in the background. Returns `@{ Proc; StdoutTask; StderrTask; Stopwatch }`; throws if it can't start. | | `Wait-PkProcess -Process -TimeoutMs` | Wait for exit with a timeout that can't overrun. Returns `$true` if it exited. | | `Complete-PkProcess -Job` | Collect exit code and output from a finished one (`-Job` is what `Start-PkProcess` returned). Same result as `Invoke-PkProcessWithTimeout`. | +| `Stop-PkProcessTree -Process` | Kill a program and every program it started. Returns `$false` if it had already exited. | `Invoke-PkProcessWithTimeout` and `Start-PkProcess` also take `-WorkingDirectory`. @@ -285,7 +289,7 @@ Other pieces: `Request-PkAbort` triggers the same abort from code. `Test-PkAbort - After changing `ParallelKit.ps1` itself, start a new PowerShell window. A session keeps the kit's compiled parts until it closes; the kit tells you if it finds an older copy loaded. - One run at a time per process. The abort state is shared. A nested `Invoke-PkRun` just runs inside the outer one. - A worker stuck inside a long .NET call can't be interrupted. Force-stop stops waiting for it and returns. The worker is NOT killed: once that call returns it can carry on to the end of its script. Before any step that can't be undone (deleting a source file, say), have the worker check `Test-PkCancelRequested` and skip the step if it's true. -- `Invoke-PkProcessBatch` and `Invoke-PkProcessDispatch` have no per-program time limit: a program that hangs holds its slot until it exits or the run is aborted. With the dispatcher you can add one through `-JobPollCompleted`, using the stopwatch the default starter puts on each job: `{ param($job) if ($job['Stopwatch'].Elapsed.TotalMinutes -gt 10) { try { $job['Proc'].Kill() } catch { } }; $job['Proc'].HasExited }`. The killed job then goes through `-JobFinalize` like any other, with a non-zero exit code. +- `Invoke-PkProcessDispatch` has no per-program time limit (`Invoke-PkProcessBatch` has `-TimeoutSeconds`): a program that hangs holds its slot until it exits or the run is aborted. You can add one through `-JobPollCompleted`, using the stopwatch the default starter puts on each job: `{ param($job) if ($job['Stopwatch'].Elapsed.TotalMinutes -gt 10) { $null = Stop-PkProcessTree -Process $job['Proc'] }; $job['Proc'].HasExited }`. The killed job then goes through `-JobFinalize` like any other, with a non-zero exit code. - Programs run hidden, and there's no way to send them input. A program that stops to ask a question will sit there until it's killed. - A program's output is collected in memory and handed over when it exits; there's no live line-by-line output. A program that prints gigabytes will use that much memory.