Invoke-PkProcessBatch: add -TimeoutSeconds

ParallelKit.Harness.ps1

@@ -1049,7 +1049,7 @@ Test-Assert ([PkState]::SourceCount -eq 0) "and nothing is left registered"
 # Every batch result has every documented field, so a strict-mode caller
 # can read any of them without a "property cannot be found" error.
 Reset-Harness
-$fields = @('Command', 'ExitCode', 'Stdout', 'Stderr', 'StreamTimedOut', 'StreamError', 'StartError', 'Aborted')
+$fields = @('Command', 'ExitCode', 'Stdout', 'Stderr', 'StreamTimedOut', 'StreamError', 'StartError', 'Aborted', 'TimedOut')
 $batch = Invoke-PkProcessBatch -MaxConcurrent 1 -Commands @(
     @{ FilePath = $cmdExe; Arguments = '/c exit 0' },
     @{ FilePath = 'C:\definitely\not\here.exe' })
@@ -1065,6 +1065,34 @@ Test-Assert ($batch.Aborted -and $batch.Results.Count -eq 2 -and @($batch.Result
 Test-Assert (@($missing).Count -eq 0) ("and their results carry every field too (missing: " + (@($missing) -join ', ') + ")")
 Reset-PkAbortState
 
+# -TimeoutSeconds kills a program that overruns; the rest run normally.
+Reset-Harness
+$completed = [System.Collections.Generic.List[object]]::new()
+$sw = [System.Diagnostics.Stopwatch]::StartNew()
+$batch = Invoke-PkProcessBatch -MaxConcurrent 2 -TimeoutSeconds 2 -OnComplete { param($c, $r) $completed.Add($r) }.GetNewClosure() -Commands @(
+    @{ FilePath = $pingExe; Arguments = '-n 30 127.0.0.1'; Name = 'slow' },
+    @{ FilePath = $cmdExe; Arguments = '/c exit 0'; Name = 'fast' })
+$sw.Stop()
+$slow = @($batch.Results | Where-Object { $_.Command['Name'] -eq 'slow' })
+$fast = @($batch.Results | Where-Object { $_.Command['Name'] -eq 'fast' })
+Test-Assert ($slow.Count -eq 1 -and $slow[0].TimedOut -and $slow[0].ExitCode -ne 0 -and $sw.Elapsed.TotalSeconds -lt 15) ("batch: -TimeoutSeconds kills an overrunning program and marks it TimedOut (" + [int]$sw.Elapsed.TotalSeconds + "s)")
+Test-Assert ($fast.Count -eq 1 -and -not $fast[0].TimedOut -and $fast[0].ExitCode -eq 0 -and -not $batch.Aborted) "and the others finish normally, not TimedOut"
+Test-Assert ($completed.Count -eq 2) "a timed-out program still goes through -OnComplete"
+# A program whose own child holds the output open: the whole tree is killed,
+# so the output closes at once instead of after the 15-second stream wait.
+$sw = [System.Diagnostics.Stopwatch]::StartNew()
+$batch = Invoke-PkProcessBatch -MaxConcurrent 1 -TimeoutSeconds 2 -Commands @(@{ FilePath = $cmdExe; Arguments = ('/c ""' + $pingExe + '" -n 30 127.0.0.1"') })
+$sw.Stop()
+$r = @($batch.Results)[0]
+Test-Assert ($r.TimedOut -and -not $r.StreamTimedOut -and $sw.Elapsed.TotalSeconds -lt 10) ("batch: a timeout kills the program's children too (" + [int]$sw.Elapsed.TotalSeconds + "s, StreamTimedOut=" + $r.StreamTimedOut + ")")
+$j = Start-PkProcess -FilePath $cmdExe -Arguments '/c exit 0'
+$null = Wait-PkProcess -Process $j.Proc -TimeoutMs 5000
+Test-Assert (-not (Stop-PkProcessTree -Process $j.Proc)) "Stop-PkProcessTree reports `$false for a program that already exited"
+$j.Proc.Dispose()
+$threw = $false
+try { $null = Invoke-PkProcessBatch -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) -TimeoutSeconds -1 } catch { $threw = ($_.ToString() -like '*-TimeoutSeconds*') }
+Test-Assert $threw "a negative -TimeoutSeconds is refused"
+
 # An aborted group still gets -GroupCompleteCallback (the README warns about
 # this - a caller zipping a folder there must check for the abort first).
 Reset-Harness

ParallelKit.ps1

@@ -559,6 +559,32 @@ function Start-PkProcess {
     }
 }
 
+function Stop-PkProcessTree {
+    # Kills a process and every process it started (taskkill /T /F), so
+    # none of them is left holding its output open. Falls back to killing
+    # just the process. Returns $true if this call killed it, $false if it
+    # had already exited on its own.
+    param([Parameter(Mandatory)][System.Diagnostics.Process]$Process)
+    if ($Process.HasExited) { return $false }
+    $killed = $false
+    $psi = [System.Diagnostics.ProcessStartInfo]::new((Join-Path $env:SystemRoot 'System32\taskkill.exe'), ('/T /F /PID ' + $Process.Id))
+    $psi.UseShellExecute = $false
+    $psi.CreateNoWindow = $true
+    $psi.RedirectStandardOutput = $true
+    $psi.RedirectStandardError = $true
+    try {
+        $taskkill = [System.Diagnostics.Process]::Start($psi)
+        try {
+            if ($taskkill.WaitForExit(10000)) { $killed = ($taskkill.ExitCode -eq 0) }
+        } finally { $taskkill.Dispose() }
+    } catch { }  # taskkill missing or blocked: the fallback below still kills the process itself
+    if (-not $killed -and -not $Process.HasExited) {
+        try { $Process.Kill(); $killed = $true } catch { }  # it exited between the check and the kill
+    }
+    if ($killed) { $null = $Process.WaitForExit(5000) }
+    return $killed
+}
+
 function Wait-PkProcess {
     # Waits on the raw process handle. Returns $true if it exited in time.
     # (Process.WaitForExit(int) also waits on the redirected streams and can
@@ -1148,8 +1174,10 @@ function Invoke-PkProcessBatch {
     # Each command is @{ FilePath = ...; Arguments = ...; WorkingDirectory = ... (optional); <anything else> }.
     # $OnComplete { param($command, $result) } fires as each one finishes
     # (not for one that couldn't start or was stopped by an abort).
+    # $TimeoutSeconds > 0 kills a program still running after that long; its
+    # result has TimedOut = $true (0 = no limit).
     # Returns @{ Results; Aborted; ForceKilledCount }; each result is
-    # @{ Command; ExitCode; Stdout; Stderr; StreamTimedOut; StreamError; StartError; Aborted },
+    # @{ Command; ExitCode; Stdout; Stderr; StreamTimedOut; StreamError; StartError; Aborted; TimedOut },
     # every field always present - keep the three places that build one in sync.
     # $Commands may be $null: a foreach that matches nothing yields $null, not
     # an empty array, and an empty batch should just return no results.
@@ -1160,9 +1188,12 @@ function Invoke-PkProcessBatch {
         [hashtable[]]$Commands,
         [int]$MaxConcurrent = 4,
         [scriptblock]$OnComplete = $null,
-        [int]$AbortTimeoutSeconds = 60
+        [int]$AbortTimeoutSeconds = 60,
+        [int]$TimeoutSeconds = 0
     )
 
+    if ($TimeoutSeconds -lt 0) { throw "Invoke-PkProcessBatch: -TimeoutSeconds must be 0 (no limit) or more." }
+
     $results = [System.Collections.Generic.List[hashtable]]::new()
     $state   = @{ Given = $false }
 
@@ -1187,30 +1218,47 @@ function Invoke-PkProcessBatch {
         try {
             $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments $job['Arguments'] -WorkingDirectory $job['WorkingDirectory']
         } catch {
-            $results.Add(@{ Command = $job['Command']; ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; StartError = [string]$_; Aborted = $false })
+            $results.Add(@{ Command = $job['Command']; ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; StartError = [string]$_; Aborted = $false; TimedOut = $false })
             throw
         }
         $job['Proc'] = $started.Proc; $job['StdoutTask'] = $started.StdoutTask; $job['StderrTask'] = $started.StderrTask
+        $job['Stopwatch'] = $started.Stopwatch; $job['TimedOut'] = $false
         return $job
     }.GetNewClosure()
 
     $jobFinalize = {
         param($job, $pendingQueue)
         $r = Complete-PkProcess -Job $job
-        $r.Command = $job['Command']; $r.StartError = $null; $r.Aborted = $false
+        $r.Command = $job['Command']; $r.StartError = $null; $r.Aborted = $false; $r.TimedOut = [bool]$job['TimedOut']
         $results.Add($r)
         if ($null -ne $OnComplete) { $null = & $OnComplete $job['Command'] $r }
         return @{ Success = ($r.ExitCode -eq 0) }
     }.GetNewClosure()
 
     $jobAbortCleanup = {
         param($job)
-        $results.Add(@{ Command = $job['Command']; ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; StartError = $null; Aborted = $true })
+        $results.Add(@{ Command = $job['Command']; ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; StartError = $null; Aborted = $true; TimedOut = $false })
     }.GetNewClosure()
 
-    $dispatch = Invoke-PkProcessDispatch -GroupProvider $groupProvider -JobBuilder $jobBuilder -JobStarter $jobStarter `
-        -JobFinalize $jobFinalize -JobAbortCleanup $jobAbortCleanup -MaxConcurrent $MaxConcurrent `
-        -AbortTimeoutSeconds $AbortTimeoutSeconds -MaxQueueAhead 1
+    $dispatchParams = @{
+        GroupProvider = $groupProvider; JobBuilder = $jobBuilder; JobStarter = $jobStarter
+        JobFinalize = $jobFinalize; JobAbortCleanup = $jobAbortCleanup; MaxConcurrent = $MaxConcurrent
+        AbortTimeoutSeconds = $AbortTimeoutSeconds; MaxQueueAhead = 1
+    }
+    if ($TimeoutSeconds -gt 0) {
+        # Kill an overrunning program; the next poll sees it exited and it
+        # goes through $jobFinalize like any other, flagged TimedOut.
+        $dispatchParams['JobPollCompleted'] = {
+            param($job)
+            if (-not $job['Proc'].HasExited -and $job['Stopwatch'].Elapsed.TotalSeconds -ge $TimeoutSeconds) {
+                # Only flagged if this call did the killing: a program that
+                # exited on its own right at the deadline didn't time out.
+                if (Stop-PkProcessTree -Process $job['Proc']) { $job['TimedOut'] = $true }
+            }
+            return $job['Proc'].HasExited
+        }.GetNewClosure()
+    }
+    $dispatch = Invoke-PkProcessDispatch @dispatchParams
 
     return @{ Results = $results; Aborted = $dispatch.Aborted; ForceKilledCount = $dispatch.ForceKilledCount }
 }
@@ -1221,7 +1269,7 @@ function Invoke-PkProcessBatch {
 
 function Get-PkWorkerSafeFunctionNames {
     # Kit functions every worker runspace gets automatically.
-    return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool',
+    return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', 'Stop-PkProcessTree',
              'Start-PkProcess', 'Wait-PkProcess', 'Complete-PkProcess', 'Invoke-PkProcessWithTimeout',
              'Invoke-PkProcessDispatch', 'Invoke-PkProcessDispatchCore', 'Invoke-PkProcessBatch',
              'Register-PkCancellationSource', 'Unregister-PkCancellationSource')

README.md

@@ -119,18 +119,21 @@ Long parameter lists in these examples use splatting (`@batchParams`) instead of
 
 Each command is a hashtable with `FilePath`, `Arguments` and optionally `WorkingDirectory`, plus any keys of your own. `Arguments` is passed to the program as one string, exactly as written, so put quotes around paths that may contain spaces (as above). Programs run hidden, with their output captured.
 
-`Invoke-PkProcessBatch` returns `@{ Results; Aborted; ForceKilledCount }`. Each result is `@{ Command; ExitCode; Stdout; Stderr; StreamTimedOut; StreamError; StartError; Aborted }`, with every field always present:
+`Invoke-PkProcessBatch` returns `@{ Results; Aborted; ForceKilledCount }`. Each result is `@{ Command; ExitCode; Stdout; Stderr; StreamTimedOut; StreamError; StartError; Aborted; TimedOut }`, with every field always present:
 
 - `Command` is your original command hashtable, so extra keys like `Source` above come back with it.
 - `ExitCode` is whatever the program returned. The kit doesn't treat a non-zero code as a failure; that's for you to decide.
 - `StartError` is set when the program couldn't be started (not found, say).
 - `Aborted` is `$true` for a command an abort killed or kept from starting.
+- `TimedOut` is `$true` for a program `-TimeoutSeconds` killed.
 - `StreamTimedOut` / `StreamError` mean the output may be incomplete. The output read gives up 15 seconds after the program exits, which only happens when a program it started is still holding the output open.
 
 `Results` is in the order commands finished, not the order you gave them. Use `Command` to match them up.
 
 `-OnComplete` runs as each command finishes, but only for commands that actually ran. Check `Results` for the ones that couldn't start or were aborted.
 
+`-TimeoutSeconds 600` kills any program still running after 10 minutes. Its result has `TimedOut = $true`, still goes through `-OnComplete`, and the rest of the batch carries on. The default, 0, means no limit. Any programs it started are killed with it.
+
 `Invoke-PkRun { ... }` is what makes Ctrl+C graceful. Whatever the block outputs is passed through, so `$batch` above gets the batch result. It also stops the PC idle-sleeping until the run ends (`-KeepAwake $false` to opt out); the display can still turn off. Without `Invoke-PkRun` the engines still work, but Ctrl+C kills the script the normal way. See "Abort" below for what it means for your own code inside the braces.
 
 ## Run PowerShell code in parallel
@@ -238,6 +241,7 @@ Closures: build every callback that uses your local variables with `{ ... }.GetN
 | `Start-PkProcess -FilePath -Arguments` | Start one hidden, with output captured in the background. Returns `@{ Proc; StdoutTask; StderrTask; Stopwatch }`; throws if it can't start. |
 | `Wait-PkProcess -Process -TimeoutMs` | Wait for exit with a timeout that can't overrun. Returns `$true` if it exited. |
 | `Complete-PkProcess -Job` | Collect exit code and output from a finished one (`-Job` is what `Start-PkProcess` returned). Same result as `Invoke-PkProcessWithTimeout`. |
+| `Stop-PkProcessTree -Process` | Kill a program and every program it started. Returns `$false` if it had already exited. |
 
 `Invoke-PkProcessWithTimeout` and `Start-PkProcess` also take `-WorkingDirectory`.
 
@@ -285,7 +289,7 @@ Other pieces: `Request-PkAbort` triggers the same abort from code. `Test-PkAbort
 - After changing `ParallelKit.ps1` itself, start a new PowerShell window. A session keeps the kit's compiled parts until it closes; the kit tells you if it finds an older copy loaded.
 - One run at a time per process. The abort state is shared. A nested `Invoke-PkRun` just runs inside the outer one.
 - A worker stuck inside a long .NET call can't be interrupted. Force-stop stops waiting for it and returns. The worker is NOT killed: once that call returns it can carry on to the end of its script. Before any step that can't be undone (deleting a source file, say), have the worker check `Test-PkCancelRequested` and skip the step if it's true.
-- `Invoke-PkProcessBatch` and `Invoke-PkProcessDispatch` have no per-program time limit: a program that hangs holds its slot until it exits or the run is aborted. With the dispatcher you can add one through `-JobPollCompleted`, using the stopwatch the default starter puts on each job: `{ param($job) if ($job['Stopwatch'].Elapsed.TotalMinutes -gt 10) { try { $job['Proc'].Kill() } catch { } }; $job['Proc'].HasExited }`. The killed job then goes through `-JobFinalize` like any other, with a non-zero exit code.
+- `Invoke-PkProcessDispatch` has no per-program time limit (`Invoke-PkProcessBatch` has `-TimeoutSeconds`): a program that hangs holds its slot until it exits or the run is aborted. You can add one through `-JobPollCompleted`, using the stopwatch the default starter puts on each job: `{ param($job) if ($job['Stopwatch'].Elapsed.TotalMinutes -gt 10) { $null = Stop-PkProcessTree -Process $job['Proc'] }; $job['Proc'].HasExited }`. The killed job then goes through `-JobFinalize` like any other, with a non-zero exit code.
 - Programs run hidden, and there's no way to send them input. A program that stops to ask a question will sit there until it's killed.
 - A program's output is collected in memory and handed over when it exits; there's no live line-by-line output. A program that prints gigabytes will use that much memory.