diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index 7525cf4..39bfebe 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -406,12 +406,8 @@ $f = Test-PkWorkerSafety -WorkerScript { param($Id) Get-HarnessUnsafeState } -Sh Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'ScriptVariable') "a shared function using `$script: is flagged" $f = Test-PkWorkerSafety -WorkerScript { param($Id) $script:top = 1; $sb = { $script:nested = 2 } } Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'ScriptVariable') "in the worker body, only `$script: inside a nested scriptblock is flagged" -$f = Test-PkWorkerSafety -WorkerScript { param($Id) Start-Sleep -Milliseconds 5; Get-HarnessUsesCmdlet } -SharedFunctionNames @('Get-HarnessUsesCmdlet') -Test-Assert (@($f | Where-Object { $_.Rule -eq 'Cmdlet' }).Count -eq 2) "cmdlets are flagged in the worker body and in shared functions" -$f = Test-PkWorkerSafety -WorkerScript { param($Id) @(1, 2) | ForEach-Object { $_ } } -Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'Cmdlet') "a pipeline cmdlet is flagged" -$f = Test-PkWorkerSafety -WorkerScript { param($Id) sleep 1; 1..2 | % { $_ }; & 'Get-Date' } -Test-Assert (@($f | Where-Object { $_.Rule -eq 'Cmdlet' }).Count -eq 3) ("cmdlets are still caught through aliases and a literal & call (got " + @($f | Where-Object { $_.Rule -eq 'Cmdlet' }).Count + ")") +$f = Test-PkWorkerSafety -WorkerScript { param($Id) Start-Sleep -Milliseconds 5; @(1, 2) | Where-Object { $_ -gt 1 } | ForEach-Object { $_ }; Get-HarnessUsesCmdlet } -SharedFunctionNames @('Get-HarnessUsesCmdlet') +Test-Assert ($f.Count -eq 0) ("cmdlets, including ones that take a scriptblock, pass the check in the worker body and in shared functions (got " + $f.Count + " finding(s))") $f = Test-PkWorkerSafety -WorkerScript { param($Id) Get-HarnessDelegate } -SharedFunctionNames @('Get-HarnessDelegate') Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'DelegateCast') "a scriptblock-to-delegate cast is flagged" $f = Test-PkWorkerSafety -WorkerScript { param($Id) Get-HarnessIgnored } -SharedFunctionNames @('Get-HarnessIgnored') @@ -419,8 +415,41 @@ Test-Assert ($f.Count -eq 0) "'# pk:ignore' on the line accepts a finding" $f = Test-PkWorkerSafety -WorkerScript { param($Id) Write-PkMessage "x"; if (Test-PkCancelRequested) { return }; $null = Start-PkProcess -FilePath 'x' } Test-Assert ($f.Count -eq 0) "the kit's own worker functions never need listing" $threw = $false -try { $null = Invoke-PkRunspaceJobs -WorkerScript { param($Id) Start-Sleep 1 } -JobParams (New-JobList -Jobs @(@{ Id = 1 })) -MaxParallel 1 -CheckWorkerSafety } catch { $threw = ($_.ToString() -like '*Start-Sleep*') } +try { $null = Invoke-PkRunspaceJobs -WorkerScript { param($Id) $null = [Func[int]]{ 1 } } -JobParams (New-JobList -Jobs @(@{ Id = 1 })) -MaxParallel 1 -CheckWorkerSafety } catch { $threw = ($_.ToString() -like '*Func*') } Test-Assert $threw "-CheckWorkerSafety refuses to launch unsafe worker code, naming the problem" +$threw = $false +try { $null = Invoke-PkRunspaceJobs -WorkerScript { param($Id) $null = [Func[int]]{ 1 } } -JobParams (New-JobList -Jobs @(@{ Id = 1 })) -MaxParallel 1 } catch { $threw = ($_.ToString() -like '*Func*') } +Test-Assert $threw "the worker-safety check runs without being asked" +$out = $null +try { $out = Invoke-PkRunspaceJobs -WorkerScript { param($Id) $null = [Func[int]]{ 1 } } -JobParams (New-JobList -Jobs @(@{ Id = 1 })) -MaxParallel 1 -CheckWorkerSafety:$false } catch { Write-Host " opt-out threw: $_" } +Test-Assert ($null -ne $out -and $out.CrashedCount -eq 0) "-CheckWorkerSafety:`$false turns the check off" +$results = [System.Collections.Generic.List[object]]::new() +$out = Invoke-PkRunspaceJobs -WorkerScript { param($Id) return @(1, 2, 3) | Where-Object { $_ -gt 1 } | Measure-Object -Sum | ForEach-Object { $_.Sum + $Id } } ` + -JobParams (New-JobList -Jobs @(@{ Id = 10 }, @{ Id = 20 })) -MaxParallel 2 -ResultHandler { param($r) $results.Add($r) }.GetNewClosure() +Test-Assert ($out.CrashedCount -eq 0 -and (@($results | Sort-Object) -join ',') -eq '15,25') "workers can run cmdlets, including ones that take a scriptblock" +$results = [System.Collections.Generic.List[object]]::new() +$out = Invoke-PkRunspaceJobs -WorkerScript { param($Path) return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash } ` + -JobParams (New-JobList -Jobs @(@{ Path = $cmdExe })) -MaxParallel 1 -ResultHandler { param($r) $results.Add($r) }.GetNewClosure() +Test-Assert ($out.CrashedCount -eq 0 -and $results.Count -eq 1 -and $results[0] -eq (Get-FileHash -LiteralPath $cmdExe -Algorithm SHA256).Hash) "a function from a module (Get-FileHash) needn't be shared, and autoloads in the worker" + +# A scriptblock passed as a job value runs against the main thread's session +# state - refused at launch, with the pass-as-text workaround. +$mainThreadBlock = { param($x) $x * 2 } +$threw = $false +try { $null = Invoke-PkRunspaceJobs -WorkerScript { param($Fn) } -JobParams (New-JobList -Jobs @(@{ Fn = 'ok' }, @{ Fn = $mainThreadBlock })) -MaxParallel 1 } catch { $threw = ($_.ToString() -like "*job 1*'Fn'*Create*Nothing was started*") } +Test-Assert $threw "a scriptblock job value is refused before any worker starts, naming the job and key" +$threw = $false +try { $null = Invoke-PkRunspaceJobs -WorkerScript { param($Fns) } -JobParams (New-JobList -Jobs @(@{ Fns = @(1, $mainThreadBlock) })) -MaxParallel 1 } catch { $threw = ($_.ToString() -like "*'Fns'*") } +Test-Assert $threw "a scriptblock inside an array job value is refused too" +$threw = $false +$providerState = @{ Given = 0 } +$provider = { if ($providerState.Given -ge 2) { return $null }; $providerState.Given++; if ($providerState.Given -eq 2) { return @{ Fn = $mainThreadBlock } }; return @{ Fn = 'ok' } }.GetNewClosure() +try { $null = Invoke-PkRunspaceJobs -WorkerScript { param($Fn) } -NextJobProvider $provider -MaxParallel 1 } catch { $threw = ($_.ToString() -like "*-NextJobProvider*'Fn'*") } +Test-Assert $threw "a scriptblock job value from -NextJobProvider is refused" +$results = [System.Collections.Generic.List[object]]::new() +$out = Invoke-PkRunspaceJobs -WorkerScript { param($FnText, $Id) $fn = [scriptblock]::Create($FnText); return (& $fn $Id) } ` + -JobParams (New-JobList -Jobs @(@{ FnText = $mainThreadBlock.ToString(); Id = 4 })) -MaxParallel 1 -ResultHandler { param($r) $results.Add($r) }.GetNewClosure() +Test-Assert ($out.CrashedCount -eq 0 -and $results.Count -eq 1 -and $results[0] -eq 8) "the pass-as-text workaround runs" Write-Host "" # ============================================================================== @@ -629,6 +658,12 @@ $f = Test-PkWorkerSafety -WorkerScript { param($Id) $lazy = [System.Lazy[object] Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'DelegateCast') "a scriptblock passed to a .NET constructor is flagged" $f = Test-PkWorkerSafety -WorkerScript { param($Id) $x = @(1, 2).ForEach({ $_ }); $y = @(1, 2).Where({ $_ }); $z = { 1 }.GetNewClosure() } Test-Assert ($f.Count -eq 0) "PowerShell's own .ForEach()/.Where()/.GetNewClosure() are not flagged" +$f = Test-PkWorkerSafety -WorkerScript { param($Id) $factory = { 1 }; $lazy = [System.Lazy[object]]::new($factory) } +Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'DelegateCast' -and $f[0].Text -like '*$factory*') "a variable holding a scriptblock, passed to a .NET constructor, is flagged by name" +$f = Test-PkWorkerSafety -WorkerScript { param($Id) $add = { param($k, $old) $old + 1 }.GetNewClosure(); $d = [System.Collections.Concurrent.ConcurrentDictionary[string,int]]::new(); $null = $d.AddOrUpdate('a', 1, $add) } +Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'DelegateCast') "a variable holding a .GetNewClosure() block, passed to a .NET method, is flagged" +$f = Test-PkWorkerSafety -WorkerScript { param($Id) $sb = { $_ * 2 }; $n = 'x'; $x = @(1, 2).ForEach($sb); $null = $n.Contains($n) } +Test-Assert ($f.Count -eq 0) "a scriptblock variable given to .ForEach(), and other variables given to .NET methods, are not flagged" $f = Test-PkWorkerSafety -WorkerScript { param($Id) $list = [System.Collections.Generic.List[scriptblock]]::new(); $list.Add({ 1 }) # pk:ignore } Test-Assert ($f.Count -eq 0) "'# pk:ignore' accepts a method that really does take a scriptblock" diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index c06634b..b4aa176 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -2,12 +2,8 @@ # graceful Ctrl+C/window-close handling for Windows PowerShell 5.1. # # Use: . "\ParallelKit.ps1" (dot-source it; do NOT Import-Module it) -# Read README.md first - the three worker-code rules there are what keep this -# from deadlocking. -# -# Split off from Manga Conversion Script -# (https://github.com/HalRiveria/Manga-Conversion-Script), where these pieces -# were debugged against real hangs and deadlocks. No dependency on that project. +# Read README.md first - the two worker-code rules there are what keep this +# from hanging or crashing. # # Works under Set-StrictMode: job hashtables are read as $job['Name'], never # $job.Name, because under strict mode the dot form THROWS for a key that @@ -1350,7 +1346,8 @@ function Invoke-PkRunspaceJobs { [scriptblock]$JobCrashedHandler = $null, [int]$DrainTimeoutSeconds = 60, [string]$AbortWaitMessage, - [switch]$CheckWorkerSafety + # On by default; -CheckWorkerSafety:$false turns it off. + [switch]$CheckWorkerSafety = $true ) if ($null -eq $JobParams -and $null -eq $NextJobProvider) { @@ -1408,6 +1405,10 @@ function Invoke-PkRunspaceJobs { throw ("Invoke-PkRunspaceJobs: job " + $jobIndex + " supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it. Nothing was started.") } } + if ($CheckWorkerSafety) { + $sbKey = Get-PkScriptBlockJobKey -Job $JobParams[$jobIndex] + if ($null -ne $sbKey) { throw ((Get-PkScriptBlockJobMessage -Job ("job " + $jobIndex) -Key $sbKey) + " Nothing was started.") } + } } } @@ -1482,6 +1483,10 @@ function Invoke-PkRunspaceJobs { throw ("Invoke-PkRunspaceJobs: a job supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it.") } } + if ($CheckWorkerSafety -and $null -eq $JobParams) { + $sbKey = Get-PkScriptBlockJobKey -Job $nextJob + if ($null -ne $sbKey) { throw (Get-PkScriptBlockJobMessage -Job "a job from -NextJobProvider" -Key $sbKey) } + } $ps = [PowerShell]::Create() $ps.RunspacePool = $pool @@ -1573,7 +1578,7 @@ function Invoke-PkRunspaceJobs { } # ============================================================================== -# Worker-safety check (see README.md, "The three rules") +# Worker-safety check (see README.md, "The two rules") # ============================================================================== function Get-PkCommandKind { @@ -1585,9 +1590,38 @@ function Get-PkCommandKind { $hops++ } if ($null -eq $cmd) { return 'Unknown' } + # A function from a module (Get-FileHash, say) autoloads in a worker + # like a cmdlet does; only functions defined in your session need sharing. + if ($cmd.CommandType -eq 'Function' -and -not [string]::IsNullOrEmpty($cmd.ModuleName)) { return 'ModuleFunction' } return [string]$cmd.CommandType } +function Get-PkScriptBlockJobKey { + # The first key of a job hashtable whose value is a scriptblock, or holds + # one (an array/list item or a hashtable value - one level down), else + # $null. A scriptblock belongs to the runspace that made it: run in a + # worker it executes against the main thread's session state, which can + # corrupt it (the kit's own functions vanish mid-run), crash, or hang. + param([Parameter(Mandatory)][hashtable]$Job) + foreach ($key in $Job.Keys) { + $value = $Job[$key] + if ($value -is [scriptblock]) { return [string]$key } + if ($value -is [System.Collections.IDictionary]) { + foreach ($inner in $value.Values) { if ($inner -is [scriptblock]) { return [string]$key } } + } elseif ($value -is [System.Collections.IList]) { + foreach ($inner in $value) { if ($inner -is [scriptblock]) { return [string]$key } } + } + } + return $null +} + +function Get-PkScriptBlockJobMessage { + # Shared by the fixed-list and -NextJobProvider checks in Invoke-PkRunspaceJobs. + param([string]$Job, [string]$Key) + return ("Invoke-PkRunspaceJobs: " + $Job + " passes a scriptblock in '" + $Key + "'. A scriptblock made on the main thread and run in a worker can corrupt, crash or hang the run. " + + "Pass its text instead (`$sb.ToString()) and rebuild it in the worker with [scriptblock]::Create(`$text), or pass -CheckWorkerSafety:`$false to skip this check.") +} + function Test-PkWorkerSafety { # Static check of a worker script and the functions shared with it. # Returns a list of findings (empty = clean); each is @@ -1595,12 +1629,12 @@ function Test-PkWorkerSafety { # # MissingFunction the worker calls one of your functions that isn't in -SharedFunctionNames # ScriptVariable a shared function (or a nested scriptblock or function in the worker) uses $script: - # Cmdlet worker-reachable code calls a cmdlet # DelegateCast worker-reachable code casts a scriptblock to a .NET delegate, or passes # one straight into a .NET method/constructor (which converts it to one) # - # Not visible from here: callbacks handed to a worker as job parameters, - # and commands invoked through a variable (& $cmd). + # Not visible from here: commands invoked through a variable (& $cmd). + # Scriptblocks handed to a worker as job values are refused separately, + # by Invoke-PkRunspaceJobs (see Get-PkScriptBlockJobKey). param( [Parameter(Mandatory)] [scriptblock]$WorkerScript, @@ -1639,9 +1673,6 @@ function Test-PkWorkerSafety { if ($kind -eq 'Function' -and -not $listed.Contains($cmdName)) { $findings.Add(@{ Rule = 'MissingFunction'; Where = $t.Where; Line = $c.Extent.StartLineNumber Text = ($t.Where + " (line " + $c.Extent.StartLineNumber + ") calls '" + $cmdName + "', which is not in -SharedFunctionNames") }) - } elseif ($kind -eq 'Cmdlet') { - $findings.Add(@{ Rule = 'Cmdlet'; Where = $t.Where; Line = $c.Extent.StartLineNumber - Text = ($t.Where + " (line " + $c.Extent.StartLineNumber + ") calls the cmdlet '" + $cmdName + "' - use a raw .NET call instead") }) } } @@ -1671,7 +1702,7 @@ function Test-PkWorkerSafety { if ($cv.Type.TypeName.Name -notmatch '^(System\.)?(Func|Action|Predicate|Comparison|Converter)(\[|`|$)') { continue } if ($cv.Extent.StartScriptPosition.Line -match '#\s*pk:ignore') { continue } $findings.Add(@{ Rule = 'DelegateCast'; Where = $t.Where; Line = $cv.Extent.StartLineNumber - Text = ($t.Where + " (line " + $cv.Extent.StartLineNumber + ") casts a scriptblock to the delegate [" + $cv.Type.TypeName.Name + "] - this can deadlock in a worker") }) + Text = ($t.Where + " (line " + $cv.Extent.StartLineNumber + ") casts a scriptblock to the delegate [" + $cv.Type.TypeName.Name + "] - this can hang or crash a worker") }) } # The implicit form: a scriptblock literal passed as an argument to @@ -1679,18 +1710,36 @@ function Test-PkWorkerSafety { # [System.Lazy[object]]::new({ ... }). No cast is written, but .NET # receives a delegate all the same. .ForEach()/.Where() are # PowerShell's own and take real scriptblocks, so they're fine. + # The same goes for a variable assigned a scriptblock literal (or + # { }.GetNewClosure()) anywhere in this code, then passed as + # $lazy = [System.Lazy[object]]::new($factory). By name, not by flow: + # a variable reassigned to something else in between is still flagged. + $scriptBlockVars = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($assign in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.AssignmentStatementAst] }, $true)) { + $target = $assign.Left + if ($target -is [System.Management.Automation.Language.ConvertExpressionAst]) { $target = $target.Child } + if ($target -isnot [System.Management.Automation.Language.VariableExpressionAst]) { continue } + $rhs = $assign.Right + if ($rhs -is [System.Management.Automation.Language.PipelineAst] -and $rhs.PipelineElements.Count -eq 1) { $rhs = $rhs.PipelineElements[0] } + if ($rhs -is [System.Management.Automation.Language.CommandExpressionAst]) { $rhs = $rhs.Expression } + if ($rhs -is [System.Management.Automation.Language.InvokeMemberExpressionAst] -and [string]$rhs.Member.Extent.Text -eq 'GetNewClosure') { $rhs = $rhs.Expression } + if ($rhs -is [System.Management.Automation.Language.ScriptBlockExpressionAst]) { $null = $scriptBlockVars.Add($target.VariablePath.UserPath) } + } foreach ($call in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.InvokeMemberExpressionAst] }, $true)) { if ($null -eq $call.Arguments) { continue } $memberName = [string]$call.Member.Extent.Text if ($memberName -in @('ForEach', 'Where')) { continue } - $passesScriptBlock = $false + $passed = $null foreach ($argument in $call.Arguments) { - if ($argument -is [System.Management.Automation.Language.ScriptBlockExpressionAst]) { $passesScriptBlock = $true } + if ($argument -is [System.Management.Automation.Language.ScriptBlockExpressionAst]) { $passed = 'a scriptblock' } + elseif ($argument -is [System.Management.Automation.Language.VariableExpressionAst] -and $scriptBlockVars.Contains($argument.VariablePath.UserPath)) { + $passed = ("the scriptblock in `$" + $argument.VariablePath.UserPath) + } } - if (-not $passesScriptBlock) { continue } + if ($null -eq $passed) { continue } if ($call.Extent.StartScriptPosition.Line -match '#\s*pk:ignore') { continue } $findings.Add(@{ Rule = 'DelegateCast'; Where = $t.Where; Line = $call.Extent.StartLineNumber - Text = ($t.Where + " (line " + $call.Extent.StartLineNumber + ") passes a scriptblock to ." + $memberName + "() - .NET turns it into a delegate, which can deadlock in a worker (add '# pk:ignore' if this method really takes a scriptblock)") }) + Text = ($t.Where + " (line " + $call.Extent.StartLineNumber + ") passes " + $passed + " to ." + $memberName + "() - .NET turns it into a delegate, which can hang or crash a worker (add '# pk:ignore' if this method really takes a scriptblock)") }) } } diff --git a/README.md b/README.md index 08c2f2b..976d2c3 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ It was pulled out of [Manga Conversion Script](https://github.com/HalRiveria/Man Built with Claude Code. -**Contents:** [Quick start](#quick-start) · [Who it's for](#who-its-for) · [Why Not Use X?](#why-not-use-x) · [Setup](#setup) · [Run a list of programs](#run-a-list-of-programs) · [Run PowerShell code in parallel](#run-powershell-code-in-parallel) · [The three rules for worker code](#the-three-rules-for-worker-code) · [Process dispatch with groups](#process-dispatch-with-groups) · [Single processes](#single-processes) · [Messages](#messages) · [Abort](#abort) · [Choosing limits](#choosing-limits) · [Limits](#limits) · [Testing](#testing) · [FAQ](#faq) +**Contents:** [Quick start](#quick-start) · [Who it's for](#who-its-for) · [Why Not Use X?](#why-not-use-x) · [Setup](#setup) · [Run a list of programs](#run-a-list-of-programs) · [Run PowerShell code in parallel](#run-powershell-code-in-parallel) · [The two rules for worker code](#the-two-rules-for-worker-code) · [Process dispatch with groups](#process-dispatch-with-groups) · [Single processes](#single-processes) · [Messages](#messages) · [Abort](#abort) · [Choosing limits](#choosing-limits) · [Limits](#limits) · [Testing](#testing) · [FAQ](#faq) ## Quick start @@ -136,13 +136,10 @@ Each command is a hashtable with `FilePath`, `Arguments` and optionally `Working ## Run PowerShell code in parallel ```powershell -# Worker code may only use .NET calls, not cmdlets - see "The three rules" below. +# Worker code can use cmdlets - see "The two rules" below for what it can't do. function Get-FileSha { param([string]$Path) - $sha = [System.Security.Cryptography.SHA256]::Create() - $stream = [System.IO.File]::OpenRead($Path) - try { return [System.BitConverter]::ToString($sha.ComputeHash($stream)).Replace('-', '') } - finally { $stream.Dispose(); $sha.Dispose() } + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash } $jobs = [System.Collections.Generic.List[hashtable]]::new() @@ -153,7 +150,6 @@ $jobsParams = @{ MaxParallel = 4 JobParams = $jobs SharedFunctionNames = @('Get-FileSha') - CheckWorkerSafety = $true WorkerScript = { param([string]$Path) if (Test-PkCancelRequested) { return } @@ -178,27 +174,25 @@ Each worker runs in its own runspace: a separate PowerShell session inside the s - If `-ResultHandler`, `-PollCallback` or `-NextJobProvider` throws, the run stops: workers still going are told to stop and the exception comes out of the call. - Returns `@{ ForceStoppedCount; CrashedCount; Aborted }`. -## The three rules for worker code +## The two rules for worker code These only apply if you run PowerShell code in parallel with `Invoke-PkRunspaceJobs` (or call a dispatcher from inside one of its workers). Running external programs with `Invoke-PkProcessBatch` or `Invoke-PkProcessDispatch` on the main thread never touches them. -Worker code is the worker script, every function in `-SharedFunctionNames`, and anything they call. Breaking these rules doesn't fail loudly. It hangs, or quietly reads nothing. +Worker code is the worker script, every function in `-SharedFunctionNames`, and anything they call. Breaking these rules doesn't fail loudly. It hangs, crashes, or quietly reads nothing. 1. **No `$script:` variables.** In a worker they can silently read as empty. Pass state in as job parameters. (`$script:` at the very top level of the worker script is fine; inside a function or a nested `{ }` block it isn't.) -2. **No cmdlets.** Two fresh runspaces calling the same cmdlet for the first time at the same moment can deadlock in PowerShell 5.1. Use .NET calls: `[System.IO.File]::Exists($p)` instead of `Test-Path`, `[System.Threading.Thread]::Sleep(100)` instead of `Start-Sleep`, a `foreach` loop instead of `ForEach-Object`, `$null = ...` instead of `| Out-Null`, `[DateTime]::Now` instead of `Get-Date`, `Write-PkMessage` instead of `Write-Host`. -3. **No scriptblocks handed to .NET as delegates** (`[Func[...]]{ }`, `[Action]{ }`, `System.Lazy`, `ConcurrentDictionary.GetOrAdd($k, { })`). Nested, they deadlock. Use plain loops and `[System.Threading.Monitor]`. +2. **A scriptblock only runs in the runspace that made it.** Don't hand one to .NET as a delegate (`[Func[...]]{ }`, `[Action]{ }`, a `System.Lazy` factory, `ConcurrentDictionary.GetOrAdd($k, { })`, a comparer, an event or task callback): use plain loops and `[System.Threading.Monitor]` instead. Don't pass one in as a job value either: pass its text (`$sb.ToString()`) and rebuild it in the worker with `[scriptblock]::Create($text)`. -Only worker code is affected. Callbacks that run on the main thread (`-ResultHandler`, `-PollCallback`, `-NextJobProvider`, `-OnComplete`, the message handler) are not worker code and can use anything, cmdlets included. +Everything else is fine in a worker, cmdlets included. Callbacks that run on the main thread (`-ResultHandler`, `-PollCallback`, `-NextJobProvider`, `-OnComplete`, the message handler) are not worker code and can use anything. -`Test-PkWorkerSafety -WorkerScript $w -SharedFunctionNames $names` checks all three, plus missing shared functions, and returns a list of findings. For rule 3 it flags both written casts and scriptblocks passed straight into a .NET call. Call it from your tests, or pass `-CheckWorkerSafety` to `Invoke-PkRunspaceJobs` to refuse to launch unsafe code. `# pk:ignore` on a line accepts a finding. It can't see callbacks you pass to a worker as job parameters, or commands run through a variable (`& $cmd`). +`Invoke-PkRunspaceJobs` checks both rules before it starts anything, and refuses to launch code that breaks them, naming the line or the job key. Pass `-CheckWorkerSafety:$false` to turn that off. The code check is also available on its own as `Test-PkWorkerSafety -WorkerScript $w -SharedFunctionNames $names`, which returns a list of findings (including functions the worker calls but you didn't share), for use in your own tests. For rule 2 it flags written casts, and scriptblocks passed into a .NET method or constructor, either written in the call or through a variable they were assigned to. `# pk:ignore` on a line accepts a finding. It can't see commands run through a variable (`& $cmd`), or a scriptblock buried more than one level deep in a job value. ### Why these rules exist -"No cmdlets" in PowerShell code sounds extreme, so here's where each rule comes from. All three were found chasing real, intermittent hangs, and none of them is in Microsoft's documentation as far as we could find. +Both were found chasing real, intermittent failures, and neither is in Microsoft's documentation as far as we could find. -- **Cmdlets.** The first time a cmdlet runs in a fresh runspace, PowerShell 5.1 sets it up, and that setup isn't safe when two runspaces do it at the same moment. With just two workers, the original project hung in roughly 1 run in 8. Every cmdlet tried could trigger it, down to `Start-Sleep` and `Out-Null`. Replacing every cmdlet on the worker side with .NET calls took that to 0 hangs in 50 stress runs. The kit's own worker-side functions follow the same rule. - **`$script:` variables.** A function listed in `-SharedFunctionNames` is recreated inside the worker from its source text, so it's no longer attached to your script and its `$script:` points somewhere else. Inside a `.GetNewClosure()` block, `$script:` points at a private copy. Either way the read comes back empty, with no error. -- **Delegates.** When .NET calls a scriptblock through a delegate, PowerShell can hand the call to the runspace's own thread instead of running it on the spot. Nested, the inner call waits for a thread that is itself waiting on that call. It's timing-dependent: it showed up in about 1 or 2 runs in 100, and was only pinned down from process dumps. +- **Scriptblocks.** A scriptblock belongs to the runspace that created it, and runs against that runspace's session state wherever it's called from ([PowerShell issue #4003](https://github.com/PowerShell/PowerShell/issues/4003)). When a delegate makes .NET call one from another runspace's thread, PowerShell can hand the call back to the owning runspace's thread. If that thread is itself waiting, and the scriptblock runs another scriptblock inside it (a `Where-Object { }` filter, a nested delegate), it deadlocks. Even when it doesn't deadlock, two threads using one session state at once can crash the process. A scriptblock passed in as a job value is the same case: in stress tests, 8 workers running one passed from the main thread broke 88 of 90 runs, usually by corrupting the main thread's session state so the kit's own functions vanished mid-run, even when the scriptblock held nothing but .NET calls. Passed as text and rebuilt in the worker, the same code ran clean in all 30 runs. ## Process dispatch with groups @@ -309,17 +303,13 @@ GitHub runs the same harness on Windows PowerShell 5.1 for every pull request an ## FAQ -**No cmdlets? Isn't the point of PowerShell to use cmdlets?** +**Can I use cmdlets in worker code?** -The rule only covers code running inside `Invoke-PkRunspaceJobs` workers. Your main script, every main-thread callback, and the whole external-program side (`Invoke-PkProcessBatch`, `Invoke-PkProcessDispatch`) when called from your main script can use cmdlets freely. Inside workers it's because of a race in PowerShell 5.1: two fresh runspaces running the same cmdlet for the first time at the same moment can deadlock. [Why these rules exist](#why-these-rules-exist) has the details. +Yes. What a worker can't do is run a scriptblock that came from another runspace: one handed to .NET as a delegate, or one passed in as a job value. [The two rules](#the-two-rules-for-worker-code) cover it, and the kit checks for both before it starts. -**What happens if I use a cmdlet in a worker anyway?** +**Do ThreadJob and PoshRSJob have the same scriptblock problem?** -Most runs work. Some hang at random, with no error. In the original project it was about 1 run in 8 with just two workers, so a test run passing tells you very little. `-CheckWorkerSafety` refuses to start worker code that uses a cmdlet, and `# pk:ignore` on a line accepts it if you've decided the risk is fine. - -**Do ThreadJob and PoshRSJob have the same cmdlet problem?** - -It hasn't been tested with them. The race is in how PowerShell 5.1 sets up cmdlets in a new runspace, not in this kit, so other runspace-based tools on 5.1 may hit it too, but that's not something that's been reproduced. +It hasn't been tested with them. The cause is in PowerShell itself, not in this kit: a scriptblock stays tied to the runspace that made it. So any runspace-based tool that lets a scriptblock cross from one runspace to another can hit it. This kit refuses to launch when it sees one. **Does it work in PowerShell 7, or on Linux or macOS?**