Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 42 additions & 7 deletions ParallelKit.Harness.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -406,21 +406,50 @@ $f = Test-PkWorkerSafety -WorkerScript { param($Id) Get-HarnessUnsafeState } -Sh
Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'ScriptVariable') "a shared function using `$script: is flagged"
$f = Test-PkWorkerSafety -WorkerScript { param($Id) $script:top = 1; $sb = { $script:nested = 2 } }
Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'ScriptVariable') "in the worker body, only `$script: inside a nested scriptblock is flagged"
$f = Test-PkWorkerSafety -WorkerScript { param($Id) Start-Sleep -Milliseconds 5; Get-HarnessUsesCmdlet } -SharedFunctionNames @('Get-HarnessUsesCmdlet')
Test-Assert (@($f | Where-Object { $_.Rule -eq 'Cmdlet' }).Count -eq 2) "cmdlets are flagged in the worker body and in shared functions"
$f = Test-PkWorkerSafety -WorkerScript { param($Id) @(1, 2) | ForEach-Object { $_ } }
Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'Cmdlet') "a pipeline cmdlet is flagged"
$f = Test-PkWorkerSafety -WorkerScript { param($Id) sleep 1; 1..2 | % { $_ }; & 'Get-Date' }
Test-Assert (@($f | Where-Object { $_.Rule -eq 'Cmdlet' }).Count -eq 3) ("cmdlets are still caught through aliases and a literal & call (got " + @($f | Where-Object { $_.Rule -eq 'Cmdlet' }).Count + ")")
$f = Test-PkWorkerSafety -WorkerScript { param($Id) Start-Sleep -Milliseconds 5; @(1, 2) | Where-Object { $_ -gt 1 } | ForEach-Object { $_ }; Get-HarnessUsesCmdlet } -SharedFunctionNames @('Get-HarnessUsesCmdlet')
Test-Assert ($f.Count -eq 0) ("cmdlets, including ones that take a scriptblock, pass the check in the worker body and in shared functions (got " + $f.Count + " finding(s))")
$f = Test-PkWorkerSafety -WorkerScript { param($Id) Get-HarnessDelegate } -SharedFunctionNames @('Get-HarnessDelegate')
Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'DelegateCast') "a scriptblock-to-delegate cast is flagged"
$f = Test-PkWorkerSafety -WorkerScript { param($Id) Get-HarnessIgnored } -SharedFunctionNames @('Get-HarnessIgnored')
Test-Assert ($f.Count -eq 0) "'# pk:ignore' on the line accepts a finding"
$f = Test-PkWorkerSafety -WorkerScript { param($Id) Write-PkMessage "x"; if (Test-PkCancelRequested) { return }; $null = Start-PkProcess -FilePath 'x' }
Test-Assert ($f.Count -eq 0) "the kit's own worker functions never need listing"
$threw = $false
try { $null = Invoke-PkRunspaceJobs -WorkerScript { param($Id) Start-Sleep 1 } -JobParams (New-JobList -Jobs @(@{ Id = 1 })) -MaxParallel 1 -CheckWorkerSafety } catch { $threw = ($_.ToString() -like '*Start-Sleep*') }
try { $null = Invoke-PkRunspaceJobs -WorkerScript { param($Id) $null = [Func[int]]{ 1 } } -JobParams (New-JobList -Jobs @(@{ Id = 1 })) -MaxParallel 1 -CheckWorkerSafety } catch { $threw = ($_.ToString() -like '*Func*') }
Test-Assert $threw "-CheckWorkerSafety refuses to launch unsafe worker code, naming the problem"
$threw = $false
try { $null = Invoke-PkRunspaceJobs -WorkerScript { param($Id) $null = [Func[int]]{ 1 } } -JobParams (New-JobList -Jobs @(@{ Id = 1 })) -MaxParallel 1 } catch { $threw = ($_.ToString() -like '*Func*') }
Test-Assert $threw "the worker-safety check runs without being asked"
$out = $null
try { $out = Invoke-PkRunspaceJobs -WorkerScript { param($Id) $null = [Func[int]]{ 1 } } -JobParams (New-JobList -Jobs @(@{ Id = 1 })) -MaxParallel 1 -CheckWorkerSafety:$false } catch { Write-Host " opt-out threw: $_" }
Test-Assert ($null -ne $out -and $out.CrashedCount -eq 0) "-CheckWorkerSafety:`$false turns the check off"
$results = [System.Collections.Generic.List[object]]::new()
$out = Invoke-PkRunspaceJobs -WorkerScript { param($Id) return @(1, 2, 3) | Where-Object { $_ -gt 1 } | Measure-Object -Sum | ForEach-Object { $_.Sum + $Id } } `
-JobParams (New-JobList -Jobs @(@{ Id = 10 }, @{ Id = 20 })) -MaxParallel 2 -ResultHandler { param($r) $results.Add($r) }.GetNewClosure()
Test-Assert ($out.CrashedCount -eq 0 -and (@($results | Sort-Object) -join ',') -eq '15,25') "workers can run cmdlets, including ones that take a scriptblock"
$results = [System.Collections.Generic.List[object]]::new()
$out = Invoke-PkRunspaceJobs -WorkerScript { param($Path) return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash } `
-JobParams (New-JobList -Jobs @(@{ Path = $cmdExe })) -MaxParallel 1 -ResultHandler { param($r) $results.Add($r) }.GetNewClosure()
Test-Assert ($out.CrashedCount -eq 0 -and $results.Count -eq 1 -and $results[0] -eq (Get-FileHash -LiteralPath $cmdExe -Algorithm SHA256).Hash) "a function from a module (Get-FileHash) needn't be shared, and autoloads in the worker"

# A scriptblock passed as a job value runs against the main thread's session
# state - refused at launch, with the pass-as-text workaround.
$mainThreadBlock = { param($x) $x * 2 }
$threw = $false
try { $null = Invoke-PkRunspaceJobs -WorkerScript { param($Fn) } -JobParams (New-JobList -Jobs @(@{ Fn = 'ok' }, @{ Fn = $mainThreadBlock })) -MaxParallel 1 } catch { $threw = ($_.ToString() -like "*job 1*'Fn'*Create*Nothing was started*") }
Test-Assert $threw "a scriptblock job value is refused before any worker starts, naming the job and key"
$threw = $false
try { $null = Invoke-PkRunspaceJobs -WorkerScript { param($Fns) } -JobParams (New-JobList -Jobs @(@{ Fns = @(1, $mainThreadBlock) })) -MaxParallel 1 } catch { $threw = ($_.ToString() -like "*'Fns'*") }
Test-Assert $threw "a scriptblock inside an array job value is refused too"
$threw = $false
$providerState = @{ Given = 0 }
$provider = { if ($providerState.Given -ge 2) { return $null }; $providerState.Given++; if ($providerState.Given -eq 2) { return @{ Fn = $mainThreadBlock } }; return @{ Fn = 'ok' } }.GetNewClosure()
try { $null = Invoke-PkRunspaceJobs -WorkerScript { param($Fn) } -NextJobProvider $provider -MaxParallel 1 } catch { $threw = ($_.ToString() -like "*-NextJobProvider*'Fn'*") }
Test-Assert $threw "a scriptblock job value from -NextJobProvider is refused"
$results = [System.Collections.Generic.List[object]]::new()
$out = Invoke-PkRunspaceJobs -WorkerScript { param($FnText, $Id) $fn = [scriptblock]::Create($FnText); return (& $fn $Id) } `
-JobParams (New-JobList -Jobs @(@{ FnText = $mainThreadBlock.ToString(); Id = 4 })) -MaxParallel 1 -ResultHandler { param($r) $results.Add($r) }.GetNewClosure()
Test-Assert ($out.CrashedCount -eq 0 -and $results.Count -eq 1 -and $results[0] -eq 8) "the pass-as-text workaround runs"
Write-Host ""

# ==============================================================================
Expand Down Expand Up @@ -629,6 +658,12 @@ $f = Test-PkWorkerSafety -WorkerScript { param($Id) $lazy = [System.Lazy[object]
Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'DelegateCast') "a scriptblock passed to a .NET constructor is flagged"
$f = Test-PkWorkerSafety -WorkerScript { param($Id) $x = @(1, 2).ForEach({ $_ }); $y = @(1, 2).Where({ $_ }); $z = { 1 }.GetNewClosure() }
Test-Assert ($f.Count -eq 0) "PowerShell's own .ForEach()/.Where()/.GetNewClosure() are not flagged"
$f = Test-PkWorkerSafety -WorkerScript { param($Id) $factory = { 1 }; $lazy = [System.Lazy[object]]::new($factory) }
Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'DelegateCast' -and $f[0].Text -like '*$factory*') "a variable holding a scriptblock, passed to a .NET constructor, is flagged by name"
$f = Test-PkWorkerSafety -WorkerScript { param($Id) $add = { param($k, $old) $old + 1 }.GetNewClosure(); $d = [System.Collections.Concurrent.ConcurrentDictionary[string,int]]::new(); $null = $d.AddOrUpdate('a', 1, $add) }
Test-Assert ($f.Count -eq 1 -and $f[0].Rule -eq 'DelegateCast') "a variable holding a .GetNewClosure() block, passed to a .NET method, is flagged"
$f = Test-PkWorkerSafety -WorkerScript { param($Id) $sb = { $_ * 2 }; $n = 'x'; $x = @(1, 2).ForEach($sb); $null = $n.Contains($n) }
Test-Assert ($f.Count -eq 0) "a scriptblock variable given to .ForEach(), and other variables given to .NET methods, are not flagged"
$f = Test-PkWorkerSafety -WorkerScript { param($Id) $list = [System.Collections.Generic.List[scriptblock]]::new(); $list.Add({ 1 }) # pk:ignore
}
Test-Assert ($f.Count -eq 0) "'# pk:ignore' accepts a method that really does take a scriptblock"
Expand Down
87 changes: 68 additions & 19 deletions ParallelKit.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,8 @@
# graceful Ctrl+C/window-close handling for Windows PowerShell 5.1.
#
# Use: . "<path>\ParallelKit.ps1" (dot-source it; do NOT Import-Module it)
# Read README.md first - the three worker-code rules there are what keep this
# from deadlocking.
#
# Split off from Manga Conversion Script
# (https://github.com/HalRiveria/Manga-Conversion-Script), where these pieces
# were debugged against real hangs and deadlocks. No dependency on that project.
# Read README.md first - the two worker-code rules there are what keep this
# from hanging or crashing.
#
# Works under Set-StrictMode: job hashtables are read as $job['Name'], never
# $job.Name, because under strict mode the dot form THROWS for a key that
Expand Down Expand Up @@ -1350,7 +1346,8 @@ function Invoke-PkRunspaceJobs {
[scriptblock]$JobCrashedHandler = $null,
[int]$DrainTimeoutSeconds = 60,
[string]$AbortWaitMessage,
[switch]$CheckWorkerSafety
# On by default; -CheckWorkerSafety:$false turns it off.
[switch]$CheckWorkerSafety = $true
)

if ($null -eq $JobParams -and $null -eq $NextJobProvider) {
Expand Down Expand Up @@ -1408,6 +1405,10 @@ function Invoke-PkRunspaceJobs {
throw ("Invoke-PkRunspaceJobs: job " + $jobIndex + " supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it. Nothing was started.")
}
}
if ($CheckWorkerSafety) {
$sbKey = Get-PkScriptBlockJobKey -Job $JobParams[$jobIndex]
if ($null -ne $sbKey) { throw ((Get-PkScriptBlockJobMessage -Job ("job " + $jobIndex) -Key $sbKey) + " Nothing was started.") }
}
}
}

Expand Down Expand Up @@ -1482,6 +1483,10 @@ function Invoke-PkRunspaceJobs {
throw ("Invoke-PkRunspaceJobs: a job supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it.")
}
}
if ($CheckWorkerSafety -and $null -eq $JobParams) {
$sbKey = Get-PkScriptBlockJobKey -Job $nextJob
if ($null -ne $sbKey) { throw (Get-PkScriptBlockJobMessage -Job "a job from -NextJobProvider" -Key $sbKey) }
}

$ps = [PowerShell]::Create()
$ps.RunspacePool = $pool
Expand Down Expand Up @@ -1573,7 +1578,7 @@ function Invoke-PkRunspaceJobs {
}

# ==============================================================================
# Worker-safety check (see README.md, "The three rules")
# Worker-safety check (see README.md, "The two rules")
# ==============================================================================

function Get-PkCommandKind {
Expand All @@ -1585,22 +1590,51 @@ function Get-PkCommandKind {
$hops++
}
if ($null -eq $cmd) { return 'Unknown' }
# A function from a module (Get-FileHash, say) autoloads in a worker
# like a cmdlet does; only functions defined in your session need sharing.
if ($cmd.CommandType -eq 'Function' -and -not [string]::IsNullOrEmpty($cmd.ModuleName)) { return 'ModuleFunction' }
return [string]$cmd.CommandType
}

function Get-PkScriptBlockJobKey {
# The first key of a job hashtable whose value is a scriptblock, or holds
# one (an array/list item or a hashtable value - one level down), else
# $null. A scriptblock belongs to the runspace that made it: run in a
# worker it executes against the main thread's session state, which can
# corrupt it (the kit's own functions vanish mid-run), crash, or hang.
param([Parameter(Mandatory)][hashtable]$Job)
foreach ($key in $Job.Keys) {
$value = $Job[$key]
if ($value -is [scriptblock]) { return [string]$key }
if ($value -is [System.Collections.IDictionary]) {
foreach ($inner in $value.Values) { if ($inner -is [scriptblock]) { return [string]$key } }
} elseif ($value -is [System.Collections.IList]) {
foreach ($inner in $value) { if ($inner -is [scriptblock]) { return [string]$key } }
}
}
return $null
}

function Get-PkScriptBlockJobMessage {
# Shared by the fixed-list and -NextJobProvider checks in Invoke-PkRunspaceJobs.
param([string]$Job, [string]$Key)
return ("Invoke-PkRunspaceJobs: " + $Job + " passes a scriptblock in '" + $Key + "'. A scriptblock made on the main thread and run in a worker can corrupt, crash or hang the run. " +
"Pass its text instead (`$sb.ToString()) and rebuild it in the worker with [scriptblock]::Create(`$text), or pass -CheckWorkerSafety:`$false to skip this check.")
}

function Test-PkWorkerSafety {
# Static check of a worker script and the functions shared with it.
# Returns a list of findings (empty = clean); each is
# @{ Rule; Where; Line; Text }. Put "# pk:ignore" on a line to accept it.
#
# MissingFunction the worker calls one of your functions that isn't in -SharedFunctionNames
# ScriptVariable a shared function (or a nested scriptblock or function in the worker) uses $script:
# Cmdlet worker-reachable code calls a cmdlet
# DelegateCast worker-reachable code casts a scriptblock to a .NET delegate, or passes
# one straight into a .NET method/constructor (which converts it to one)
#
# Not visible from here: callbacks handed to a worker as job parameters,
# and commands invoked through a variable (& $cmd).
# Not visible from here: commands invoked through a variable (& $cmd).
# Scriptblocks handed to a worker as job values are refused separately,
# by Invoke-PkRunspaceJobs (see Get-PkScriptBlockJobKey).
param(
[Parameter(Mandatory)]
[scriptblock]$WorkerScript,
Expand Down Expand Up @@ -1639,9 +1673,6 @@ function Test-PkWorkerSafety {
if ($kind -eq 'Function' -and -not $listed.Contains($cmdName)) {
$findings.Add(@{ Rule = 'MissingFunction'; Where = $t.Where; Line = $c.Extent.StartLineNumber
Text = ($t.Where + " (line " + $c.Extent.StartLineNumber + ") calls '" + $cmdName + "', which is not in -SharedFunctionNames") })
} elseif ($kind -eq 'Cmdlet') {
$findings.Add(@{ Rule = 'Cmdlet'; Where = $t.Where; Line = $c.Extent.StartLineNumber
Text = ($t.Where + " (line " + $c.Extent.StartLineNumber + ") calls the cmdlet '" + $cmdName + "' - use a raw .NET call instead") })
}
}

Expand Down Expand Up @@ -1671,26 +1702,44 @@ function Test-PkWorkerSafety {
if ($cv.Type.TypeName.Name -notmatch '^(System\.)?(Func|Action|Predicate|Comparison|Converter)(\[|`|$)') { continue }
if ($cv.Extent.StartScriptPosition.Line -match '#\s*pk:ignore') { continue }
$findings.Add(@{ Rule = 'DelegateCast'; Where = $t.Where; Line = $cv.Extent.StartLineNumber
Text = ($t.Where + " (line " + $cv.Extent.StartLineNumber + ") casts a scriptblock to the delegate [" + $cv.Type.TypeName.Name + "] - this can deadlock in a worker") })
Text = ($t.Where + " (line " + $cv.Extent.StartLineNumber + ") casts a scriptblock to the delegate [" + $cv.Type.TypeName.Name + "] - this can hang or crash a worker") })
}

# The implicit form: a scriptblock literal passed as an argument to
# a method or constructor - $dict.GetOrAdd($k, { ... }),
# [System.Lazy[object]]::new({ ... }). No cast is written, but .NET
# receives a delegate all the same. .ForEach()/.Where() are
# PowerShell's own and take real scriptblocks, so they're fine.
# The same goes for a variable assigned a scriptblock literal (or
# { }.GetNewClosure()) anywhere in this code, then passed as
# $lazy = [System.Lazy[object]]::new($factory). By name, not by flow:
# a variable reassigned to something else in between is still flagged.
$scriptBlockVars = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
foreach ($assign in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.AssignmentStatementAst] }, $true)) {
$target = $assign.Left
if ($target -is [System.Management.Automation.Language.ConvertExpressionAst]) { $target = $target.Child }
if ($target -isnot [System.Management.Automation.Language.VariableExpressionAst]) { continue }
$rhs = $assign.Right
if ($rhs -is [System.Management.Automation.Language.PipelineAst] -and $rhs.PipelineElements.Count -eq 1) { $rhs = $rhs.PipelineElements[0] }
if ($rhs -is [System.Management.Automation.Language.CommandExpressionAst]) { $rhs = $rhs.Expression }
if ($rhs -is [System.Management.Automation.Language.InvokeMemberExpressionAst] -and [string]$rhs.Member.Extent.Text -eq 'GetNewClosure') { $rhs = $rhs.Expression }
if ($rhs -is [System.Management.Automation.Language.ScriptBlockExpressionAst]) { $null = $scriptBlockVars.Add($target.VariablePath.UserPath) }
}
foreach ($call in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.InvokeMemberExpressionAst] }, $true)) {
if ($null -eq $call.Arguments) { continue }
$memberName = [string]$call.Member.Extent.Text
if ($memberName -in @('ForEach', 'Where')) { continue }
$passesScriptBlock = $false
$passed = $null
foreach ($argument in $call.Arguments) {
if ($argument -is [System.Management.Automation.Language.ScriptBlockExpressionAst]) { $passesScriptBlock = $true }
if ($argument -is [System.Management.Automation.Language.ScriptBlockExpressionAst]) { $passed = 'a scriptblock' }
elseif ($argument -is [System.Management.Automation.Language.VariableExpressionAst] -and $scriptBlockVars.Contains($argument.VariablePath.UserPath)) {
$passed = ("the scriptblock in `$" + $argument.VariablePath.UserPath)
}
}
if (-not $passesScriptBlock) { continue }
if ($null -eq $passed) { continue }
if ($call.Extent.StartScriptPosition.Line -match '#\s*pk:ignore') { continue }
$findings.Add(@{ Rule = 'DelegateCast'; Where = $t.Where; Line = $call.Extent.StartLineNumber
Text = ($t.Where + " (line " + $call.Extent.StartLineNumber + ") passes a scriptblock to ." + $memberName + "() - .NET turns it into a delegate, which can deadlock in a worker (add '# pk:ignore' if this method really takes a scriptblock)") })
Text = ($t.Where + " (line " + $call.Extent.StartLineNumber + ") passes " + $passed + " to ." + $memberName + "() - .NET turns it into a delegate, which can hang or crash a worker (add '# pk:ignore' if this method really takes a scriptblock)") })
}
}

Expand Down
Loading
Loading