From 0280736f113f10cf4b030e948dfd86cdda01e047 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:51:04 -0400 Subject: [PATCH 1/3] Load the kit as an in-memory module Dot-sourced, the kit's functions ran in the caller's scope chain. A callback built without .GetNewClosure() resolved names like $results, $state, $job, $outcome, $TimeoutSeconds or $ScriptBlock to the kit's own locals instead of the caller's: silently wrong results, and before v2.1 an endless loop. Every internal helper also landed in the caller's session and tab completion. Dot-sourcing the file now loads its body as an in-memory module named ParallelKit and imports the public functions globally. Nothing is installed or put on $env:PSModulePath. A plain callback now sees the caller's variables and functions, and the internal helpers stay inside the module. Details: - New-Module runs with -ErrorAction Stop, so a throw in the module body (the stale-types check) still stops the script loading the kit. - An earlier copy is removed first, so loading again leaves one module. - The batch's own .GetNewClosure() callbacks take the now-private New-PkBatchResult with them, since a closure only sees global functions. - -SharedFunctionNames and Test-PkWorkerSafety find the caller's functions through the caller's session state, with an unbound scriptblock run by InvokeScript. That finds a script's own functions when it's run from a prompt, functions defined inside a function, and a module's private functions. The last case didn't work before. SessionState.InvokeCommand.GetCommand would have found only global functions. The lookups for all names happen in one call. Callbacks given to an engine inside a worker runspace still need .GetNewClosure(): the kit's functions there are plain copies, not a module. A drive made with New-PSDrive inside a function is no longer visible to the kit; the README says to use -Scope Script. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 3 + ParallelKit.Harness.ps1 | 101 ++++++++++++++++++-- ParallelKit.ps1 | 201 ++++++++++++++++++++++++++++------------ README.md | 17 ++-- 4 files changed, 244 insertions(+), 78 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8ca9f44 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=crlf +*.ps1 text eol=crlf +*.md text eol=crlf diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index 7f00eb8..c587b8e 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -283,7 +283,7 @@ if ($compiled) { Test-Assert ($byName['default'] -eq $u8Text -and $byName['own'] -ne $u8Text) "batch: -OutputEncoding applies to every command, and a command's own OutputEncoding key overrides it" try { [System.IO.File]::Delete($u8Exe) } catch { } # temp file only; best effort } -Test-Assert ((Resolve-PkEncoding -Encoding '437').CodePage -eq 437) "a code page given as text ('437') is accepted" +Test-Assert ((& (Get-Module ParallelKit) Resolve-PkEncoding -Encoding '437').CodePage -eq 437) "a code page given as text ('437') is accepted" # -OutputEncoding mustn't replace PowerShell's own $OutputEncoding inside the # caller's callbacks. $seen = @{ Value = 'unset' } @@ -825,11 +825,11 @@ Remove-Module -Name $modName # Job-value scan: nested values, typed arrays, and live shared collections. $mainThreadBlock = { 1 } -Test-Assert ((Get-PkScriptBlockJobKey -Job @{ Steps = @(@{ Name = 'a'; Run = $mainThreadBlock }) }) -eq 'Steps') "a scriptblock two levels down in a job value is found" -Test-Assert ((Get-PkScriptBlockJobKey -Job @{ Opts = [pscustomobject]@{ OnDone = $mainThreadBlock } }) -eq 'Opts') "a scriptblock in a [pscustomobject] property is found" +Test-Assert ((& (Get-Module ParallelKit) Get-PkScriptBlockJobKey -Job @{ Steps = @(@{ Name = 'a'; Run = $mainThreadBlock }) }) -eq 'Steps') "a scriptblock two levels down in a job value is found" +Test-Assert ((& (Get-Module ParallelKit) Get-PkScriptBlockJobKey -Job @{ Opts = [pscustomobject]@{ OnDone = $mainThreadBlock } }) -eq 'Opts') "a scriptblock in a [pscustomobject] property is found" $bigBytes = [byte[]]::new(20MB) $sw = [System.Diagnostics.Stopwatch]::StartNew() -$key = Get-PkScriptBlockJobKey -Job @{ Data = $bigBytes; Paths = [string[]]::new(200000) } +$key = & (Get-Module ParallelKit) Get-PkScriptBlockJobKey -Job @{ Data = $bigBytes; Paths = [string[]]::new(200000) } $sw.Stop() Test-Assert ($null -eq $key -and $sw.Elapsed.TotalSeconds -lt 1) ("a 20 MB byte[] and a 200k string[] are skipped, not walked (" + [int]$sw.Elapsed.TotalMilliseconds + "ms)") $sink = [System.Collections.ArrayList]::Synchronized([System.Collections.ArrayList]::new()) @@ -840,7 +840,7 @@ $writerHandle = $writer.BeginInvoke() $scanError = $null $sw = [System.Diagnostics.Stopwatch]::StartNew() while ($sw.Elapsed.TotalSeconds -lt 3 -and $null -eq $scanError) { - try { $null = Get-PkScriptBlockJobKey -Job @{ Sink = $sink } } catch { $scanError = $_ } + try { $null = & (Get-Module ParallelKit) Get-PkScriptBlockJobKey -Job @{ Sink = $sink } } catch { $scanError = $_ } } $null = $writer.BeginStop($null, $null) $writer.Dispose() @@ -868,7 +868,7 @@ Test-Assert ($probeThreads -ge 128) ("and raises the thread pool by itself (mini # A script run the way a prompt runs it (& script, or .\script.ps1) holds # the kit's functions in its own scope, where a .GetNewClosure() block can't -# see them. The kit makes them global too, so the batch (whose callbacks are +# see them. The kit's public functions are imported globally, so the batch (whose callbacks are # closures) and a caller's closure that calls a kit function both work. # Needs a fresh process, started with -Command rather than -File. $promptScript = Join-Path $env:TEMP ("pk-prompt-" + [guid]::NewGuid().ToString("N") + ".ps1") @@ -1486,11 +1486,11 @@ Test-Assert ([PkConsoleCtrl]::SkipDrain) "a press during an abort started from c # The force-stop notice says why. Reset-Harness -$reasons = @((Get-PkForceStopReason -TimeoutSeconds 5)) +$reasons = @((& (Get-Module ParallelKit) Get-PkForceStopReason -TimeoutSeconds 5)) $null = [PkConsoleCtrl]::OnInterrupt('Ctrl+C'); [PkConsoleCtrl]::SkipDrain = $true -$reasons += Get-PkForceStopReason -TimeoutSeconds 5 +$reasons += & (Get-Module ParallelKit) Get-PkForceStopReason -TimeoutSeconds 5 [PkConsoleCtrl]::IsCloseEvent = $true -$reasons += Get-PkForceStopReason -TimeoutSeconds 5 +$reasons += & (Get-Module ParallelKit) Get-PkForceStopReason -TimeoutSeconds 5 Test-Assert ($reasons[0] -like '*within 5s*' -and $reasons[1] -like '*pressed again*' -and $reasons[2] -like '*window is closing*') ("the force-stop notice gives the real reason (" + ($reasons -join ' | ') + ")") # Presses from another thread while a run waits, as the native handler would. @@ -2023,6 +2023,89 @@ $null = Invoke-PkProcessDispatch -MaxConcurrent 4 ` Test-Assert ($dispatchSeen.Alive -ge 0 -and $dispatchSeen.Alive -le 15) ("dispatcher: a finished job's output can be freed before its group ends (" + $dispatchSeen.Alive + " of 29 still held)") Write-Host "" +# ============================================================================== +Write-Host "=== Z: the kit loads as an in-memory module ===" -ForegroundColor Cyan +Reset-Harness +# Only the public functions are visible, and they come from the module. +$missingPublic = @($publicFunctions | Where-Object { -not (Get-Command -Name $_ -CommandType Function -ErrorAction SilentlyContinue) }) +$visibleInternal = @('Invoke-PkProcessDispatchCore', 'Add-PkCrashedJob', 'New-PkBatchResult', 'Get-PkCallerFunction' | Where-Object { Get-Command -Name $_ -ErrorAction SilentlyContinue }) +Test-Assert ($missingPublic.Count -eq 0 -and $visibleInternal.Count -eq 0 -and (Get-Command Invoke-PkRun).ModuleName -eq 'ParallelKit') ("only the public functions are visible, from module ParallelKit (missing: " + ($missingPublic -join ', ') + "; internal visible: " + ($visibleInternal -join ', ') + ")") + +# A callback built without .GetNewClosure() sees the caller's variables, +# not ones of the same name inside the kit. +$results = [System.Collections.Generic.List[object]]::new() +$state = @{ Count = 0 } +$TimeoutSeconds = 99 +$sawTimeout = @{ Value = $null } +$plainBatch = Invoke-PkProcessBatch -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }, @{ FilePath = $cmdExe; Arguments = '/c exit 0' }) -OnComplete { + param($c, $r) + $results.Add('mine') + $state.Count++ + $sawTimeout.Value = $TimeoutSeconds +} +$foreign = @($plainBatch.Results | Where-Object { $_ -isnot [System.Management.Automation.PSCustomObject] }).Count +Test-Assert ($plainBatch.Results.Count -eq 2 -and $foreign -eq 0 -and $results.Count -eq 2 -and $state.Count -eq 2) ("a plain -OnComplete's `$results/`$state are the caller's, not the batch's (batch results " + $plainBatch.Results.Count + ", caller's " + $results.Count + "/" + $state.Count + ")") +Test-Assert ($sawTimeout.Value -eq 99) ("a plain -OnComplete reads the caller's `$TimeoutSeconds, not the batch's parameter (" + $sawTimeout.Value + ")") +Remove-Variable -Name TimeoutSeconds +$outcome = @{} +$plainJobs = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams (New-JobList -Jobs @(@{ Id = 1 })) -WorkerScript { param([int]$Id) @{ Id = ("job" + $Id) } } -ResultHandler { param($r) $outcome[$r.Id] = 1 } +Test-Assert ($outcome.Count -eq 1 -and @($plainJobs.PSObject.Properties).Count -eq 4) ("a plain -ResultHandler writes to the caller's `$outcome, not the kit's return value (caller's " + $outcome.Count + ")") +$ScriptBlock = 'mine' +$fromRun = Invoke-PkRun { $ScriptBlock } +Remove-Variable -Name ScriptBlock +Test-Assert ($fromRun -eq 'mine') ("Invoke-PkRun's block reads the caller's `$ScriptBlock, not Invoke-PkRun's parameter (" + $fromRun + ")") + +# -SharedFunctionNames and Test-PkWorkerSafety find the caller's functions +# however the script was started. A script run from a prompt (& script) +# keeps its functions in its own scope, not global, so this needs a fresh +# process started with -Command. +$lookupScript = Join-Path $env:TEMP ("pk-lookup-" + [guid]::NewGuid().ToString("N") + ".ps1") +[System.IO.File]::WriteAllText($lookupScript, @" +. '$kitPath' +function Get-LookupDouble { param(`$N) 2 * `$N } +function Get-LookupState { `$script:lookupState } +`$out = [System.Collections.Generic.List[object]]::new() +`$null = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams @(@{ N = 3 }) -SharedFunctionNames Get-LookupDouble -WorkerScript { param(`$N) Get-LookupDouble -N `$N } -ResultHandler { param(`$r) `$out.Add(`$r) }.GetNewClosure() +Write-Output ('script fn=' + (`$out -join ',')) +function Invoke-LookupOuter { + function Get-LookupInner { param(`$N) 10 * `$N } + `$inner = [System.Collections.Generic.List[object]]::new() + `$null = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams @(@{ N = 4 }) -SharedFunctionNames Get-LookupInner -WorkerScript { param(`$N) Get-LookupInner -N `$N } -ResultHandler { param(`$r) `$inner.Add(`$r) }.GetNewClosure() + Write-Output ('inner fn=' + (`$inner -join ',')) +} +Invoke-LookupOuter +Write-Output ('unlisted=' + (@(Test-PkWorkerSafety -WorkerScript { param(`$N) Get-LookupDouble -N `$N }) | ForEach-Object { `$_.Rule })) +Write-Output ('script var=' + (@(Test-PkWorkerSafety -WorkerScript { param(`$N) Get-LookupState } -SharedFunctionNames Get-LookupState) | ForEach-Object { `$_.Rule })) +"@) +$lookup = Invoke-PkProcessWithTimeout -FilePath $selfExe -Arguments (ConvertTo-PkArgumentString @('-NoProfile', '-ExecutionPolicy', 'Bypass', '-Command', ("& '" + $lookupScript + "'"))) -TimeoutMs 60000 +[System.IO.File]::Delete($lookupScript) +$lookupOut = ([string]$lookup.Stdout + [string]$lookup.Stderr).Trim() -replace '\s+', ' ' +Test-Assert ($lookupOut -match 'script fn=6' -and $lookupOut -match 'inner fn=40') ("in a script run from a prompt, -SharedFunctionNames finds the script's functions and ones defined inside a function (" + $lookupOut + ")") +Test-Assert ($lookupOut -match 'unlisted=MissingFunction' -and $lookupOut -match 'script var=ScriptVariable') "and Test-PkWorkerSafety still reads their source: unlisted ones and `$script: use are flagged" + +# A module's private function can be shared, when the script that loaded +# the kit calls into that module. +$modDir = Join-Path $env:TEMP ("pk-harness-mod-" + $PID) +$null = New-Item -ItemType Directory -Force -Path $modDir +$modPath = Join-Path $modDir 'PkHarnessMod.psm1' +[System.IO.File]::WriteAllText($modPath, @' +function Get-ModPrivate { param($N) 100 + $N } +function Invoke-ModWork { + $out = [System.Collections.Generic.List[object]]::new() + $null = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams @(@{ N = 5 }) -SharedFunctionNames Get-ModPrivate -WorkerScript { param($N) Get-ModPrivate -N $N } -ResultHandler { param($r) $out.Add($r) }.GetNewClosure() + 'module private=' + ($out -join ',') +} +Export-ModuleMember -Function Invoke-ModWork +'@) +$modScript = Join-Path $modDir 'use.ps1' +[System.IO.File]::WriteAllText($modScript, ". '" + $kitPath + "'`r`nImport-Module '" + $modPath + "'`r`nInvoke-ModWork`r`n. '" + $kitPath + "'`r`n'kit modules=' + @(Get-Module -Name ParallelKit).Count`r`n") +$modRun = Invoke-PkProcessWithTimeout -FilePath $selfExe -Arguments (ConvertTo-PkArgumentString @('-NoProfile', '-ExecutionPolicy', 'Bypass', '-Command', ("& '" + $modScript + "'"))) -TimeoutMs 60000 +Remove-Item -LiteralPath $modDir -Recurse -Force -ErrorAction SilentlyContinue +$modOut = ([string]$modRun.Stdout + [string]$modRun.Stderr).Trim() -replace '\s+', ' ' +Test-Assert ($modOut -match 'module private=105') ("a module's private function can be shared with workers (" + $modOut + ")") +Test-Assert ($modOut -match 'kit modules=1') "loading the kit a second time leaves one ParallelKit module, not two" +Write-Host "" + Set-PkMessageHandler -Handler $null Reset-PkAbortState diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index 02c2a76..a9ca516 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -5,7 +5,7 @@ # Copyright (c) 2026 HalRiveria. MIT License: keep this notice with any copy; # the full text is in LICENSE in the repository. # -# Use: . "\ParallelKit.ps1" (dot-source it; do NOT Import-Module it) +# Use: . "\ParallelKit.ps1" (dot-source it; it loads itself as an in-memory module) # Read README.md first - the two worker-code rules there are what keep this # from hanging or crashing. # @@ -25,6 +25,17 @@ if ($ExecutionContext.SessionState.LanguageMode -ne 'FullLanguage') { "Ask whoever manages the machine to allow this script, or run it where that policy doesn't apply.") } +# Everything below loads as an in-memory module named ParallelKit. The kit's +# own frames then live in the module's session state, so a callback you pass +# in sees your variables, never the kit's locals, and the internal helpers +# stay out of your session. The public functions are imported globally, so +# they work from any .GetNewClosure() block and for the rest of the session. +# A copy loaded earlier in this session is dropped first, so only one is +# listed. -ErrorAction Stop: without it, a throw in the module body (the +# stale-types check below, say) wouldn't stop the script that loads the kit. +Get-Module -Name ParallelKit | Remove-Module +New-Module -Name ParallelKit -ErrorAction Stop -ScriptBlock { + # ============================================================================== # Native types (guarded so the file can be dot-sourced more than once) # ============================================================================== @@ -632,10 +643,6 @@ function Invoke-PkRun { # # For the length of the run it also keeps the PC from idle-sleeping and turns off # the console's QuickEdit mode. Whatever the block outputs is passed through. - # - # The block runs inside this function, so a variable of yours named $ScriptBlock, - # $KeepAwake or $DisableQuickEdit reads this function's value instead. - # Add .GetNewClosure() to the block if that could happen. # .PARAMETER ScriptBlock # The work to run. # .PARAMETER DisableQuickEdit @@ -1669,6 +1676,9 @@ function Invoke-PkProcessBatch { $results = [System.Collections.Generic.List[object]]::new() $state = @{ Given = $false } + # The closures below can't see the module's private functions (a + # .GetNewClosure() block only sees global ones), so take this one along. + $pkNewBatchResult = ${function:New-PkBatchResult} $groupProvider = { if ($state.Given) { return $null } @@ -1693,7 +1703,7 @@ function Invoke-PkProcessBatch { try { $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments $job['Arguments'] -WorkingDirectory $job['WorkingDirectory'] -OutputEncoding $job['OutputEncoding'] } catch { - $results.Add((New-PkBatchResult -Command $job['Command'] -StartError ([string]$_))) + $results.Add((& $pkNewBatchResult -Command $job['Command'] -StartError ([string]$_))) throw } $job['Proc'] = $started.Proc; $job['StdoutTask'] = $started.StdoutTask; $job['StderrTask'] = $started.StderrTask @@ -1703,7 +1713,7 @@ function Invoke-PkProcessBatch { $jobFinalize = { param($job, $pendingQueue) - $r = New-PkBatchResult -Command $job['Command'] -Completed (Complete-PkProcess -Job $job) -TimedOut ([bool]$job['TimedOut']) + $r = & $pkNewBatchResult -Command $job['Command'] -Completed (Complete-PkProcess -Job $job) -TimedOut ([bool]$job['TimedOut']) $results.Add($r) if ($null -ne $OnComplete) { $null = & $OnComplete $job['Command'] $r } return @{ Success = ($r.ExitCode -eq 0) } @@ -1711,7 +1721,7 @@ function Invoke-PkProcessBatch { $jobAbortCleanup = { param($job) - $results.Add((New-PkBatchResult -Command $job['Command'] -Aborted)) + $results.Add((& $pkNewBatchResult -Command $job['Command'] -Aborted)) }.GetNewClosure() $dispatchParams = @{ @@ -1765,8 +1775,8 @@ function Get-PkHandlerParamCount { function Add-PkCrashedJob { # A worker threw: count it, record it with its job on the run's outcome, # announce it, and tell -JobCrashedHandler. Shared by the main loop and - # the abort drain. pk-prefixed parameters: a handler built without - # .GetNewClosure() can see them. + # the abort drain. pk-prefixed parameters: when the kit was dot-sourced + # rather than a module, a handler built without .GetNewClosure() saw them. param( [Parameter(Mandatory)][System.Collections.Specialized.OrderedDictionary]$PkOutcome, [hashtable]$PkJob, @@ -1801,10 +1811,10 @@ function Receive-PkCompletedHandles { $stillActive = [System.Collections.Generic.List[hashtable]]::new() foreach ($h in $Handles) { if ($h.Handle.IsCompleted) { - # Not $results: a -ResultHandler built without .GetNewClosure() sees - # this function's locals, and $results.Add($r) there would grow the - # collection this loop walks, so it never ends. Same in - # Invoke-PkRunspaceJobs. + # Not $results: when the kit was dot-sourced rather than a module, a + # -ResultHandler built without .GetNewClosure() saw this function's + # locals, and $results.Add($r) there grew the collection this loop + # walks, so it never ended. Same in Invoke-PkRunspaceJobs. $pkWorkerOutput = $null try { $pkWorkerOutput = $h.PS.EndInvoke($h.Handle) @@ -1982,7 +1992,7 @@ function Invoke-PkRunspaceJobs { if ($null -ne $JobParams -and $JobParams.Count -eq 0) { return [pscustomobject]$outcome } if ($CheckWorkerSafety) { - $safetyFindings = Test-PkWorkerSafety -WorkerScript $WorkerScript -SharedFunctionNames $SharedFunctionNames + $safetyFindings = Test-PkWorkerSafetyCore -WorkerScript $WorkerScript -SharedFunctionNames $SharedFunctionNames -SessionState $PSCmdlet.SessionState if ($safetyFindings.Count -gt 0) { $safetyLines = foreach ($f in $safetyFindings) { " " + $f.Text } throw ("Invoke-PkRunspaceJobs: worker code breaks the worker-safety rules (see README.md):`r`n" + ($safetyLines -join "`r`n")) @@ -2039,13 +2049,19 @@ function Invoke-PkRunspaceJobs { $allShared = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) foreach ($n in (Get-PkWorkerSafeFunctionNames)) { $null = $allShared.Add($n) } foreach ($n in $SharedFunctionNames) { $null = $allShared.Add($n) } + # The kit's worker functions come from this module; yours from where + # you called this (a script's own functions, functions inside a + # function, a module's private ones). + $kitWorkerNames = [System.Collections.Generic.HashSet[string]]::new([string[]]@(Get-PkWorkerSafeFunctionNames), [System.StringComparer]::OrdinalIgnoreCase) + $callerFns = Get-PkCallerFunction -SessionState $PSCmdlet.SessionState -Name @($allShared | Where-Object { -not $kitWorkerNames.Contains($_) }) foreach ($sharedFn in $allShared) { - try { - $fnDef = (Get-Item ("Function:\" + $sharedFn) -ErrorAction Stop).Definition - } catch { - throw ("Invoke-PkRunspaceJobs: -SharedFunctionNames entry '" + $sharedFn + "' does not resolve to a defined function - check for a typo, or that the file defining it was dot-sourced before this call. Original error: " + $_) + $fnInfo = $null + if ($kitWorkerNames.Contains($sharedFn)) { $fnInfo = Get-Item -LiteralPath ("Function:\" + $sharedFn) -ErrorAction SilentlyContinue } + else { $fnInfo = $callerFns[$sharedFn] } + if ($null -eq $fnInfo) { + throw ("Invoke-PkRunspaceJobs: -SharedFunctionNames entry '" + $sharedFn + "' does not resolve to a defined function - check for a typo, or that the file defining it was dot-sourced before this call.") } - $iss.Commands.Add([System.Management.Automation.Runspaces.SessionStateFunctionEntry]::new($sharedFn, $fnDef)) + $iss.Commands.Add([System.Management.Automation.Runspaces.SessionStateFunctionEntry]::new($sharedFn, $fnInfo.Definition)) } # Global variables in every worker runspace: how kit functions know # they're in a worker, and where they find the run's token. @@ -2128,10 +2144,11 @@ function Invoke-PkRunspaceJobs { $stillPending = [System.Collections.Generic.List[hashtable]]::new() foreach ($h in $handles) { if ($h.Handle.IsCompleted) { - # Not $results: a -ResultHandler or -JobCrashedHandler built - # without .GetNewClosure() sees this function's locals, and - # $results.Add($r) there would grow the collection this loop - # walks, so it never ends. Same in Receive-PkCompletedHandles. + # Not $results: when the kit was dot-sourced rather than a + # module, a -ResultHandler or -JobCrashedHandler built without + # .GetNewClosure() saw this function's locals, and + # $results.Add($r) there grew the collection this loop + # walks, so it never ended. Same in Receive-PkCompletedHandles. $pkWorkerOutput = $null try { $pkWorkerOutput = $h.PS.EndInvoke($h.Handle) @@ -2205,24 +2222,73 @@ function Invoke-PkRunspaceJobs { # Worker-safety check (see README.md, "The two rules") # ============================================================================== +function Get-PkCallerFunction { + # Looks function names up in the CALLER's session state, from the caller's + # current scope: a script's own functions (.\script.ps1), functions + # defined inside the calling function, a calling module's private ones. + # Get-Item Function: here would search this module (plus global), and + # SessionState.InvokeCommand.GetCommand only finds global functions. + # An unbound scriptblock (made from text) run with InvokeScript(SessionState) + # runs in that session state, dot-sourced into the caller's current scope. + # So the work happens inside & { } and the names are embedded as quoted + # literals: nothing is assigned in the caller's scope. + # With -FollowAlias, an alias is followed to the function it points at. + # Returns a hashtable: requested name -> FunctionInfo (missing = not found). + param( + [Parameter(Mandatory)][System.Management.Automation.SessionState]$SessionState, + [string[]]$Name, + [switch]$FollowAlias + ) + $found = @{} + $names = @($Name | Where-Object { -not [string]::IsNullOrEmpty($_) }) + if ($names.Count -eq 0) { return $found } + $q = [string][char]39 + $quoted = foreach ($n in $names) { $q + $n.Replace($q, $q + $q) + $q } + $follow = '$false' + if ($FollowAlias) { $follow = '$true' } + $text = @' +& { + foreach ($pkName in @(__NAMES__)) { + $pkCmd = $pkName + if (__FOLLOW__) { + $pkHops = 0 + $pkAlias = Get-Item -LiteralPath ('Alias:' + $pkCmd) -ErrorAction SilentlyContinue + while ($null -ne $pkAlias -and $pkHops -lt 5) { + $pkCmd = $pkAlias.Definition + $pkAlias = Get-Item -LiteralPath ('Alias:' + $pkCmd) -ErrorAction SilentlyContinue + $pkHops++ + } + } + $pkFn = Get-Item -LiteralPath ('Function:' + $pkCmd) -ErrorAction SilentlyContinue + if ($null -ne $pkFn) { [pscustomobject]@{ Name = $pkName; Fn = $pkFn } } + } +} +'@ + $text = $text.Replace('__NAMES__', ($quoted -join ', ')).Replace('__FOLLOW__', $follow) + $lookup = [scriptblock]::Create($text) + foreach ($hit in $ExecutionContext.InvokeCommand.InvokeScript($SessionState, $lookup, @())) { + $found[[string]$hit.Name] = $hit.Fn + } + return $found +} + function Get-PkCommandKind { - # What a worker would need to run $Name: 'Function' (one of your session's - # functions - must be shared), 'UnloadableModuleFunction' (from a module a - # worker can't autoload), or 'Other' (cmdlet, program, autoloadable module - # function, or unknown). Reads the Alias: and Function: drives directly: - # Get-Command on a name that doesn't resolve searches every module on - # $env:PSModulePath, and the check runs on every Invoke-PkRunspaceJobs call. - param([string]$Name) - $hops = 0 - $alias = Get-Item -LiteralPath ("Alias:\" + $Name) -ErrorAction SilentlyContinue - while ($null -ne $alias -and $hops -lt 5) { - $Name = $alias.Definition - $alias = Get-Item -LiteralPath ("Alias:\" + $Name) -ErrorAction SilentlyContinue - $hops++ - } - $fn = Get-Item -LiteralPath ("Function:\" + $Name) -ErrorAction SilentlyContinue + # What a worker would need to run a command, given what the caller's + # session resolves its name to ($Fn from Get-PkCallerFunction -FollowAlias, + # or $null): 'Function' (one of your session's functions - must be + # shared), 'UnloadableModuleFunction' (from a module a worker can't + # autoload), or 'Other' (cmdlet, program, autoloadable module function, + # or unknown). Works from the Alias: and Function: drives rather than + # Get-Command: Get-Command on a name that doesn't resolve searches every + # module on $env:PSModulePath, and the check runs on every + # Invoke-PkRunspaceJobs call. + param([System.Management.Automation.FunctionInfo]$Fn) + $fn = $Fn if ($null -eq $fn) { return 'Other' } if ([string]::IsNullOrEmpty($fn.ModuleName)) { return 'Function' } + # The kit's own: a worker gets only the worker-safe ones, so treat the + # rest like a function of yours (needs listing), as before the kit was a module. + if ($fn.ModuleName -eq 'ParallelKit') { return 'Function' } # A module function (Get-FileHash, say) autoloads in a worker like a # cmdlet does - but only if the module can be found by name. One imported # by path from elsewhere can't. @@ -2319,6 +2385,18 @@ function Test-PkWorkerSafety { [scriptblock]$WorkerScript, [string[]]$SharedFunctionNames = @() ) + # Your functions are looked up where you called this from. + return ,(Test-PkWorkerSafetyCore -WorkerScript $WorkerScript -SharedFunctionNames $SharedFunctionNames -SessionState $PSCmdlet.SessionState) +} + +function Test-PkWorkerSafetyCore { + # Test-PkWorkerSafety's body. -SessionState is the caller's, for finding + # their functions (Invoke-PkRunspaceJobs passes its own caller's). + param( + [Parameter(Mandatory)][scriptblock]$WorkerScript, + [string[]]$SharedFunctionNames = @(), + [Parameter(Mandatory)][System.Management.Automation.SessionState]$SessionState + ) $findings = [System.Collections.Generic.List[object]]::new() $kitNames = [System.Collections.Generic.HashSet[string]]::new([string[]]@(Get-PkWorkerSafeFunctionNames), [System.StringComparer]::OrdinalIgnoreCase) @@ -2329,9 +2407,10 @@ function Test-PkWorkerSafety { # What to check: the worker body, plus every shared function of yours. $targets = [System.Collections.Generic.List[hashtable]]::new() $targets.Add(@{ Where = 'worker script'; Ast = $WorkerScript.Ast; IsWorkerBody = $true }) + $callerFns = Get-PkCallerFunction -SessionState $SessionState -Name @($SharedFunctionNames | Where-Object { -not $kitNames.Contains($_) }) foreach ($n in $SharedFunctionNames) { if ($kitNames.Contains($n)) { continue } - $fn = Get-Item ("Function:\" + $n) -ErrorAction SilentlyContinue + $fn = $callerFns[$n] if ($null -eq $fn) { $findings.Add([pscustomobject][ordered]@{ Rule = 'MissingFunction'; Where = '-SharedFunctionNames'; Line = 0; Text = ("-SharedFunctionNames lists '" + $n + "', which is not a defined function") }) continue @@ -2339,6 +2418,16 @@ function Test-PkWorkerSafety { $targets.Add(@{ Where = ("function " + $n); Ast = $fn.ScriptBlock.Ast; IsWorkerBody = $false }) } + # Every command name the code calls, resolved in the caller's session in + # one lookup rather than one per name. + $cmdNames = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($t in $targets) { + foreach ($c in $t.Ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.CommandAst] }, $true)) { + $cmdName = $c.GetCommandName() + if ($cmdName) { $null = $cmdNames.Add($cmdName) } + } + } + $resolvedFns = Get-PkCallerFunction -SessionState $SessionState -Name @($cmdNames) -FollowAlias $kindCache = @{} foreach ($t in $targets) { $ast = $t.Ast @@ -2347,7 +2436,7 @@ function Test-PkWorkerSafety { $cmdName = $c.GetCommandName() if (-not $cmdName) { continue } if ($c.Extent.StartScriptPosition.Line -match '#\s*pk:ignore') { continue } - if (-not $kindCache.ContainsKey($cmdName)) { $kindCache[$cmdName] = Get-PkCommandKind -Name $cmdName } + if (-not $kindCache.ContainsKey($cmdName)) { $kindCache[$cmdName] = Get-PkCommandKind -Fn $resolvedFns[$cmdName] } $kind = $kindCache[$cmdName] if ($kind -eq 'Function' -and -not $listed.Contains($cmdName)) { $findings.Add([pscustomobject][ordered]@{ Rule = 'MissingFunction'; Where = $t.Where; Line = $c.Extent.StartLineNumber @@ -2435,23 +2524,17 @@ function Test-PkWorkerSafety { } # ============================================================================== -# Make the kit's functions global +# Public functions (the ones README.md documents) # ============================================================================== -# Dot-sourced inside a script run from a prompt (.\script.ps1 or & script), -# the functions above exist only in that script's scope. A .GetNewClosure() -# block - the batch's own callbacks, or yours - can only see global -# functions, so it couldn't find them. Defining each one globally too means -# kit functions work from any closure, however the script was started. -# Only the kit's own functions (Pk-named, plus Initialize-ParallelKit): the -# kit may be pasted into a larger script whose other functions are none of -# its business. Run in a child scope so the loop's variables don't touch -# the caller's. -& { - param($kitAst) - foreach ($fnAst in $kitAst.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false)) { - if ($fnAst.Name -notlike '*-Pk*' -and $fnAst.Name -ne 'Initialize-ParallelKit') { continue } - $fn = Get-Item -LiteralPath ("Function:\" + $fnAst.Name) -ErrorAction SilentlyContinue - if ($null -ne $fn) { $null = Set-Item -Path ("Function:\global:" + $fnAst.Name) -Value $fn.ScriptBlock } - } -} $MyInvocation.MyCommand.ScriptBlock.Ast +# Imported globally (Import-Module -Global below): a .GetNewClosure() block +# can only see global functions, so this is what lets the batch's own +# callbacks, and yours, call them however your script was started. +Export-ModuleMember -Function @( + 'Set-PkMessageHandler', 'Write-PkMessage', 'Initialize-ParallelKit', 'Register-PkCancellationSource', + 'Unregister-PkCancellationSource', 'Reset-PkAbortState', 'Request-PkAbort', 'Test-PkAbortSignal', 'Test-PkCancelRequested', + 'Get-PkAbortTriggerLabel', 'Invoke-PkRun', 'ConvertTo-PkArgumentString', 'Start-PkProcess', 'Stop-PkProcessTree', + 'Wait-PkProcess', 'Complete-PkProcess', 'Invoke-PkProcessWithTimeout', 'Invoke-PkProcessDispatch', 'Invoke-PkProcessBatch', + 'Invoke-PkRunspaceJobs', 'Test-PkWorkerSafety') + +} | Import-Module -Global diff --git a/README.md b/README.md index 9666598..77fcb75 100644 --- a/README.md +++ b/README.md @@ -95,9 +95,9 @@ It works whether or not your script uses `Set-StrictMode`. The version is in the comment block at the top of the file, so a copy kept in your project still says which one it is. That block also carries the licence notice; keep it with the file. -Dot-source it; don't `Import-Module` it. As a module it couldn't see the functions you list in `-SharedFunctionNames`. +Dot-source it. It loads itself as an in-memory module named `ParallelKit`: nothing is installed and nothing goes on `$env:PSModulePath`, but `Get-Module` lists it while it's loaded. Dot-sourcing it again replaces the copy already loaded. -The kit's functions are also defined globally, so they stay available for the rest of that PowerShell session, and a `.GetNewClosure()` block can call them however your script was started. +Its public functions are imported globally, so they stay available for the rest of that PowerShell session, and a `.GetNewClosure()` block can call them however your script was started. Its internal helpers stay inside the module. `Initialize-ParallelKit` is optional. It registers the Ctrl+C handler, sets up child-process cleanup and sizes the thread pool up front; the engines set up what they need on first use either way. It returns an object with `AbortHandler`, `ChildProcessJob` and `ThreadPool`, each `$true` if that part worked (a failed abort handler or child-process cleanup also prints a warning). @@ -125,7 +125,7 @@ Long parameter lists in these examples use splatting (`@batchParams`) instead of Each command is a hashtable with `FilePath`, `Arguments` and optionally `WorkingDirectory`, plus any keys of your own. `Arguments` is passed to the program as one string, exactly as written. `ConvertTo-PkArgumentString @(...)` builds that string from a list, quoting each argument so the program gets exactly that list back, spaces, quotes and trailing backslashes included. It follows the rules most Windows programs use; `cmd.exe /c` parses its own way. Programs run hidden, with their output captured. -Programs run in your current location (`Get-Location`) unless a command says otherwise, and a relative `WorkingDirectory`, or a `FilePath` with a folder in it like `.\tools\cjxl.exe`, is relative to that location too. Drives PowerShell knows, like `~` or one made with `New-PSDrive`, work in both. A bare name like `cjxl.exe` is found the way Windows finds programs, mostly on `PATH`. A `.bat` or `.cmd` program can't run in a network-share (`\\server\share`) folder: `cmd.exe` falls back to `C:\Windows` there, so give such commands a local `WorkingDirectory`. +Programs run in your current location (`Get-Location`) unless a command says otherwise, and a relative `WorkingDirectory`, or a `FilePath` with a folder in it like `.\tools\cjxl.exe`, is relative to that location too. Drives PowerShell knows, like `~` or one made with `New-PSDrive`, work in both. A drive you make with `New-PSDrive` inside a function is visible only to that function, so make it with `-Scope Script` (or `Global`) if the kit should use it. A bare name like `cjxl.exe` is found the way Windows finds programs, mostly on `PATH`. A `.bat` or `.cmd` program can't run in a network-share (`\\server\share`) folder: `cmd.exe` falls back to `C:\Windows` there, so give such commands a local `WorkingDirectory`. `Invoke-PkProcessBatch` returns an object with `Results`, `Aborted` and `ForceKilledCount`. Each result is an object with `Command`, `ExitCode`, `Stdout`, `Stderr`, `StreamTimedOut`, `StreamError`, `StartError`, `Aborted` and `TimedOut`, every field always present, so `$batch.Results | Format-Table` reads well: @@ -254,7 +254,7 @@ All of these callbacks run on whichever thread called the dispatcher: the main t - An exception from one of your callbacks stops the dispatcher: running processes are killed and the exception is rethrown. - Returns an object with `GroupResults`, `Aborted`, `ForceKilledCount` and `JobStarterFailedCount`. `GroupResults` holds one object with `GroupInfo`, `Completed` and `Aborted` per group, where `GroupInfo` is the group value. -Closures: build every callback that uses your local variables with `{ ... }.GetNewClosure()`. Without it, a variable name that happens to match one inside the kit silently reads the kit's value instead of yours. And keep any state a callback changes in a hashtable (`$state = @{ Count = 0 }`, then `$state.Count++`). A plain variable assigned inside a closure doesn't stick between calls. A closure can't see functions your script defines when the script is run from a prompt (`.\script.ps1`); the kit's own functions always work, but define any of yours that a closure calls as `function global:Get-Thing { ... }`. +Callbacks: a plain `{ ... }` callback sees your variables and functions from where you called the kit, never the kit's own internals, so `.GetNewClosure()` isn't needed for that. The one exception is a callback you give to an engine inside a worker runspace (a dispatcher per runspace job, say): there, build it with `.GetNewClosure()`, or a variable name that matches one inside the kit reads the kit's value instead of yours. Either way, keep any state a callback changes in a hashtable (`$state = @{ Count = 0 }`, then `$state.Count++`). A plain variable assigned inside a callback doesn't stick between calls. A `.GetNewClosure()` block can't see functions your script defines when the script is run from a prompt (`.\script.ps1`); the kit's own functions always work, but define any of yours that a closure calls as `function global:Get-Thing { ... }`. ## Single processes @@ -301,9 +301,6 @@ Three things to know: - **Your script carries on after an abort.** The engines return with `Aborted = $true` and the rest of your script runs. If an abort should end it, say so: `if ($batch.Aborted) { exit 1 }`. `Invoke-PkRun` also turns off the console's QuickEdit mode for the length of the run (`-DisableQuickEdit $false` to opt out). With QuickEdit on in the classic console window, one stray click starts a text selection, and the script freezes at its next console write until the selection is cleared. That looks exactly like a hang. It's turned back on afterwards if it was on before. - -The block you give `Invoke-PkRun` runs inside it, so a variable of yours named like one of its own (`$ScriptBlock`, `$KeepAwake`, `$DisableQuickEdit`) reads its value instead of yours. Add `.GetNewClosure()` to the block if that could happen. - Other pieces: `Request-PkAbort` triggers the same abort from code. `Test-PkAbortSignal` asks whether one is under way. `Register-PkCancellationSource` adds your own `CancellationTokenSource` to the set an abort cancels (a `-Cts` you pass to `Invoke-PkRunspaceJobs` is added automatically; a `-CancelToken` you pass to the dispatcher isn't cancelled by an abort, but the dispatcher stops either way). `Get-PkAbortTriggerLabel` says which kind of abort it was. ## Choosing limits @@ -354,9 +351,9 @@ PowerShell 7 on Windows, yes: Windows PowerShell 5.1 is the main target, but the No. Nothing it does needs elevation. -**Why a file to dot-source instead of a module?** +**Is it a module?** -As a module it couldn't find the functions you list in `-SharedFunctionNames`, since a module has its own scope. It's also one file you can copy into a project and keep at a known version, with nothing to install on machines where installing modules isn't allowed. +It is. Dot-sourcing the file loads it as an in-memory module, so there's nothing to install and nothing on `$env:PSModulePath`. The functions you list in `-SharedFunctionNames` are found from wherever you call the kit: a script's own functions, functions inside a function, or a module's private functions. You keep one file at a known version, which you can copy into a project even on machines where installing modules isn't allowed. **What happens to running programs when I press Ctrl+C?** @@ -380,7 +377,7 @@ No. The engines take a list of hashtables (`-Commands`, `-JobParams`) rather tha **Which functions are meant to be used?** -The ones in this README: the engines, the [single-process functions](#single-processes), and the abort and message functions named in their sections. Dot-sourcing also brings in internal helpers (`Invoke-PkProcessDispatchCore`, `Get-PkCommandKind` and so on), which show up in tab completion. Those can change between versions; don't call them. +The ones in this README: the engines, the [single-process functions](#single-processes), and the abort and message functions named in their sections. Everything else is an internal helper that stays inside the module and can change between versions. **Does a program I start get my Ctrl+C too?** From 48948ceabe0b376b08f67efc8b1f327646093d99 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:55:35 -0400 Subject: [PATCH 2/3] Still flag worker calls to the kit's private helpers; cross-reference the public lists Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 5 ++++- ParallelKit.ps1 | 13 ++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index c587b8e..e3ff15e 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -503,7 +503,8 @@ Test-Assert ($leaked -eq $false) "each run starts with a clean abort state" Write-Host "" # Every public function has Get-Help help: a synopsis, and a description for -# each of its parameters. +# each of its parameters. The same list is the Export-ModuleMember list at the +# end of ParallelKit.ps1; keep the two in step. $publicFunctions = @('Set-PkMessageHandler', 'Write-PkMessage', 'Initialize-ParallelKit', 'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'Reset-PkAbortState', 'Request-PkAbort', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Get-PkAbortTriggerLabel', 'Invoke-PkRun', 'ConvertTo-PkArgumentString', 'Start-PkProcess', 'Stop-PkProcessTree', @@ -2049,6 +2050,8 @@ Test-Assert ($sawTimeout.Value -eq 99) ("a plain -OnComplete reads the caller's Remove-Variable -Name TimeoutSeconds $outcome = @{} $plainJobs = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams (New-JobList -Jobs @(@{ Id = 1 })) -WorkerScript { param([int]$Id) @{ Id = ("job" + $Id) } } -ResultHandler { param($r) $outcome[$r.Id] = 1 } +$privateCall = @(Test-PkWorkerSafety -WorkerScript { param($N) Invoke-PkMessageHandler -Message 'x' -Level Info }) +Test-Assert ($privateCall.Count -eq 1 -and $privateCall[0].Rule -eq 'MissingFunction') ("worker code calling one of the kit's private helpers is still flagged (" + (@($privateCall | ForEach-Object { $_.Rule }) -join ',') + ")") Test-Assert ($outcome.Count -eq 1 -and @($plainJobs.PSObject.Properties).Count -eq 4) ("a plain -ResultHandler writes to the caller's `$outcome, not the kit's return value (caller's " + $outcome.Count + ")") $ScriptBlock = 'mine' $fromRun = Invoke-PkRun { $ScriptBlock } diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index a9ca516..7e39f8d 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -2436,7 +2436,16 @@ function Test-PkWorkerSafetyCore { $cmdName = $c.GetCommandName() if (-not $cmdName) { continue } if ($c.Extent.StartScriptPosition.Line -match '#\s*pk:ignore') { continue } - if (-not $kindCache.ContainsKey($cmdName)) { $kindCache[$cmdName] = Get-PkCommandKind -Fn $resolvedFns[$cmdName] } + if (-not $kindCache.ContainsKey($cmdName)) { + $fnForKind = $resolvedFns[$cmdName] + if ($null -eq $fnForKind) { + # One of the kit's private helpers: invisible to the caller, + # but a worker can't run it unless it's worker-safe either. + $ownFn = Get-Item -LiteralPath ('Function:' + $cmdName) -ErrorAction SilentlyContinue + if ($null -ne $ownFn -and $ownFn.ModuleName -eq 'ParallelKit') { $fnForKind = $ownFn } + } + $kindCache[$cmdName] = Get-PkCommandKind -Fn $fnForKind + } $kind = $kindCache[$cmdName] if ($kind -eq 'Function' -and -not $listed.Contains($cmdName)) { $findings.Add([pscustomobject][ordered]@{ Rule = 'MissingFunction'; Where = $t.Where; Line = $c.Extent.StartLineNumber @@ -2530,6 +2539,8 @@ function Test-PkWorkerSafetyCore { # Imported globally (Import-Module -Global below): a .GetNewClosure() block # can only see global functions, so this is what lets the batch's own # callbacks, and yours, call them however your script was started. +# The same list is $publicFunctions in ParallelKit.Harness.ps1 (which checks +# each has Get-Help help and is visible); keep the two in step. Export-ModuleMember -Function @( 'Set-PkMessageHandler', 'Write-PkMessage', 'Initialize-ParallelKit', 'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'Reset-PkAbortState', 'Request-PkAbort', 'Test-PkAbortSignal', 'Test-PkCancelRequested', From b93c493d7183374b1d3bdda47529b6f2574f86ba Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:03:54 -0400 Subject: [PATCH 3/3] README: drop the module FAQ entry Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 77fcb75..f9dca3f 100644 --- a/README.md +++ b/README.md @@ -351,10 +351,6 @@ PowerShell 7 on Windows, yes: Windows PowerShell 5.1 is the main target, but the No. Nothing it does needs elevation. -**Is it a module?** - -It is. Dot-sourcing the file loads it as an in-memory module, so there's nothing to install and nothing on `$env:PSModulePath`. The functions you list in `-SharedFunctionNames` are found from wherever you call the kit: a script's own functions, functions inside a function, or a module's private functions. You keep one file at a known version, which you can copy into a project even on machines where installing modules isn't allowed. - **What happens to running programs when I press Ctrl+C?** Inside `Invoke-PkRun`, no new ones start, and the running ones get up to 60 seconds by default to finish. Anything still running after that is killed, and `-JobAbortCleanup` is called for each so you can delete its partial output. Press Ctrl+C again to kill them straight away instead of waiting. See [Abort](#abort).