From c4f5217e926a7ea72fd9f9aa7557d0fc6fad970c Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:05:21 -0400 Subject: [PATCH 1/3] Add -OutputEncoding for decoding program output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Output was always decoded in the console's code page, so a program that writes UTF-8 came back garbled on a default console (code page 437): "café" read as box-drawing characters. -OutputEncoding takes a name ('utf-8', 'utf8'), a code page number or an Encoding, on Start-PkProcess, Invoke-PkProcessWithTimeout and Invoke-PkProcessBatch, plus an OutputEncoding key on batch commands and dispatcher jobs. Unset, behaviour is unchanged, since Windows' own tools write in the console code page. An unknown name fails with a clear message. Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 26 ++++++++++++++++++++++ ParallelKit.ps1 | 49 ++++++++++++++++++++++++++++++++++------- README.md | 2 ++ 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index d2e55e7..270f4f5 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -209,6 +209,32 @@ if ($compiled) { Test-Assert (($received -join '|') -eq ($expected -join '|')) ("a program receives exactly the arguments given (got " + ($received -join ' ') + ")") try { [System.IO.File]::Delete($argvExe) } catch { } # temp file only; best effort } + +# -OutputEncoding: a program that writes UTF-8 bytes comes back intact when +# told so, and garbled when decoded as code page 437 (a default console). +$u8Exe = Join-Path $env:TEMP ("pk-u8-" + [guid]::NewGuid().ToString("N") + ".exe") +$u8Src = [System.IO.Path]::ChangeExtension($u8Exe, '.cs') +[System.IO.File]::WriteAllText($u8Src, 'public static class PkU8 { public static void Main() { string t = new string(new char[] { (char)0x63, (char)0x61, (char)0x66, (char)0xE9, (char)0x20, (char)0x65E5, (char)0x672C, (char)0x8A9E }); byte[] b = new System.Text.UTF8Encoding(false).GetBytes(t); var o = System.Console.OpenStandardOutput(); o.Write(b, 0, b.Length); o.Flush(); var e = System.Console.OpenStandardError(); e.Write(b, 0, b.Length); e.Flush(); } }') +$csc = $null +try { $csc = Invoke-PkProcessWithTimeout -FilePath $cscExe -Arguments (ConvertTo-PkArgumentString @('/nologo', ('/out:' + $u8Exe), $u8Src)) } catch { Write-Host " csc.exe failed to run: $_" } +[System.IO.File]::Delete($u8Src) +$compiled = ($null -ne $csc -and $csc.ExitCode -eq 0 -and [System.IO.File]::Exists($u8Exe)) +Test-Assert $compiled "the UTF-8 test program compiles" +if ($compiled) { + $u8Text = 'caf' + [char]0xE9 + ' ' + [char]0x65E5 + [char]0x672C + [char]0x8A9E + $r = Invoke-PkProcessWithTimeout -FilePath $u8Exe -OutputEncoding 'utf-8' + $codes = { param($s) (([string]$s).ToCharArray() | ForEach-Object { '{0:X4}' -f [int]$_ }) -join ' ' } + Test-Assert ($r.Stdout -eq $u8Text -and $r.Stderr -eq $u8Text) ("-OutputEncoding 'utf-8' decodes a program's UTF-8 output, stdout and stderr (got " + (& $codes $r.Stdout) + " / " + (& $codes $r.Stderr) + ")") + $r = Invoke-PkProcessWithTimeout -FilePath $u8Exe -OutputEncoding 437 + Test-Assert ($r.Stdout -ne $u8Text) "and decoding it as code page 437 garbles it, so the option really takes effect" + $batch = Invoke-PkProcessBatch -OutputEncoding 'utf8' -Commands @(@{ FilePath = $u8Exe; Name = 'default' }, @{ FilePath = $u8Exe; Name = 'own'; OutputEncoding = 437 }) + $byName = @{}; foreach ($br in $batch.Results) { $byName[$br.Command['Name']] = $br.Stdout } + Test-Assert ($byName['default'] -eq $u8Text -and $byName['own'] -ne $u8Text) "batch: -OutputEncoding applies to every command, and a command's own OutputEncoding key overrides it" + try { [System.IO.File]::Delete($u8Exe) } catch { } # temp file only; best effort +} +$threw = $false +try { $null = Invoke-PkProcessBatch -OutputEncoding 'no-such-encoding' -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) } catch { $threw = ($_.ToString() -like "*'no-such-encoding' isn't an encoding*") } +Test-Assert $threw "an unknown -OutputEncoding fails the call with a clear message" Write-Host "" # ============================================================================== diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index 0401054..41d986b 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -569,16 +569,37 @@ function ConvertTo-PkArgumentString { return ($parts -join ' ') } +function Resolve-PkEncoding { + # -OutputEncoding as given (an Encoding, a code page number, or a name + # like 'utf-8' or 'utf8') -> an Encoding, or $null for "not set". + param($Encoding) + if ($null -eq $Encoding -or ($Encoding -is [string] -and $Encoding -eq '')) { return $null } + if ($Encoding -is [System.Text.Encoding]) { return $Encoding } + # .NET Framework doesn't know the name 'utf8'; no BOM either way, since + # this only decodes. + if ($Encoding -is [string] -and $Encoding -in @('utf8', 'utf-8')) { return [System.Text.UTF8Encoding]::new($false) } + try { + if ($Encoding -is [int]) { return [System.Text.Encoding]::GetEncoding([int]$Encoding) } + return [System.Text.Encoding]::GetEncoding([string]$Encoding) + } catch { + throw ("OutputEncoding '" + $Encoding + "' isn't an encoding .NET knows. Use a name like 'utf-8' or 'windows-1252', or a code page number like 65001.") + } +} + function Start-PkProcess { # Starts a hidden child process with stdout/stderr redirected and being # read asynchronously (so a chatty process can't block on a full pipe). + # $OutputEncoding is how the program's output is decoded ('utf-8', a + # code page number, or an Encoding); unset, it's the console's code page. # Returns @{ Proc; StdoutTask; StderrTask; Stopwatch }. param( [Parameter(Mandatory)] [string]$FilePath, [string]$Arguments = "", - [string]$WorkingDirectory = "" + [string]$WorkingDirectory = "", + $OutputEncoding = $null ) + $encoding = Resolve-PkEncoding -Encoding $OutputEncoding # Self-starting: a caller who never ran Initialize-ParallelKit still # gets children that die with the script, and output reads that # aren't starved of threads. Both are cheap no-ops once done. @@ -589,6 +610,10 @@ function Start-PkProcess { $psi.FileName = $FilePath $psi.Arguments = $Arguments $psi.UseShellExecute = $false + if ($null -ne $encoding) { + $psi.StandardOutputEncoding = $encoding + $psi.StandardErrorEncoding = $encoding + } $psi.RedirectStandardOutput = $true $psi.RedirectStandardError = $true $psi.CreateNoWindow = $true @@ -703,11 +728,12 @@ function Invoke-PkProcessWithTimeout { [int]$TimeoutMs = 60000, [int]$StreamTimeoutMs = 15000, [string]$WorkingDirectory = "", + $OutputEncoding = $null, # For a program that has to run BECAUSE of an abort (cleanup, a # restore step): keep going even though the run is being aborted. [switch]$IgnoreAbort ) - $job = Start-PkProcess -FilePath $FilePath -Arguments $Arguments -WorkingDirectory $WorkingDirectory + $job = Start-PkProcess -FilePath $FilePath -Arguments $Arguments -WorkingDirectory $WorkingDirectory -OutputEncoding $OutputEncoding try { # Wait in short slices so Ctrl+C (or the run's cancel token) is # noticed within a moment, not after the whole time limit. @@ -806,7 +832,7 @@ function Invoke-PkProcessDispatch { if ($null -eq $JobStarter) { $JobStarter = { param($job) - $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments ([string]$job['Arguments']) -WorkingDirectory ([string]$job['WorkingDirectory']) + $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments ([string]$job['Arguments']) -WorkingDirectory ([string]$job['WorkingDirectory']) -OutputEncoding $job['OutputEncoding'] $job['Proc'] = $started.Proc; $job['StdoutTask'] = $started.StdoutTask; $job['StderrTask'] = $started.StderrTask $job['Stopwatch'] = $started.Stopwatch return $job @@ -1223,7 +1249,8 @@ function Invoke-PkProcessDispatchCore { function Invoke-PkProcessBatch { # The simple case: run a flat list of commands, $MaxConcurrent at a time. - # Each command is @{ FilePath = ...; Arguments = ...; WorkingDirectory = ... (optional); }. + # Each command is @{ FilePath = ...; Arguments = ...; WorkingDirectory = ... (optional); + # OutputEncoding = ... (optional, overrides -OutputEncoding); }. # $OnComplete { param($command, $result) } fires as each one finishes # (not for one that couldn't start or was stopped by an abort). # $TimeoutSeconds > 0 kills a program still running after that long; its @@ -1241,9 +1268,13 @@ function Invoke-PkProcessBatch { [int]$MaxConcurrent = 4, [scriptblock]$OnComplete = $null, [int]$AbortTimeoutSeconds = 60, - [int]$TimeoutSeconds = 0 + [int]$TimeoutSeconds = 0, + $OutputEncoding = $null ) + # Checked once here, so a bad name fails the call instead of every command. + $null = Resolve-PkEncoding -Encoding $OutputEncoding + if ($TimeoutSeconds -lt 0) { throw "Invoke-PkProcessBatch: -TimeoutSeconds must be 0 (no limit) or more." } $results = [System.Collections.Generic.List[hashtable]]::new() @@ -1260,7 +1291,9 @@ function Invoke-PkProcessBatch { $jobs = [System.Collections.Generic.List[hashtable]]::new() foreach ($c in $Commands) { if ($null -eq $c) { continue } - $jobs.Add(@{ FilePath = $c['FilePath']; Arguments = [string]$c['Arguments']; WorkingDirectory = [string]$c['WorkingDirectory']; Command = $c }) + $encoding = $OutputEncoding + if ($c.ContainsKey('OutputEncoding')) { $encoding = $c['OutputEncoding'] } + $jobs.Add(@{ FilePath = $c['FilePath']; Arguments = [string]$c['Arguments']; WorkingDirectory = [string]$c['WorkingDirectory']; OutputEncoding = $encoding; Command = $c }) } return $jobs }.GetNewClosure() @@ -1268,7 +1301,7 @@ function Invoke-PkProcessBatch { $jobStarter = { param($job) try { - $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments $job['Arguments'] -WorkingDirectory $job['WorkingDirectory'] + $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments $job['Arguments'] -WorkingDirectory $job['WorkingDirectory'] -OutputEncoding $job['OutputEncoding'] } catch { $results.Add(@{ Command = $job['Command']; ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; StartError = [string]$_; Aborted = $false; TimedOut = $false }) throw @@ -1321,7 +1354,7 @@ function Invoke-PkProcessBatch { function Get-PkWorkerSafeFunctionNames { # Kit functions every worker runspace gets automatically. - return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', 'Stop-PkProcessTree', + return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', 'Stop-PkProcessTree', 'Resolve-PkEncoding', 'Start-PkProcess', 'Wait-PkProcess', 'Complete-PkProcess', 'Invoke-PkProcessWithTimeout', 'Invoke-PkProcessDispatch', 'Invoke-PkProcessDispatchCore', 'Invoke-PkProcessBatch', 'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'ConvertTo-PkArgumentString') diff --git a/README.md b/README.md index 59ee3cb..a22c218 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,8 @@ Long parameter lists in these examples use splatting (`@batchParams`) instead of Each command is a hashtable with `FilePath`, `Arguments` and optionally `WorkingDirectory`, plus any keys of your own. `Arguments` is passed to the program as one string, exactly as written. `ConvertTo-PkArgumentString @(...)` builds that string from a list, quoting each argument so the program gets exactly that list back, spaces, quotes and trailing backslashes included. It follows the rules most Windows programs use; `cmd.exe /c` parses its own way. Programs run hidden, with their output captured. +Output is decoded in the console's code page unless you say otherwise, which is right for Windows' own tools. Many cross-platform tools write UTF-8 instead, and their non-ASCII output (accented or Japanese file names, say) comes back garbled. Pass `-OutputEncoding 'utf-8'` for the whole batch, or put an `OutputEncoding` key on one command. It also takes a code page number (`65001`, `437`) or a `[System.Text.Encoding]`, and the same option exists on `Invoke-PkProcessWithTimeout` and `Start-PkProcess` (and as a job key for the dispatcher's default starter). + `Invoke-PkProcessBatch` returns `@{ Results; Aborted; ForceKilledCount }`. Each result is `@{ Command; ExitCode; Stdout; Stderr; StreamTimedOut; StreamError; StartError; Aborted; TimedOut }`, with every field always present: - `Command` is your original command hashtable, so extra keys like `Source` above come back with it. From 780252e3c2075a10a2c0820401f1d75d01278a77 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:05:30 -0400 Subject: [PATCH 2/3] README: move the OutputEncoding paragraph after the timeout one Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a22c218..537e6a3 100644 --- a/README.md +++ b/README.md @@ -119,8 +119,6 @@ Long parameter lists in these examples use splatting (`@batchParams`) instead of Each command is a hashtable with `FilePath`, `Arguments` and optionally `WorkingDirectory`, plus any keys of your own. `Arguments` is passed to the program as one string, exactly as written. `ConvertTo-PkArgumentString @(...)` builds that string from a list, quoting each argument so the program gets exactly that list back, spaces, quotes and trailing backslashes included. It follows the rules most Windows programs use; `cmd.exe /c` parses its own way. Programs run hidden, with their output captured. -Output is decoded in the console's code page unless you say otherwise, which is right for Windows' own tools. Many cross-platform tools write UTF-8 instead, and their non-ASCII output (accented or Japanese file names, say) comes back garbled. Pass `-OutputEncoding 'utf-8'` for the whole batch, or put an `OutputEncoding` key on one command. It also takes a code page number (`65001`, `437`) or a `[System.Text.Encoding]`, and the same option exists on `Invoke-PkProcessWithTimeout` and `Start-PkProcess` (and as a job key for the dispatcher's default starter). - `Invoke-PkProcessBatch` returns `@{ Results; Aborted; ForceKilledCount }`. Each result is `@{ Command; ExitCode; Stdout; Stderr; StreamTimedOut; StreamError; StartError; Aborted; TimedOut }`, with every field always present: - `Command` is your original command hashtable, so extra keys like `Source` above come back with it. @@ -136,6 +134,8 @@ Output is decoded in the console's code page unless you say otherwise, which is `-TimeoutSeconds 600` kills any program still running after 10 minutes. Its result has `TimedOut = $true`, still goes through `-OnComplete`, and the rest of the batch carries on. The default, 0, means no limit. Any programs it started are killed with it. +Output is decoded in the console's code page unless you say otherwise, which is right for Windows' own tools. Many cross-platform tools write UTF-8 instead, and their non-ASCII output (accented or Japanese file names, say) comes back garbled. Pass `-OutputEncoding 'utf-8'` for the whole batch, or put an `OutputEncoding` key on one command. It also takes a code page number (`65001`, `437`) or a `[System.Text.Encoding]`, and the same option exists on `Invoke-PkProcessWithTimeout` and `Start-PkProcess` (and as a job key for the dispatcher's default starter). + `Invoke-PkRun { ... }` is what makes Ctrl+C graceful. Whatever the block outputs is passed through, so `$batch` above gets the batch result. It also stops the PC idle-sleeping until the run ends (`-KeepAwake $false` to opt out); the display can still turn off. Without `Invoke-PkRun` the engines still work, but Ctrl+C kills the script the normal way. See "Abort" below for what it means for your own code inside the braces. ## Run PowerShell code in parallel From 714ffb2f0a2ac88589cd29c9de3b6b18f490a926 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:04:00 -0400 Subject: [PATCH 3/3] Batch: don't let -OutputEncoding replace PowerShell's own $OutputEncoding in callbacks The parameter shares its name with PowerShell's preference variable, and the batch's closures captured it, so an -OnComplete scriptblock saw the kit's value (empty, or a string) instead of PowerShell's Encoding. The value now moves to a kit-private name before the closures are built. Also asserts that a code page given as text ('437') works. Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 9 +++++++++ ParallelKit.ps1 | 7 ++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index 270f4f5..102d91d 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -232,6 +232,15 @@ if ($compiled) { Test-Assert ($byName['default'] -eq $u8Text -and $byName['own'] -ne $u8Text) "batch: -OutputEncoding applies to every command, and a command's own OutputEncoding key overrides it" try { [System.IO.File]::Delete($u8Exe) } catch { } # temp file only; best effort } +Test-Assert ((Resolve-PkEncoding -Encoding '437').CodePage -eq 437) "a code page given as text ('437') is accepted" +# -OutputEncoding mustn't replace PowerShell's own $OutputEncoding inside the +# caller's callbacks. +$seen = @{ Value = 'unset' } +$null = Invoke-PkProcessBatch -OutputEncoding 'utf-8' -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) -OnComplete { param($c, $r) $seen.Value = $OutputEncoding }.GetNewClosure() +Test-Assert ($seen.Value -is [System.Text.Encoding]) ("inside -OnComplete, `$OutputEncoding is still PowerShell's own (got [" + $seen.Value + "])") +$seen.Value = 'unset' +$null = Invoke-PkProcessBatch -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) -OnComplete { param($c, $r) $seen.Value = $OutputEncoding } +Test-Assert ($seen.Value -is [System.Text.Encoding]) ("and in an -OnComplete without .GetNewClosure() too (got [" + $seen.Value + "])") $threw = $false try { $null = Invoke-PkProcessBatch -OutputEncoding 'no-such-encoding' -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) } catch { $threw = ($_.ToString() -like "*'no-such-encoding' isn't an encoding*") } Test-Assert $threw "an unknown -OutputEncoding fails the call with a clear message" diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index 41d986b..31b5e83 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -1274,6 +1274,11 @@ function Invoke-PkProcessBatch { # Checked once here, so a bad name fails the call instead of every command. $null = Resolve-PkEncoding -Encoding $OutputEncoding + # $OutputEncoding is also PowerShell's own preference variable. Move the + # value to another name before the closures below are built, or they'd + # capture it and every -OnComplete would see this instead of PowerShell's. + $batchEncoding = $OutputEncoding + Remove-Variable -Name OutputEncoding -Scope Local if ($TimeoutSeconds -lt 0) { throw "Invoke-PkProcessBatch: -TimeoutSeconds must be 0 (no limit) or more." } @@ -1291,7 +1296,7 @@ function Invoke-PkProcessBatch { $jobs = [System.Collections.Generic.List[hashtable]]::new() foreach ($c in $Commands) { if ($null -eq $c) { continue } - $encoding = $OutputEncoding + $encoding = $batchEncoding if ($c.ContainsKey('OutputEncoding')) { $encoding = $c['OutputEncoding'] } $jobs.Add(@{ FilePath = $c['FilePath']; Arguments = [string]$c['Arguments']; WorkingDirectory = [string]$c['WorkingDirectory']; OutputEncoding = $encoding; Command = $c }) }