diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index d2e55e7..102d91d 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -209,6 +209,41 @@ if ($compiled) { Test-Assert (($received -join '|') -eq ($expected -join '|')) ("a program receives exactly the arguments given (got " + ($received -join ' ') + ")") try { [System.IO.File]::Delete($argvExe) } catch { } # temp file only; best effort } + +# -OutputEncoding: a program that writes UTF-8 bytes comes back intact when +# told so, and garbled when decoded as code page 437 (a default console). +$u8Exe = Join-Path $env:TEMP ("pk-u8-" + [guid]::NewGuid().ToString("N") + ".exe") +$u8Src = [System.IO.Path]::ChangeExtension($u8Exe, '.cs') +[System.IO.File]::WriteAllText($u8Src, 'public static class PkU8 { public static void Main() { string t = new string(new char[] { (char)0x63, (char)0x61, (char)0x66, (char)0xE9, (char)0x20, (char)0x65E5, (char)0x672C, (char)0x8A9E }); byte[] b = new System.Text.UTF8Encoding(false).GetBytes(t); var o = System.Console.OpenStandardOutput(); o.Write(b, 0, b.Length); o.Flush(); var e = System.Console.OpenStandardError(); e.Write(b, 0, b.Length); e.Flush(); } }') +$csc = $null +try { $csc = Invoke-PkProcessWithTimeout -FilePath $cscExe -Arguments (ConvertTo-PkArgumentString @('/nologo', ('/out:' + $u8Exe), $u8Src)) } catch { Write-Host " csc.exe failed to run: $_" } +[System.IO.File]::Delete($u8Src) +$compiled = ($null -ne $csc -and $csc.ExitCode -eq 0 -and [System.IO.File]::Exists($u8Exe)) +Test-Assert $compiled "the UTF-8 test program compiles" +if ($compiled) { + $u8Text = 'caf' + [char]0xE9 + ' ' + [char]0x65E5 + [char]0x672C + [char]0x8A9E + $r = Invoke-PkProcessWithTimeout -FilePath $u8Exe -OutputEncoding 'utf-8' + $codes = { param($s) (([string]$s).ToCharArray() | ForEach-Object { '{0:X4}' -f [int]$_ }) -join ' ' } + Test-Assert ($r.Stdout -eq $u8Text -and $r.Stderr -eq $u8Text) ("-OutputEncoding 'utf-8' decodes a program's UTF-8 output, stdout and stderr (got " + (& $codes $r.Stdout) + " / " + (& $codes $r.Stderr) + ")") + $r = Invoke-PkProcessWithTimeout -FilePath $u8Exe -OutputEncoding 437 + Test-Assert ($r.Stdout -ne $u8Text) "and decoding it as code page 437 garbles it, so the option really takes effect" + $batch = Invoke-PkProcessBatch -OutputEncoding 'utf8' -Commands @(@{ FilePath = $u8Exe; Name = 'default' }, @{ FilePath = $u8Exe; Name = 'own'; OutputEncoding = 437 }) + $byName = @{}; foreach ($br in $batch.Results) { $byName[$br.Command['Name']] = $br.Stdout } + Test-Assert ($byName['default'] -eq $u8Text -and $byName['own'] -ne $u8Text) "batch: -OutputEncoding applies to every command, and a command's own OutputEncoding key overrides it" + try { [System.IO.File]::Delete($u8Exe) } catch { } # temp file only; best effort +} +Test-Assert ((Resolve-PkEncoding -Encoding '437').CodePage -eq 437) "a code page given as text ('437') is accepted" +# -OutputEncoding mustn't replace PowerShell's own $OutputEncoding inside the +# caller's callbacks. +$seen = @{ Value = 'unset' } +$null = Invoke-PkProcessBatch -OutputEncoding 'utf-8' -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) -OnComplete { param($c, $r) $seen.Value = $OutputEncoding }.GetNewClosure() +Test-Assert ($seen.Value -is [System.Text.Encoding]) ("inside -OnComplete, `$OutputEncoding is still PowerShell's own (got [" + $seen.Value + "])") +$seen.Value = 'unset' +$null = Invoke-PkProcessBatch -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) -OnComplete { param($c, $r) $seen.Value = $OutputEncoding } +Test-Assert ($seen.Value -is [System.Text.Encoding]) ("and in an -OnComplete without .GetNewClosure() too (got [" + $seen.Value + "])") +$threw = $false +try { $null = Invoke-PkProcessBatch -OutputEncoding 'no-such-encoding' -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) } catch { $threw = ($_.ToString() -like "*'no-such-encoding' isn't an encoding*") } +Test-Assert $threw "an unknown -OutputEncoding fails the call with a clear message" Write-Host "" # ============================================================================== diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index 0401054..31b5e83 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -569,16 +569,37 @@ function ConvertTo-PkArgumentString { return ($parts -join ' ') } +function Resolve-PkEncoding { + # -OutputEncoding as given (an Encoding, a code page number, or a name + # like 'utf-8' or 'utf8') -> an Encoding, or $null for "not set". + param($Encoding) + if ($null -eq $Encoding -or ($Encoding -is [string] -and $Encoding -eq '')) { return $null } + if ($Encoding -is [System.Text.Encoding]) { return $Encoding } + # .NET Framework doesn't know the name 'utf8'; no BOM either way, since + # this only decodes. + if ($Encoding -is [string] -and $Encoding -in @('utf8', 'utf-8')) { return [System.Text.UTF8Encoding]::new($false) } + try { + if ($Encoding -is [int]) { return [System.Text.Encoding]::GetEncoding([int]$Encoding) } + return [System.Text.Encoding]::GetEncoding([string]$Encoding) + } catch { + throw ("OutputEncoding '" + $Encoding + "' isn't an encoding .NET knows. Use a name like 'utf-8' or 'windows-1252', or a code page number like 65001.") + } +} + function Start-PkProcess { # Starts a hidden child process with stdout/stderr redirected and being # read asynchronously (so a chatty process can't block on a full pipe). + # $OutputEncoding is how the program's output is decoded ('utf-8', a + # code page number, or an Encoding); unset, it's the console's code page. # Returns @{ Proc; StdoutTask; StderrTask; Stopwatch }. param( [Parameter(Mandatory)] [string]$FilePath, [string]$Arguments = "", - [string]$WorkingDirectory = "" + [string]$WorkingDirectory = "", + $OutputEncoding = $null ) + $encoding = Resolve-PkEncoding -Encoding $OutputEncoding # Self-starting: a caller who never ran Initialize-ParallelKit still # gets children that die with the script, and output reads that # aren't starved of threads. Both are cheap no-ops once done. @@ -589,6 +610,10 @@ function Start-PkProcess { $psi.FileName = $FilePath $psi.Arguments = $Arguments $psi.UseShellExecute = $false + if ($null -ne $encoding) { + $psi.StandardOutputEncoding = $encoding + $psi.StandardErrorEncoding = $encoding + } $psi.RedirectStandardOutput = $true $psi.RedirectStandardError = $true $psi.CreateNoWindow = $true @@ -703,11 +728,12 @@ function Invoke-PkProcessWithTimeout { [int]$TimeoutMs = 60000, [int]$StreamTimeoutMs = 15000, [string]$WorkingDirectory = "", + $OutputEncoding = $null, # For a program that has to run BECAUSE of an abort (cleanup, a # restore step): keep going even though the run is being aborted. [switch]$IgnoreAbort ) - $job = Start-PkProcess -FilePath $FilePath -Arguments $Arguments -WorkingDirectory $WorkingDirectory + $job = Start-PkProcess -FilePath $FilePath -Arguments $Arguments -WorkingDirectory $WorkingDirectory -OutputEncoding $OutputEncoding try { # Wait in short slices so Ctrl+C (or the run's cancel token) is # noticed within a moment, not after the whole time limit. @@ -806,7 +832,7 @@ function Invoke-PkProcessDispatch { if ($null -eq $JobStarter) { $JobStarter = { param($job) - $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments ([string]$job['Arguments']) -WorkingDirectory ([string]$job['WorkingDirectory']) + $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments ([string]$job['Arguments']) -WorkingDirectory ([string]$job['WorkingDirectory']) -OutputEncoding $job['OutputEncoding'] $job['Proc'] = $started.Proc; $job['StdoutTask'] = $started.StdoutTask; $job['StderrTask'] = $started.StderrTask $job['Stopwatch'] = $started.Stopwatch return $job @@ -1223,7 +1249,8 @@ function Invoke-PkProcessDispatchCore { function Invoke-PkProcessBatch { # The simple case: run a flat list of commands, $MaxConcurrent at a time. - # Each command is @{ FilePath = ...; Arguments = ...; WorkingDirectory = ... (optional); }. + # Each command is @{ FilePath = ...; Arguments = ...; WorkingDirectory = ... (optional); + # OutputEncoding = ... (optional, overrides -OutputEncoding); }. # $OnComplete { param($command, $result) } fires as each one finishes # (not for one that couldn't start or was stopped by an abort). # $TimeoutSeconds > 0 kills a program still running after that long; its @@ -1241,9 +1268,18 @@ function Invoke-PkProcessBatch { [int]$MaxConcurrent = 4, [scriptblock]$OnComplete = $null, [int]$AbortTimeoutSeconds = 60, - [int]$TimeoutSeconds = 0 + [int]$TimeoutSeconds = 0, + $OutputEncoding = $null ) + # Checked once here, so a bad name fails the call instead of every command. + $null = Resolve-PkEncoding -Encoding $OutputEncoding + # $OutputEncoding is also PowerShell's own preference variable. Move the + # value to another name before the closures below are built, or they'd + # capture it and every -OnComplete would see this instead of PowerShell's. + $batchEncoding = $OutputEncoding + Remove-Variable -Name OutputEncoding -Scope Local + if ($TimeoutSeconds -lt 0) { throw "Invoke-PkProcessBatch: -TimeoutSeconds must be 0 (no limit) or more." } $results = [System.Collections.Generic.List[hashtable]]::new() @@ -1260,7 +1296,9 @@ function Invoke-PkProcessBatch { $jobs = [System.Collections.Generic.List[hashtable]]::new() foreach ($c in $Commands) { if ($null -eq $c) { continue } - $jobs.Add(@{ FilePath = $c['FilePath']; Arguments = [string]$c['Arguments']; WorkingDirectory = [string]$c['WorkingDirectory']; Command = $c }) + $encoding = $batchEncoding + if ($c.ContainsKey('OutputEncoding')) { $encoding = $c['OutputEncoding'] } + $jobs.Add(@{ FilePath = $c['FilePath']; Arguments = [string]$c['Arguments']; WorkingDirectory = [string]$c['WorkingDirectory']; OutputEncoding = $encoding; Command = $c }) } return $jobs }.GetNewClosure() @@ -1268,7 +1306,7 @@ function Invoke-PkProcessBatch { $jobStarter = { param($job) try { - $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments $job['Arguments'] -WorkingDirectory $job['WorkingDirectory'] + $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments $job['Arguments'] -WorkingDirectory $job['WorkingDirectory'] -OutputEncoding $job['OutputEncoding'] } catch { $results.Add(@{ Command = $job['Command']; ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; StartError = [string]$_; Aborted = $false; TimedOut = $false }) throw @@ -1321,7 +1359,7 @@ function Invoke-PkProcessBatch { function Get-PkWorkerSafeFunctionNames { # Kit functions every worker runspace gets automatically. - return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', 'Stop-PkProcessTree', + return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', 'Stop-PkProcessTree', 'Resolve-PkEncoding', 'Start-PkProcess', 'Wait-PkProcess', 'Complete-PkProcess', 'Invoke-PkProcessWithTimeout', 'Invoke-PkProcessDispatch', 'Invoke-PkProcessDispatchCore', 'Invoke-PkProcessBatch', 'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'ConvertTo-PkArgumentString') diff --git a/README.md b/README.md index 59ee3cb..537e6a3 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,8 @@ Each command is a hashtable with `FilePath`, `Arguments` and optionally `Working `-TimeoutSeconds 600` kills any program still running after 10 minutes. Its result has `TimedOut = $true`, still goes through `-OnComplete`, and the rest of the batch carries on. The default, 0, means no limit. Any programs it started are killed with it. +Output is decoded in the console's code page unless you say otherwise, which is right for Windows' own tools. Many cross-platform tools write UTF-8 instead, and their non-ASCII output (accented or Japanese file names, say) comes back garbled. Pass `-OutputEncoding 'utf-8'` for the whole batch, or put an `OutputEncoding` key on one command. It also takes a code page number (`65001`, `437`) or a `[System.Text.Encoding]`, and the same option exists on `Invoke-PkProcessWithTimeout` and `Start-PkProcess` (and as a job key for the dispatcher's default starter). + `Invoke-PkRun { ... }` is what makes Ctrl+C graceful. Whatever the block outputs is passed through, so `$batch` above gets the batch result. It also stops the PC idle-sleeping until the run ends (`-KeepAwake $false` to opt out); the display can still turn off. Without `Invoke-PkRun` the engines still work, but Ctrl+C kills the script the normal way. See "Abort" below for what it means for your own code inside the braces. ## Run PowerShell code in parallel