Add -OutputEncoding for decoding program output

ParallelKit.Harness.ps1

@@ -209,6 +209,41 @@ if ($compiled) {
     Test-Assert (($received -join '|') -eq ($expected -join '|')) ("a program receives exactly the arguments given (got " + ($received -join ' ') + ")")
     try { [System.IO.File]::Delete($argvExe) } catch { }  # temp file only; best effort
 }
+
+# -OutputEncoding: a program that writes UTF-8 bytes comes back intact when
+# told so, and garbled when decoded as code page 437 (a default console).
+$u8Exe = Join-Path $env:TEMP ("pk-u8-" + [guid]::NewGuid().ToString("N") + ".exe")
+$u8Src = [System.IO.Path]::ChangeExtension($u8Exe, '.cs')
+[System.IO.File]::WriteAllText($u8Src, 'public static class PkU8 { public static void Main() { string t = new string(new char[] { (char)0x63, (char)0x61, (char)0x66, (char)0xE9, (char)0x20, (char)0x65E5, (char)0x672C, (char)0x8A9E }); byte[] b = new System.Text.UTF8Encoding(false).GetBytes(t); var o = System.Console.OpenStandardOutput(); o.Write(b, 0, b.Length); o.Flush(); var e = System.Console.OpenStandardError(); e.Write(b, 0, b.Length); e.Flush(); } }')
+$csc = $null
+try { $csc = Invoke-PkProcessWithTimeout -FilePath $cscExe -Arguments (ConvertTo-PkArgumentString @('/nologo', ('/out:' + $u8Exe), $u8Src)) } catch { Write-Host "  csc.exe failed to run: $_" }
+[System.IO.File]::Delete($u8Src)
+$compiled = ($null -ne $csc -and $csc.ExitCode -eq 0 -and [System.IO.File]::Exists($u8Exe))
+Test-Assert $compiled "the UTF-8 test program compiles"
+if ($compiled) {
+    $u8Text = 'caf' + [char]0xE9 + ' ' + [char]0x65E5 + [char]0x672C + [char]0x8A9E
+    $r = Invoke-PkProcessWithTimeout -FilePath $u8Exe -OutputEncoding 'utf-8'
+    $codes = { param($s) (([string]$s).ToCharArray() | ForEach-Object { '{0:X4}' -f [int]$_ }) -join ' ' }
+    Test-Assert ($r.Stdout -eq $u8Text -and $r.Stderr -eq $u8Text) ("-OutputEncoding 'utf-8' decodes a program's UTF-8 output, stdout and stderr (got " + (& $codes $r.Stdout) + " / " + (& $codes $r.Stderr) + ")")
+    $r = Invoke-PkProcessWithTimeout -FilePath $u8Exe -OutputEncoding 437
+    Test-Assert ($r.Stdout -ne $u8Text) "and decoding it as code page 437 garbles it, so the option really takes effect"
+    $batch = Invoke-PkProcessBatch -OutputEncoding 'utf8' -Commands @(@{ FilePath = $u8Exe; Name = 'default' }, @{ FilePath = $u8Exe; Name = 'own'; OutputEncoding = 437 })
+    $byName = @{}; foreach ($br in $batch.Results) { $byName[$br.Command['Name']] = $br.Stdout }
+    Test-Assert ($byName['default'] -eq $u8Text -and $byName['own'] -ne $u8Text) "batch: -OutputEncoding applies to every command, and a command's own OutputEncoding key overrides it"
+    try { [System.IO.File]::Delete($u8Exe) } catch { }  # temp file only; best effort
+}
+Test-Assert ((Resolve-PkEncoding -Encoding '437').CodePage -eq 437) "a code page given as text ('437') is accepted"
+# -OutputEncoding mustn't replace PowerShell's own $OutputEncoding inside the
+# caller's callbacks.
+$seen = @{ Value = 'unset' }
+$null = Invoke-PkProcessBatch -OutputEncoding 'utf-8' -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) -OnComplete { param($c, $r) $seen.Value = $OutputEncoding }.GetNewClosure()
+Test-Assert ($seen.Value -is [System.Text.Encoding]) ("inside -OnComplete, `$OutputEncoding is still PowerShell's own (got [" + $seen.Value + "])")
+$seen.Value = 'unset'
+$null = Invoke-PkProcessBatch -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) -OnComplete { param($c, $r) $seen.Value = $OutputEncoding }
+Test-Assert ($seen.Value -is [System.Text.Encoding]) ("and in an -OnComplete without .GetNewClosure() too (got [" + $seen.Value + "])")
+$threw = $false
+try { $null = Invoke-PkProcessBatch -OutputEncoding 'no-such-encoding' -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) } catch { $threw = ($_.ToString() -like "*'no-such-encoding' isn't an encoding*") }
+Test-Assert $threw "an unknown -OutputEncoding fails the call with a clear message"
 Write-Host ""
 
 # ==============================================================================

ParallelKit.ps1

@@ -569,16 +569,37 @@ function ConvertTo-PkArgumentString {
     return ($parts -join ' ')
 }
 
+function Resolve-PkEncoding {
+    # -OutputEncoding as given (an Encoding, a code page number, or a name
+    # like 'utf-8' or 'utf8') -> an Encoding, or $null for "not set".
+    param($Encoding)
+    if ($null -eq $Encoding -or ($Encoding -is [string] -and $Encoding -eq '')) { return $null }
+    if ($Encoding -is [System.Text.Encoding]) { return $Encoding }
+    # .NET Framework doesn't know the name 'utf8'; no BOM either way, since
+    # this only decodes.
+    if ($Encoding -is [string] -and $Encoding -in @('utf8', 'utf-8')) { return [System.Text.UTF8Encoding]::new($false) }
+    try {
+        if ($Encoding -is [int]) { return [System.Text.Encoding]::GetEncoding([int]$Encoding) }
+        return [System.Text.Encoding]::GetEncoding([string]$Encoding)
+    } catch {
+        throw ("OutputEncoding '" + $Encoding + "' isn't an encoding .NET knows. Use a name like 'utf-8' or 'windows-1252', or a code page number like 65001.")
+    }
+}
+
 function Start-PkProcess {
     # Starts a hidden child process with stdout/stderr redirected and being
     # read asynchronously (so a chatty process can't block on a full pipe).
+    # $OutputEncoding is how the program's output is decoded ('utf-8', a
+    # code page number, or an Encoding); unset, it's the console's code page.
     # Returns @{ Proc; StdoutTask; StderrTask; Stopwatch }.
     param(
         [Parameter(Mandatory)]
         [string]$FilePath,
         [string]$Arguments = "",
-        [string]$WorkingDirectory = ""
+        [string]$WorkingDirectory = "",
+        $OutputEncoding = $null
     )
+    $encoding = Resolve-PkEncoding -Encoding $OutputEncoding
     # Self-starting: a caller who never ran Initialize-ParallelKit still
     # gets children that die with the script, and output reads that
     # aren't starved of threads. Both are cheap no-ops once done.
@@ -589,6 +610,10 @@ function Start-PkProcess {
     $psi.FileName               = $FilePath
     $psi.Arguments              = $Arguments
     $psi.UseShellExecute        = $false
+    if ($null -ne $encoding) {
+        $psi.StandardOutputEncoding = $encoding
+        $psi.StandardErrorEncoding  = $encoding
+    }
     $psi.RedirectStandardOutput = $true
     $psi.RedirectStandardError  = $true
     $psi.CreateNoWindow         = $true
@@ -703,11 +728,12 @@ function Invoke-PkProcessWithTimeout {
         [int]$TimeoutMs = 60000,
         [int]$StreamTimeoutMs = 15000,
         [string]$WorkingDirectory = "",
+        $OutputEncoding = $null,
         # For a program that has to run BECAUSE of an abort (cleanup, a
         # restore step): keep going even though the run is being aborted.
         [switch]$IgnoreAbort
     )
-    $job = Start-PkProcess -FilePath $FilePath -Arguments $Arguments -WorkingDirectory $WorkingDirectory
+    $job = Start-PkProcess -FilePath $FilePath -Arguments $Arguments -WorkingDirectory $WorkingDirectory -OutputEncoding $OutputEncoding
     try {
         # Wait in short slices so Ctrl+C (or the run's cancel token) is
         # noticed within a moment, not after the whole time limit.
@@ -806,7 +832,7 @@ function Invoke-PkProcessDispatch {
     if ($null -eq $JobStarter) {
         $JobStarter = {
             param($job)
-            $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments ([string]$job['Arguments']) -WorkingDirectory ([string]$job['WorkingDirectory'])
+            $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments ([string]$job['Arguments']) -WorkingDirectory ([string]$job['WorkingDirectory']) -OutputEncoding $job['OutputEncoding']
             $job['Proc'] = $started.Proc; $job['StdoutTask'] = $started.StdoutTask; $job['StderrTask'] = $started.StderrTask
             $job['Stopwatch'] = $started.Stopwatch
             return $job
@@ -1223,7 +1249,8 @@ function Invoke-PkProcessDispatchCore {
 
 function Invoke-PkProcessBatch {
     # The simple case: run a flat list of commands, $MaxConcurrent at a time.
-    # Each command is @{ FilePath = ...; Arguments = ...; WorkingDirectory = ... (optional); <anything else> }.
+    # Each command is @{ FilePath = ...; Arguments = ...; WorkingDirectory = ... (optional);
+    # OutputEncoding = ... (optional, overrides -OutputEncoding); <anything else> }.
     # $OnComplete { param($command, $result) } fires as each one finishes
     # (not for one that couldn't start or was stopped by an abort).
     # $TimeoutSeconds > 0 kills a program still running after that long; its
@@ -1241,9 +1268,18 @@ function Invoke-PkProcessBatch {
         [int]$MaxConcurrent = 4,
         [scriptblock]$OnComplete = $null,
         [int]$AbortTimeoutSeconds = 60,
-        [int]$TimeoutSeconds = 0
+        [int]$TimeoutSeconds = 0,
+        $OutputEncoding = $null
     )
 
+    # Checked once here, so a bad name fails the call instead of every command.
+    $null = Resolve-PkEncoding -Encoding $OutputEncoding
+    # $OutputEncoding is also PowerShell's own preference variable. Move the
+    # value to another name before the closures below are built, or they'd
+    # capture it and every -OnComplete would see this instead of PowerShell's.
+    $batchEncoding = $OutputEncoding
+    Remove-Variable -Name OutputEncoding -Scope Local
+
     if ($TimeoutSeconds -lt 0) { throw "Invoke-PkProcessBatch: -TimeoutSeconds must be 0 (no limit) or more." }
 
     $results = [System.Collections.Generic.List[hashtable]]::new()
@@ -1260,15 +1296,17 @@ function Invoke-PkProcessBatch {
         $jobs = [System.Collections.Generic.List[hashtable]]::new()
         foreach ($c in $Commands) {
             if ($null -eq $c) { continue }
-            $jobs.Add(@{ FilePath = $c['FilePath']; Arguments = [string]$c['Arguments']; WorkingDirectory = [string]$c['WorkingDirectory']; Command = $c })
+            $encoding = $batchEncoding
+            if ($c.ContainsKey('OutputEncoding')) { $encoding = $c['OutputEncoding'] }
+            $jobs.Add(@{ FilePath = $c['FilePath']; Arguments = [string]$c['Arguments']; WorkingDirectory = [string]$c['WorkingDirectory']; OutputEncoding = $encoding; Command = $c })
         }
         return $jobs
     }.GetNewClosure()
 
     $jobStarter = {
         param($job)
         try {
-            $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments $job['Arguments'] -WorkingDirectory $job['WorkingDirectory']
+            $started = Start-PkProcess -FilePath $job['FilePath'] -Arguments $job['Arguments'] -WorkingDirectory $job['WorkingDirectory'] -OutputEncoding $job['OutputEncoding']
         } catch {
             $results.Add(@{ Command = $job['Command']; ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; StartError = [string]$_; Aborted = $false; TimedOut = $false })
             throw
@@ -1321,7 +1359,7 @@ function Invoke-PkProcessBatch {
 
 function Get-PkWorkerSafeFunctionNames {
     # Kit functions every worker runspace gets automatically.
-    return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', 'Stop-PkProcessTree',
+    return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', 'Stop-PkProcessTree', 'Resolve-PkEncoding',
              'Start-PkProcess', 'Wait-PkProcess', 'Complete-PkProcess', 'Invoke-PkProcessWithTimeout',
              'Invoke-PkProcessDispatch', 'Invoke-PkProcessDispatchCore', 'Invoke-PkProcessBatch',
              'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'ConvertTo-PkArgumentString')

README.md

@@ -134,6 +134,8 @@ Each command is a hashtable with `FilePath`, `Arguments` and optionally `Working
 
 `-TimeoutSeconds 600` kills any program still running after 10 minutes. Its result has `TimedOut = $true`, still goes through `-OnComplete`, and the rest of the batch carries on. The default, 0, means no limit. Any programs it started are killed with it.
 
+Output is decoded in the console's code page unless you say otherwise, which is right for Windows' own tools. Many cross-platform tools write UTF-8 instead, and their non-ASCII output (accented or Japanese file names, say) comes back garbled. Pass `-OutputEncoding 'utf-8'` for the whole batch, or put an `OutputEncoding` key on one command. It also takes a code page number (`65001`, `437`) or a `[System.Text.Encoding]`, and the same option exists on `Invoke-PkProcessWithTimeout` and `Start-PkProcess` (and as a job key for the dispatcher's default starter).
+
 `Invoke-PkRun { ... }` is what makes Ctrl+C graceful. Whatever the block outputs is passed through, so `$batch` above gets the batch result. It also stops the PC idle-sleeping until the run ends (`-KeepAwake $false` to opt out); the display can still turn off. Without `Invoke-PkRun` the engines still work, but Ctrl+C kills the script the normal way. See "Abort" below for what it means for your own code inside the braces.
 
 ## Run PowerShell code in parallel