diff --git a/.github/workflows/platform-vitest-main.yaml b/.github/workflows/platform-vitest-main.yaml index cb56e1dfab8..4917fed1f65 100644 --- a/.github/workflows/platform-vitest-main.yaml +++ b/.github/workflows/platform-vitest-main.yaml @@ -339,15 +339,7 @@ jobs: shell: powershell run: | . "$env:TRUSTED_WSL_HELPER" - $script = @' - set -euo pipefail - service docker stop >/dev/null 2>&1 || true - if docker info >/dev/null 2>&1; then - echo "Docker must remain unavailable during the non-live Vitest suite." >&2 - exit 1 - fi - '@ - Invoke-WslScript -Distro $env:WSL_DISTRO -User root -Script $script + Stop-WslContainerRuntime -Distro $env:WSL_DISTRO - name: Install reviewed Node.js and npm in WSL shell: powershell @@ -394,6 +386,7 @@ jobs: . "$env:TRUSTED_WSL_HELPER" $testUser = ConvertTo-BashLiteral -Value $env:WSL_TEST_USER $workdir = ConvertTo-BashLiteral -Value $env:WSL_WORKDIR + $runtimeCheck = Get-WslContainerRuntimeUnavailableScript $script = @" set -euo pipefail id -un | grep -Fxq $testUser @@ -401,6 +394,7 @@ jobs: export NPM_CONFIG_CACHE="`$HOME/.npm" export NEMOCLAW_EXEC_TIMEOUT=60000 export NEMOCLAW_TEST_TIMEOUT=60000 + $runtimeCheck npx vitest run --testTimeout 60000 --shard='${{ matrix.shard }}/4' "@ Invoke-WslScript -Distro $env:WSL_DISTRO -User $env:WSL_TEST_USER -Script $script @@ -438,12 +432,11 @@ jobs: $script = @" set -euo pipefail usermod -aG docker $testUser - service docker start - docker info podman --version ip -Version "@ Invoke-WslScript -Distro $env:WSL_DISTRO -User root -Script $script + Start-WslContainerRuntime -Distro $env:WSL_DISTRO - id: wsl_docker name: Detect Docker availability in WSL diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index 01545387e05..2b48762ee0c 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -895,7 +895,7 @@ describe("rebuild destroy phase", () => { expect(mocks.captureOpenshell).toHaveBeenCalledWith( ["sandbox", "get", "-g", "nemoclaw", "alpha"], - expect.objectContaining({ timeout: 15_000 }), + expect.any(Object), ); expect(onDeleted).not.toHaveBeenCalled(); expect(onDeleteStateAmbiguous).toHaveBeenCalledOnce(); diff --git a/test/automation/e2e/platform-vitest-main-workflow.test.ts b/test/automation/e2e/platform-vitest-main-workflow.test.ts index 563d2b75690..d29f3719daa 100644 --- a/test/automation/e2e/platform-vitest-main-workflow.test.ts +++ b/test/automation/e2e/platform-vitest-main-workflow.test.ts @@ -148,6 +148,7 @@ describe("platform evidence workflow", () => { const stopped = step("wsl-vitest", "Stop WSL container runtime before non-live tests").run ?? ""; const runtime = step("wsl-vitest", "Start the WSL container runtime").run ?? ""; + const suite = step("wsl-vitest", "Run full Vitest suite in WSL").run ?? ""; const installIndex = steps.findIndex((entry) => entry.name === "Install Ubuntu dependencies"); const stoppedIndex = steps.findIndex( (entry) => entry.name === "Stop WSL container runtime before non-live tests", @@ -169,12 +170,11 @@ describe("platform evidence workflow", () => { expect(install).toContain("'zip'"); expect(install).toContain("'gnu-coreutils'"); expect(install).not.toContain("service docker start"); - expect(stopped).toContain("service docker stop"); - expect(stopped).toContain("if docker info >/dev/null 2>&1; then"); - expect(stopped).toContain("exit 1"); + expect(stopped).toContain("Stop-WslContainerRuntime -Distro $env:WSL_DISTRO"); + expect(suite).toContain("$runtimeCheck = Get-WslContainerRuntimeUnavailableScript"); + expect(suite).toContain("$runtimeCheck\nnpx vitest run"); expect(runtime).not.toContain("Install-WslUbuntuDependencies"); - expect(runtime).toContain("service docker start"); - expect(runtime).toContain("docker info"); + expect(runtime).toContain("Start-WslContainerRuntime -Distro $env:WSL_DISTRO"); expect(runtime).toContain("podman --version"); expect(runtime).toContain("ip -Version"); expect(step("wsl-vitest", "Resolve workspace paths for WSL").run).toContain( diff --git a/test/automation/e2e/wsl-ci-helper.test.ts b/test/automation/e2e/wsl-ci-helper.test.ts index c013489de7c..30020be22d9 100644 --- a/test/automation/e2e/wsl-ci-helper.test.ts +++ b/test/automation/e2e/wsl-ci-helper.test.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { beforeAll, describe, expect, it } from "vitest"; import { testTimeout, testTimeoutOptions } from "../../helpers/timeouts"; +import { runWslContainerRuntimeScript } from "../../support/wsl-container-runtime-fixture"; import { POWERSHELL_BATCH_EXEC_TIMEOUT_MS, type PowerShellBatchCase, @@ -43,6 +44,167 @@ describe("trusted WSL CI helper", () => { BATCH_TIMEOUT, ); + itPowerShell( + "keeps Docker unavailable across distro restarts and preserves shutdown failures", + ` +. ${JSON.stringify(WSL_CI_HELPER)} +Get-WslContainerRuntimeStopScript +`, + (result) => { + expect(result.status).toBe(0); + expect( + runWslContainerRuntimeScript(`${result.stdout}\nsimulate_wsl_restart\n! docker info`), + ).toMatchObject({ + status: 0, + masked: true, + commands: [ + "systemctl mask --now docker.service docker.socket", + "docker info", + "docker info", + ], + }); + expect(runWslContainerRuntimeScript(result.stdout, "mask-fails")).toMatchObject({ + status: 23, + stderr: "systemd stop failed\n", + commands: ["systemctl mask --now docker.service docker.socket"], + }); + expect(runWslContainerRuntimeScript(result.stdout, "reachable")).toMatchObject({ + status: 1, + masked: true, + stderr: expect.stringContaining("Docker must remain unavailable"), + }); + }, + ); + + itPowerShell( + "restores Docker for live tests and propagates the health deadline", + ` +. ${JSON.stringify(WSL_CI_HELPER)} +Get-WslContainerRuntimeStartScript +`, + (result) => { + expect(result.status).toBe(0); + expect(runWslContainerRuntimeScript(result.stdout, "normal", true)).toMatchObject({ + status: 0, + masked: false, + commands: [ + "systemctl unmask docker.service docker.socket", + "systemctl start docker.service", + "timeout 30s", + "docker info", + "docker info", + ], + }); + expect(runWslContainerRuntimeScript(result.stdout, "health-timeout", true)).toMatchObject({ + status: 124, + masked: false, + }); + }, + ); + + itPowerShell( + "restarts only the requested distro once when the systemd bus is unavailable", + ` +. ${JSON.stringify(WSL_CI_HELPER)} +$script:calls = @() +$script:probes = 0 +function Invoke-WslNativeOutput { + param([string[]]$ArgumentList) + $script:probeCommand = $ArgumentList[-1] + $script:probes += 1 + if ($script:probes -eq 1) { + return [pscustomobject]@{ ExitCode = 1; Output = @('Failed to connect to system scope bus via local transport: Connection refused') } + } + return [pscustomobject]@{ ExitCode = 0; Output = @('259') } +} +function Invoke-WslNative { + param([string[]]$ArgumentList, [switch]$MergeError) + $script:calls += $ArgumentList -join ' ' + return 0 +} +function Invoke-WslScript { + param([string]$Distro, [string]$User, [string]$Script) + $script:calls += "stop $Distro as $User" +} +Stop-WslContainerRuntime -Distro 'Ubuntu CI' +Stop-WslContainerRuntime -Distro 'Ubuntu CI' +[pscustomobject]@{ calls = @($script:calls); probeCommand = $script:probeCommand } | ConvertTo-Json -Compress +`, + (result) => { + expect(result.status).toBe(0); + const observed = JSON.parse(result.stdout.trim().split(/\r?\n/).at(-1)!); + expect(observed.calls).toEqual([ + "--terminate Ubuntu CI", + "stop Ubuntu CI as root", + "stop Ubuntu CI as root", + ]); + expect(runWslContainerRuntimeScript(observed.probeCommand, "bus-unavailable")).toMatchObject({ + status: 1, + stdout: "Failed to connect to bus: Connection refused\n", + stderr: "", + }); + }, + ); + + itPowerShell( + "stops after one unsuccessful systemd recovery without running the Docker script", + ` +. ${JSON.stringify(WSL_CI_HELPER)} +$script:terminations = 0 +$script:probes = 0 +function Invoke-WslNativeOutput { + param([string[]]$ArgumentList) + $script:probes += 1 + return [pscustomobject]@{ ExitCode = 1; Output = @('Failed to connect to bus: Connection refused') } +} +function Invoke-WslNative { param([string[]]$ArgumentList, [switch]$MergeError) $script:terminations += 1; return 0 } +function Start-Sleep { param([int]$Seconds) } +function Invoke-WslScript { throw 'Docker script must not run' } +try { Stop-WslContainerRuntime -Distro Ubuntu } catch { $script:failure = $_.Exception.Message } +[pscustomobject]@{ terminations = $script:terminations; probes = $script:probes; failure = $script:failure } | ConvertTo-Json -Compress +`, + (result) => { + expect(result.status).toBe(0); + expect(JSON.parse(result.stdout.trim().split(/\r?\n/).at(-1)!)).toMatchObject({ + terminations: 1, + probes: 21, + failure: expect.stringContaining("after one distro restart"), + }); + }, + ); + + itPowerShell( + "rejects unrelated service-manager errors without restarting the distro", + ` +. ${JSON.stringify(WSL_CI_HELPER)} +function Invoke-WslNativeOutput { param([string[]]$ArgumentList) return [pscustomobject]@{ ExitCode = 127; Output = @('systemctl: command not found') } } +function Invoke-WslNative { throw 'Distro must not restart' } +function Invoke-WslScript { throw 'Docker script must not run' } +Stop-WslContainerRuntime -Distro Ubuntu +`, + (result) => { + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "WSL service-manager probe failed: systemctl: command not found", + ); + }, + ); + + itPowerShell( + "propagates a failed distro termination after the manager probe times out", + ` +. ${JSON.stringify(WSL_CI_HELPER)} +function Invoke-WslNativeOutput { param([string[]]$ArgumentList) return [pscustomobject]@{ ExitCode = 124; Output = @() } } +function Invoke-WslNative { param([string[]]$ArgumentList, [switch]$MergeError) return 5 } +function Invoke-WslScript { throw 'Docker script must not run' } +Stop-WslContainerRuntime -Distro Ubuntu +`, + (result) => { + expect(result.status).toBe(1); + expect(result.stderr).toContain("WSL distro termination failed with exit code 5."); + }, + ); + itPowerShell( "converts drive paths with spaces and quotes without changing their data", ` diff --git a/test/e2e/README.md b/test/e2e/README.md index 5524f9b2916..d67c793e666 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -374,6 +374,12 @@ Each macOS Vitest shard has a 30-minute budget. The independent macOS live E2E job has a 150-minute budget, including its 70-minute live test and cleanup. The first WSL shard has a 180-minute budget for root-required contracts and live E2E; the other shards have 90 minutes. +WSL setup checks the systemd manager after package installation. An unavailable bus or a timed-out +probe permits one restart of the job's Ubuntu distro, followed by bounded readiness probes. +Other errors stop setup. The helper masks `docker.service` and `docker.socket` in the job-owned +distro so the masks survive a restart. The test script checks Docker again immediately before +Vitest. The helper removes the masks and requires Docker health before the live step. + The independent macOS job and WSL shard 1 run focused live E2E only when the run tests `main` and Docker is available. Otherwise, the workflow records the skip and retains the platform contract evidence. Therefore, the workflow is platform evidence, not `Release qualification`. diff --git a/test/support/wsl-container-runtime-fixture.ts b/test/support/wsl-container-runtime-fixture.ts new file mode 100644 index 00000000000..b91148cdaa4 --- /dev/null +++ b/test/support/wsl-container-runtime-fixture.ts @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +/** Model Docker socket activation while executing the helper's real shell script. */ +export function runWslContainerRuntimeScript( + script: string, + mode: "normal" | "mask-fails" | "reachable" | "health-timeout" | "bus-unavailable" = "normal", + initiallyMasked = false, +) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-wsl-runtime-")); + try { + if (initiallyMasked) fs.writeFileSync(path.join(directory, "masked"), ""); + const result = spawnSync( + "bash", + [ + "-c", + String.raw` +systemctl() { + printf 'systemctl %s\n' "$*" >> "$WSL_RUNTIME_FIXTURE_DIR/commands" + case "$1" in + mask) + if [ "$WSL_RUNTIME_FIXTURE_MODE" = mask-fails ]; then + echo 'systemd stop failed' >&2 + return 23 + fi + case " $* " in + *' --runtime '*) touch "$WSL_RUNTIME_FIXTURE_DIR/runtime-masked" ;; + *) touch "$WSL_RUNTIME_FIXTURE_DIR/masked" ;; + esac + ;; + unmask) + rm -f "$WSL_RUNTIME_FIXTURE_DIR/runtime-masked" + case " $* " in + *' --runtime '*) ;; + *) rm -f "$WSL_RUNTIME_FIXTURE_DIR/masked" ;; + esac + ;; + start) test ! -f "$WSL_RUNTIME_FIXTURE_DIR/masked" && test ! -f "$WSL_RUNTIME_FIXTURE_DIR/runtime-masked" ;; + show) + if [ "$WSL_RUNTIME_FIXTURE_MODE" = bus-unavailable ]; then + echo 'Failed to connect to bus: Connection refused' >&2 + return 1 + fi + echo 'ActiveState=inactive' + ;; + *) return 99 ;; + esac +} +docker() { + printf 'docker %s\n' "$*" >> "$WSL_RUNTIME_FIXTURE_DIR/commands" + # An unmasked socket activates Docker when a client connects. + [ "$WSL_RUNTIME_FIXTURE_MODE" = reachable ] || { + [ ! -f "$WSL_RUNTIME_FIXTURE_DIR/masked" ] && [ ! -f "$WSL_RUNTIME_FIXTURE_DIR/runtime-masked" ] + } +} +simulate_wsl_restart() { + rm -f "$WSL_RUNTIME_FIXTURE_DIR/runtime-masked" +} +timeout() { + printf 'timeout %s\n' "$1" >> "$WSL_RUNTIME_FIXTURE_DIR/commands" + if [ "$WSL_RUNTIME_FIXTURE_MODE" = health-timeout ]; then return 124; fi + shift + "$@" +} +export -f systemctl docker +` + script, + ], + { + encoding: "utf8", + env: { + ...process.env, + WSL_RUNTIME_FIXTURE_DIR: directory, + WSL_RUNTIME_FIXTURE_MODE: mode, + }, + timeout: 5_000, + }, + ); + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr, + masked: + fs.existsSync(path.join(directory, "masked")) || + fs.existsSync(path.join(directory, "runtime-masked")), + commands: fs.readFileSync(path.join(directory, "commands"), "utf8").trim().split("\n"), + }; + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } +} diff --git a/tools/wsl/ci-helper.ps1 b/tools/wsl/ci-helper.ps1 index 002c0fa31c5..9449540c57c 100644 --- a/tools/wsl/ci-helper.ps1 +++ b/tools/wsl/ci-helper.ps1 @@ -299,6 +299,78 @@ function Install-WslUbuntuDependencies { Invoke-WslScript @invokeParameters } +function Repair-WslSystemdAfterInstall { + param([Parameter(Mandatory = $true)] [string]$Distro) + + # Windows PowerShell 5.1 can throw on redirected native stderr before + # returning an exit code. Capture expected manager errors inside Linux. + $probe = @('-d', $Distro, '--user', 'root', '--', 'bash', '-c', + 'timeout 10s systemctl show --property=Version --value 2>&1') + $result = Invoke-WslNativeOutput -ArgumentList $probe + if ($result.ExitCode -eq 0) { return } + + $diagnostic = @($result.Output) -join "`n" + if ($result.ExitCode -ne 124 -and + $diagnostic -notmatch 'Failed to connect.*bus|Transport endpoint is not connected') { + throw "WSL service-manager probe failed: $diagnostic" + } + + # Package installation can leave the CI distro's systemd bus unavailable. + # Restart only that distro, once, before tests or credentials enter it. + Write-Host "Restarting CI distro '$Distro' after an unavailable systemd bus: $diagnostic" + $exitCode = Invoke-WslNative -ArgumentList @('--terminate', $Distro) -MergeError + if ($exitCode -ne 0) { throw "WSL distro termination failed with exit code $exitCode." } + + for ($attempt = 0; $attempt -lt 20; $attempt++) { + $result = Invoke-WslNativeOutput -ArgumentList $probe + if ($result.ExitCode -eq 0) { return } + if ($attempt -lt 19) { Start-Sleep -Seconds 1 } + } + throw "WSL systemd did not become available after one distro restart: $(@($result.Output) -join "`n")" +} + +function Get-WslContainerRuntimeUnavailableScript { + return @' +if docker info >/dev/null 2>&1; then + echo "Docker must remain unavailable during the non-live Vitest suite." >&2 + systemctl show --property=LoadState,ActiveState,SubState docker.service docker.socket >&2 || true + exit 1 +fi +'@ +} + +function Get-WslContainerRuntimeStopScript { + # This CI-owned distro can restart between steps; keep masks until live testing. + $script = @' +set -euo pipefail +systemctl mask --now docker.service docker.socket +'@ + return $script + "`n" + (Get-WslContainerRuntimeUnavailableScript) +} + +function Stop-WslContainerRuntime { + param([Parameter(Mandatory = $true)] [string]$Distro) + + Repair-WslSystemdAfterInstall -Distro $Distro + Invoke-WslScript -Distro $Distro -User root -Script (Get-WslContainerRuntimeStopScript) +} + +function Get-WslContainerRuntimeStartScript { + return @' +set -euo pipefail +systemctl unmask docker.service docker.socket +systemctl start docker.service +timeout 30s bash -c 'until docker info >/dev/null 2>&1; do sleep 1; done' +docker info +'@ +} + +function Start-WslContainerRuntime { + param([Parameter(Mandatory = $true)] [string]$Distro) + + Invoke-WslScript -Distro $Distro -User root -Script (Get-WslContainerRuntimeStartScript) +} + function Get-WslNodeInstallScript { return @' set -euo pipefail