From e1cc07a4af542b18ee7bf969e434f66bc8704133 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 12 Nov 2026 20:01:52 -0300 Subject: [PATCH 1/2] test: Remote-SSH in a real VS Code against a spin e2e stack task test:remote-ssh installs the packaged VSIX and Microsoft's Remote-SSH from the Marketplace into VS Code stable and opens vscode-remote:// ssh-remote+spin./workspace the way a reconnect does, nothing of Spinbox's run first. Each case runs in bubblewrap with a home of its own over $HOME (ssh reads ~/.ssh from the password database), a wrapper around spin that stamps each call, and a probe extension that says what the window is connected to, runs a command in its terminal, and closes notifications when asked. Every check is read from VS Code's logs, the probe and the stack: - running and stopped: Spinbox activates on onResolveRemoteAuthority: ssh-remote, its spin code --start ends before Remote-SSH's resolver is called, the workspace (stopped one included) is running, and the terminal's command is read back with spin exec; - unenrolled: Spinbox shows key_not_enrolled, Remote-SSH never resolves, and the window leaves the remote once the message is closed. The unenrolled case fails on main: closing the message lets activation resolve, and VS Code then calls Remote-SSH's resolver while the window unloads, which runs into spin's askpass box, times out after ConnectTimeout, and shows Remote-SSH's own "Could not establish connection" dialog; the window is local only 17-85 s later (four runs). Holding activation instead leaves the unload waiting on the resolution forever (join.disconnectRemote), so it is not a fix. It needs a stack and bwrap, so no workflow runs it. Co-Authored-By: Claude Opus 5.5 --- Taskfile.yml | 14 ++ test/remote-ssh/probe/extension.cjs | 61 +++++ test/remote-ssh/probe/package.json | 19 ++ test/remote-ssh/run.mjs | 370 ++++++++++++++++++++++++++++ 4 files changed, 464 insertions(+) create mode 100644 test/remote-ssh/probe/extension.cjs create mode 100644 test/remote-ssh/probe/package.json create mode 100644 test/remote-ssh/run.mjs diff --git a/Taskfile.yml b/Taskfile.yml index 792b05d..71e1440 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -54,6 +54,20 @@ tasks: - task: build - '{{if .XVFB}}xvfb-run -a {{end}}pnpm test:integration' + test:remote-ssh: + desc: >- + The packaged VSIX and Microsoft's Remote-SSH in a real VS Code stable, opening the workspaces + of a spin e2e stack as a reconnect does (test/remote-ssh/run.mjs): Spinbox readies each before + Remote-SSH resolves it, and a refused key closes the remote with Spinbox's message. Needs the + stack - SPIN_STACK= after its `task e2e:up` and `task e2e:workspaces` - and + bwrap, so no workflow runs it. Under xvfb-run when it is installed. + vars: + XVFB: + sh: command -v xvfb-run || true + cmds: + - task: package + - '{{if .XVFB}}xvfb-run -a {{end}}node test/remote-ssh/run.mjs' + package: desc: Package the extension into spinbox.vsix with vsce, without the CLI - what CI's gate checks packages. package:all makes the ones that are published. cmds: diff --git a/test/remote-ssh/probe/extension.cjs b/test/remote-ssh/probe/extension.cjs new file mode 100644 index 0000000..faa1fb3 --- /dev/null +++ b/test/remote-ssh/probe/extension.cjs @@ -0,0 +1,61 @@ +// The lane's eyes in the window (test/remote-ssh/run.mjs): each thing it sees is a JSON line +// appended to SPINBOX_PROBE_OUT. It activates as Spinbox does, on the resolution of ssh-remote, +// so it is there while Spinbox's activation holds Remote-SSH back, and again in the window a +// closed remote leaves. +// +// - activate: the remote the window is for, and its folder; +// - terminal: in a window connected to a workspace, what a command run in its terminal wrote, +// read back over the remote's file system; +// - dismissed: the notifications closed when the lane asked, by creating SPINBOX_PROBE_DISMISS, +// as a person closes one: a notification nobody answers holds its caller forever. +const fs = require("node:fs") +const vscode = require("vscode") + +const out = process.env.SPINBOX_PROBE_OUT +const dismiss = process.env.SPINBOX_PROBE_DISMISS + +function say(event, fields = {}) { + if (out) fs.appendFileSync(out, `${JSON.stringify({ t: Date.now(), event, ...fields })}\n`) +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +exports.activate = () => { + const folder = vscode.workspace.workspaceFolders?.[0]?.uri + say("activate", { remoteName: vscode.env.remoteName ?? null, folder: folder?.toString() ?? null }) + if (dismiss) void dismissWhenAsked() + if (vscode.env.remoteName === "ssh-remote" && folder !== undefined) void runInTerminal(folder) +} + +async function dismissWhenAsked() { + for (;;) { + if (fs.existsSync(dismiss)) { + fs.rmSync(dismiss, { force: true }) + await vscode.commands.executeCommand("notifications.clearAll") + say("dismissed") + } + await sleep(500) + } +} + +// The terminal opens once the remote is connected; until then the file is not there to read. +async function runInTerminal(folder) { + const path = `/tmp/remote-ssh-probe-${process.pid}-${Date.now()}` + const terminal = vscode.window.createTerminal({ name: "remote-ssh probe" }) + terminal.sendText(`echo "$(hostname) $(id -un) $PWD" > ${path}`) + const file = folder.with({ path }) + const end = Date.now() + 300_000 + while (Date.now() < end) { + try { + const body = Buffer.from(await vscode.workspace.fs.readFile(file)) + .toString("utf8") + .trim() + if (body !== "") { + say("terminal", { path, output: body }) + return + } + } catch {} + await sleep(1_000) + } + say("terminal", { path, output: null }) +} diff --git a/test/remote-ssh/probe/package.json b/test/remote-ssh/probe/package.json new file mode 100644 index 0000000..ac4922f --- /dev/null +++ b/test/remote-ssh/probe/package.json @@ -0,0 +1,19 @@ +{ + "name": "remote-ssh-probe", + "displayName": "The remote-ssh lane's probe", + "description": "Says to test/remote-ssh/run.mjs what the window is connected to, runs a command in its terminal, and dismisses the notifications the lane asks it to.", + "version": "0.0.0", + "publisher": "spin-stack", + "private": true, + "engines": { + "vscode": "^1.146.0" + }, + "extensionKind": [ + "ui" + ], + "main": "./extension.cjs", + "activationEvents": [ + "*", + "onResolveRemoteAuthority:ssh-remote" + ] +} diff --git a/test/remote-ssh/run.mjs b/test/remote-ssh/run.mjs new file mode 100644 index 0000000..afb3c9f --- /dev/null +++ b/test/remote-ssh/run.mjs @@ -0,0 +1,370 @@ +// The Spinbox VSIX and Microsoft's Remote-SSH, installed into a real VS Code stable, opening the +// workspaces of a spin e2e stack (spin's `task e2e:up` and `task e2e:workspaces`) the way a +// reconnect or a recent folder does: VS Code started on vscode-remote://ssh-remote+spin., with +// nothing of Spinbox's run first. What src/remote/connect.ts claims is checked from what each +// side logs: +// +// - running: Spinbox activates on onResolveRemoteAuthority:ssh-remote and its `spin code` ends +// before Remote-SSH resolves the authority, Remote-SSH connects, and a command typed into the +// window's terminal runs in the workspace; +// - stopped: the same, with the workspace started by that `spin code --start`, and no error of +// Remote-SSH's; +// - unenrolled: with a key the account does not have, Spinbox says key_not_enrolled (whose +// action is Add Key), Remote-SSH never resolves, and once the message is closed - READ_MS +// after it is said, as a person reads it - the window leaves the remote. +// +// SPIN_STACK names the spin checkout whose stack is up: its bin/spin, and the fixture user's home +// hack/e2estack keeps. Each case runs in bubblewrap with a home of its own bound over $HOME, since +// ssh finds ~/.ssh from the password database and not from HOME, so neither spin code's ssh config +// nor the stack's session ever touches the real one. CASES=running,stopped,unenrolled picks some. +// Needs bwrap, ssh-keygen and a display: `task test:remote-ssh` runs it under xvfb-run. +import { spawn, spawnSync } from "node:child_process" +import { + appendFileSync, + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + writeFileSync, +} from "node:fs" +import { homedir, tmpdir } from "node:os" +import { dirname, join, resolve } from "node:path" +import { fileURLToPath } from "node:url" +import { + downloadAndUnzipVSCode, + resolveCliArgsFromVSCodeExecutablePath, +} from "@vscode/test-electron" + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "../..") +const stack = process.env.SPIN_STACK +if (!stack) { + console.error("SPIN_STACK: the spin checkout whose e2e stack is up") + process.exit(2) +} +const spin = resolve(stack, "bin/spin") +const person = resolve(stack, "_output/run/e2e/people/user") +const home = homedir() +const work = mkdtempSync(join(tmpdir(), "spinbox-remote-ssh-")) +const vsix = process.env.SPINBOX_VSIX ?? resolve(root, "spinbox.vsix") +const only = process.env.CASES?.split(",") +const WAIT_MS = 360_000 +// How long the unenrolled case leaves Spinbox's message up before closing it. +const READ_MS = Number(process.env.READ_MS ?? 30_000) + +const say = (line) => console.log(`==> ${line}`) + +// VS Code stable, as Marketplace users have it, with Remote-SSH from the Marketplace beside the +// VSIX: neither is a development copy. +const exe = await downloadAndUnzipVSCode({ + version: "stable", + cachePath: resolve(root, ".vscode-test"), +}) +const [cli] = resolveCliArgsFromVSCodeExecutablePath(exe) +const version = spawnSync(cli, ["--version"], { encoding: "utf8" }).stdout.split("\n") +const extensions = join(work, "extensions") +const installed = spawnSync( + cli, + [ + `--extensions-dir=${extensions}`, + `--user-data-dir=${join(work, "install")}`, + "--install-extension", + "ms-vscode-remote.remote-ssh", + "--install-extension", + vsix, + ], + { encoding: "utf8" }, +) +if (installed.status !== 0) throw new Error(`installing the extensions: ${installed.stderr}`) +const listed = readdirSync(extensions).filter((d) => d.includes(".")) +say(`VS Code ${version[0]} (${version[1]}), ${listed.join(", ")}`) + +// The person's home for one case: their session and the stack's root, and their enrolled key, or +// a key of nobody's. +const KEPT = [ + ".spin/config.yaml", + ".trust/stack-root.pem", + ".ssh/id_ed25519", + ".ssh/id_ed25519.pub", +] + +function homeFor(dir, enrolled) { + for (const f of KEPT) { + mkdirSync(dirname(join(dir, f)), { recursive: true, mode: 0o700 }) + if (enrolled || !f.startsWith(".ssh/")) copyFileSync(join(person, f), join(dir, f)) + } + if (!enrolled) { + const key = join(dir, ".ssh/id_ed25519") + const r = spawnSync("ssh-keygen", ["-q", "-t", "ed25519", "-N", "", "-f", key]) + if (r.status !== 0) throw new Error("ssh-keygen failed") + } +} + +const trust = `/etc/ssl/certs:${join(home, ".trust")}` + +// The CLI as the person, outside VS Code, for what the stack says of a workspace. +function asPerson(...args) { + const r = spawnSync(spin, args, { + encoding: "utf8", + env: { ...process.env, HOME: person, SSL_CERT_DIR: `/etc/ssl/certs:${join(person, ".trust")}` }, + }) + return { ok: r.status === 0, out: r.stdout.trim(), err: r.stderr.trim() } +} + +function stateOf(name) { + const r = asPerson("-o", "json", "workspace", "list") + if (!r.ok) throw new Error(`listing workspaces: ${r.err}`) + return JSON.parse(r.out).items.find((w) => w.name === name)?.state +} + +const lines = (body) => body.split("\n").filter((l) => l !== "") + +// Every line VS Code's logs hold in the files whose names end with suffix. +function logOf(userData, suffix) { + const found = [] + const walk = (dir) => { + for (const e of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, e.name) + if (e.isDirectory()) walk(p) + else if (e.name.endsWith(suffix)) found.push(...lines(readFileSync(p, "utf8"))) + } + } + if (existsSync(join(userData, "logs"))) walk(join(userData, "logs")) + return found +} + +const read = (path) => (existsSync(path) ? lines(readFileSync(path, "utf8")) : []) + +// [hh:mm:ss.mmm] at the start of a Remote-SSH line, as epoch ms of the day it ran. +function timeOf(line, day) { + const m = /^\[(\d{2}):(\d{2}):(\d{2})\.(\d{3})\]/.exec(line) + if (m === null) return undefined + const d = new Date(day) + d.setHours(Number(m[1]), Number(m[2]), Number(m[3]), Number(m[4])) + return d.getTime() +} + +async function openCase(name, workspace, { enrolled, until }) { + const dir = join(work, name) + const caseHome = join(dir, "home") + const userData = join(dir, "user-data") + homeFor(caseHome, enrolled) + // An empty system ssh config: in bubblewrap's namespace root's files are nobody's, which ssh + // refuses to read. + writeFileSync(join(dir, "ssh_config"), "") + // Each spin the extension runs, said with when it started and ended, around the real one. + const calls = join(dir, "spin-calls.log") + const wrapper = join(dir, "spin") + const stamp = (what) => `echo "$(date +%s%3N) ${what} $*" >> ${calls}` + writeFileSync( + wrapper, + `#!/bin/sh\n${stamp("start")}\n${spin} "$@"\nrc=$?\n${stamp("end $rc")}\nexit $rc\n`, + ) + chmodSync(wrapper, 0o755) + mkdirSync(join(userData, "User"), { recursive: true }) + writeFileSync( + join(userData, "User/settings.json"), + JSON.stringify({ + "spinbox.cliPath": wrapper, + "spinbox.telemetry": false, + "telemetry.telemetryLevel": "off", + "update.mode": "none", + "extensions.autoUpdate": false, + "extensions.autoCheckUpdates": false, + "workbench.startupEditor": "none", + "security.workspace.trust.enabled": false, + }), + ) + const probeOut = join(dir, "probe.jsonl") + const dismiss = join(dir, "dismiss") + const folder = `vscode-remote://ssh-remote+spin.${workspace}/workspace` + const started = Date.now() + const bwrap = [ + ["--die-with-parent"], + ["--dev-bind", "/", "/"], + ["--bind", caseHome, home], + ["--bind", root, root], + ["--bind", resolve(stack), resolve(stack)], + ["--tmpfs", "/tmp"], + ["--bind", "/tmp/.X11-unix", "/tmp/.X11-unix"], + ["--ro-bind", join(dir, "ssh_config"), "/etc/ssh/ssh_config"], + ["--tmpfs", "/etc/ssh/ssh_config.d"], + process.env.XAUTHORITY ? ["--bind", process.env.XAUTHORITY, process.env.XAUTHORITY] : [], + ].flat() + const code = spawn( + "bwrap", + [ + ...bwrap, + exe, + "--no-sandbox", + "--disable-gpu", + `--user-data-dir=${userData}`, + `--extensions-dir=${extensions}`, + `--extensionDevelopmentPath=${join(root, "test/remote-ssh/probe")}`, + `--folder-uri=${folder}`, + "--log=trace", + "--skip-welcome", + "--skip-release-notes", + "--disable-workspace-trust", + "--new-window", + ], + { + detached: true, + stdio: "ignore", + env: { + ...process.env, + SSL_CERT_DIR: trust, + SPINBOX_PROBE_OUT: probeOut, + SPINBOX_PROBE_DISMISS: dismiss, + }, + }, + ) + const seen = () => ({ + started, + probe: read(probeOut).map((l) => JSON.parse(l)), + calls: read(calls), + spinbox: logOf(userData, "Spinbox.log"), + remoteSSH: logOf(userData, "Remote - SSH.log"), + exthost: logOf(userData, "exthost.log"), + }) + let s = seen() + const end = Date.now() + WAIT_MS + while (Date.now() < end && !until(s, dismiss)) { + await new Promise((r) => setTimeout(r, 1_000)) + s = seen() + } + try { + process.kill(-code.pid, "SIGTERM") + } catch {} + await new Promise((r) => setTimeout(r, 2_000)) + return seen() +} + +const failures = [] +function check(name, ok, evidence) { + say(`${ok ? "ok " : "FAIL"} ${name}`) + for (const line of [evidence].flat().filter((l) => l !== undefined)) console.log(` ${line}`) + if (!ok) failures.push(name) +} + +// Spinbox's activation by the resolution, the `spin code ` it ran and when that ended, +// and the first line Remote-SSH says resolving the authority with. +function ordering(s, workspace) { + const activated = s.exthost.find( + (l) => + l.includes("_doActivateExtension spin-stack.spinbox,") && + l.includes("onResolveRemoteAuthority:ssh-remote"), + ) + const start = s.calls.find((l) => / start code /.test(l) && l.includes(` ${workspace} `)) + const end = s.calls.find((l) => / end \d+ code /.test(l) && l.includes(` ${workspace} `)) + const resolving = s.remoteSSH.find((l) => /SSH Resolver called/.test(l)) + return { + activated, + start, + end, + endMs: end === undefined ? undefined : Number(end.split(" ")[0]), + rc: end === undefined ? undefined : Number(end.split(" ")[2]), + resolving, + resolvingMs: resolving === undefined ? undefined : timeOf(resolving, s.started), + } +} + +// What Remote-SSH says when it gives up on a connection; its server's own lines ([server]) are the +// remote's, and say nothing of the connection. +const GAVE_UP = /resolver error|could not establish|failed to connect|connection (closed|refused)/i +const errorsOf = (s) => s.remoteSSH.filter((l) => !l.includes("[server]") && GAVE_UP.test(l)) + +const connected = (s) => s.probe.some((e) => e.event === "terminal") + +async function connects(name, workspace) { + const s = await openCase(name, workspace, { enrolled: true, until: connected }) + const o = ordering(s, workspace) + check( + `${name}: Spinbox ran spin code --start before Remote-SSH resolved`, + o.activated !== undefined && + o.rc === 0 && + o.start?.includes("--start") && + o.endMs <= o.resolvingMs, + [o.activated?.slice(0, 160), o.start, o.end, `Remote-SSH: ${o.resolving}`], + ) + const t = s.probe.find((e) => e.event === "terminal") + const there = t?.output ? asPerson("exec", workspace, "--no-tty", "--", "cat", t.path) : undefined + check( + `${name}: a command run in the window's terminal ran in ${workspace}`, + there?.ok === true && there.out === t.output, + [`terminal wrote ${t?.path}: ${t?.output}`, `spin exec ${workspace} cat: ${there?.out}`], + ) + const errors = errorsOf(s) + check(`${name}: Remote-SSH said no error`, errors.length === 0, errors.slice(0, 5)) +} + +const cases = { + running: async () => { + const state = stateOf("e2e-running") + check("running: e2e-running is running", state === "WORKSPACE_STATE_RUNNING", state) + await connects("running", "e2e-running") + }, + stopped: async () => { + const before = stateOf("e2e-stopped") + check("stopped: e2e-stopped is stopped", before === "WORKSPACE_STATE_STOPPED", before) + await connects("stopped", "e2e-stopped") + const after = stateOf("e2e-stopped") + check("stopped: Spinbox started it", after === "WORKSPACE_STATE_RUNNING", after) + // Left as e2e:workspaces makes it, for the next run. + asPerson("workspace", "stop", "e2e-stopped", "--wait", "--timeout", "3m") + }, + unenrolled: async () => { + let refusedAt + const s = await openCase("unenrolled", "e2e-running", { + enrolled: false, + until: (s, dismiss) => { + if (refusedAt === undefined && s.spinbox.some((l) => l.includes("[key_not_enrolled]"))) { + refusedAt = Date.now() + } + if ( + refusedAt !== undefined && + Date.now() - refusedAt >= READ_MS && + !s.probe.some((e) => e.event === "dismissed") && + !existsSync(dismiss) + ) { + appendFileSync(dismiss, "") + } + const activations = s.probe.filter((e) => e.event === "activate") + return activations.length >= 2 && activations.at(-1).remoteName === null + }, + }) + const refused = s.spinbox.find((l) => l.includes("[key_not_enrolled]")) + check( + "unenrolled: Spinbox showed key_not_enrolled (Add Key)", + refused !== undefined, + refused?.slice(0, 160), + ) + const o = ordering(s, "e2e-running") + const dismissed = s.probe.find((e) => e.event === "dismissed") + check( + "unenrolled: Remote-SSH did not resolve while the message was up", + o.resolving === undefined || o.resolvingMs >= dismissed?.t, + [o.end, `message closed at ${dismissed?.t}`, `Remote-SSH: ${o.resolving}`], + ) + check("unenrolled: Remote-SSH never resolved", o.resolving === undefined, o.resolving) + const errors = errorsOf(s) + check("unenrolled: Remote-SSH said no error", errors.length === 0, errors.slice(0, 5)) + const left = s.probe.filter((e) => e.event === "activate").at(-1) + check( + "unenrolled: the window left the remote within 5 s of the message closing", + left?.remoteName === null && dismissed !== undefined && left.t - dismissed.t <= 5_000, + `message closed at ${dismissed?.t}, window local at ${left?.t} (remoteName=${left?.remoteName})`, + ) + }, +} + +for (const [name, run] of Object.entries(cases)) { + if (only === undefined || only.includes(name)) await run() +} +say(`logs in ${work}`) +if (failures.length > 0) { + console.error(`${failures.length} failed: ${failures.join("; ")}`) + process.exit(1) +} From 77199127b990b2d78ff919cab71658074e8ec36c Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 12 Nov 2026 20:05:41 -0300 Subject: [PATCH 2/2] test: the remote-ssh probe under testdata, as sfw leaves it sfw check holds every package.json outside testdata to pnpm; the probe is an extension VS Code loads, never installed, as testdata/resolver is. Co-Authored-By: Claude Opus 5.5 --- test/remote-ssh/run.mjs | 2 +- test/remote-ssh/{ => testdata}/probe/extension.cjs | 0 test/remote-ssh/{ => testdata}/probe/package.json | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename test/remote-ssh/{ => testdata}/probe/extension.cjs (100%) rename test/remote-ssh/{ => testdata}/probe/package.json (100%) diff --git a/test/remote-ssh/run.mjs b/test/remote-ssh/run.mjs index afb3c9f..ee48c9b 100644 --- a/test/remote-ssh/run.mjs +++ b/test/remote-ssh/run.mjs @@ -202,7 +202,7 @@ async function openCase(name, workspace, { enrolled, until }) { "--disable-gpu", `--user-data-dir=${userData}`, `--extensions-dir=${extensions}`, - `--extensionDevelopmentPath=${join(root, "test/remote-ssh/probe")}`, + `--extensionDevelopmentPath=${join(root, "test/remote-ssh/testdata/probe")}`, `--folder-uri=${folder}`, "--log=trace", "--skip-welcome", diff --git a/test/remote-ssh/probe/extension.cjs b/test/remote-ssh/testdata/probe/extension.cjs similarity index 100% rename from test/remote-ssh/probe/extension.cjs rename to test/remote-ssh/testdata/probe/extension.cjs diff --git a/test/remote-ssh/probe/package.json b/test/remote-ssh/testdata/probe/package.json similarity index 100% rename from test/remote-ssh/probe/package.json rename to test/remote-ssh/testdata/probe/package.json