diff --git a/packages/runtime-node/src/process-protocol.ts b/packages/runtime-node/src/process-protocol.ts index 51053ea..a8a8f42 100644 --- a/packages/runtime-node/src/process-protocol.ts +++ b/packages/runtime-node/src/process-protocol.ts @@ -364,7 +364,7 @@ export interface RuntimeWarningMessage { } export interface RuntimeStoppedMessage { - reason: "parent" | "SIGINT" | "SIGTERM"; + reason: "parent" | "SIGHUP" | "SIGINT" | "SIGTERM"; type: "stopped"; } diff --git a/packages/runtime-node/src/runner.ts b/packages/runtime-node/src/runner.ts index e3bf0aa..dfc1bc3 100644 --- a/packages/runtime-node/src/runner.ts +++ b/packages/runtime-node/src/runner.ts @@ -1428,13 +1428,19 @@ export async function runRuntimeProcess(): Promise { send({ ...errorDetails(error), type: "background-error" }); }); - async function stop(reason: RuntimeStoppedMessage["reason"]): Promise { - if (runtime !== undefined) { - await runtime.close(reason); - } - // `orange dev` exits after this process, so the Engine must exit first. - await stopRivetEngineAfterLastDevSession(); - resolveCompletion?.(); + // A closed terminal sends SIGHUP here while `orange dev` also sends shutdown. A second stop + // must wait for the Engine too, or this process exits before it. + let stopping: Promise | undefined; + function stop(reason: RuntimeStoppedMessage["reason"]): Promise { + stopping ??= (async () => { + if (runtime !== undefined) { + await runtime.close(reason); + } + // `orange dev` exits after this process, so the Engine must exit first. + await stopRivetEngineAfterLastDevSession(); + resolveCompletion?.(); + })(); + return stopping; } process.on("message", (rawMessage: unknown) => { @@ -1465,6 +1471,9 @@ export async function runRuntimeProcess(): Promise { }); process.once("SIGINT", () => void stop("SIGINT")); process.once("SIGTERM", () => void stop("SIGTERM")); + // A closed terminal sends SIGHUP to this process too, and can send it more than once. Node's + // default exits without stopping the Engine. + process.on("SIGHUP", () => void stop("SIGHUP")); // The IPC channel closes when the parent dies, so the runtime stops with it. process.once("disconnect", () => void stop("parent")); // Runs on every exit, also one during startup before a registry exists. diff --git a/src/cli/closed-output.ts b/src/cli/closed-output.ts index 90b5ec8..1380f74 100644 --- a/src/cli/closed-output.ts +++ b/src/cli/closed-output.ts @@ -1,6 +1,7 @@ // A reader such as `head` or `jq` can close the pipe before the output ends. The rest of the output // then has nowhere to go: Orange drops it, and a command that runs until it is stopped, such as // `orange dev`, stops as for Ctrl-C. A short command finishes and exits with its usual code. +// A closed terminal is the same case: a write to it fails with EIO instead of EPIPE. const closedStreams = new WeakSet(); const listeners = new Set<() => void>(); @@ -12,10 +13,10 @@ function markClosed(): void { for (const listener of listeners) listener(); } -/** Drops the output of `stream` after its reader closes the pipe, instead of crashing on EPIPE. */ +/** Drops the output of `stream` after its reader closes the pipe or terminal, instead of crashing on EPIPE or EIO. */ export function dropOutputAfterReaderCloses(stream: NodeJS.WriteStream): void { stream.on("error", (error: NodeJS.ErrnoException) => { - if (error.code !== "EPIPE") throw error; + if (error.code !== "EPIPE" && error.code !== "EIO") throw error; if (closedStreams.has(stream)) return; closedStreams.add(stream); stream.write = (...args: unknown[]): boolean => { diff --git a/src/commands/dev-session.ts b/src/commands/dev-session.ts index 5ca26b1..03c8d4d 100644 --- a/src/commands/dev-session.ts +++ b/src/commands/dev-session.ts @@ -2,10 +2,10 @@ import { spawn } from "node:child_process"; import { onOutputClosed } from "../cli/closed-output.js"; -export type StopReason = "SIGINT" | "SIGTERM" | "key"; +export type StopReason = "SIGHUP" | "SIGINT" | "SIGTERM" | "key"; export interface StopRequests { - // Resolves on the first Ctrl-C, SIGTERM, or `x`. + // Resolves on the first Ctrl-C, SIGTERM, SIGHUP, or `x`. readonly first: Promise; // Resolves on the second, which forces the stop. readonly second: Promise; @@ -25,14 +25,23 @@ export function listenForStopRequests(): StopRequests { }; const onSigint = (): void => request("SIGINT"); const onSigterm = (): void => request("SIGTERM"); + // After the terminal closes or the reader of the output goes away, nobody is left to force the + // stop, and these events can come more than once. So they only start a stop. + const startStop = (reason: StopReason): void => { + if (count === 0) request(reason); + }; + // A closed terminal sends SIGHUP. Node's default exits at once, which leaves the Engine running. + const onSighup = (): void => startStop("SIGHUP"); process.on("SIGINT", onSigint); process.on("SIGTERM", onSigterm); + process.on("SIGHUP", onSighup); // Nobody reads the output any more, as after `orange dev --json | head`. - const stopOnClosedOutput = onOutputClosed(() => request("SIGTERM")); + const stopOnClosedOutput = onOutputClosed(() => startStop("SIGTERM")); return { dispose(): void { process.off("SIGINT", onSigint); process.off("SIGTERM", onSigterm); + process.off("SIGHUP", onSighup); stopOnClosedOutput(); }, first: first.promise, diff --git a/tests/cli/orange-dev-shared-engine.test.ts b/tests/cli/orange-dev-shared-engine.test.ts index a82eade..cba6ace 100644 --- a/tests/cli/orange-dev-shared-engine.test.ts +++ b/tests/cli/orange-dev-shared-engine.test.ts @@ -46,3 +46,20 @@ test("the orange dev that started the shared Engine exits and the other keeps se await project.dispose(); } }); + +// Closing the terminal of orange dev stops it, but the Engine runs in its own process group and +// gets no SIGHUP. A next orange dev would reuse an Engine left running. +test("orange dev stops its Engine when its terminal closes", { timeout: 120_000 }, async () => { + const project = await copyProject(join(repositoryRoot, "examples", "hello-durable-object")); + try { + const dev = await startOrangeDev(project.directory, "wrangler.jsonc", { terminal: true }); + assert.deepEqual(await dev.json("/alice"), { greeting: "Hello, alice!", visits: 1 }); + const pid = await enginePid(project.directory); + + await dev.hangUp(); + assert.equal(processIsAlive(pid), false, "orange dev exited and left the Engine running"); + } finally { + await stopProjectEngines(project.directory); + await project.dispose(); + } +}); diff --git a/tests/harness/orange-dev.ts b/tests/harness/orange-dev.ts index 675de98..69617a9 100644 --- a/tests/harness/orange-dev.ts +++ b/tests/harness/orange-dev.ts @@ -34,6 +34,11 @@ export interface RunningOrangeDev { kill(): Promise; // Resolves with the exit code once orange dev exits on its own or after close(). exited: Promise; + /** + * Closes the terminal of orange dev, as closing its window does, and resolves after orange dev + * exits. It needs `startOrangeDev(..., { terminal: true })`. + */ + hangUp(): Promise; /** Fetches `path` from orange dev, asserts a 200, and returns the parsed JSON body. A failure shows the body and the output. */ json(path: string, init?: RequestInit): Promise; origin: string; @@ -155,16 +160,27 @@ export interface OrangeCli { runtime: string; } +// `script` runs the command on a new terminal, as a terminal window does. +function inTerminal(command: string, args: readonly string[]): [string, string[]] { + if (process.platform === "darwin") return ["script", ["-q", "/dev/null", command, ...args]]; + const line = [command, ...args].map((arg) => `'${arg.replaceAll("'", "'\\''")}'`).join(" "); + return ["script", ["-qfec", `exec ${line}`, "/dev/null"]]; +} + function spawnOrange( args: readonly string[], environment: NodeJS.ProcessEnv = {}, cli?: OrangeCli, + terminal = false, ): ChildProcess { - const [command, prefix] = + const [runtime, prefix] = cli === undefined ? [process.execPath, ["--import", "tsx", cliEntrypoint]] : [cli.runtime, [cli.entrypoint]]; - return spawn(command, [...prefix, ...args], { + const [command, commandArgs] = terminal + ? inTerminal(runtime, [...prefix, ...args]) + : [runtime, [...prefix, ...args]]; + return spawn(command, commandArgs, { cwd: repositoryRoot, env: { ...process.env, ...environment, CI: "1", NO_COLOR: "1" }, stdio: ["ignore", "pipe", "pipe"], @@ -226,6 +242,8 @@ export interface StartOrangeDevOptions { args?: readonly string[]; cli?: OrangeCli; environment?: NodeJS.ProcessEnv; + // Runs orange dev on a terminal, so that hangUp() can close it. + terminal?: boolean; } // More than one config starts a multi-Worker session. The first Worker serves the port. @@ -253,6 +271,7 @@ export async function startOrangeDev( ...(options.actorLifecycleLogs === true ? { RIVET_LOG_LEVEL: "info" } : {}), }, options.cli, + options.terminal, ); const exited = new Promise((resolveExit) => { child.once("exit", (code) => resolveExit(code)); @@ -329,6 +348,35 @@ export async function startOrangeDev( } }, exited, + async hangUp(): Promise { + if (options.terminal !== true) throw new Error("hangUp() needs startOrangeDev(..., { terminal: true })"); + closed = true; + // orange dev and its runtime, under `script`. + const pids = child.pid === undefined ? [] : descendantPids(child.pid); + // The terminal closes, so writes to it fail and orange dev gets SIGHUP. The shell of a + // terminal window also sends SIGHUP to the job, and the kernel sends one more when the + // shell exits. The Rivet Engine runs in its own process group, so it gets none. + child.kill("SIGKILL"); + for (let signal = 0; signal < 2; signal += 1) { + // Two signals sent at once can merge into one. + await delay(200); + for (const pid of pids) { + try { + process.kill(pid, "SIGHUP"); + } catch { + // It already exited. + } + } + } + const deadline = Date.now() + 30_000; + while (pids.some(processIsAlive)) { + if (Date.now() > deadline) { + for (const pid of pids.filter(processIsAlive)) process.kill(pid, "SIGKILL"); + throw new Error(`orange dev did not exit after its terminal closed:\n${output()}`); + } + await delay(25); + } + }, async kill(): Promise { if (closed) return; closed = true;