From 20c64211620b2a3fe297a9e1e9b0b9c7da4767c5 Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Sat, 14 Nov 2026 19:40:27 +0400 Subject: [PATCH] fix(dev): stop the Rivet Engine when the terminal closes Closing the terminal sends SIGHUP to orange dev and its runtime. Node's default exits at once, so the Engine, in its own process group, kept running and the next orange dev reused it. Treat SIGHUP as a stop request in both processes. A repeated SIGHUP or an EIO write to the closed terminal only starts a stop, so it can no longer force-kill the runtime before it stops the Engine. --- packages/runtime-node/src/process-protocol.ts | 2 +- packages/runtime-node/src/runner.ts | 23 +++++--- src/cli/closed-output.ts | 5 +- src/commands/dev-session.ts | 15 ++++-- tests/cli/orange-dev-shared-engine.test.ts | 17 ++++++ tests/harness/orange-dev.ts | 52 ++++++++++++++++++- 6 files changed, 99 insertions(+), 15 deletions(-) diff --git a/packages/runtime-node/src/process-protocol.ts b/packages/runtime-node/src/process-protocol.ts index 51053ea..a8a8f42 100644 --- a/packages/runtime-node/src/process-protocol.ts +++ b/packages/runtime-node/src/process-protocol.ts @@ -364,7 +364,7 @@ export interface RuntimeWarningMessage { } export interface RuntimeStoppedMessage { - reason: "parent" | "SIGINT" | "SIGTERM"; + reason: "parent" | "SIGHUP" | "SIGINT" | "SIGTERM"; type: "stopped"; } diff --git a/packages/runtime-node/src/runner.ts b/packages/runtime-node/src/runner.ts index e3bf0aa..dfc1bc3 100644 --- a/packages/runtime-node/src/runner.ts +++ b/packages/runtime-node/src/runner.ts @@ -1428,13 +1428,19 @@ export async function runRuntimeProcess(): Promise { send({ ...errorDetails(error), type: "background-error" }); }); - async function stop(reason: RuntimeStoppedMessage["reason"]): Promise { - if (runtime !== undefined) { - await runtime.close(reason); - } - // `orange dev` exits after this process, so the Engine must exit first. - await stopRivetEngineAfterLastDevSession(); - resolveCompletion?.(); + // A closed terminal sends SIGHUP here while `orange dev` also sends shutdown. A second stop + // must wait for the Engine too, or this process exits before it. + let stopping: Promise | undefined; + function stop(reason: RuntimeStoppedMessage["reason"]): Promise { + stopping ??= (async () => { + if (runtime !== undefined) { + await runtime.close(reason); + } + // `orange dev` exits after this process, so the Engine must exit first. + await stopRivetEngineAfterLastDevSession(); + resolveCompletion?.(); + })(); + return stopping; } process.on("message", (rawMessage: unknown) => { @@ -1465,6 +1471,9 @@ export async function runRuntimeProcess(): Promise { }); process.once("SIGINT", () => void stop("SIGINT")); process.once("SIGTERM", () => void stop("SIGTERM")); + // A closed terminal sends SIGHUP to this process too, and can send it more than once. Node's + // default exits without stopping the Engine. + process.on("SIGHUP", () => void stop("SIGHUP")); // The IPC channel closes when the parent dies, so the runtime stops with it. process.once("disconnect", () => void stop("parent")); // Runs on every exit, also one during startup before a registry exists. diff --git a/src/cli/closed-output.ts b/src/cli/closed-output.ts index 90b5ec8..1380f74 100644 --- a/src/cli/closed-output.ts +++ b/src/cli/closed-output.ts @@ -1,6 +1,7 @@ // A reader such as `head` or `jq` can close the pipe before the output ends. The rest of the output // then has nowhere to go: Orange drops it, and a command that runs until it is stopped, such as // `orange dev`, stops as for Ctrl-C. A short command finishes and exits with its usual code. +// A closed terminal is the same case: a write to it fails with EIO instead of EPIPE. const closedStreams = new WeakSet(); const listeners = new Set<() => void>(); @@ -12,10 +13,10 @@ function markClosed(): void { for (const listener of listeners) listener(); } -/** Drops the output of `stream` after its reader closes the pipe, instead of crashing on EPIPE. */ +/** Drops the output of `stream` after its reader closes the pipe or terminal, instead of crashing on EPIPE or EIO. */ export function dropOutputAfterReaderCloses(stream: NodeJS.WriteStream): void { stream.on("error", (error: NodeJS.ErrnoException) => { - if (error.code !== "EPIPE") throw error; + if (error.code !== "EPIPE" && error.code !== "EIO") throw error; if (closedStreams.has(stream)) return; closedStreams.add(stream); stream.write = (...args: unknown[]): boolean => { diff --git a/src/commands/dev-session.ts b/src/commands/dev-session.ts index 5ca26b1..03c8d4d 100644 --- a/src/commands/dev-session.ts +++ b/src/commands/dev-session.ts @@ -2,10 +2,10 @@ import { spawn } from "node:child_process"; import { onOutputClosed } from "../cli/closed-output.js"; -export type StopReason = "SIGINT" | "SIGTERM" | "key"; +export type StopReason = "SIGHUP" | "SIGINT" | "SIGTERM" | "key"; export interface StopRequests { - // Resolves on the first Ctrl-C, SIGTERM, or `x`. + // Resolves on the first Ctrl-C, SIGTERM, SIGHUP, or `x`. readonly first: Promise; // Resolves on the second, which forces the stop. readonly second: Promise; @@ -25,14 +25,23 @@ export function listenForStopRequests(): StopRequests { }; const onSigint = (): void => request("SIGINT"); const onSigterm = (): void => request("SIGTERM"); + // After the terminal closes or the reader of the output goes away, nobody is left to force the + // stop, and these events can come more than once. So they only start a stop. + const startStop = (reason: StopReason): void => { + if (count === 0) request(reason); + }; + // A closed terminal sends SIGHUP. Node's default exits at once, which leaves the Engine running. + const onSighup = (): void => startStop("SIGHUP"); process.on("SIGINT", onSigint); process.on("SIGTERM", onSigterm); + process.on("SIGHUP", onSighup); // Nobody reads the output any more, as after `orange dev --json | head`. - const stopOnClosedOutput = onOutputClosed(() => request("SIGTERM")); + const stopOnClosedOutput = onOutputClosed(() => startStop("SIGTERM")); return { dispose(): void { process.off("SIGINT", onSigint); process.off("SIGTERM", onSigterm); + process.off("SIGHUP", onSighup); stopOnClosedOutput(); }, first: first.promise, diff --git a/tests/cli/orange-dev-shared-engine.test.ts b/tests/cli/orange-dev-shared-engine.test.ts index a82eade..cba6ace 100644 --- a/tests/cli/orange-dev-shared-engine.test.ts +++ b/tests/cli/orange-dev-shared-engine.test.ts @@ -46,3 +46,20 @@ test("the orange dev that started the shared Engine exits and the other keeps se await project.dispose(); } }); + +// Closing the terminal of orange dev stops it, but the Engine runs in its own process group and +// gets no SIGHUP. A next orange dev would reuse an Engine left running. +test("orange dev stops its Engine when its terminal closes", { timeout: 120_000 }, async () => { + const project = await copyProject(join(repositoryRoot, "examples", "hello-durable-object")); + try { + const dev = await startOrangeDev(project.directory, "wrangler.jsonc", { terminal: true }); + assert.deepEqual(await dev.json("/alice"), { greeting: "Hello, alice!", visits: 1 }); + const pid = await enginePid(project.directory); + + await dev.hangUp(); + assert.equal(processIsAlive(pid), false, "orange dev exited and left the Engine running"); + } finally { + await stopProjectEngines(project.directory); + await project.dispose(); + } +}); diff --git a/tests/harness/orange-dev.ts b/tests/harness/orange-dev.ts index 675de98..69617a9 100644 --- a/tests/harness/orange-dev.ts +++ b/tests/harness/orange-dev.ts @@ -34,6 +34,11 @@ export interface RunningOrangeDev { kill(): Promise; // Resolves with the exit code once orange dev exits on its own or after close(). exited: Promise; + /** + * Closes the terminal of orange dev, as closing its window does, and resolves after orange dev + * exits. It needs `startOrangeDev(..., { terminal: true })`. + */ + hangUp(): Promise; /** Fetches `path` from orange dev, asserts a 200, and returns the parsed JSON body. A failure shows the body and the output. */ json(path: string, init?: RequestInit): Promise; origin: string; @@ -155,16 +160,27 @@ export interface OrangeCli { runtime: string; } +// `script` runs the command on a new terminal, as a terminal window does. +function inTerminal(command: string, args: readonly string[]): [string, string[]] { + if (process.platform === "darwin") return ["script", ["-q", "/dev/null", command, ...args]]; + const line = [command, ...args].map((arg) => `'${arg.replaceAll("'", "'\\''")}'`).join(" "); + return ["script", ["-qfec", `exec ${line}`, "/dev/null"]]; +} + function spawnOrange( args: readonly string[], environment: NodeJS.ProcessEnv = {}, cli?: OrangeCli, + terminal = false, ): ChildProcess { - const [command, prefix] = + const [runtime, prefix] = cli === undefined ? [process.execPath, ["--import", "tsx", cliEntrypoint]] : [cli.runtime, [cli.entrypoint]]; - return spawn(command, [...prefix, ...args], { + const [command, commandArgs] = terminal + ? inTerminal(runtime, [...prefix, ...args]) + : [runtime, [...prefix, ...args]]; + return spawn(command, commandArgs, { cwd: repositoryRoot, env: { ...process.env, ...environment, CI: "1", NO_COLOR: "1" }, stdio: ["ignore", "pipe", "pipe"], @@ -226,6 +242,8 @@ export interface StartOrangeDevOptions { args?: readonly string[]; cli?: OrangeCli; environment?: NodeJS.ProcessEnv; + // Runs orange dev on a terminal, so that hangUp() can close it. + terminal?: boolean; } // More than one config starts a multi-Worker session. The first Worker serves the port. @@ -253,6 +271,7 @@ export async function startOrangeDev( ...(options.actorLifecycleLogs === true ? { RIVET_LOG_LEVEL: "info" } : {}), }, options.cli, + options.terminal, ); const exited = new Promise((resolveExit) => { child.once("exit", (code) => resolveExit(code)); @@ -329,6 +348,35 @@ export async function startOrangeDev( } }, exited, + async hangUp(): Promise { + if (options.terminal !== true) throw new Error("hangUp() needs startOrangeDev(..., { terminal: true })"); + closed = true; + // orange dev and its runtime, under `script`. + const pids = child.pid === undefined ? [] : descendantPids(child.pid); + // The terminal closes, so writes to it fail and orange dev gets SIGHUP. The shell of a + // terminal window also sends SIGHUP to the job, and the kernel sends one more when the + // shell exits. The Rivet Engine runs in its own process group, so it gets none. + child.kill("SIGKILL"); + for (let signal = 0; signal < 2; signal += 1) { + // Two signals sent at once can merge into one. + await delay(200); + for (const pid of pids) { + try { + process.kill(pid, "SIGHUP"); + } catch { + // It already exited. + } + } + } + const deadline = Date.now() + 30_000; + while (pids.some(processIsAlive)) { + if (Date.now() > deadline) { + for (const pid of pids.filter(processIsAlive)) process.kill(pid, "SIGKILL"); + throw new Error(`orange dev did not exit after its terminal closed:\n${output()}`); + } + await delay(25); + } + }, async kill(): Promise { if (closed) return; closed = true;