Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/runtime-node/src/process-protocol.ts
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,7 @@ export interface RuntimeWarningMessage {
}

export interface RuntimeStoppedMessage {
reason: "parent" | "SIGINT" | "SIGTERM";
reason: "parent" | "SIGHUP" | "SIGINT" | "SIGTERM";
type: "stopped";
}

Expand Down
23 changes: 16 additions & 7 deletions packages/runtime-node/src/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1428,13 +1428,19 @@ export async function runRuntimeProcess(): Promise<void> {
send({ ...errorDetails(error), type: "background-error" });
});

async function stop(reason: RuntimeStoppedMessage["reason"]): Promise<void> {
if (runtime !== undefined) {
await runtime.close(reason);
}
// `orange dev` exits after this process, so the Engine must exit first.
await stopRivetEngineAfterLastDevSession();
resolveCompletion?.();
// A closed terminal sends SIGHUP here while `orange dev` also sends shutdown. A second stop
// must wait for the Engine too, or this process exits before it.
let stopping: Promise<void> | undefined;
function stop(reason: RuntimeStoppedMessage["reason"]): Promise<void> {
stopping ??= (async () => {
if (runtime !== undefined) {
await runtime.close(reason);
}
// `orange dev` exits after this process, so the Engine must exit first.
await stopRivetEngineAfterLastDevSession();
resolveCompletion?.();
})();
return stopping;
}

process.on("message", (rawMessage: unknown) => {
Expand Down Expand Up @@ -1465,6 +1471,9 @@ export async function runRuntimeProcess(): Promise<void> {
});
process.once("SIGINT", () => void stop("SIGINT"));
process.once("SIGTERM", () => void stop("SIGTERM"));
// A closed terminal sends SIGHUP to this process too, and can send it more than once. Node's
// default exits without stopping the Engine.
process.on("SIGHUP", () => void stop("SIGHUP"));
// The IPC channel closes when the parent dies, so the runtime stops with it.
process.once("disconnect", () => void stop("parent"));
// Runs on every exit, also one during startup before a registry exists.
Expand Down
5 changes: 3 additions & 2 deletions src/cli/closed-output.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// A reader such as `head` or `jq` can close the pipe before the output ends. The rest of the output
// then has nowhere to go: Orange drops it, and a command that runs until it is stopped, such as
// `orange dev`, stops as for Ctrl-C. A short command finishes and exits with its usual code.
// A closed terminal is the same case: a write to it fails with EIO instead of EPIPE.

const closedStreams = new WeakSet<NodeJS.WriteStream>();
const listeners = new Set<() => void>();
Expand All @@ -12,10 +13,10 @@ function markClosed(): void {
for (const listener of listeners) listener();
}

/** Drops the output of `stream` after its reader closes the pipe, instead of crashing on EPIPE. */
/** Drops the output of `stream` after its reader closes the pipe or terminal, instead of crashing on EPIPE or EIO. */
export function dropOutputAfterReaderCloses(stream: NodeJS.WriteStream): void {
stream.on("error", (error: NodeJS.ErrnoException) => {
if (error.code !== "EPIPE") throw error;
if (error.code !== "EPIPE" && error.code !== "EIO") throw error;
if (closedStreams.has(stream)) return;
closedStreams.add(stream);
stream.write = (...args: unknown[]): boolean => {
Expand Down
15 changes: 12 additions & 3 deletions src/commands/dev-session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@ import { spawn } from "node:child_process";

import { onOutputClosed } from "../cli/closed-output.js";

export type StopReason = "SIGINT" | "SIGTERM" | "key";
export type StopReason = "SIGHUP" | "SIGINT" | "SIGTERM" | "key";

export interface StopRequests {
// Resolves on the first Ctrl-C, SIGTERM, or `x`.
// Resolves on the first Ctrl-C, SIGTERM, SIGHUP, or `x`.
readonly first: Promise<StopReason>;
// Resolves on the second, which forces the stop.
readonly second: Promise<StopReason>;
Expand All @@ -25,14 +25,23 @@ export function listenForStopRequests(): StopRequests {
};
const onSigint = (): void => request("SIGINT");
const onSigterm = (): void => request("SIGTERM");
// After the terminal closes or the reader of the output goes away, nobody is left to force the
// stop, and these events can come more than once. So they only start a stop.
const startStop = (reason: StopReason): void => {
if (count === 0) request(reason);
};
// A closed terminal sends SIGHUP. Node's default exits at once, which leaves the Engine running.
const onSighup = (): void => startStop("SIGHUP");
process.on("SIGINT", onSigint);
process.on("SIGTERM", onSigterm);
process.on("SIGHUP", onSighup);
// Nobody reads the output any more, as after `orange dev --json | head`.
const stopOnClosedOutput = onOutputClosed(() => request("SIGTERM"));
const stopOnClosedOutput = onOutputClosed(() => startStop("SIGTERM"));
return {
dispose(): void {
process.off("SIGINT", onSigint);
process.off("SIGTERM", onSigterm);
process.off("SIGHUP", onSighup);
stopOnClosedOutput();
},
first: first.promise,
Expand Down
17 changes: 17 additions & 0 deletions tests/cli/orange-dev-shared-engine.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,20 @@ test("the orange dev that started the shared Engine exits and the other keeps se
await project.dispose();
}
});

// Closing the terminal of orange dev stops it, but the Engine runs in its own process group and
// gets no SIGHUP. A next orange dev would reuse an Engine left running.
test("orange dev stops its Engine when its terminal closes", { timeout: 120_000 }, async () => {
const project = await copyProject(join(repositoryRoot, "examples", "hello-durable-object"));
try {
const dev = await startOrangeDev(project.directory, "wrangler.jsonc", { terminal: true });
assert.deepEqual(await dev.json("/alice"), { greeting: "Hello, alice!", visits: 1 });
const pid = await enginePid(project.directory);

await dev.hangUp();
assert.equal(processIsAlive(pid), false, "orange dev exited and left the Engine running");
} finally {
await stopProjectEngines(project.directory);
await project.dispose();
}
});
52 changes: 50 additions & 2 deletions tests/harness/orange-dev.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ export interface RunningOrangeDev {
kill(): Promise<void>;
// Resolves with the exit code once orange dev exits on its own or after close().
exited: Promise<number | null>;
/**
* Closes the terminal of orange dev, as closing its window does, and resolves after orange dev
* exits. It needs `startOrangeDev(..., { terminal: true })`.
*/
hangUp(): Promise<void>;
/** Fetches `path` from orange dev, asserts a 200, and returns the parsed JSON body. A failure shows the body and the output. */
json(path: string, init?: RequestInit): Promise<unknown>;
origin: string;
Expand Down Expand Up @@ -155,16 +160,27 @@ export interface OrangeCli {
runtime: string;
}

// `script` runs the command on a new terminal, as a terminal window does.
function inTerminal(command: string, args: readonly string[]): [string, string[]] {
if (process.platform === "darwin") return ["script", ["-q", "/dev/null", command, ...args]];
const line = [command, ...args].map((arg) => `'${arg.replaceAll("'", "'\\''")}'`).join(" ");
return ["script", ["-qfec", `exec ${line}`, "/dev/null"]];
}

function spawnOrange(
args: readonly string[],
environment: NodeJS.ProcessEnv = {},
cli?: OrangeCli,
terminal = false,
): ChildProcess {
const [command, prefix] =
const [runtime, prefix] =
cli === undefined
? [process.execPath, ["--import", "tsx", cliEntrypoint]]
: [cli.runtime, [cli.entrypoint]];
return spawn(command, [...prefix, ...args], {
const [command, commandArgs] = terminal
? inTerminal(runtime, [...prefix, ...args])
: [runtime, [...prefix, ...args]];
return spawn(command, commandArgs, {
cwd: repositoryRoot,
env: { ...process.env, ...environment, CI: "1", NO_COLOR: "1" },
stdio: ["ignore", "pipe", "pipe"],
Expand Down Expand Up @@ -226,6 +242,8 @@ export interface StartOrangeDevOptions {
args?: readonly string[];
cli?: OrangeCli;
environment?: NodeJS.ProcessEnv;
// Runs orange dev on a terminal, so that hangUp() can close it.
terminal?: boolean;
}

// More than one config starts a multi-Worker session. The first Worker serves the port.
Expand Down Expand Up @@ -253,6 +271,7 @@ export async function startOrangeDev(
...(options.actorLifecycleLogs === true ? { RIVET_LOG_LEVEL: "info" } : {}),
},
options.cli,
options.terminal,
);
const exited = new Promise<number | null>((resolveExit) => {
child.once("exit", (code) => resolveExit(code));
Expand Down Expand Up @@ -329,6 +348,35 @@ export async function startOrangeDev(
}
},
exited,
async hangUp(): Promise<void> {
if (options.terminal !== true) throw new Error("hangUp() needs startOrangeDev(..., { terminal: true })");
closed = true;
// orange dev and its runtime, under `script`.
const pids = child.pid === undefined ? [] : descendantPids(child.pid);
// The terminal closes, so writes to it fail and orange dev gets SIGHUP. The shell of a
// terminal window also sends SIGHUP to the job, and the kernel sends one more when the
// shell exits. The Rivet Engine runs in its own process group, so it gets none.
child.kill("SIGKILL");
for (let signal = 0; signal < 2; signal += 1) {
// Two signals sent at once can merge into one.
await delay(200);
for (const pid of pids) {
try {
process.kill(pid, "SIGHUP");
} catch {
// It already exited.
}
}
}
const deadline = Date.now() + 30_000;
while (pids.some(processIsAlive)) {
if (Date.now() > deadline) {
for (const pid of pids.filter(processIsAlive)) process.kill(pid, "SIGKILL");
throw new Error(`orange dev did not exit after its terminal closed:\n${output()}`);
}
await delay(25);
}
},
async kill(): Promise<void> {
if (closed) return;
closed = true;
Expand Down
Loading