diff --git a/.changeset/sharder-exit-hooks.md b/.changeset/sharder-exit-hooks.md new file mode 100644 index 00000000..f79c1f47 --- /dev/null +++ b/.changeset/sharder-exit-hooks.md @@ -0,0 +1,5 @@ +--- +"@wolfstar/plugin-sharder": minor +--- + +`ShardClient#addExitHook` runs a hook before the shard stops with `process.exit` (on `exit()`, `restart()`, a close from the manager, or the manager going away), which skips `beforeExit`, so traces or metrics can be flushed first. `ShardManager#addDestroyHook` runs a hook at the end of `destroy()`, once the shards are closed. diff --git a/packages/plugin-sharder/README.md b/packages/plugin-sharder/README.md index 46c03c7d..4d334826 100644 --- a/packages/plugin-sharder/README.md +++ b/packages/plugin-sharder/README.md @@ -197,6 +197,19 @@ first and closes the old one once the new one is ready, for close to no downtime `manager.reshard(layout)` does the same for a whole new layout. When the manager dies, a process shard emits `disconnect` and exits when nothing listens to it; a worker thread stops with it. +A shard stops with `process.exit`, which skips `beforeExit`, so whatever a library flushes there +(traces, metrics) is lost. `shard.addExitHook(hook)` runs a hook first: on `shard.exit()` and +`shard.restart()`, after the close handler when the manager closes the shard, and when the manager +is gone and nothing listens to `disconnect`. The hooks count towards the close timeout. +`manager.addDestroyHook(hook)` runs a hook at the end of `manager.destroy()`, once the shards are +closed, even when closing one failed. In both, a hook that throws is reported through `error` and +the next ones still run, and the call returns a function that removes the hook. + +```ts +shard.addExitHook(() => metrics.flush()); +manager.addDestroyHook(() => metrics.flush()); +``` + ### Gateway shards The manager paces identifies across every process: `shard.identifyThrottler` is `@discordjs/ws`'s diff --git a/packages/plugin-sharder/src/ShardClient.ts b/packages/plugin-sharder/src/ShardClient.ts index 0ab50914..234966fd 100644 --- a/packages/plugin-sharder/src/ShardClient.ts +++ b/packages/plugin-sharder/src/ShardClient.ts @@ -54,6 +54,11 @@ export interface ClientTransport { exit(code: number): void; } +/** + * Runs before a shard's process or thread stops, see {@link ShardClient.addExitHook}. + */ +export type ShardExitHook = () => unknown; + /** * The options of a {@link ShardClient}. */ @@ -207,6 +212,9 @@ export class ShardClient extends EventEmitter { #watchdog: NodeJS.Timeout | null = null; #requestHandler: RequestHandler<{ from: number | null }> | null = null; #closeHandler: (() => unknown) | null = null; + readonly #exitHooks = new Set(); + // The hooks in progress, shared by the paths that stop the shard, so none exits before the first run is done. + #exiting: Promise | null = null; #shardHandler: ShardHandler | null = null; public constructor(options: ShardClientOptions = {}) { @@ -227,7 +235,7 @@ export class ShardClient extends EventEmitter { this.#transport.onDisconnect(() => { this.#stopWatchdog(); if (this.listenerCount("disconnect") > 0) this.emit("disconnect"); - else this.#transport.exit(0); + else void this.#runExitHooks().finally(() => this.#transport.exit(0)); }); this.#pingTimeout = context.pingTimeout; @@ -455,6 +463,28 @@ export class ShardClient extends EventEmitter { return this; } + /** + * Runs a hook before the shard stops: on {@link ShardClient.exit}, on {@link ShardClient.restart}, once the close + * handler of a close from the manager resolved, and when the manager is gone and nothing listens to `disconnect`. + * The place to flush what is left to export, such as traces: the shard stops with `process.exit`, which skips + * `beforeExit`. + * + * @remarks + * The hooks run one after the other, in the order they were added, before the shard tells the manager it stops. A + * hook that throws is reported through the `error` event (or logged when nothing listens) and does not stop the + * others, nor the shard. They run once: a path that stops the shard while they run (the manager going away during + * `exit()`) waits for them. Adding the same function twice adds it once. + * + * @param hook The hook. + * @returns A function that removes the hook. + */ + public addExitHook(hook: ShardExitHook): () => void { + this.#exitHooks.add(hook); + return () => { + this.#exitHooks.delete(hook); + }; + } + /** * Sets how the manager starts and closes the gateway shards of this shard, for `manager.startShard` & co. * @@ -486,10 +516,25 @@ export class ShardClient extends EventEmitter { async #stop(status: ShardStatus, code: number): Promise { this.#stopWatchdog(); + await this.#runExitHooks(); await this.#signal(status).catch(() => undefined); this.#transport.exit(code); } + #runExitHooks(): Promise { + this.#exiting ??= (async () => { + // A copy: a hook added while the hooks run is not run this time. + for (const hook of Array.from(this.#exitHooks)) { + try { + await hook(); + } catch (error) { + this.#report(error); + } + } + })(); + return this.#exiting; + } + async #write(packet: Packet): Promise { await this.#transport.send(await this.#codec.encode(packet, { channelId: this.id })); } diff --git a/packages/plugin-sharder/src/ShardManager.ts b/packages/plugin-sharder/src/ShardManager.ts index 0987f8f3..13b469bc 100644 --- a/packages/plugin-sharder/src/ShardManager.ts +++ b/packages/plugin-sharder/src/ShardManager.ts @@ -79,6 +79,11 @@ export interface ShardLayoutOptions { recommended?: RecommendedShardCountOptions; } +/** + * Runs at the end of {@link ShardManager.destroy}, see {@link ShardManager.addDestroyHook}. + */ +export type ShardManagerDestroyHook = () => unknown; + /** * The options of a {@link ShardManager}. */ @@ -314,6 +319,7 @@ export class ShardManager extends EventEmitter { readonly #readyTimes: number[] = []; #queue: Promise = Promise.resolve(); #initialized = false; + readonly #destroyHooks = new Set(); public constructor(options: ShardManagerOptions = {}) { super(); @@ -622,9 +628,42 @@ export class ShardManager extends EventEmitter { * Closes every shard for good, and releases the strategy. */ public async destroy(): Promise { - await Promise.all(this.channels.map((channel) => channel.close())); - if (this.#initialized) await this.strategy.destroy?.(); - this.#initialized = false; + try { + // Every close settles before the hooks run, so a shard still stopping does not emit after them. + const closed = await Promise.allSettled(this.channels.map((channel) => channel.close())); + const failed = closed.find((result) => result.status === "rejected"); + if (failed) throw failed.reason; + if (this.#initialized) await this.strategy.destroy?.(); + this.#initialized = false; + } finally { + // Even when closing failed: the process is likely to exit next, and what is left should still go out. + for (const hook of Array.from(this.#destroyHooks)) { + try { + await hook(); + } catch (error) { + this.reportError(error); + } + } + } + } + + /** + * Runs a hook at the end of every {@link ShardManager.destroy}, once the shards are closed: the place to flush what + * is left to export, such as traces, before the manager's process exits. + * + * @remarks + * The hooks run one after the other, in the order they were added, even when closing the shards failed. A hook that + * throws is reported through the `error` event (or logged when nothing listens) and does not stop the others. Adding + * the same function twice adds it once. + * + * @param hook The hook. + * @returns A function that removes the hook. + */ + public addDestroyHook(hook: ShardManagerDestroyHook): () => void { + this.#destroyHooks.add(hook); + return () => { + this.#destroyHooks.delete(hook); + }; } /** diff --git a/packages/plugin-sharder/tests/hooks.test.ts b/packages/plugin-sharder/tests/hooks.test.ts new file mode 100644 index 00000000..668d3812 --- /dev/null +++ b/packages/plugin-sharder/tests/hooks.test.ts @@ -0,0 +1,222 @@ +import { describe, expect, test, vi } from "vitest"; +import { ShardClient, type ClientTransport } from "../src/index.js"; +import { createManager, readyScript } from "./helpers.js"; + +describe("ShardClient#addExitHook", () => { + test("GIVEN exit() THEN the hooks run in order before the manager sees the shard stop", async () => { + const order: string[] = []; + const { manager, strategy } = createManager( + (client) => { + client.addExitHook(() => order.push("first")); + client.addExitHook(async () => { + order.push("second"); + }); + void client.ready(); + }, + { shards: 1 }, + ); + manager.on("shardExit", () => order.push("shardExit")); + await manager.spawn(); + + await strategy.clients[0]!.exit(); + + await expect.poll(() => order).toEqual(["first", "second", "shardExit"]); + expect(manager.channels[0]!.stopped).toBe(true); + await manager.destroy(); + }); + + test("GIVEN a close from the manager THEN the hooks run once the close handler resolved", async () => { + const order: string[] = []; + const { manager } = createManager( + (client, spawns) => { + if (spawns > 0) return void client.ready(); + client.setCloseHandler(() => order.push("close")); + client.addExitHook(() => order.push("exit hook")); + void client.ready(); + }, + { shards: 1 }, + ); + await manager.spawn(); + + await manager.restart(0); + + expect(order).toEqual(["close", "exit hook"]); + await manager.destroy(); + }); + + test("GIVEN a failing hook THEN it is reported, the next hooks run and the shard still stops", async () => { + const failure = new Error("flush failed"); + const errors: unknown[] = []; + const next = vi.fn(); + const { manager, strategy } = createManager( + (client) => { + client.on("error", (error) => errors.push(error)); + client.addExitHook(() => { + throw failure; + }); + client.addExitHook(next); + void client.ready(); + }, + { shards: 1 }, + ); + await manager.spawn(); + + await strategy.clients[0]!.exit(); + + expect(errors).toEqual([failure]); + expect(next).toHaveBeenCalledTimes(1); + await expect.poll(() => manager.channels[0]!.stopped).toBe(true); + await manager.destroy(); + }); + + test("GIVEN a removed hook THEN the shard stops without running it", async () => { + const hook = vi.fn(); + const { manager, strategy } = createManager( + (client) => { + client.addExitHook(hook)(); + void client.ready(); + }, + { shards: 1 }, + ); + await manager.spawn(); + + await strategy.clients[0]!.exit(); + + expect(hook).not.toHaveBeenCalled(); + await manager.destroy(); + }); + + test("GIVEN the manager is gone and nothing listens to disconnect THEN the hooks run before the shard exits", async () => { + const { manager, strategy } = createManager(readyScript, { shards: 1 }); + await manager.spawn(); + const { context } = strategy.spawned[0]!; + + const order: string[] = []; + let disconnect: () => void = () => undefined; + const transport: ClientTransport = { + send: async () => undefined, + onMessage: () => undefined, + onDisconnect: (listener) => { + disconnect = listener; + }, + exit: (code) => order.push(`exit ${code}`), + }; + const client = new ShardClient({ context, transport }); + client.addExitHook(async () => { + order.push("hook"); + }); + + disconnect(); + + await expect.poll(() => order).toEqual(["hook", "exit 0"]); + await manager.destroy(); + }); +}); + +describe("overlapping stops", () => { + test("GIVEN the manager going away during exit() THEN the hooks run once and the shard exits after them", async () => { + const { manager, strategy } = createManager(readyScript, { shards: 1 }); + await manager.spawn(); + const { context } = strategy.spawned[0]!; + + const order: string[] = []; + let disconnect: () => void = () => undefined; + const transport: ClientTransport = { + send: async () => undefined, + onMessage: () => undefined, + onDisconnect: (listener) => { + disconnect = listener; + }, + exit: (code) => order.push(`exit ${code}`), + }; + const client = new ShardClient({ context, transport }); + let release: () => void = () => undefined; + const hook = vi.fn(async () => { + order.push("flush start"); + await new Promise((resolve) => { + release = resolve; + }); + order.push("flush end"); + }); + client.addExitHook(hook); + + const exiting = client.exit(); + disconnect(); + await Promise.resolve(); + expect(order).toEqual(["flush start"]); + release(); + await exiting; + + await expect.poll(() => order).toEqual(["flush start", "flush end", "exit 0", "exit 0"]); + expect(hook).toHaveBeenCalledTimes(1); + await manager.destroy(); + }); +}); + +describe("ShardManager#addDestroyHook", () => { + test("GIVEN hooks THEN they run in order once the shards are closed", async () => { + const { manager } = createManager(readyScript, { shards: 1 }); + const order: string[] = []; + manager.on("shardExit", () => order.push("shardExit")); + manager.addDestroyHook(() => order.push("first")); + manager.addDestroyHook(async () => { + order.push("second"); + }); + await manager.spawn(); + + await manager.destroy(); + + expect(order).toEqual(["shardExit", "first", "second"]); + }); + + test("GIVEN closing a shard fails THEN the hooks still run and destroy rejects", async () => { + const { manager } = createManager(readyScript, { shards: 1 }); + await manager.spawn(); + const failure = new Error("close failed"); + vi.spyOn(manager.channels[0]!, "close").mockRejectedValue(failure); + const hook = vi.fn(); + manager.addDestroyHook(hook); + + await expect(manager.destroy()).rejects.toBe(failure); + expect(hook).toHaveBeenCalledTimes(1); + }); + + test("GIVEN one shard failing to close THEN the hooks wait for the others to close", async () => { + const { manager } = createManager(readyScript, { shards: 2 }); + await manager.spawn(); + const [first, second] = manager.channels; + const failure = new Error("close failed"); + const order: string[] = []; + vi.spyOn(first!, "close").mockRejectedValue(failure); + const close = second!.close.bind(second); + vi.spyOn(second!, "close").mockImplementation(async (...args) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + await close(...args); + order.push("second closed"); + }); + manager.addDestroyHook(() => order.push("hook")); + + await expect(manager.destroy()).rejects.toBe(failure); + expect(order).toEqual(["second closed", "hook"]); + }); + + test("GIVEN a failing hook THEN it is reported and the next hooks still run", async () => { + const { manager } = createManager(readyScript, { shards: 1 }); + const errors: unknown[] = []; + manager.on("error", (error) => errors.push(error)); + const failure = new Error("flush failed"); + const next = vi.fn(); + manager.addDestroyHook(() => { + throw failure; + }); + manager.addDestroyHook(next); + const remove = manager.addDestroyHook(vi.fn()); + remove(); + await manager.spawn(); + + await manager.destroy(); + + expect(errors).toEqual([failure]); + expect(next).toHaveBeenCalledTimes(1); + }); +});