From 89af4ad77e168d35a287568104dfd54df23236a3 Mon Sep 17 00:00:00 2001 From: RedStar Date: Fri, 13 Nov 2026 17:21:14 +0100 Subject: [PATCH 1/3] feat(plugin-gateway): add dispatch wrappers and destroy hooks addDispatchWrapper runs code around the processing of each dispatch received from Discord, so a tracing span or an AsyncLocalStorage store covers the cache writes and the listeners. addDestroyHook runs at the end of every destroy(), once the shards are disconnected and the dispatches processed, to flush what is left before the process exits. Refs #261 --- .changeset/gateway-dispatch-wrappers.md | 5 + packages/plugin-gateway/README.md | 25 +++ packages/plugin-gateway/src/GatewayClient.ts | 119 ++++++++++- packages/plugin-gateway/tests/hooks.test.ts | 209 +++++++++++++++++++ 4 files changed, 349 insertions(+), 9 deletions(-) create mode 100644 .changeset/gateway-dispatch-wrappers.md create mode 100644 packages/plugin-gateway/tests/hooks.test.ts diff --git a/.changeset/gateway-dispatch-wrappers.md b/.changeset/gateway-dispatch-wrappers.md new file mode 100644 index 00000000..4eea127a --- /dev/null +++ b/.changeset/gateway-dispatch-wrappers.md @@ -0,0 +1,5 @@ +--- +"@wolfstar/plugin-gateway": minor +--- + +`GatewayClient#addDispatchWrapper` runs code around the processing of each dispatch received from Discord, such as a tracing span or an `AsyncLocalStorage` store that the cache writes and listeners run in, and `GatewayClient#addDestroyHook` runs a hook at the end of every `destroy()`, once the shards are disconnected and the dispatches processed, to flush what is left before the process exits. diff --git a/packages/plugin-gateway/README.md b/packages/plugin-gateway/README.md index 9c4b4b48..674192aa 100644 --- a/packages/plugin-gateway/README.md +++ b/packages/plugin-gateway/README.md @@ -246,6 +246,31 @@ transport.receive(async ({ payload, shardId, state }) => { [`@wolfstar/plugin-broker`](https://www.npmjs.com/package/@wolfstar/plugin-broker) wires this up over Redis Streams with `forwardGatewayDispatches` and `replayGatewayDispatches`. +### Wrapping dispatches and shutting down + +`client.addDispatchWrapper(wrapper)` runs code around the processing of each dispatch received from Discord, for what +an event cannot do: a tracing span that the cache writes and listeners become children of, or an `AsyncLocalStorage` +store they can read. The wrapper gets `{ payload, shardId, partition }` and a `run` function, which it must call once +and return or await. `run` rejects when the processing fails, and the client still reports the failure through `error`. +Wrappers nest in the order they were added, the first one outermost. Dispatches replayed with `replayDispatch` are not +wrapped. + +`client.addDestroyHook(hook)` runs a hook at the end of every `client.destroy()`, once the shards are disconnected and +every dispatch is processed, even when disconnecting failed: the place to flush what is left to export before the +process exits. A hook that throws is reported through `error` and does not stop the next ones. + +Both return a function that removes what they added. + +```ts +import { AsyncLocalStorage } from "node:async_hooks"; + +const dispatchContext = new AsyncLocalStorage<{ type: string; shardId: number }>(); +client.addDispatchWrapper(({ payload, shardId }, run) => + dispatchContext.run({ type: payload.t, shardId }, run), +); +client.addDestroyHook(() => metrics.flush()); +``` + ## Listeners Since `GatewayClient` emits on the client itself, gateway events are handled by regular listener diff --git a/packages/plugin-gateway/src/GatewayClient.ts b/packages/plugin-gateway/src/GatewayClient.ts index 22f26ec1..683e3b77 100644 --- a/packages/plugin-gateway/src/GatewayClient.ts +++ b/packages/plugin-gateway/src/GatewayClient.ts @@ -325,6 +325,37 @@ export interface GatewayClientDestroyOptions { resumable?: boolean; } +/** + * The dispatch a {@link DispatchWrapper} wraps. + */ +export interface WrappedDispatch { + /** The payload Discord sent. */ + payload: GatewayDispatchPayload; + /** The gateway shard it came from. */ + shardId: number; + /** The partition it is ordered in (`guild:`, `channel:`...), or `null` for one ordered by shard only. */ + partition: string | null; +} + +/** + * Wraps the processing of each dispatch, for code that has to run around it rather than react to an event: a tracing + * span the cache and listener work becomes the child of, a timer, an `AsyncLocalStorage` store. + * + * @param dispatch The dispatch being processed. + * @param run Processes it: writes it into the cache and emits its events. Call it exactly once and return (or await) + * its promise. It rejects when the processing fails, which the client then reports as for any dispatch. + */ +export type DispatchWrapper = ( + dispatch: WrappedDispatch, + run: () => Promise, +) => Promise; + +/** + * Runs at the end of {@link GatewayClient.destroy}, once the shards are disconnected and every dispatch is processed: + * flushing what is left to export, such as traces, before the process exits. + */ +export type GatewayClientDestroyHook = () => unknown; + /** * A {@link Client} that, on top of serving HTTP interactions, connects to the Discord gateway, writes every dispatch * into its cache, and emits {@link GatewayEventMap} events carrying structures. @@ -508,6 +539,11 @@ export class GatewayClient extends Client { // The `destroy` in progress, which a concurrent call joins instead of changing `#keepSessions` under it. #destroying: Promise | null = null; + // See `addDispatchWrapper`: the first added is the outermost. + #dispatchWrappers: readonly DispatchWrapper[] = []; + + readonly #destroyHooks = new Set(); + public constructor(options: GatewayClientOptions) { super(options); container.gatewayClient = this; @@ -627,18 +663,70 @@ export class GatewayClient extends Client { } private async disconnect(resumable: boolean): Promise { - this.#keepSessions = resumable; try { - // Discord invalidates the session of a connection closed with 1000 or 1001, the default is 1000. - await this.gateway.destroy( - this.#keepSessions ? { code: CloseCodes.Resuming, reason: "Resumable shutdown" } : {}, - ); + this.#keepSessions = resumable; + try { + // Discord invalidates the session of a connection closed with 1000 or 1001, the default is 1000. + await this.gateway.destroy( + this.#keepSessions ? { code: CloseCodes.Resuming, reason: "Resumable shutdown" } : {}, + ); + } finally { + this.#keepSessions = false; + } + + await this.#sessions?.flush(); + await this.idle(); } finally { - this.#keepSessions = false; + // Even when disconnecting failed: the process is likely to exit next, and what is left should still go out. + await this.#runDestroyHooks(); } + } + + /** + * Wraps the processing of every dispatch received from now on, see {@link DispatchWrapper}. + * + * @remarks + * Wrappers nest in the order they were added: the first one added is the outermost. The dispatch timeout and the + * error reporting of the client are outside of every wrapper. + * + * @param wrapper The wrapper. + * @returns A function that removes the wrapper, for the dispatches received after it is called. + */ + public addDispatchWrapper(wrapper: DispatchWrapper): () => void { + this.#dispatchWrappers = [...this.#dispatchWrappers, wrapper]; + return () => { + this.#dispatchWrappers = this.#dispatchWrappers.filter((entry) => entry !== wrapper); + }; + } + + /** + * Runs a hook at the end of every {@link GatewayClient.destroy}, see {@link GatewayClientDestroyHook}. + * + * @remarks + * The hooks run one after the other, in the order they were added, even when disconnecting the shards failed. A hook + * that throws is reported through the `error` event (or logged when nothing listens) and does not stop the others. + * Adding the same function twice adds it once. + * + * @param hook The hook. + * @returns A function that removes the hook. + */ + public addDestroyHook(hook: GatewayClientDestroyHook): () => void { + this.#destroyHooks.add(hook); + return () => { + this.#destroyHooks.delete(hook); + }; + } - await this.#sessions?.flush(); - await this.idle(); + async #runDestroyHooks(): Promise { + // A copy: a hook added while the hooks run waits for the next destroy. + for (const hook of Array.from(this.#destroyHooks)) { + try { + await hook(); + } catch (error) { + if (this.listenerCount("error") > 0) this.emit("error", error); + else this.logger.error("[Gateway] A destroy hook failed:", error); + } + } } /** @@ -1262,7 +1350,7 @@ export class GatewayClient extends Client { timer?.unref?.(); try { - await this.handleDispatch(payload, shardId); + await this.#wrapDispatch({ payload, shardId, partition }); } catch (error) { this.reportError(error, payload.t, shardId); } finally { @@ -1270,6 +1358,19 @@ export class GatewayClient extends Client { } } + #wrapDispatch(dispatch: WrappedDispatch): Promise { + let run = () => this.handleDispatch(dispatch.payload, dispatch.shardId); + // Read once, so a wrapper added or removed meanwhile only changes the next dispatches. + const wrappers = this.#dispatchWrappers; + for (let index = wrappers.length - 1; index >= 0; index--) { + const wrapper = wrappers[index]!; + const inner = run; + run = () => wrapper(dispatch, inner); + } + + return run(); + } + private sessionCallbacks( options: GatewayClientOptions, ): Pick { diff --git a/packages/plugin-gateway/tests/hooks.test.ts b/packages/plugin-gateway/tests/hooks.test.ts new file mode 100644 index 00000000..2c7c03ce --- /dev/null +++ b/packages/plugin-gateway/tests/hooks.test.ts @@ -0,0 +1,209 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { WebSocketShardEvents } from "@discordjs/ws"; +import { createInMemoryCache } from "@wolfstar/plugin-cache"; +import { + GatewayDispatchEvents, + GatewayOpcodes, + type APIUser, + type GatewayDispatchPayload, +} from "discord-api-types/v10"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { GatewayClient, type WrappedDispatch } from "../src/index.js"; + +const user: APIUser = { + id: "266624760782258186", + username: "wolf", + discriminator: "0", + global_name: null, + avatar: null, +}; + +function createClient() { + return new GatewayClient({ + discordPublicKey: "0".repeat(64), + discordToken: "test-token", + clientId: "266624760782258186", + intents: 0, + cache: createInMemoryCache(), + }); +} + +function send(client: GatewayClient, id: string, shardId = 0) { + const payload = { + op: GatewayOpcodes.Dispatch, + s: 1, + t: GatewayDispatchEvents.MessageCreate, + d: { + id, + channel_id: "20", + guild_id: "10", + author: user, + content: id, + mentions: [], + mention_roles: [], + attachments: [], + embeds: [], + timestamp: "2024-01-01T00:00:00.000Z", + edited_timestamp: null, + type: 0, + }, + } as GatewayDispatchPayload; + client.gateway.emit(WebSocketShardEvents.Dispatch, payload, shardId); +} + +function collectErrors(client: GatewayClient) { + const errors: unknown[] = []; + client.on("error", (error) => errors.push(error)); + return errors; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("GatewayClient#addDispatchWrapper", () => { + test("GIVEN wrappers THEN they nest, the first added outermost, around the processing of the dispatch", async () => { + const client = createClient(); + const calls: string[] = []; + const seen: WrappedDispatch[] = []; + client.addDispatchWrapper(async (dispatch, run) => { + seen.push(dispatch); + calls.push("outer:start"); + await run(); + calls.push("outer:end"); + }); + client.addDispatchWrapper(async (_dispatch, run) => { + calls.push("inner:start"); + await run(); + calls.push("inner:end"); + }); + client.on("messageCreate", () => calls.push("messageCreate")); + + send(client, "1", 3); + await client.idle(); + + expect(calls).toEqual([ + "outer:start", + "inner:start", + "messageCreate", + "inner:end", + "outer:end", + ]); + expect(seen).toEqual([ + { + payload: expect.objectContaining({ t: GatewayDispatchEvents.MessageCreate }), + shardId: 3, + partition: "guild:10", + }, + ]); + }); + + test("GIVEN a wrapper that enters an AsyncLocalStorage THEN the listeners of the dispatch run in its store", async () => { + const client = createClient(); + const storage = new AsyncLocalStorage(); + client.addDispatchWrapper((dispatch, run) => storage.run(`shard ${dispatch.shardId}`, run)); + const stores: (string | undefined)[] = []; + client.on("messageCreate", () => stores.push(storage.getStore())); + + send(client, "1", 2); + await client.idle(); + + expect(stores).toEqual(["shard 2"]); + }); + + test("GIVEN a failing dispatch THEN run rejects in the wrapper and the client still reports it", async () => { + const client = createClient(); + const failure = new Error("listener failed"); + vi.spyOn(client.cache!.messages!, "upsert").mockRejectedValue(failure); + const errors = collectErrors(client); + const rejections: unknown[] = []; + client.addDispatchWrapper(async (_dispatch, run) => { + try { + await run(); + } catch (error) { + rejections.push(error); + throw error; + } + }); + + send(client, "1"); + await client.idle(); + + expect(rejections).toHaveLength(1); + expect(errors).toHaveLength(1); + }); + + test("GIVEN a removed wrapper THEN the next dispatches skip it", async () => { + const client = createClient(); + const wrapper = vi.fn((_dispatch: WrappedDispatch, run: () => Promise) => run()); + const remove = client.addDispatchWrapper(wrapper); + + send(client, "1"); + await client.idle(); + remove(); + send(client, "2"); + await client.idle(); + + expect(wrapper).toHaveBeenCalledTimes(1); + }); +}); + +describe("GatewayClient#addDestroyHook", () => { + test("GIVEN hooks THEN they run in order once the shards are disconnected and the dispatches processed", async () => { + const client = createClient(); + const calls: string[] = []; + vi.spyOn(client.gateway, "destroy").mockImplementation(async () => { + calls.push("gateway.destroy"); + }); + vi.spyOn(client, "idle").mockImplementation(async () => { + calls.push("idle"); + }); + client.addDestroyHook(() => calls.push("first")); + client.addDestroyHook(async () => { + calls.push("second"); + }); + + await client.destroy(); + + expect(calls).toEqual(["gateway.destroy", "idle", "first", "second"]); + }); + + test("GIVEN a gateway that fails to disconnect THEN the hooks still run and destroy rejects", async () => { + const client = createClient(); + const failure = new Error("close failed"); + vi.spyOn(client.gateway, "destroy").mockRejectedValue(failure); + const hook = vi.fn(); + client.addDestroyHook(hook); + + await expect(client.destroy()).rejects.toBe(failure); + expect(hook).toHaveBeenCalledTimes(1); + }); + + test("GIVEN a failing hook THEN it is reported and the next hooks still run", async () => { + const client = createClient(); + vi.spyOn(client.gateway, "destroy").mockResolvedValue(); + const errors = collectErrors(client); + const failure = new Error("flush failed"); + const next = vi.fn(); + client.addDestroyHook(() => { + throw failure; + }); + client.addDestroyHook(next); + + await client.destroy(); + + expect(errors).toEqual([failure]); + expect(next).toHaveBeenCalledTimes(1); + }); + + test("GIVEN a removed hook THEN destroy does not run it", async () => { + const client = createClient(); + vi.spyOn(client.gateway, "destroy").mockResolvedValue(); + const hook = vi.fn(); + client.addDestroyHook(hook)(); + + await client.destroy(); + + expect(hook).not.toHaveBeenCalled(); + }); +}); From 2da444fbd62eb5f35e4d283cf7d66c2dd84ca209 Mon Sep 17 00:00:00 2001 From: RedStar Date: Fri, 13 Nov 2026 17:24:01 +0100 Subject: [PATCH 2/3] feat(plugin-sharder): add shard exit hooks and manager destroy hooks A shard stops with process.exit, which skips beforeExit, so whatever a library flushes there is lost. ShardClient#addExitHook runs a hook first, on exit(), restart(), a close from the manager, and when the manager is gone. ShardManager#addDestroyHook runs one at the end of destroy(), once the shards are closed. Refs #261 --- .changeset/sharder-exit-hooks.md | 5 + packages/plugin-sharder/README.md | 13 ++ packages/plugin-sharder/src/ShardClient.ts | 41 ++++- packages/plugin-sharder/src/ShardManager.ts | 42 ++++- packages/plugin-sharder/tests/hooks.test.ts | 163 ++++++++++++++++++++ 5 files changed, 260 insertions(+), 4 deletions(-) create mode 100644 .changeset/sharder-exit-hooks.md create mode 100644 packages/plugin-sharder/tests/hooks.test.ts diff --git a/.changeset/sharder-exit-hooks.md b/.changeset/sharder-exit-hooks.md new file mode 100644 index 00000000..f79c1f47 --- /dev/null +++ b/.changeset/sharder-exit-hooks.md @@ -0,0 +1,5 @@ +--- +"@wolfstar/plugin-sharder": minor +--- + +`ShardClient#addExitHook` runs a hook before the shard stops with `process.exit` (on `exit()`, `restart()`, a close from the manager, or the manager going away), which skips `beforeExit`, so traces or metrics can be flushed first. `ShardManager#addDestroyHook` runs a hook at the end of `destroy()`, once the shards are closed. diff --git a/packages/plugin-sharder/README.md b/packages/plugin-sharder/README.md index 46c03c7d..4d334826 100644 --- a/packages/plugin-sharder/README.md +++ b/packages/plugin-sharder/README.md @@ -197,6 +197,19 @@ first and closes the old one once the new one is ready, for close to no downtime `manager.reshard(layout)` does the same for a whole new layout. When the manager dies, a process shard emits `disconnect` and exits when nothing listens to it; a worker thread stops with it. +A shard stops with `process.exit`, which skips `beforeExit`, so whatever a library flushes there +(traces, metrics) is lost. `shard.addExitHook(hook)` runs a hook first: on `shard.exit()` and +`shard.restart()`, after the close handler when the manager closes the shard, and when the manager +is gone and nothing listens to `disconnect`. The hooks count towards the close timeout. +`manager.addDestroyHook(hook)` runs a hook at the end of `manager.destroy()`, once the shards are +closed, even when closing one failed. In both, a hook that throws is reported through `error` and +the next ones still run, and the call returns a function that removes the hook. + +```ts +shard.addExitHook(() => metrics.flush()); +manager.addDestroyHook(() => metrics.flush()); +``` + ### Gateway shards The manager paces identifies across every process: `shard.identifyThrottler` is `@discordjs/ws`'s diff --git a/packages/plugin-sharder/src/ShardClient.ts b/packages/plugin-sharder/src/ShardClient.ts index 0ab50914..17270797 100644 --- a/packages/plugin-sharder/src/ShardClient.ts +++ b/packages/plugin-sharder/src/ShardClient.ts @@ -54,6 +54,11 @@ export interface ClientTransport { exit(code: number): void; } +/** + * Runs before a shard's process or thread stops, see {@link ShardClient.addExitHook}. + */ +export type ShardExitHook = () => unknown; + /** * The options of a {@link ShardClient}. */ @@ -207,6 +212,7 @@ export class ShardClient extends EventEmitter { #watchdog: NodeJS.Timeout | null = null; #requestHandler: RequestHandler<{ from: number | null }> | null = null; #closeHandler: (() => unknown) | null = null; + readonly #exitHooks = new Set(); #shardHandler: ShardHandler | null = null; public constructor(options: ShardClientOptions = {}) { @@ -227,7 +233,7 @@ export class ShardClient extends EventEmitter { this.#transport.onDisconnect(() => { this.#stopWatchdog(); if (this.listenerCount("disconnect") > 0) this.emit("disconnect"); - else this.#transport.exit(0); + else void this.#runExitHooks().finally(() => this.#transport.exit(0)); }); this.#pingTimeout = context.pingTimeout; @@ -455,6 +461,27 @@ export class ShardClient extends EventEmitter { return this; } + /** + * Runs a hook before the shard stops: on {@link ShardClient.exit}, on {@link ShardClient.restart}, once the close + * handler of a close from the manager resolved, and when the manager is gone and nothing listens to `disconnect`. + * The place to flush what is left to export, such as traces: the shard stops with `process.exit`, which skips + * `beforeExit`. + * + * @remarks + * The hooks run one after the other, in the order they were added, before the shard tells the manager it stops. A + * hook that throws is reported through the `error` event (or logged when nothing listens) and does not stop the + * others, nor the shard. Adding the same function twice adds it once. + * + * @param hook The hook. + * @returns A function that removes the hook. + */ + public addExitHook(hook: ShardExitHook): () => void { + this.#exitHooks.add(hook); + return () => { + this.#exitHooks.delete(hook); + }; + } + /** * Sets how the manager starts and closes the gateway shards of this shard, for `manager.startShard` & co. * @@ -486,10 +513,22 @@ export class ShardClient extends EventEmitter { async #stop(status: ShardStatus, code: number): Promise { this.#stopWatchdog(); + await this.#runExitHooks(); await this.#signal(status).catch(() => undefined); this.#transport.exit(code); } + async #runExitHooks(): Promise { + // A copy: a hook added while the hooks run is not run this time. + for (const hook of Array.from(this.#exitHooks)) { + try { + await hook(); + } catch (error) { + this.#report(error); + } + } + } + async #write(packet: Packet): Promise { await this.#transport.send(await this.#codec.encode(packet, { channelId: this.id })); } diff --git a/packages/plugin-sharder/src/ShardManager.ts b/packages/plugin-sharder/src/ShardManager.ts index 0987f8f3..04500236 100644 --- a/packages/plugin-sharder/src/ShardManager.ts +++ b/packages/plugin-sharder/src/ShardManager.ts @@ -82,6 +82,11 @@ export interface ShardLayoutOptions { /** * The options of a {@link ShardManager}. */ +/** + * Runs at the end of {@link ShardManager.destroy}, see {@link ShardManager.addDestroyHook}. + */ +export type ShardManagerDestroyHook = () => unknown; + export interface ShardManagerOptions extends ShardLayoutOptions { /** * How to spawn the shards: a {@link ChannelStrategy}, or the name of a registered one (`"fork"`, `"cluster"`, @@ -314,6 +319,7 @@ export class ShardManager extends EventEmitter { readonly #readyTimes: number[] = []; #queue: Promise = Promise.resolve(); #initialized = false; + readonly #destroyHooks = new Set(); public constructor(options: ShardManagerOptions = {}) { super(); @@ -622,9 +628,39 @@ export class ShardManager extends EventEmitter { * Closes every shard for good, and releases the strategy. */ public async destroy(): Promise { - await Promise.all(this.channels.map((channel) => channel.close())); - if (this.#initialized) await this.strategy.destroy?.(); - this.#initialized = false; + try { + await Promise.all(this.channels.map((channel) => channel.close())); + if (this.#initialized) await this.strategy.destroy?.(); + this.#initialized = false; + } finally { + // Even when closing failed: the process is likely to exit next, and what is left should still go out. + for (const hook of Array.from(this.#destroyHooks)) { + try { + await hook(); + } catch (error) { + this.reportError(error); + } + } + } + } + + /** + * Runs a hook at the end of every {@link ShardManager.destroy}, once the shards are closed: the place to flush what + * is left to export, such as traces, before the manager's process exits. + * + * @remarks + * The hooks run one after the other, in the order they were added, even when closing the shards failed. A hook that + * throws is reported through the `error` event (or logged when nothing listens) and does not stop the others. Adding + * the same function twice adds it once. + * + * @param hook The hook. + * @returns A function that removes the hook. + */ + public addDestroyHook(hook: ShardManagerDestroyHook): () => void { + this.#destroyHooks.add(hook); + return () => { + this.#destroyHooks.delete(hook); + }; } /** diff --git a/packages/plugin-sharder/tests/hooks.test.ts b/packages/plugin-sharder/tests/hooks.test.ts new file mode 100644 index 00000000..98f5d172 --- /dev/null +++ b/packages/plugin-sharder/tests/hooks.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, test, vi } from "vitest"; +import { ShardClient, type ClientTransport } from "../src/index.js"; +import { createManager, readyScript } from "./helpers.js"; + +describe("ShardClient#addExitHook", () => { + test("GIVEN exit() THEN the hooks run in order before the manager sees the shard stop", async () => { + const order: string[] = []; + const { manager, strategy } = createManager( + (client) => { + client.addExitHook(() => order.push("first")); + client.addExitHook(async () => { + order.push("second"); + }); + void client.ready(); + }, + { shards: 1 }, + ); + manager.on("shardExit", () => order.push("shardExit")); + await manager.spawn(); + + await strategy.clients[0]!.exit(); + + await expect.poll(() => order).toEqual(["first", "second", "shardExit"]); + expect(manager.channels[0]!.stopped).toBe(true); + await manager.destroy(); + }); + + test("GIVEN a close from the manager THEN the hooks run once the close handler resolved", async () => { + const order: string[] = []; + const { manager } = createManager( + (client, spawns) => { + if (spawns > 0) return void client.ready(); + client.setCloseHandler(() => order.push("close")); + client.addExitHook(() => order.push("exit hook")); + void client.ready(); + }, + { shards: 1 }, + ); + await manager.spawn(); + + await manager.restart(0); + + expect(order).toEqual(["close", "exit hook"]); + await manager.destroy(); + }); + + test("GIVEN a failing hook THEN it is reported, the next hooks run and the shard still stops", async () => { + const failure = new Error("flush failed"); + const errors: unknown[] = []; + const next = vi.fn(); + const { manager, strategy } = createManager( + (client) => { + client.on("error", (error) => errors.push(error)); + client.addExitHook(() => { + throw failure; + }); + client.addExitHook(next); + void client.ready(); + }, + { shards: 1 }, + ); + await manager.spawn(); + + await strategy.clients[0]!.exit(); + + expect(errors).toEqual([failure]); + expect(next).toHaveBeenCalledTimes(1); + await expect.poll(() => manager.channels[0]!.stopped).toBe(true); + await manager.destroy(); + }); + + test("GIVEN a removed hook THEN the shard stops without running it", async () => { + const hook = vi.fn(); + const { manager, strategy } = createManager( + (client) => { + client.addExitHook(hook)(); + void client.ready(); + }, + { shards: 1 }, + ); + await manager.spawn(); + + await strategy.clients[0]!.exit(); + + expect(hook).not.toHaveBeenCalled(); + await manager.destroy(); + }); + + test("GIVEN the manager is gone and nothing listens to disconnect THEN the hooks run before the shard exits", async () => { + const { manager, strategy } = createManager(readyScript, { shards: 1 }); + await manager.spawn(); + const { context } = strategy.spawned[0]!; + + const order: string[] = []; + let disconnect: () => void = () => undefined; + const transport: ClientTransport = { + send: async () => undefined, + onMessage: () => undefined, + onDisconnect: (listener) => { + disconnect = listener; + }, + exit: (code) => order.push(`exit ${code}`), + }; + const client = new ShardClient({ context, transport }); + client.addExitHook(async () => { + order.push("hook"); + }); + + disconnect(); + + await expect.poll(() => order).toEqual(["hook", "exit 0"]); + await manager.destroy(); + }); +}); + +describe("ShardManager#addDestroyHook", () => { + test("GIVEN hooks THEN they run in order once the shards are closed", async () => { + const { manager } = createManager(readyScript, { shards: 1 }); + const order: string[] = []; + manager.on("shardExit", () => order.push("shardExit")); + manager.addDestroyHook(() => order.push("first")); + manager.addDestroyHook(async () => { + order.push("second"); + }); + await manager.spawn(); + + await manager.destroy(); + + expect(order).toEqual(["shardExit", "first", "second"]); + }); + + test("GIVEN closing a shard fails THEN the hooks still run and destroy rejects", async () => { + const { manager } = createManager(readyScript, { shards: 1 }); + await manager.spawn(); + const failure = new Error("close failed"); + vi.spyOn(manager.channels[0]!, "close").mockRejectedValue(failure); + const hook = vi.fn(); + manager.addDestroyHook(hook); + + await expect(manager.destroy()).rejects.toBe(failure); + expect(hook).toHaveBeenCalledTimes(1); + }); + + test("GIVEN a failing hook THEN it is reported and the next hooks still run", async () => { + const { manager } = createManager(readyScript, { shards: 1 }); + const errors: unknown[] = []; + manager.on("error", (error) => errors.push(error)); + const failure = new Error("flush failed"); + const next = vi.fn(); + manager.addDestroyHook(() => { + throw failure; + }); + manager.addDestroyHook(next); + const remove = manager.addDestroyHook(vi.fn()); + remove(); + await manager.spawn(); + + await manager.destroy(); + + expect(errors).toEqual([failure]); + expect(next).toHaveBeenCalledTimes(1); + }); +}); From 7ebafc2e690fa6f1284e7dd23841d89a4354eae8 Mon Sep 17 00:00:00 2001 From: RedStar Date: Fri, 13 Nov 2026 17:32:17 +0100 Subject: [PATCH 3/3] fix(plugin-sharder): run exit hooks once and destroy hooks after every close Share the exit hooks in progress across the paths that stop a shard, so the manager going away during exit() waits for the first flush instead of starting another and exiting before it is done. Let every shard close settle before the destroy hooks run, and keep the JSDoc of ShardManagerOptions attached to it. --- packages/plugin-sharder/src/ShardClient.ts | 24 +++++---- packages/plugin-sharder/src/ShardManager.ts | 11 ++-- packages/plugin-sharder/tests/hooks.test.ts | 59 +++++++++++++++++++++ 3 files changed, 81 insertions(+), 13 deletions(-) diff --git a/packages/plugin-sharder/src/ShardClient.ts b/packages/plugin-sharder/src/ShardClient.ts index 17270797..234966fd 100644 --- a/packages/plugin-sharder/src/ShardClient.ts +++ b/packages/plugin-sharder/src/ShardClient.ts @@ -213,6 +213,8 @@ export class ShardClient extends EventEmitter { #requestHandler: RequestHandler<{ from: number | null }> | null = null; #closeHandler: (() => unknown) | null = null; readonly #exitHooks = new Set(); + // The hooks in progress, shared by the paths that stop the shard, so none exits before the first run is done. + #exiting: Promise | null = null; #shardHandler: ShardHandler | null = null; public constructor(options: ShardClientOptions = {}) { @@ -470,7 +472,8 @@ export class ShardClient extends EventEmitter { * @remarks * The hooks run one after the other, in the order they were added, before the shard tells the manager it stops. A * hook that throws is reported through the `error` event (or logged when nothing listens) and does not stop the - * others, nor the shard. Adding the same function twice adds it once. + * others, nor the shard. They run once: a path that stops the shard while they run (the manager going away during + * `exit()`) waits for them. Adding the same function twice adds it once. * * @param hook The hook. * @returns A function that removes the hook. @@ -518,15 +521,18 @@ export class ShardClient extends EventEmitter { this.#transport.exit(code); } - async #runExitHooks(): Promise { - // A copy: a hook added while the hooks run is not run this time. - for (const hook of Array.from(this.#exitHooks)) { - try { - await hook(); - } catch (error) { - this.#report(error); + #runExitHooks(): Promise { + this.#exiting ??= (async () => { + // A copy: a hook added while the hooks run is not run this time. + for (const hook of Array.from(this.#exitHooks)) { + try { + await hook(); + } catch (error) { + this.#report(error); + } } - } + })(); + return this.#exiting; } async #write(packet: Packet): Promise { diff --git a/packages/plugin-sharder/src/ShardManager.ts b/packages/plugin-sharder/src/ShardManager.ts index 04500236..13b469bc 100644 --- a/packages/plugin-sharder/src/ShardManager.ts +++ b/packages/plugin-sharder/src/ShardManager.ts @@ -79,14 +79,14 @@ export interface ShardLayoutOptions { recommended?: RecommendedShardCountOptions; } -/** - * The options of a {@link ShardManager}. - */ /** * Runs at the end of {@link ShardManager.destroy}, see {@link ShardManager.addDestroyHook}. */ export type ShardManagerDestroyHook = () => unknown; +/** + * The options of a {@link ShardManager}. + */ export interface ShardManagerOptions extends ShardLayoutOptions { /** * How to spawn the shards: a {@link ChannelStrategy}, or the name of a registered one (`"fork"`, `"cluster"`, @@ -629,7 +629,10 @@ export class ShardManager extends EventEmitter { */ public async destroy(): Promise { try { - await Promise.all(this.channels.map((channel) => channel.close())); + // Every close settles before the hooks run, so a shard still stopping does not emit after them. + const closed = await Promise.allSettled(this.channels.map((channel) => channel.close())); + const failed = closed.find((result) => result.status === "rejected"); + if (failed) throw failed.reason; if (this.#initialized) await this.strategy.destroy?.(); this.#initialized = false; } finally { diff --git a/packages/plugin-sharder/tests/hooks.test.ts b/packages/plugin-sharder/tests/hooks.test.ts index 98f5d172..668d3812 100644 --- a/packages/plugin-sharder/tests/hooks.test.ts +++ b/packages/plugin-sharder/tests/hooks.test.ts @@ -113,6 +113,46 @@ describe("ShardClient#addExitHook", () => { }); }); +describe("overlapping stops", () => { + test("GIVEN the manager going away during exit() THEN the hooks run once and the shard exits after them", async () => { + const { manager, strategy } = createManager(readyScript, { shards: 1 }); + await manager.spawn(); + const { context } = strategy.spawned[0]!; + + const order: string[] = []; + let disconnect: () => void = () => undefined; + const transport: ClientTransport = { + send: async () => undefined, + onMessage: () => undefined, + onDisconnect: (listener) => { + disconnect = listener; + }, + exit: (code) => order.push(`exit ${code}`), + }; + const client = new ShardClient({ context, transport }); + let release: () => void = () => undefined; + const hook = vi.fn(async () => { + order.push("flush start"); + await new Promise((resolve) => { + release = resolve; + }); + order.push("flush end"); + }); + client.addExitHook(hook); + + const exiting = client.exit(); + disconnect(); + await Promise.resolve(); + expect(order).toEqual(["flush start"]); + release(); + await exiting; + + await expect.poll(() => order).toEqual(["flush start", "flush end", "exit 0", "exit 0"]); + expect(hook).toHaveBeenCalledTimes(1); + await manager.destroy(); + }); +}); + describe("ShardManager#addDestroyHook", () => { test("GIVEN hooks THEN they run in order once the shards are closed", async () => { const { manager } = createManager(readyScript, { shards: 1 }); @@ -141,6 +181,25 @@ describe("ShardManager#addDestroyHook", () => { expect(hook).toHaveBeenCalledTimes(1); }); + test("GIVEN one shard failing to close THEN the hooks wait for the others to close", async () => { + const { manager } = createManager(readyScript, { shards: 2 }); + await manager.spawn(); + const [first, second] = manager.channels; + const failure = new Error("close failed"); + const order: string[] = []; + vi.spyOn(first!, "close").mockRejectedValue(failure); + const close = second!.close.bind(second); + vi.spyOn(second!, "close").mockImplementation(async (...args) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + await close(...args); + order.push("second closed"); + }); + manager.addDestroyHook(() => order.push("hook")); + + await expect(manager.destroy()).rejects.toBe(failure); + expect(order).toEqual(["second closed", "hook"]); + }); + test("GIVEN a failing hook THEN it is reported and the next hooks still run", async () => { const { manager } = createManager(readyScript, { shards: 1 }); const errors: unknown[] = [];