Skip to content

Commit 67cab6c

Browse files
ericallamericallam
committed
fix(cli,core): stop dev workers spinning at 100% CPU after parent CLI disconnect
Orphaned trigger-dev-run-worker and trigger-dev-index-worker processes were getting stuck in an uncaughtException feedback loop when the parent CLI closed the IPC channel: a periodic IPC send via process.send would throw ERR_IPC_CHANNEL_CLOSED, which re-entered the same handler that itself called process.send. The loop was amplified by source-map-support's prepareStackTrace running every iteration. - @trigger.dev/core: ZodIpcConnection drops packets when process.connected is false and swallows synchronous send errors, so closed-channel sends no longer throw out of the IPC layer. - trigger.dev (cli-v3): dev-run-worker and dev-index-worker now exit cleanly via process.on("disconnect") instead of being re-parented to init. - trigger.dev (cli-v3): all four worker entry points wrap their uncaughtException process.send calls in safeSend, which checks process.connected and swallows synchronous throws so a closed channel can never re-enter the handler.
1 parent 19c1675 commit 67cab6c

6 files changed

Lines changed: 160 additions & 92 deletions

File tree

‎.changeset/dev-worker-disconnect-loop.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@trigger.dev/core": patch
3+
"trigger.dev": patch
4+
---
5+
6+
Fix dev workers spinning at 100% CPU after the parent CLI disconnects. Orphaned `trigger-dev-run-worker` (and indexer) processes were caught in an `uncaughtException` feedback loop: a periodic IPC send via `process.send` would throw `ERR_IPC_CHANNEL_CLOSED` once the parent closed the channel, which re-entered the same handler that itself called `process.send`, scheduled via `setImmediate` and amplified by source-map-support's `prepareStackTrace`. Fixed by (1) silently dropping packets in `ZodIpcConnection` when the channel is disconnected, (2) adding a `process.on("disconnect", ...)` handler in dev workers so they exit cleanly when the CLI closes the IPC channel, and (3) wrapping all `uncaughtException`-path `process.send` calls in a `safeSend` guard that checks `process.connected` and swallows synchronous throws.

‎packages/cli-v3/src/entryPoints/dev-index-worker.ts‎

Lines changed: 36 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -27,30 +27,45 @@ sourceMapSupport.install({
2727
hookRequire: false,
2828
});
2929

30+
// If the parent CLI closes the IPC channel, exit cleanly instead of being
31+
// re-parented to init and busy-looping on `process.send` against a dead channel.
32+
process.on("disconnect", () => {
33+
process.exit(0);
34+
});
35+
36+
function safeSend(message: unknown) {
37+
if (!process.connected || !process.send) {
38+
return;
39+
}
40+
try {
41+
process.send(message);
42+
} catch {
43+
// swallow: a throw here would re-enter this handler and busy-loop the worker
44+
}
45+
}
46+
3047
process.on("uncaughtException", function (error, origin) {
3148
if (error instanceof Error) {
32-
process.send &&
33-
process.send({
34-
type: "UNCAUGHT_EXCEPTION",
35-
payload: {
36-
error: { name: error.name, message: error.message, stack: error.stack },
37-
origin,
38-
},
39-
version: "v1",
40-
});
49+
safeSend({
50+
type: "UNCAUGHT_EXCEPTION",
51+
payload: {
52+
error: { name: error.name, message: error.message, stack: error.stack },
53+
origin,
54+
},
55+
version: "v1",
56+
});
4157
} else {
42-
process.send &&
43-
process.send({
44-
type: "UNCAUGHT_EXCEPTION",
45-
payload: {
46-
error: {
47-
name: "Error",
48-
message: typeof error === "string" ? error : JSON.stringify(error),
49-
},
50-
origin,
58+
safeSend({
59+
type: "UNCAUGHT_EXCEPTION",
60+
payload: {
61+
error: {
62+
name: "Error",
63+
message: typeof error === "string" ? error : JSON.stringify(error),
5164
},
52-
version: "v1",
53-
});
65+
origin,
66+
},
67+
version: "v1",
68+
});
5469
}
5570
});
5671

@@ -183,7 +198,7 @@ await sendMessageInCatalog(
183198
importErrors,
184199
},
185200
async (msg) => {
186-
process.send?.(msg);
201+
safeSend(msg);
187202
}
188203
).catch((err) => {
189204
if (err instanceof ZodSchemaParsedError) {

‎packages/cli-v3/src/entryPoints/dev-run-worker.ts‎

Lines changed: 40 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -77,37 +77,53 @@ sourceMapSupport.install({
7777
hookRequire: false,
7878
});
7979

80+
// If the parent CLI closes the IPC channel (process restart, crash, lost
81+
// handle), exit cleanly instead of being re-parented to init and busy-looping
82+
// on `process.send` that throws against a dead channel.
83+
process.on("disconnect", () => {
84+
process.exit(0);
85+
});
86+
87+
function safeSend(message: unknown) {
88+
if (!process.connected || !process.send) {
89+
return;
90+
}
91+
try {
92+
process.send(message);
93+
} catch {
94+
// swallow: a throw here would re-enter this handler and busy-loop the worker
95+
}
96+
}
97+
8098
process.on("uncaughtException", function (error, origin) {
8199
logError("Uncaught exception", { error, origin });
82100
if (error instanceof Error) {
83-
process.send &&
84-
process.send({
85-
type: "EVENT",
86-
message: {
87-
type: "UNCAUGHT_EXCEPTION",
88-
payload: {
89-
error: { name: error.name, message: error.message, stack: error.stack },
90-
origin,
91-
},
92-
version: "v1",
101+
safeSend({
102+
type: "EVENT",
103+
message: {
104+
type: "UNCAUGHT_EXCEPTION",
105+
payload: {
106+
error: { name: error.name, message: error.message, stack: error.stack },
107+
origin,
93108
},
94-
});
109+
version: "v1",
110+
},
111+
});
95112
} else {
96-
process.send &&
97-
process.send({
98-
type: "EVENT",
99-
message: {
100-
type: "UNCAUGHT_EXCEPTION",
101-
payload: {
102-
error: {
103-
name: "Error",
104-
message: typeof error === "string" ? error : JSON.stringify(error),
105-
},
106-
origin,
113+
safeSend({
114+
type: "EVENT",
115+
message: {
116+
type: "UNCAUGHT_EXCEPTION",
117+
payload: {
118+
error: {
119+
name: "Error",
120+
message: typeof error === "string" ? error : JSON.stringify(error),
107121
},
108-
version: "v1",
122+
origin,
109123
},
110-
});
124+
version: "v1",
125+
},
126+
});
111127
}
112128
});
113129

‎packages/cli-v3/src/entryPoints/managed-index-worker.ts‎

Lines changed: 31 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -27,30 +27,39 @@ sourceMapSupport.install({
2727
hookRequire: false,
2828
});
2929

30+
function safeSend(message: unknown) {
31+
if (!process.connected || !process.send) {
32+
return;
33+
}
34+
try {
35+
process.send(message);
36+
} catch {
37+
// swallow: a throw here would re-enter this handler and busy-loop the worker
38+
}
39+
}
40+
3041
process.on("uncaughtException", function (error, origin) {
3142
if (error instanceof Error) {
32-
process.send &&
33-
process.send({
34-
type: "UNCAUGHT_EXCEPTION",
35-
payload: {
36-
error: { name: error.name, message: error.message, stack: error.stack },
37-
origin,
38-
},
39-
version: "v1",
40-
});
43+
safeSend({
44+
type: "UNCAUGHT_EXCEPTION",
45+
payload: {
46+
error: { name: error.name, message: error.message, stack: error.stack },
47+
origin,
48+
},
49+
version: "v1",
50+
});
4151
} else {
42-
process.send &&
43-
process.send({
44-
type: "UNCAUGHT_EXCEPTION",
45-
payload: {
46-
error: {
47-
name: "Error",
48-
message: typeof error === "string" ? error : JSON.stringify(error),
49-
},
50-
origin,
52+
safeSend({
53+
type: "UNCAUGHT_EXCEPTION",
54+
payload: {
55+
error: {
56+
name: "Error",
57+
message: typeof error === "string" ? error : JSON.stringify(error),
5158
},
52-
version: "v1",
53-
});
59+
origin,
60+
},
61+
version: "v1",
62+
});
5463
}
5564
});
5665

@@ -191,7 +200,7 @@ await sendMessageInCatalog(
191200
importErrors,
192201
},
193202
async (msg) => {
194-
process.send?.(msg);
203+
safeSend(msg);
195204
}
196205
).catch((err) => {
197206
if (err instanceof ZodSchemaParsedError) {
@@ -200,7 +209,7 @@ await sendMessageInCatalog(
200209
"TASKS_FAILED_TO_PARSE",
201210
{ zodIssues: err.error.issues, tasks },
202211
async (msg) => {
203-
process.send?.(msg);
212+
safeSend(msg);
204213
}
205214
);
206215
} else {

‎packages/cli-v3/src/entryPoints/managed-run-worker.ts‎

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -77,37 +77,46 @@ sourceMapSupport.install({
7777
hookRequire: false,
7878
});
7979

80+
function safeSend(message: unknown) {
81+
if (!process.connected || !process.send) {
82+
return;
83+
}
84+
try {
85+
process.send(message);
86+
} catch {
87+
// swallow: a throw here would re-enter this handler and busy-loop the worker
88+
}
89+
}
90+
8091
process.on("uncaughtException", function (error, origin) {
8192
console.error("Uncaught exception", { error, origin });
8293
if (error instanceof Error) {
83-
process.send &&
84-
process.send({
85-
type: "EVENT",
86-
message: {
87-
type: "UNCAUGHT_EXCEPTION",
88-
payload: {
89-
error: { name: error.name, message: error.message, stack: error.stack },
90-
origin,
91-
},
92-
version: "v1",
94+
safeSend({
95+
type: "EVENT",
96+
message: {
97+
type: "UNCAUGHT_EXCEPTION",
98+
payload: {
99+
error: { name: error.name, message: error.message, stack: error.stack },
100+
origin,
93101
},
94-
});
102+
version: "v1",
103+
},
104+
});
95105
} else {
96-
process.send &&
97-
process.send({
98-
type: "EVENT",
99-
message: {
100-
type: "UNCAUGHT_EXCEPTION",
101-
payload: {
102-
error: {
103-
name: "Error",
104-
message: typeof error === "string" ? error : JSON.stringify(error),
105-
},
106-
origin,
106+
safeSend({
107+
type: "EVENT",
108+
message: {
109+
type: "UNCAUGHT_EXCEPTION",
110+
payload: {
111+
error: {
112+
name: "Error",
113+
message: typeof error === "string" ? error : JSON.stringify(error),
107114
},
108-
version: "v1",
115+
origin,
109116
},
110-
});
117+
version: "v1",
118+
},
119+
});
111120
}
112121
});
113122

‎packages/core/src/v3/zodIpc.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ interface ZodIpcConnectionOptions<
143143
process: {
144144
send?: (message: any) => any;
145145
on?: (event: "message", listener: (message: any) => void) => void;
146+
connected?: boolean;
146147
};
147148
handlers?: ZodIpcMessageHandlers<TListenCatalog, TEmitCatalog>;
148149
}
@@ -257,7 +258,19 @@ export class ZodIpcConnection<
257258
}
258259

259260
async #sendPacket(packet: Packet) {
260-
await this.opts.process.send?.(packet);
261+
// When the IPC channel is closed (e.g. parent process exited), there is no
262+
// recipient — drop the packet rather than letting `process.send` throw
263+
// ERR_IPC_CHANNEL_CLOSED, which would otherwise propagate as an
264+
// uncaughtException and re-enter any handler that itself calls `process.send`.
265+
if (this.opts.process.connected === false) {
266+
return;
267+
}
268+
269+
try {
270+
await this.opts.process.send?.(packet);
271+
} catch {
272+
// swallow: channel raced from open to closed between the check and the send
273+
}
261274
}
262275

263276
async send<K extends GetSocketMessagesWithoutCallback<TEmitCatalog>>(

0 commit comments

Comments
 (0)