Skip to content

Commit 17efdcf

Browse files
authored
Merge pull request #81 from AlexDeveloperUwU/captcha-worker-cancellation
fix(captcha): cancel timed-out worker races
2 parents 39f3444 + 4079a45 commit 17efdcf

4 files changed

Lines changed: 156 additions & 22 deletions

File tree

‎src/proxy/captcha-pool.ts‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -259,15 +259,21 @@ export class CaptchaTokenPool {
259259
// grind through its retry budget (~30-45s) — cap the client-facing wait
260260
// and let the background waves finish the job instead.
261261
const raceDeadlineMs = Number(process.env.CAPTCHA_SOLVE_RACE_DEADLINE_MS || 35_000);
262-
param = await Promise.race([
263-
this.solveRaced(cfg),
264-
new Promise<never>((_, rej) =>
265-
setTimeout(
266-
() => rej(new Error(`captcha take deadline (${raceDeadlineMs}ms)`)),
267-
Math.max(1_000, raceDeadlineMs),
268-
),
269-
),
270-
]);
262+
const solveAbort = new AbortController();
263+
let deadlineTimer: ReturnType<typeof setTimeout> | undefined;
264+
try {
265+
param = await Promise.race([
266+
this.solveRaced(cfg, solveAbort.signal),
267+
new Promise<never>((_, rej) => {
268+
deadlineTimer = setTimeout(() => {
269+
solveAbort.abort(new Error(`captcha take deadline (${raceDeadlineMs}ms)`));
270+
rej(new Error(`captcha take deadline (${raceDeadlineMs}ms)`));
271+
}, Math.max(1_000, raceDeadlineMs));
272+
}),
273+
]);
274+
} finally {
275+
if (deadlineTimer !== undefined) clearTimeout(deadlineTimer);
276+
}
271277
} catch (err) {
272278
// Mints fail in clusters (pe-stall storms, F008 velocity). Background
273279
// refill waves keep retrying — give them a short window to land a
@@ -578,15 +584,18 @@ export class CaptchaTokenPool {
578584
}
579585
}
580586

581-
private async solveFresh(cfg: CaptchaConfig): Promise<string> {
587+
private async solveFresh(cfg: CaptchaConfig, signal?: AbortSignal): Promise<string> {
582588
this.activeSolves += 1;
583589
this.lastSolveAt = Date.now();
584590
try {
585591
let lastErr: string | null = null;
586592
let sawIpBlock = false;
587593
for (let attempt = 1; attempt <= this.opts.solveRetries; attempt += 1) {
588594
try {
589-
const param = await runCaptchaSolve(cfg.sceneId, cfg.region, cfg.prefix);
595+
if (signal?.aborted) {
596+
throw signal.reason instanceof Error ? signal.reason : new Error("captcha solve aborted");
597+
}
598+
const param = await runCaptchaSolve(cfg.sceneId, cfg.region, cfg.prefix, signal);
590599
if (!param) {
591600
lastErr = "solver returned empty";
592601
continue;
@@ -599,6 +608,7 @@ export class CaptchaTokenPool {
599608
return param;
600609
} catch (err) {
601610
lastErr = err instanceof Error ? err.message : String(err);
611+
if (signal?.aborted) throw err;
602612
if (isCaptchaDuplicateError(lastErr)) {
603613
lastErr = `duplicate certifyId (F008)`;
604614
} else if (isCaptchaIpBlockError(lastErr)) {
@@ -633,16 +643,16 @@ export class CaptchaTokenPool {
633643
* sequential retry chain. Extra successes are banked into the pool instead
634644
* of being wasted; the race only rejects when every racer fails.
635645
*/
636-
private solveRaced(cfg: CaptchaConfig): Promise<string> {
646+
private solveRaced(cfg: CaptchaConfig, signal?: AbortSignal): Promise<string> {
637647
const racers = Math.max(1, Math.min(this.opts.emptyTakeRace, this.opts.solveConcurrency));
638-
if (racers <= 1) return this.solveFresh(cfg);
648+
if (racers <= 1) return this.solveFresh(cfg, signal);
639649

640650
return new Promise<string>((resolve, reject) => {
641651
let pending = racers;
642652
let served = false;
643653
let lastErr: unknown = null;
644654
for (let i = 0; i < racers; i += 1) {
645-
this.solveFresh(cfg).then(
655+
this.solveFresh(cfg, signal).then(
646656
(param) => {
647657
if (served) {
648658
// Lost the race but minted a valid token — bank it.

‎src/proxy/captcha-solver.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@ import { solveViaWorkerOrInProcess, stopCaptchaWorkers } from "./captcha-worker-
1010

1111
const BACKEND = process.env.ZCODE_CAPTCHA_BACKEND?.trim().toLowerCase() || "happy";
1212

13-
export async function runCaptchaSolve(scene: string, region: string, prefix: string): Promise<string> {
13+
export async function runCaptchaSolve(scene: string, region: string, prefix: string, signal?: AbortSignal): Promise<string> {
1414
if (BACKEND !== "happy") {
1515
throw new Error(`captcha backend "${BACKEND}" is not available; use ZCODE_CAPTCHA_BACKEND=happy`);
1616
}
17-
return solveViaWorkerOrInProcess({ scene, region, prefix });
17+
return solveViaWorkerOrInProcess({ scene, region, prefix, signal });
1818
}
1919

2020
/** Worker-per-solve needs no concurrency plumbing — kept for the pool API. */

‎src/proxy/captcha-worker-dispatch.test.ts‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ describe("warm worker pool", () => {
134134
const warmDir = fs.mkdtempSync(path.join(os.tmpdir(), "cap-warm-"));
135135
const markerPath = path.join(warmDir, "spawns.txt");
136136
const warmFixture = path.join(warmDir, "warm-worker.mjs");
137+
const hangingFixture = path.join(warmDir, "hanging-worker.mjs");
137138
fs.writeFileSync(
138139
warmFixture,
139140
[
@@ -147,6 +148,32 @@ describe("warm worker pool", () => {
147148
].join("\n"),
148149
"utf8",
149150
);
151+
fs.writeFileSync(
152+
hangingFixture,
153+
[
154+
'import { parentPort } from "node:worker_threads";',
155+
'import { appendFileSync } from "node:fs";',
156+
`appendFileSync(${JSON.stringify(markerPath)}, "spawn\\n");`,
157+
"if (!parentPort) throw new Error('fixture requires parent port');",
158+
"parentPort.on('message', () => {});",
159+
].join("\n"),
160+
"utf8",
161+
);
162+
const failThenHangFixture = path.join(warmDir, "fail-then-hang-worker.mjs");
163+
fs.writeFileSync(
164+
failThenHangFixture,
165+
[
166+
'import { parentPort } from "node:worker_threads";',
167+
'import { appendFileSync } from "node:fs";',
168+
`appendFileSync(${JSON.stringify(markerPath)}, "spawn\\n");`,
169+
"if (!parentPort) throw new Error('fixture requires parent port');",
170+
"let n = 0;",
171+
"parentPort.on('message', (m) => {",
172+
" if (n++ === 0) parentPort.postMessage({ id: m.id, ok: false, error: 'boom' });",
173+
"});",
174+
].join("\n"),
175+
"utf8",
176+
);
150177
const spawnCount = (): number =>
151178
fs.existsSync(markerPath)
152179
? fs.readFileSync(markerPath, "utf8").split("\n").filter((l) => l === "spawn").length
@@ -182,6 +209,79 @@ describe("warm worker pool", () => {
182209
// The pool was emptied -> the next solve had to spawn a fresh worker.
183210
expect(spawnCount()).toBe(2);
184211
});
212+
213+
test("aborting a worker-per-solve solve mid-flight terminates the hung worker", async () => {
214+
process.env.ZCODE_CAPTCHA_WORKER = hangingFixture;
215+
delete process.env.ZCODE_CAPTCHA_WARM_WORKERS;
216+
mock.module("./captcha-worker-asset.js", () => ({ default: null }));
217+
__resetCaptchaWorkerDispatchForTest();
218+
219+
const controller = new AbortController();
220+
const pending = solveViaWorkerOrInProcess({
221+
scene: "abort",
222+
region: "r",
223+
prefix: "p",
224+
signal: controller.signal,
225+
});
226+
await new Promise((r) => setTimeout(r, 100));
227+
const t0 = Date.now();
228+
controller.abort(new Error("request deadline"));
229+
await expect(pending).rejects.toThrow("request deadline");
230+
expect(Date.now() - t0).toBeLessThan(5_000);
231+
});
232+
233+
test("aborting a warm solve mid-flight drops the handle; the next solve respawns", async () => {
234+
process.env.ZCODE_CAPTCHA_WORKER = hangingFixture;
235+
process.env.ZCODE_CAPTCHA_WARM_WORKERS = "1";
236+
mock.module("./captcha-worker-asset.js", () => ({ default: null }));
237+
__resetCaptchaWorkerDispatchForTest();
238+
const baseline = spawnCount();
239+
240+
const abortAfterDispatch = async (scene: string): Promise<void> => {
241+
const controller = new AbortController();
242+
const pending = solveViaWorkerOrInProcess({ scene, region: "r", prefix: "p", signal: controller.signal });
243+
await new Promise((r) => setTimeout(r, 100));
244+
const t0 = Date.now();
245+
controller.abort(new Error("request deadline"));
246+
await expect(pending).rejects.toThrow("request deadline");
247+
expect(Date.now() - t0).toBeLessThan(5_000);
248+
};
249+
250+
await abortAfterDispatch("warm-abort-1");
251+
await abortAfterDispatch("warm-abort-2");
252+
expect(spawnCount() - baseline).toBe(2);
253+
});
254+
255+
test("a late abort from a finished warm solve does not kill the next take on the same handle", async () => {
256+
process.env.ZCODE_CAPTCHA_WORKER = failThenHangFixture;
257+
process.env.ZCODE_CAPTCHA_WARM_WORKERS = "1";
258+
mock.module("./captcha-worker-asset.js", () => ({ default: null }));
259+
__resetCaptchaWorkerDispatchForTest();
260+
const baseline = spawnCount();
261+
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
262+
263+
const controllerA = new AbortController();
264+
await expect(
265+
solveViaWorkerOrInProcess({ scene: "a", region: "r", prefix: "p", signal: controllerA.signal }),
266+
).rejects.toThrow("boom");
267+
268+
const controllerB = new AbortController();
269+
let settledB = false;
270+
const pendingB = solveViaWorkerOrInProcess({ scene: "b", region: "r", prefix: "p", signal: controllerB.signal });
271+
pendingB.then(
272+
() => { settledB = true; },
273+
() => { settledB = true; },
274+
);
275+
await sleep(100);
276+
277+
controllerA.abort(new Error("late abort A"));
278+
await sleep(100);
279+
expect(settledB).toBe(false);
280+
281+
controllerB.abort(new Error("B deadline"));
282+
await expect(pendingB).rejects.toThrow("B deadline");
283+
expect(spawnCount() - baseline).toBe(1);
284+
});
185285
});
186286

187287
// File-level cleanup: runs after every describe above, so shared fixtures

‎src/proxy/captcha-worker-dispatch.ts‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -181,8 +181,16 @@ export async function solveViaWorkerOrInProcess(req: {
181181
scene: string;
182182
region: string;
183183
prefix: string;
184+
signal?: AbortSignal;
184185
}): Promise<string> {
186+
const throwIfAborted = () => {
187+
if (req.signal?.aborted) {
188+
throw req.signal.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted");
189+
}
190+
};
191+
throwIfAborted();
185192
const entryPath = await getWorkerEntryPath();
193+
throwIfAborted();
186194
if (entryPath === null) {
187195
if (strictWorkerRequired()) {
188196
const looked = resolveWorkerEntryCandidates({
@@ -209,7 +217,7 @@ export async function solveViaWorkerOrInProcess(req: {
209217
*/
210218
async function solveInWorkerDegrading(
211219
entryPath: string,
212-
req: { scene: string; region: string; prefix: string },
220+
req: { scene: string; region: string; prefix: string; signal?: AbortSignal },
213221
): Promise<string> {
214222
try {
215223
return await solveInWorker(entryPath, req);
@@ -368,21 +376,31 @@ function acquireWarm(entryPath: string): { handle?: WarmHandle; spawnError?: Err
368376
return spawnWarm(entryPath);
369377
}
370378

371-
function dispatchToWarm(h: WarmHandle, req: { scene: string; region: string; prefix: string }): Promise<string> {
379+
function dispatchToWarm(
380+
h: WarmHandle,
381+
req: { scene: string; region: string; prefix: string; signal?: AbortSignal },
382+
): Promise<string> {
372383
return new Promise<string>((resolve, reject) => {
373384
const id = ++nextSolveId;
385+
const onAbort = () => {
386+
if (h.pending?.id !== id) return;
387+
killWarm(h, req.signal?.reason instanceof Error ? req.signal.reason.message : "captcha solve aborted");
388+
dropFromPool(h);
389+
reject(req.signal?.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted"));
390+
};
374391
const timer = setTimeout(() => {
375392
killWarm(h, `captcha worker timeout (${SOLVE_WORKER_TIMEOUT_MS}ms)`);
376393
dropFromPool(h);
377394
noteWarmDeath(h);
378395
}, SOLVE_WORKER_TIMEOUT_MS);
379396
h.busy = true;
380397
h.pending = { id, timer, resolve, reject };
398+
req.signal?.addEventListener("abort", onAbort, { once: true });
381399
h.worker.postMessage({ id, scene: req.scene, region: req.region, prefix: req.prefix });
382400
});
383401
}
384402

385-
async function warmSolve(entryPath: string, req: { scene: string; region: string; prefix: string }): Promise<string> {
403+
async function warmSolve(entryPath: string, req: { scene: string; region: string; prefix: string; signal?: AbortSignal }): Promise<string> {
386404
const acq = acquireWarm(entryPath);
387405
if (acq.spawnError) {
388406
if (strictWorkerRequired()) {
@@ -448,22 +466,28 @@ export function stopCaptchaWorkers(): void {
448466
*/
449467
function solveInWorker(
450468
entryPath: string,
451-
req: { scene: string; region: string; prefix: string },
469+
req: { scene: string; region: string; prefix: string; signal?: AbortSignal },
452470
): Promise<string> {
453471
const id = ++nextSolveId;
454472
return new Promise<string>((resolve, reject) => {
455473
let settled = false;
456474
let worker: Worker | null = null;
475+
let timer!: ReturnType<typeof setTimeout>;
457476
const settle = (fn: () => void) => {
458477
if (settled) return;
459478
settled = true;
460479
clearTimeout(timer);
480+
req.signal?.removeEventListener("abort", onAbort);
461481
try { worker?.terminate(); } catch {}
462482
fn();
463483
};
464-
const timer = setTimeout(() => {
484+
const onAbort = () => {
485+
settle(() => reject(req.signal?.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted")));
486+
};
487+
timer = setTimeout(() => {
465488
settle(() => reject(new Error(`captcha worker timeout (${SOLVE_WORKER_TIMEOUT_MS}ms)`)));
466489
}, SOLVE_WORKER_TIMEOUT_MS);
490+
req.signal?.addEventListener("abort", onAbort, { once: true });
467491

468492
try {
469493
worker = new Worker(entryPath);
@@ -473,7 +497,7 @@ function solveInWorker(
473497
settle(() => reject(new WorkerUnavailableError(msg)));
474498
return;
475499
}
476-
const msg: SolveRequest = { id, ...req };
500+
const msg: SolveRequest = { id, scene: req.scene, region: req.region, prefix: req.prefix };
477501
worker.on("message", (m: SolveResponse) => {
478502
if (!m || m.id !== id) return;
479503
if (m.ok) settle(() => resolve(m.param));

0 commit comments

Comments
 (0)