diff --git a/src/proxy/captcha-pool.ts b/src/proxy/captcha-pool.ts index afc67c7..2c9f016 100644 --- a/src/proxy/captcha-pool.ts +++ b/src/proxy/captcha-pool.ts @@ -259,15 +259,21 @@ export class CaptchaTokenPool { // grind through its retry budget (~30-45s) — cap the client-facing wait // and let the background waves finish the job instead. const raceDeadlineMs = Number(process.env.CAPTCHA_SOLVE_RACE_DEADLINE_MS || 35_000); - param = await Promise.race([ - this.solveRaced(cfg), - new Promise((_, rej) => - setTimeout( - () => rej(new Error(`captcha take deadline (${raceDeadlineMs}ms)`)), - Math.max(1_000, raceDeadlineMs), - ), - ), - ]); + const solveAbort = new AbortController(); + let deadlineTimer: ReturnType | undefined; + try { + param = await Promise.race([ + this.solveRaced(cfg, solveAbort.signal), + new Promise((_, rej) => { + deadlineTimer = setTimeout(() => { + solveAbort.abort(new Error(`captcha take deadline (${raceDeadlineMs}ms)`)); + rej(new Error(`captcha take deadline (${raceDeadlineMs}ms)`)); + }, Math.max(1_000, raceDeadlineMs)); + }), + ]); + } finally { + if (deadlineTimer !== undefined) clearTimeout(deadlineTimer); + } } catch (err) { // Mints fail in clusters (pe-stall storms, F008 velocity). Background // refill waves keep retrying — give them a short window to land a @@ -578,7 +584,7 @@ export class CaptchaTokenPool { } } - private async solveFresh(cfg: CaptchaConfig): Promise { + private async solveFresh(cfg: CaptchaConfig, signal?: AbortSignal): Promise { this.activeSolves += 1; this.lastSolveAt = Date.now(); try { @@ -586,7 +592,10 @@ export class CaptchaTokenPool { let sawIpBlock = false; for (let attempt = 1; attempt <= this.opts.solveRetries; attempt += 1) { try { - const param = await runCaptchaSolve(cfg.sceneId, cfg.region, cfg.prefix); + if (signal?.aborted) { + throw signal.reason instanceof Error ? signal.reason : new Error("captcha solve aborted"); + } + const param = await runCaptchaSolve(cfg.sceneId, cfg.region, cfg.prefix, signal); if (!param) { lastErr = "solver returned empty"; continue; @@ -599,6 +608,7 @@ export class CaptchaTokenPool { return param; } catch (err) { lastErr = err instanceof Error ? err.message : String(err); + if (signal?.aborted) throw err; if (isCaptchaDuplicateError(lastErr)) { lastErr = `duplicate certifyId (F008)`; } else if (isCaptchaIpBlockError(lastErr)) { @@ -633,16 +643,16 @@ export class CaptchaTokenPool { * sequential retry chain. Extra successes are banked into the pool instead * of being wasted; the race only rejects when every racer fails. */ - private solveRaced(cfg: CaptchaConfig): Promise { + private solveRaced(cfg: CaptchaConfig, signal?: AbortSignal): Promise { const racers = Math.max(1, Math.min(this.opts.emptyTakeRace, this.opts.solveConcurrency)); - if (racers <= 1) return this.solveFresh(cfg); + if (racers <= 1) return this.solveFresh(cfg, signal); return new Promise((resolve, reject) => { let pending = racers; let served = false; let lastErr: unknown = null; for (let i = 0; i < racers; i += 1) { - this.solveFresh(cfg).then( + this.solveFresh(cfg, signal).then( (param) => { if (served) { // Lost the race but minted a valid token — bank it. diff --git a/src/proxy/captcha-solver.ts b/src/proxy/captcha-solver.ts index 13f8786..3290fa1 100644 --- a/src/proxy/captcha-solver.ts +++ b/src/proxy/captcha-solver.ts @@ -10,11 +10,11 @@ import { solveViaWorkerOrInProcess, stopCaptchaWorkers } from "./captcha-worker- const BACKEND = process.env.ZCODE_CAPTCHA_BACKEND?.trim().toLowerCase() || "happy"; -export async function runCaptchaSolve(scene: string, region: string, prefix: string): Promise { +export async function runCaptchaSolve(scene: string, region: string, prefix: string, signal?: AbortSignal): Promise { if (BACKEND !== "happy") { throw new Error(`captcha backend "${BACKEND}" is not available; use ZCODE_CAPTCHA_BACKEND=happy`); } - return solveViaWorkerOrInProcess({ scene, region, prefix }); + return solveViaWorkerOrInProcess({ scene, region, prefix, signal }); } /** Worker-per-solve needs no concurrency plumbing — kept for the pool API. */ diff --git a/src/proxy/captcha-worker-dispatch.test.ts b/src/proxy/captcha-worker-dispatch.test.ts index 76893a0..741353a 100644 --- a/src/proxy/captcha-worker-dispatch.test.ts +++ b/src/proxy/captcha-worker-dispatch.test.ts @@ -134,6 +134,7 @@ describe("warm worker pool", () => { const warmDir = fs.mkdtempSync(path.join(os.tmpdir(), "cap-warm-")); const markerPath = path.join(warmDir, "spawns.txt"); const warmFixture = path.join(warmDir, "warm-worker.mjs"); + const hangingFixture = path.join(warmDir, "hanging-worker.mjs"); fs.writeFileSync( warmFixture, [ @@ -147,6 +148,32 @@ describe("warm worker pool", () => { ].join("\n"), "utf8", ); + fs.writeFileSync( + hangingFixture, + [ + 'import { parentPort } from "node:worker_threads";', + 'import { appendFileSync } from "node:fs";', + `appendFileSync(${JSON.stringify(markerPath)}, "spawn\\n");`, + "if (!parentPort) throw new Error('fixture requires parent port');", + "parentPort.on('message', () => {});", + ].join("\n"), + "utf8", + ); + const failThenHangFixture = path.join(warmDir, "fail-then-hang-worker.mjs"); + fs.writeFileSync( + failThenHangFixture, + [ + 'import { parentPort } from "node:worker_threads";', + 'import { appendFileSync } from "node:fs";', + `appendFileSync(${JSON.stringify(markerPath)}, "spawn\\n");`, + "if (!parentPort) throw new Error('fixture requires parent port');", + "let n = 0;", + "parentPort.on('message', (m) => {", + " if (n++ === 0) parentPort.postMessage({ id: m.id, ok: false, error: 'boom' });", + "});", + ].join("\n"), + "utf8", + ); const spawnCount = (): number => fs.existsSync(markerPath) ? fs.readFileSync(markerPath, "utf8").split("\n").filter((l) => l === "spawn").length @@ -182,6 +209,79 @@ describe("warm worker pool", () => { // The pool was emptied -> the next solve had to spawn a fresh worker. expect(spawnCount()).toBe(2); }); + + test("aborting a worker-per-solve solve mid-flight terminates the hung worker", async () => { + process.env.ZCODE_CAPTCHA_WORKER = hangingFixture; + delete process.env.ZCODE_CAPTCHA_WARM_WORKERS; + mock.module("./captcha-worker-asset.js", () => ({ default: null })); + __resetCaptchaWorkerDispatchForTest(); + + const controller = new AbortController(); + const pending = solveViaWorkerOrInProcess({ + scene: "abort", + region: "r", + prefix: "p", + signal: controller.signal, + }); + await new Promise((r) => setTimeout(r, 100)); + const t0 = Date.now(); + controller.abort(new Error("request deadline")); + await expect(pending).rejects.toThrow("request deadline"); + expect(Date.now() - t0).toBeLessThan(5_000); + }); + + test("aborting a warm solve mid-flight drops the handle; the next solve respawns", async () => { + process.env.ZCODE_CAPTCHA_WORKER = hangingFixture; + process.env.ZCODE_CAPTCHA_WARM_WORKERS = "1"; + mock.module("./captcha-worker-asset.js", () => ({ default: null })); + __resetCaptchaWorkerDispatchForTest(); + const baseline = spawnCount(); + + const abortAfterDispatch = async (scene: string): Promise => { + const controller = new AbortController(); + const pending = solveViaWorkerOrInProcess({ scene, region: "r", prefix: "p", signal: controller.signal }); + await new Promise((r) => setTimeout(r, 100)); + const t0 = Date.now(); + controller.abort(new Error("request deadline")); + await expect(pending).rejects.toThrow("request deadline"); + expect(Date.now() - t0).toBeLessThan(5_000); + }; + + await abortAfterDispatch("warm-abort-1"); + await abortAfterDispatch("warm-abort-2"); + expect(spawnCount() - baseline).toBe(2); + }); + + test("a late abort from a finished warm solve does not kill the next take on the same handle", async () => { + process.env.ZCODE_CAPTCHA_WORKER = failThenHangFixture; + process.env.ZCODE_CAPTCHA_WARM_WORKERS = "1"; + mock.module("./captcha-worker-asset.js", () => ({ default: null })); + __resetCaptchaWorkerDispatchForTest(); + const baseline = spawnCount(); + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + + const controllerA = new AbortController(); + await expect( + solveViaWorkerOrInProcess({ scene: "a", region: "r", prefix: "p", signal: controllerA.signal }), + ).rejects.toThrow("boom"); + + const controllerB = new AbortController(); + let settledB = false; + const pendingB = solveViaWorkerOrInProcess({ scene: "b", region: "r", prefix: "p", signal: controllerB.signal }); + pendingB.then( + () => { settledB = true; }, + () => { settledB = true; }, + ); + await sleep(100); + + controllerA.abort(new Error("late abort A")); + await sleep(100); + expect(settledB).toBe(false); + + controllerB.abort(new Error("B deadline")); + await expect(pendingB).rejects.toThrow("B deadline"); + expect(spawnCount() - baseline).toBe(1); + }); }); // File-level cleanup: runs after every describe above, so shared fixtures diff --git a/src/proxy/captcha-worker-dispatch.ts b/src/proxy/captcha-worker-dispatch.ts index cc784b5..953a3c3 100644 --- a/src/proxy/captcha-worker-dispatch.ts +++ b/src/proxy/captcha-worker-dispatch.ts @@ -181,8 +181,16 @@ export async function solveViaWorkerOrInProcess(req: { scene: string; region: string; prefix: string; + signal?: AbortSignal; }): Promise { + const throwIfAborted = () => { + if (req.signal?.aborted) { + throw req.signal.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted"); + } + }; + throwIfAborted(); const entryPath = await getWorkerEntryPath(); + throwIfAborted(); if (entryPath === null) { if (strictWorkerRequired()) { const looked = resolveWorkerEntryCandidates({ @@ -209,7 +217,7 @@ export async function solveViaWorkerOrInProcess(req: { */ async function solveInWorkerDegrading( entryPath: string, - req: { scene: string; region: string; prefix: string }, + req: { scene: string; region: string; prefix: string; signal?: AbortSignal }, ): Promise { try { return await solveInWorker(entryPath, req); @@ -368,9 +376,18 @@ function acquireWarm(entryPath: string): { handle?: WarmHandle; spawnError?: Err return spawnWarm(entryPath); } -function dispatchToWarm(h: WarmHandle, req: { scene: string; region: string; prefix: string }): Promise { +function dispatchToWarm( + h: WarmHandle, + req: { scene: string; region: string; prefix: string; signal?: AbortSignal }, +): Promise { return new Promise((resolve, reject) => { const id = ++nextSolveId; + const onAbort = () => { + if (h.pending?.id !== id) return; + killWarm(h, req.signal?.reason instanceof Error ? req.signal.reason.message : "captcha solve aborted"); + dropFromPool(h); + reject(req.signal?.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted")); + }; const timer = setTimeout(() => { killWarm(h, `captcha worker timeout (${SOLVE_WORKER_TIMEOUT_MS}ms)`); dropFromPool(h); @@ -378,11 +395,12 @@ function dispatchToWarm(h: WarmHandle, req: { scene: string; region: string; pre }, SOLVE_WORKER_TIMEOUT_MS); h.busy = true; h.pending = { id, timer, resolve, reject }; + req.signal?.addEventListener("abort", onAbort, { once: true }); h.worker.postMessage({ id, scene: req.scene, region: req.region, prefix: req.prefix }); }); } -async function warmSolve(entryPath: string, req: { scene: string; region: string; prefix: string }): Promise { +async function warmSolve(entryPath: string, req: { scene: string; region: string; prefix: string; signal?: AbortSignal }): Promise { const acq = acquireWarm(entryPath); if (acq.spawnError) { if (strictWorkerRequired()) { @@ -448,22 +466,28 @@ export function stopCaptchaWorkers(): void { */ function solveInWorker( entryPath: string, - req: { scene: string; region: string; prefix: string }, + req: { scene: string; region: string; prefix: string; signal?: AbortSignal }, ): Promise { const id = ++nextSolveId; return new Promise((resolve, reject) => { let settled = false; let worker: Worker | null = null; + let timer!: ReturnType; const settle = (fn: () => void) => { if (settled) return; settled = true; clearTimeout(timer); + req.signal?.removeEventListener("abort", onAbort); try { worker?.terminate(); } catch {} fn(); }; - const timer = setTimeout(() => { + const onAbort = () => { + settle(() => reject(req.signal?.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted"))); + }; + timer = setTimeout(() => { settle(() => reject(new Error(`captcha worker timeout (${SOLVE_WORKER_TIMEOUT_MS}ms)`))); }, SOLVE_WORKER_TIMEOUT_MS); + req.signal?.addEventListener("abort", onAbort, { once: true }); try { worker = new Worker(entryPath); @@ -473,7 +497,7 @@ function solveInWorker( settle(() => reject(new WorkerUnavailableError(msg))); return; } - const msg: SolveRequest = { id, ...req }; + const msg: SolveRequest = { id, scene: req.scene, region: req.region, prefix: req.prefix }; worker.on("message", (m: SolveResponse) => { if (!m || m.id !== id) return; if (m.ok) settle(() => resolve(m.param));