Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 24 additions & 14 deletions src/proxy/captcha-pool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -259,15 +259,21 @@ export class CaptchaTokenPool {
// grind through its retry budget (~30-45s) — cap the client-facing wait
// and let the background waves finish the job instead.
const raceDeadlineMs = Number(process.env.CAPTCHA_SOLVE_RACE_DEADLINE_MS || 35_000);
param = await Promise.race([
this.solveRaced(cfg),
new Promise<never>((_, rej) =>
setTimeout(
() => rej(new Error(`captcha take deadline (${raceDeadlineMs}ms)`)),
Math.max(1_000, raceDeadlineMs),
),
),
]);
const solveAbort = new AbortController();
let deadlineTimer: ReturnType<typeof setTimeout> | undefined;
try {
param = await Promise.race([
this.solveRaced(cfg, solveAbort.signal),
new Promise<never>((_, rej) => {
deadlineTimer = setTimeout(() => {
solveAbort.abort(new Error(`captcha take deadline (${raceDeadlineMs}ms)`));
rej(new Error(`captcha take deadline (${raceDeadlineMs}ms)`));
}, Math.max(1_000, raceDeadlineMs));
}),
]);
} finally {
if (deadlineTimer !== undefined) clearTimeout(deadlineTimer);
}
} catch (err) {
// Mints fail in clusters (pe-stall storms, F008 velocity). Background
// refill waves keep retrying — give them a short window to land a
Expand Down Expand Up @@ -578,15 +584,18 @@ export class CaptchaTokenPool {
}
}

private async solveFresh(cfg: CaptchaConfig): Promise<string> {
private async solveFresh(cfg: CaptchaConfig, signal?: AbortSignal): Promise<string> {
this.activeSolves += 1;
this.lastSolveAt = Date.now();
try {
let lastErr: string | null = null;
let sawIpBlock = false;
for (let attempt = 1; attempt <= this.opts.solveRetries; attempt += 1) {
try {
const param = await runCaptchaSolve(cfg.sceneId, cfg.region, cfg.prefix);
if (signal?.aborted) {
throw signal.reason instanceof Error ? signal.reason : new Error("captcha solve aborted");
}
const param = await runCaptchaSolve(cfg.sceneId, cfg.region, cfg.prefix, signal);
if (!param) {
lastErr = "solver returned empty";
continue;
Expand All @@ -599,6 +608,7 @@ export class CaptchaTokenPool {
return param;
} catch (err) {
lastErr = err instanceof Error ? err.message : String(err);
if (signal?.aborted) throw err;
if (isCaptchaDuplicateError(lastErr)) {
lastErr = `duplicate certifyId (F008)`;
} else if (isCaptchaIpBlockError(lastErr)) {
Expand Down Expand Up @@ -633,16 +643,16 @@ export class CaptchaTokenPool {
* sequential retry chain. Extra successes are banked into the pool instead
* of being wasted; the race only rejects when every racer fails.
*/
private solveRaced(cfg: CaptchaConfig): Promise<string> {
private solveRaced(cfg: CaptchaConfig, signal?: AbortSignal): Promise<string> {
const racers = Math.max(1, Math.min(this.opts.emptyTakeRace, this.opts.solveConcurrency));
if (racers <= 1) return this.solveFresh(cfg);
if (racers <= 1) return this.solveFresh(cfg, signal);

return new Promise<string>((resolve, reject) => {
let pending = racers;
let served = false;
let lastErr: unknown = null;
for (let i = 0; i < racers; i += 1) {
this.solveFresh(cfg).then(
this.solveFresh(cfg, signal).then(
(param) => {
if (served) {
// Lost the race but minted a valid token — bank it.
Expand Down
4 changes: 2 additions & 2 deletions src/proxy/captcha-solver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ import { solveViaWorkerOrInProcess, stopCaptchaWorkers } from "./captcha-worker-

const BACKEND = process.env.ZCODE_CAPTCHA_BACKEND?.trim().toLowerCase() || "happy";

export async function runCaptchaSolve(scene: string, region: string, prefix: string): Promise<string> {
export async function runCaptchaSolve(scene: string, region: string, prefix: string, signal?: AbortSignal): Promise<string> {
if (BACKEND !== "happy") {
throw new Error(`captcha backend "${BACKEND}" is not available; use ZCODE_CAPTCHA_BACKEND=happy`);
}
return solveViaWorkerOrInProcess({ scene, region, prefix });
return solveViaWorkerOrInProcess({ scene, region, prefix, signal });
}

/** Worker-per-solve needs no concurrency plumbing — kept for the pool API. */
Expand Down
100 changes: 100 additions & 0 deletions src/proxy/captcha-worker-dispatch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ describe("warm worker pool", () => {
const warmDir = fs.mkdtempSync(path.join(os.tmpdir(), "cap-warm-"));
const markerPath = path.join(warmDir, "spawns.txt");
const warmFixture = path.join(warmDir, "warm-worker.mjs");
const hangingFixture = path.join(warmDir, "hanging-worker.mjs");
fs.writeFileSync(
warmFixture,
[
Expand All @@ -147,6 +148,32 @@ describe("warm worker pool", () => {
].join("\n"),
"utf8",
);
fs.writeFileSync(
hangingFixture,
[
'import { parentPort } from "node:worker_threads";',
'import { appendFileSync } from "node:fs";',
`appendFileSync(${JSON.stringify(markerPath)}, "spawn\\n");`,
"if (!parentPort) throw new Error('fixture requires parent port');",
"parentPort.on('message', () => {});",
].join("\n"),
"utf8",
);
const failThenHangFixture = path.join(warmDir, "fail-then-hang-worker.mjs");
fs.writeFileSync(
failThenHangFixture,
[
'import { parentPort } from "node:worker_threads";',
'import { appendFileSync } from "node:fs";',
`appendFileSync(${JSON.stringify(markerPath)}, "spawn\\n");`,
"if (!parentPort) throw new Error('fixture requires parent port');",
"let n = 0;",
"parentPort.on('message', (m) => {",
" if (n++ === 0) parentPort.postMessage({ id: m.id, ok: false, error: 'boom' });",
"});",
].join("\n"),
"utf8",
);
const spawnCount = (): number =>
fs.existsSync(markerPath)
? fs.readFileSync(markerPath, "utf8").split("\n").filter((l) => l === "spawn").length
Expand Down Expand Up @@ -182,6 +209,79 @@ describe("warm worker pool", () => {
// The pool was emptied -> the next solve had to spawn a fresh worker.
expect(spawnCount()).toBe(2);
});

test("aborting a worker-per-solve solve mid-flight terminates the hung worker", async () => {
process.env.ZCODE_CAPTCHA_WORKER = hangingFixture;
delete process.env.ZCODE_CAPTCHA_WARM_WORKERS;
mock.module("./captcha-worker-asset.js", () => ({ default: null }));
__resetCaptchaWorkerDispatchForTest();

const controller = new AbortController();
const pending = solveViaWorkerOrInProcess({
scene: "abort",
region: "r",
prefix: "p",
signal: controller.signal,
});
await new Promise((r) => setTimeout(r, 100));
const t0 = Date.now();
controller.abort(new Error("request deadline"));
await expect(pending).rejects.toThrow("request deadline");
expect(Date.now() - t0).toBeLessThan(5_000);
});

test("aborting a warm solve mid-flight drops the handle; the next solve respawns", async () => {
process.env.ZCODE_CAPTCHA_WORKER = hangingFixture;
process.env.ZCODE_CAPTCHA_WARM_WORKERS = "1";
mock.module("./captcha-worker-asset.js", () => ({ default: null }));
__resetCaptchaWorkerDispatchForTest();
const baseline = spawnCount();

const abortAfterDispatch = async (scene: string): Promise<void> => {
const controller = new AbortController();
const pending = solveViaWorkerOrInProcess({ scene, region: "r", prefix: "p", signal: controller.signal });
await new Promise((r) => setTimeout(r, 100));
const t0 = Date.now();
controller.abort(new Error("request deadline"));
await expect(pending).rejects.toThrow("request deadline");
expect(Date.now() - t0).toBeLessThan(5_000);
};

await abortAfterDispatch("warm-abort-1");
await abortAfterDispatch("warm-abort-2");
expect(spawnCount() - baseline).toBe(2);
});

test("a late abort from a finished warm solve does not kill the next take on the same handle", async () => {
process.env.ZCODE_CAPTCHA_WORKER = failThenHangFixture;
process.env.ZCODE_CAPTCHA_WARM_WORKERS = "1";
mock.module("./captcha-worker-asset.js", () => ({ default: null }));
__resetCaptchaWorkerDispatchForTest();
const baseline = spawnCount();
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

const controllerA = new AbortController();
await expect(
solveViaWorkerOrInProcess({ scene: "a", region: "r", prefix: "p", signal: controllerA.signal }),
).rejects.toThrow("boom");

const controllerB = new AbortController();
let settledB = false;
const pendingB = solveViaWorkerOrInProcess({ scene: "b", region: "r", prefix: "p", signal: controllerB.signal });
pendingB.then(
() => { settledB = true; },
() => { settledB = true; },
);
await sleep(100);

controllerA.abort(new Error("late abort A"));
await sleep(100);
expect(settledB).toBe(false);

controllerB.abort(new Error("B deadline"));
await expect(pendingB).rejects.toThrow("B deadline");
expect(spawnCount() - baseline).toBe(1);
});
});

// File-level cleanup: runs after every describe above, so shared fixtures
Expand Down
36 changes: 30 additions & 6 deletions src/proxy/captcha-worker-dispatch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -181,8 +181,16 @@ export async function solveViaWorkerOrInProcess(req: {
scene: string;
region: string;
prefix: string;
signal?: AbortSignal;
}): Promise<string> {
const throwIfAborted = () => {
if (req.signal?.aborted) {
throw req.signal.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted");
}
};
throwIfAborted();
const entryPath = await getWorkerEntryPath();
throwIfAborted();
if (entryPath === null) {
if (strictWorkerRequired()) {
const looked = resolveWorkerEntryCandidates({
Expand All @@ -209,7 +217,7 @@ export async function solveViaWorkerOrInProcess(req: {
*/
async function solveInWorkerDegrading(
entryPath: string,
req: { scene: string; region: string; prefix: string },
req: { scene: string; region: string; prefix: string; signal?: AbortSignal },
): Promise<string> {
try {
return await solveInWorker(entryPath, req);
Expand Down Expand Up @@ -368,21 +376,31 @@ function acquireWarm(entryPath: string): { handle?: WarmHandle; spawnError?: Err
return spawnWarm(entryPath);
}

function dispatchToWarm(h: WarmHandle, req: { scene: string; region: string; prefix: string }): Promise<string> {
function dispatchToWarm(
h: WarmHandle,
req: { scene: string; region: string; prefix: string; signal?: AbortSignal },
): Promise<string> {
return new Promise<string>((resolve, reject) => {
const id = ++nextSolveId;
const onAbort = () => {
if (h.pending?.id !== id) return;
killWarm(h, req.signal?.reason instanceof Error ? req.signal.reason.message : "captcha solve aborted");
dropFromPool(h);
reject(req.signal?.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted"));
};
const timer = setTimeout(() => {
killWarm(h, `captcha worker timeout (${SOLVE_WORKER_TIMEOUT_MS}ms)`);
dropFromPool(h);
noteWarmDeath(h);
}, SOLVE_WORKER_TIMEOUT_MS);
h.busy = true;
h.pending = { id, timer, resolve, reject };
req.signal?.addEventListener("abort", onAbort, { once: true });
h.worker.postMessage({ id, scene: req.scene, region: req.region, prefix: req.prefix });
});
}

async function warmSolve(entryPath: string, req: { scene: string; region: string; prefix: string }): Promise<string> {
async function warmSolve(entryPath: string, req: { scene: string; region: string; prefix: string; signal?: AbortSignal }): Promise<string> {
const acq = acquireWarm(entryPath);
if (acq.spawnError) {
if (strictWorkerRequired()) {
Expand Down Expand Up @@ -448,22 +466,28 @@ export function stopCaptchaWorkers(): void {
*/
function solveInWorker(
entryPath: string,
req: { scene: string; region: string; prefix: string },
req: { scene: string; region: string; prefix: string; signal?: AbortSignal },
): Promise<string> {
const id = ++nextSolveId;
return new Promise<string>((resolve, reject) => {
let settled = false;
let worker: Worker | null = null;
let timer!: ReturnType<typeof setTimeout>;
const settle = (fn: () => void) => {
if (settled) return;
settled = true;
clearTimeout(timer);
req.signal?.removeEventListener("abort", onAbort);
try { worker?.terminate(); } catch {}
fn();
};
const timer = setTimeout(() => {
const onAbort = () => {
settle(() => reject(req.signal?.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted")));
};
timer = setTimeout(() => {
settle(() => reject(new Error(`captcha worker timeout (${SOLVE_WORKER_TIMEOUT_MS}ms)`)));
}, SOLVE_WORKER_TIMEOUT_MS);
req.signal?.addEventListener("abort", onAbort, { once: true });

try {
worker = new Worker(entryPath);
Expand All @@ -473,7 +497,7 @@ function solveInWorker(
settle(() => reject(new WorkerUnavailableError(msg)));
return;
}
const msg: SolveRequest = { id, ...req };
const msg: SolveRequest = { id, scene: req.scene, region: req.region, prefix: req.prefix };
worker.on("message", (m: SolveResponse) => {
if (!m || m.id !== id) return;
if (m.ok) settle(() => resolve(m.param));
Expand Down
Loading