From b84c3ea1a3b540ff04e0f48a3b781072e48fd0bc Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Thu, 24 Sep 2026 16:58:24 +0000 Subject: [PATCH] Re-queue behind a dead BullMQ job, not just an absent one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sweep asked whether a BullMQ job existed for a stranded row and skipped the row if one did. Existence is the wrong question: BullMQ retains a job after it finishes, so a failed or completed job is still findable by id and is emphatically not scheduled. Every row whose attempt had failed sat queued forever behind the corpse of the attempt that failed it — which is exactly the stranding this sweep was written to undo. It cost a working deploy. With the GIF frame-pattern bug fixed and shipped and the row reset to queued, nothing rendered for five minutes and the sweep logged nothing, because it was skipping the row on every pass. A job in a live state is still left alone. A dead one is removed so its id is free, and the row is scheduled again. Co-Authored-By: Claude Opus 5 (1M context) --- lib/ads/video/sweep.ts | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/ads/video/sweep.ts b/lib/ads/video/sweep.ts index d7b362a..6c93dab 100644 --- a/lib/ads/video/sweep.ts +++ b/lib/ads/video/sweep.ts @@ -49,12 +49,24 @@ export async function processDueVideoRenders( const hash = row.render_hash as string; const profile = (row.output_profile as string) ?? "default"; - // Already scheduled: re-adding is harmless because BullMQ dedupes on the - // job id, but checking first keeps the log honest about what this did. - const existing = await queue.getJob(renderJobId(hash, profile)).catch(() => null); + // Already scheduled? Existence is not the question — BullMQ retains a job + // after it finishes, so a failed or completed job is still findable and is + // emphatically not scheduled. Treating "a job exists" as "work is coming" + // left rows queued forever behind the corpse of the attempt that failed + // them, which is precisely the stranding this sweep exists to undo. + // + // A job in a live state is left alone. A dead one is removed so the id is + // free, then re-added below. + const jobId = renderJobId(hash, profile); + const existing = await queue.getJob(jobId).catch(() => null); if (existing) { - skipped++; - continue; + const jobState = await existing.getState().catch(() => "unknown"); + if (jobState === "failed" || jobState === "completed") { + await existing.remove().catch(() => {}); + } else { + skipped++; + continue; + } } // A row with no design cannot be rendered and re-queueing it forever would