From 62384848d641f6d66cdebfcaef87a8d6d95e4226 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 30 Aug 2026 04:50:35 +0000 Subject: [PATCH] Stop losing a post's images when the image provider blinks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every post published since 2026-08-24 went out with no hero and no inline art. Nothing was broken in the image code and nothing was in the database to say so: gpt-image-2 is the only thing in the pipeline that draws, and unlike the article text — which falls back openai -> anthropic — an image failure has nowhere to go. So each call is wrapped in a try/catch that warns and carries on, because losing an illustration should not cost the article. That trade was right; the silence around it was not. OpenAI's quota lapsed for a few days, four posts published image-less, the only trace was a console.warn on a worker, and the first report came from someone looking at the blog. Worse, a failed inline image had its marker deleted, which threw away both where the image belonged and the brief it would have been drawn from — so even once the provider recovered, those images were gone for good. Three changes, none of which make an image failure fatal: - A failed inline image now leaves a self-describing PENDING marker instead of a hole. It is an HTML comment, so readers see nothing, but it carries the ordinal, kind, alt and prompt needed to draw it later. The ordinal matters: it names the object in storage, so repairing image 2 cannot overwrite image 1. - The reasons land on lx_article.generation_error — a column that already existed and was never written — so "which posts lost art, and why" is a query instead of a log trawl nobody runs. - repairMissingArticleImages() backfills them. The hero prompt derives entirely from stored columns, and pending inline markers carry their own brief, so images stay recoverable for as long as the row does. The worker sweep calls it two articles at a time, since four "high" 1536x1024 renders is real money and about a minute of wall clock. A provider outage is now a delay rather than a permanent hole. The five posts already affected have had their heroes generated and backfilled by hand; their inline images predate the marker fix and are not recoverable. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RL1QYJhEEqWzs7FsQQBAMG --- lib/lx/articleGen.ts | 100 ++++++++++++-- lib/lx/guestPostGen.ts | 39 ++++-- lib/lx/repairImages.ts | 206 ++++++++++++++++++++++++++++ tests/autoblog-image-repair.test.ts | 74 ++++++++++ worker/index.ts | 16 +++ 5 files changed, 411 insertions(+), 24 deletions(-) create mode 100644 lib/lx/repairImages.ts create mode 100644 tests/autoblog-image-repair.test.ts diff --git a/lib/lx/articleGen.ts b/lib/lx/articleGen.ts index 16f36bee..961470c9 100644 --- a/lib/lx/articleGen.ts +++ b/lib/lx/articleGen.ts @@ -914,6 +914,65 @@ export async function uploadImage( return data.publicUrl; } +// An inline image that fails to generate used to have its marker deleted, +// which threw away the only record of where the image belonged and what it +// was meant to show — the post published a paragraph short of its plan and +// nothing could put it back. Leave a self-describing marker instead: it is +// an HTML comment, so it renders as nothing for the reader, and the repair +// sweep can regenerate exactly that image later. See repairImages.ts. +const PENDING_INLINE_RE = //g; + +// `--` cannot appear inside an HTML comment, and a quote would end the +// attribute early. Neither survives a round trip, so drop both. +function markerAttr(value: string): string { + return value.replace(/--+/g, " ").replace(/["<>]/g, "").trim(); +} + +export function pendingInlineMarker(spec: { + // 1-based position of this image in the article, which is also what names + // its object in storage. Carried on the marker so a repair that fills in + // only image 2 cannot overwrite image 1. + index: number; + kind?: string | null; + alt?: string | null; + prompt: string; +}): string { + const kind = markerAttr(spec.kind ?? "concept"); + const alt = markerAttr(spec.alt ?? ""); + const prompt = markerAttr(spec.prompt); + return ``; +} + +export type PendingInlineImage = { + raw: string; + index: number; + kind: string; + alt: string; + prompt: string; +}; + +export function parsePendingInlineMarkers(markdown: string): PendingInlineImage[] { + const out: PendingInlineImage[] = []; + for (const m of markdown.matchAll(PENDING_INLINE_RE)) { + const attrs = m[1] ?? ""; + const read = (name: string) => + new RegExp(`${name}="([^"]*)"`).exec(attrs)?.[1] ?? ""; + const prompt = read("prompt"); + // A marker with no prompt can't be regenerated — skip it rather than + // sending the image model an empty brief. + if (!prompt) continue; + const index = Number.parseInt(read("n"), 10); + out.push({ + raw: m[0], + index: Number.isFinite(index) && index > 0 ? index : out.length + 1, + kind: read("kind") || "concept", + alt: read("alt"), + prompt, + }); + } + return out; +} + export function validateInternalLinks( markdown: string, expected: string[], @@ -1342,6 +1401,11 @@ export async function generateArticle( // Featured image + inline section images, generated in parallel so the // 30–60s image latency stacks once instead of 4×. let imageUrl: string | null = null; + // Every image call is best-effort so a provider blip can't cost us the + // article. That silence is the trap: for a week in August every post + // published image-less and nothing recorded why. Collect the reasons and + // store them on the row. + const imageFailures: string[] = []; const inlineImageUrls: Array = new Array( article.inline_image_prompts.length, ).fill(null); @@ -1360,11 +1424,11 @@ export async function generateArticle( }); if (bytes) imageUrl = await uploadImage(supabase, typedSite.id, finalSlug, bytes); + if (!imageUrl) imageFailures.push("hero: no image returned"); } catch (err) { - console.warn( - "[lx] hero image generation failed, continuing without", - err instanceof Error ? err.message : err, - ); + const msg = err instanceof Error ? err.message : String(err); + imageFailures.push(`hero: ${msg}`); + console.warn("[lx] hero image generation failed, continuing without", msg); } })(), ...article.inline_image_prompts.map(async (p, i) => { @@ -1383,25 +1447,33 @@ export async function generateArticle( bytes, ); } + if (!inlineImageUrls[i]) { + imageFailures.push(`inline ${i + 1}: no image returned`); + } } catch (err) { - console.warn( - `[lx] inline image ${i + 1} failed, continuing without`, - err instanceof Error ? err.message : err, - ); + const msg = err instanceof Error ? err.message : String(err); + imageFailures.push(`inline ${i + 1}: ${msg}`); + console.warn(`[lx] inline image ${i + 1} failed, continuing without`, msg); } }), ]); - // Substitute the inline-image markers in markdown. A missing/failed - // image strips the marker rather than leaving a visible comment. + // Substitute the inline-image markers in markdown. A missing/failed image + // leaves a PENDING marker — invisible to the reader, but enough for the + // repair sweep to generate that one image later instead of losing it. let bodyWithImages = article.markdown_body; for (let i = 0; i < article.inline_image_prompts.length; i++) { const marker = new RegExp(``, "g"); const url = inlineImageUrls[i]; - const alt = article.inline_image_prompts[i]?.alt ?? ""; + const spec = article.inline_image_prompts[i]; + const alt = spec?.alt ?? ""; bodyWithImages = bodyWithImages.replace( marker, - url ? `![${alt.replace(/[\[\]]/g, "")}](${url})` : "", + url + ? `![${alt.replace(/[\[\]]/g, "")}](${url})` + : spec + ? pendingInlineMarker({ index: i + 1, kind: spec.kind, alt, prompt: spec.prompt }) + : "", ); } @@ -1464,6 +1536,10 @@ export async function generateArticle( // a blog whose scores are creeping up is drifting before it fails. slop_score: gate?.score ?? null, slop_issues: gate ? gate.issues : [], + generation_error: + imageFailures.length > 0 + ? `images: ${imageFailures.join("; ")}`.slice(0, 2000) + : null, status: "ready", }) .select("id") diff --git a/lib/lx/guestPostGen.ts b/lib/lx/guestPostGen.ts index e8900d5e..aff83203 100644 --- a/lib/lx/guestPostGen.ts +++ b/lib/lx/guestPostGen.ts @@ -33,6 +33,7 @@ import { generateImage, generateInlineImage, normalizeArticleOutput, + pendingInlineMarker, refundCredit, ensureTableOfContentsLinks, slugify, @@ -210,8 +211,10 @@ export async function generateGuestPost( const baseSlug = article.slug || slugify(article.title); const finalSlug = await uniqueSlug(supabase, target.id, baseSlug); - // Hero + inline images. + // Hero + inline images. Best-effort, so record why each one is missing + // rather than publishing image-less in silence — see articleGen. let imageUrl: string | null = null; + const imageFailures: string[] = []; const inlineImageUrls: Array = new Array( article.inline_image_prompts.length, ).fill(null); @@ -228,11 +231,11 @@ export async function generateGuestPost( brand: target.domain ?? null, }); if (bytes) imageUrl = await uploadImage(supabase, target.id, finalSlug, bytes); + if (!imageUrl) imageFailures.push("hero: no image returned"); } catch (err) { - console.warn( - "[lx] guest hero image failed, continuing without", - err instanceof Error ? err.message : err, - ); + const msg = err instanceof Error ? err.message : String(err); + imageFailures.push(`hero: ${msg}`); + console.warn("[lx] guest hero image failed, continuing without", msg); } })(), ...article.inline_image_prompts.map(async (p, i) => { @@ -251,24 +254,32 @@ export async function generateGuestPost( bytes, ); } + if (!inlineImageUrls[i]) { + imageFailures.push(`inline ${i + 1}: no image returned`); + } } catch (err) { - console.warn( - `[lx] guest inline image ${i + 1} failed, continuing without`, - err instanceof Error ? err.message : err, - ); + const msg = err instanceof Error ? err.message : String(err); + imageFailures.push(`inline ${i + 1}: ${msg}`); + console.warn(`[lx] guest inline image ${i + 1} failed, continuing without`, msg); } }), ]); - // Substitute the inline image markers + strip any pandoc heading IDs. + // Substitute the inline image markers + strip any pandoc heading IDs. A + // failed image leaves a PENDING marker for the repair sweep to fill in. let bodyWithImages = article.markdown_body; for (let i = 0; i < article.inline_image_prompts.length; i++) { const marker = new RegExp(``, "g"); const url = inlineImageUrls[i]; - const alt = article.inline_image_prompts[i]?.alt ?? ""; + const spec = article.inline_image_prompts[i]; + const alt = spec?.alt ?? ""; bodyWithImages = bodyWithImages.replace( marker, - url ? `![${alt.replace(/[\[\]]/g, "")}](${url})` : "", + url + ? `![${alt.replace(/[\[\]]/g, "")}](${url})` + : spec + ? pendingInlineMarker({ index: i + 1, kind: spec.kind, alt, prompt: spec.prompt }) + : "", ); } bodyWithImages = bodyWithImages.replace( @@ -316,6 +327,10 @@ export async function generateGuestPost( tags: article.tags, internal_links: internalLinksPayload, outbound_links: outboundLinksPayload, + generation_error: + imageFailures.length > 0 + ? `images: ${imageFailures.join("; ")}`.slice(0, 2000) + : null, status: "ready", }) .select("id") diff --git a/lib/lx/repairImages.ts b/lib/lx/repairImages.ts new file mode 100644 index 00000000..1266cc0a --- /dev/null +++ b/lib/lx/repairImages.ts @@ -0,0 +1,206 @@ +// Backfill images onto posts that published without them. +// +// Image generation is the one step of the article pipeline with no second +// provider: the text falls back openai -> anthropic, but gpt-image-2 is the +// only thing that draws. Every image call is therefore best-effort, wrapped +// in a try/catch that warns and carries on, because losing an illustration +// is not worth losing the article. The cost of that trade only showed up in +// August 2026: OpenAI's quota lapsed for a few days, every post in the +// window published with a null hero and no inline art, and because the +// failure was a console.warn on a worker nobody read, the first report came +// from someone looking at the blog. +// +// This closes the loop. A post that lost its images keeps enough state to +// get them back — the hero prompt is derived entirely from columns we +// store (title, excerpt, tags, niche), and a failed inline image now leaves +// a PENDING marker naming its own prompt. So the images are recoverable for +// as long as the row exists, and a transient outage becomes a delay rather +// than a permanent hole. +// +// Deliberately small per pass: four "high" 1536x1024 images is real money +// and ~60s of wall clock, so the sweep takes a couple of articles at a time +// and lets the next tick pick up the rest. + +import type { SupabaseClient } from "@supabase/supabase-js"; +import type OpenAI from "openai"; +import { markdownToHtml } from "../markdown"; +import { + generateImage, + generateInlineImage, + parsePendingInlineMarkers, + uploadImage, + type BannerStyle, +} from "./articleGen"; + +// How far back to look. An article old enough to have been read already is +// not worth paying to re-illustrate. +const MAX_AGE_DAYS = 45; + +export type ImageRepairResult = { + scanned: number; + articlesRepaired: number; + heroesRestored: number; + inlineRestored: number; + failures: string[]; +}; + +type ArticleRow = { + id: string; + site_id: string; + slug: string; + title: string; + excerpt: string | null; + meta_description: string | null; + tags: string[] | null; + image_url: string | null; + content_markdown: string; + lx_site: { + niche: string | null; + target_audiences: string[] | null; + domain: string | null; + banner_style: string | null; + } | null; +}; + +export async function repairMissingArticleImages( + supabase: SupabaseClient, + openai: OpenAI, + opts: { limit?: number; siteId?: string; now?: Date } = {}, +): Promise { + const limit = opts.limit ?? 2; + const since = new Date( + (opts.now ?? new Date()).getTime() - MAX_AGE_DAYS * 24 * 60 * 60 * 1000, + ).toISOString(); + + const result: ImageRepairResult = { + scanned: 0, + articlesRepaired: 0, + heroesRestored: 0, + inlineRestored: 0, + failures: [], + }; + + // Two independent symptoms of the same outage: a null hero, or an inline + // marker still waiting. PostgREST `or` keeps it to one round trip. + let query = supabase + .from("lx_article") + .select( + "id, site_id, slug, title, excerpt, meta_description, tags, image_url, content_markdown, lx_site!lx_article_site_id_fkey(niche, target_audiences, domain, banner_style)", + ) + .gte("created_at", since) + .or("image_url.is.null,content_markdown.like.*INLINE_IMAGE_PENDING*") + .order("created_at", { ascending: false }) + .limit(limit); + if (opts.siteId) query = query.eq("site_id", opts.siteId); + + const { data, error } = await query; + if (error) { + result.failures.push(`query: ${error.message}`); + return result; + } + + const rows = (data ?? []) as unknown as ArticleRow[]; + result.scanned = rows.length; + + for (const row of rows) { + const site = row.lx_site; + let markdown = row.content_markdown; + let heroUrl = row.image_url; + let changed = false; + + // Hero. Rebuilt from stored columns, so it comes out equivalent to + // what the original run would have produced. + if (!heroUrl) { + try { + const bytes = await generateImage(openai, { + title: row.title, + excerpt: row.excerpt, + metaDescription: row.meta_description, + tags: row.tags, + niche: site?.niche ?? null, + audiences: site?.target_audiences ?? null, + brand: site?.domain ?? null, + style: (site?.banner_style as BannerStyle | null) ?? null, + }); + if (bytes) { + heroUrl = await uploadImage(supabase, row.site_id, row.slug, bytes); + if (heroUrl) { + changed = true; + result.heroesRestored += 1; + } + } + } catch (err) { + result.failures.push( + `${row.slug} hero: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + // Inline. Each marker carries the brief it was generated from, and is + // replaced in place so the image lands back in its own section. + const pending = parsePendingInlineMarkers(markdown); + for (const spec of pending) { + try { + const bytes = await generateInlineImage(openai, spec.prompt, site?.niche ?? null, { + kind: spec.kind, + }); + if (!bytes) continue; + const url = await uploadImage( + supabase, + row.site_id, + `${row.slug}-inline-${spec.index}`, + bytes, + ); + if (!url) continue; + markdown = markdown.replace( + spec.raw, + `![${spec.alt.replace(/[\[\]]/g, "")}](${url})`, + ); + changed = true; + result.inlineRestored += 1; + } catch (err) { + result.failures.push( + `${row.slug} inline ${spec.index}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + if (!changed) continue; + + // Re-render only when the body actually moved; a hero-only repair + // leaves the HTML untouched. + const patch: Record = { image_url: heroUrl }; + if (markdown !== row.content_markdown) { + try { + patch.content_markdown = markdown; + patch.content_html = await markdownToHtml(markdown); + } catch (err) { + result.failures.push( + `${row.slug} render: ${err instanceof Error ? err.message : String(err)}`, + ); + continue; + } + } + + // Clear the recorded reason only once nothing is outstanding, so a + // partial recovery still reads as needing attention. + const stillMissing = + !heroUrl || parsePendingInlineMarkers(markdown).length > 0; + if (!stillMissing) patch.generation_error = null; + + const { error: updErr } = await supabase + .from("lx_article") + .update(patch) + .eq("id", row.id); + if (updErr) { + result.failures.push(`${row.slug} update: ${updErr.message}`); + continue; + } + result.articlesRepaired += 1; + console.log( + `[lx repair] images restored for ${row.slug} (hero=${heroUrl ? "yes" : "no"}, inline=${pending.length})`, + ); + } + + return result; +} diff --git a/tests/autoblog-image-repair.test.ts b/tests/autoblog-image-repair.test.ts new file mode 100644 index 00000000..f29e3d93 --- /dev/null +++ b/tests/autoblog-image-repair.test.ts @@ -0,0 +1,74 @@ +import { describe, it, expect } from "vitest"; +import { + parsePendingInlineMarkers, + pendingInlineMarker, +} from "@/lib/lx/articleGen"; + +/** + * When an inline image fails to generate, the marker used to be deleted and + * the image was gone for good — that is how a few days of OpenAI quota + * trouble in August 2026 left published posts permanently short of their + * art. The marker now survives the failure carrying its own brief, which is + * the only reason repairMissingArticleImages() can put the image back. + */ +describe("pending inline image markers", () => { + it("round-trips the brief needed to regenerate the image", () => { + const marker = pendingInlineMarker({ + index: 2, + kind: "chart", + alt: "Latency by transport", + prompt: "A bar chart comparing WebRTC and HLS end-to-end latency", + }); + const [parsed] = parsePendingInlineMarkers(`intro\n\n${marker}\n\nbody`); + expect(parsed.index).toBe(2); + expect(parsed.kind).toBe("chart"); + expect(parsed.alt).toBe("Latency by transport"); + expect(parsed.prompt).toBe( + "A bar chart comparing WebRTC and HLS end-to-end latency", + ); + expect(parsed.raw).toBe(marker); + }); + + it("stays a well-formed HTML comment when the brief contains -- or quotes", () => { + const marker = pendingInlineMarker({ + index: 1, + kind: "flow", + alt: 'the "handoff" path', + prompt: 'A flow -- from client --> edge -- with a "control" hop', + }); + // A stray `--` would close the comment early and leak prompt text into + // the rendered post; a stray quote would truncate the attribute. + expect(marker.indexOf("--", 4)).toBe(marker.length - 3); + expect(marker).not.toContain('"the "handoff"'); + + const [parsed] = parsePendingInlineMarkers(marker); + expect(parsed.kind).toBe("flow"); + expect(parsed.prompt).toContain("from client"); + }); + + it("finds every pending marker and ignores rendered images", () => { + const body = [ + "## One", + pendingInlineMarker({ index: 1, kind: "chart", alt: "a", prompt: "first" }), + "## Two", + "![already here](https://example.com/x.png)", + "## Three", + pendingInlineMarker({ index: 3, kind: "concept", alt: "c", prompt: "third" }), + ].join("\n\n"); + const found = parsePendingInlineMarkers(body); + expect(found.map((f) => f.prompt)).toEqual(["first", "third"]); + // The ordinal is what names the object in storage, so a repair that + // fills image 3 must not write over image 1. + expect(found.map((f) => f.index)).toEqual([1, 3]); + }); + + it("skips a marker with no prompt rather than briefing the model on nothing", () => { + expect( + parsePendingInlineMarkers(''), + ).toEqual([]); + }); + + it("returns nothing for a body that has no markers", () => { + expect(parsePendingInlineMarkers("## Heading\n\nJust prose.")).toEqual([]); + }); +}); diff --git a/worker/index.ts b/worker/index.ts index b1d5f992..220146f3 100644 --- a/worker/index.ts +++ b/worker/index.ts @@ -34,6 +34,7 @@ import Anthropic from "@anthropic-ai/sdk"; import { generateArticle } from "../lib/lx/articleGen"; import { deliverArticle } from "../lib/lx/webhookDeliver"; import { repairStuckLxJobs } from "../lib/lx/repair"; +import { repairMissingArticleImages } from "../lib/lx/repairImages"; import { processDueSocialFeeds } from "../lib/sp/feedAutopost"; import { processBrowserPost } from "../lib/sp/browserPost"; import { getOrMintInstallationToken } from "../lib/github/installations"; @@ -1306,6 +1307,21 @@ async function lxSweep() { console.log("[worker] lx sweep recovered", repaired); } + // Posts that published without their art. Image generation has no second + // provider, so an OpenAI outage silently ships image-less articles; this + // fills them back in once the provider recovers. Bounded per tick because + // each article is up to four "high" 1536x1024 renders. + if (openai) { + try { + const images = await repairMissingArticleImages(supabase, openai, { limit: 2 }); + if (images.articlesRepaired > 0 || images.failures.length > 0) { + console.log("[worker] lx image repair", images); + } + } catch (err) { + console.warn("[worker] lx image repair crashed", err); + } + } + // Guest-post requests are persisted before the worker is notified. If // that fire-and-forget notify fails, the row stays queued forever unless // the worker polls it. This sweep is the durable queue fallback.