From eba50294bb3eb37cb4433f20a42ac6967234b5d9 Mon Sep 17 00:00:00 2001 From: Tomasz Plonka <4361591+PlkMarudny@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:52:35 +0300 Subject: [PATCH 1/2] fix: speed up export --- CHANGELOG.md | 11 ++ CLAUDE.md | 9 +- README.md | 6 +- app.js | 375 +++++++++++++++++++++++++++++++----------- encode-profiles.js | 15 +- server.js | 3 +- test/rest-api.test.js | 48 ++++++ 7 files changed, 359 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 466449a..ffb0d4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 bars collapse via **◂** / **▸** beside the master strip (master L/R stay visible). ### Changed +- Fast export no longer waits for each JPEG HTTP POST before drawing the next + frame. Uploads are pipelined and batched (same pattern as WebCodecs), and + JPEG encode of frame *n* overlaps seek/draw of frame *n+1*. ffmpeg's + image2pipe input is declared as mjpeg (no stdin probe) and given a larger + packet queue so a batched POST does not stall or fail to open. - `POST /api/export/begin` now **requires** `fps` (pass `project.fps`) instead of defaulting to 30, and takes `mode: "jpeg" | "annexb"`. Callers that relied on the old default must send the value; a missing or non-numeric `fps` is a @@ -53,6 +58,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 that omit `panSchema: 1` will be migrated again on the next open. ### Fixed +- Fast / WebCodecs export no longer throws “tainted canvases may not be exported” + for animated SVG overlays (rasterized via a same-origin blob instead of a + `data:` URL) or for other-origin footage that sends CORS (reload with + `crossOrigin=anonymous` for the encode). A clip whose server omits + `Access-Control-Allow-Origin` still cannot be JPEG-encoded — import it into + `./media` instead. - MCP `initialize` no longer echoes an unsupported `protocolVersion`. Missing or unknown versions now negotiate to `2025-11-25` instead of claiming a revision the server does not speak (#58). - `CLAUDE.md` pointed agents at `fablecut_docs {section:"props"}`, which matches no `## ` heading and returns nothing useful; it now names a real section. - Audio graph teardown on project reload — clip chains diff --git a/CLAUDE.md b/CLAUDE.md index 47c8a80..c8545b2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -455,8 +455,11 @@ obvious cuts were missed, raise it if motion is being misread as cuts. redirects to those. Remote SVG is refused (`/media/*.svg` is served same-origin as `image/svg+xml` — a scripted SVG opened as a document would run on the editor origin). Does not write `project.json` — register the media - afterwards (UI and `fablecut_import_media` do this). Do **not** put the - HTTPS URL in `media.src`: canvas CORS would break thumbs, FX and export. + afterwards (UI and `fablecut_import_media` do this). Do **not** put a + raw `https://` URL (or another origin, including `localhost` on a different + port) in `media.src`: canvas CORS would taint the compositor and Fast / + WebCodecs export cannot JPEG-encode. Import into `./media` so `src` is + `/media/…`, or serve the remote with `Access-Control-Allow-Origin`. - `POST /api/analyze` — body `{src:"/media/ref.mp4", threshold?, music?}`: analyze a reference video into an edit blueprint (see "Remake a reference video"); extracts its music into ./media. `GET /api/analyze?src=…` returns the cached blueprint. @@ -471,7 +474,7 @@ obvious cuts were missed, raise it if motion is being misread as cuts. `mode` is `"jpeg"` (default, Fast) or `"annexb"` (WebCodecs H.264 elementary stream); jpeg **400** if `profile` is not a defined id, or if ffmpeg rejects its args in the dry run) · `POST /api/export/frame?id=` (JPEG body for jpeg mode; Annex-B bytes for - annexb — one POST may carry several concatenated AUs. Must be after audio; + annexb — one POST may carry several concatenated JPEGs or AUs. Must be after audio; ffmpeg is spawned on the first frame in both modes) · `POST /api/export/audio?id=` (WAV body — must be sent before the first frame) · `POST /api/export/end?id=[&discard=1]` → `{src}` under `/exports/` diff --git a/README.md b/README.md index 2d22144..ee7ed2d 100644 --- a/README.md +++ b/README.md @@ -193,8 +193,10 @@ same time. - Fast export: browser renders every frame + an offline audio mix; ffmpeg encodes them via an **encoding profile** from `encoding-profiles.json` - (keeps rendering if you switch tabs). The Export dialog has a profile - selector; pin a project default with `encodeProfile` in `project.json` + (keeps rendering if you switch tabs). Frame uploads are batched and overlap + with the next frame's render so HTTP round-trips do not stall the compositor. + The Export dialog has a profile selector; pin a project default with + `encodeProfile` in `project.json` - WebCodecs export: the browser HW-encodes Annex-B H.264; the server stream-copies and muxes audio. Faster uploads; bitrate/VBR-CBR in the Export dialog. Unavailable while an export frame is set (use Fast) diff --git a/app.js b/app.js index 1dec82e..ff0710e 100644 --- a/app.js +++ b/app.js @@ -529,6 +529,13 @@ const ctx2d = els.preview.getContext("2d"); /* ── Utils ─────────────────────────────────────────────────────────────── */ const uid = () => Math.random().toString(36).slice(2, 9); const clamp = (v, a, b) => Math.min(b, Math.max(a, v)); +/** Other host/port than the editor. drawImage of that media taints the canvas + * unless it was fetched with CORS — Fast/WebCodecs export then cannot toBlob. */ +function isCrossOriginSrc(src) { + if (!src) return false; + try { return new URL(src, location.href).origin !== location.origin; } + catch { return false; } +} function escapeHtml(s) { return String(s ?? "") .replace(/&/g, "&") @@ -1067,11 +1074,21 @@ function probeAV(src, kind) { }); } function loadImage(src) { - return new Promise((res, rej) => { const i = new Image(); i.onload = () => res(i); i.onerror = rej; i.src = src; }); + return new Promise((res, rej) => { + const i = new Image(); + // Must be set before src. Same-origin stays unset so /media needs no ACAO; + // cross-origin needs CORS or drawImage taints the canvas. + if (isCrossOriginSrc(src)) i.crossOrigin = "anonymous"; + i.onload = () => res(i); + i.onerror = rej; + i.src = src; + }); } async function grabThumb(m) { const v = document.createElement("video"); - v.muted = true; v.preload = "auto"; v.src = m.src; + v.muted = true; v.preload = "auto"; + if (isCrossOriginSrc(m.src)) v.crossOrigin = "anonymous"; + v.src = m.src; await new Promise((res, rej) => { v.onloadeddata = res; v.onerror = rej; }); v.currentTime = Math.min(0.5, (v.duration || 1) / 2); await new Promise((res) => { v.onseeked = res; setTimeout(res, 1500); }); @@ -4866,23 +4883,47 @@ async function loadSvgMedia(m) { aux.svgFrames = new Map(); // quantized t -> HTMLImageElement (small LRU) aux.svgPending = null; runtime.mediaAux.set(m.id, aux); - if (!aux.svgAnimated) aux.img = await loadImage(m.src); + if (!aux.svgAnimated) aux.img = await rasterizeSvgMarkup(txt); state.dirtyTimeline = true; } -function svgUrlAt(aux, t) { +function svgMarkupAt(aux, t) { + if (!aux.svgAnimated) return aux.svgText; const style = ``; - const txt = aux.svgText.replace(/(]*>)/i, `$1${style}`); - return "data:image/svg+xml;charset=utf-8," + encodeURIComponent(txt); -} -function renderSvgFrame(aux, t) { + return aux.svgText.replace(/(]*>)/i, `$1${style}`); +} +function closeSvgFrame(img) { + if (img && typeof img.close === "function") try { img.close(); } catch { } +} +function pruneSvgFrames(aux) { + if (!aux.svgFrames || aux.svgFrames.size <= 90) return; + const k = aux.svgFrames.keys().next().value; + closeSvgFrame(aux.svgFrames.get(k)); + aux.svgFrames.delete(k); +} +/** Rasterize SVG markup without tainting the compositor canvas. + * data: URLs (and in some browsers) mark the bitmap dirty + * so toBlob throws. A same-origin blob + createImageBitmap stays origin-clean. */ +async function rasterizeSvgMarkup(markup) { + const blob = new Blob([markup], { type: "image/svg+xml;charset=utf-8" }); + if (typeof createImageBitmap === "function") { + try { + const bmp = await createImageBitmap(blob); + if (bmp.width && bmp.height) return bmp; + try { bmp.close(); } catch { } + } catch { /* Safari / empty SVG bitmaps */ } + } return new Promise((resolve, reject) => { const img = new Image(); - img.onload = () => resolve(img); - img.onerror = () => reject(new Error("svg raster failed")); - img.src = svgUrlAt(aux, t); + const url = URL.createObjectURL(blob); + img.onload = () => { URL.revokeObjectURL(url); resolve(img); }; + img.onerror = () => { URL.revokeObjectURL(url); reject(new Error("svg raster failed")); }; + img.src = url; }); } +function renderSvgFrame(aux, t) { + return rasterizeSvgMarkup(svgMarkupAt(aux, t)); +} /* Preview path: returns the best already-rasterized frame and schedules the exact one; export path awaits prepareSvgFrame() instead. */ function getSvgImage(c, t) { @@ -4896,7 +4937,7 @@ function getSvgImage(c, t) { if (!aux.svgPending) { aux.svgPending = renderSvgFrame(aux, q).then((img) => { aux.svgFrames.set(q, img); - if (aux.svgFrames.size > 90) aux.svgFrames.delete(aux.svgFrames.keys().next().value); + pruneSvgFrames(aux); aux.lastImg = img; }).catch(() => { }).finally(() => { aux.svgPending = null; }); } @@ -4913,7 +4954,7 @@ async function prepareSvgFrame(c, t) { try { const img = await renderSvgFrame(aux, q); aux.svgFrames.set(q, img); - if (aux.svgFrames.size > 90) aux.svgFrames.delete(aux.svgFrames.keys().next().value); + pruneSvgFrames(aux); aux.lastImg = img; } catch { } } @@ -5954,6 +5995,8 @@ function ensureFont(name) { runtime.googleLoaded.add(name); const link = document.createElement("link"); link.rel = "stylesheet"; + // CORS mode so @font-face files stay origin-clean when fillText hits the canvas. + link.crossOrigin = "anonymous"; link.href = "https://fonts.googleapis.com/css2?family=" + encodeURIComponent(name).replace(/%20/g, "+") + ":ital,wght@0,300..900;1,300..900&display=swap"; document.head.appendChild(link); @@ -6213,16 +6256,166 @@ function startChosenExport() { /* ── Fast export ── */ let renderCancelled = false; +let exportAbort = null; let exportCropCanvas = null; +let exportCropCtx = null; +function canvasToJpeg(canvas, quality) { + return new Promise((res, rej) => { + const fail = (e) => { + const tainted = e && (e.name === "SecurityError" || /taint/i.test(String(e.message || e))); + rej(tainted + ? new Error("Tainted canvas — a clip is from another origin (not /media or /library) without CORS, or an SVG could not be rasterized cleanly. Import the file into the project, or serve it with Access-Control-Allow-Origin.") + : (e || new Error("frame encode failed"))); + }; + try { + canvas.toBlob((blob) => blob ? res(blob) : fail(new Error("frame encode failed")), "image/jpeg", quality); + } catch (e) { fail(e); } + }); +} function previewToExportBlob(quality = 0.95) { const ef = getExportFrame(); - if (!ef) return new Promise((res) => els.preview.toBlob(res, "image/jpeg", quality)); + if (!ef) return canvasToJpeg(els.preview, quality); if (!exportCropCanvas) exportCropCanvas = document.createElement("canvas"); - exportCropCanvas.width = ef.w; - exportCropCanvas.height = ef.h; - exportCropCanvas.getContext("2d").drawImage( - els.preview, ef.x, ef.y, ef.w, ef.h, 0, 0, ef.w, ef.h); - return new Promise((res) => exportCropCanvas.toBlob(res, "image/jpeg", quality)); + if (exportCropCanvas.width !== ef.w) exportCropCanvas.width = ef.w; + if (exportCropCanvas.height !== ef.h) exportCropCanvas.height = ef.h; + if (!exportCropCtx) exportCropCtx = exportCropCanvas.getContext("2d", { alpha: false }); + exportCropCtx.drawImage(els.preview, ef.x, ef.y, ef.w, ef.h, 0, 0, ef.w, ef.h); + return canvasToJpeg(exportCropCanvas, quality); +} +/** Preview may already be tainted (cross-origin PiP, old data: SVG). Resetting + * width clears the bitmap and the origin-clean flag; export redraws each frame. */ +function resetExportCanvases() { + els.preview.width = els.preview.width; + if (exportCropCanvas) { + exportCropCanvas.width = exportCropCanvas.width; + exportCropCtx = null; + } + adjScratch.width = adjScratch.width; + scratch.width = scratch.width; +} +function waitMediaEl(el, ms = 2500) { + if (el.readyState >= 2 && !(el.error)) return Promise.resolve(); + return new Promise((res) => { + const done = () => { + clearTimeout(tm); + el.removeEventListener("loadeddata", done); + el.removeEventListener("error", done); + res(); + }; + const tm = setTimeout(done, ms); + el.addEventListener("loadeddata", done); + el.addEventListener("error", done); + }); +} +/** Reload other-origin video/images with crossOrigin=anonymous for the export + * compositor. Preview leaves them no-cors so a stream without ACAO still plays. */ +async function armExportCors() { + const jobs = []; + for (const c of project.clips) { + if (c.kind !== "video" || !isTrackEnabled(c.track)) continue; + const m = getMedia(c.mediaId); + if (!m || !isCrossOriginSrc(m.src)) continue; + const el = getClipEl(c); + if (!el || el.crossOrigin === "anonymous") continue; + const t = el.currentTime; + el.crossOrigin = "anonymous"; + el.src = m.src; + jobs.push(waitMediaEl(el).then(() => { + try { if (Number.isFinite(t)) el.currentTime = t; } catch { } + })); + } + for (const m of project.media) { + if (m.kind !== "image" || !isCrossOriginSrc(m.src)) continue; + const aux = runtime.mediaAux.get(m.id); + if (!aux?.img || aux.img.crossOrigin === "anonymous") continue; + jobs.push(loadImage(m.src).then((img) => { + runtime.mediaAux.set(m.id, { ...aux, img }); + }).catch(() => { })); + } + await Promise.all(jobs); + const missing = []; + for (const c of project.clips) { + if (c.kind !== "video" || !isTrackEnabled(c.track)) continue; + const m = getMedia(c.mediaId); + if (!m || !isCrossOriginSrc(m.src)) continue; + const el = runtime.clipEls.get(c.id); + if (el && (el.error || el.readyState < 2)) missing.push(m.name || m.src); + } + if (missing.length) + toast("No CORS on " + missing.join(", ") + " — blank in the export. Import into the project or add Access-Control-Allow-Origin."); +} +/* Sequential /frame POSTs, batched. Concurrent bodies can still race on ffmpeg + stdin if they complete out of order, so the client starts each fetch only + after the previous one settles; rendering runs ahead under backpressure. + JPEG Fast and WebCodecs Annex-B both concatenate on this path — image2pipe + and H.264 start-codes are self-delimiting. */ +function createExportUploader(sessId, { batchItems, batchBytes, signal, getError, setError }) { + let batch = []; + let packed = 0; + let sentFirst = false; + let uploadTail = Promise.resolve(); + let uploadsInFlight = 0; + const cancelled = () => renderCancelled || !!(signal && signal.aborted); + const concatChunks = (parts, n) => { + if (parts.length === 1) return parts[0]; + if (parts[0] instanceof Uint8Array) { + const out = new Uint8Array(n); + let o = 0; + for (const p of parts) { out.set(p, o); o += p.byteLength; } + return out; + } + return new Blob(parts); + }; + const enqueueUpload = (body) => { + uploadsInFlight++; + const p = uploadTail.catch(() => {}).then(async () => { + if (cancelled()) throw new Error("cancelled"); + const err = getError(); + if (err) throw err; + const r = await fetch("/api/export/frame?id=" + sessId, { + method: "POST", body, signal, + }); + if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || "frame upload failed"); + }); + uploadTail = p.catch((err) => { + if (!getError()) setError(err); + }).finally(() => { uploadsInFlight--; }); + return p; + }; + const flush = (force) => { + if (!batch.length) return; + const first = !sentFirst; + if (!force && !first && packed < batchBytes && batch.length < batchItems) return; + sentFirst = true; + const parts = batch, n = packed; + batch = []; + packed = 0; + enqueueUpload(concatChunks(parts, n)); + }; + return { + push(chunk) { + const size = chunk.size ?? chunk.byteLength ?? 0; + batch.push(chunk); + packed += size; + flush(false); + }, + flush, + done() { flush(true); return uploadTail; }, + waitBackpressure(max = 2) { + return new Promise((res, rej) => { + const tick = () => { + if (cancelled()) { clearInterval(poll); rej(new Error("cancelled")); } + else { + const err = getError(); + if (err) { clearInterval(poll); rej(err); } + else if (uploadsInFlight <= max) { clearInterval(poll); res(); } + } + }; + const poll = setInterval(tick, 20); + tick(); + }); + }, + }; } /* ── Fast / WebCodecs frame sync ── HTMLVideoElement has no “step one frame” API — assigning currentTime always @@ -6506,6 +6699,8 @@ async function fastExport() { if (state.exporting) return; pause(); state.exporting = true; state.rendering = true; renderCancelled = false; + exportAbort = new AbortController(); + const signal = exportAbort.signal; els.exportOverlay.classList.remove("hidden"); els.exportProgress.style.width = "0%"; els.exportNote.textContent = "Rendering frames → ffmpeg. You can switch tabs; export continues."; @@ -6513,6 +6708,8 @@ async function fastExport() { const fps = projectFps(), dur = Math.max(1 / fps, projDur()); const frames = Math.max(1, Math.round(dur * fps)); let sessId = null; + let uploadError = null; + const setError = (err) => { if (!uploadError) uploadError = err; }; try { els.exportTitle.textContent = "Mixing audio…"; const wav = await renderAudioMix(dur); @@ -6529,31 +6726,55 @@ async function fastExport() { // will actually feed it, so -map based profiles are checked correctly hasAudio: !!wav, }), + signal, }).then((r) => r.json()); if (!begin.id) throw new Error(begin.error || "export begin failed"); sessId = begin.id; if (wav) { - const r = await fetch("/api/export/audio?id=" + sessId, { method: "POST", body: wav }); + const r = await fetch("/api/export/audio?id=" + sessId, { method: "POST", body: wav, signal }); if (!r.ok) throw new Error("audio upload failed"); } try { await document.fonts.ready; } catch { } + resetExportCanvases(); + await armExportCors(); + // JPEG bodies are ~10× Annex-B AUs; batch a handful so HTTP headers stop + // dominating, without holding many uncompressed frames in RAM. + const up = createExportUploader(sessId, { + batchItems: 8, batchBytes: 512 * 1024, signal, + getError: () => uploadError, setError, + }); + let jpegPending = null; for (let f = 0; f < frames; f++) { - if (renderCancelled) throw new Error("cancelled"); + if (renderCancelled || signal.aborted) throw new Error("cancelled"); + if (uploadError) throw uploadError; + await up.waitBackpressure(2); const t = f / fps; state.time = t; // playhead follows the render await seekVideosTo(t); await prepareFrameAssets(t); // exact SVG frames + AI masks drawFrame(t); - const blob = await previewToExportBlob(jpegQ); - if (!blob) throw new Error("frame encode failed"); - const r = await fetch("/api/export/frame?id=" + sessId, { method: "POST", body: blob }); - if (!r.ok) throw new Error((await r.json()).error || "frame upload failed"); + // Snapshot now (toBlob captures at call time) so the next seek/draw can + // overlap this frame's JPEG encode. + const thisJpeg = previewToExportBlob(jpegQ); + if (jpegPending) { + const blob = await jpegPending; + if (!blob) throw new Error("frame encode failed"); + up.push(blob); + } + jpegPending = thisJpeg; const pct = ((f + 1) / frames) * 100; els.exportProgress.style.width = pct.toFixed(1) + "%"; els.exportTitle.textContent = `Rendering… ${pct.toFixed(0)}%`; } + if (jpegPending) { + const blob = await jpegPending; + if (!blob) throw new Error("frame encode failed"); + up.push(blob); + } els.exportTitle.textContent = "Encoding…"; - const end = await fetch("/api/export/end?id=" + sessId, { method: "POST" }).then((r) => r.json()); + await up.done(); + if (uploadError) throw uploadError; + const end = await fetch("/api/export/end?id=" + sessId, { method: "POST", signal }).then((r) => r.json()); if (!end.src) throw new Error(end.error || "encode failed"); const a = document.createElement("a"); a.href = end.src; @@ -6561,8 +6782,10 @@ async function fastExport() { a.click(); } catch (e) { if (sessId) fetch("/api/export/end?id=" + sessId + "&discard=1", { method: "POST" }).catch(() => { }); - if (String(e.message) !== "cancelled") alert("Export failed: " + e.message); + const msg = e?.name === "AbortError" ? "cancelled" : String(e.message || e); + if (msg !== "cancelled") alert("Export failed: " + msg); } finally { + exportAbort = null; restoreExportVideoState(); state.exporting = false; state.rendering = false; els.exportOverlay.classList.add("hidden"); @@ -6572,11 +6795,6 @@ async function fastExport() { } /* ── WebCodecs export (browser H.264 → server mux) ── */ -/* Uploads MUST be strictly sequential — concurrent /frame POSTs race on the - same ffmpeg stdin and deadlock the pipe. Annex-B AUs concatenate (start - codes), so we batch them: first AU starts ffmpeg, later POSTs carry ~12 AUs - or ~128 KiB so HTTP headers stop dominating the payload. */ -let webCodecsAbort = null; function waitEncodeQueue(encoder, max = 2, { signal, getError } = {}) { const cancelled = () => renderCancelled || !!(signal && signal.aborted); const failed = () => (getError ? getError() : null); @@ -6612,8 +6830,8 @@ async function webCodecsExport() { if (!state.webCodecs) { startExport(); return; } pause(); state.exporting = true; state.rendering = true; renderCancelled = false; - webCodecsAbort = new AbortController(); - const signal = webCodecsAbort.signal; + exportAbort = new AbortController(); + const signal = exportAbort.signal; els.exportOverlay.classList.remove("hidden"); els.exportProgress.style.width = "0%"; els.exportNote.textContent = "Encoding with WebCodecs → ffmpeg mux. You can switch tabs; export continues."; @@ -6624,61 +6842,8 @@ async function webCodecsExport() { let sessId = null; let encoder = null; let uploadError = null; - // single-flight POST chain; each body may be several concatenated Annex-B AUs - const BATCH_AUS = 12; - const BATCH_BYTES = 128 * 1024; - let batch = []; - let batchBytes = 0; - let sentFirstAu = false; - let uploadTail = Promise.resolve(); - let uploadsInFlight = 0; - const concatAus = (parts, n) => { - if (parts.length === 1) return parts[0]; - const out = new Uint8Array(n); - let o = 0; - for (const p of parts) { out.set(p, o); o += p.length; } - return out; - }; - const enqueueUpload = (buf) => { - uploadsInFlight++; - // recover from a prior rejection so one failed POST doesn't stall the chain - const p = uploadTail.catch(() => {}).then(async () => { - if (renderCancelled || signal.aborted) throw new Error("cancelled"); - if (uploadError) throw uploadError; - const r = await fetch("/api/export/frame?id=" + sessId, { - method: "POST", body: buf, signal, - }); - if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || "frame upload failed"); - }); - uploadTail = p.catch((err) => { - if (!uploadError) uploadError = err; - }).finally(() => { uploadsInFlight--; }); - return p; - }; - const flushBatch = (force) => { - if (!batch.length) return; - const first = !sentFirstAu; - if (!force && !first && batchBytes < BATCH_BYTES && batch.length < BATCH_AUS) return; - sentFirstAu = true; - const parts = batch, n = batchBytes; - batch = []; - batchBytes = 0; - enqueueUpload(concatAus(parts, n)); - }; - const enqueueAu = (buf) => { - batch.push(buf); - batchBytes += buf.length; - flushBatch(false); - }; - const waitUploadBackpressure = (max = 2) => new Promise((res, rej) => { - const tick = () => { - if (renderCancelled || signal.aborted) { clearInterval(poll); rej(new Error("cancelled")); } - else if (uploadError) { clearInterval(poll); rej(uploadError); } - else if (uploadsInFlight <= max) { clearInterval(poll); res(); } - }; - const poll = setInterval(tick, 20); - tick(); - }); + const setError = (err) => { if (!uploadError) uploadError = err; }; + let up = null; try { els.exportTitle.textContent = "Mixing audio…"; const wav = await renderAudioMix(dur); @@ -6700,6 +6865,10 @@ async function webCodecsExport() { const r = await fetch("/api/export/audio?id=" + sessId, { method: "POST", body: wav, signal }); if (!r.ok) throw new Error("audio upload failed"); } + up = createExportUploader(sessId, { + batchItems: 12, batchBytes: 128 * 1024, signal, + getError: () => uploadError, setError, + }); // Always encode at project/frame resolution (not display CSS size). const w = Math.max(2, project.width | 0 || 1280); @@ -6716,7 +6885,7 @@ async function webCodecsExport() { if (uploadError || renderCancelled || signal.aborted) return; const buf = new Uint8Array(chunk.byteLength); chunk.copyTo(buf); - enqueueAu(buf); + up.push(buf); }, error: (e) => { uploadError = e; }, }); @@ -6726,11 +6895,13 @@ async function webCodecsExport() { latencyMode: "quality", }); try { await document.fonts.ready; } catch { } + resetExportCanvases(); + await armExportCors(); for (let f = 0; f < frames; f++) { if (renderCancelled || signal.aborted) throw new Error("cancelled"); if (uploadError) throw uploadError; - await waitUploadBackpressure(2); + await up.waitBackpressure(2); await waitEncodeQueue(encoder, 2, { signal, getError: () => uploadError }); const t = f / fps; state.time = t; @@ -6740,10 +6911,17 @@ async function webCodecsExport() { // Absolute µs timestamps; duration = delta so average rate stays exact // (constant Math.round(1e6/fps) drifts, e.g. 33333µs → avg 1000000/33333). const ts = Math.round(f * 1e6 / fps); - const frame = new VideoFrame(els.preview, { - timestamp: ts, - duration: Math.round((f + 1) * 1e6 / fps) - ts, - }); + let frame; + try { + frame = new VideoFrame(els.preview, { + timestamp: ts, + duration: Math.round((f + 1) * 1e6 / fps) - ts, + }); + } catch (e) { + if (e && (e.name === "SecurityError" || /taint/i.test(String(e.message || e)))) + throw new Error("Tainted canvas — a clip is from another origin (not /media or /library) without CORS. Import the file into the project, or serve it with Access-Control-Allow-Origin."); + throw e; + } try { encoder.encode(frame, { keyFrame: f === 0 || f % keyEvery === 0 }); } finally { @@ -6755,8 +6933,7 @@ async function webCodecsExport() { } els.exportTitle.textContent = "Finishing…"; await encoder.flush(); - flushBatch(true); - await uploadTail; + await up.done(); if (uploadError) throw uploadError; encoder.close(); encoder = null; @@ -6772,7 +6949,7 @@ async function webCodecsExport() { const msg = e?.name === "AbortError" ? "cancelled" : String(e.message || e); if (msg !== "cancelled") alert("Export failed: " + msg); } finally { - webCodecsAbort = null; + exportAbort = null; restoreExportVideoState(); state.exporting = false; state.rendering = false; els.exportOverlay.classList.add("hidden"); @@ -6890,7 +7067,7 @@ $("exportWcMode")?.addEventListener("change", async () => { $("btnCancelExport").addEventListener("click", () => { if (state.rendering) { renderCancelled = true; - try { webCodecsAbort?.abort(); } catch { } + try { exportAbort?.abort(); } catch { } } else finishExport(false); }); $("btnPlay").addEventListener("click", () => state.playing ? pause() : play()); diff --git a/encode-profiles.js b/encode-profiles.js index 2a96fcd..042c109 100644 --- a/encode-profiles.js +++ b/encode-profiles.js @@ -13,8 +13,8 @@ conversion + tags from profile.color) and the output path; the profile owns everything in between: - ffmpeg -y -f image2pipe -framerate -i - [-i audio.wav] - + ffmpeg -y -thread_queue_size 64 -f image2pipe -framerate -c:v mjpeg + -i - [-i audio.wav] ═══════════════════════════════════════════════════════════════════════════ */ "use strict"; const fs = require("fs"); @@ -215,7 +215,16 @@ function withJpegColor(profile) { mix (when the timeline has any) is already on disk by the time we spawn. */ function buildExportArgs(profile, { fps, wavPath, outPath }) { // -hide_banner so a failure's stderr tail is the actual error, not the build config - const args = ["-y", "-hide_banner", "-f", "image2pipe", "-framerate", String(fps), "-i", "-"]; + // thread_queue_size: batched JPEG POSTs can dump many packets at once; the + // default queue of 8 blocks stdin (and the HTTP handler) until x264 catches up. + const args = [ + "-y", "-hide_banner", + "-thread_queue_size", "64", + // -c:v mjpeg on the INPUT so ffmpeg does not have to probe stdin. image2pipe + // alone fails the probe when the first write and stdin EOF arrive together + // (short exports / batched POSTs); the later profile -c:v is the encoder. + "-f", "image2pipe", "-framerate", String(fps), "-c:v", "mjpeg", "-i", "-", + ]; if (wavPath) args.push("-i", wavPath); args.push(...withJpegColor(profile), outPath); return args; diff --git a/server.js b/server.js index 2b156ca..ce74486 100644 --- a/server.js +++ b/server.js @@ -179,7 +179,8 @@ function faststart(file) { return maybeFaststart(file); } /* ── Export sessions ── Two modes share the same HTTP session API: - jpeg — browser streams JPEGs; ffmpeg encodes via an encoding profile (Fast) + jpeg — browser streams JPEGs (one POST may concatenate several); + ffmpeg encodes via an encoding profile (Fast) annexb — browser streams Annex-B H.264 (one POST may concatenate several AUs); ffmpeg stream-copies (WebCodecs) Both spawn on the FIRST frame, not here: the audio mix is uploaded between diff --git a/test/rest-api.test.js b/test/rest-api.test.js index 39c5fc8..6cb5640 100644 --- a/test/rest-api.test.js +++ b/test/rest-api.test.js @@ -9,6 +9,7 @@ const fs = require("node:fs"); const http = require("node:http"); const os = require("node:os"); const path = require("node:path"); +const { spawnSync } = require("node:child_process"); const { makeDataDir, readProject, seedProject, startServer, rawGet } = require("./helpers"); const boot = async (t, project) => { @@ -201,6 +202,53 @@ test("POST /api/export/begin validates the encoding profile", async (t) => { assert.equal(end.status, 200); }); +test("buildExportArgs sizes the image2pipe queue for batched JPEGs", () => { + const { buildExportArgs, resolveProfile } = require("../encode-profiles"); + const args = buildExportArgs(resolveProfile("draft"), { fps: 30, outPath: "out.mp4" }); + const i = args.indexOf("-thread_queue_size"); + assert.ok(i >= 0, "image2pipe input should set -thread_queue_size"); + assert.equal(args[i + 1], "64"); + assert.ok(i < args.indexOf("-i"), "the queue size applies to the JPEG stdin input"); + const c = args.indexOf("-c:v"); + assert.equal(args[c + 1], "mjpeg", "input codec must be set before -i so stdin does not need a probe"); + assert.ok(c < args.indexOf("-i")); +}); + +test("POST /api/export/frame accepts concatenated JPEGs in one body", async (t) => { + const { dir, base } = await boot(t); + const ffmpeg = await (await fetch(base + "/api/export/ffmpeg")).json(); + if (!ffmpeg.available) return; + + const jpegPath = path.join(dir, "frame.jpg"); + const made = spawnSync("ffmpeg", [ + "-y", "-hide_banner", "-loglevel", "error", + "-f", "lavfi", "-i", "color=c=black:s=64x64:d=0.1", + "-frames:v", "1", jpegPath, + ], { encoding: "utf8" }); + if (made.status !== 0) return; + const jpeg = fs.readFileSync(jpegPath); + assert.ok(jpeg.length > 0); + + const begin = await fetch(base + "/api/export/begin", { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ fps: 30, name: "batch-jpeg", profile: "draft", hasAudio: false }), + }); + const sess = await begin.json(); + assert.equal(begin.status, 200, sess.error); + + const frame = await fetch(base + "/api/export/frame?id=" + encodeURIComponent(sess.id), { + method: "POST", body: Buffer.concat([jpeg, jpeg, jpeg]), + }); + assert.equal(frame.status, 200, (await frame.json().catch(() => ({}))).error); + + const end = await fetch(base + "/api/export/end?id=" + encodeURIComponent(sess.id), { method: "POST" }); + const out = await end.json(); + assert.equal(end.status, 200, out.error); + assert.match(out.src, /^\/exports\//); + const file = path.join(dir, "exports", decodeURIComponent(out.src.split("/").pop())); + assert.ok(fs.existsSync(file), "batched JPEGs should mux into a finished file"); +}); + test("the app shell and its assets are served", async (t) => { const { base } = await boot(t); const index = await fetch(base + "/"); From 6c1ed376ce6c625462286042eb149ee7b5a84907 Mon Sep 17 00:00:00 2001 From: Tomasz Plonka <4361591+PlkMarudny@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:29:27 +0300 Subject: [PATCH 2/2] feat: use CreateBitmap for screenshotting, faster than before --- CHANGELOG.md | 9 +- CLAUDE.md | 15 ++-- README.md | 10 +-- app.js | 196 +++++++++++++++++++++++++++++++++--------- encode-profiles.js | 67 ++++++++------- server.js | 14 ++- test/rest-api.test.js | 49 ++++++----- 7 files changed, 249 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffb0d4f..e8beace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,10 +43,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Fast export no longer waits for each JPEG HTTP POST before drawing the next - frame. Uploads are pipelined and batched (same pattern as WebCodecs), and - JPEG encode of frame *n* overlaps seek/draw of frame *n+1*. ffmpeg's - image2pipe input is declared as mjpeg (no stdin probe) and given a larger - packet queue so a batched POST does not stall or fail to open. + frame. The compositor snapshots via `transferToImageBitmap` and keeps going; + JPEG encode runs in workers and uploads overlap in batches. Piping uncompressed + RGBA over HTTP was slower (1080p ≈ 8 MiB/frame); the UI is back on JPEG + image2pipe. `pixelFormat: "rgba"` remains on `/api/export/begin` for callers + that want raw frames. - `POST /api/export/begin` now **requires** `fps` (pass `project.fps`) instead of defaulting to 30, and takes `mode: "jpeg" | "annexb"`. Callers that relied on the old default must send the value; a missing or non-numeric `fps` is a diff --git a/CLAUDE.md b/CLAUDE.md index c8545b2..9ed17d2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -469,13 +469,14 @@ obvious cuts were missed, raise it if motion is being misread as cuts. - Fast / WebCodecs export (browser compositor → server ffmpeg): `GET /api/export/ffmpeg` → `{available}` · `GET /api/export/profiles[?detail=1]` → `{default, profiles, issues}` · `POST /api/export/begin` - `{fps,name,mode?,profile?,hasAudio?}` → `{id,mode,profile?,label,summary}` + `{fps,name,mode?,profile?,hasAudio?,pixelFormat?,width?,height?}` → `{id,mode,profile?,label,summary}` (`fps` is required — pass `project.fps`, no server-side default; `mode` is `"jpeg"` (default, Fast) or `"annexb"` (WebCodecs H.264 elementary stream); - jpeg **400** if `profile` is not a defined id, or if ffmpeg rejects its args in the dry run) - · `POST /api/export/frame?id=` (JPEG body for jpeg mode; Annex-B bytes for - annexb — one POST may carry several concatenated JPEGs or AUs. Must be after audio; - ffmpeg is spawned on the first frame in both modes) + jpeg **400** if `profile` is not a defined id, or if ffmpeg rejects its args in the dry run; + optional `pixelFormat:"rgba"` plus `width`/`height` pipes raw canvas frames instead of JPEG) + · `POST /api/export/frame?id=` (JPEG body for jpeg mode; raw RGBA if `begin` + used `pixelFormat:"rgba"`; Annex-B for annexb — one POST may concatenate frames + or AUs. Must be after audio; ffmpeg is spawned on the first frame in both modes) · `POST /api/export/audio?id=` (WAV body — must be sent before the first frame) · `POST /api/export/end?id=[&discard=1]` → `{src}` under `/exports/` @@ -656,7 +657,7 @@ Export is **one ffmpeg pass**. The server owns the input side and the output pat `args` is everything in between (plus JPEG color conversion derived from `color`): ``` -ffmpeg -y -f image2pipe -framerate -i - [-i audio.wav] exports/ +ffmpeg -y -f image2pipe -framerate -i - [-i audio.wav] exports/ ``` - **There is no allow-list.** Any codec, filter, container or flag your local ffmpeg @@ -678,6 +679,8 @@ ffmpeg -y -f image2pipe -framerate -i - [-i audio.wav] { - const fail = (e) => { - const tainted = e && (e.name === "SecurityError" || /taint/i.test(String(e.message || e))); - rej(tainted - ? new Error("Tainted canvas — a clip is from another origin (not /media or /library) without CORS, or an SVG could not be rasterized cleanly. Import the file into the project, or serve it with Access-Control-Allow-Origin.") - : (e || new Error("frame encode failed"))); - }; - try { - canvas.toBlob((blob) => blob ? res(blob) : fail(new Error("frame encode failed")), "image/jpeg", quality); - } catch (e) { fail(e); } - }); +function canvasTaintError(e) { + const tainted = e && (e.name === "SecurityError" || /taint/i.test(String(e.message || e))); + return tainted + ? new Error("Tainted canvas — a clip is from another origin (not /media or /library) without CORS, or an SVG could not be rasterized cleanly. Import the file into the project, or serve it with Access-Control-Allow-Origin.") + : (e || new Error("frame encode failed")); } -function previewToExportBlob(quality = 0.95) { +function exportSourceCanvas() { const ef = getExportFrame(); - if (!ef) return canvasToJpeg(els.preview, quality); + if (!ef) return els.preview; if (!exportCropCanvas) exportCropCanvas = document.createElement("canvas"); if (exportCropCanvas.width !== ef.w) exportCropCanvas.width = ef.w; if (exportCropCanvas.height !== ef.h) exportCropCanvas.height = ef.h; if (!exportCropCtx) exportCropCtx = exportCropCanvas.getContext("2d", { alpha: false }); exportCropCtx.drawImage(els.preview, ef.x, ef.y, ef.w, ef.h, 0, 0, ef.w, ef.h); - return canvasToJpeg(exportCropCanvas, quality); + return exportCropCanvas; +} +let exportSnapOff = null, exportSnapCtx = null; +/** Synchronous snapshot so the compositor can draw the next frame immediately. + * JPEG encode runs off-thread from the ImageBitmap. */ +function snapshotExportFrame() { + const src = exportSourceCanvas(); + try { + if (typeof OffscreenCanvas === "function") { + if (!exportSnapOff || exportSnapOff.width !== src.width || exportSnapOff.height !== src.height) { + exportSnapOff = new OffscreenCanvas(src.width, src.height); + exportSnapCtx = exportSnapOff.getContext("2d", { alpha: false }); + } + exportSnapCtx.drawImage(src, 0, 0); + if (typeof exportSnapOff.transferToImageBitmap === "function") + return { kind: "bmp", bmp: exportSnapOff.transferToImageBitmap() }; + } + const img = src.getContext("2d").getImageData(0, 0, src.width, src.height); + return { kind: "rgba", data: img.data, w: src.width, h: src.height }; + } catch (e) { throw canvasTaintError(e); } +} +const JPEG_WORKER_SRC = `"use strict"; +self.onmessage = async (e) => { + const { id, bmp, quality } = e.data; + try { + const c = new OffscreenCanvas(bmp.width, bmp.height); + const ctx = c.getContext("2d", { alpha: false }); + ctx.drawImage(bmp, 0, 0); + bmp.close(); + const blob = await c.convertToBlob({ type: "image/jpeg", quality: quality || 0.92 }); + const buf = await blob.arrayBuffer(); + self.postMessage({ id, buf }, [buf]); + } catch (err) { + try { bmp.close(); } catch {} + self.postMessage({ id, error: String(err && err.message || err) }); + } +}; +`; +function createJpegWorkers(n) { + if (typeof Worker !== "function" || typeof OffscreenCanvas !== "function") return null; + let url; + try { url = URL.createObjectURL(new Blob([JPEG_WORKER_SRC], { type: "text/javascript" })); } + catch { return null; } + const workers = []; + try { + for (let i = 0; i < n; i++) workers.push(new Worker(url)); + } catch { + for (const w of workers) try { w.terminate(); } catch { } + URL.revokeObjectURL(url); + return null; + } + URL.revokeObjectURL(url); + const pending = new Map(); + let nextId = 0, rr = 0; + for (const w of workers) { + w.onmessage = (e) => { + const rec = pending.get(e.data.id); + if (!rec) return; + pending.delete(e.data.id); + if (e.data.error) rec.reject(new Error(e.data.error)); + else rec.resolve(e.data.buf); + }; + w.onerror = () => {}; + } + return { + encode(bmp, quality) { + const id = nextId++; + const w = workers[rr++ % workers.length]; + return new Promise((resolve, reject) => { + pending.set(id, { resolve, reject }); + try { w.postMessage({ id, bmp, quality }, [bmp]); } + catch (e) { pending.delete(id); reject(e); } + }); + }, + terminate() { + for (const rec of pending.values()) rec.reject(new Error("cancelled")); + pending.clear(); + for (const w of workers) try { w.terminate(); } catch { } + }, + }; +} +function jpegFromBitmap(bmp, quality) { + const c = document.createElement("canvas"); + c.width = bmp.width; c.height = bmp.height; + c.getContext("2d", { alpha: false }).drawImage(bmp, 0, 0); + try { bmp.close(); } catch { } + return new Promise((res, rej) => { + try { + c.toBlob((b) => { + if (!b) return rej(new Error("frame encode failed")); + b.arrayBuffer().then(res, rej); + }, "image/jpeg", quality); + } catch (e) { rej(canvasTaintError(e)); } + }); } /** Preview may already be tainted (cross-origin PiP, old data: SVG). Resetting * width clears the bitmap and the origin-clean flag; export redraws each frame. */ @@ -6292,6 +6378,8 @@ function resetExportCanvases() { } adjScratch.width = adjScratch.width; scratch.width = scratch.width; + exportSnapOff = null; + exportSnapCtx = null; } function waitMediaEl(el, ms = 2500) { if (el.readyState >= 2 && !(el.error)) return Promise.resolve(); @@ -6347,8 +6435,7 @@ async function armExportCors() { /* Sequential /frame POSTs, batched. Concurrent bodies can still race on ffmpeg stdin if they complete out of order, so the client starts each fetch only after the previous one settles; rendering runs ahead under backpressure. - JPEG Fast and WebCodecs Annex-B both concatenate on this path — image2pipe - and H.264 start-codes are self-delimiting. */ + JPEG Fast and WebCodecs Annex-B both concatenate on this path. */ function createExportUploader(sessId, { batchItems, batchBytes, signal, getError, setError }) { let batch = []; let packed = 0; @@ -6423,8 +6510,8 @@ function createExportUploader(sessId, { batchItems, batchBytes, signal, getError • already within ½ media-frame → no-op • forward through a shot, buffered → play + requestVideoFrameCallback. If the last compositor tick was faster than a timeline frame, leave the - element playing (no play/pause per frame). If it was slower (typical Fast - JPEG upload), pause after the hit so the file cannot overrun. + element playing (no play/pause per frame). If it was slower, pause after + the hit so the file cannot overrun. • reverse / large jump / unbuffered / overshoot → hard seek Incoming clips are seek-prefetched ~1 s before they become active. */ let exportSeekClock = 0; @@ -6551,7 +6638,7 @@ async function seekVideosTo(t) { const now = performance.now(); const frameMs = 1000 / fps; // Last compositor+upload tick faster than a timeline frame → decoder can - // stay in play() through the shot. Slower (Fast JPEG) → pause so we don't + // stay in play() through the shot. Slower → pause so we don't // overshoot and seek backwards every frame. const keepPlaying = exportSeekClock > 0 && (now - exportSeekClock) < frameMs * 1.4; exportSeekClock = now; @@ -6722,8 +6809,6 @@ async function fastExport() { fps, name: project.name.replace(/[^\w\- ]+/g, "") || "export", profile: profileId, - // lets the server dry-run the profile with the same input count we - // will actually feed it, so -map based profiles are checked correctly hasAudio: !!wav, }), signal, @@ -6737,43 +6822,72 @@ async function fastExport() { try { await document.fonts.ready; } catch { } resetExportCanvases(); await armExportCors(); - // JPEG bodies are ~10× Annex-B AUs; batch a handful so HTTP headers stop - // dominating, without holding many uncompressed frames in RAM. + // JPEG off the compositor thread: snapshot is sync, encode/upload run ahead + // under backpressure. Awaiting toBlob every frame was slower than ffmpeg. const up = createExportUploader(sessId, { batchItems: 8, batchBytes: 512 * 1024, signal, getError: () => uploadError, setError, }); - let jpegPending = null; + const workers = createJpegWorkers(Math.min(3, Math.max(1, (navigator.hardwareConcurrency || 2) - 1))); + let pixelChain = Promise.resolve(); + let pixelsInflight = 0; + const waitPixels = (max) => new Promise((res, rej) => { + const tick = () => { + if (renderCancelled || signal.aborted) { clearInterval(poll); rej(new Error("cancelled")); } + else if (uploadError) { clearInterval(poll); rej(uploadError); } + else if (pixelsInflight <= max) { clearInterval(poll); res(); } + }; + const poll = setInterval(tick, 4); + tick(); + }); + try { for (let f = 0; f < frames; f++) { if (renderCancelled || signal.aborted) throw new Error("cancelled"); if (uploadError) throw uploadError; + await waitPixels(3); await up.waitBackpressure(2); const t = f / fps; - state.time = t; // playhead follows the render + state.time = t; await seekVideosTo(t); - await prepareFrameAssets(t); // exact SVG frames + AI masks + await prepareFrameAssets(t); drawFrame(t); - // Snapshot now (toBlob captures at call time) so the next seek/draw can - // overlap this frame's JPEG encode. - const thisJpeg = previewToExportBlob(jpegQ); - if (jpegPending) { - const blob = await jpegPending; - if (!blob) throw new Error("frame encode failed"); - up.push(blob); + const snap = snapshotExportFrame(); + pixelsInflight++; + let jpegP; + if (snap.kind === "bmp") { + jpegP = workers ? workers.encode(snap.bmp, jpegQ) : jpegFromBitmap(snap.bmp, jpegQ); + } else { + const c = document.createElement("canvas"); + c.width = snap.w; c.height = snap.h; + c.getContext("2d", { alpha: false }).putImageData(new ImageData(snap.data, snap.w, snap.h), 0, 0); + jpegP = new Promise((res, rej) => { + try { + c.toBlob((b) => { + if (!b) return rej(new Error("frame encode failed")); + b.arrayBuffer().then(res, rej); + }, "image/jpeg", jpegQ); + } catch (e) { rej(canvasTaintError(e)); } + }); } - jpegPending = thisJpeg; + pixelChain = pixelChain.then(async () => { + if (renderCancelled || signal.aborted) throw new Error("cancelled"); + if (uploadError) throw uploadError; + const buf = await jpegP; + if (!buf || !(buf.byteLength || buf.size || buf.length)) throw new Error("frame encode failed"); + up.push(buf instanceof ArrayBuffer ? new Uint8Array(buf) : buf); + }).catch((err) => { setError(err); throw err; }).finally(() => { pixelsInflight--; }); const pct = ((f + 1) / frames) * 100; els.exportProgress.style.width = pct.toFixed(1) + "%"; els.exportTitle.textContent = `Rendering… ${pct.toFixed(0)}%`; } - if (jpegPending) { - const blob = await jpegPending; - if (!blob) throw new Error("frame encode failed"); - up.push(blob); - } + await pixelChain; + if (uploadError) throw uploadError; els.exportTitle.textContent = "Encoding…"; await up.done(); if (uploadError) throw uploadError; + } finally { + try { workers?.terminate(); } catch { } + } const end = await fetch("/api/export/end?id=" + sessId, { method: "POST", signal }).then((r) => r.json()); if (!end.src) throw new Error(end.error || "encode failed"); const a = document.createElement("a"); diff --git a/encode-profiles.js b/encode-profiles.js index 042c109..6da098f 100644 --- a/encode-profiles.js +++ b/encode-profiles.js @@ -9,12 +9,14 @@ nothing but a smaller set of usable formats. Typos are caught by dryRunProfile against the real ffmpeg build instead, which also knows which encoders it has. - Export is ONE ffmpeg pass; this module owns the input side (JPEG color - conversion + tags from profile.color) and the output path; the profile owns - everything in between: + Export is ONE ffmpeg pass; this module owns the input side (JPEG image2pipe + by default, or canvas RGBA when pixelFormat is "rgba") plus color conversion + and tags from profile.color; the profile owns everything in between: - ffmpeg -y -thread_queue_size 64 -f image2pipe -framerate -c:v mjpeg - -i - [-i audio.wav] + ffmpeg -y -thread_queue_size 64 -f image2pipe -framerate + -c:v mjpeg -i - [-i audio.wav] + + `pixelFormat: "rgba"` switches stdin to `-f rawvideo -pix_fmt rgba`. ═══════════════════════════════════════════════════════════════════════════ */ "use strict"; const fs = require("fs"); @@ -182,13 +184,14 @@ function listProfilesPublic(detail) { return out; } -/* JPEG frames from the browser are full-range BT.601 (JFIF). Convert them to - the profile's output matrix/range and tag the stream, otherwise x264 emits - bt470bg/pc/unknown and players do the wrong YUV→RGB conversion — darker than - preview. Independent of -pix_fmt (420 vs 422/10-bit). */ -function jpegColorVf(color) { +/* Canvas RGBA is full-range BT.709-ish (sRGB). JPEG/JFIF is full-range BT.601. + Convert to the profile's output matrix/range and tag the stream, otherwise + x264 emits bt470bg/pc/unknown and players do the wrong YUV→RGB conversion. + Independent of -pix_fmt (420 vs 422/10-bit). */ +function inputColorVf(color, inMatrix) { const c = color || DEFAULT_COLOR; - return `scale=in_range=full:in_color_matrix=bt601:out_range=${c.range}:out_color_matrix=${c.matrix}`; + const matrix = inMatrix === "bt601" ? "bt601" : "bt709"; + return `scale=in_range=full:in_color_matrix=${matrix}:out_range=${c.range}:out_color_matrix=${c.matrix}`; } function jpegColorTags(color) { const c = color || DEFAULT_COLOR; @@ -200,9 +203,9 @@ function jpegColorTags(color) { ]; } -function withJpegColor(profile) { +function withInputColor(profile, inMatrix) { const color = profile.color || DEFAULT_COLOR; - const vf = jpegColorVf(color); + const vf = inputColorVf(color, inMatrix); const args = stripColorArgs((profile.args || []).slice()); const vfAt = args.indexOf("-vf"); if (vfAt >= 0 && args[vfAt + 1] != null) args[vfAt + 1] = `${vf},${args[vfAt + 1]}`; @@ -211,22 +214,28 @@ function withJpegColor(profile) { return args; } -/* The single export pass. Frames arrive on stdin as a JPEG stream; the audio - mix (when the timeline has any) is already on disk by the time we spawn. */ -function buildExportArgs(profile, { fps, wavPath, outPath }) { - // -hide_banner so a failure's stderr tail is the actual error, not the build config - // thread_queue_size: batched JPEG POSTs can dump many packets at once; the - // default queue of 8 blocks stdin (and the HTTP handler) until x264 catches up. - const args = [ - "-y", "-hide_banner", - "-thread_queue_size", "64", - // -c:v mjpeg on the INPUT so ffmpeg does not have to probe stdin. image2pipe - // alone fails the probe when the first write and stdin EOF arrive together - // (short exports / batched POSTs); the later profile -c:v is the encoder. - "-f", "image2pipe", "-framerate", String(fps), "-c:v", "mjpeg", "-i", "-", - ]; +/* The single export pass. Fast export sends JPEG image2pipe by default; + `pixelFormat: "rgba"` is uncompressed canvas frames. Audio mix is on disk + before spawn. */ +function buildExportArgs(profile, { fps, wavPath, outPath, pixelFormat, width, height } = {}) { + const args = ["-y", "-hide_banner"]; + const raw = pixelFormat === "rgba"; + if (raw) { + const w = Math.max(2, width | 0), h = Math.max(2, height | 0); + // Small queue: each raw frame is width×height×4 bytes (1080p ≈ 8 MiB). + args.push( + "-thread_queue_size", "8", + "-f", "rawvideo", "-pix_fmt", "rgba", + "-s", `${w}x${h}`, "-framerate", String(fps), "-i", "-", + ); + } else { + args.push( + "-thread_queue_size", "64", + "-f", "image2pipe", "-framerate", String(fps), "-c:v", "mjpeg", "-i", "-", + ); + } if (wavPath) args.push("-i", wavPath); - args.push(...withJpegColor(profile), outPath); + args.push(...withInputColor(profile, raw ? "bt709" : "bt601"), outPath); return args; } @@ -242,7 +251,7 @@ function dryRunProfile(profile, { fps = 30, hasAudio = true } = {}) { const args = ["-y", "-hide_banner", "-f", "lavfi", "-i", `color=c=black:s=64x64:r=${fps}:d=0.1`]; if (hasAudio) args.push("-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo"); - args.push("-t", "0.1", ...withJpegColor(profile), out); + args.push("-t", "0.1", ...withInputColor(profile, "bt709"), out); return new Promise((resolve) => { let stderr = ""; let settled = false; diff --git a/server.js b/server.js index ce74486..7196bd1 100644 --- a/server.js +++ b/server.js @@ -207,12 +207,18 @@ function attachProc(sess, proc) { proc.stdin.on("error", () => {}); // EPIPE if ffmpeg dies mid-stream sess.done = new Promise((res) => proc.on("close", res)); } -async function beginExport(fps, name, profileId, hasAudio, mode) { +async function beginExport(fps, name, profileId, hasAudio, mode, extra = {}) { const m = mode === "annexb" ? "annexb" : "jpeg"; const rate = Number(fps); if (!Number.isFinite(rate) || rate <= 0) { throw new Error("export fps required (pass project.fps)"); } + const pixelFormat = extra.pixelFormat === "rgba" ? "rgba" : "jpeg"; + const width = Math.max(0, extra.width | 0); + const height = Math.max(0, extra.height | 0); + if (pixelFormat === "rgba" && (width < 2 || height < 2)) { + throw new Error("export width/height required for rgba"); + } const id = Date.now().toString(36) + Math.random().toString(36).slice(2, 7); const safe = safeName(name || "export"); // Same filesystem as the finished file so renameSync(partPath, out) cannot EXDEV. @@ -241,6 +247,7 @@ async function beginExport(fps, name, profileId, hasAudio, mode) { const { outPath, partPath } = reserveExportPaths(EXPORTS_DIR, safe, profile.extension); const sess = { mode: m, proc: null, fps: rate, profile, name: safe, hasAudio: !!hasAudio, + pixelFormat, width, height, dir, wav: null, partPath, outPath, stderr: "", done: null, lastTouch: Date.now(), err: () => sess.stderr.trim().split("\n").filter(Boolean).slice(-3) @@ -255,6 +262,7 @@ async function beginExport(fps, name, profileId, hasAudio, mode) { function startEncoder(sess) { const proc = spawn("ffmpeg", buildExportArgs(sess.profile, { fps: sess.fps, wavPath: sess.wav, outPath: sess.partPath, + pixelFormat: sess.pixelFormat, width: sess.width, height: sess.height, }), { stdio: ["pipe", "ignore", "pipe"] }); proc.stderr.on("data", (d) => { sess.stderr = (sess.stderr + d).slice(-2000); }); // EPIPE on end()/late writes is normal once ffmpeg has exited; writeExportFrame @@ -510,10 +518,10 @@ const server = http.createServer(async (req, res) => { const mode = opts.mode === "annexb" ? "annexb" : "jpeg"; if (mode !== "annexb" && opts.profile) resolveProfile(opts.profile); // 400, not 500, on a bad id — even without ffmpeg if (!HAS_FFMPEG) { sendJSON(res, 400, { error: "ffmpeg not found on PATH" }); return; } - sendJSON(res, 200, await beginExport(opts.fps, opts.name, opts.profile, opts.hasAudio !== false, mode)); + sendJSON(res, 200, await beginExport(opts.fps, opts.name, opts.profile, opts.hasAudio !== false, mode, opts)); } catch (e) { // an unusable profile is the caller's problem, not a server fault - const bad = /^Unknown encoding profile|was rejected by ffmpeg|export fps required/.test(e.message || ""); + const bad = /^Unknown encoding profile|was rejected by ffmpeg|export fps required|export width\/height required/.test(e.message || ""); sendJSON(res, bad ? 400 : 500, { error: String(e.message || e) }); } return; diff --git a/test/rest-api.test.js b/test/rest-api.test.js index 6cb5640..a10b209 100644 --- a/test/rest-api.test.js +++ b/test/rest-api.test.js @@ -9,7 +9,6 @@ const fs = require("node:fs"); const http = require("node:http"); const os = require("node:os"); const path = require("node:path"); -const { spawnSync } = require("node:child_process"); const { makeDataDir, readProject, seedProject, startServer, rawGet } = require("./helpers"); const boot = async (t, project) => { @@ -202,51 +201,55 @@ test("POST /api/export/begin validates the encoding profile", async (t) => { assert.equal(end.status, 200); }); -test("buildExportArgs sizes the image2pipe queue for batched JPEGs", () => { +test("buildExportArgs uses raw RGBA when the client asks for it", () => { + const { buildExportArgs, resolveProfile } = require("../encode-profiles"); + const args = buildExportArgs(resolveProfile("draft"), { + fps: 50, outPath: "out.mp4", pixelFormat: "rgba", width: 1920, height: 1080, + }); + assert.ok(args.includes("rawvideo")); + assert.equal(args[args.indexOf("-s") + 1], "1920x1080"); + assert.ok(args.indexOf("-f") < args.indexOf("-i")); +}); + +test("buildExportArgs still accepts JPEG image2pipe for older clients", () => { const { buildExportArgs, resolveProfile } = require("../encode-profiles"); const args = buildExportArgs(resolveProfile("draft"), { fps: 30, outPath: "out.mp4" }); const i = args.indexOf("-thread_queue_size"); - assert.ok(i >= 0, "image2pipe input should set -thread_queue_size"); - assert.equal(args[i + 1], "64"); - assert.ok(i < args.indexOf("-i"), "the queue size applies to the JPEG stdin input"); - const c = args.indexOf("-c:v"); - assert.equal(args[c + 1], "mjpeg", "input codec must be set before -i so stdin does not need a probe"); - assert.ok(c < args.indexOf("-i")); + assert.ok(i >= 0); + assert.equal(args[args.indexOf("-c:v") + 1], "mjpeg"); + assert.ok(args.indexOf("-c:v") < args.indexOf("-i")); }); -test("POST /api/export/frame accepts concatenated JPEGs in one body", async (t) => { +test("POST /api/export/frame accepts concatenated RGBA frames", async (t) => { const { dir, base } = await boot(t); const ffmpeg = await (await fetch(base + "/api/export/ffmpeg")).json(); if (!ffmpeg.available) return; - const jpegPath = path.join(dir, "frame.jpg"); - const made = spawnSync("ffmpeg", [ - "-y", "-hide_banner", "-loglevel", "error", - "-f", "lavfi", "-i", "color=c=black:s=64x64:d=0.1", - "-frames:v", "1", jpegPath, - ], { encoding: "utf8" }); - if (made.status !== 0) return; - const jpeg = fs.readFileSync(jpegPath); - assert.ok(jpeg.length > 0); + const w = 64, h = 64; + const frame = Buffer.alloc(w * h * 4, 0); + for (let i = 3; i < frame.length; i += 4) frame[i] = 255; const begin = await fetch(base + "/api/export/begin", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ fps: 30, name: "batch-jpeg", profile: "draft", hasAudio: false }), + body: JSON.stringify({ + fps: 30, name: "batch-rgba", profile: "draft", hasAudio: false, + pixelFormat: "rgba", width: w, height: h, + }), }); const sess = await begin.json(); assert.equal(begin.status, 200, sess.error); - const frame = await fetch(base + "/api/export/frame?id=" + encodeURIComponent(sess.id), { - method: "POST", body: Buffer.concat([jpeg, jpeg, jpeg]), + const posted = await fetch(base + "/api/export/frame?id=" + encodeURIComponent(sess.id), { + method: "POST", body: Buffer.concat([frame, frame, frame]), }); - assert.equal(frame.status, 200, (await frame.json().catch(() => ({}))).error); + assert.equal(posted.status, 200, (await posted.json().catch(() => ({}))).error); const end = await fetch(base + "/api/export/end?id=" + encodeURIComponent(sess.id), { method: "POST" }); const out = await end.json(); assert.equal(end.status, 200, out.error); assert.match(out.src, /^\/exports\//); const file = path.join(dir, "exports", decodeURIComponent(out.src.split("/").pop())); - assert.ok(fs.existsSync(file), "batched JPEGs should mux into a finished file"); + assert.ok(fs.existsSync(file), "batched RGBA frames should mux into a finished file"); }); test("the app shell and its assets are served", async (t) => {