diff --git a/src/agents/createOverlayPositionAgent.ts b/src/agents/createOverlayPositionAgent.ts new file mode 100644 index 0000000..3847fc6 --- /dev/null +++ b/src/agents/createOverlayPositionAgent.ts @@ -0,0 +1,32 @@ +import { ToolLoopAgent, Output, stepCountIs } from "ai"; +import { z } from "zod"; +import { IMAGE_DETECTION_MODEL } from "./imageDetectionModel"; +import { OVERLAY_POSITIONS } from "../content/overlayPosition"; + +const overlayPositionSchema = z.object({ + position: z.enum(OVERLAY_POSITIONS), +}); + +const instructions = `You analyze an editorial image to determine the best corner to place small overlay images (like playlist covers or DSP logos) so they do not obscure the subject. + +Choose exactly ONE of four positions: top-left, top-right, bottom-left, bottom-right. + +Rules: +- Pick the corner that has the LEAST visual importance — the emptiest or most uniform area. +- Avoid placing overlays on top of the subject's face, hands, or other key focal points. +- If the subject is centered and all corners are roughly equal, prefer "top-left" as the default. +- If the background is blurred or uniform on one side, prefer that side. +- Consider that overlays will be 250×250 pixels stacked vertically in the chosen corner of a 720×1280 (9:16 portrait) frame.`; + +/** + * Creates a ToolLoopAgent that analyzes an editorial image and returns + * the best corner position for overlay images. + */ +export function createOverlayPositionAgent() { + return new ToolLoopAgent({ + model: IMAGE_DETECTION_MODEL, + instructions, + output: Output.object({ schema: overlayPositionSchema }), + stopWhen: stepCountIs(1), + }); +} diff --git a/src/content/__tests__/buildFilterComplex.test.ts b/src/content/__tests__/buildFilterComplex.test.ts index d232a19..0c902c3 100644 --- a/src/content/__tests__/buildFilterComplex.test.ts +++ b/src/content/__tests__/buildFilterComplex.test.ts @@ -43,4 +43,45 @@ describe("buildFilterComplex", () => { expect(result).toContain("[out]"); }); + + it("positions overlays in top-right when specified", () => { + const result = buildFilterComplex({ + overlayCount: 1, + captionFilters: [], + overlayPosition: "top-right", + }); + + // 720 - 30 - 150 = 540 + expect(result).toMatch(/overlay=540:30/); + }); + + it("positions overlays in bottom-left when specified", () => { + const result = buildFilterComplex({ + overlayCount: 1, + captionFilters: [], + overlayPosition: "bottom-left", + }); + + // 1280 - 30 - 150 = 1100 + expect(result).toMatch(/overlay=30:1100/); + }); + + it("positions overlays in bottom-right when specified", () => { + const result = buildFilterComplex({ + overlayCount: 1, + captionFilters: [], + overlayPosition: "bottom-right", + }); + + expect(result).toMatch(/overlay=540:1100/); + }); + + it("defaults to top-left when no overlayPosition provided", () => { + const result = buildFilterComplex({ + overlayCount: 1, + captionFilters: [], + }); + + expect(result).toMatch(/overlay=30:30/); + }); }); diff --git a/src/content/__tests__/overlayPosition.test.ts b/src/content/__tests__/overlayPosition.test.ts new file mode 100644 index 0000000..201d461 --- /dev/null +++ b/src/content/__tests__/overlayPosition.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect } from "vitest"; + +import { calculateOverlayCoordinates } from "../overlayPosition"; + +describe("calculateOverlayCoordinates", () => { + it("top-left: first overlay at (30, 30)", () => { + expect(calculateOverlayCoordinates("top-left", 0)).toEqual({ x: 30, y: 30 }); + }); + + it("top-left: second overlay stacks below", () => { + expect(calculateOverlayCoordinates("top-left", 1)).toEqual({ x: 30, y: 300 }); + }); + + it("top-right: first overlay at right edge", () => { + // 720 - 30 - 250 = 440 + expect(calculateOverlayCoordinates("top-right", 0)).toEqual({ x: 440, y: 30 }); + }); + + it("top-right: second overlay stacks below", () => { + expect(calculateOverlayCoordinates("top-right", 1)).toEqual({ x: 440, y: 300 }); + }); + + it("bottom-left: first overlay near bottom edge", () => { + // 1280 - 30 - 250 = 1000 + expect(calculateOverlayCoordinates("bottom-left", 0)).toEqual({ x: 30, y: 1000 }); + }); + + it("bottom-left: second overlay stacks upward", () => { + // 1000 - 270 = 730 + expect(calculateOverlayCoordinates("bottom-left", 1)).toEqual({ x: 30, y: 730 }); + }); + + it("bottom-right: first overlay at bottom-right corner", () => { + expect(calculateOverlayCoordinates("bottom-right", 0)).toEqual({ x: 440, y: 1000 }); + }); + + it("bottom-right: second overlay stacks upward", () => { + expect(calculateOverlayCoordinates("bottom-right", 1)).toEqual({ x: 440, y: 730 }); + }); + + it("respects custom overlay size for positioning", () => { + // 720 - 30 - 150 = 540, 1280 - 30 - 150 = 1100 + expect(calculateOverlayCoordinates("bottom-right", 0, 150)).toEqual({ x: 540, y: 1100 }); + }); + + it("stacks with custom overlay size", () => { + // 1100 - (150 + 20) = 930 + expect(calculateOverlayCoordinates("bottom-right", 1, 150)).toEqual({ x: 540, y: 930 }); + }); +}); diff --git a/src/content/buildFfmpegArgs.ts b/src/content/buildFfmpegArgs.ts index 8e553a4..d9e022a 100644 --- a/src/content/buildFfmpegArgs.ts +++ b/src/content/buildFfmpegArgs.ts @@ -1,5 +1,6 @@ import { buildFilterComplex } from "./buildFilterComplex"; import { escapeDrawtext } from "./escapeDrawtext"; +import type { OverlayPosition } from "./overlayPosition"; /** Video frame dimensions */ const FRAME_HEIGHT = 1280; @@ -23,6 +24,7 @@ export function buildFfmpegArgs({ audioDurationSeconds, hasAudio, overlayImagePaths, + overlayPosition, }: { videoPath: string; audioPath: string; @@ -32,6 +34,7 @@ export function buildFfmpegArgs({ audioDurationSeconds: number; hasAudio: boolean; overlayImagePaths: string[]; + overlayPosition?: OverlayPosition; }): string[] { const { lines, fontSize, lineHeight, position } = captionLayout; @@ -85,6 +88,7 @@ export function buildFfmpegArgs({ const filterComplex = buildFilterComplex({ overlayCount: overlayImagePaths.length, captionFilters, + overlayPosition, }); args.push("-filter_complex", filterComplex); diff --git a/src/content/buildFilterComplex.ts b/src/content/buildFilterComplex.ts index 673858f..e7a73f7 100644 --- a/src/content/buildFilterComplex.ts +++ b/src/content/buildFilterComplex.ts @@ -1,11 +1,7 @@ -/** Video frame dimensions */ -const FRAME_WIDTH = 720; -const FRAME_HEIGHT = 1280; +import { calculateOverlayCoordinates, type OverlayPosition } from "./overlayPosition"; /** Overlay image size */ const OVERLAY_SIZE = 150; -const EDGE_PADDING = 30; -const OVERLAY_GAP = 20; /** * Builds the ffmpeg filter_complex string for video with overlay images and captions. @@ -14,14 +10,17 @@ const OVERLAY_GAP = 20; * * @param overlayCount - Number of overlay image inputs (starting at input index 1) * @param captionFilters - Array of drawtext filter strings for captions + * @param overlayPosition - Corner to place overlays (default: top-left) * @returns The complete filter_complex string */ export function buildFilterComplex({ overlayCount, captionFilters, + overlayPosition = "top-left", }: { overlayCount: number; captionFilters: string[]; + overlayPosition?: OverlayPosition; }): string { const parts: string[] = []; @@ -34,11 +33,10 @@ export function buildFilterComplex({ parts.push(`[${inputIdx}:v]scale=${OVERLAY_SIZE}:${OVERLAY_SIZE}[ovr_${i}]`); } - // Chain overlays — stacked vertically from top-left + // Chain overlays at the chosen position let prevLabel = "video_base"; for (let i = 0; i < overlayCount; i++) { - const x = EDGE_PADDING; - const y = EDGE_PADDING + i * (OVERLAY_SIZE + OVERLAY_GAP); + const { x, y } = calculateOverlayCoordinates(overlayPosition, i, OVERLAY_SIZE); const outLabel = i < overlayCount - 1 ? `ovr_out_${i}` : "ovr_final"; parts.push(`[${prevLabel}][ovr_${i}]overlay=${x}:${y}[${outLabel}]`); prevLabel = outLabel; diff --git a/src/content/buildStaticImageArgs.ts b/src/content/buildStaticImageArgs.ts new file mode 100644 index 0000000..e6bb8ba --- /dev/null +++ b/src/content/buildStaticImageArgs.ts @@ -0,0 +1,64 @@ +import { calculateOverlayCoordinates, type OverlayPosition } from "./overlayPosition"; + +/** Overlay image size */ +const OVERLAY_SIZE = 250; + +/** + * Builds ffmpeg arguments to render a static image with overlay images. + * + * Pipeline: crop 16:9 → 9:16, scale to 720×1280, overlay images at the chosen corner. + * Outputs a single PNG frame. + */ +export function buildStaticImageArgs({ + imagePath, + overlayImagePaths, + outputPath, + overlayPosition = "top-left", +}: { + imagePath: string; + overlayImagePaths: string[]; + outputPath: string; + overlayPosition?: OverlayPosition; +}): string[] { + const hasOverlays = overlayImagePaths.length > 0; + const args = ["-y"]; + + if (hasOverlays) { + args.push("-i", imagePath); + for (const p of overlayImagePaths) { + args.push("-i", p); + } + + const parts: string[] = []; + + // Crop + scale base image + parts.push("[0:v]crop=ih*9/16:ih,scale=720:1280[img_base]"); + + // Scale each overlay + for (let i = 0; i < overlayImagePaths.length; i++) { + parts.push(`[${1 + i}:v]scale=${OVERLAY_SIZE}:${OVERLAY_SIZE}[ovr_${i}]`); + } + + // Chain overlays at the chosen position + let prevLabel = "img_base"; + for (let i = 0; i < overlayImagePaths.length; i++) { + const { x, y } = calculateOverlayCoordinates(overlayPosition, i); + const outLabel = i < overlayImagePaths.length - 1 ? `ovr_out_${i}` : "out"; + parts.push(`[${prevLabel}][ovr_${i}]overlay=${x}:${y}[${outLabel}]`); + prevLabel = outLabel; + } + + args.push("-filter_complex", parts.join(";")); + args.push("-map", "[out]"); + args.push("-frames:v", "1", outputPath); + } else { + args.push( + "-i", imagePath, + "-vf", "crop=ih*9/16:ih,scale=720:1280", + "-frames:v", "1", + outputPath, + ); + } + + return args; +} diff --git a/src/content/overlayPosition.ts b/src/content/overlayPosition.ts new file mode 100644 index 0000000..95a79ee --- /dev/null +++ b/src/content/overlayPosition.ts @@ -0,0 +1,36 @@ +/** Supported overlay positions on the 9:16 frame. */ +export const OVERLAY_POSITIONS = ["top-left", "top-right", "bottom-left", "bottom-right"] as const; +export type OverlayPosition = (typeof OVERLAY_POSITIONS)[number]; + +/** Video frame dimensions */ +const FRAME_WIDTH = 720; +const FRAME_HEIGHT = 1280; + +/** Default overlay image size */ +export const DEFAULT_OVERLAY_SIZE = 250; +const EDGE_PADDING = 30; +const OVERLAY_GAP = 20; + +/** + * Calculates x,y coordinates for an overlay image at the given position. + * + * Overlays stack vertically from the chosen corner: + * - top-left / top-right: first overlay nearest the top, subsequent ones below + * - bottom-left / bottom-right: first overlay nearest the bottom, subsequent ones above + */ +export function calculateOverlayCoordinates( + position: OverlayPosition, + index: number, + overlaySize: number = DEFAULT_OVERLAY_SIZE, +): { x: number; y: number } { + const isRight = position === "top-right" || position === "bottom-right"; + const isBottom = position === "bottom-left" || position === "bottom-right"; + + const x = isRight ? FRAME_WIDTH - EDGE_PADDING - overlaySize : EDGE_PADDING; + + const y = isBottom + ? FRAME_HEIGHT - EDGE_PADDING - overlaySize - index * (overlaySize + OVERLAY_GAP) + : EDGE_PADDING + index * (overlaySize + OVERLAY_GAP); + + return { x, y }; +} diff --git a/src/content/renderFinalVideo.ts b/src/content/renderFinalVideo.ts index 063ad47..3b15e23 100644 --- a/src/content/renderFinalVideo.ts +++ b/src/content/renderFinalVideo.ts @@ -8,6 +8,7 @@ import { calculateCaptionLayout } from "./calculateCaptionLayout"; import { stripEmoji } from "./stripEmoji"; import { downloadOverlayImages } from "./downloadOverlayImages"; import { downloadMediaToFile } from "./downloadMediaToFile"; +import type { OverlayPosition } from "./overlayPosition"; import { runFfmpeg } from "./runFfmpeg"; import { uploadToFalStorage } from "./uploadToFalStorage"; @@ -19,6 +20,7 @@ export interface RenderFinalVideoInput { captionText: string; hasAudio: boolean; overlayImageUrls?: string[]; + overlayPosition?: OverlayPosition; } export interface RenderFinalVideoOutput { @@ -61,6 +63,7 @@ export async function renderFinalVideo( audioDurationSeconds: input.audioDurationSeconds, hasAudio: input.hasAudio, overlayImagePaths: overlayPaths, + overlayPosition: input.overlayPosition, }); logStep("Running ffmpeg render", true, { diff --git a/src/content/resolveOverlayPosition.ts b/src/content/resolveOverlayPosition.ts new file mode 100644 index 0000000..a0381a4 --- /dev/null +++ b/src/content/resolveOverlayPosition.ts @@ -0,0 +1,36 @@ +import { logStep } from "../sandboxes/logStep"; +import { createOverlayPositionAgent } from "../agents/createOverlayPositionAgent"; +import type { OverlayPosition } from "./overlayPosition"; + +/** + * Analyzes the editorial image to determine the best corner for overlay placement. + * Falls back to "top-left" on any failure. + */ +export async function resolveOverlayPosition( + editorialImageUrl: string, +): Promise { + try { + const agent = createOverlayPositionAgent(); + const { output } = await agent.generate({ + messages: [ + { + role: "user", + content: [ + { type: "image", image: editorialImageUrl }, + { type: "text", text: "Analyze this editorial image and determine the best corner to place overlay images." }, + ], + }, + ], + }); + + const position = output?.position ?? "top-left"; + logStep("Overlay position resolved", false, { editorialImageUrl, position }); + return position as OverlayPosition; + } catch (err) { + logStep("Overlay position analysis failed, defaulting to top-left", false, { + editorialImageUrl, + error: err instanceof Error ? err.message : String(err), + }); + return "top-left"; + } +} diff --git a/src/tasks/createContentTask.ts b/src/tasks/createContentTask.ts index cad1e64..3f34de7 100644 --- a/src/tasks/createContentTask.ts +++ b/src/tasks/createContentTask.ts @@ -15,6 +15,7 @@ import { generateVideo, } from "../recoup/contentApi"; import { resolveCaptionText } from "../content/resolveCaptionText"; +import { resolveOverlayPosition } from "../content/resolveOverlayPosition"; /** * Content-creation task — full pipeline that generates a social-ready video. @@ -92,6 +93,11 @@ export const createContentTask = schemaTask({ upscale: payload.upscale, }); + // --- Step 6b: Resolve overlay position (AI-based, editorial templates only) --- + const overlayPosition = template.usesImageOverlay && editorialImageUrl + ? await resolveOverlayPosition(editorialImageUrl) + : undefined; + // --- Step 7: Generate video (API) --- const motionPrompt = buildMotionPrompt(template); let audioUrl: string | undefined; @@ -141,6 +147,7 @@ export const createContentTask = schemaTask({ captionText, hasAudio: payload.lipsync, overlayImageUrls: template.usesImageOverlay ? additionalImageUrls : undefined, + overlayPosition, }); // --- Return result ---