feat: AI-based overlay image position for editorial templates

src/agents/createOverlayPositionAgent.ts

@@ -0,0 +1,32 @@
+import { ToolLoopAgent, Output, stepCountIs } from "ai";
+import { z } from "zod";
+import { IMAGE_DETECTION_MODEL } from "./imageDetectionModel";
+import { OVERLAY_POSITIONS } from "../content/overlayPosition";
+
+const overlayPositionSchema = z.object({
+  position: z.enum(OVERLAY_POSITIONS),
+});
+
+const instructions = `You analyze an editorial image to determine the best corner to place small overlay images (like playlist covers or DSP logos) so they do not obscure the subject.
+
+Choose exactly ONE of four positions: top-left, top-right, bottom-left, bottom-right.
+
+Rules:
+- Pick the corner that has the LEAST visual importance — the emptiest or most uniform area.
+- Avoid placing overlays on top of the subject's face, hands, or other key focal points.
+- If the subject is centered and all corners are roughly equal, prefer "top-left" as the default.
+- If the background is blurred or uniform on one side, prefer that side.
+- Consider that overlays will be 250×250 pixels stacked vertically in the chosen corner of a 720×1280 (9:16 portrait) frame.`;
+
+/**
+ * Creates a ToolLoopAgent that analyzes an editorial image and returns
+ * the best corner position for overlay images.
+ */
+export function createOverlayPositionAgent() {
+  return new ToolLoopAgent({
+    model: IMAGE_DETECTION_MODEL,
+    instructions,
+    output: Output.object({ schema: overlayPositionSchema }),
+    stopWhen: stepCountIs(1),
+  });
+}

src/content/__tests__/buildFilterComplex.test.ts

@@ -43,4 +43,45 @@ describe("buildFilterComplex", () => {
 
     expect(result).toContain("[out]");
   });
+
+  it("positions overlays in top-right when specified", () => {
+    const result = buildFilterComplex({
+      overlayCount: 1,
+      captionFilters: [],
+      overlayPosition: "top-right",
+    });
+
+    // 720 - 30 - 150 = 540
+    expect(result).toMatch(/overlay=540:30/);
+  });
+
+  it("positions overlays in bottom-left when specified", () => {
+    const result = buildFilterComplex({
+      overlayCount: 1,
+      captionFilters: [],
+      overlayPosition: "bottom-left",
+    });
+
+    // 1280 - 30 - 150 = 1100
+    expect(result).toMatch(/overlay=30:1100/);
+  });
+
+  it("positions overlays in bottom-right when specified", () => {
+    const result = buildFilterComplex({
+      overlayCount: 1,
+      captionFilters: [],
+      overlayPosition: "bottom-right",
+    });
+
+    expect(result).toMatch(/overlay=540:1100/);
+  });
+
+  it("defaults to top-left when no overlayPosition provided", () => {
+    const result = buildFilterComplex({
+      overlayCount: 1,
+      captionFilters: [],
+    });
+
+    expect(result).toMatch(/overlay=30:30/);
+  });
 });

src/content/__tests__/overlayPosition.test.ts

@@ -0,0 +1,50 @@
+import { describe, it, expect } from "vitest";
+
+import { calculateOverlayCoordinates } from "../overlayPosition";
+
+describe("calculateOverlayCoordinates", () => {
+  it("top-left: first overlay at (30, 30)", () => {
+    expect(calculateOverlayCoordinates("top-left", 0)).toEqual({ x: 30, y: 30 });
+  });
+
+  it("top-left: second overlay stacks below", () => {
+    expect(calculateOverlayCoordinates("top-left", 1)).toEqual({ x: 30, y: 300 });
+  });
+
+  it("top-right: first overlay at right edge", () => {
+    // 720 - 30 - 250 = 440
+    expect(calculateOverlayCoordinates("top-right", 0)).toEqual({ x: 440, y: 30 });
+  });
+
+  it("top-right: second overlay stacks below", () => {
+    expect(calculateOverlayCoordinates("top-right", 1)).toEqual({ x: 440, y: 300 });
+  });
+
+  it("bottom-left: first overlay near bottom edge", () => {
+    // 1280 - 30 - 250 = 1000
+    expect(calculateOverlayCoordinates("bottom-left", 0)).toEqual({ x: 30, y: 1000 });
+  });
+
+  it("bottom-left: second overlay stacks upward", () => {
+    // 1000 - 270 = 730
+    expect(calculateOverlayCoordinates("bottom-left", 1)).toEqual({ x: 30, y: 730 });
+  });
+
+  it("bottom-right: first overlay at bottom-right corner", () => {
+    expect(calculateOverlayCoordinates("bottom-right", 0)).toEqual({ x: 440, y: 1000 });
+  });
+
+  it("bottom-right: second overlay stacks upward", () => {
+    expect(calculateOverlayCoordinates("bottom-right", 1)).toEqual({ x: 440, y: 730 });
+  });
+
+  it("respects custom overlay size for positioning", () => {
+    // 720 - 30 - 150 = 540, 1280 - 30 - 150 = 1100
+    expect(calculateOverlayCoordinates("bottom-right", 0, 150)).toEqual({ x: 540, y: 1100 });
+  });
+
+  it("stacks with custom overlay size", () => {
+    // 1100 - (150 + 20) = 930
+    expect(calculateOverlayCoordinates("bottom-right", 1, 150)).toEqual({ x: 540, y: 930 });
+  });
+});

src/content/buildFfmpegArgs.ts

@@ -1,5 +1,6 @@
 import { buildFilterComplex } from "./buildFilterComplex";
 import { escapeDrawtext } from "./escapeDrawtext";
+import type { OverlayPosition } from "./overlayPosition";
 
 /** Video frame dimensions */
 const FRAME_HEIGHT = 1280;
@@ -23,6 +24,7 @@ export function buildFfmpegArgs({
   audioDurationSeconds,
   hasAudio,
   overlayImagePaths,
+  overlayPosition,
 }: {
   videoPath: string;
   audioPath: string;
@@ -32,6 +34,7 @@ export function buildFfmpegArgs({
   audioDurationSeconds: number;
   hasAudio: boolean;
   overlayImagePaths: string[];
+  overlayPosition?: OverlayPosition;
 }): string[] {
   const { lines, fontSize, lineHeight, position } = captionLayout;
 
@@ -85,6 +88,7 @@ export function buildFfmpegArgs({
     const filterComplex = buildFilterComplex({
       overlayCount: overlayImagePaths.length,
       captionFilters,
+      overlayPosition,
     });
 
     args.push("-filter_complex", filterComplex);

src/content/buildFilterComplex.ts

@@ -1,11 +1,7 @@
-/** Video frame dimensions */
-const FRAME_WIDTH = 720;
-const FRAME_HEIGHT = 1280;
+import { calculateOverlayCoordinates, type OverlayPosition } from "./overlayPosition";
 
 /** Overlay image size */
 const OVERLAY_SIZE = 150;
-const EDGE_PADDING = 30;
-const OVERLAY_GAP = 20;
 
 /**
  * Builds the ffmpeg filter_complex string for video with overlay images and captions.
@@ -14,14 +10,17 @@ const OVERLAY_GAP = 20;
  *
  * @param overlayCount - Number of overlay image inputs (starting at input index 1)
  * @param captionFilters - Array of drawtext filter strings for captions
+ * @param overlayPosition - Corner to place overlays (default: top-left)
  * @returns The complete filter_complex string
  */
 export function buildFilterComplex({
   overlayCount,
   captionFilters,
+  overlayPosition = "top-left",
 }: {
   overlayCount: number;
   captionFilters: string[];
+  overlayPosition?: OverlayPosition;
 }): string {
   const parts: string[] = [];
 
@@ -34,11 +33,10 @@ export function buildFilterComplex({
     parts.push(`[${inputIdx}:v]scale=${OVERLAY_SIZE}:${OVERLAY_SIZE}[ovr_${i}]`);
   }
 
-  // Chain overlays — stacked vertically from top-left
+  // Chain overlays at the chosen position
   let prevLabel = "video_base";
   for (let i = 0; i < overlayCount; i++) {
-    const x = EDGE_PADDING;
-    const y = EDGE_PADDING + i * (OVERLAY_SIZE + OVERLAY_GAP);
+    const { x, y } = calculateOverlayCoordinates(overlayPosition, i, OVERLAY_SIZE);
     const outLabel = i < overlayCount - 1 ? `ovr_out_${i}` : "ovr_final";
     parts.push(`[${prevLabel}][ovr_${i}]overlay=${x}:${y}[${outLabel}]`);
     prevLabel = outLabel;

src/content/buildStaticImageArgs.ts

@@ -0,0 +1,64 @@
+import { calculateOverlayCoordinates, type OverlayPosition } from "./overlayPosition";
+
+/** Overlay image size */
+const OVERLAY_SIZE = 250;
+
+/**
+ * Builds ffmpeg arguments to render a static image with overlay images.
+ *
+ * Pipeline: crop 16:9 → 9:16, scale to 720×1280, overlay images at the chosen corner.
+ * Outputs a single PNG frame.
+ */
+export function buildStaticImageArgs({
+  imagePath,
+  overlayImagePaths,
+  outputPath,
+  overlayPosition = "top-left",
+}: {
+  imagePath: string;
+  overlayImagePaths: string[];
+  outputPath: string;
+  overlayPosition?: OverlayPosition;
+}): string[] {
+  const hasOverlays = overlayImagePaths.length > 0;
+  const args = ["-y"];
+
+  if (hasOverlays) {
+    args.push("-i", imagePath);
+    for (const p of overlayImagePaths) {
+      args.push("-i", p);
+    }
+
+    const parts: string[] = [];
+
+    // Crop + scale base image
+    parts.push("[0:v]crop=ih*9/16:ih,scale=720:1280[img_base]");
+
+    // Scale each overlay
+    for (let i = 0; i < overlayImagePaths.length; i++) {
+      parts.push(`[${1 + i}:v]scale=${OVERLAY_SIZE}:${OVERLAY_SIZE}[ovr_${i}]`);
+    }
+
+    // Chain overlays at the chosen position
+    let prevLabel = "img_base";
+    for (let i = 0; i < overlayImagePaths.length; i++) {
+      const { x, y } = calculateOverlayCoordinates(overlayPosition, i);
+      const outLabel = i < overlayImagePaths.length - 1 ? `ovr_out_${i}` : "out";
+      parts.push(`[${prevLabel}][ovr_${i}]overlay=${x}:${y}[${outLabel}]`);
+      prevLabel = outLabel;
+    }
+
+    args.push("-filter_complex", parts.join(";"));
+    args.push("-map", "[out]");
+    args.push("-frames:v", "1", outputPath);
+  } else {
+    args.push(
+      "-i", imagePath,
+      "-vf", "crop=ih*9/16:ih,scale=720:1280",
+      "-frames:v", "1",
+      outputPath,
+    );
+  }
+
+  return args;
+}

src/content/overlayPosition.ts

@@ -0,0 +1,36 @@
+/** Supported overlay positions on the 9:16 frame. */
+export const OVERLAY_POSITIONS = ["top-left", "top-right", "bottom-left", "bottom-right"] as const;
+export type OverlayPosition = (typeof OVERLAY_POSITIONS)[number];
+
+/** Video frame dimensions */
+const FRAME_WIDTH = 720;
+const FRAME_HEIGHT = 1280;
+
+/** Default overlay image size */
+export const DEFAULT_OVERLAY_SIZE = 250;
+const EDGE_PADDING = 30;
+const OVERLAY_GAP = 20;
+
+/**
+ * Calculates x,y coordinates for an overlay image at the given position.
+ *
+ * Overlays stack vertically from the chosen corner:
+ * - top-left / top-right: first overlay nearest the top, subsequent ones below
+ * - bottom-left / bottom-right: first overlay nearest the bottom, subsequent ones above
+ */
+export function calculateOverlayCoordinates(
+  position: OverlayPosition,
+  index: number,
+  overlaySize: number = DEFAULT_OVERLAY_SIZE,
+): { x: number; y: number } {
+  const isRight = position === "top-right" || position === "bottom-right";
+  const isBottom = position === "bottom-left" || position === "bottom-right";
+
+  const x = isRight ? FRAME_WIDTH - EDGE_PADDING - overlaySize : EDGE_PADDING;
+
+  const y = isBottom
+    ? FRAME_HEIGHT - EDGE_PADDING - overlaySize - index * (overlaySize + OVERLAY_GAP)
+    : EDGE_PADDING + index * (overlaySize + OVERLAY_GAP);
+
+  return { x, y };
+}

src/content/renderFinalVideo.ts

@@ -8,6 +8,7 @@ import { calculateCaptionLayout } from "./calculateCaptionLayout";
 import { stripEmoji } from "./stripEmoji";
 import { downloadOverlayImages } from "./downloadOverlayImages";
 import { downloadMediaToFile } from "./downloadMediaToFile";
+import type { OverlayPosition } from "./overlayPosition";
 import { runFfmpeg } from "./runFfmpeg";
 import { uploadToFalStorage } from "./uploadToFalStorage";
 
@@ -19,6 +20,7 @@ export interface RenderFinalVideoInput {
   captionText: string;
   hasAudio: boolean;
   overlayImageUrls?: string[];
+  overlayPosition?: OverlayPosition;
 }
 
 export interface RenderFinalVideoOutput {
@@ -61,6 +63,7 @@ export async function renderFinalVideo(
       audioDurationSeconds: input.audioDurationSeconds,
       hasAudio: input.hasAudio,
       overlayImagePaths: overlayPaths,
+      overlayPosition: input.overlayPosition,
     });
 
     logStep("Running ffmpeg render", true, {

src/content/resolveOverlayPosition.ts

@@ -0,0 +1,36 @@
+import { logStep } from "../sandboxes/logStep";
+import { createOverlayPositionAgent } from "../agents/createOverlayPositionAgent";
+import type { OverlayPosition } from "./overlayPosition";
+
+/**
+ * Analyzes the editorial image to determine the best corner for overlay placement.
+ * Falls back to "top-left" on any failure.
+ */
+export async function resolveOverlayPosition(
+  editorialImageUrl: string,
+): Promise<OverlayPosition> {
+  try {
+    const agent = createOverlayPositionAgent();
+    const { output } = await agent.generate({
+      messages: [
+        {
+          role: "user",
+          content: [
+            { type: "image", image: editorialImageUrl },
+            { type: "text", text: "Analyze this editorial image and determine the best corner to place overlay images." },
+          ],
+        },
+      ],
+    });
+
+    const position = output?.position ?? "top-left";
+    logStep("Overlay position resolved", false, { editorialImageUrl, position });
+    return position as OverlayPosition;
+  } catch (err) {
+    logStep("Overlay position analysis failed, defaulting to top-left", false, {
+      editorialImageUrl,
+      error: err instanceof Error ? err.message : String(err),
+    });
+    return "top-left";
+  }
+}

src/tasks/createContentTask.ts

@@ -15,6 +15,7 @@ import {
   generateVideo,
 } from "../recoup/contentApi";
 import { resolveCaptionText } from "../content/resolveCaptionText";
+import { resolveOverlayPosition } from "../content/resolveOverlayPosition";
 
 /**
  * Content-creation task — full pipeline that generates a social-ready video.
@@ -92,6 +93,11 @@ export const createContentTask = schemaTask({
       upscale: payload.upscale,
     });
 
+    // --- Step 6b: Resolve overlay position (AI-based, editorial templates only) ---
+    const overlayPosition = template.usesImageOverlay && editorialImageUrl
+      ? await resolveOverlayPosition(editorialImageUrl)
+      : undefined;
+
     // --- Step 7: Generate video (API) ---
     const motionPrompt = buildMotionPrompt(template);
     let audioUrl: string | undefined;
@@ -141,6 +147,7 @@ export const createContentTask = schemaTask({
       captionText,
       hasAudio: payload.lipsync,
       overlayImageUrls: template.usesImageOverlay ? additionalImageUrls : undefined,
+      overlayPosition,
     });
 
     // --- Return result ---