feat(apify): one consolidated new-posts digest per scrape batch

lib/apify/__tests__/apifyWebhookHandler.test.ts

@@ -1,4 +1,6 @@
 import { describe, it, expect, vi, beforeEach } from "vitest";
+import { completeApifyScraperRun } from "@/lib/supabase/apify_scraper_runs/completeApifyScraperRun";
+import { maybeSendScrapeDigest } from "@/lib/apify/digest/maybeSendScrapeDigest";
 import { NextRequest } from "next/server";
 import { apifyWebhookHandler } from "../apifyWebhookHandler";
 import { handleInstagramProfileScraperResults } from "../instagram/handleInstagramProfileScraperResults";
@@ -44,6 +46,13 @@ const baseBody = {
   resource: { defaultDatasetId: "ds_1" },
 };
 
+vi.mock("@/lib/supabase/apify_scraper_runs/completeApifyScraperRun", () => ({
+  completeApifyScraperRun: vi.fn(async () => null),
+}));
+vi.mock("@/lib/apify/digest/maybeSendScrapeDigest", () => ({
+  maybeSendScrapeDigest: vi.fn(async () => null),
+}));
+
 describe("apifyWebhookHandler", () => {
   beforeEach(() => vi.clearAllMocks());
 
@@ -60,6 +69,38 @@ describe("apifyWebhookHandler", () => {
     expect(handleInstagramCommentsScraper).not.toHaveBeenCalled();
   });
 
+  it("records batch completion + triggers the digest check when the payload carries a run id (chat#1855)", async () => {
+    vi.mocked(handleInstagramProfileScraperResults).mockResolvedValue({
+      posts: [],
+      newPostUrls: ["https://instagram.com/p/new1"],
+    } as never);
+    vi.mocked(completeApifyScraperRun).mockResolvedValue({
+      run_id: "run-9",
+      batch_id: "batch-7",
+    } as never);
+
+    const res = await apifyWebhookHandler(
+      makeRequest({
+        ...baseBody,
+        eventData: { actorId: "dSCLg0C3YEZ83HzYX" },
+        resource: { ...baseBody.resource, id: "run-9" },
+      }),
+    );
+
+    expect(res.status).toBe(200);
+    expect(completeApifyScraperRun).toHaveBeenCalledWith("run-9", ["https://instagram.com/p/new1"]);
+    expect(maybeSendScrapeDigest).toHaveBeenCalledWith("batch-7");
+  });
+
+  it("skips digest bookkeeping when the payload has no run id", async () => {
+    vi.mocked(handleInstagramProfileScraperResults).mockResolvedValue({ posts: [] } as never);
+    await apifyWebhookHandler(
+      makeRequest({ ...baseBody, eventData: { actorId: "dSCLg0C3YEZ83HzYX" } }),
+    );
+    expect(completeApifyScraperRun).not.toHaveBeenCalled();
+    expect(maybeSendScrapeDigest).not.toHaveBeenCalled();
+  });
+
   it("dispatches comments scraper for the IG comments actor", async () => {
     vi.mocked(handleInstagramCommentsScraper).mockResolvedValue({
       comments: [],

lib/apify/apifyWebhookHandler.ts

@@ -1,6 +1,8 @@
 import { NextRequest, NextResponse } from "next/server";
 import { validateApifyWebhookRequest } from "@/lib/apify/validateApifyWebhookRequest";
 import { getApifyResultHandler } from "@/lib/apify/getApifyResultHandler";
+import { completeApifyScraperRun } from "@/lib/supabase/apify_scraper_runs/completeApifyScraperRun";
+import { maybeSendScrapeDigest } from "@/lib/apify/digest/maybeSendScrapeDigest";
 
 /**
  * Handler for `POST /api/apify`. Always responds 200 so Apify does not
@@ -27,6 +29,22 @@ export async function apifyWebhookHandler(request: NextRequest): Promise<NextRes
 
   try {
     const result = await handler(validated);
+
+    // Digest-batch bookkeeping (chat#1855): record this run's genuinely-new
+    // posts and, when it was the batch's last completion, send the single
+    // consolidated digest. Never fails the webhook.
+    const runId = validated.resource.id;
+    if (runId) {
+      try {
+        const newPostUrls =
+          (result as { newPostUrls?: string[] } | null | undefined)?.newPostUrls ?? [];
+        const run = await completeApifyScraperRun(runId, newPostUrls);
+        await maybeSendScrapeDigest(run?.batch_id);
+      } catch (digestError) {
+        console.error("[WARN] scrape digest bookkeeping failed:", digestError);
+      }
+    }
+
     return NextResponse.json(result, { status: 200 });
   } catch (error) {
     console.error("[ERROR] apifyWebhookHandler:", error);

lib/apify/digest/__tests__/maybeSendScrapeDigest.test.ts

@@ -0,0 +1,73 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { maybeSendScrapeDigest } from "@/lib/apify/digest/maybeSendScrapeDigest";
+import { selectApifyScraperRunsByBatch } from "@/lib/supabase/apify_scraper_runs/selectApifyScraperRunsByBatch";
+import { getScrapeDigestRecipients } from "@/lib/apify/digest/getScrapeDigestRecipients";
+import { sendScrapeDigestEmail } from "@/lib/apify/digest/sendScrapeDigestEmail";
+
+vi.mock("@/lib/supabase/apify_scraper_runs/selectApifyScraperRunsByBatch", () => ({
+  selectApifyScraperRunsByBatch: vi.fn(),
+}));
+vi.mock("@/lib/apify/digest/getScrapeDigestRecipients", () => ({
+  getScrapeDigestRecipients: vi.fn(),
+}));
+vi.mock("@/lib/apify/digest/sendScrapeDigestEmail", () => ({
+  sendScrapeDigestEmail: vi.fn(),
+}));
+
+const run = (over: Record<string, unknown>) => ({
+  run_id: "r1",
+  batch_id: "b1",
+  account_id: "acct-1",
+  social_id: "s1",
+  platform: "instagram",
+  completed_at: "2026-07-07T00:00:00Z",
+  new_post_urls: [],
+  ...over,
+});
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  vi.mocked(getScrapeDigestRecipients).mockResolvedValue(["owner@example.com"]);
+  vi.mocked(sendScrapeDigestEmail).mockResolvedValue({ id: "email-1" } as never);
+});
+
+describe("maybeSendScrapeDigest", () => {
+  it("does nothing while sibling runs are still incomplete", async () => {
+    vi.mocked(selectApifyScraperRunsByBatch).mockResolvedValue([
+      run({}),
+      run({ run_id: "r2", platform: "tiktok", completed_at: null }),
+    ] as never);
+    expect(await maybeSendScrapeDigest("b1")).toBeNull();
+    expect(sendScrapeDigestEmail).not.toHaveBeenCalled();
+  });
+
+  it("sends ONE digest with per-platform new posts when the batch completes", async () => {
+    vi.mocked(selectApifyScraperRunsByBatch).mockResolvedValue([
+      run({ new_post_urls: ["https://instagram.com/p/1"] }),
+      run({ run_id: "r2", platform: "tiktok", new_post_urls: ["https://tiktok.com/v/2"] }),
+      run({ run_id: "r3", platform: "x", new_post_urls: [] }),
+    ] as never);
+    await maybeSendScrapeDigest("b1");
+    expect(sendScrapeDigestEmail).toHaveBeenCalledOnce();
+    const arg = vi.mocked(sendScrapeDigestEmail).mock.calls[0][0];
+    expect(arg.sections).toEqual([
+      { platform: "instagram", postUrls: ["https://instagram.com/p/1"] },
+      { platform: "tiktok", postUrls: ["https://tiktok.com/v/2"] },
+    ]); // x omitted — nothing new
+    expect(arg.emails).toEqual(["owner@example.com"]);
+  });
+
+  it("sends nothing when the batch completes with zero new posts anywhere", async () => {
+    vi.mocked(selectApifyScraperRunsByBatch).mockResolvedValue([
+      run({}),
+      run({ run_id: "r2", platform: "tiktok" }),
+    ] as never);
+    expect(await maybeSendScrapeDigest("b1")).toBeNull();
+    expect(sendScrapeDigestEmail).not.toHaveBeenCalled();
+  });
+
+  it("no-ops for a null batch id (legacy runs)", async () => {
+    expect(await maybeSendScrapeDigest(null)).toBeNull();
+    expect(selectApifyScraperRunsByBatch).not.toHaveBeenCalled();
+  });
+});

lib/apify/digest/getScrapeDigestRecipients.ts

@@ -0,0 +1,28 @@
+import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccountSocials";
+import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccountArtistIds";
+import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails";
+
+/**
+ * Resolves the digest recipients for a set of scraped socials: every owner
+ * of every artist watching any of them (same chain the per-platform alert
+ * used). Recipients span tenants — senders must BCC (chat#1855).
+ */
+export async function getScrapeDigestRecipients(socialIds: string[]): Promise<string[]> {
+  const uniqueSocialIds = Array.from(new Set(socialIds.filter(Boolean)));
+  if (!uniqueSocialIds.length) return [];
+
+  const accountSocials = (
+    await Promise.all(
+      uniqueSocialIds.map(socialId => selectAccountSocials({ socialId, limit: 10000 })),
+    )
+  ).flat();
+
+  const accountArtistIds = await getAccountArtistIds({
+    artistIds: accountSocials.map(a => a.account_id),
+  });
+  const uniqueAccountIds = Array.from(
+    new Set(accountArtistIds.map(a => a.account_id).filter((id): id is string => Boolean(id))),
+  );
+  const accountEmails = await selectAccountEmails({ accountIds: uniqueAccountIds });
+  return Array.from(new Set(accountEmails.map(e => e.email).filter(Boolean)));
+}

lib/apify/digest/maybeSendScrapeDigest.ts

@@ -0,0 +1,27 @@
+import { selectApifyScraperRunsByBatch } from "@/lib/supabase/apify_scraper_runs/selectApifyScraperRunsByBatch";
+import { getScrapeDigestRecipients } from "@/lib/apify/digest/getScrapeDigestRecipients";
+import { sendScrapeDigestEmail } from "@/lib/apify/digest/sendScrapeDigestEmail";
+
+/**
+ * Batch-completion check for the one-digest-per-scrape design (chat#1855):
+ * called after each platform run's results are processed. Sends the single
+ * consolidated digest when (a) every sibling run in the batch has completed
+ * and (b) at least one platform found genuinely new posts. Platforms with
+ * nothing new are omitted; a batch with nothing new sends nothing.
+ */
+export async function maybeSendScrapeDigest(batchId: string | null | undefined) {
+  if (!batchId) return null;
+
+  const runs = await selectApifyScraperRunsByBatch(batchId);
+  if (!runs.length || runs.some(r => !r.completed_at)) return null;
+
+  const sections = runs
+    .filter(r => (r.new_post_urls?.length ?? 0) > 0)
+    .map(r => ({ platform: r.platform ?? "other", postUrls: r.new_post_urls ?? [] }));
+  if (!sections.length) return null;
+
+  const emails = await getScrapeDigestRecipients(
+    runs.map(r => r.social_id).filter((id): id is string => Boolean(id)),
+  );
+  return await sendScrapeDigestEmail({ emails, sections });
+}

lib/apify/digest/sendScrapeDigestEmail.ts

@@ -0,0 +1,38 @@
+import { sendEmailWithResend } from "@/lib/emails/sendEmail";
+import { RECOUP_FROM_EMAIL } from "@/lib/const";
+
+export type ScrapeDigestSection = { platform: string; postUrls: string[] };
+export type ScrapeDigestInput = {
+  emails: string[];
+  sections: ScrapeDigestSection[];
+  artistName?: string | null;
+};
+
+/**
+ * Sends the consolidated new-posts digest: one email per scrape batch,
+ * one section per platform that found genuinely new posts. Deterministic
+ * body, direct links to each new post. Recipients span tenants — BCC only
+ * (chat#1855).
+ */
+export async function sendScrapeDigestEmail({ emails, sections, artistName }: ScrapeDigestInput) {
+  if (!emails.length || !sections.length) return null;
+
+  const total = sections.reduce((n, s) => n + s.postUrls.length, 0);
+  const who = artistName || "your artist";
+  const sectionHtml = sections
+    .map(
+      s =>
+        `<h3 style="margin:16px 0 4px">${s.platform}</h3><ul>${s.postUrls
+          .map(u => `<li><a href="${u.startsWith("http") ? u : `https://${u}`}">${u}</a></li>`)
+          .join("")}</ul>`,
+    )
+    .join("");
+
+  return await sendEmailWithResend({
+    from: RECOUP_FROM_EMAIL,
+    to: [RECOUP_FROM_EMAIL],
+    bcc: emails,
+    subject: `${who}: ${total} new post${total === 1 ? "" : "s"} found across ${sections.length} platform${sections.length === 1 ? "" : "s"}`,
+    html: `<p>New posts found for ${who}:</p>${sectionHtml}`,
+  });
+}

lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts

@@ -13,6 +13,7 @@ import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccoun
 import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails";
 import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave";
 import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls";
+import { selectApifyScraperRun } from "@/lib/supabase/apify_scraper_runs/selectApifyScraperRun";
 
 vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn() } }));
 
@@ -28,6 +29,9 @@ vi.mock("../handleInstagramProfileFollowUpRuns", () => ({
 }));
 vi.mock("@/lib/apify/sendApifyWebhookEmail", () => ({ sendApifyWebhookEmail: vi.fn() }));
 vi.mock("@/lib/socials/filterNewPostUrls", () => ({ filterNewPostUrls: vi.fn() }));
+vi.mock("@/lib/supabase/apify_scraper_runs/selectApifyScraperRun", () => ({
+  selectApifyScraperRun: vi.fn(async () => null),
+}));
 vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ upsertSocials: vi.fn() }));
 vi.mock("@/lib/supabase/socials/selectSocials", () => ({
   selectSocials: vi.fn(),
@@ -128,4 +132,36 @@ describe("handleInstagramProfileScraperResults", () => {
     expect(handleInstagramProfileFollowUpRuns).toHaveBeenCalledOnce();
     expect(result.social).toEqual({ id: "s1" });
   });
+
+  it("suppresses the solo email for digest-batch runs (webhook layer sends ONE digest)", async () => {
+    mockDataset([
+      {
+        latestPosts: [{ url: "u-new", timestamp: "t" }],
+        username: "alice",
+        url: "instagram.com/alice",
+        profilePicUrl: "https://a",
+        fullName: "Alice",
+      },
+    ]);
+    vi.mocked(selectApifyScraperRun).mockResolvedValue({
+      run_id: "run-1",
+      batch_id: "b1",
+    } as never);
+    vi.mocked(upsertPosts).mockResolvedValue({ data: null, error: null } as never);
+    vi.mocked(getPosts).mockResolvedValue([{ id: "p1", post_url: "u-new" }] as never);
+    vi.mocked(uploadLinkToArweave).mockResolvedValue(null);
+    vi.mocked(upsertSocials).mockResolvedValue([] as never);
+    vi.mocked(selectSocials).mockResolvedValue([{ id: "s1" }] as never);
+    vi.mocked(selectAccountSocials).mockResolvedValue([{ account_id: "a1" }] as never);
+    vi.mocked(getAccountArtistIds).mockResolvedValue([{ account_id: "a1" }] as never);
+    vi.mocked(selectAccountEmails).mockResolvedValue([{ email: "x@y.com" }] as never);
+
+    const result = await handleInstagramProfileScraperResults({
+      ...(payload as Record<string, unknown>),
+      resource: { defaultDatasetId: "ds_1", id: "run-1" },
+    } as never);
+
+    expect(sendApifyWebhookEmail).not.toHaveBeenCalled(); // digest covers it
+    expect(result.newPostUrls).toEqual(["u-new"]);
+  });
 });

lib/apify/instagram/handleInstagramProfileScraperResults.ts

@@ -15,6 +15,7 @@ import { getFetchableUrl } from "@/lib/arweave/getFetchableUrl";
 import type { ApifyInstagramProfileResult } from "@/lib/apify/types";
 import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest";
 import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls";
+import { selectApifyScraperRun } from "@/lib/supabase/apify_scraper_runs/selectApifyScraperRun";
 import type { TablesInsert } from "@/types/database.types";
 
 /**
@@ -98,11 +99,16 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP
 
   // Email + follow-up scrape are independent side effects; isolate so a
   // mail outage doesn't block comment scraping and vice versa.
+  // Digest-batch runs get ONE consolidated email from the webhook layer —
+  // suppress the per-platform solo email for them (chat#1855). Legacy runs
+  // (no batch registration) keep the immediate alert.
+  const registeredRun = parsed.resource.id ? await selectApifyScraperRun(parsed.resource.id) : null;
+
   let sentEmails = null;
   try {
     // Only notify when the scrape actually found posts new to the platform —
     // otherwise every scrape re-announces the profile's recent feed.
-    if (newPostUrls.length > 0) {
+    if (newPostUrls.length > 0 && !registeredRun?.batch_id) {
       sentEmails = await sendApifyWebhookEmail(
         firstResult,
         accountEmails.map(e => e.email).filter(Boolean),
@@ -118,5 +124,5 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP
     console.error("[WARN] follow-up scrape failed:", error);
   }
 
-  return { posts, social, accountSocials, accountEmails, sentEmails };
+  return { posts, social, accountSocials, accountEmails, sentEmails, newPostUrls };
 }

lib/apify/scrapeProfileUrlBatch.ts

@@ -1,5 +1,7 @@
 import { ProfileScrapeResult, ScrapeProfileResult, scrapeProfileUrl } from "./scrapeProfileUrl";
 
+export type BatchProfileScrapeResult = ProfileScrapeResult & { profileUrl: string | null };
+
 type ScrapeProfileUrlBatchInput = {
   profileUrl: string | null | undefined;
   username: string | null | undefined;
@@ -8,18 +10,22 @@ type ScrapeProfileUrlBatchInput = {
 export const scrapeProfileUrlBatch = async (
   inputs: ScrapeProfileUrlBatchInput[],
   posts?: number,
-): Promise<ProfileScrapeResult[]> => {
+): Promise<BatchProfileScrapeResult[]> => {
   const results = await Promise.all(
-    inputs.map(({ profileUrl, username }) =>
-      scrapeProfileUrl(profileUrl ?? null, username ?? "", posts),
-    ),
+    inputs.map(async ({ profileUrl, username }) => {
+      const result = await scrapeProfileUrl(profileUrl ?? null, username ?? "", posts);
+      return result ? { ...result, profileUrl: profileUrl ?? null } : null;
+    }),
   );
 
   return results
-    .filter((result): result is ScrapeProfileResult => result !== null)
-    .map(({ runId, datasetId, error }) => ({
+    .filter(
+      (result): result is ScrapeProfileResult & { profileUrl: string | null } => result !== null,
+    )
+    .map(({ runId, datasetId, error, profileUrl }) => ({
       runId,
       datasetId,
       error,
+      profileUrl,
     }));
 };

lib/apify/tiktok/__tests__/handleTiktokProfileScraperResults.test.ts

@@ -1,6 +1,9 @@
 import { describe, it, expect, vi, beforeEach } from "vitest";
 import { handleTiktokProfileScraperResults } from "@/lib/apify/tiktok/handleTiktokProfileScraperResults";
 const listItems = vi.fn();
+vi.mock("@/lib/socials/filterNewPostUrls", () => ({
+  filterNewPostUrls: vi.fn(async (urls: string[]) => urls),
+}));
 vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn(() => ({ listItems })) } }));
 const upsertSocials = vi.fn();
 vi.mock("@/lib/supabase/socials/upsertSocials", () => ({