feat(apify): scrape alerts fire only on genuinely new posts (diff against stored posts)

lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts

@@ -12,6 +12,7 @@ import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccou
 import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccountArtistIds";
 import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails";
 import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave";
+import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls";
 
 vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn() } }));
 
@@ -26,6 +27,7 @@ vi.mock("../handleInstagramProfileFollowUpRuns", () => ({
   handleInstagramProfileFollowUpRuns: vi.fn(),
 }));
 vi.mock("@/lib/apify/sendApifyWebhookEmail", () => ({ sendApifyWebhookEmail: vi.fn() }));
+vi.mock("@/lib/socials/filterNewPostUrls", () => ({ filterNewPostUrls: vi.fn() }));
 vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ upsertSocials: vi.fn() }));
 vi.mock("@/lib/supabase/socials/selectSocials", () => ({
   selectSocials: vi.fn(),
@@ -51,7 +53,11 @@ const payload = {
 } as never;
 
 describe("handleInstagramProfileScraperResults", () => {
-  beforeEach(() => vi.clearAllMocks());
+  beforeEach(() => {
+    vi.clearAllMocks();
+    // default: everything scraped counts as new (per-test overrides below)
+    vi.mocked(filterNewPostUrls).mockImplementation(async urls => urls);
+  });
 
   it("short-circuits when the dataset has no latest posts", async () => {
     mockDataset([{ username: "alice" }]);
@@ -92,4 +98,34 @@ describe("handleInstagramProfileScraperResults", () => {
     expect(result.social).toEqual({ id: "s1" });
     expect(result.posts).toEqual(posts);
   });
+
+  it("skips the alert email when no scraped post is genuinely new (chat#1855)", async () => {
+    mockDataset([
+      {
+        latestPosts: [{ url: "u1", timestamp: "t" }],
+        username: "alice",
+        url: "instagram.com/alice",
+        profilePicUrl: "https://a",
+        fullName: "Alice",
+      },
+    ]);
+    vi.mocked(filterNewPostUrls).mockResolvedValue([]); // all already stored
+    vi.mocked(upsertPosts).mockResolvedValue({ data: null, error: null } as never);
+    vi.mocked(getPosts).mockResolvedValue([{ id: "p1", post_url: "u1" }] as never);
+    vi.mocked(uploadLinkToArweave).mockResolvedValue(null);
+    vi.mocked(upsertSocials).mockResolvedValue([] as never);
+    vi.mocked(selectSocials).mockResolvedValue([{ id: "s1" }] as never);
+    vi.mocked(selectAccountSocials).mockResolvedValue([{ account_id: "a1" }] as never);
+    vi.mocked(getAccountArtistIds).mockResolvedValue([{ account_id: "a1" }] as never);
+    vi.mocked(selectAccountEmails).mockResolvedValue([{ email: "x@y.com" }] as never);
+
+    const result = await handleInstagramProfileScraperResults(payload);
+
+    expect(sendApifyWebhookEmail).not.toHaveBeenCalled();
+    // persistence is unaffected — only the notification is gated
+    expect(upsertPosts).toHaveBeenCalledOnce();
+    expect(upsertSocialPosts).toHaveBeenCalledOnce();
+    expect(handleInstagramProfileFollowUpRuns).toHaveBeenCalledOnce();
+    expect(result.social).toEqual({ id: "s1" });
+  });
 });

lib/apify/instagram/handleInstagramProfileScraperResults.ts

@@ -14,6 +14,7 @@ import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave";
 import { getFetchableUrl } from "@/lib/arweave/getFetchableUrl";
 import type { ApifyInstagramProfileResult } from "@/lib/apify/types";
 import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest";
+import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls";
 import type { TablesInsert } from "@/types/database.types";
 
 /**
@@ -40,6 +41,9 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP
     post.url ? [{ post_url: post.url, updated_at: post.timestamp }] : [],
   );
   if (postRows.length === 0) return { posts: [], social: null };
+  // Diff BEFORE upserting — afterwards every scraped post exists and nothing
+  // is distinguishable as new (chat#1855).
+  const newPostUrls = await filterNewPostUrls(postRows.map(p => p.post_url));
   await upsertPosts(postRows);
   const posts = await getPosts({ postUrls: postRows.map(p => p.post_url) });
 
@@ -96,10 +100,14 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP
   // mail outage doesn't block comment scraping and vice versa.
   let sentEmails = null;
   try {
-    sentEmails = await sendApifyWebhookEmail(
-      firstResult,
-      accountEmails.map(e => e.email).filter(Boolean),
-    );
+    // Only notify when the scrape actually found posts new to the platform —
+    // otherwise every scrape re-announces the profile's recent feed.
+    if (newPostUrls.length > 0) {
+      sentEmails = await sendApifyWebhookEmail(
+        firstResult,
+        accountEmails.map(e => e.email).filter(Boolean),
+      );
+    }
   } catch (error) {
     console.error("[WARN] webhook email failed:", error);
   }

lib/socials/__tests__/filterNewPostUrls.test.ts

@@ -0,0 +1,25 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls";
+import { getPosts } from "@/lib/supabase/posts/getPosts";
+
+vi.mock("@/lib/supabase/posts/getPosts", () => ({ getPosts: vi.fn() }));
+
+beforeEach(() => vi.clearAllMocks());
+
+describe("filterNewPostUrls", () => {
+  it("returns only URLs not already stored in posts", async () => {
+    vi.mocked(getPosts).mockResolvedValue([{ post_url: "u1" }, { post_url: "u3" }] as never);
+    expect(await filterNewPostUrls(["u1", "u2", "u3", "u4"])).toEqual(["u2", "u4"]);
+    expect(getPosts).toHaveBeenCalledWith({ postUrls: ["u1", "u2", "u3", "u4"] });
+  });
+
+  it("returns every URL when none are stored", async () => {
+    vi.mocked(getPosts).mockResolvedValue([] as never);
+    expect(await filterNewPostUrls(["u1"])).toEqual(["u1"]);
+  });
+
+  it("returns [] for empty input without querying", async () => {
+    expect(await filterNewPostUrls([])).toEqual([]);
+    expect(getPosts).not.toHaveBeenCalled();
+  });
+});

lib/socials/filterNewPostUrls.ts

@@ -0,0 +1,21 @@
+import { getPosts } from "@/lib/supabase/posts/getPosts";
+
+/**
+ * Returns the subset of post URLs that are genuinely new to the platform —
+ * i.e. not already present in `posts`. Must run BEFORE the scrape results are
+ * upserted (afterwards everything exists and nothing is "new").
+ *
+ * Used to gate scrape-alert notifications: a scrape re-reads a profile's
+ * whole recent feed, so without this diff every scrape re-announces old
+ * posts as new (recoupable/chat#1855).
+ *
+ * @param postUrls - Candidate post URLs from a scrape result.
+ * @returns URLs with no existing `posts` row.
+ */
+export async function filterNewPostUrls(postUrls: string[]): Promise<string[]> {
+  if (!postUrls.length) return [];
+
+  const existing = await getPosts({ postUrls });
+  const known = new Set((existing ?? []).map(post => post.post_url));
+  return postUrls.filter(url => !known.has(url));
+}