diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts index 9092a197b..bab22c438 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts @@ -12,6 +12,7 @@ import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccou import { getAccountArtistIds } from "@/lib/supabase/account_artist_ids/getAccountArtistIds"; import selectAccountEmails from "@/lib/supabase/account_emails/selectAccountEmails"; import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave"; +import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls"; vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn() } })); @@ -26,6 +27,7 @@ vi.mock("../handleInstagramProfileFollowUpRuns", () => ({ handleInstagramProfileFollowUpRuns: vi.fn(), })); vi.mock("@/lib/apify/sendApifyWebhookEmail", () => ({ sendApifyWebhookEmail: vi.fn() })); +vi.mock("@/lib/socials/filterNewPostUrls", () => ({ filterNewPostUrls: vi.fn() })); vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ upsertSocials: vi.fn() })); vi.mock("@/lib/supabase/socials/selectSocials", () => ({ selectSocials: vi.fn(), @@ -51,7 +53,11 @@ const payload = { } as never; describe("handleInstagramProfileScraperResults", () => { - beforeEach(() => vi.clearAllMocks()); + beforeEach(() => { + vi.clearAllMocks(); + // default: everything scraped counts as new (per-test overrides below) + vi.mocked(filterNewPostUrls).mockImplementation(async urls => urls); + }); it("short-circuits when the dataset has no latest posts", async () => { mockDataset([{ username: "alice" }]); @@ -92,4 +98,34 @@ describe("handleInstagramProfileScraperResults", () => { expect(result.social).toEqual({ id: "s1" }); expect(result.posts).toEqual(posts); }); + + it("skips the alert email when no scraped post is genuinely new (chat#1855)", async () => { + mockDataset([ + { + latestPosts: [{ url: "u1", timestamp: "t" }], + username: "alice", + url: "instagram.com/alice", + profilePicUrl: "https://a", + fullName: "Alice", + }, + ]); + vi.mocked(filterNewPostUrls).mockResolvedValue([]); // all already stored + vi.mocked(upsertPosts).mockResolvedValue({ data: null, error: null } as never); + vi.mocked(getPosts).mockResolvedValue([{ id: "p1", post_url: "u1" }] as never); + vi.mocked(uploadLinkToArweave).mockResolvedValue(null); + vi.mocked(upsertSocials).mockResolvedValue([] as never); + vi.mocked(selectSocials).mockResolvedValue([{ id: "s1" }] as never); + vi.mocked(selectAccountSocials).mockResolvedValue([{ account_id: "a1" }] as never); + vi.mocked(getAccountArtistIds).mockResolvedValue([{ account_id: "a1" }] as never); + vi.mocked(selectAccountEmails).mockResolvedValue([{ email: "x@y.com" }] as never); + + const result = await handleInstagramProfileScraperResults(payload); + + expect(sendApifyWebhookEmail).not.toHaveBeenCalled(); + // persistence is unaffected — only the notification is gated + expect(upsertPosts).toHaveBeenCalledOnce(); + expect(upsertSocialPosts).toHaveBeenCalledOnce(); + expect(handleInstagramProfileFollowUpRuns).toHaveBeenCalledOnce(); + expect(result.social).toEqual({ id: "s1" }); + }); }); diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index ff8a971c5..4774c17f0 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -14,6 +14,7 @@ import { uploadLinkToArweave } from "@/lib/arweave/uploadLinkToArweave"; import { getFetchableUrl } from "@/lib/arweave/getFetchableUrl"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; +import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls"; import type { TablesInsert } from "@/types/database.types"; /** @@ -40,6 +41,9 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP post.url ? [{ post_url: post.url, updated_at: post.timestamp }] : [], ); if (postRows.length === 0) return { posts: [], social: null }; + // Diff BEFORE upserting — afterwards every scraped post exists and nothing + // is distinguishable as new (chat#1855). + const newPostUrls = await filterNewPostUrls(postRows.map(p => p.post_url)); await upsertPosts(postRows); const posts = await getPosts({ postUrls: postRows.map(p => p.post_url) }); @@ -96,10 +100,14 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP // mail outage doesn't block comment scraping and vice versa. let sentEmails = null; try { - sentEmails = await sendApifyWebhookEmail( - firstResult, - accountEmails.map(e => e.email).filter(Boolean), - ); + // Only notify when the scrape actually found posts new to the platform — + // otherwise every scrape re-announces the profile's recent feed. + if (newPostUrls.length > 0) { + sentEmails = await sendApifyWebhookEmail( + firstResult, + accountEmails.map(e => e.email).filter(Boolean), + ); + } } catch (error) { console.error("[WARN] webhook email failed:", error); } diff --git a/lib/socials/__tests__/filterNewPostUrls.test.ts b/lib/socials/__tests__/filterNewPostUrls.test.ts new file mode 100644 index 000000000..0f4be5305 --- /dev/null +++ b/lib/socials/__tests__/filterNewPostUrls.test.ts @@ -0,0 +1,25 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; + +vi.mock("@/lib/supabase/posts/getPosts", () => ({ getPosts: vi.fn() })); + +beforeEach(() => vi.clearAllMocks()); + +describe("filterNewPostUrls", () => { + it("returns only URLs not already stored in posts", async () => { + vi.mocked(getPosts).mockResolvedValue([{ post_url: "u1" }, { post_url: "u3" }] as never); + expect(await filterNewPostUrls(["u1", "u2", "u3", "u4"])).toEqual(["u2", "u4"]); + expect(getPosts).toHaveBeenCalledWith({ postUrls: ["u1", "u2", "u3", "u4"] }); + }); + + it("returns every URL when none are stored", async () => { + vi.mocked(getPosts).mockResolvedValue([] as never); + expect(await filterNewPostUrls(["u1"])).toEqual(["u1"]); + }); + + it("returns [] for empty input without querying", async () => { + expect(await filterNewPostUrls([])).toEqual([]); + expect(getPosts).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/socials/filterNewPostUrls.ts b/lib/socials/filterNewPostUrls.ts new file mode 100644 index 000000000..c71d3e670 --- /dev/null +++ b/lib/socials/filterNewPostUrls.ts @@ -0,0 +1,21 @@ +import { getPosts } from "@/lib/supabase/posts/getPosts"; + +/** + * Returns the subset of post URLs that are genuinely new to the platform — + * i.e. not already present in `posts`. Must run BEFORE the scrape results are + * upserted (afterwards everything exists and nothing is "new"). + * + * Used to gate scrape-alert notifications: a scrape re-reads a profile's + * whole recent feed, so without this diff every scrape re-announces old + * posts as new (recoupable/chat#1855). + * + * @param postUrls - Candidate post URLs from a scrape result. + * @returns URLs with no existing `posts` row. + */ +export async function filterNewPostUrls(postUrls: string[]): Promise { + if (!postUrls.length) return []; + + const existing = await getPosts({ postUrls }); + const known = new Set((existing ?? []).map(post => post.post_url)); + return postUrls.filter(url => !known.has(url)); +}