-
Notifications
You must be signed in to change notification settings - Fork 9
feat(apify): deterministic digest template — direct links to each new post, no LLM body #761
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: feat/scrape-digest-aggregation
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| import { describe, it, expect, vi, beforeEach } from "vitest"; | ||
| import { sendApifyWebhookEmail } from "@/lib/apify/sendApifyWebhookEmail"; | ||
| import { RECOUP_FROM_EMAIL } from "@/lib/const"; | ||
|
|
||
| const sendEmailWithResend = vi.fn(); | ||
| vi.mock("@/lib/emails/sendEmail", () => ({ | ||
| sendEmailWithResend: (...a: unknown[]) => sendEmailWithResend(...a), | ||
| })); | ||
| vi.mock("@/lib/composio/getFrontendBaseUrl", () => ({ | ||
| getFrontendBaseUrl: () => "https://chat.recoupable.dev", | ||
| })); | ||
|
|
||
| const PROFILE = { | ||
| fullName: "Ashnikko", | ||
| username: "ashnikko", | ||
| url: "https://instagram.com/ashnikko", | ||
| profilePicUrl: "https://example.com/pic.jpg", | ||
| biography: "bio", | ||
| followersCount: 100, | ||
| followsCount: 10, | ||
| latestPosts: [], | ||
| } as never; | ||
| const NEW_URLS = ["https://instagram.com/p/new1"]; | ||
|
|
||
| beforeEach(() => { | ||
| vi.clearAllMocks(); | ||
| sendEmailWithResend.mockResolvedValue({ id: "email-1" }); | ||
| }); | ||
|
|
||
| describe("sendApifyWebhookEmail", () => { | ||
| it("BCCs recipients so no account can see another's address (chat#1855)", async () => { | ||
| const recipients = ["manager@customer-a.com", "label@customer-b.com"]; | ||
| await sendApifyWebhookEmail(PROFILE, recipients, NEW_URLS); | ||
| const payload = sendEmailWithResend.mock.calls[0][0] as Record<string, unknown>; | ||
| expect(payload.bcc).toEqual(recipients); | ||
| expect(payload.to).toEqual([RECOUP_FROM_EMAIL]); | ||
| }); | ||
|
|
||
| it("never carries more than our own address in to/cc, regardless of recipient count", async () => { | ||
| const recipients = Array.from({ length: 25 }, (_, i) => `user${i}@example.com`); | ||
| await sendApifyWebhookEmail(PROFILE, recipients, NEW_URLS); | ||
| const payload = sendEmailWithResend.mock.calls[0][0] as Record<string, unknown>; | ||
| expect([payload.to, payload.cc].flat().filter(Boolean)).toEqual([RECOUP_FROM_EMAIL]); | ||
| }); | ||
|
|
||
| it("links the new posts in the deterministic body — no vendor jargon", async () => { | ||
| await sendApifyWebhookEmail(PROFILE, ["a@b.com"], NEW_URLS); | ||
| const payload = sendEmailWithResend.mock.calls[0][0] as Record<string, string>; | ||
| expect(payload.html).toContain('href="https://instagram.com/p/new1"'); | ||
| expect((payload.html + payload.subject).toLowerCase()).not.toContain("apify"); | ||
| }); | ||
|
|
||
| it("returns null and sends nothing when there are no recipients", async () => { | ||
| expect(await sendApifyWebhookEmail(PROFILE, [], NEW_URLS)).toBeNull(); | ||
| expect(sendEmailWithResend).not.toHaveBeenCalled(); | ||
| }); | ||
|
|
||
| it("returns null and sends nothing when no post is genuinely new", async () => { | ||
| expect(await sendApifyWebhookEmail(PROFILE, ["a@b.com"], [])).toBeNull(); | ||
| expect(sendEmailWithResend).not.toHaveBeenCalled(); | ||
| }); | ||
| }); | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| import { describe, it, expect } from "vitest"; | ||
| import { renderScrapeDigestHtml } from "@/lib/apify/digest/renderScrapeDigestHtml"; | ||
|
|
||
| const SECTIONS = [ | ||
| { | ||
| platform: "instagram", | ||
| postUrls: ["https://instagram.com/p/abc", "https://instagram.com/p/def"], | ||
| }, | ||
| { platform: "tiktok", postUrls: ["https://tiktok.com/@a/video/1"] }, | ||
| ]; | ||
|
|
||
| describe("renderScrapeDigestHtml", () => { | ||
| it("links every new post, grouped under its platform", () => { | ||
| const { html } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: No test for null/undefined artistName. The fallback Prompt for AI agents |
||
| for (const s of SECTIONS) for (const u of s.postUrls) expect(html).toContain(`href="${u}"`); | ||
| expect(html.toLowerCase()).toContain("instagram"); | ||
| expect(html.toLowerCase()).toContain("tiktok"); | ||
| }); | ||
|
|
||
| it("is deterministic — identical input renders identical output", () => { | ||
| const a = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); | ||
| const b = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); | ||
| expect(a).toEqual(b); | ||
| }); | ||
|
|
||
| it("never leaks internal vendor jargon to customers", () => { | ||
| const { html, subject } = renderScrapeDigestHtml({ | ||
| sections: SECTIONS, | ||
| artistName: "Ashnikko", | ||
| }); | ||
| expect((html + subject).toLowerCase()).not.toContain("apify"); | ||
| expect((html + subject).toLowerCase()).not.toContain("dataset"); | ||
| }); | ||
|
|
||
| it("counts posts and platforms in the subject", () => { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Singular subject form is untested: no case covers 1 post or 1 platform, where the pluralization code ( Prompt for AI agents |
||
| const { subject } = renderScrapeDigestHtml({ sections: SECTIONS, artistName: "Ashnikko" }); | ||
| expect(subject).toBe("Ashnikko: 3 new posts across 2 platforms"); | ||
| }); | ||
| }); | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| import { getFrontendBaseUrl } from "@/lib/composio/getFrontendBaseUrl"; | ||
| import type { ScrapeDigestSection } from "@/lib/apify/digest/sendScrapeDigestEmail"; | ||
|
|
||
| const PLATFORM_LABELS: Record<string, string> = { | ||
| instagram: "Instagram", | ||
| tiktok: "TikTok", | ||
| x: "X", | ||
| twitter: "X", | ||
| youtube: "YouTube", | ||
| facebook: "Facebook", | ||
| threads: "Threads", | ||
| }; | ||
|
|
||
| /** | ||
| * Deterministic house-style renderer for the new-posts digest (chat#1855). | ||
| * Replaces the per-send LLM body: identical input renders identical output, | ||
| * every genuinely-new post is linked directly, and internal vendor jargon | ||
| * never reaches customers. | ||
| */ | ||
| export function renderScrapeDigestHtml({ | ||
| sections, | ||
| artistName, | ||
| }: { | ||
| sections: ScrapeDigestSection[]; | ||
| artistName?: string | null; | ||
| }): { subject: string; html: string } { | ||
| const who = artistName || "Your artist"; | ||
| const total = sections.reduce((n, s) => n + s.postUrls.length, 0); | ||
| const subject = `${who}: ${total} new post${total === 1 ? "" : "s"} across ${sections.length} platform${sections.length === 1 ? "" : "s"}`; | ||
|
|
||
| const sectionHtml = sections | ||
| .map(section => { | ||
| const label = PLATFORM_LABELS[section.platform.toLowerCase()] ?? section.platform; | ||
| const items = section.postUrls | ||
| .map(url => { | ||
| const href = url.startsWith("http") ? url : `https://${url}`; | ||
| return `<li style="margin:4px 0"><a href="${href}" style="color:#111;text-decoration:underline">${href}</a></li>`; | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P1: The new renderer inserts Prompt for AI agents |
||
| }) | ||
| .join(""); | ||
| return `<h3 style="margin:20px 0 6px;font-size:15px">${label}</h3><ul style="margin:0;padding-left:20px">${items}</ul>`; | ||
| }) | ||
| .join(""); | ||
|
|
||
| const chatUrl = `${getFrontendBaseUrl()}/?q=${encodeURIComponent("tell me about my artist's latest posts")}`; | ||
| const html = `<div style="font-family:ui-sans-serif,system-ui,sans-serif;color:#111;max-width:560px"> | ||
| <h2 style="font-size:18px;margin:0 0 8px">New posts from ${who}</h2> | ||
| <p style="margin:0 0 4px">We found ${total} new post${total === 1 ? "" : "s"} since we last checked:</p> | ||
| ${sectionHtml} | ||
| <p style="margin:24px 0 0"><a href="${chatUrl}" style="color:#111">Ask Recoup about these posts →</a></p> | ||
| </div>`; | ||
|
|
||
| return { subject, html }; | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,6 @@ | ||
| import { sendEmailWithResend } from "@/lib/emails/sendEmail"; | ||
| import { RECOUP_FROM_EMAIL } from "@/lib/const"; | ||
| import { renderScrapeDigestHtml } from "@/lib/apify/digest/renderScrapeDigestHtml"; | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: The batch digest sender still composes its own inline HTML/subject and never calls Prompt for AI agents |
||
|
|
||
| export type ScrapeDigestSection = { platform: string; postUrls: string[] }; | ||
| export type ScrapeDigestInput = { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,54 +1,39 @@ | ||
| import generateText from "@/lib/ai/generateText"; | ||
| import { sendEmailWithResend } from "@/lib/emails/sendEmail"; | ||
| import { RECOUP_FROM_EMAIL } from "@/lib/const"; | ||
| import { getFrontendBaseUrl } from "@/lib/composio/getFrontendBaseUrl"; | ||
| import { renderScrapeDigestHtml } from "@/lib/apify/digest/renderScrapeDigestHtml"; | ||
| import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; | ||
|
|
||
| /** | ||
| * Sends an Apify-webhook summary email to the given recipients using | ||
| * Resend. Generates the email body via an LLM based on the first | ||
| * profile result. | ||
| * Sends the legacy single-platform Instagram alert (non-batch scrape runs). | ||
| * Batch runs get the consolidated digest instead. Body comes from the shared | ||
| * deterministic renderer — the per-send LLM body is gone (chat#1855): stable | ||
| * branding, direct links to the new posts, no vendor jargon. | ||
| * | ||
| * @param profile - First dataset result from the profile scraper. | ||
| * @param emails - Recipient email addresses. | ||
| * @returns The Resend response, or `null` when there are no recipients. | ||
| * @param emails - Recipient email addresses (cross-tenant — BCC only). | ||
| * @param newPostUrls - Post URLs genuinely new to the platform. | ||
| * @returns The Resend response, or `null` when there is nothing to send. | ||
| */ | ||
| export async function sendApifyWebhookEmail( | ||
| profile: ApifyInstagramProfileResult, | ||
| emails: string[], | ||
| newPostUrls: string[] = [], | ||
| ) { | ||
| if (!emails?.length) return null; | ||
| if (!emails?.length || !newPostUrls.length) return null; | ||
|
|
||
| const prompt = `You have a new Apify dataset update. Here is the data: | ||
|
|
||
| Key Data | ||
| Full Name: ${profile.fullName} | ||
| Username: ${profile.username} | ||
| Profile URL: ${profile.url} | ||
| Profile Picture: ${profile.profilePicUrl} | ||
| Biography: ${profile.biography} | ||
| Followers: ${profile.followersCount} | ||
| Following: ${profile.followsCount} | ||
| Latest Posts: ${(Array.isArray(profile.latestPosts) ? profile.latestPosts : []).map(p => JSON.stringify(p)).join(", ")} | ||
| `; | ||
|
|
||
| const { text } = await generateText({ | ||
| system: `you are a record label services manager for Recoup. | ||
| write beautiful html email. | ||
| subject: New Apify Dataset Notification. you're notifying music managers about new posts being available for one of their roster musician's Instagram profile. | ||
| include a link to view the instagram profile. | ||
| call to action is to open a chat link to learn more about the latest posts using Recoup Chat (AI Agents): ${getFrontendBaseUrl()}/?q=tell%20me%20about%20my%20latest%20Ig%20posts | ||
| You'll be passed a dataset summary for a musician profile and their latest posts on instagram. | ||
| your goal is to get the recipient to click a cta link to open a chat link to learn more about the latest posts using Recoup Chat (AI Agents). | ||
| only include the email body html. | ||
| no headers or subject`, | ||
| prompt, | ||
| const { subject, html } = renderScrapeDigestHtml({ | ||
| sections: [{ platform: "instagram", postUrls: newPostUrls }], | ||
| artistName: profile.fullName ?? profile.username ?? null, | ||
| }); | ||
|
|
||
| // Recipients span multiple customer accounts (the social's watchers are | ||
| // resolved cross-tenant), so they must NEVER share a visible To: line — | ||
| // BCC only, with ourselves as the required To: (chat#1855). | ||
| return await sendEmailWithResend({ | ||
| from: RECOUP_FROM_EMAIL, | ||
| to: emails, | ||
| subject: `${profile.fullName ?? profile.username ?? "Your artist"} has new posts on Instagram`, | ||
| html: text, | ||
| to: [RECOUP_FROM_EMAIL], | ||
| bcc: emails, | ||
| subject, | ||
| html, | ||
| }); | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P3: Custom agent: Enforce Clear Code Style and Maintainability Practices
The PROFILE test fixture uses
as neverwhich completely erases type checking on this object. SinceApifyInstagramProfileResulthas all-optional fields, the literal already satisfies the interface — no assertion is needed. Removingas never(or switching toas ApifyInstagramProfileResultif you want explicit annotation) would let TypeScript verify the fixture stays in sync with the type definition.Prompt for AI agents