From cdce9792ba0942eec4b32353a45cec1ff047b0f1 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Tue, 28 Jul 2026 12:04:36 +0000 Subject: [PATCH] feat(leads): track opens, and read replies out of the mailbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The funnel could count sends and nothing else, because a send is something this application does and a reply is something that happens elsewhere. Everything past "contacted" waited on somebody remembering to mark it, so a funnel left alone reported a reply rate of structurally zero — which reads as a verdict on the campaign rather than on the bookkeeping. Opens are tracked per send rather than per campaign. A campaign-level pixel can only say that somebody opened something, which is not a fact anybody can act on; per send it says which person opened which email, which is what decides who to chase. The honest caveat is built in rather than written on a help page: privacy proxies fetch every image the moment mail arrives, so loads carrying a proxy fingerprint or arriving implausibly fast are counted separately and kept out of the total. The number is a floor. The UI says so, and the open rate divides by tracked sends rather than by every send ever, so a healthy campaign does not appear to decline as its pre-tracking history accumulates behind it. Replies come from the mailbox connected in #126, which has stored IMAP settings all along and never been read. Two findings from running it against a real inbox, both now fixed and pinned: A cap on an oldest-first walk reads two-week-old mail forever and never reaches today. The first run scanned 200 messages and matched nothing; searching first and taking the newest slice found a match in the same window. And that match was wrong. A Huntress marketing blast came from a contacted domain, from a plausible address, and was counted as a reply — marking the prospect replied. Its headers say plainly what it was: List-Unsubscribe, Feedback-ID, no In-Reply-To, no Re:. So bulk fingerprints now count as automatic, and the domain fallback requires some evidence of being a response at all. Sends record their own Message-Id, so a reply that names it in In-Reply-To matches the exact send rather than being inferred from a domain. Auto-replies are recorded and flagged, never counted. Verified against the live mailbox: the newsletter no longer matches, and three helpdesk acknowledgements carrying Auto-Submitted: auto-replied are kept out of the reply rate. The bad row and the prospect status it changed have been reverted. Co-Authored-By: Claude Opus 5 (1M context) --- app/(app)/projects/[id]/leads/page.tsx | 20 +- app/(marketing)/lead-generation/page.tsx | 49 +- app/api/cron/outreach-replies/route.ts | 47 ++ app/api/o/[token]/route.ts | 45 ++ components/leads/funnel-panel.tsx | 19 +- components/leads/replies-panel.tsx | 50 ++ lib/email.ts | 24 +- lib/outreach/funnel.ts | 60 +- lib/outreach/openTracking.ts | 146 +++++ lib/outreach/pipeline.ts | 14 + lib/outreach/replyScan.ts | 551 ++++++++++++++++++ next.config.ts | 4 +- package-lock.json | 230 ++++++++ package.json | 1 + ...60729020000_outreach_opens_and_replies.sql | 78 +++ .../20260729030000_reply_scan_cron.sql | 25 + tests/funnel.test.ts | 62 +- tests/open-tracking.test.ts | 100 ++++ tests/reply-scan.test.ts | 231 ++++++++ 19 files changed, 1713 insertions(+), 43 deletions(-) create mode 100644 app/api/cron/outreach-replies/route.ts create mode 100644 app/api/o/[token]/route.ts create mode 100644 components/leads/replies-panel.tsx create mode 100644 lib/outreach/openTracking.ts create mode 100644 lib/outreach/replyScan.ts create mode 100644 supabase/migrations/20260729020000_outreach_opens_and_replies.sql create mode 100644 supabase/migrations/20260729030000_reply_scan_cron.sql create mode 100644 tests/open-tracking.test.ts create mode 100644 tests/reply-scan.test.ts diff --git a/app/(app)/projects/[id]/leads/page.tsx b/app/(app)/projects/[id]/leads/page.tsx index 50d70fcb..965e6302 100644 --- a/app/(app)/projects/[id]/leads/page.tsx +++ b/app/(app)/projects/[id]/leads/page.tsx @@ -10,6 +10,7 @@ import { SenderAddress } from "@/components/leads/sender-address"; import { MailboxConnect, type ConnectedMailbox } from "@/components/leads/mailbox-connect"; import { SeedLogins } from "@/components/leads/seed-logins"; import { FunnelPanel } from "@/components/leads/funnel-panel"; +import { RepliesPanel, type ReplyRow } from "@/components/leads/replies-panel"; import { campaignFunnels, projectFunnel } from "@/lib/outreach/funnel"; import { listSeedCredentials, type StoredSeedCredential } from "@/lib/outreach/seedCredentials"; import { RefreshLeads } from "@/components/leads/refresh-leads"; @@ -41,6 +42,8 @@ type SendRow = { subject: string | null; dry_run: boolean; sent_at: string; + opened_at: string | null; + open_count: number | null; }; const STATUS_TONE: Record = { @@ -79,7 +82,7 @@ export default async function LeadsPage({ .limit(200), supabase .from("outreach_sends") - .select("channel, step, recipient, subject, dry_run, sent_at") + .select("channel, step, recipient, subject, dry_run, sent_at, opened_at, open_count") .eq("project_id", projectId) .order("sent_at", { ascending: false }) .limit(10), @@ -164,6 +167,16 @@ export default async function LeadsPage({ campaignFunnels(projectId), ]); + // What came back. Read from the connected mailbox by the reply-scan cron, + // so this fills without anyone marking anything by hand. + const { data: replyData } = await supabase + .from("outreach_replies") + .select("from_email, subject, snippet, received_at, auto_reply") + .eq("project_id", projectId) + .order("received_at", { ascending: false }) + .limit(20); + const replies = (replyData as ReplyRow[] | null) ?? []; + let seedCredentials: StoredSeedCredential[] = []; if (orgId) seedCredentials = await listSeedCredentials(orgId); @@ -225,6 +238,8 @@ export default async function LeadsPage({ + + {" "} {s.sent_at.slice(0, 16).replace("T", " ")} {s.channel} step {s.step} → {s.recipient} {s.subject ? ` — ${s.subject}` : ""} + {s.opened_at + ? ` · opened${(s.open_count ?? 0) > 1 ? ` ×${s.open_count}` : ""}` + : ""} ))} diff --git a/app/(marketing)/lead-generation/page.tsx b/app/(marketing)/lead-generation/page.tsx index c87f59fe..bc561970 100644 --- a/app/(marketing)/lead-generation/page.tsx +++ b/app/(marketing)/lead-generation/page.tsx @@ -5,7 +5,7 @@ import { LEADS_PER_CHARGE } from "@/lib/outreach/billing"; export const metadata = { title: "Lead generation & cold outreach", description: - "Point CrawlProof at a directory or a search, and it finds the companies, finds the people, finds the addresses, and writes the email — from your own mailbox, grounded in facts you supply. 3 credits a run.", + "Point CrawlProof at a directory or a search, and it finds the companies, finds the people, finds the addresses, writes the email, and reads the replies — from your own mailbox, grounded in facts you supply. 3 credits a run.", alternates: { canonical: "/lead-generation" }, openGraph: { title: "Lead generation & cold outreach · CrawlProof", @@ -18,10 +18,12 @@ export const metadata = { /** * The sales page for project-level lead generation. * - * Every claim here is one the pipeline actually makes good on. Two things it - * deliberately does not claim: open tracking and automatic reply detection. - * Neither is built, and a lead-gen page that oversells is the exact thing the - * drafting guard exists to stop the product doing to other people's inboxes. + * Every claim here is one the pipeline actually makes good on — a lead-gen + * page that oversells is the exact thing the drafting guard exists to stop the + * product doing to other people's inboxes. Where a number has a known error + * bar, the page says so rather than rounding it away: open tracking measures a + * floor, not a count, and claiming otherwise would be the first dishonest + * sentence on it. */ const LADDER = [ @@ -161,7 +163,7 @@ export default function LeadGenerationPage() { /> @@ -200,13 +202,40 @@ export default function LeadGenerationPage() {

+
+

It reads the answers too

+

+ The same mailbox connection that sends is checked for what comes back, so a lead that + answers is marked as answered without anybody remembering to do it — and the follow-up + sequence stops on its own. +

+
+ + + + +
+
+

Your numbers, not somebody else's

- Sends, replies and closes are counted at three levels: the whole project, each campaign, - and each individual run. Rates stay hidden until there is enough volume to mean anything — - one reply out of three sends is not a 33% reply rate, and showing it as one invites a - decision the sample cannot support. + Sends, opens, replies and closes are counted at three levels: the whole project, each + campaign, and each individual run. Rates stay hidden until there is enough volume to mean + anything — one reply out of three sends is not a 33% reply rate, and showing it as one + invites a decision the sample cannot support.

diff --git a/app/api/cron/outreach-replies/route.ts b/app/api/cron/outreach-replies/route.ts new file mode 100644 index 00000000..cb1243df --- /dev/null +++ b/app/api/cron/outreach-replies/route.ts @@ -0,0 +1,47 @@ +import { NextResponse } from "next/server"; +import { env } from "@/lib/env"; +import { scanAllMailboxes } from "@/lib/outreach/replyScan"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; +// IMAP is a conversation, not a request: connect, authenticate, select, fetch. +// A busy mailbox on a slow server can take a while and the default budget is +// not generous enough to finish. +export const maxDuration = 300; + +/** + * Read the connected mailboxes and mark who replied. + * + * Everything past "contacted" used to depend on a human marking it, which + * means anyone who did not got a reply rate of structurally zero — a number + * that reads as a verdict on the campaign when it is really a verdict on the + * bookkeeping. + * + * Auto-replies are recorded and flagged rather than counted. An out-of-office + * is a real fact about the recipient and not an answer to the pitch. + */ +export async function GET(req: Request) { + return POST(req); +} + +export async function POST(req: Request) { + const incoming = + req.headers.get("x-cron-secret") ?? + req.headers.get("authorization")?.replace(/^Bearer\s+/i, ""); + if (incoming !== env.cronSecret) { + return NextResponse.json({ ok: false, error: "unauthorized" }, { status: 401 }); + } + + const results = await scanAllMailboxes(); + + return NextResponse.json({ + ok: true, + mailboxes: results.length, + scanned: results.reduce((n, r) => n + r.scanned, 0), + replies: results.reduce((n, r) => n + r.matched, 0), + autoReplies: results.reduce((n, r) => n + r.autoReplies, 0), + // Per mailbox, so one broken credential is visible rather than being + // averaged into an otherwise healthy total. + results, + }); +} diff --git a/app/api/o/[token]/route.ts b/app/api/o/[token]/route.ts new file mode 100644 index 00000000..b6a12b6d --- /dev/null +++ b/app/api/o/[token]/route.ts @@ -0,0 +1,45 @@ +import { NextResponse } from "next/server"; +import { PIXEL_GIF, recordOpen } from "@/lib/outreach/openTracking"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +/** + * The open-tracking pixel. + * + * Always 200s with the same one-pixel GIF, whatever the token turns out to be. + * A 404 for an unknown token would let anyone test tokens for validity, and a + * broken image in somebody's inbox is a worse outcome than a lost datapoint. + * + * Recording happens before the response for the same reason a send is logged + * before it is reported: an open that is served but not counted is invisible, + * and there is no retry for an image load. + */ +export async function GET( + req: Request, + { params }: { params: Promise<{ token: string }> }, +) { + const { token } = await params; + + try { + await recordOpen({ + // The .gif suffix is cosmetic — it exists so the URL reads as an image. + token: token.replace(/\.(gif|png)$/i, ""), + userAgent: req.headers.get("user-agent"), + }); + } catch { + // A failure to record is not a reason to break the email. + } + + return new NextResponse(new Uint8Array(PIXEL_GIF), { + status: 200, + headers: { + "content-type": "image/gif", + "content-length": String(PIXEL_GIF.length), + // Every layer asked not to cache: a cached pixel is one open forever. + "cache-control": "no-store, no-cache, must-revalidate, private, max-age=0", + pragma: "no-cache", + expires: "0", + }, + }); +} diff --git a/components/leads/funnel-panel.tsx b/components/leads/funnel-panel.tsx index 51234337..ea9e36e0 100644 --- a/components/leads/funnel-panel.tsx +++ b/components/leads/funnel-panel.tsx @@ -21,6 +21,12 @@ function Row({ label, f }: { label: string; f: FunnelCounts }) { {label} {f.sent} + + {f.tracked ? f.opened : "—"} + {f.openRate !== null && ( + {pct(f.openRate)} + )} + {f.contacted} {f.replied} {f.won} @@ -59,11 +65,12 @@ export function FunnelPanel({
- +
+ @@ -89,8 +96,14 @@ export function FunnelPanel({

Reply rate is of people contacted; close rate is of people who replied, since a deal comes - out of a conversation. Replies are recorded when you mark a lead — nothing reads your inbox - yet. + out of a conversation. Replies are read from your connected mailbox — out-of-office and + bounce messages are recorded but not counted as replies. +

+ +

+ Opens are a floor, not a count. Mail clients that block images report nothing, and loads + from privacy proxies are discarded rather than counted — so the real number is higher than + this one, never lower. Open rate is of tracked sends only.

); diff --git a/components/leads/replies-panel.tsx b/components/leads/replies-panel.tsx new file mode 100644 index 00000000..0b8753a5 --- /dev/null +++ b/components/leads/replies-panel.tsx @@ -0,0 +1,50 @@ +export type ReplyRow = { + from_email: string; + subject: string | null; + snippet: string | null; + received_at: string; + auto_reply: boolean; +}; + +/** + * What came back. + * + * The funnel above reports how many replied; this is the part that decides + * what to do next, which is never a count. Auto-replies are shown rather than + * hidden — an out-of-office is worth knowing about even though it is not an + * answer — but they are visibly separated, because burying them in the same + * list is how a reply rate ends up meaning nothing. + */ +export function RepliesPanel({ replies }: { replies: ReplyRow[] }) { + if (!replies.length) return null; + + return ( +
+
+

Replies

+

+ Read from your connected mailbox. Answer from your own mail client — this is a record, + not an inbox. +

+
+ +
    + {replies.map((r, i) => ( +
  • +

    + {r.from_email} + {r.auto_reply && auto-reply} + + {r.received_at.slice(0, 16).replace("T", " ")} + +

    + {r.subject &&

    {r.subject}

    } + {r.snippet && ( +

    {r.snippet}

    + )} +
  • + ))} +
+
+ ); +} diff --git a/lib/email.ts b/lib/email.ts index 3c540899..0284bb1d 100644 --- a/lib/email.ts +++ b/lib/email.ts @@ -948,6 +948,11 @@ export function coldOutreachEmailHtml(input: { reportUrl: string | null; unsubscribeUrl: string; postalAddress: string; + /** + * Open-tracking pixel markup, when this send is being tracked. Absent for + * dry runs, which are never delivered and so can never be opened. + */ + trackingPixel?: string | null; }): string { const paragraphs = input.bodyText .split(/\n{2,}/) @@ -973,13 +978,21 @@ export function coldOutreachEmailHtml(input: { ` : ""; + // Last, after the visible content. A client that stops rendering partway + // through has already shown the mail, and one that never reaches the pixel + // was never going to report a meaningful open anyway. + const pixelRow = input.trackingPixel + ? `` + : ""; + const innerHtml = ` ${reportBlock} - `; + + ${pixelRow}`; // Why-you-got-this, opt-out, postal address — the three things a cold // commercial email legally and ethically has to carry. @@ -1056,7 +1069,7 @@ export async function sendColdOutreachEmail(input: { * replies come back to them; omitting it uses the shared Resend sender. */ mailbox?: SenderMailbox | null; -}): Promise<{ sent: boolean; error?: string; provider?: string }> { +}): Promise<{ sent: boolean; error?: string; provider?: string; messageId?: string }> { const headers = { "List-Unsubscribe": `<${input.unsubscribeUrl}>`, "List-Unsubscribe-Post": "List-Unsubscribe=One-Click", @@ -1073,7 +1086,10 @@ export async function sendColdOutreachEmail(input: { secure: input.mailbox.secure, auth: { user: input.mailbox.user, pass: input.mailbox.pass }, }); - await transporter.sendMail({ + // The Message-Id is what a reply names in In-Reply-To. Keeping it is + // what lets reply detection match by thread rather than by guessing + // from the sender's domain. + const info = await transporter.sendMail({ from: input.mailbox.fromEmail, to: input.to, subject: input.subject, @@ -1082,7 +1098,7 @@ export async function sendColdOutreachEmail(input: { headers, }); transporter.close(); - return { sent: true, provider: "mailbox" }; + return { sent: true, provider: "mailbox", messageId: info.messageId }; } catch (error) { // No silent fallback to the shared sender: the user asked for mail to // come from their address, and quietly sending as someone else would diff --git a/lib/outreach/funnel.ts b/lib/outreach/funnel.ts index 4d1dae19..80e46571 100644 --- a/lib/outreach/funnel.ts +++ b/lib/outreach/funnel.ts @@ -19,6 +19,20 @@ const MIN_SENDS_FOR_RATE = 20; export type FunnelCounts = { /** Live sends. Dry runs are excluded — nobody received them. */ sent: number; + /** + * Sends whose pixel was loaded by something that looked like a person. + * Proxy prefetches are excluded, so this understates rather than inflates. + */ + opened: number; + /** Null until enough tracked sends exist for the ratio to mean anything. */ + openRate: number | null; + /** + * Sends carrying a pixel at all. Anything sent before open tracking, and + * anything sent through a client that strips images, is not measurable — + * dividing opens by every send ever would report a falling open rate as + * old sends accumulate. + */ + tracked: number; /** Distinct people contacted, which is what a rate should divide by. */ contacted: number; replied: number; @@ -33,10 +47,18 @@ export type FunnelCounts = { export type CampaignFunnel = FunnelCounts & { campaign: string }; -function rates(counts: Omit): FunnelCounts { +function rates( + counts: Omit, +): FunnelCounts { + // Divided by tracked sends, not all sends: an untracked send cannot report + // an open, and counting it in the denominator would show a healthy campaign + // slowly declining as its pre-tracking history piles up. + const openRate = + counts.tracked >= MIN_SENDS_FOR_RATE ? counts.opened / counts.tracked : null; if (counts.sent < MIN_SENDS_FOR_RATE) { return { ...counts, + openRate, replyRate: null, closeRate: null, rateNote: `needs ${MIN_SENDS_FOR_RATE - counts.sent} more sends before a rate means anything`, @@ -48,6 +70,7 @@ function rates(counts: Omit 0 ? counts.won / counts.replied : null; return { ...counts, + openRate, replyRate, closeRate, rateNote: counts.replied === 0 ? "no replies yet, so there is no close rate to show" : null, @@ -59,10 +82,10 @@ function rates(counts: Omit { const sb = serviceClient(); - const [{ count: sent }, { data: prospects }] = await Promise.all([ + const [{ data: sends }, { data: prospects }] = await Promise.all([ sb .from("outreach_sends") - .select("id", { count: "exact", head: true }) + .select("track_token, opened_at") .eq("project_id", projectId) .eq("channel", "email") .eq("dry_run", false), @@ -74,7 +97,10 @@ export async function projectFunnel(projectId: string): Promise { .in("status", ["contacted", "replied", "won", "lost"]), ]); - return rates(tally(sent ?? 0, (prospects as { status: string }[] | null) ?? [])); + const rows = (sends as SendRow[] | null) ?? []; + return rates( + tally(rows, (prospects as { status: string }[] | null) ?? []), + ); } /** Per campaign, ordered by volume so the busiest reads first. */ @@ -83,7 +109,7 @@ export async function campaignFunnels(projectId: string): Promise [c.id, c.name]), ); - const sentByCampaign = new Map(); - for (const s of ((sends as { campaign: string | null }[] | null) ?? [])) { + const sendsByCampaign = new Map(); + for (const s of ((sends as (SendRow & { campaign: string | null })[] | null) ?? [])) { const key = s.campaign ?? "(manual)"; - sentByCampaign.set(key, (sentByCampaign.get(key) ?? 0) + 1); + const list = sendsByCampaign.get(key) ?? []; + list.push(s); + sendsByCampaign.set(key, list); } const statusesByCampaign = new Map(); @@ -119,19 +147,25 @@ export async function campaignFunnels(projectId: string): Promise ({ campaign, - ...rates(tally(sentByCampaign.get(campaign) ?? 0, statusesByCampaign.get(campaign) ?? [])), + ...rates(tally(sendsByCampaign.get(campaign) ?? [], statusesByCampaign.get(campaign) ?? [])), })) .sort((a, b) => b.sent - a.sent); } +/** What the funnel needs off a send row. */ +type SendRow = { track_token: string | null; opened_at: string | null }; + function tally( - sent: number, + sends: SendRow[], prospects: { status: string }[], -): Omit { +): Omit { + const sent = sends.length; + const tracked = sends.filter((s) => s.track_token).length; + const opened = sends.filter((s) => s.opened_at).length; let replied = 0; let won = 0; let lost = 0; @@ -147,5 +181,5 @@ function tally( replied += 1; } } - return { sent, contacted: prospects.length, replied, won, lost }; + return { sent, tracked, opened, contacted: prospects.length, replied, won, lost }; } diff --git a/lib/outreach/openTracking.ts b/lib/outreach/openTracking.ts new file mode 100644 index 00000000..b498f214 --- /dev/null +++ b/lib/outreach/openTracking.ts @@ -0,0 +1,146 @@ +// Open tracking, one token per send. +// +// A pixel keyed to the campaign can only ever say "somebody opened something", +// which is not a fact anybody can act on. Keyed to the send it says which +// person opened which email, which is what decides who to follow up. +// +// The honest caveat is built in rather than written on a help page. Mail +// privacy proxies — Apple Mail Privacy Protection, Gmail's image cache — fetch +// every image the moment a message arrives, whether or not a human ever looks +// at it. An open counted from one of those is a lie, so fetches that carry a +// proxy's fingerprint or arrive implausibly fast are recorded separately and +// kept out of the count. + +import crypto from "node:crypto"; +import { serviceClient } from "@/lib/supabase/service"; + +/** 1×1 transparent GIF. Smaller than the equivalent PNG and older than most mail clients. */ +export const PIXEL_GIF = Buffer.from( + "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", + "base64", +); + +/** + * Below this, a fetch is the recipient's mail server, not the recipient. + * + * A human cannot receive, notice and open an email inside ten seconds often + * enough to matter; a caching proxy does it every time. Discarding the window + * costs a few genuine same-second opens and removes a systematic overcount. + */ +const PREFETCH_WINDOW_MS = 10_000; + +/** User-agent fragments belonging to something that fetches on the recipient's behalf. */ +const PROXY_AGENTS = [ + "googleimageproxy", + "yahoomailproxy", + "proofpoint", + "barracuda", + "mimecast", + "symantec", + "microsoft office", + "bingpreview", + "skypeuripreview", + "slackbot", + "whatsapp", + "twitterbot", + "facebookexternalhit", +]; + +export function newTrackToken(): string { + return crypto.randomBytes(16).toString("hex"); +} + +export function pixelUrl(siteBase: string, token: string): string { + // .gif on the end so clients that sniff by extension are satisfied, and so + // the URL reads as an image to a human looking at raw source. + return `${siteBase}/api/o/${token}.gif`; +} + +/** + * The tracking pixel, as HTML. + * + * Empty alt and role="presentation" so a screen reader does not announce a + * spacer image, and inline dimensions so a client that blocks images shows + * nothing rather than a broken-image box in the middle of the mail. + */ +export function pixelHtml(url: string): string { + return ``; +} + +/** + * Whether a fetch looks like a proxy rather than a person. + * + * Deliberately generous: counting a proxy as an open corrupts the one number + * the funnel exists to report, while discarding a real open only understates + * it. An understated open rate is a survivable error; an inflated one gets + * somebody to keep sending into a void. + */ +export function looksLikePrefetch(input: { + userAgent: string | null; + sentAt: Date; + now: Date; +}): boolean { + const ua = (input.userAgent ?? "").toLowerCase(); + if (!ua) return true; // No client identifies itself as nothing. A scanner does. + if (PROXY_AGENTS.some((p) => ua.includes(p))) return true; + const elapsed = input.now.getTime() - input.sentAt.getTime(); + // Negative elapsed means clock skew between the sender and this request; a + // fetch that appears to precede its own email is not evidence of anything. + if (elapsed < PREFETCH_WINDOW_MS) return true; + return false; +} + +export type OpenResult = + | { kind: "unknown" } + | { kind: "prefetch" } + | { kind: "open"; first: boolean }; + +/** + * Record one image load. + * + * Never throws and never reveals whether the token matched: the route serves + * the same pixel either way, so a token cannot be probed for validity. + */ +export async function recordOpen(input: { + token: string; + userAgent: string | null; + now?: Date; +}): Promise { + const now = input.now ?? new Date(); + const sb = serviceClient(); + + const { data } = await sb + .from("outreach_sends") + .select("id, sent_at, opened_at, open_count, prefetch_count, dry_run") + .eq("track_token", input.token) + .maybeSingle(); + + const row = data as Record | null; + if (!row) return { kind: "unknown" }; + // A dry run was never delivered, so anything fetching its pixel is us. + if (row.dry_run) return { kind: "unknown" }; + + const sentAt = new Date(row.sent_at as string); + if (looksLikePrefetch({ userAgent: input.userAgent, sentAt, now })) { + await sb + .from("outreach_sends") + .update({ prefetch_count: ((row.prefetch_count as number) ?? 0) + 1 }) + .eq("id", row.id as string); + return { kind: "prefetch" }; + } + + const first = !row.opened_at; + await sb + .from("outreach_sends") + .update({ + // opened_at is the first open and never moves; last_opened_at is the + // most recent. Collapsing them into one column would answer "have they + // looked again" by destroying the answer to "when did they first look". + opened_at: first ? now.toISOString() : (row.opened_at as string), + last_opened_at: now.toISOString(), + open_count: ((row.open_count as number) ?? 0) + 1, + }) + .eq("id", row.id as string); + + return { kind: "open", first }; +} diff --git a/lib/outreach/pipeline.ts b/lib/outreach/pipeline.ts index a8f33c95..8c9c6461 100644 --- a/lib/outreach/pipeline.ts +++ b/lib/outreach/pipeline.ts @@ -50,6 +50,7 @@ import { loadProjectMailbox } from "./senderMailbox"; import { loadRecipientContext, recipientContextPrompt } from "./recipientContext"; import { upsertContact } from "./contacts"; import { contactsFromDocuments, teamPageLinks } from "./documents"; +import { newTrackToken, pixelHtml, pixelUrl } from "./openTracking"; export type ProspectRow = { id: string; @@ -891,10 +892,17 @@ export async function sendProspectEmail(input: { } const unsubscribeUrl = `${siteBase()}/unsubscribe/${input.prospect.unsubscribe_token}`; + // Minted before the send, because the token has to be inside the message + // that goes out — it cannot be attached to the row afterwards. Dry runs get + // none: nothing was delivered, so nothing can open it. + const trackToken = input.dryRun ? null : newTrackToken(); let failed: string | null = null; // Defaults to the shared sender; a connected mailbox overrides it so the // pitch arrives from the user's own address. let provider = "resend"; + // Recorded so a reply naming it in In-Reply-To can be matched to this exact + // send rather than inferred from the sender's domain. + let providerMessageId: string | null = null; if (!input.dryRun) { const mailbox = await loadProjectMailbox(input.prospect.project_id); const res = await sendColdOutreachEmail({ @@ -906,12 +914,14 @@ export async function sendProspectEmail(input: { reportUrl: facts.reportUrl, unsubscribeUrl, postalAddress: postal.address ?? "", + trackingPixel: trackToken ? pixelHtml(pixelUrl(siteBase(), trackToken)) : null, }), unsubscribeUrl, replyTo: input.replyTo ?? undefined, mailbox, }); provider = res.provider ?? "resend"; + providerMessageId = res.messageId ?? null; if (!res.sent) failed = res.error ?? "send failed"; } @@ -927,7 +937,11 @@ export async function sendProspectEmail(input: { body: input.body, target_url: input.prospect.site_url, provider: input.dryRun ? "dry-run" : provider, + provider_message_id: providerMessageId, dry_run: input.dryRun || !!failed, + // Only on a send that actually left. A token on a failed send would sit + // in the table waiting for an open that cannot happen. + track_token: failed ? null : trackToken, }); if (failed) return { ok: false, reason: failed }; diff --git a/lib/outreach/replyScan.ts b/lib/outreach/replyScan.ts new file mode 100644 index 00000000..6ebf4042 --- /dev/null +++ b/lib/outreach/replyScan.ts @@ -0,0 +1,551 @@ +// Reply detection: read the mailbox we sent from and notice who answered. +// +// The funnel could count sends and nothing else, because a send is something +// this application does and a reply is something that happens somewhere else. +// Everything past "contacted" therefore depended on a human remembering to +// mark it, which means a reply rate of structurally zero for anyone who did +// not — and a rate of zero is worse than no rate, because it reads as a +// verdict on the campaign rather than on the bookkeeping. +// +// The mailbox connected in #126 already stores IMAP settings and an encrypted +// password. This is what finally reads them. +// +// The judgement call throughout is what counts as a reply. An out-of-office is +// not one. A bounce is not one. Both arrive addressed exactly like one, and +// counting either would inflate the single number this exists to report — so +// they are recorded, flagged, and kept out of the count. + +import { serviceClient } from "@/lib/supabase/service"; +import { decryptSecret } from "@/lib/sp/vault"; + +/** How far back a mailbox with no cursor looks on its first scan. */ +const FIRST_SCAN_DAYS = 14; +/** + * Ceiling per mailbox per run, so one busy inbox cannot hold up the rest. + * + * Applied to the newest messages in the window rather than the first ones the + * server hands over. A cap on an oldest-first walk silently reads two-week-old + * mail forever and never sees today's replies. + */ +const MAX_MESSAGES_PER_SCAN = 200; +const SNIPPET_CHARS = 280; + +export type ReplyMailbox = { + configId: string; + organizationId: string; + host: string; + port: number; + secure: boolean; + user: string; + pass: string; + fromEmail: string; + lastScanAt: Date | null; +}; + +// ---------------------------------------------------------------- classify + +/** + * Headers that mean a machine sent this. + * + * RFC 3834 defines Auto-Submitted for exactly this purpose and the value "no" + * is the one that means a human. Everything else here is what real mail + * systems send instead of implementing it. + */ +const AUTO_HEADERS: Array<[string, RegExp]> = [ + ["auto-submitted", /^(?!no\b)/i], + ["precedence", /\b(bulk|auto_reply|junk|list)\b/i], + ["x-autoreply", /./], + ["x-autorespond", /./], + ["x-auto-response-suppress", /./], + ["x-mailer-daemon-error", /./], + ["list-id", /./], + // Bulk-mail fingerprints. A company we contacted may also be sending us + // their newsletter, and it arrives from a real human-looking address at the + // right domain — the exact shape of a reply, carrying none of its meaning. + ["list-unsubscribe", /./], + ["feedback-id", /./], + ["x-campaign", /./], + ["x-campaignid", /./], + ["x-mailchimp-id", /./], + ["x-csa-complaints", /./], +]; + +const AUTO_SUBJECT = + // "Auto:" with the colon is Outlook's out-of-office prefix. The colon is + // what keeps it from matching a person writing about automobiles. + /^\s*(re:\s*)?(out of (the )?office|automatic(ally)? repl|auto(-| )?repl|autoreply|auto:\s|undeliverable|undelivered|delivery (status notification|has failed|failure)|mail delivery (failed|subsystem)|returned mail|absence|away from|vacation|thank you for (your|contacting)|we('| ha)ve received your)/i; + +const AUTO_SENDER = + /^(mailer-daemon|postmaster|no-?reply|noreply|do-?not-?reply|bounce|bounces|notifications?|automated?|support-?bot)@/i; + +/** + * Whether a message is a machine's answer rather than a person's. + * + * Biased towards calling things automatic. A human reply wrongly flagged shows + * up in the list for the user to correct; an out-of-office wrongly counted + * silently inflates the reply rate on every campaign that touches a company + * with a holiday policy. + */ +export function isAutoReply(input: { + headers: Record; + subject: string | null; + from: string; +}): boolean { + const h: Record = {}; + for (const [k, v] of Object.entries(input.headers)) h[k.toLowerCase()] = v; + + for (const [name, pattern] of AUTO_HEADERS) { + const value = h[name]; + if (value !== undefined && pattern.test(value.trim())) return true; + } + if (input.subject && AUTO_SUBJECT.test(input.subject)) return true; + if (AUTO_SENDER.test(input.from.trim().toLowerCase())) return true; + return false; +} + +/** Parse a raw header block into a lowercase-keyed map. Folded lines are joined. */ +export function parseHeaders(raw: string): Record { + const out: Record = {}; + let currentKey = ""; + for (const line of raw.split(/\r?\n/)) { + if (!line.trim()) continue; + // A leading space or tab continues the previous header rather than + // starting a new one — long References lists arrive folded this way. + if (/^[ \t]/.test(line) && currentKey) { + out[currentKey] += ` ${line.trim()}`; + continue; + } + const idx = line.indexOf(":"); + if (idx <= 0) continue; + currentKey = line.slice(0, idx).trim().toLowerCase(); + out[currentKey] = line.slice(idx + 1).trim(); + } + return out; +} + +/** + * A readable excerpt of the message. + * + * Quoted history is dropped: nearly every reply carries a copy of the mail we + * sent, and an excerpt of our own outreach tells the reader nothing about the + * answer. + */ +export function snippetOf(body: string): string { + const withoutHtml = decodeQuotedPrintable(body) + .replace(//gi, " ") + .replace(/<[^>]+>/g, " "); + const lines = withoutHtml.split(/\r?\n/); + const kept: string[] = []; + for (const line of lines) { + const t = line.trim(); + if (t.startsWith(">")) break; // quoted original + if (/^on .{0,80}\bwrote:$/i.test(t)) break; // the usual attribution line + if (/^-{2,}\s*(original message|forwarded message)/i.test(t)) break; + if (/^from:\s.+@/i.test(t) && kept.length) break; // Outlook's quoting style + kept.push(t); + } + return kept + .join(" ") + .replace(/ /gi, " ") + .replace(/&/gi, "&") + .replace(/\s+/g, " ") + .trim() + .slice(0, SNIPPET_CHARS); +} + +/** + * Whether this message is a response to something rather than a fresh send. + * + * A reply carries threading headers or a Re: subject. Requiring one of those + * is what stops a company's marketing list — which arrives from their domain, + * from a plausible address, addressed to us — from being counted as an answer + * to a pitch nobody read. + */ +export function looksLikeResponse(input: { + headers: Record; + subject: string | null; +}): boolean { + const h: Record = {}; + for (const [k, v] of Object.entries(input.headers)) h[k.toLowerCase()] = v; + if (h["in-reply-to"] || h["references"]) return true; + return /^\s*(re|aw|sv|antw|res|r)\s*:/i.test(input.subject ?? ""); +} + +/** Message-Ids this message claims to answer, from either threading header. */ +export function referencedMessageIds(headers: Record): string[] { + const h: Record = {}; + for (const [k, v] of Object.entries(headers)) h[k.toLowerCase()] = v; + const raw = `${h["in-reply-to"] ?? ""} ${h["references"] ?? ""}`; + return [...raw.matchAll(/<([^>]+)>/g)].map((m) => m[1]); +} + +/** + * Undo quoted-printable, which is how most mail encodes anything non-ASCII. + * + * Without it an em-dash arrives as "=E2=80=94" and a snippet of a perfectly + * ordinary sentence reads as line noise. Applied unconditionally: the encoding + * is a no-op on text that does not use it. + */ +export function decodeQuotedPrintable(text: string): string { + const unfolded = text.replace(/=\r?\n/g, ""); + // Runs of =XX are decoded together so multi-byte UTF-8 sequences survive; + // decoding each byte separately would produce mojibake for every accent. + return unfolded.replace(/(?:=[0-9A-Fa-f]{2})+/g, (run) => { + const bytes = Buffer.from( + run.split("=").filter(Boolean).map((h) => parseInt(h, 16)), + ); + return bytes.toString("utf8"); + }); +} + +/** The bare address out of a From header or an envelope entry. */ +export function addressOf(value: string | null | undefined): string { + if (!value) return ""; + const angled = value.match(/<([^>]+)>/); + const raw = angled ? angled[1] : value; + return raw.trim().toLowerCase(); +} + +// ------------------------------------------------------------------ loading + +/** Mailboxes with usable IMAP settings. A send-only mailbox cannot be scanned. */ +export async function loadReplyMailboxes(): Promise { + const sb = serviceClient(); + const { data } = await sb + .from("organization_outreach_configs") + .select( + "id, organization_id, from_email, imap_host, imap_port, imap_secure, imap_user, enc_smtp_user, enc_smtp_pass, last_reply_scan_at", + ) + .eq("channel", "email") + .eq("provider", "smtp") + .eq("enabled", true) + .not("imap_host", "is", null); + + const out: ReplyMailbox[] = []; + for (const r of (data as Record[] | null) ?? []) { + const host = (r.imap_host as string | null) ?? ""; + const fromEmail = (r.from_email as string | null) ?? ""; + const encPass = r.enc_smtp_pass as string | null; + if (!host || !fromEmail || !encPass) continue; + try { + out.push({ + configId: r.id as string, + organizationId: r.organization_id as string, + host, + port: (r.imap_port as number | null) ?? 993, + secure: (r.imap_secure as boolean | null) ?? true, + // The IMAP username is usually the SMTP one; fall back to the address. + user: + (r.imap_user as string | null) ?? + (r.enc_smtp_user ? decryptSecret(r.enc_smtp_user as string) : fromEmail), + pass: decryptSecret(encPass), + fromEmail, + lastScanAt: r.last_reply_scan_at ? new Date(r.last_reply_scan_at as string) : null, + }); + } catch { + // A credential we cannot decrypt is skipped rather than failing the run; + // the vault key may have rotated under an old row. + } + } + return out; +} + +/** Where a scan should start reading from. */ +export function scanSince(lastScanAt: Date | null, now: Date): Date { + if (!lastScanAt) return new Date(now.getTime() - FIRST_SCAN_DAYS * 86_400_000); + // A day of overlap. IMAP SINCE has date granularity, and a reply that lands + // mid-scan would otherwise fall in the gap between two runs. + return new Date(lastScanAt.getTime() - 86_400_000); +} + +// ------------------------------------------------------------------ matching + +export type ReplyMatch = { + projectId: string | null; + ownerId: string; + prospectId: string | null; + sendId: string | null; +}; + +/** + * Tie an incoming address back to something we sent. + * + * Two passes, because the address that answers is often not the address that + * was written to: mail to info@ gets answered by a named person at the same + * company. Matching the exact recipient first keeps that fallback from + * claiming a reply for the wrong prospect when both exist. + */ +export async function matchReply(input: { + organizationId: string; + fromEmail: string; + /** Message-Ids this mail answers, from In-Reply-To / References. */ + references?: string[]; + /** Whether it carries any sign of being a response at all. */ + isResponse?: boolean; +}): Promise { + const sb = serviceClient(); + const from = input.fromEmail.toLowerCase(); + const domain = from.split("@")[1] ?? ""; + if (!domain) return null; + + const { data: projectRows } = await sb + .from("projects") + .select("id") + .eq("organization_id", input.organizationId); + const projectIds = ((projectRows as { id: string }[] | null) ?? []).map((p) => p.id); + if (!projectIds.length) return null; + + // Strongest evidence first: this message names the Message-Id of something + // we sent. Nothing else can be confused for that. + for (const ref of input.references ?? []) { + const { data: threaded } = await sb + .from("outreach_sends") + .select("id, project_id, owner_id, prospect_id") + .in("project_id", projectIds) + .eq("channel", "email") + .eq("dry_run", false) + .eq("provider_message_id", ref.startsWith("<") ? ref : `<${ref}>`) + .limit(1); + const hit = ((threaded as Record[] | null) ?? [])[0]; + if (hit) { + return { + projectId: (hit.project_id as string | null) ?? null, + ownerId: hit.owner_id as string, + prospectId: (hit.prospect_id as string | null) ?? null, + sendId: hit.id as string, + }; + } + } + + const { data: sendRows } = await sb + .from("outreach_sends") + .select("id, project_id, owner_id, prospect_id, recipient, sent_at") + .in("project_id", projectIds) + .eq("channel", "email") + .eq("dry_run", false) + .ilike("recipient", from) + .order("sent_at", { ascending: false }) + .limit(1); + + const exact = ((sendRows as Record[] | null) ?? [])[0]; + if (exact) { + return { + projectId: (exact.project_id as string | null) ?? null, + ownerId: exact.owner_id as string, + prospectId: (exact.prospect_id as string | null) ?? null, + sendId: exact.id as string, + }; + } + + // Nobody at that exact address was written to. Somebody at that company may + // have been — but only if this looks like a response at all. Without that + // guard a contacted company's marketing list counts as a reply, which is + // precisely what happened the first time this ran against a real mailbox. + if (!input.isResponse) return null; + + const { data: prospectRows } = await sb + .from("outreach_prospects") + .select("id, project_id, owner_id, status") + .in("project_id", projectIds) + .eq("channel", "email") + .eq("target_key", domain) + .in("status", ["contacted", "replied", "won"]) + .limit(1); + + const byDomain = ((prospectRows as Record[] | null) ?? [])[0]; + if (!byDomain) return null; + return { + projectId: (byDomain.project_id as string | null) ?? null, + ownerId: byDomain.owner_id as string, + prospectId: byDomain.id as string, + sendId: null, + }; +} + +// ------------------------------------------------------------------ recording + +export async function recordReply(input: { + match: ReplyMatch; + fromEmail: string; + subject: string | null; + snippet: string; + messageId: string | null; + receivedAt: Date; + autoReply: boolean; +}): Promise<"recorded" | "duplicate"> { + const sb = serviceClient(); + + const { error } = await sb.from("outreach_replies").insert({ + project_id: input.match.projectId, + owner_id: input.match.ownerId, + prospect_id: input.match.prospectId, + send_id: input.match.sendId, + from_email: input.fromEmail, + subject: input.subject, + snippet: input.snippet, + message_id: input.messageId, + received_at: input.receivedAt.toISOString(), + auto_reply: input.autoReply, + }); + // The unique index on (owner_id, message_id) is what makes a re-scan safe, + // so a conflict here is the expected path rather than a failure. + if (error) return "duplicate"; + + // An auto-reply moves nothing. The prospect has not answered; their mail + // server has. + if (!input.autoReply && input.match.prospectId) { + const { data: current } = await sb + .from("outreach_prospects") + .select("status") + .eq("id", input.match.prospectId) + .maybeSingle(); + const status = (current as { status?: string } | null)?.status; + // Never walk an outcome backwards: a prospect already marked won or lost + // has been judged by a human, and a later reply does not undo that. + if (status === "contacted") { + await sb + .from("outreach_prospects") + .update({ status: "replied", replied_at: input.receivedAt.toISOString() }) + .eq("id", input.match.prospectId); + } + } + + return "recorded"; +} + +// ------------------------------------------------------------------ scanning + +export type ScanResult = { + mailbox: string; + scanned: number; + matched: number; + autoReplies: number; + error: string | null; +}; + +/** + * Read one mailbox and record anything that answers outreach. + * + * imapflow is imported lazily: it opens TLS sockets and is needed only by this + * path, while everything else importing this module wants the pure helpers + * above. + */ +export async function scanMailbox(box: ReplyMailbox, now = new Date()): Promise { + const result: ScanResult = { + mailbox: box.fromEmail, + scanned: 0, + matched: 0, + autoReplies: 0, + error: null, + }; + + const { ImapFlow } = await import("imapflow"); + const client = new ImapFlow({ + host: box.host, + port: box.port, + secure: box.secure, + auth: { user: box.user, pass: box.pass }, + // imapflow logs every command at info level by default, which would put + // the mailbox password's neighbouring traffic into the app logs. + logger: false, + }); + + try { + await client.connect(); + const lock = await client.getMailboxLock("INBOX"); + try { + const since = scanSince(box.lastScanAt, now); + // Search first, then fetch the newest slice. A plain ranged fetch walks + // the mailbox oldest-first, so on a busy inbox the per-run ceiling is + // spent on old mail and the scan never reaches the replies it is looking + // for — which is exactly what a first scan of a real mailbox did. + const uids = (await client.search({ since }, { uid: true })) || []; + const recent = uids.slice(-MAX_MESSAGES_PER_SCAN); + if (!recent.length) { + result.scanned = 0; + } else + for await (const msg of client.fetch( + recent, + { + uid: true, + envelope: true, + headers: [ + "auto-submitted", + "precedence", + "x-autoreply", + "x-autorespond", + "x-auto-response-suppress", + "list-id", + "in-reply-to", + ], + bodyParts: ["1", "text"], + }, + { uid: true }, + )) { + result.scanned += 1; + + const from = addressOf( + msg.envelope?.from?.[0]?.address ?? msg.envelope?.from?.[0]?.name ?? null, + ); + if (!from) continue; + // Our own sent mail, if the account files it in INBOX. + if (from === box.fromEmail.toLowerCase()) continue; + + const headers = parseHeaders(msg.headers?.toString("utf8") ?? ""); + const subject = msg.envelope?.subject ?? null; + const auto = isAutoReply({ headers, subject, from }); + + const match = await matchReply({ + organizationId: box.organizationId, + fromEmail: from, + references: referencedMessageIds(headers), + isResponse: looksLikeResponse({ headers, subject }), + }); + if (!match) continue; + + const part = msg.bodyParts?.get("1") ?? msg.bodyParts?.get("text"); + const recorded = await recordReply({ + match, + fromEmail: from, + subject, + snippet: snippetOf(part?.toString("utf8") ?? ""), + messageId: msg.envelope?.messageId ?? null, + receivedAt: msg.envelope?.date ?? now, + autoReply: auto, + }); + if (recorded === "duplicate") continue; + if (auto) result.autoReplies += 1; + else result.matched += 1; + } + } finally { + lock.release(); + } + await client.logout(); + } catch (error) { + result.error = error instanceof Error ? error.message.slice(0, 300) : "IMAP scan failed"; + try { + await client.close(); + } catch { + // Already gone. + } + } + + const sb = serviceClient(); + await sb + .from("organization_outreach_configs") + // The cursor advances even after an error, deliberately bounded by the + // day of overlap in scanSince. A mailbox that fails every run would + // otherwise widen its window forever until the scan times out. + .update({ last_reply_scan_at: now.toISOString(), reply_scan_error: result.error }) + .eq("id", box.configId); + + return result; +} + +/** Every connected mailbox, one after another. */ +export async function scanAllMailboxes(now = new Date()): Promise { + const boxes = await loadReplyMailboxes(); + const out: ScanResult[] = []; + for (const box of boxes) out.push(await scanMailbox(box, now)); + return out; +} diff --git a/next.config.ts b/next.config.ts index d8caf2b6..d78029f7 100644 --- a/next.config.ts +++ b/next.config.ts @@ -9,7 +9,9 @@ const nextConfig: NextConfig = { // Playwright launches a real Chromium binary and resolves it through its own // package layout, so bundling it into a server chunk breaks that lookup at // runtime. Keep it external and let Node require it from node_modules. - serverExternalPackages: ["playwright", "playwright-core"], + // imapflow opens raw TLS sockets and resolves its own compiled deps at + // runtime; bundling it breaks both. nodemailer is here for the same reason. + serverExternalPackages: ["playwright", "playwright-core", "imapflow", "nodemailer"], experimental: { serverActions: { bodySizeLimit: "2mb", diff --git a/package-lock.json b/package-lock.json index 91e64f76..1c477047 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@types/nodemailer": "^8.0.0", "bullmq": "^5.79.2", "cheerio": "^1.0.0", + "imapflow": "^1.6.1", "ioredis": "^5.11.1", "linkinator": "^7.6.1", "marked": "^14.1.3", @@ -1559,6 +1560,12 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", + "license": "MIT" + }, "node_modules/@playwright/browser-chromium": { "version": "1.48.0", "resolved": "https://registry.npmjs.org/@playwright/browser-chromium/-/browser-chromium-1.48.0.tgz", @@ -2716,6 +2723,17 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/@zone-eu/mailsplit": { + "version": "5.4.14", + "resolved": "https://registry.npmjs.org/@zone-eu/mailsplit/-/mailsplit-5.4.14.tgz", + "integrity": "sha512-rz0FQOhN3Vq1XrSeSSa9+dPcaFbBxmQPjiZm6zS9oxdVHV7rOWIAYX3yP2YAUf0qBncY8CI+NogzPCmMVrMXcw==", + "license": "(MIT OR EUPL-1.1+)", + "dependencies": { + "libbase64": "1.3.0", + "libmime": "5.4.1", + "libqp": "2.1.1" + } + }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -2787,6 +2805,15 @@ "node": ">=12" } }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", @@ -3610,6 +3637,15 @@ "node": ">= 0.8" } }, + "node_modules/encoding-japanese": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/encoding-japanese/-/encoding-japanese-2.2.0.tgz", + "integrity": "sha512-EuJWwlHPZ1LbADuKTClvHtwbaFn4rOD+dRAbWysqEOXRc2Uui0hJInNJrsdH0c+OhJA4nrCBdSkW4DD5YxAo6A==", + "license": "MIT", + "engines": { + "node": ">=8.10.0" + } + }, "node_modules/encoding-sniffer": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", @@ -4277,6 +4313,38 @@ "node": ">=0.10.0" } }, + "node_modules/imapflow": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/imapflow/-/imapflow-1.6.1.tgz", + "integrity": "sha512-lBkh0ZcKeYHDS3sWfxEcD7j24frrrn4wtbe0UUtLqdLwMSvm9Mygx8j7ZtzWgnvqI7V1DkNfcsidJ/qasb7FBA==", + "license": "MIT", + "dependencies": { + "@zone-eu/mailsplit": "5.4.14", + "encoding-japanese": "2.2.0", + "iconv-lite": "0.7.3", + "libbase64": "1.3.0", + "libmime": "5.4.1", + "libqp": "2.1.1", + "pino": "10.3.1", + "socks": "2.8.9" + } + }, + "node_modules/imapflow/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/immer": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/immer/-/immer-10.2.0.tgz", @@ -4449,6 +4517,46 @@ "url": "https://ko-fi.com/killymxi" } }, + "node_modules/libbase64": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-1.3.0.tgz", + "integrity": "sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg==", + "license": "MIT" + }, + "node_modules/libmime": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/libmime/-/libmime-5.4.1.tgz", + "integrity": "sha512-0wHGhsofo9IdQPenr3BBHXuxcwMq4atFUTsZ9Ogc1OvI5h4rUdDIrBQEN9JHjCXfDMrE59LUMJWsTD82wTYk8A==", + "license": "MIT", + "dependencies": { + "encoding-japanese": "2.2.0", + "iconv-lite": "0.7.3", + "libbase64": "1.3.0", + "libqp": "2.1.1" + } + }, + "node_modules/libmime/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/libqp": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/libqp/-/libqp-2.1.1.tgz", + "integrity": "sha512-0Wd+GPz1O134cP62YU2GTOPNA7Qgl09XwCqM5zpBv87ERCXdfDtyKXvV7c9U22yWJh44QZqBocFnXN11K96qow==", + "license": "MIT" + }, "node_modules/libsql": { "version": "0.5.29", "resolved": "https://registry.npmjs.org/libsql/-/libsql-0.5.29.tgz", @@ -5331,6 +5439,15 @@ ], "license": "MIT" }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -5514,6 +5631,43 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pino": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", + "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^4.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz", + "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", + "license": "MIT" + }, "node_modules/pkce-challenge": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", @@ -5657,6 +5811,22 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, "node_modules/promise-limit": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/promise-limit/-/promise-limit-2.7.0.tgz", @@ -5724,6 +5894,12 @@ "integrity": "sha512-lT5yCqEBgfoMYpf3F2xQRK7zEr1rhIIZuceDK6+xRkJQ4NMbHTwXqk4NkwDwQMNqXgG9r9fyHnzwNVs6zV5KRw==", "license": "MIT" }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, "node_modules/range-parser": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", @@ -5860,6 +6036,15 @@ } } }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, "node_modules/recharts": { "version": "3.8.1", "resolved": "https://registry.npmjs.org/recharts/-/recharts-3.8.1.tgz", @@ -6019,6 +6204,15 @@ "node": ">= 18" } }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -6281,6 +6475,15 @@ "npm": ">= 3.0.0" } }, + "node_modules/sonic-boom": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", + "integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -6290,6 +6493,15 @@ "node": ">=0.10.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/srcset": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/srcset/-/srcset-5.0.3.tgz", @@ -6398,6 +6610,24 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/thread-stream": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", + "integrity": "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==", + "license": "MIT", + "dependencies": { + "real-require": "^1.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/thread-stream/node_modules/real-require": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-1.0.0.tgz", + "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==", + "license": "MIT" + }, "node_modules/three": { "version": "0.184.0", "resolved": "https://registry.npmjs.org/three/-/three-0.184.0.tgz", diff --git a/package.json b/package.json index fc9238cd..c3f0a728 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "@types/nodemailer": "^8.0.0", "bullmq": "^5.79.2", "cheerio": "^1.0.0", + "imapflow": "^1.6.1", "ioredis": "^5.11.1", "linkinator": "^7.6.1", "marked": "^14.1.3", diff --git a/supabase/migrations/20260729020000_outreach_opens_and_replies.sql b/supabase/migrations/20260729020000_outreach_opens_and_replies.sql new file mode 100644 index 00000000..69c3e418 --- /dev/null +++ b/supabase/migrations/20260729020000_outreach_opens_and_replies.sql @@ -0,0 +1,78 @@ +-- Close the outreach loop: what got opened, and what came back. +-- +-- Until now the only measurable event was the send. Everything after it — +-- opened, replied, won — depended on somebody remembering to mark it by hand, +-- so a funnel left alone reported a reply rate of structurally zero. These two +-- columns' worth of bookkeeping is what turns it into a measurement. + +-- ------------------------------------------------------------------- opens + +-- Per send, never per campaign. A campaign-level pixel can tell you that +-- somebody opened something, which is not a fact anybody can act on. +alter table public.outreach_sends + add column if not exists track_token text, + add column if not exists opened_at timestamptz, + add column if not exists last_opened_at timestamptz, + add column if not exists open_count integer not null default 0, + -- Mail privacy proxies fetch every image the instant a message arrives, + -- before a human has seen anything. Keeping the discarded ones visible is + -- the difference between a number that is wrong and a number that is + -- honest about its own error bar. + add column if not exists prefetch_count integer not null default 0; + +create unique index if not exists outreach_sends_track_token_idx + on public.outreach_sends (track_token) + where track_token is not null; + +comment on column public.outreach_sends.track_token is + 'Per-send opaque token behind the tracking pixel URL. Null for dry runs and for sends that predate open tracking.'; +comment on column public.outreach_sends.prefetch_count is + 'Image loads discarded as proxy prefetch rather than counted as opens.'; + +-- ----------------------------------------------------------------- replies + +alter table public.outreach_prospects + add column if not exists replied_at timestamptz; + +-- The reply itself, not just the fact of one. "Did anyone answer" is +-- answerable from a status; "what did they say" is the question that decides +-- what to do next, and it is the reason to keep the row. +create table if not exists public.outreach_replies ( + id uuid primary key default gen_random_uuid(), + project_id uuid references public.projects(id) on delete cascade, + owner_id uuid not null references public.profiles(id) on delete cascade, + prospect_id uuid references public.outreach_prospects(id) on delete set null, + send_id uuid references public.outreach_sends(id) on delete set null, + + from_email text not null, + subject text, + -- A short excerpt, not the message. Enough to tell "interested" from "take + -- me off this list" in a list view without storing somebody's whole mail. + snippet text, + message_id text, + received_at timestamptz not null, + -- Auto-replies are recorded and flagged rather than dropped: an out-of- + -- office is a real fact about the recipient, and counting it as a reply + -- would inflate the one rate the funnel exists to report. + auto_reply boolean not null default false, + created_at timestamptz not null default now() +); + +-- Message-Id is unique per message, so scanning the same mailbox twice cannot +-- record the same reply twice. Scoped by owner because two accounts may each +-- legitimately hold a copy of the same message. +create unique index if not exists outreach_replies_message_idx + on public.outreach_replies (owner_id, message_id) + where message_id is not null; + +create index if not exists outreach_replies_project_idx + on public.outreach_replies (project_id, received_at desc); + +comment on table public.outreach_replies is + 'Replies matched to outreach by scanning the connected mailbox over IMAP. Auto-replies are flagged, not counted.'; + +-- Where the last scan got to, so a scan reads new mail rather than the whole +-- mailbox. Per mailbox, because that is what the cursor describes. +alter table public.organization_outreach_configs + add column if not exists last_reply_scan_at timestamptz, + add column if not exists reply_scan_error text; diff --git a/supabase/migrations/20260729030000_reply_scan_cron.sql b/supabase/migrations/20260729030000_reply_scan_cron.sql new file mode 100644 index 00000000..7441ae8a --- /dev/null +++ b/supabase/migrations/20260729030000_reply_scan_cron.sql @@ -0,0 +1,25 @@ +-- Scan connected mailboxes for replies. +-- +-- Every twenty minutes. A reply is not urgent — nobody is going to answer it +-- inside the window either way — but it does gate the follow-up sequence, and +-- a campaign that keeps chasing someone who already answered is the specific +-- failure this is meant to prevent. Twenty minutes is short enough that the +-- next tick's follow-ups see it. +-- +-- Offset from the outreach cron so a mailbox is not being read and written in +-- the same instant. + +select cron.schedule( + 'crawlproof-outreach-replies', + '11,31,51 * * * *', + $$ + select net.http_post( + url := (select value from public.cron_config where key = 'site_url') || '/api/cron/outreach-replies', + headers := jsonb_build_object( + 'content-type', 'application/json', + 'x-cron-secret', (select value from public.cron_config where key = 'cron_secret') + ), + body := '{}'::jsonb + ); + $$ +); diff --git a/tests/funnel.test.ts b/tests/funnel.test.ts index c063c73a..156da767 100644 --- a/tests/funnel.test.ts +++ b/tests/funnel.test.ts @@ -4,14 +4,12 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; // and is what these cover. A fake client keeps the maths under test without // standing up a database. const rows: Record = { sends: [], prospects: [], campaigns: [] }; -let sendCount = 0; vi.mock("@/lib/supabase/service", () => ({ serviceClient: () => ({ from(table: string) { const chain = { - select: (_cols: string, opts?: { head?: boolean }) => - opts?.head ? { ...chain, count: sendCount } : chain, + select: () => chain, eq: () => chain, in: () => chain, then: undefined, @@ -20,8 +18,7 @@ vi.mock("@/lib/supabase/service", () => ({ const key = table === "outreach_sends" ? "sends" : table === "outreach_prospects" ? "prospects" : "campaigns"; Object.assign(chain, { - then: (resolve: (v: unknown) => void) => - resolve({ data: rows[key], count: table === "outreach_sends" ? sendCount : null }), + then: (resolve: (v: unknown) => void) => resolve({ data: rows[key] }), }); return chain; }, @@ -30,6 +27,21 @@ vi.mock("@/lib/supabase/service", () => ({ const { projectFunnel } = await import("@/lib/outreach/funnel"); +/** + * n live sends, of which `tracked` carry a pixel and `opened` were opened. + * + * Sends are rows rather than a count because the funnel has to distinguish a + * send that could report an open from one that never could. + */ +function sends(n: number, opts: { tracked?: number; opened?: number } = {}) { + const tracked = opts.tracked ?? 0; + const opened = opts.opened ?? 0; + return Array.from({ length: n }, (_, i) => ({ + track_token: i < tracked ? `t${i}` : null, + opened_at: i < opened ? "2026-07-28T00:00:00Z" : null, + })); +} + function prospects(spec: Record) { const out: { status: string }[] = []; for (const [status, n] of Object.entries(spec)) { @@ -42,12 +54,11 @@ beforeEach(() => { rows.sends = []; rows.prospects = []; rows.campaigns = []; - sendCount = 0; }); describe("projectFunnel", () => { it("withholds rates until the sample can carry one", async () => { - sendCount = 3; + rows.sends = sends(3); rows.prospects = prospects({ contacted: 2, replied: 1 }); const f = await projectFunnel("p1"); // One reply in three sends is not a 33% reply rate. @@ -57,7 +68,7 @@ describe("projectFunnel", () => { }); it("computes reply rate over people contacted, not sends", async () => { - sendCount = 60; // follow-ups mean sends exceed people + rows.sends = sends(60); // follow-ups mean sends exceed people rows.prospects = prospects({ contacted: 36, replied: 4 }); const f = await projectFunnel("p1"); expect(f.contacted).toBe(40); @@ -66,7 +77,7 @@ describe("projectFunnel", () => { }); it("counts won and lost as having replied", async () => { - sendCount = 40; + rows.sends = sends(40); rows.prospects = prospects({ contacted: 20, replied: 5, won: 3, lost: 2 }); const f = await projectFunnel("p1"); // A deal or a rejection both required a conversation first. @@ -76,7 +87,7 @@ describe("projectFunnel", () => { }); it("computes close rate over replies, not over everyone contacted", async () => { - sendCount = 40; + rows.sends = sends(40); rows.prospects = prospects({ contacted: 30, replied: 6, won: 4 }); const f = await projectFunnel("p1"); expect(f.replied).toBe(10); @@ -86,7 +97,7 @@ describe("projectFunnel", () => { }); it("says there is no close rate rather than showing zero", async () => { - sendCount = 40; + rows.sends = sends(40); rows.prospects = prospects({ contacted: 40 }); const f = await projectFunnel("p1"); expect(f.replyRate).toBe(0); @@ -100,3 +111,32 @@ describe("projectFunnel", () => { expect(f.replyRate).toBeNull(); }); }); + +describe("open rate", () => { + it("divides by tracked sends, not by every send ever", async () => { + // 20 sends predate open tracking and could never report one. Counting + // them in the denominator would show a healthy campaign declining as its + // own history accumulated behind it. + rows.sends = sends(40, { tracked: 20, opened: 10 }); + rows.prospects = prospects({ contacted: 40 }); + const f = await projectFunnel("p1"); + expect(f.tracked).toBe(20); + expect(f.opened).toBe(10); + expect(f.openRate).toBeCloseTo(0.5); + }); + + it("withholds the rate until enough sends carry a pixel", async () => { + rows.sends = sends(40, { tracked: 5, opened: 3 }); + rows.prospects = prospects({ contacted: 40 }); + const f = await projectFunnel("p1"); + // Three opens out of five is not a 60% open rate. + expect(f.openRate).toBeNull(); + }); + + it("reports no opens rather than nothing when tracking is on and quiet", async () => { + rows.sends = sends(40, { tracked: 40, opened: 0 }); + rows.prospects = prospects({ contacted: 40 }); + const f = await projectFunnel("p1"); + expect(f.openRate).toBe(0); + }); +}); diff --git a/tests/open-tracking.test.ts b/tests/open-tracking.test.ts new file mode 100644 index 00000000..e947ec05 --- /dev/null +++ b/tests/open-tracking.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect } from "vitest"; +import { + PIXEL_GIF, + looksLikePrefetch, + newTrackToken, + pixelHtml, + pixelUrl, +} from "@/lib/outreach/openTracking"; + +const SENT = new Date("2026-07-28T12:00:00Z"); +const later = (ms: number) => new Date(SENT.getTime() + ms); + +describe("the pixel itself", () => { + it("is a real GIF", () => { + // A malformed image is a broken-image box in a stranger's inbox. + expect(PIXEL_GIF.subarray(0, 6).toString("ascii")).toBe("GIF89a"); + }); + + it("is small enough to be free", () => { + expect(PIXEL_GIF.length).toBeLessThan(100); + }); + + it("renders as an invisible, unannounced image", () => { + const html = pixelHtml("https://x.test/api/o/abc.gif"); + // A screen reader announcing "image" mid-email would give the tracking + // away to exactly the readers least able to avoid it. + expect(html).toContain('alt=""'); + expect(html).toContain('role="presentation"'); + expect(html).toContain("width=\"1\""); + }); + + it("mints a token that is not guessable from the last one", () => { + const a = newTrackToken(); + const b = newTrackToken(); + expect(a).not.toBe(b); + expect(a).toMatch(/^[0-9a-f]{32}$/); + }); + + it("builds a URL that reads as an image", () => { + expect(pixelUrl("https://x.test", "abc")).toBe("https://x.test/api/o/abc.gif"); + }); +}); + +describe("telling a person from a proxy", () => { + it("counts a plausible human open", () => { + expect( + looksLikePrefetch({ + userAgent: "Mozilla/5.0 (Macintosh) AppleWebKit/605.1.15", + sentAt: SENT, + now: later(4 * 3600_000), + }), + ).toBe(false); + }); + + it("discards Gmail's image cache", () => { + expect( + looksLikePrefetch({ + userAgent: "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko GoogleImageProxy", + sentAt: SENT, + now: later(4 * 3600_000), + }), + ).toBe(true); + }); + + it("discards a fetch that beats the recipient to the message", () => { + // Nobody receives, notices and opens an email in two seconds often enough + // to matter; a caching proxy does it every time. + expect(looksLikePrefetch({ userAgent: "Mozilla/5.0", sentAt: SENT, now: later(2000) })).toBe( + true, + ); + }); + + it("discards a fetch that appears to predate its own email", () => { + // Clock skew between sender and receiver, not evidence of anything. + expect(looksLikePrefetch({ userAgent: "Mozilla/5.0", sentAt: SENT, now: later(-5000) })).toBe( + true, + ); + }); + + it("discards a client that identifies itself as nothing", () => { + expect(looksLikePrefetch({ userAgent: null, sentAt: SENT, now: later(3600_000) })).toBe(true); + expect(looksLikePrefetch({ userAgent: "", sentAt: SENT, now: later(3600_000) })).toBe(true); + }); + + it("discards link-preview bots", () => { + // A forwarded email pasted into Slack fetches every image in it. + for (const ua of ["Slackbot-LinkExpanding 1.0", "facebookexternalhit/1.1", "WhatsApp/2.0"]) { + expect(looksLikePrefetch({ sentAt: SENT, now: later(3600_000), userAgent: ua })).toBe(true); + } + }); + + it("errs towards discarding", () => { + // An understated open rate is survivable. An inflated one keeps somebody + // sending into a void because the numbers looked fine. + const suspicious = ["Microsoft Office Outlook", "Barracuda", "Mimecast", "Proofpoint"]; + for (const ua of suspicious) { + expect(looksLikePrefetch({ sentAt: SENT, now: later(3600_000), userAgent: ua })).toBe(true); + } + }); +}); diff --git a/tests/reply-scan.test.ts b/tests/reply-scan.test.ts new file mode 100644 index 00000000..e7549f35 --- /dev/null +++ b/tests/reply-scan.test.ts @@ -0,0 +1,231 @@ +import { describe, it, expect } from "vitest"; +import { + addressOf, + decodeQuotedPrintable, + isAutoReply, + looksLikeResponse, + parseHeaders, + referencedMessageIds, + scanSince, + snippetOf, +} from "@/lib/outreach/replyScan"; + +const human = { headers: {}, subject: "Re: quick question", from: "jane@acme.test" }; + +describe("isAutoReply", () => { + it("lets a real reply through", () => { + expect(isAutoReply(human)).toBe(false); + }); + + it("catches RFC 3834 Auto-Submitted", () => { + expect(isAutoReply({ ...human, headers: { "Auto-Submitted": "auto-replied" } })).toBe(true); + }); + + it("treats Auto-Submitted: no as the human it means", () => { + // The spec's value for "a person sent this". Reading the header's mere + // presence as automatic would discard every reply from a compliant client. + expect(isAutoReply({ ...human, headers: { "Auto-Submitted": "no" } })).toBe(false); + }); + + it("catches the headers sent instead of implementing the spec", () => { + for (const h of ["X-Autoreply", "X-Autorespond", "X-Auto-Response-Suppress"]) { + expect(isAutoReply({ ...human, headers: { [h]: "yes" } })).toBe(true); + } + expect(isAutoReply({ ...human, headers: { Precedence: "bulk" } })).toBe(true); + }); + + it("catches a mailing list", () => { + expect(isAutoReply({ ...human, headers: { "List-Id": "" } })).toBe(true); + }); + + it("catches out-of-office by subject", () => { + const subjects = [ + "Out of Office", + "Re: Out of the office until Monday", + "Automatic reply: quick question", + "Auto: I am away", + "Undeliverable: quick question", + "Delivery Status Notification (Failure)", + "Thank you for contacting us", + ]; + for (const subject of subjects) { + expect(isAutoReply({ ...human, subject })).toBe(true); + } + }); + + it("catches a bounce by sender", () => { + for (const from of [ + "MAILER-DAEMON@acme.test", + "postmaster@acme.test", + "no-reply@acme.test", + "noreply@acme.test", + ]) { + expect(isAutoReply({ ...human, from })).toBe(true); + } + }); + + it("does not mistake a person for a robot", () => { + // These sit near the patterns above without matching them, and each one is + // a real prospect whose answer would otherwise be discarded. + expect(isAutoReply({ ...human, from: "norbert@acme.test" })).toBe(false); + expect(isAutoReply({ ...human, subject: "Re: your office move" })).toBe(false); + expect(isAutoReply({ ...human, subject: "Thanks — can we talk Thursday?" })).toBe(false); + }); +}); + +describe("parseHeaders", () => { + it("reads a header block case-insensitively", () => { + expect(parseHeaders("Auto-Submitted: auto-replied\r\nPrecedence: bulk")).toEqual({ + "auto-submitted": "auto-replied", + precedence: "bulk", + }); + }); + + it("joins folded continuation lines", () => { + // Long References lists arrive wrapped; treating the continuation as a + // new header loses the tail of the value. + const parsed = parseHeaders("References: \r\n \r\n\t"); + expect(parsed.references).toBe(" "); + }); + + it("survives junk", () => { + expect(parseHeaders("not a header\r\n\r\nX: 1")).toEqual({ x: "1" }); + }); +}); + +describe("snippetOf", () => { + it("stops at the quoted original", () => { + // Nearly every reply carries a copy of what we sent, and an excerpt of our + // own outreach says nothing about the answer. + const body = "Sure, Thursday works.\n\nOn Tue, 28 Jul 2026, Anthony wrote:\n> our pitch here"; + expect(snippetOf(body)).toBe("Sure, Thursday works."); + }); + + it("stops at a bare quote marker", () => { + expect(snippetOf("Interested.\n> original")).toBe("Interested."); + }); + + it("stops at Outlook's quoting style", () => { + const body = "Yes please.\n\nFrom: anthony@crawlproof.com\nSent: Tuesday"; + expect(snippetOf(body)).toBe("Yes please."); + }); + + it("strips markup from an HTML-only reply", () => { + expect(snippetOf("

Happy to chat.

")).toBe("Happy to chat."); + }); + + it("is bounded", () => { + expect(snippetOf("x".repeat(5000)).length).toBeLessThanOrEqual(280); + }); +}); + +describe("addressOf", () => { + it("pulls the address out of a display name", () => { + expect(addressOf("Jane Doe ")).toBe("jane@acme.test"); + }); + + it("passes a bare address through", () => { + expect(addressOf(" Jane@Acme.test ")).toBe("jane@acme.test"); + }); + + it("has nothing to say about nothing", () => { + expect(addressOf(null)).toBe(""); + }); +}); + +describe("scanSince", () => { + const now = new Date("2026-07-28T12:00:00Z"); + + it("looks back a fortnight on a first scan", () => { + // Not the whole mailbox: a first scan of ten years of mail would time out + // and match nothing, since none of it predates the campaign. + const since = scanSince(null, now); + expect(Math.round((now.getTime() - since.getTime()) / 86_400_000)).toBe(14); + }); + + it("overlaps the previous scan by a day", () => { + // IMAP SINCE has date granularity, so a reply landing mid-scan would fall + // into the gap between two runs. + const last = new Date("2026-07-28T11:00:00Z"); + expect(scanSince(last, now).getTime()).toBe(last.getTime() - 86_400_000); + }); +}); + +describe("a reply has to look like one", () => { + // The first live run against a real mailbox counted a Huntress marketing + // blast as a reply: it came from a contacted domain, from a plausible + // address, and matched on domain alone. Every case here is drawn from that + // message's actual headers. + const blast = { + headers: { + "List-Unsubscribe": "", + "Feedback-ID": "aecl4ke:aig3kzw8:aib1s:HubSpot", + }, + subject: "Here’s what hackers can expect", + from: "marketing@huntress.com", + }; + + it("flags bulk mail as automatic", () => { + expect(isAutoReply(blast)).toBe(true); + }); + + it("flags List-Unsubscribe on its own", () => { + // The single most reliable marker that something is a mailing rather than + // a message: no human writing to one person adds an unsubscribe link. + expect( + isAutoReply({ headers: { "List-Unsubscribe": "" }, subject: "Hi", from: "a@b.test" }), + ).toBe(true); + }); + + it("does not read a marketing subject as a response", () => { + expect(looksLikeResponse(blast)).toBe(false); + }); + + it("reads a Re: subject as a response", () => { + expect(looksLikeResponse({ headers: {}, subject: "Re: quick question" })).toBe(true); + }); + + it("reads threading headers as a response even without Re:", () => { + // Some clients rewrite the subject entirely; the header is the fact. + expect(looksLikeResponse({ headers: { "In-Reply-To": "" }, subject: "Thursday?" })).toBe( + true, + ); + }); + + it("accepts the non-English Re: prefixes", () => { + for (const s of ["AW: Frage", "SV: spørsmål", "Antw: vraag"]) { + expect(looksLikeResponse({ headers: {}, subject: s })).toBe(true); + } + }); +}); + +describe("referencedMessageIds", () => { + it("collects ids from both threading headers", () => { + expect( + referencedMessageIds({ "In-Reply-To": "", References: " " }), + ).toEqual(["a@x", "b@x", "c@x"]); + }); + + it("has nothing to say about a fresh message", () => { + expect(referencedMessageIds({ Subject: "hi" })).toEqual([]); + }); +}); + +describe("decodeQuotedPrintable", () => { + it("decodes a multi-byte character as one character", () => { + // Decoding each byte separately is how "…" becomes three mojibake glyphs. + expect(decodeQuotedPrintable("threats=E2=80=A6and so many")).toBe("threats…and so many"); + }); + + it("joins soft line breaks", () => { + expect(decodeQuotedPrintable("can be a=\r\nvoided")).toBe("can be avoided"); + }); + + it("leaves plain text alone", () => { + expect(decodeQuotedPrintable("nothing to do here")).toBe("nothing to do here"); + }); + + it("reaches the snippet", () => { + expect(snippetOf("threats=E2=80=A6and more")).toBe("threats…and more"); + }); +});
Campaign SentOpened People Replied Won
${input.trackingPixel}
${paragraphs}