From cdce9792ba0942eec4b32353a45cec1ff047b0f1 Mon Sep 17 00:00:00 2001
From: Anthony Ettinger
Date: Tue, 28 Jul 2026 12:04:36 +0000
Subject: [PATCH] feat(leads): track opens, and read replies out of the mailbox
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The funnel could count sends and nothing else, because a send is
something this application does and a reply is something that happens
elsewhere. Everything past "contacted" waited on somebody remembering to
mark it, so a funnel left alone reported a reply rate of structurally
zero — which reads as a verdict on the campaign rather than on the
bookkeeping.
Opens are tracked per send rather than per campaign. A campaign-level
pixel can only say that somebody opened something, which is not a fact
anybody can act on; per send it says which person opened which email,
which is what decides who to chase. The honest caveat is built in rather
than written on a help page: privacy proxies fetch every image the moment
mail arrives, so loads carrying a proxy fingerprint or arriving
implausibly fast are counted separately and kept out of the total. The
number is a floor. The UI says so, and the open rate divides by tracked
sends rather than by every send ever, so a healthy campaign does not
appear to decline as its pre-tracking history accumulates behind it.
Replies come from the mailbox connected in #126, which has stored IMAP
settings all along and never been read. Two findings from running it
against a real inbox, both now fixed and pinned:
A cap on an oldest-first walk reads two-week-old mail forever and never
reaches today. The first run scanned 200 messages and matched nothing;
searching first and taking the newest slice found a match in the same
window.
And that match was wrong. A Huntress marketing blast came from a
contacted domain, from a plausible address, and was counted as a reply —
marking the prospect replied. Its headers say plainly what it was:
List-Unsubscribe, Feedback-ID, no In-Reply-To, no Re:. So bulk
fingerprints now count as automatic, and the domain fallback requires
some evidence of being a response at all. Sends record their own
Message-Id, so a reply that names it in In-Reply-To matches the exact
send rather than being inferred from a domain.
Auto-replies are recorded and flagged, never counted. Verified against
the live mailbox: the newsletter no longer matches, and three helpdesk
acknowledgements carrying Auto-Submitted: auto-replied are kept out of
the reply rate. The bad row and the prospect status it changed have been
reverted.
Co-Authored-By: Claude Opus 5 (1M context)
---
app/(app)/projects/[id]/leads/page.tsx | 20 +-
app/(marketing)/lead-generation/page.tsx | 49 +-
app/api/cron/outreach-replies/route.ts | 47 ++
app/api/o/[token]/route.ts | 45 ++
components/leads/funnel-panel.tsx | 19 +-
components/leads/replies-panel.tsx | 50 ++
lib/email.ts | 24 +-
lib/outreach/funnel.ts | 60 +-
lib/outreach/openTracking.ts | 146 +++++
lib/outreach/pipeline.ts | 14 +
lib/outreach/replyScan.ts | 551 ++++++++++++++++++
next.config.ts | 4 +-
package-lock.json | 230 ++++++++
package.json | 1 +
...60729020000_outreach_opens_and_replies.sql | 78 +++
.../20260729030000_reply_scan_cron.sql | 25 +
tests/funnel.test.ts | 62 +-
tests/open-tracking.test.ts | 100 ++++
tests/reply-scan.test.ts | 231 ++++++++
19 files changed, 1713 insertions(+), 43 deletions(-)
create mode 100644 app/api/cron/outreach-replies/route.ts
create mode 100644 app/api/o/[token]/route.ts
create mode 100644 components/leads/replies-panel.tsx
create mode 100644 lib/outreach/openTracking.ts
create mode 100644 lib/outreach/replyScan.ts
create mode 100644 supabase/migrations/20260729020000_outreach_opens_and_replies.sql
create mode 100644 supabase/migrations/20260729030000_reply_scan_cron.sql
create mode 100644 tests/open-tracking.test.ts
create mode 100644 tests/reply-scan.test.ts
diff --git a/app/(app)/projects/[id]/leads/page.tsx b/app/(app)/projects/[id]/leads/page.tsx
index 50d70fcb..965e6302 100644
--- a/app/(app)/projects/[id]/leads/page.tsx
+++ b/app/(app)/projects/[id]/leads/page.tsx
@@ -10,6 +10,7 @@ import { SenderAddress } from "@/components/leads/sender-address";
import { MailboxConnect, type ConnectedMailbox } from "@/components/leads/mailbox-connect";
import { SeedLogins } from "@/components/leads/seed-logins";
import { FunnelPanel } from "@/components/leads/funnel-panel";
+import { RepliesPanel, type ReplyRow } from "@/components/leads/replies-panel";
import { campaignFunnels, projectFunnel } from "@/lib/outreach/funnel";
import { listSeedCredentials, type StoredSeedCredential } from "@/lib/outreach/seedCredentials";
import { RefreshLeads } from "@/components/leads/refresh-leads";
@@ -41,6 +42,8 @@ type SendRow = {
subject: string | null;
dry_run: boolean;
sent_at: string;
+ opened_at: string | null;
+ open_count: number | null;
};
const STATUS_TONE: Record = {
@@ -79,7 +82,7 @@ export default async function LeadsPage({
.limit(200),
supabase
.from("outreach_sends")
- .select("channel, step, recipient, subject, dry_run, sent_at")
+ .select("channel, step, recipient, subject, dry_run, sent_at, opened_at, open_count")
.eq("project_id", projectId)
.order("sent_at", { ascending: false })
.limit(10),
@@ -164,6 +167,16 @@ export default async function LeadsPage({
campaignFunnels(projectId),
]);
+ // What came back. Read from the connected mailbox by the reply-scan cron,
+ // so this fills without anyone marking anything by hand.
+ const { data: replyData } = await supabase
+ .from("outreach_replies")
+ .select("from_email, subject, snippet, received_at, auto_reply")
+ .eq("project_id", projectId)
+ .order("received_at", { ascending: false })
+ .limit(20);
+ const replies = (replyData as ReplyRow[] | null) ?? [];
+
let seedCredentials: StoredSeedCredential[] = [];
if (orgId) seedCredentials = await listSeedCredentials(orgId);
@@ -225,6 +238,8 @@ export default async function LeadsPage({
+
+
{" "}
{s.sent_at.slice(0, 16).replace("T", " ")} {s.channel} step {s.step} → {s.recipient}
{s.subject ? ` — ${s.subject}` : ""}
+ {s.opened_at
+ ? ` · opened${(s.open_count ?? 0) > 1 ? ` ×${s.open_count}` : ""}`
+ : ""}
))}
diff --git a/app/(marketing)/lead-generation/page.tsx b/app/(marketing)/lead-generation/page.tsx
index c87f59fe..bc561970 100644
--- a/app/(marketing)/lead-generation/page.tsx
+++ b/app/(marketing)/lead-generation/page.tsx
@@ -5,7 +5,7 @@ import { LEADS_PER_CHARGE } from "@/lib/outreach/billing";
export const metadata = {
title: "Lead generation & cold outreach",
description:
- "Point CrawlProof at a directory or a search, and it finds the companies, finds the people, finds the addresses, and writes the email — from your own mailbox, grounded in facts you supply. 3 credits a run.",
+ "Point CrawlProof at a directory or a search, and it finds the companies, finds the people, finds the addresses, writes the email, and reads the replies — from your own mailbox, grounded in facts you supply. 3 credits a run.",
alternates: { canonical: "/lead-generation" },
openGraph: {
title: "Lead generation & cold outreach · CrawlProof",
@@ -18,10 +18,12 @@ export const metadata = {
/**
* The sales page for project-level lead generation.
*
- * Every claim here is one the pipeline actually makes good on. Two things it
- * deliberately does not claim: open tracking and automatic reply detection.
- * Neither is built, and a lead-gen page that oversells is the exact thing the
- * drafting guard exists to stop the product doing to other people's inboxes.
+ * Every claim here is one the pipeline actually makes good on — a lead-gen
+ * page that oversells is the exact thing the drafting guard exists to stop the
+ * product doing to other people's inboxes. Where a number has a known error
+ * bar, the page says so rather than rounding it away: open tracking measures a
+ * floor, not a count, and claiming otherwise would be the first dishonest
+ * sentence on it.
*/
const LADDER = [
@@ -161,7 +163,7 @@ export default function LeadGenerationPage() {
/>
@@ -200,13 +202,40 @@ export default function LeadGenerationPage() {
+
+ It reads the answers too
+
+ The same mailbox connection that sends is checked for what comes back, so a lead that
+ answers is marked as answered without anybody remembering to do it — and the follow-up
+ sequence stops on its own.
+
+
+
+
+
+
+
+
+
Your numbers, not somebody else's
- Sends, replies and closes are counted at three levels: the whole project, each campaign,
- and each individual run. Rates stay hidden until there is enough volume to mean anything —
- one reply out of three sends is not a 33% reply rate, and showing it as one invites a
- decision the sample cannot support.
+ Sends, opens, replies and closes are counted at three levels: the whole project, each
+ campaign, and each individual run. Rates stay hidden until there is enough volume to mean
+ anything — one reply out of three sends is not a 33% reply rate, and showing it as one
+ invites a decision the sample cannot support.
diff --git a/app/api/cron/outreach-replies/route.ts b/app/api/cron/outreach-replies/route.ts
new file mode 100644
index 00000000..cb1243df
--- /dev/null
+++ b/app/api/cron/outreach-replies/route.ts
@@ -0,0 +1,47 @@
+import { NextResponse } from "next/server";
+import { env } from "@/lib/env";
+import { scanAllMailboxes } from "@/lib/outreach/replyScan";
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+// IMAP is a conversation, not a request: connect, authenticate, select, fetch.
+// A busy mailbox on a slow server can take a while and the default budget is
+// not generous enough to finish.
+export const maxDuration = 300;
+
+/**
+ * Read the connected mailboxes and mark who replied.
+ *
+ * Everything past "contacted" used to depend on a human marking it, which
+ * means anyone who did not got a reply rate of structurally zero — a number
+ * that reads as a verdict on the campaign when it is really a verdict on the
+ * bookkeeping.
+ *
+ * Auto-replies are recorded and flagged rather than counted. An out-of-office
+ * is a real fact about the recipient and not an answer to the pitch.
+ */
+export async function GET(req: Request) {
+ return POST(req);
+}
+
+export async function POST(req: Request) {
+ const incoming =
+ req.headers.get("x-cron-secret") ??
+ req.headers.get("authorization")?.replace(/^Bearer\s+/i, "");
+ if (incoming !== env.cronSecret) {
+ return NextResponse.json({ ok: false, error: "unauthorized" }, { status: 401 });
+ }
+
+ const results = await scanAllMailboxes();
+
+ return NextResponse.json({
+ ok: true,
+ mailboxes: results.length,
+ scanned: results.reduce((n, r) => n + r.scanned, 0),
+ replies: results.reduce((n, r) => n + r.matched, 0),
+ autoReplies: results.reduce((n, r) => n + r.autoReplies, 0),
+ // Per mailbox, so one broken credential is visible rather than being
+ // averaged into an otherwise healthy total.
+ results,
+ });
+}
diff --git a/app/api/o/[token]/route.ts b/app/api/o/[token]/route.ts
new file mode 100644
index 00000000..b6a12b6d
--- /dev/null
+++ b/app/api/o/[token]/route.ts
@@ -0,0 +1,45 @@
+import { NextResponse } from "next/server";
+import { PIXEL_GIF, recordOpen } from "@/lib/outreach/openTracking";
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+/**
+ * The open-tracking pixel.
+ *
+ * Always 200s with the same one-pixel GIF, whatever the token turns out to be.
+ * A 404 for an unknown token would let anyone test tokens for validity, and a
+ * broken image in somebody's inbox is a worse outcome than a lost datapoint.
+ *
+ * Recording happens before the response for the same reason a send is logged
+ * before it is reported: an open that is served but not counted is invisible,
+ * and there is no retry for an image load.
+ */
+export async function GET(
+ req: Request,
+ { params }: { params: Promise<{ token: string }> },
+) {
+ const { token } = await params;
+
+ try {
+ await recordOpen({
+ // The .gif suffix is cosmetic — it exists so the URL reads as an image.
+ token: token.replace(/\.(gif|png)$/i, ""),
+ userAgent: req.headers.get("user-agent"),
+ });
+ } catch {
+ // A failure to record is not a reason to break the email.
+ }
+
+ return new NextResponse(new Uint8Array(PIXEL_GIF), {
+ status: 200,
+ headers: {
+ "content-type": "image/gif",
+ "content-length": String(PIXEL_GIF.length),
+ // Every layer asked not to cache: a cached pixel is one open forever.
+ "cache-control": "no-store, no-cache, must-revalidate, private, max-age=0",
+ pragma: "no-cache",
+ expires: "0",
+ },
+ });
+}
diff --git a/components/leads/funnel-panel.tsx b/components/leads/funnel-panel.tsx
index 51234337..ea9e36e0 100644
--- a/components/leads/funnel-panel.tsx
+++ b/components/leads/funnel-panel.tsx
@@ -21,6 +21,12 @@ function Row({ label, f }: { label: string; f: FunnelCounts }) {
| {label} |
{f.sent} |
+
+ {f.tracked ? f.opened : "—"}
+ {f.openRate !== null && (
+ {pct(f.openRate)}
+ )}
+ |
{f.contacted} |
{f.replied} |
{f.won} |
@@ -59,11 +65,12 @@ export function FunnelPanel({
-
+
| Campaign |
Sent |
+ Opened |
People |
Replied |
Won |
@@ -89,8 +96,14 @@ export function FunnelPanel({
Reply rate is of people contacted; close rate is of people who replied, since a deal comes
- out of a conversation. Replies are recorded when you mark a lead — nothing reads your inbox
- yet.
+ out of a conversation. Replies are read from your connected mailbox — out-of-office and
+ bounce messages are recorded but not counted as replies.
+
+
+
+ Opens are a floor, not a count. Mail clients that block images report nothing, and loads
+ from privacy proxies are discarded rather than counted — so the real number is higher than
+ this one, never lower. Open rate is of tracked sends only.
);
diff --git a/components/leads/replies-panel.tsx b/components/leads/replies-panel.tsx
new file mode 100644
index 00000000..0b8753a5
--- /dev/null
+++ b/components/leads/replies-panel.tsx
@@ -0,0 +1,50 @@
+export type ReplyRow = {
+ from_email: string;
+ subject: string | null;
+ snippet: string | null;
+ received_at: string;
+ auto_reply: boolean;
+};
+
+/**
+ * What came back.
+ *
+ * The funnel above reports how many replied; this is the part that decides
+ * what to do next, which is never a count. Auto-replies are shown rather than
+ * hidden — an out-of-office is worth knowing about even though it is not an
+ * answer — but they are visibly separated, because burying them in the same
+ * list is how a reply rate ends up meaning nothing.
+ */
+export function RepliesPanel({ replies }: { replies: ReplyRow[] }) {
+ if (!replies.length) return null;
+
+ return (
+
+
+
Replies
+
+ Read from your connected mailbox. Answer from your own mail client — this is a record,
+ not an inbox.
+
+
+
+
+
+ );
+}
diff --git a/lib/email.ts b/lib/email.ts
index 3c540899..0284bb1d 100644
--- a/lib/email.ts
+++ b/lib/email.ts
@@ -948,6 +948,11 @@ export function coldOutreachEmailHtml(input: {
reportUrl: string | null;
unsubscribeUrl: string;
postalAddress: string;
+ /**
+ * Open-tracking pixel markup, when this send is being tracked. Absent for
+ * dry runs, which are never delivered and so can never be opened.
+ */
+ trackingPixel?: string | null;
}): string {
const paragraphs = input.bodyText
.split(/\n{2,}/)
@@ -973,13 +978,21 @@ export function coldOutreachEmailHtml(input: {
`
: "";
+ // Last, after the visible content. A client that stops rendering partway
+ // through has already shown the mail, and one that never reaches the pixel
+ // was never going to report a meaningful open anyway.
+ const pixelRow = input.trackingPixel
+ ? `| ${input.trackingPixel} |
`
+ : "";
+
const innerHtml = `
|
${paragraphs}
|
${reportBlock}
- |
`;
+ |
+ ${pixelRow}`;
// Why-you-got-this, opt-out, postal address — the three things a cold
// commercial email legally and ethically has to carry.
@@ -1056,7 +1069,7 @@ export async function sendColdOutreachEmail(input: {
* replies come back to them; omitting it uses the shared Resend sender.
*/
mailbox?: SenderMailbox | null;
-}): Promise<{ sent: boolean; error?: string; provider?: string }> {
+}): Promise<{ sent: boolean; error?: string; provider?: string; messageId?: string }> {
const headers = {
"List-Unsubscribe": `<${input.unsubscribeUrl}>`,
"List-Unsubscribe-Post": "List-Unsubscribe=One-Click",
@@ -1073,7 +1086,10 @@ export async function sendColdOutreachEmail(input: {
secure: input.mailbox.secure,
auth: { user: input.mailbox.user, pass: input.mailbox.pass },
});
- await transporter.sendMail({
+ // The Message-Id is what a reply names in In-Reply-To. Keeping it is
+ // what lets reply detection match by thread rather than by guessing
+ // from the sender's domain.
+ const info = await transporter.sendMail({
from: input.mailbox.fromEmail,
to: input.to,
subject: input.subject,
@@ -1082,7 +1098,7 @@ export async function sendColdOutreachEmail(input: {
headers,
});
transporter.close();
- return { sent: true, provider: "mailbox" };
+ return { sent: true, provider: "mailbox", messageId: info.messageId };
} catch (error) {
// No silent fallback to the shared sender: the user asked for mail to
// come from their address, and quietly sending as someone else would
diff --git a/lib/outreach/funnel.ts b/lib/outreach/funnel.ts
index 4d1dae19..80e46571 100644
--- a/lib/outreach/funnel.ts
+++ b/lib/outreach/funnel.ts
@@ -19,6 +19,20 @@ const MIN_SENDS_FOR_RATE = 20;
export type FunnelCounts = {
/** Live sends. Dry runs are excluded — nobody received them. */
sent: number;
+ /**
+ * Sends whose pixel was loaded by something that looked like a person.
+ * Proxy prefetches are excluded, so this understates rather than inflates.
+ */
+ opened: number;
+ /** Null until enough tracked sends exist for the ratio to mean anything. */
+ openRate: number | null;
+ /**
+ * Sends carrying a pixel at all. Anything sent before open tracking, and
+ * anything sent through a client that strips images, is not measurable —
+ * dividing opens by every send ever would report a falling open rate as
+ * old sends accumulate.
+ */
+ tracked: number;
/** Distinct people contacted, which is what a rate should divide by. */
contacted: number;
replied: number;
@@ -33,10 +47,18 @@ export type FunnelCounts = {
export type CampaignFunnel = FunnelCounts & { campaign: string };
-function rates(counts: Omit): FunnelCounts {
+function rates(
+ counts: Omit,
+): FunnelCounts {
+ // Divided by tracked sends, not all sends: an untracked send cannot report
+ // an open, and counting it in the denominator would show a healthy campaign
+ // slowly declining as its pre-tracking history piles up.
+ const openRate =
+ counts.tracked >= MIN_SENDS_FOR_RATE ? counts.opened / counts.tracked : null;
if (counts.sent < MIN_SENDS_FOR_RATE) {
return {
...counts,
+ openRate,
replyRate: null,
closeRate: null,
rateNote: `needs ${MIN_SENDS_FOR_RATE - counts.sent} more sends before a rate means anything`,
@@ -48,6 +70,7 @@ function rates(counts: Omit 0 ? counts.won / counts.replied : null;
return {
...counts,
+ openRate,
replyRate,
closeRate,
rateNote: counts.replied === 0 ? "no replies yet, so there is no close rate to show" : null,
@@ -59,10 +82,10 @@ function rates(counts: Omit {
const sb = serviceClient();
- const [{ count: sent }, { data: prospects }] = await Promise.all([
+ const [{ data: sends }, { data: prospects }] = await Promise.all([
sb
.from("outreach_sends")
- .select("id", { count: "exact", head: true })
+ .select("track_token, opened_at")
.eq("project_id", projectId)
.eq("channel", "email")
.eq("dry_run", false),
@@ -74,7 +97,10 @@ export async function projectFunnel(projectId: string): Promise {
.in("status", ["contacted", "replied", "won", "lost"]),
]);
- return rates(tally(sent ?? 0, (prospects as { status: string }[] | null) ?? []));
+ const rows = (sends as SendRow[] | null) ?? [];
+ return rates(
+ tally(rows, (prospects as { status: string }[] | null) ?? []),
+ );
}
/** Per campaign, ordered by volume so the busiest reads first. */
@@ -83,7 +109,7 @@ export async function campaignFunnels(projectId: string): Promise [c.id, c.name]),
);
- const sentByCampaign = new Map();
- for (const s of ((sends as { campaign: string | null }[] | null) ?? [])) {
+ const sendsByCampaign = new Map();
+ for (const s of ((sends as (SendRow & { campaign: string | null })[] | null) ?? [])) {
const key = s.campaign ?? "(manual)";
- sentByCampaign.set(key, (sentByCampaign.get(key) ?? 0) + 1);
+ const list = sendsByCampaign.get(key) ?? [];
+ list.push(s);
+ sendsByCampaign.set(key, list);
}
const statusesByCampaign = new Map();
@@ -119,19 +147,25 @@ export async function campaignFunnels(projectId: string): Promise ({
campaign,
- ...rates(tally(sentByCampaign.get(campaign) ?? 0, statusesByCampaign.get(campaign) ?? [])),
+ ...rates(tally(sendsByCampaign.get(campaign) ?? [], statusesByCampaign.get(campaign) ?? [])),
}))
.sort((a, b) => b.sent - a.sent);
}
+/** What the funnel needs off a send row. */
+type SendRow = { track_token: string | null; opened_at: string | null };
+
function tally(
- sent: number,
+ sends: SendRow[],
prospects: { status: string }[],
-): Omit {
+): Omit {
+ const sent = sends.length;
+ const tracked = sends.filter((s) => s.track_token).length;
+ const opened = sends.filter((s) => s.opened_at).length;
let replied = 0;
let won = 0;
let lost = 0;
@@ -147,5 +181,5 @@ function tally(
replied += 1;
}
}
- return { sent, contacted: prospects.length, replied, won, lost };
+ return { sent, tracked, opened, contacted: prospects.length, replied, won, lost };
}
diff --git a/lib/outreach/openTracking.ts b/lib/outreach/openTracking.ts
new file mode 100644
index 00000000..b498f214
--- /dev/null
+++ b/lib/outreach/openTracking.ts
@@ -0,0 +1,146 @@
+// Open tracking, one token per send.
+//
+// A pixel keyed to the campaign can only ever say "somebody opened something",
+// which is not a fact anybody can act on. Keyed to the send it says which
+// person opened which email, which is what decides who to follow up.
+//
+// The honest caveat is built in rather than written on a help page. Mail
+// privacy proxies — Apple Mail Privacy Protection, Gmail's image cache — fetch
+// every image the moment a message arrives, whether or not a human ever looks
+// at it. An open counted from one of those is a lie, so fetches that carry a
+// proxy's fingerprint or arrive implausibly fast are recorded separately and
+// kept out of the count.
+
+import crypto from "node:crypto";
+import { serviceClient } from "@/lib/supabase/service";
+
+/** 1×1 transparent GIF. Smaller than the equivalent PNG and older than most mail clients. */
+export const PIXEL_GIF = Buffer.from(
+ "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7",
+ "base64",
+);
+
+/**
+ * Below this, a fetch is the recipient's mail server, not the recipient.
+ *
+ * A human cannot receive, notice and open an email inside ten seconds often
+ * enough to matter; a caching proxy does it every time. Discarding the window
+ * costs a few genuine same-second opens and removes a systematic overcount.
+ */
+const PREFETCH_WINDOW_MS = 10_000;
+
+/** User-agent fragments belonging to something that fetches on the recipient's behalf. */
+const PROXY_AGENTS = [
+ "googleimageproxy",
+ "yahoomailproxy",
+ "proofpoint",
+ "barracuda",
+ "mimecast",
+ "symantec",
+ "microsoft office",
+ "bingpreview",
+ "skypeuripreview",
+ "slackbot",
+ "whatsapp",
+ "twitterbot",
+ "facebookexternalhit",
+];
+
+export function newTrackToken(): string {
+ return crypto.randomBytes(16).toString("hex");
+}
+
+export function pixelUrl(siteBase: string, token: string): string {
+ // .gif on the end so clients that sniff by extension are satisfied, and so
+ // the URL reads as an image to a human looking at raw source.
+ return `${siteBase}/api/o/${token}.gif`;
+}
+
+/**
+ * The tracking pixel, as HTML.
+ *
+ * Empty alt and role="presentation" so a screen reader does not announce a
+ * spacer image, and inline dimensions so a client that blocks images shows
+ * nothing rather than a broken-image box in the middle of the mail.
+ */
+export function pixelHtml(url: string): string {
+ return `
`;
+}
+
+/**
+ * Whether a fetch looks like a proxy rather than a person.
+ *
+ * Deliberately generous: counting a proxy as an open corrupts the one number
+ * the funnel exists to report, while discarding a real open only understates
+ * it. An understated open rate is a survivable error; an inflated one gets
+ * somebody to keep sending into a void.
+ */
+export function looksLikePrefetch(input: {
+ userAgent: string | null;
+ sentAt: Date;
+ now: Date;
+}): boolean {
+ const ua = (input.userAgent ?? "").toLowerCase();
+ if (!ua) return true; // No client identifies itself as nothing. A scanner does.
+ if (PROXY_AGENTS.some((p) => ua.includes(p))) return true;
+ const elapsed = input.now.getTime() - input.sentAt.getTime();
+ // Negative elapsed means clock skew between the sender and this request; a
+ // fetch that appears to precede its own email is not evidence of anything.
+ if (elapsed < PREFETCH_WINDOW_MS) return true;
+ return false;
+}
+
+export type OpenResult =
+ | { kind: "unknown" }
+ | { kind: "prefetch" }
+ | { kind: "open"; first: boolean };
+
+/**
+ * Record one image load.
+ *
+ * Never throws and never reveals whether the token matched: the route serves
+ * the same pixel either way, so a token cannot be probed for validity.
+ */
+export async function recordOpen(input: {
+ token: string;
+ userAgent: string | null;
+ now?: Date;
+}): Promise {
+ const now = input.now ?? new Date();
+ const sb = serviceClient();
+
+ const { data } = await sb
+ .from("outreach_sends")
+ .select("id, sent_at, opened_at, open_count, prefetch_count, dry_run")
+ .eq("track_token", input.token)
+ .maybeSingle();
+
+ const row = data as Record | null;
+ if (!row) return { kind: "unknown" };
+ // A dry run was never delivered, so anything fetching its pixel is us.
+ if (row.dry_run) return { kind: "unknown" };
+
+ const sentAt = new Date(row.sent_at as string);
+ if (looksLikePrefetch({ userAgent: input.userAgent, sentAt, now })) {
+ await sb
+ .from("outreach_sends")
+ .update({ prefetch_count: ((row.prefetch_count as number) ?? 0) + 1 })
+ .eq("id", row.id as string);
+ return { kind: "prefetch" };
+ }
+
+ const first = !row.opened_at;
+ await sb
+ .from("outreach_sends")
+ .update({
+ // opened_at is the first open and never moves; last_opened_at is the
+ // most recent. Collapsing them into one column would answer "have they
+ // looked again" by destroying the answer to "when did they first look".
+ opened_at: first ? now.toISOString() : (row.opened_at as string),
+ last_opened_at: now.toISOString(),
+ open_count: ((row.open_count as number) ?? 0) + 1,
+ })
+ .eq("id", row.id as string);
+
+ return { kind: "open", first };
+}
diff --git a/lib/outreach/pipeline.ts b/lib/outreach/pipeline.ts
index a8f33c95..8c9c6461 100644
--- a/lib/outreach/pipeline.ts
+++ b/lib/outreach/pipeline.ts
@@ -50,6 +50,7 @@ import { loadProjectMailbox } from "./senderMailbox";
import { loadRecipientContext, recipientContextPrompt } from "./recipientContext";
import { upsertContact } from "./contacts";
import { contactsFromDocuments, teamPageLinks } from "./documents";
+import { newTrackToken, pixelHtml, pixelUrl } from "./openTracking";
export type ProspectRow = {
id: string;
@@ -891,10 +892,17 @@ export async function sendProspectEmail(input: {
}
const unsubscribeUrl = `${siteBase()}/unsubscribe/${input.prospect.unsubscribe_token}`;
+ // Minted before the send, because the token has to be inside the message
+ // that goes out — it cannot be attached to the row afterwards. Dry runs get
+ // none: nothing was delivered, so nothing can open it.
+ const trackToken = input.dryRun ? null : newTrackToken();
let failed: string | null = null;
// Defaults to the shared sender; a connected mailbox overrides it so the
// pitch arrives from the user's own address.
let provider = "resend";
+ // Recorded so a reply naming it in In-Reply-To can be matched to this exact
+ // send rather than inferred from the sender's domain.
+ let providerMessageId: string | null = null;
if (!input.dryRun) {
const mailbox = await loadProjectMailbox(input.prospect.project_id);
const res = await sendColdOutreachEmail({
@@ -906,12 +914,14 @@ export async function sendProspectEmail(input: {
reportUrl: facts.reportUrl,
unsubscribeUrl,
postalAddress: postal.address ?? "",
+ trackingPixel: trackToken ? pixelHtml(pixelUrl(siteBase(), trackToken)) : null,
}),
unsubscribeUrl,
replyTo: input.replyTo ?? undefined,
mailbox,
});
provider = res.provider ?? "resend";
+ providerMessageId = res.messageId ?? null;
if (!res.sent) failed = res.error ?? "send failed";
}
@@ -927,7 +937,11 @@ export async function sendProspectEmail(input: {
body: input.body,
target_url: input.prospect.site_url,
provider: input.dryRun ? "dry-run" : provider,
+ provider_message_id: providerMessageId,
dry_run: input.dryRun || !!failed,
+ // Only on a send that actually left. A token on a failed send would sit
+ // in the table waiting for an open that cannot happen.
+ track_token: failed ? null : trackToken,
});
if (failed) return { ok: false, reason: failed };
diff --git a/lib/outreach/replyScan.ts b/lib/outreach/replyScan.ts
new file mode 100644
index 00000000..6ebf4042
--- /dev/null
+++ b/lib/outreach/replyScan.ts
@@ -0,0 +1,551 @@
+// Reply detection: read the mailbox we sent from and notice who answered.
+//
+// The funnel could count sends and nothing else, because a send is something
+// this application does and a reply is something that happens somewhere else.
+// Everything past "contacted" therefore depended on a human remembering to
+// mark it, which means a reply rate of structurally zero for anyone who did
+// not — and a rate of zero is worse than no rate, because it reads as a
+// verdict on the campaign rather than on the bookkeeping.
+//
+// The mailbox connected in #126 already stores IMAP settings and an encrypted
+// password. This is what finally reads them.
+//
+// The judgement call throughout is what counts as a reply. An out-of-office is
+// not one. A bounce is not one. Both arrive addressed exactly like one, and
+// counting either would inflate the single number this exists to report — so
+// they are recorded, flagged, and kept out of the count.
+
+import { serviceClient } from "@/lib/supabase/service";
+import { decryptSecret } from "@/lib/sp/vault";
+
+/** How far back a mailbox with no cursor looks on its first scan. */
+const FIRST_SCAN_DAYS = 14;
+/**
+ * Ceiling per mailbox per run, so one busy inbox cannot hold up the rest.
+ *
+ * Applied to the newest messages in the window rather than the first ones the
+ * server hands over. A cap on an oldest-first walk silently reads two-week-old
+ * mail forever and never sees today's replies.
+ */
+const MAX_MESSAGES_PER_SCAN = 200;
+const SNIPPET_CHARS = 280;
+
+export type ReplyMailbox = {
+ configId: string;
+ organizationId: string;
+ host: string;
+ port: number;
+ secure: boolean;
+ user: string;
+ pass: string;
+ fromEmail: string;
+ lastScanAt: Date | null;
+};
+
+// ---------------------------------------------------------------- classify
+
+/**
+ * Headers that mean a machine sent this.
+ *
+ * RFC 3834 defines Auto-Submitted for exactly this purpose and the value "no"
+ * is the one that means a human. Everything else here is what real mail
+ * systems send instead of implementing it.
+ */
+const AUTO_HEADERS: Array<[string, RegExp]> = [
+ ["auto-submitted", /^(?!no\b)/i],
+ ["precedence", /\b(bulk|auto_reply|junk|list)\b/i],
+ ["x-autoreply", /./],
+ ["x-autorespond", /./],
+ ["x-auto-response-suppress", /./],
+ ["x-mailer-daemon-error", /./],
+ ["list-id", /./],
+ // Bulk-mail fingerprints. A company we contacted may also be sending us
+ // their newsletter, and it arrives from a real human-looking address at the
+ // right domain — the exact shape of a reply, carrying none of its meaning.
+ ["list-unsubscribe", /./],
+ ["feedback-id", /./],
+ ["x-campaign", /./],
+ ["x-campaignid", /./],
+ ["x-mailchimp-id", /./],
+ ["x-csa-complaints", /./],
+];
+
+const AUTO_SUBJECT =
+ // "Auto:" with the colon is Outlook's out-of-office prefix. The colon is
+ // what keeps it from matching a person writing about automobiles.
+ /^\s*(re:\s*)?(out of (the )?office|automatic(ally)? repl|auto(-| )?repl|autoreply|auto:\s|undeliverable|undelivered|delivery (status notification|has failed|failure)|mail delivery (failed|subsystem)|returned mail|absence|away from|vacation|thank you for (your|contacting)|we('| ha)ve received your)/i;
+
+const AUTO_SENDER =
+ /^(mailer-daemon|postmaster|no-?reply|noreply|do-?not-?reply|bounce|bounces|notifications?|automated?|support-?bot)@/i;
+
+/**
+ * Whether a message is a machine's answer rather than a person's.
+ *
+ * Biased towards calling things automatic. A human reply wrongly flagged shows
+ * up in the list for the user to correct; an out-of-office wrongly counted
+ * silently inflates the reply rate on every campaign that touches a company
+ * with a holiday policy.
+ */
+export function isAutoReply(input: {
+ headers: Record;
+ subject: string | null;
+ from: string;
+}): boolean {
+ const h: Record = {};
+ for (const [k, v] of Object.entries(input.headers)) h[k.toLowerCase()] = v;
+
+ for (const [name, pattern] of AUTO_HEADERS) {
+ const value = h[name];
+ if (value !== undefined && pattern.test(value.trim())) return true;
+ }
+ if (input.subject && AUTO_SUBJECT.test(input.subject)) return true;
+ if (AUTO_SENDER.test(input.from.trim().toLowerCase())) return true;
+ return false;
+}
+
+/** Parse a raw header block into a lowercase-keyed map. Folded lines are joined. */
+export function parseHeaders(raw: string): Record {
+ const out: Record = {};
+ let currentKey = "";
+ for (const line of raw.split(/\r?\n/)) {
+ if (!line.trim()) continue;
+ // A leading space or tab continues the previous header rather than
+ // starting a new one — long References lists arrive folded this way.
+ if (/^[ \t]/.test(line) && currentKey) {
+ out[currentKey] += ` ${line.trim()}`;
+ continue;
+ }
+ const idx = line.indexOf(":");
+ if (idx <= 0) continue;
+ currentKey = line.slice(0, idx).trim().toLowerCase();
+ out[currentKey] = line.slice(idx + 1).trim();
+ }
+ return out;
+}
+
+/**
+ * A readable excerpt of the message.
+ *
+ * Quoted history is dropped: nearly every reply carries a copy of the mail we
+ * sent, and an excerpt of our own outreach tells the reader nothing about the
+ * answer.
+ */
+export function snippetOf(body: string): string {
+ const withoutHtml = decodeQuotedPrintable(body)
+ .replace(/