feat(leads): track opens, and read replies out of the mailbox

app/(app)/projects/[id]/leads/page.tsx

@@ -10,6 +10,7 @@ import { SenderAddress } from "@/components/leads/sender-address";
 import { MailboxConnect, type ConnectedMailbox } from "@/components/leads/mailbox-connect";
 import { SeedLogins } from "@/components/leads/seed-logins";
 import { FunnelPanel } from "@/components/leads/funnel-panel";
+import { RepliesPanel, type ReplyRow } from "@/components/leads/replies-panel";
 import { campaignFunnels, projectFunnel } from "@/lib/outreach/funnel";
 import { listSeedCredentials, type StoredSeedCredential } from "@/lib/outreach/seedCredentials";
 import { RefreshLeads } from "@/components/leads/refresh-leads";
@@ -41,6 +42,8 @@ type SendRow = {
   subject: string | null;
   dry_run: boolean;
   sent_at: string;
+  opened_at: string | null;
+  open_count: number | null;
 };
 
 const STATUS_TONE: Record<string, string> = {
@@ -79,7 +82,7 @@ export default async function LeadsPage({
       .limit(200),
     supabase
       .from("outreach_sends")
-      .select("channel, step, recipient, subject, dry_run, sent_at")
+      .select("channel, step, recipient, subject, dry_run, sent_at, opened_at, open_count")
       .eq("project_id", projectId)
       .order("sent_at", { ascending: false })
       .limit(10),
@@ -164,6 +167,16 @@ export default async function LeadsPage({
     campaignFunnels(projectId),
   ]);
 
+  // What came back. Read from the connected mailbox by the reply-scan cron,
+  // so this fills without anyone marking anything by hand.
+  const { data: replyData } = await supabase
+    .from("outreach_replies")
+    .select("from_email, subject, snippet, received_at, auto_reply")
+    .eq("project_id", projectId)
+    .order("received_at", { ascending: false })
+    .limit(20);
+  const replies = (replyData as ReplyRow[] | null) ?? [];
+
   let seedCredentials: StoredSeedCredential[] = [];
   if (orgId) seedCredentials = await listSeedCredentials(orgId);
 
@@ -225,6 +238,8 @@ export default async function LeadsPage({
 
       <FunnelPanel project={funnel} campaigns={perCampaignFunnel} />
 
+      <RepliesPanel replies={replies} />
+
       <SeedLogins
         projectId={projectId}
         waitingHosts={waitingHosts}
@@ -329,6 +344,9 @@ export default async function LeadsPage({
                 </span>{" "}
                 {s.sent_at.slice(0, 16).replace("T", " ")} {s.channel} step {s.step} → {s.recipient}
                 {s.subject ? ` — ${s.subject}` : ""}
+                {s.opened_at
+                  ? ` · opened${(s.open_count ?? 0) > 1 ? ` ×${s.open_count}` : ""}`
+                  : ""}
               </li>
             ))}
           </ul>

app/(marketing)/lead-generation/page.tsx

@@ -5,7 +5,7 @@ import { LEADS_PER_CHARGE } from "@/lib/outreach/billing";
 export const metadata = {
   title: "Lead generation & cold outreach",
   description:
-    "Point CrawlProof at a directory or a search, and it finds the companies, finds the people, finds the addresses, and writes the email — from your own mailbox, grounded in facts you supply. 3 credits a run.",
+    "Point CrawlProof at a directory or a search, and it finds the companies, finds the people, finds the addresses, writes the email, and reads the replies — from your own mailbox, grounded in facts you supply. 3 credits a run.",
   alternates: { canonical: "/lead-generation" },
   openGraph: {
     title: "Lead generation & cold outreach · CrawlProof",
@@ -18,10 +18,12 @@ export const metadata = {
 /**
  * The sales page for project-level lead generation.
  *
- * Every claim here is one the pipeline actually makes good on. Two things it
- * deliberately does not claim: open tracking and automatic reply detection.
- * Neither is built, and a lead-gen page that oversells is the exact thing the
- * drafting guard exists to stop the product doing to other people's inboxes.
+ * Every claim here is one the pipeline actually makes good on — a lead-gen
+ * page that oversells is the exact thing the drafting guard exists to stop the
+ * product doing to other people's inboxes. Where a number has a known error
+ * bar, the page says so rather than rounding it away: open tracking measures a
+ * floor, not a count, and claiming otherwise would be the first dishonest
+ * sentence on it.
  */
 
 const LADDER = [
@@ -161,7 +163,7 @@ export default function LeadGenerationPage() {
           />
           <Card
             title="Follow-ups that stop"
-            body="At most two follow-ups on a schedule, and marking a lead replied ends the sequence. Nobody gets a fourth email."
+            body="At most two follow-ups on a schedule, and a reply ends the sequence — detected from your mailbox, not waiting on you to log it. Nobody gets a fourth email."
           />
         </div>
       </section>
@@ -200,13 +202,40 @@ export default function LeadGenerationPage() {
         </p>
       </section>
 
+      <section className="mt-20">
+        <h2 className="text-2xl font-bold">It reads the answers too</h2>
+        <p className="mt-2 max-w-3xl text-[var(--color-muted)]">
+          The same mailbox connection that sends is checked for what comes back, so a lead that
+          answers is marked as answered without anybody remembering to do it — and the follow-up
+          sequence stops on its own.
+        </p>
+        <div className="mt-6 grid gap-4 sm:grid-cols-2">
+          <Card
+            title="Out-of-office is not a reply"
+            body="Auto-responders, bounces and mailing-list traffic are recorded and flagged rather than counted. Counting them would inflate the one number worth reporting."
+          />
+          <Card
+            title="Answers from a colleague still count"
+            body="Mail to info@ often gets answered by a named person. A reply from anyone at the company is matched back to the lead it belongs to."
+          />
+          <Card
+            title="Opens, honestly"
+            body="A per-email pixel, with loads from privacy proxies discarded rather than counted. It measures a floor: the real number is higher, never lower."
+          />
+          <Card
+            title="Nothing is marked by hand"
+            body="Sent, opened, replied and closed are recorded as they happen — so a reply rate of zero means nobody answered, not that nobody logged it."
+          />
+        </div>
+      </section>
+
       <section className="mt-20">
         <h2 className="text-2xl font-bold">Your numbers, not somebody else&apos;s</h2>
         <p className="mt-2 max-w-3xl text-[var(--color-muted)]">
-          Sends, replies and closes are counted at three levels: the whole project, each campaign,
-          and each individual run. Rates stay hidden until there is enough volume to mean anything —
-          one reply out of three sends is not a 33% reply rate, and showing it as one invites a
-          decision the sample cannot support.
+          Sends, opens, replies and closes are counted at three levels: the whole project, each
+          campaign, and each individual run. Rates stay hidden until there is enough volume to mean
+          anything — one reply out of three sends is not a 33% reply rate, and showing it as one
+          invites a decision the sample cannot support.
         </p>
       </section>
 

app/api/cron/outreach-replies/route.ts

@@ -0,0 +1,47 @@
+import { NextResponse } from "next/server";
+import { env } from "@/lib/env";
+import { scanAllMailboxes } from "@/lib/outreach/replyScan";
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+// IMAP is a conversation, not a request: connect, authenticate, select, fetch.
+// A busy mailbox on a slow server can take a while and the default budget is
+// not generous enough to finish.
+export const maxDuration = 300;
+
+/**
+ * Read the connected mailboxes and mark who replied.
+ *
+ * Everything past "contacted" used to depend on a human marking it, which
+ * means anyone who did not got a reply rate of structurally zero — a number
+ * that reads as a verdict on the campaign when it is really a verdict on the
+ * bookkeeping.
+ *
+ * Auto-replies are recorded and flagged rather than counted. An out-of-office
+ * is a real fact about the recipient and not an answer to the pitch.
+ */
+export async function GET(req: Request) {
+  return POST(req);
+}
+
+export async function POST(req: Request) {
+  const incoming =
+    req.headers.get("x-cron-secret") ??
+    req.headers.get("authorization")?.replace(/^Bearer\s+/i, "");
+  if (incoming !== env.cronSecret) {
+    return NextResponse.json({ ok: false, error: "unauthorized" }, { status: 401 });
+  }
+
+  const results = await scanAllMailboxes();
+
+  return NextResponse.json({
+    ok: true,
+    mailboxes: results.length,
+    scanned: results.reduce((n, r) => n + r.scanned, 0),
+    replies: results.reduce((n, r) => n + r.matched, 0),
+    autoReplies: results.reduce((n, r) => n + r.autoReplies, 0),
+    // Per mailbox, so one broken credential is visible rather than being
+    // averaged into an otherwise healthy total.
+    results,
+  });
+}

app/api/o/[token]/route.ts

@@ -0,0 +1,45 @@
+import { NextResponse } from "next/server";
+import { PIXEL_GIF, recordOpen } from "@/lib/outreach/openTracking";
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+/**
+ * The open-tracking pixel.
+ *
+ * Always 200s with the same one-pixel GIF, whatever the token turns out to be.
+ * A 404 for an unknown token would let anyone test tokens for validity, and a
+ * broken image in somebody's inbox is a worse outcome than a lost datapoint.
+ *
+ * Recording happens before the response for the same reason a send is logged
+ * before it is reported: an open that is served but not counted is invisible,
+ * and there is no retry for an image load.
+ */
+export async function GET(
+  req: Request,
+  { params }: { params: Promise<{ token: string }> },
+) {
+  const { token } = await params;
+
+  try {
+    await recordOpen({
+      // The .gif suffix is cosmetic — it exists so the URL reads as an image.
+      token: token.replace(/\.(gif|png)$/i, ""),
+      userAgent: req.headers.get("user-agent"),
+    });
+  } catch {
+    // A failure to record is not a reason to break the email.
+  }
+
+  return new NextResponse(new Uint8Array(PIXEL_GIF), {
+    status: 200,
+    headers: {
+      "content-type": "image/gif",
+      "content-length": String(PIXEL_GIF.length),
+      // Every layer asked not to cache: a cached pixel is one open forever.
+      "cache-control": "no-store, no-cache, must-revalidate, private, max-age=0",
+      pragma: "no-cache",
+      expires: "0",
+    },
+  });
+}

components/leads/funnel-panel.tsx

@@ -21,6 +21,12 @@ function Row({ label, f }: { label: string; f: FunnelCounts }) {
     <tr className="border-t border-[var(--color-border)]">
       <td className="py-2 pr-3 font-medium">{label}</td>
       <td className="py-2 pr-3 text-right font-mono">{f.sent}</td>
+      <td className="py-2 pr-3 text-right font-mono">
+        {f.tracked ? f.opened : "—"}
+        {f.openRate !== null && (
+          <span className="ml-1 text-xs text-[var(--color-muted)]">{pct(f.openRate)}</span>
+        )}
+      </td>
       <td className="py-2 pr-3 text-right font-mono">{f.contacted}</td>
       <td className="py-2 pr-3 text-right font-mono">{f.replied}</td>
       <td className="py-2 pr-3 text-right font-mono">{f.won}</td>
@@ -59,11 +65,12 @@ export function FunnelPanel({
       </div>
 
       <div className="mt-3 overflow-x-auto">
-        <table className="w-full min-w-[34rem] text-sm">
+        <table className="w-full min-w-[40rem] text-sm">
           <thead>
             <tr className="text-xs text-[var(--color-muted)]">
               <th className="pb-1 text-left font-medium">Campaign</th>
               <th className="pb-1 pr-3 text-right font-medium">Sent</th>
+              <th className="pb-1 pr-3 text-right font-medium">Opened</th>
               <th className="pb-1 pr-3 text-right font-medium">People</th>
               <th className="pb-1 pr-3 text-right font-medium">Replied</th>
               <th className="pb-1 pr-3 text-right font-medium">Won</th>
@@ -89,8 +96,14 @@ export function FunnelPanel({
 
       <p className="mt-2 text-xs text-[var(--color-muted)]">
         Reply rate is of people contacted; close rate is of people who replied, since a deal comes
-        out of a conversation. Replies are recorded when you mark a lead — nothing reads your inbox
-        yet.
+        out of a conversation. Replies are read from your connected mailbox — out-of-office and
+        bounce messages are recorded but not counted as replies.
+      </p>
+
+      <p className="mt-2 text-xs text-[var(--color-muted)]">
+        Opens are a floor, not a count. Mail clients that block images report nothing, and loads
+        from privacy proxies are discarded rather than counted — so the real number is higher than
+        this one, never lower. Open rate is of tracked sends only.
       </p>
     </section>
   );

components/leads/replies-panel.tsx

@@ -0,0 +1,50 @@
+export type ReplyRow = {
+  from_email: string;
+  subject: string | null;
+  snippet: string | null;
+  received_at: string;
+  auto_reply: boolean;
+};
+
+/**
+ * What came back.
+ *
+ * The funnel above reports how many replied; this is the part that decides
+ * what to do next, which is never a count. Auto-replies are shown rather than
+ * hidden — an out-of-office is worth knowing about even though it is not an
+ * answer — but they are visibly separated, because burying them in the same
+ * list is how a reply rate ends up meaning nothing.
+ */
+export function RepliesPanel({ replies }: { replies: ReplyRow[] }) {
+  if (!replies.length) return null;
+
+  return (
+    <section className="card p-4">
+      <div className="flex flex-wrap items-baseline justify-between gap-2">
+        <h2 className="text-lg font-semibold">Replies</h2>
+        <p className="text-xs text-[var(--color-muted)]">
+          Read from your connected mailbox. Answer from your own mail client — this is a record,
+          not an inbox.
+        </p>
+      </div>
+
+      <ul className="mt-3 divide-y divide-[var(--color-border)]">
+        {replies.map((r, i) => (
+          <li key={`${r.received_at}-${i}`} className="py-3">
+            <p className="flex flex-wrap items-center gap-2 text-sm font-medium">
+              {r.from_email}
+              {r.auto_reply && <span className="badge badge-unknown">auto-reply</span>}
+              <span className="font-mono text-xs font-normal text-[var(--color-muted)]">
+                {r.received_at.slice(0, 16).replace("T", " ")}
+              </span>
+            </p>
+            {r.subject && <p className="mt-1 text-sm">{r.subject}</p>}
+            {r.snippet && (
+              <p className="mt-1 text-xs text-[var(--color-muted)]">{r.snippet}</p>
+            )}
+          </li>
+        ))}
+      </ul>
+    </section>
+  );
+}

lib/email.ts

@@ -948,6 +948,11 @@ export function coldOutreachEmailHtml(input: {
   reportUrl: string | null;
   unsubscribeUrl: string;
   postalAddress: string;
+  /**
+   * Open-tracking pixel markup, when this send is being tracked. Absent for
+   * dry runs, which are never delivered and so can never be opened.
+   */
+  trackingPixel?: string | null;
 }): string {
   const paragraphs = input.bodyText
     .split(/\n{2,}/)
@@ -973,13 +978,21 @@ export function coldOutreachEmailHtml(input: {
           </tr>`
     : "";
 
+  // Last, after the visible content. A client that stops rendering partway
+  // through has already shown the mail, and one that never reaches the pixel
+  // was never going to report a meaningful open anyway.
+  const pixelRow = input.trackingPixel
+    ? `<tr><td style="padding:0;line-height:0;font-size:0;">${input.trackingPixel}</td></tr>`
+    : "";
+
   const innerHtml = `<tr>
             <td style="padding:24px 32px 0;">
               ${paragraphs}
             </td>
           </tr>
           ${reportBlock}
-          <tr><td style="padding:0 32px 20px;"></td></tr>`;
+          <tr><td style="padding:0 32px 20px;"></td></tr>
+          ${pixelRow}`;
 
   // Why-you-got-this, opt-out, postal address — the three things a cold
   // commercial email legally and ethically has to carry.
@@ -1056,7 +1069,7 @@ export async function sendColdOutreachEmail(input: {
    * replies come back to them; omitting it uses the shared Resend sender.
    */
   mailbox?: SenderMailbox | null;
-}): Promise<{ sent: boolean; error?: string; provider?: string }> {
+}): Promise<{ sent: boolean; error?: string; provider?: string; messageId?: string }> {
   const headers = {
     "List-Unsubscribe": `<${input.unsubscribeUrl}>`,
     "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",
@@ -1073,7 +1086,10 @@ export async function sendColdOutreachEmail(input: {
         secure: input.mailbox.secure,
         auth: { user: input.mailbox.user, pass: input.mailbox.pass },
       });
-      await transporter.sendMail({
+      // The Message-Id is what a reply names in In-Reply-To. Keeping it is
+      // what lets reply detection match by thread rather than by guessing
+      // from the sender's domain.
+      const info = await transporter.sendMail({
         from: input.mailbox.fromEmail,
         to: input.to,
         subject: input.subject,
@@ -1082,7 +1098,7 @@ export async function sendColdOutreachEmail(input: {
         headers,
       });
       transporter.close();
-      return { sent: true, provider: "mailbox" };
+      return { sent: true, provider: "mailbox", messageId: info.messageId };
     } catch (error) {
       // No silent fallback to the shared sender: the user asked for mail to
       // come from their address, and quietly sending as someone else would