From 204109199dde3d78ed70a157b5a7f5a63107d172 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Tue, 28 Jul 2026 16:20:59 +0000 Subject: [PATCH] feat(leads): qualify on public buying intent, not resemblance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feedback from someone who had already built a business on this premise: the expensive part of cold outreach is not assembling a list, it is that writing to people who have shown no intent converts badly however good the copy is. Firmographic targeting does not fix that — it selects for people who look like buyers rather than people who are buying. So a campaign can now require evidence that somebody actually asked, and those leads rank ahead of everything found by resemblance. Intent is scored as recency times explicitness. A named budget outranks a request for suggestions, which outranks a bare complaint, and someone leaving their current supplier ranks near the top because the budget already exists and the decision to spend it has been made. All of it decays with a half-life rather than falling off a cliff at a threshold: a 25-hour-old post is not meaningfully worse than a 23-hour-old one. Not only Reddit. The same sentence turns up on Hacker News, Stack Exchange, Quora, Indie Hackers, X, LinkedIn and contract boards, and a pipeline watching one community inherits its demographics as the entire market. Everything except Reddit is read through the search index rather than a platform API — no per-platform auth, rate limits or review, at the cost of an indexing lag that makes hours-old intent reliable and minutes-old intent occasional. Three things the scorer refuses. Intent on somebody else's topic: a flawless "anyone recommend an accountant, happy to pay" is worth nothing to a load-testing company. Anyone who wrote "no vendors" or "don't DM me", whatever else they scored — answering them is the fastest way to lose a channel for everybody. And job ads, which read exactly like budget and are not. Signals are stored as signals rather than prospects. A prospect is a company with a domain you can email; this is a person on a platform. My first pass filed them as prospects and gave every Reddit thread the target key "reddit.com", collapsing the queue into a single lead. Off unless switched on, so no running campaign is silently emptied by a feature it never opted into. Co-Authored-By: Claude Opus 5 (1M context) --- app/(app)/projects/[id]/leads/page.tsx | 16 +- app/(marketing)/lead-generation/page.tsx | 39 ++- app/actions/leads.ts | 12 + components/leads/campaign-panel.tsx | 42 +++ components/leads/intent-panel.tsx | 79 ++++++ lib/outreach/intent.ts | 266 ++++++++++++++++++ lib/outreach/intentSources.ts | 254 +++++++++++++++++ lib/outreach/runner.ts | 61 +++- .../20260729060000_intent_qualification.sql | 84 ++++++ tests/intent.test.ts | 183 ++++++++++++ 10 files changed, 1033 insertions(+), 3 deletions(-) create mode 100644 components/leads/intent-panel.tsx create mode 100644 lib/outreach/intent.ts create mode 100644 lib/outreach/intentSources.ts create mode 100644 supabase/migrations/20260729060000_intent_qualification.sql create mode 100644 tests/intent.test.ts diff --git a/app/(app)/projects/[id]/leads/page.tsx b/app/(app)/projects/[id]/leads/page.tsx index 0469689a..26130eec 100644 --- a/app/(app)/projects/[id]/leads/page.tsx +++ b/app/(app)/projects/[id]/leads/page.tsx @@ -12,6 +12,7 @@ import { SeedLogins } from "@/components/leads/seed-logins"; import { FunnelPanel } from "@/components/leads/funnel-panel"; import { RepliesPanel, type ReplyRow } from "@/components/leads/replies-panel"; import { ContactsPanel } from "@/components/leads/contacts-panel"; +import { IntentPanel, type IntentSignalRow } from "@/components/leads/intent-panel"; import { contactNiches } from "@/lib/outreach/contactsExport"; import { campaignFunnels, projectFunnel } from "@/lib/outreach/funnel"; import { listSeedCredentials, type StoredSeedCredential } from "@/lib/outreach/seedCredentials"; @@ -91,7 +92,7 @@ export default async function LeadsPage({ supabase .from("outreach_campaigns") .select( - "id, name, active, auto_send, daily_send_limit, max_score, queries, seed_urls, last_run_at, last_run_note, auth_required_hosts, pitch_mode, pitch_intro, pitch_ask, pitch_facts, scan_prospects, angle, sender_name, reply_to", + "id, name, active, auto_send, daily_send_limit, max_score, queries, seed_urls, last_run_at, last_run_note, auth_required_hosts, pitch_mode, pitch_intro, pitch_ask, pitch_facts, scan_prospects, min_intent, angle, sender_name, reply_to", ) .eq("project_id", projectId) .order("updated_at", { ascending: false }) @@ -180,6 +181,17 @@ export default async function LeadsPage({ .limit(20); const replies = (replyData as ReplyRow[] | null) ?? []; + // Public requests to buy, strongest and freshest first. A separate queue + // from the pipeline: these are people on a platform, not domains to email. + const { data: intentData } = await supabase + .from("outreach_intent_signals") + .select("source, url, title, snippet, score, tier, reasons, posted_at, status") + .eq("project_id", projectId) + .neq("status", "dismissed") + .order("score", { ascending: false }) + .limit(25); + const intentSignals = (intentData as IntentSignalRow[] | null) ?? []; + let seedCredentials: StoredSeedCredential[] = []; if (orgId) seedCredentials = await listSeedCredentials(orgId); @@ -239,6 +251,8 @@ export default async function LeadsPage({ + + diff --git a/app/(marketing)/lead-generation/page.tsx b/app/(marketing)/lead-generation/page.tsx index bc561970..71416dd7 100644 --- a/app/(marketing)/lead-generation/page.tsx +++ b/app/(marketing)/lead-generation/page.tsx @@ -5,7 +5,7 @@ import { LEADS_PER_CHARGE } from "@/lib/outreach/billing"; export const metadata = { title: "Lead generation & cold outreach", description: - "Point CrawlProof at a directory or a search, and it finds the companies, finds the people, finds the addresses, writes the email, and reads the replies — from your own mailbox, grounded in facts you supply. 3 credits a run.", + "Watch for people publicly asking to buy what you sell, and work them first. Then find the companies, the people and the addresses, write the email and read the replies — from your own mailbox, grounded in facts you supply.", alternates: { canonical: "/lead-generation" }, openGraph: { title: "Lead generation & cold outreach · CrawlProof", @@ -85,6 +85,43 @@ export default function LeadGenerationPage() {

+
+

Start with people who already asked

+

+ The hard part of cold outreach is not building the list. It is that writing to people who + have shown no intent converts badly however good the email is, and targeting by + resemblance — right industry, right headcount, right tech stack — does not fix that. It + selects for people who look like buyers, not people who are buying. +

+

+ So a campaign can watch for the opposite signal: somebody saying, in public and recently, + that they want to buy what you sell. Those leads are ranked ahead of everything found by + resemblance, because they are a different kind of lead rather than a better-matched one. +

+
+ + + + +
+

+ Reply where they asked, in public, as a reply — these are people on a platform, not + addresses to mail. The email pipeline below is for the companies you go and find. +

+
+

It reads the directory like a person would

diff --git a/app/actions/leads.ts b/app/actions/leads.ts index 7a31e350..9ab0aa0e 100644 --- a/app/actions/leads.ts +++ b/app/actions/leads.ts @@ -505,6 +505,12 @@ export async function saveCampaignAction(input: { pitchAsk?: string; pitchFacts?: string; scanProspects?: boolean; + /** + * Minimum public buying intent a lead must show, 0-100. Null or absent + * leaves the campaign selecting on resemblance alone, which is what every + * campaign did before intent existed. + */ + minIntent?: number | null; }): Promise | Err> { const auth = await requireLeadAccess(input.projectId); if (!auth.ok) return auth; @@ -572,6 +578,12 @@ export async function saveCampaignAction(input: { max_score: input.maxScore, daily_send_limit: input.dailySendLimit, auto_send: input.autoSend, + // Clamped rather than rejected: a bar above 100 silently matches + // nothing, which looks like the sweep being broken. + min_intent: + input.minIntent === null || input.minIntent === undefined + ? null + : Math.max(0, Math.min(100, Math.round(input.minIntent))), angle: input.angle?.trim() || null, sender_name: input.senderName?.trim() || null, reply_to: input.replyTo?.trim() || null, diff --git a/components/leads/campaign-panel.tsx b/components/leads/campaign-panel.tsx index 5e411ed9..774cace4 100644 --- a/components/leads/campaign-panel.tsx +++ b/components/leads/campaign-panel.tsx @@ -35,6 +35,7 @@ export type CampaignSummary = { pitch_ask: string | null; pitch_facts: string[]; scan_prospects: boolean; + min_intent: number | null; auth_required_hosts: string[]; runs: CampaignRun[]; }; @@ -83,6 +84,9 @@ export function CampaignPanel({ // scan of the prospect to contribute. Selecting the audit pitch turns it // back on, because that pitch is built from findings. const [scanProspects, setScanProspects] = useState(false); + // Null means "don't qualify on intent" — the behaviour every campaign had + // before this existed, and the default for anything already running. + const [minIntent, setMinIntent] = useState(null); const [goal, setGoal] = useState(""); const [generating, setGenerating] = useState(false); const [pitchNote, setPitchNote] = useState(null); @@ -159,6 +163,7 @@ export function CampaignPanel({ setPitchAsk(c.pitch_ask ?? ""); setPitchFacts((c.pitch_facts ?? []).join("\n")); setScanProspects(c.scan_prospects); + setMinIntent(c.min_intent ?? null); // Every field the form submits has to be loaded, not just the ones that // changed recently: save() sends the whole form, so anything left blank // here is written back as blank and silently wipes the column. @@ -191,6 +196,7 @@ export function CampaignPanel({ pitchAsk, pitchFacts, scanProspects, + minIntent, }); if (result.ok) setEditing(null); return result; @@ -492,6 +498,42 @@ export function CampaignPanel({ )} + + + {minIntent !== null && ( + + )} +

+
+

Asked to buy

+

+ Ranked by how explicitly they asked and how recently. Reply where they asked — not by + email. +

+
+ +
    + {signals.map((s) => ( +
  • +

    + {s.tier} + {s.score}/100 + + {s.source} · {age(s.posted_at)} + +

    + + {s.title || s.url} + + {s.snippet && ( +

    {s.snippet}

    + )} + {s.reasons.length > 0 && ( +

    {s.reasons.join(" · ")}

    + )} +
    • + ))} +
+
+ ); +} diff --git a/lib/outreach/intent.ts b/lib/outreach/intent.ts new file mode 100644 index 00000000..41836fce --- /dev/null +++ b/lib/outreach/intent.ts @@ -0,0 +1,266 @@ +// Buying intent, as a signal you can sort by. +// +// The premise this encodes, which came from someone who had already built a +// business on it: cold outreach to people who have shown no intent converts +// badly regardless of how good the copy is, and no amount of firmographic +// targeting fixes that. Someone who posted "anyone know a good X, happy to +// pay" four hours ago is a different prospect from a company that merely +// resembles your customers — not a better-matched one, a categorically +// different one. +// +// So intent and recency are scored as the primary signal, and everything the +// rest of the pipeline does well — finding addresses, grounding drafts — is +// downstream of choosing who to write to at all. +// +// Deliberately source-agnostic. A Reddit thread, a forum post, a job ad, a +// tweet and a status page all reduce to "someone said this, at this time", and +// a scorer that only understands one platform makes every new source a rewrite. +// The adapters live in intentSources.ts; this file knows nothing about where +// the words came from. + +/** Hours after which a signal has decayed to roughly half its strength. */ +const HALF_LIFE_HOURS = 36; +/** Past this, treat a signal as stale regardless of how strong it was. */ +const MAX_AGE_HOURS = 24 * 14; + +export type IntentTier = "purchase" | "solicitation" | "switching" | "pain" | "none"; + +type TierSpec = { + tier: Exclude; + /** Base score before recency is applied. */ + weight: number; + label: string; + patterns: RegExp[]; +}; + +/** + * Ordered strongest first. The tiers are distinct kinds of statement, not + * degrees of the same one: + * + * - purchase: money is already on the table. + * - switching: they have a supplier and are leaving. The budget exists and + * the decision to spend it has been made — which is why this outranks a + * general request for suggestions. + * - solicitation: asking the room for a recommendation. + * - pain: describing a problem without asking for anything. + */ +const TIERS: TierSpec[] = [ + { + tier: "purchase", + weight: 100, + label: "ready to pay", + patterns: [ + /\b(?:looking|want|wanting|ready|keen)\s+to\s+(?:buy|purchase|pay)\b/i, + /\b(?:willing|happy|glad)\s+to\s+pay\b/i, + /\b(?:we|i)\s+(?:have|got)\s+(?:a\s+)?budget\b/i, + /\bbudget\s+(?:of|for|is|around|up to)\b/i, + /\b(?:take|shut up and take)\s+my\s+money\b/i, + /\b(?:request for proposal|rfp|rfq)\b/i, + /\b(?:get|need|send|want)\s+(?:a\s+)?(?:quote|quotes|pricing)\b/i, + /\bwho\s+(?:do|should)\s+(?:i|we)\s+pay\b/i, + /\b(?:hire|hiring|contract|engage)\s+(?:someone|a\s+\w+|an\s+agency|a\s+freelancer)\b/i, + /\bpaid\s+(?:tool|service|plan|option|solution)\b/i, + /\bhappy\s+to\s+(?:pay|spend)\b/i, + ], + }, + { + tier: "switching", + weight: 85, + label: "leaving their current supplier", + patterns: [ + /\b(?:migrating|moving|switching|shifting)\s+(?:away\s+)?(?:from|off)\b/i, + /\blooking\s+(?:to|for)\s+(?:a\s+)?replace(?:ment)?\b/i, + /\b(?:replacing|replace)\s+(?:our|my|the)\b/i, + /\b(?:cancelled|canceling|cancelling|dropping|ditching)\s+(?:our|my)\b/i, + /\balternatives?\s+to\b/i, + /\bfed\s+up\s+with\b/i, + /\bcontract\s+(?:is\s+)?(?:up|expiring|ending|renewing)\b/i, + ], + }, + { + tier: "solicitation", + weight: 70, + label: "asking for a recommendation", + patterns: [ + /\b(?:anyone|anybody)\s+(?:know|use|used|recommend|tried)\b/i, + /\b(?:can|could)\s+(?:anyone|anybody|someone)\s+recommend\b/i, + /\brecommendations?\s+for\b/i, + /\b(?:looking|searching|hunting)\s+for\s+(?:a|an|some)\b/i, + /\bin\s+search\s+of\b/i, + /\bsuggestions?\s+for\b/i, + /\bwhat(?:'s| is| are)\s+the\s+best\b/i, + /\bwhich\s+(?:tool|service|vendor|provider|agency|platform)\b/i, + /\bwhat\s+(?:do|are)\s+you\s+(?:all\s+)?(?:use|using)\b/i, + /\bany\s+(?:tools?|services?|vendors?|agencies)\s+that\b/i, + ], + }, + { + tier: "pain", + weight: 45, + label: "describing a problem", + patterns: [ + /\b(?:struggling|struggle)\s+with\b/i, + /\b(?:sick|tired)\s+of\b/i, + /\b(?:frustrated|frustrating)\b/i, + /\bcan(?:'|no)?t\s+(?:figure|get|work)\b/i, + /\b(?:keeps|constantly)\s+(?:failing|breaking|crashing)\b/i, + /\b(?:manual|by hand)\s+(?:process|work|workflow)\b/i, + /\b(?:wasting|spending)\s+(?:hours|days|so much time)\b/i, + /\bis\s+there\s+(?:a|any)\s+(?:tool|way|service)\b/i, + ], + }, +]; + +/** + * Statements that mean this is not a lead however well it otherwise scores. + * + * These are cheap to check and expensive to get wrong in the other direction: + * messaging someone who wrote "no vendors please" is the fastest way to lose + * a channel for everybody. + */ +const DISQUALIFIERS: Array<{ pattern: RegExp; reason: string }> = [ + { pattern: /\bno\s+(?:vendors?|sales|solicitation|pitches|promos?|ads?)\b/i, reason: "asked not to be pitched" }, + { pattern: /\b(?:do\s*n(?:o|')t|please\s+don'?t)\s+(?:dm|pm|message|contact|email)\s+me\b/i, reason: "asked not to be contacted" }, + { pattern: /\bnot\s+looking\s+(?:to\s+(?:buy|pay)|for\s+(?:a\s+)?(?:vendor|tool|service))\b/i, reason: "explicitly not buying" }, + { pattern: /\b(?:free|open[- ]source)\s+only\b/i, reason: "free/open-source only" }, + { pattern: /\bno\s+paid\b/i, reason: "free/open-source only" }, + { pattern: /\bthis\s+is\s+(?:an?\s+)?(?:ad|advert|sponsored)\b/i, reason: "the post is itself an advert" }, + { pattern: /\bwe(?:'re| are)\s+hiring\b/i, reason: "a job ad, not a buyer" }, +]; + +export type IntentSignal = { + /** 0–100, recency already applied. */ + score: number; + tier: IntentTier; + /** Why it scored what it did, in the user's words where possible. */ + reasons: string[]; + /** Set when the text rules the lead out entirely. */ + disqualified: string | null; + /** Age in hours, or null when the source gave no timestamp. */ + ageHours: number | null; +}; + +/** + * How much of a signal's strength survives its age. + * + * Halves every HALF_LIFE_HOURS rather than falling off a cliff at some + * threshold, because a 25-hour-old post is not meaningfully worse than a + * 23-hour-old one and a step function would rank them as if it were. + */ +export function recencyFactor(ageHours: number | null): number { + // No timestamp is not the same as old. Most scraped directory entries have + // no date at all, and pretending they are fresh would rank them above a + // real post from yesterday — so they take a flat penalty instead. + if (ageHours === null) return 0.55; + if (ageHours < 0) return 1; + if (ageHours > MAX_AGE_HOURS) return 0; + return Math.pow(0.5, ageHours / HALF_LIFE_HOURS); +} + +/** + * Score a piece of public text for buying intent. + * + * `keywords` are the campaign's own topic words. Intent without topic match is + * somebody else's lead: "anyone recommend a good accountant" is a perfect + * solicitation and worthless to a company selling load testing. + */ +export function scoreIntent(input: { + text: string; + /** When it was posted. Null when the source does not say. */ + postedAt?: Date | null; + /** Campaign topic words. At least one must appear. */ + keywords?: string[]; + negativeKeywords?: string[]; + now?: Date; +}): IntentSignal { + const text = (input.text ?? "").slice(0, 8000); + const now = input.now ?? new Date(); + const ageHours = input.postedAt + ? Math.max(0, (now.getTime() - input.postedAt.getTime()) / 3_600_000) + : null; + + const base: IntentSignal = { score: 0, tier: "none", reasons: [], disqualified: null, ageHours }; + if (!text.trim()) return { ...base, disqualified: "no text to read" }; + + for (const d of DISQUALIFIERS) { + if (d.pattern.test(text)) return { ...base, disqualified: d.reason }; + } + + const negatives = (input.negativeKeywords ?? []).filter((k) => + k.trim() ? text.toLowerCase().includes(k.toLowerCase()) : false, + ); + if (negatives.length) { + return { ...base, disqualified: `negative match: ${negatives.join(", ")}` }; + } + + const keywords = (input.keywords ?? []).filter((k) => k.trim()); + if (keywords.length) { + const hits = keywords.filter((k) => text.toLowerCase().includes(k.toLowerCase())); + if (!hits.length) { + return { ...base, disqualified: "on-topic for nobody — no campaign keyword appears" }; + } + base.reasons.push(`mentions ${hits.slice(0, 3).join(", ")}`); + } + + const matched = TIERS.filter((t) => t.patterns.some((p) => p.test(text))); + if (!matched.length) { + return { + ...base, + score: 0, + tier: "none", + disqualified: null, + reasons: [...base.reasons, "no expression of intent — nobody asked for anything"], + }; + } + + // The strongest statement sets the score; the others add a little, because + // someone who is both leaving a supplier and asking for recommendations is + // a better prospect than someone doing either alone — but not twice as good. + const strongest = matched[0]; + const extra = matched.slice(1).reduce((n, t) => n + t.weight * 0.08, 0); + const raw = Math.min(100, strongest.weight + extra); + + const factor = recencyFactor(ageHours); + const reasons = [...base.reasons, strongest.label]; + for (const t of matched.slice(1)) reasons.push(t.label); + + if (ageHours === null) { + reasons.push("no date on the source, so treated as stale"); + } else if (ageHours <= 6) { + reasons.push("posted in the last few hours"); + } else if (ageHours <= 24) { + reasons.push("posted today"); + } else { + reasons.push(`${Math.round(ageHours / 24)}d old`); + } + + return { + score: Math.round(raw * factor), + tier: strongest.tier, + reasons, + disqualified: factor === 0 ? `older than ${Math.round(MAX_AGE_HOURS / 24)} days` : null, + ageHours, + }; +} + +/** Default bar for acting on a signal. Roughly "a fresh solicitation". */ +export const DEFAULT_MIN_INTENT = 40; + +/** + * Whether a lead clears a campaign's bar. + * + * A campaign with no bar set keeps the old behaviour, so turning intent + * scoring on does not silently empty anybody's pipeline. + */ +export function qualifies(signal: IntentSignal, minIntent: number | null | undefined): boolean { + if (signal.disqualified) return false; + if (minIntent === null || minIntent === undefined) return true; + return signal.score >= minIntent; +} + +/** One-line explanation for the run log and the leads table. */ +export function describeIntent(signal: IntentSignal): string { + if (signal.disqualified) return `no intent: ${signal.disqualified}`; + if (signal.tier === "none") return "no intent signal"; + return `${signal.score}/100 ${signal.tier} — ${signal.reasons.slice(0, 3).join("; ")}`; +} diff --git a/lib/outreach/intentSources.ts b/lib/outreach/intentSources.ts new file mode 100644 index 00000000..fbac300c --- /dev/null +++ b/lib/outreach/intentSources.ts @@ -0,0 +1,254 @@ +// Where public buying intent gets expressed, and how to go and read it. +// +// Reddit is the obvious one and the one this codebase already talks to +// natively, but it is not the only place somebody says "we need X, who should +// we pay". The same sentence turns up on Hacker News, Stack Exchange, Quora, +// X, LinkedIn, Mastodon, Indie Hackers and a long tail of forums — and a +// pipeline that only watches one of them inherits that one community's +// demographics as its entire market. +// +// Everything except Reddit is read through the search index rather than a +// platform API. That is a deliberate trade: platform APIs mean per-platform +// auth, per-platform rate limits, per-platform review processes and a +// per-platform reason to break, and the search index already crawls all of +// them with a date filter attached. It costs one SERP call per site per run +// and needs no credentials at all. +// +// The honest limitation, stated here because it decides how the results should +// be read: a search index lags. Something posted twenty minutes ago may not be +// indexed yet, so this finds hours-old intent reliably and minutes-old intent +// only sometimes. Reddit's own API does not have that lag, which is why it +// stays a first-class source rather than another site: filter. + +import { searchSerp } from "@/lib/alerts/valueserp"; +import { scoreIntent, type IntentSignal } from "./intent"; + +export type IntentSource = { + id: string; + label: string; + /** Domains to restrict a search to. */ + sites: string[]; + /** Off by default when a platform rewards volume over relevance. */ + defaultOn: boolean; +}; + +/** + * The places worth watching, and why each is on the list. + * + * Ordered by how often people state a budget outright. Q&A sites and founder + * communities do it constantly; the big social networks do it rarely but at + * enough volume to be worth one call. + */ +export const INTENT_SOURCES: IntentSource[] = [ + { + id: "reddit", + label: "Reddit", + sites: ["reddit.com"], + defaultOn: true, + }, + { + id: "forums", + label: "Q&A and developer forums", + sites: ["news.ycombinator.com", "stackexchange.com", "serverfault.com", "superuser.com"], + defaultOn: true, + }, + { + id: "founders", + label: "Founder and indie communities", + sites: ["indiehackers.com", "producthunt.com", "lobste.rs"], + defaultOn: true, + }, + { + id: "quora", + label: "Quora", + sites: ["quora.com"], + defaultOn: true, + }, + { + id: "x", + label: "X", + sites: ["x.com", "twitter.com"], + // Public search coverage of X is patchy since the API changes, so it is + // on but expected to be thin rather than reliable. + defaultOn: true, + }, + { + id: "linkedin", + label: "LinkedIn", + sites: ["linkedin.com"], + defaultOn: true, + }, + { + id: "mastodon", + label: "Mastodon and Bluesky", + sites: ["mastodon.social", "bsky.app"], + defaultOn: false, + }, + { + id: "jobs", + label: "Job and contract boards", + // A job ad for the thing you sell is a budget that has already been + // approved — it is intent wearing different clothes. + sites: ["upwork.com", "weworkremotely.com", "remoteok.com"], + defaultOn: false, + }, +]; + +/** + * Search phrasings that find a request rather than an article about one. + * + * A bare topic search returns listicles and vendor pages; these return people + * talking. Each is combined with the campaign's own topic words. + */ +export const INTENT_PHRASINGS = [ + '"anyone recommend"', + '"looking for a"', + '"any recommendations"', + '"willing to pay"', + '"alternatives to"', + '"what do you use"', +]; + +export type IntentHit = { + source: string; + url: string; + title: string; + snippet: string; + /** Parsed from the result where the index reports one. */ + postedAt: Date | null; + signal: IntentSignal; +}; + +/** + * Turn the date a search index reports into a timestamp. + * + * Indexes report these inconsistently — an absolute date on one result and "3 + * days ago" on the next — and a relative string is still worth parsing, + * because recency is the half of the score that a stale-but-strong post gets + * wrong. + */ +export function parseResultDate(raw: string | null | undefined, now = new Date()): Date | null { + if (!raw) return null; + const text = raw.trim(); + + const relative = text.match(/(\d+)\s*(minute|hour|day|week|month|year)s?\s+ago/i); + if (relative) { + const n = Number(relative[1]); + const unit = relative[2].toLowerCase(); + const ms = + unit === "minute" ? 60_000 + : unit === "hour" ? 3_600_000 + : unit === "day" ? 86_400_000 + : unit === "week" ? 604_800_000 + : unit === "month" ? 2_592_000_000 + : 31_536_000_000; + return new Date(now.getTime() - n * ms); + } + + const parsed = Date.parse(text); + if (!Number.isNaN(parsed)) { + const d = new Date(parsed); + // A date in the future is the index misreporting, not a scoop. + return d.getTime() > now.getTime() + 86_400_000 ? null : d; + } + return null; +} + +/** One search string per source, combining the topic with a request phrasing. */ +export function buildIntentQuery(input: { + source: IntentSource; + topic: string; + phrasing: string; +}): string { + const sites = input.source.sites.map((s) => `site:${s}`).join(" OR "); + return `(${sites}) ${input.topic} ${input.phrasing}`; +} + +export type IntentSweepResult = { + hits: IntentHit[]; + /** SERP calls actually spent, for the caller's budget. */ + calls: number; + notes: string[]; +}; + +/** + * Sweep the enabled sources for people asking to buy. + * + * Bounded by an explicit call budget rather than by the number of sources, so + * adding a platform to the list above cannot quietly multiply what a run + * costs. + */ +export async function sweepIntent(input: { + topic: string; + keywords: string[]; + negativeKeywords?: string[]; + sources?: string[]; + /** Maximum SERP calls this sweep may spend. */ + maxCalls: number; + /** Only consider posts newer than this. */ + recency?: "day" | "week" | "month"; + minIntent?: number; + now?: Date; +}): Promise { + const now = input.now ?? new Date(); + const enabled = INTENT_SOURCES.filter((s) => + input.sources?.length ? input.sources.includes(s.id) : s.defaultOn, + ); + const notes: string[] = []; + const seen = new Set(); + const hits: IntentHit[] = []; + let calls = 0; + + // One phrasing at a time across every source, rather than every phrasing on + // the first source — so a small budget still covers the whole list instead + // of exhausting itself on Reddit. + outer: for (const phrasing of INTENT_PHRASINGS) { + for (const source of enabled) { + if (calls >= input.maxCalls) { + notes.push(`stopped at the ${input.maxCalls}-search budget`); + break outer; + } + const res = await searchSerp({ + query: buildIntentQuery({ source, topic: input.topic, phrasing }), + recency: input.recency ?? "week", + num: 20, + }); + calls += res.calls; + if (!res.ok) { + notes.push(`${source.label}: ${res.error ?? "search failed"}`); + continue; + } + + for (const r of res.results) { + if (!r.url || seen.has(r.url)) continue; + seen.add(r.url); + const postedAt = parseResultDate(r.date, now); + const signal = scoreIntent({ + // Title and snippet are all the index gives; it is also roughly what + // a human skims before deciding whether to open the thread. + text: `${r.title ?? ""}\n${r.snippet ?? ""}`, + postedAt, + keywords: input.keywords, + negativeKeywords: input.negativeKeywords, + now, + }); + if (signal.disqualified) continue; + if (signal.score < (input.minIntent ?? 0)) continue; + hits.push({ + source: source.id, + url: r.url, + title: r.title ?? "", + snippet: r.snippet ?? "", + postedAt, + signal, + }); + } + } + } + + // Strongest and freshest first — the score already folds both in, so this is + // simply the order to work them in. + hits.sort((a, b) => b.signal.score - a.signal.score); + notes.push(`${hits.length} intent signals from ${enabled.length} sources in ${calls} searches`); + return { hits, calls, notes }; +} diff --git a/lib/outreach/runner.ts b/lib/outreach/runner.ts index 55c16aba..4267e6db 100644 --- a/lib/outreach/runner.ts +++ b/lib/outreach/runner.ts @@ -26,6 +26,8 @@ import { import { nextStepReadyAt, type OutreachStep } from "./cold"; import { leadRunBilling, outOfCreditsNote } from "./billing"; import { recordDiscoveredPeople } from "./contacts"; +import { sweepIntent } from "./intentSources"; +import { describeIntent } from "./intent"; import { LEAD_RUN_CREDITS } from "@/lib/credits"; export type CampaignRow = { @@ -51,13 +53,17 @@ export type CampaignRow = { pitch_ask: string | null; pitch_facts: string[] | null; scan_prospects: boolean; + /** Minimum intent score a lead must show. Null = no intent gate. */ + min_intent: number | null; + intent_sources: string[] | null; + intent_recency: "day" | "week" | "month" | null; sender_name: string | null; reply_to: string | null; last_run_at: string | null; }; export const CAMPAIGN_COLUMNS = - "id, project_id, owner_id, name, channel, active, queries, seed_urls, keywords, subreddits, negative_keywords, max_score, daily_send_limit, target_pipeline, auto_send, follow_ups, angle, sender_name, reply_to, last_run_at, pitch_mode, pitch_intro, pitch_ask, pitch_facts, scan_prospects"; + "id, project_id, owner_id, name, channel, active, queries, seed_urls, keywords, subreddits, negative_keywords, max_score, daily_send_limit, target_pipeline, auto_send, follow_ups, angle, sender_name, reply_to, last_run_at, pitch_mode, pitch_intro, pitch_ask, pitch_facts, scan_prospects, min_intent, intent_sources, intent_recency"; export type TickResult = { campaign: string; @@ -75,6 +81,10 @@ export type TickResult = { creditsSpent: number; /** People named by this tick and written to the shared contact record. */ peopleRecorded: number; + /** Public requests to buy that this tick found and qualified. */ + intentFound: number; + /** Candidates dropped for showing no intent, when the campaign requires it. */ + intentRejected: number; skipped: string[]; errors: string[]; }; @@ -89,6 +99,10 @@ const MAX_SEND_PER_TICK = 5; // variable cost in a run — SERP calls scale with how many prospects publish // no address — so it gets a ceiling like every other per-tick stage. const MAX_CONTACT_SEARCHES_PER_TICK = 10; +// Searches a tick may spend sweeping for public buying intent. Small, because +// it runs every fifteen minutes and the window it looks at is days wide — the +// same request does not need finding four times an hour. +const MAX_INTENT_SEARCHES_PER_TICK = 6; export async function runEmailCampaignTick(campaign: CampaignRow): Promise { const sb = serviceClient(); @@ -103,6 +117,8 @@ export async function runEmailCampaignTick(campaign: CampaignRow): Promise new Date(NOW.getTime() - h * 3_600_000); +const kw = ["load testing"]; + +const score = (text: string, hours = 2) => + scoreIntent({ text, postedAt: agoHours(hours), keywords: kw, now: NOW }); + +describe("what counts as intent", () => { + it("ranks money on the table highest", () => { + const s = score("Anyone know a good load testing service? Willing to pay."); + expect(s.tier).toBe("purchase"); + expect(s.score).toBeGreaterThan(70); + }); + + it("treats leaving a supplier as strong intent", () => { + // The budget exists and the decision to spend it has been made. + const s = score("We're migrating away from our load testing vendor."); + expect(s.tier).toBe("switching"); + expect(s.score).toBeGreaterThan(50); + }); + + it("recognises a plain request for a recommendation", () => { + expect(score("Can anyone recommend a load testing tool?").tier).toBe("solicitation"); + }); + + it("scores a bare complaint lowest of the four", () => { + const pain = score("Struggling with our load testing setup again."); + const ask = score("Can anyone recommend a load testing tool?"); + expect(pain.tier).toBe("pain"); + expect(pain.score).toBeLessThan(ask.score); + }); + + it("says plainly when nobody asked for anything", () => { + // A statement is not an invitation, and this is the case the whole + // feature exists to separate out. + const s = score("Here is our quarterly load testing report."); + expect(s.tier).toBe("none"); + expect(s.score).toBe(0); + expect(s.reasons.join(" ")).toMatch(/nobody asked/); + }); +}); + +describe("recency is half the signal", () => { + it("decays with age", () => { + const fresh = score("Willing to pay for load testing help", 1); + const old = score("Willing to pay for load testing help", 24 * 7); + expect(fresh.score).toBeGreaterThan(old.score * 2); + }); + + it("halves at the half-life", () => { + expect(recencyFactor(36)).toBeCloseTo(0.5, 2); + expect(recencyFactor(72)).toBeCloseTo(0.25, 2); + }); + + it("gives up entirely on something ancient", () => { + const s = score("Willing to pay for load testing", 24 * 30); + expect(s.score).toBe(0); + expect(s.disqualified).toMatch(/older than/); + }); + + it("penalises an undated source rather than assuming it is fresh", () => { + // Most scraped directory entries have no date. Treating them as new would + // rank them above a real post from yesterday. + const undated = scoreIntent({ text: "Willing to pay for load testing", keywords: kw, now: NOW }); + const fresh = score("Willing to pay for load testing", 1); + expect(undated.score).toBeLessThan(fresh.score); + expect(undated.reasons.join(" ")).toMatch(/no date/); + }); +}); + +describe("intent for somebody else is not a lead", () => { + it("drops a perfect request about the wrong topic", () => { + // "Anyone recommend a good accountant, happy to pay" is flawless + // solicitation and worthless to a company selling load testing. + const s = scoreIntent({ + text: "Anyone recommend a good accountant? Happy to pay.", + postedAt: agoHours(1), + keywords: kw, + now: NOW, + }); + expect(s.disqualified).toMatch(/no campaign keyword/); + }); + + it("honours the campaign's negative keywords", () => { + const s = scoreIntent({ + text: "Anyone recommend free load testing for a student project?", + postedAt: agoHours(1), + keywords: kw, + negativeKeywords: ["student"], + now: NOW, + }); + expect(s.disqualified).toMatch(/negative match/); + }); +}); + +describe("people who said not to pitch them", () => { + it("refuses someone who asked not to be pitched", () => { + // The fastest way to lose a channel for everyone is to message the person + // who wrote this. + for (const text of [ + "Looking for load testing advice. No vendors please.", + "Need load testing help — please don't DM me.", + "Load testing, free only, not looking to pay", + ]) { + expect(scoreIntent({ text, postedAt: agoHours(1), keywords: kw, now: NOW }).disqualified).toBeTruthy(); + } + }); + + it("does not mistake a job ad for a buyer", () => { + const s = score("We're hiring a load testing engineer, budget approved."); + expect(s.disqualified).toMatch(/job ad/); + }); +}); + +describe("qualifying against a campaign's bar", () => { + it("lets everything through when no bar is set", () => { + // Turning the feature on must not silently empty an existing pipeline. + const weak = score("Struggling with load testing", 24 * 5); + expect(qualifies(weak, null)).toBe(true); + expect(qualifies(weak, undefined)).toBe(true); + }); + + it("holds a weak signal below the bar", () => { + expect(qualifies(score("Struggling with load testing", 24 * 6), DEFAULT_MIN_INTENT)).toBe(false); + }); + + it("passes a fresh explicit request", () => { + expect(qualifies(score("Anyone recommend a load testing tool?", 2), DEFAULT_MIN_INTENT)).toBe(true); + }); + + it("never qualifies a disqualified signal, whatever the bar", () => { + const blocked = scoreIntent({ text: "load testing, no vendors", postedAt: agoHours(1), keywords: kw, now: NOW }); + expect(qualifies(blocked, 0)).toBe(false); + }); + + it("explains itself in one line", () => { + expect(describeIntent(score("Willing to pay for load testing", 1))).toMatch(/purchase/); + }); +}); + +describe("reading dates off a search index", () => { + it("parses a relative date", () => { + expect(parseResultDate("3 hours ago", NOW)?.toISOString()).toBe("2026-07-28T09:00:00.000Z"); + }); + + it("parses an absolute date", () => { + expect(parseResultDate("Jul 20, 2026", NOW)?.getUTCDate()).toBe(20); + }); + + it("rejects a date from the future", () => { + // The index misreporting, not a scoop. + expect(parseResultDate("Jan 1, 2030", NOW)).toBeNull(); + }); + + it("has nothing to say when the index gives no date", () => { + expect(parseResultDate(null, NOW)).toBeNull(); + expect(parseResultDate("sometime", NOW)).toBeNull(); + }); +}); + +describe("searching more than one platform", () => { + it("covers more than Reddit", () => { + // A pipeline that watches one community inherits its demographics as the + // entire market. + expect(INTENT_SOURCES.length).toBeGreaterThan(4); + expect(INTENT_SOURCES.some((s) => s.id === "reddit")).toBe(true); + expect(INTENT_SOURCES.some((s) => s.sites.some((x) => x.includes("linkedin")))).toBe(true); + }); + + it("restricts a search to its own sites", () => { + const q = buildIntentQuery({ + source: INTENT_SOURCES[0], + topic: "load testing", + phrasing: '"anyone recommend"', + }); + expect(q).toContain("site:reddit.com"); + expect(q).toContain("load testing"); + }); +});