fix(outreach): stop campaigns burning the ValueSERP plan on dead queries

lib/alerts/limits.ts

@@ -19,10 +19,20 @@ export const MAX_ACTIVE_ALERTS: Record<Plan, number> = {
 // Monthly SERP-call budget by plan — the hard cost backstop. Free is set so a
 // typical user never touches it, while a cap-abuser is bounded to well under
 // a dollar. Paid budgets assume hourly checks (24× the call volume).
+//
+// These are per-account and must stay under VALUESERP_MONTHLY_PLAN, which is
+// the whole shared bucket: the old pro budget of 200k authorised one account
+// to spend eight times everything CrawlProof buys in a month, so the backstop
+// could not actually stop anything. A pro account running 250 hourly alerts
+// needs 250 × 24 × 30 = 180k to never touch its cap, which the 25k plan cannot
+// fund — the cap is the honest number, and the plan is what to raise if a real
+// customer starts hitting it.
+export const VALUESERP_MONTHLY_PLAN = 25_000;
+
 export const SERP_CALLS_PER_MONTH: Record<Plan, number> = {
   free: 400,
-  pro: 200_000,
-  team: 1_000_000,
+  pro: 8_000,
+  team: 15_000,
 };
 
 // Hourly is currently free for everyone (no paywall on frequency yet — the

lib/alerts/valueserp.ts

@@ -50,6 +50,28 @@ export function hasValueSerpKey(): boolean {
   return Boolean(env.valueSerpApiKey);
 }
 
+/**
+ * When the account is out of credits, stop asking until this passes.
+ *
+ * ValueSERP answers an exhausted plan with HTTP 402, and the plan is a monthly
+ * bucket — once it is empty every later call in the cycle gets the same
+ * answer. Without this, a single campaign tick fires twenty-odd searches that
+ * are all guaranteed to fail, each paying a network round-trip and filing its
+ * own error, and the run summary reads as twenty distinct problems rather than
+ * one. Nothing is billed either way (a 402 consumes no credit), so this buys
+ * latency and legible errors, not money.
+ *
+ * Deliberately short. The reset time is not in the search response, so this
+ * guesses low rather than parking a working key for hours.
+ */
+const OUT_OF_CREDITS_COOLDOWN_MS = 10 * 60_000;
+let outOfCreditsUntil = 0;
+
+/** Exposed so tests can reset the module-level cooldown between cases. */
+export function resetSerpCreditCooldown(): void {
+  outOfCreditsUntil = 0;
+}
+
 /**
  * Run one ValueSERP search. Returns billable `calls` even on an empty result
  * set so the caller can debit the budget accurately. Retries once on a
@@ -63,6 +85,14 @@ export async function searchSerp(input: {
   if (!env.valueSerpApiKey) {
     return { ok: false, calls: 0, results: [], error: "VALUESERP_API_KEY not set" };
   }
+  if (Date.now() < outOfCreditsUntil) {
+    return {
+      ok: false,
+      calls: 0,
+      results: [],
+      error: "ValueSERP is out of monthly credits (HTTP 402) — not retrying until the cooldown passes",
+    };
+  }
   const num = Math.min(input.num ?? RESULTS_PER_CHECK, 100);
   const params = new URLSearchParams({
     api_key: env.valueSerpApiKey,
@@ -91,6 +121,12 @@ export async function searchSerp(input: {
       clearTimeout(timer);
       if (!res.ok) {
         lastErr = `ValueSERP HTTP ${res.status}`;
+        // 402 is an empty monthly plan, which no later call in this cycle can
+        // change. Park every caller rather than letting each one rediscover it.
+        if (res.status === 402) {
+          outOfCreditsUntil = Date.now() + OUT_OF_CREDITS_COOLDOWN_MS;
+          lastErr = "ValueSERP HTTP 402 — monthly credit plan is exhausted";
+        }
         // 4xx (bad query, out of quota) won't fix on retry.
         if (res.status < 500) return { ok: false, calls: 0, results: [], error: lastErr };
         continue;

lib/outreach/runner.ts

@@ -47,6 +47,10 @@ export type CampaignRow = {
   max_score: number;
   daily_send_limit: number;
   target_pipeline: number;
+  /** When discovery may next run. Null means now. See discoveryBackoffMinutes. */
+  discovery_backoff_until: string | null;
+  /** Consecutive discovery passes that produced nothing. Drives the back-off. */
+  discovery_dry_streak: number;
   auto_send: boolean;
   follow_ups: boolean;
   angle: string | null;
@@ -69,7 +73,7 @@ export type CampaignRow = {
 };
 
 export const CAMPAIGN_COLUMNS =
-  "id, project_id, owner_id, name, channel, active, queries, seed_urls, keywords, subreddits, negative_keywords, max_score, daily_send_limit, target_pipeline, auto_send, follow_ups, angle, sender_name, reply_to, last_run_at, pitch_mode, pitch_intro, pitch_ask, pitch_facts, scan_prospects, min_intent, intent_sources, intent_recency, sells_description, alert_email, alerts_enabled";
+  "id, project_id, owner_id, name, channel, active, queries, seed_urls, keywords, subreddits, negative_keywords, max_score, daily_send_limit, target_pipeline, discovery_backoff_until, discovery_dry_streak, auto_send, follow_ups, angle, sender_name, reply_to, last_run_at, pitch_mode, pitch_intro, pitch_ask, pitch_facts, scan_prospects, min_intent, intent_sources, intent_recency, sells_description, alert_email, alerts_enabled";
 
 export type TickResult = {
   campaign: string;
@@ -112,6 +116,30 @@ const MAX_CONTACT_SEARCHES_PER_TICK = 10;
 // same request does not need finding four times an hour.
 const MAX_INTENT_SEARCHES_PER_TICK = 6;
 
+// Back-off for a campaign whose queries have stopped producing.
+//
+// The funnel gate below counts only prospects still in flight, so a campaign
+// that has contacted everyone it found reads as empty forever and re-runs its
+// identical query list every fifteen minutes — paying full search price for
+// hosts it already has. Left alone, three campaigns at five queries a tick
+// spend ~43k searches a month against a 25k plan, and the whole quota is gone
+// in a week.
+//
+// Counting contacted prospects toward the target would stop it, but it would
+// also cap a campaign at target_pipeline leads for its entire life, which
+// turns an autopilot into a one-shot. So the gate stays as it is and an
+// unproductive pass simply waits longer before trying again: doubling from
+// half an hour up to a day. Nothing is capped — a tapped-out query set is
+// still retried daily, and one new result resets the campaign to full speed.
+const DISCOVERY_BACKOFF_BASE_MIN = 30;
+const DISCOVERY_BACKOFF_MAX_MIN = 24 * 60;
+
+/** How long a campaign should wait after `streak` consecutive empty passes. */
+export function discoveryBackoffMinutes(streak: number): number {
+  if (streak < 1) return 0;
+  return Math.min(DISCOVERY_BACKOFF_BASE_MIN * 2 ** (streak - 1), DISCOVERY_BACKOFF_MAX_MIN);
+}
+
 export async function runEmailCampaignTick(campaign: CampaignRow): Promise<TickResult> {
   const sb = serviceClient();
   // Shared across every prospect this tick researches, so the ceiling is per
@@ -266,9 +294,23 @@ export async function runEmailCampaignTick(campaign: CampaignRow): Promise<TickR
 
   // ---- 4. Top the funnel up.
   const liveCount = prospects.filter((p) => ["new", "researched", "drafted"].includes(p.status)).length;
+  // Whether the queries have earned another run yet. Checked before canSpend
+  // so a backed-off campaign costs nothing at all — not a search, not a
+  // billing round-trip.
+  const backoffUntil = campaign.discovery_backoff_until
+    ? new Date(campaign.discovery_backoff_until)
+    : null;
+  const backedOff = backoffUntil !== null && backoffUntil.getTime() > Date.now();
+  if (backedOff && backoffUntil) {
+    result.skipped.push(
+      `discovery: resting until ${backoffUntil.toISOString()} after ${campaign.discovery_dry_streak} empty pass(es)`,
+    );
+  }
   // Discovery is the most expensive stage — several search calls before a
   // single prospect exists — so it is gated like the rest.
-  if (liveCount < campaign.target_pipeline && (await canSpend())) {
+  let discoveryRan = false;
+  if (liveCount < campaign.target_pipeline && !backedOff && (await canSpend())) {
+    discoveryRan = true;
     const want = Math.min(campaign.target_pipeline - liveCount, MAX_DISCOVER_PER_TICK);
     const { data: projectRow } = await sb
       .from("projects")
@@ -399,6 +441,33 @@ export async function runEmailCampaignTick(campaign: CampaignRow): Promise<TickR
     }
   }
 
+  // Record whether the searches earned their keep, so the next tick knows
+  // whether to run them again. Only a pass that actually searched votes: a
+  // tick that skipped discovery because the funnel was full says nothing
+  // about whether the queries still work, and letting it reset the streak
+  // would put a tapped-out campaign straight back to a search every fifteen
+  // minutes the moment its pipeline drained.
+  if (discoveryRan) {
+    // People and intent signals count. A run against a people-directory names
+    // humans without adding a prospect, and a query surfacing fresh buying
+    // intent is working even when it yields no new domain — backing either
+    // off as "empty" would rest the queries that are doing their job.
+    const discoveryProduced =
+      result.discovered || result.peopleRecorded || result.intentFound;
+    const dryStreak = discoveryProduced ? 0 : (campaign.discovery_dry_streak ?? 0) + 1;
+    const restMinutes = discoveryBackoffMinutes(dryStreak);
+    await sb
+      .from("outreach_campaigns")
+      .update({
+        discovery_dry_streak: dryStreak,
+        discovery_backoff_until: restMinutes
+          ? new Date(Date.now() + restMinutes * 60_000).toISOString()
+          : null,
+      })
+      .eq("id", campaign.id);
+    if (restMinutes) result.skipped.push(`discovery: resting ${restMinutes}m (${dryStreak} empty)`);
+  }
+
   // A run that was charged and produced nothing gives the money back. Some of
   // it is genuinely spent by then — a search that returned no usable candidate
   // still cost a call — but billing for a tick with no output is a worse trade

supabase/migrations/20260813040000_outreach_discovery_backoff.sql

@@ -0,0 +1,28 @@
+-- Let a campaign rest its search queries after they stop producing.
+--
+-- The funnel gate in runEmailCampaignTick counts only prospects still in
+-- flight, so a campaign that has contacted everyone it found reads as empty
+-- forever and re-runs its identical query list every fifteen minutes, paying
+-- full search price for hosts it already has. Three active campaigns at five
+-- queries a tick worked out to ~43k ValueSERP searches a month against a 25k
+-- plan, which emptied the quota in six days and left discovery returning
+-- HTTP 402 for the rest of the cycle.
+--
+-- These two columns let an unproductive pass wait longer before trying again
+-- (30m doubling to a 24h ceiling) instead of capping how many leads a campaign
+-- may ever find. One new result clears the streak and restores full speed.
+
+alter table public.outreach_campaigns
+  add column if not exists discovery_backoff_until timestamptz,
+  add column if not exists discovery_dry_streak int not null default 0;
+
+comment on column public.outreach_campaigns.discovery_backoff_until is
+  'When discovery may next run for this campaign. Null means immediately.';
+comment on column public.outreach_campaigns.discovery_dry_streak is
+  'Consecutive discovery passes that produced no prospect, person or intent signal.';
+
+-- The runner reads active campaigns oldest-tick-first every 15 minutes; the
+-- back-off column is checked on each of those rows.
+create index if not exists outreach_campaigns_discovery_backoff_idx
+  on public.outreach_campaigns (discovery_backoff_until)
+  where active;

tests/discovery-backoff.test.ts

@@ -0,0 +1,95 @@
+// The quota guard. In August 2026 three active campaigns emptied a 25,000-call
+// ValueSERP plan in six days, and every search after that returned HTTP 402 —
+// including the ones a human typed into the lead finder. Neither failure was
+// visible as a bug: the runner was doing exactly what it was told, fifteen
+// minutes at a time, on queries that had nothing left to give.
+//
+// These cover the two pieces that stop it recurring.
+
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import { discoveryBackoffMinutes } from "@/lib/outreach/runner";
+import { SERP_CALLS_PER_MONTH, VALUESERP_MONTHLY_PLAN } from "@/lib/alerts/limits";
+
+describe("discovery back-off", () => {
+  it("does not rest a campaign that just produced something", () => {
+    expect(discoveryBackoffMinutes(0)).toBe(0);
+  });
+
+  it("starts at half an hour and doubles", () => {
+    expect(discoveryBackoffMinutes(1)).toBe(30);
+    expect(discoveryBackoffMinutes(2)).toBe(60);
+    expect(discoveryBackoffMinutes(3)).toBe(120);
+    expect(discoveryBackoffMinutes(4)).toBe(240);
+  });
+
+  it("stops doubling at a day, so a tapped-out campaign still retries daily", () => {
+    expect(discoveryBackoffMinutes(10)).toBe(24 * 60);
+    expect(discoveryBackoffMinutes(100)).toBe(24 * 60);
+  });
+
+  // The whole point of the change. At 5 queries a tick, every 15 minutes,
+  // three campaigns cost ~43k searches a month against a 25k plan; capped at
+  // one pass a day they cost ~450.
+  it("cuts a dry campaign from 96 passes a day to 1", () => {
+    const passesPerDay = (24 * 60) / discoveryBackoffMinutes(10);
+    expect(passesPerDay).toBe(1);
+    const before = (24 * 60) / 15;
+    expect(before).toBe(96);
+  });
+});
+
+describe("per-account SERP budgets", () => {
+  // The backstop that could not stop anything: a single pro account was
+  // authorised for 200k calls a month out of a shared bucket of 25k.
+  it("keeps every plan under the vendor plan it spends from", () => {
+    for (const [plan, budget] of Object.entries(SERP_CALLS_PER_MONTH)) {
+      expect(budget, `${plan} budget exceeds the ValueSERP plan`).toBeLessThanOrEqual(
+        VALUESERP_MONTHLY_PLAN,
+      );
+    }
+  });
+});
+
+describe("ValueSERP out-of-credit cooldown", () => {
+  const realFetch = globalThis.fetch;
+
+  beforeEach(() => {
+    vi.resetModules();
+    process.env.VALUESERP_API_KEY = "test-key";
+  });
+
+  afterEach(() => {
+    globalThis.fetch = realFetch;
+    vi.restoreAllMocks();
+  });
+
+  it("asks once, then answers from the cooldown instead of re-hitting a dead plan", async () => {
+    const fetchMock = vi.fn(async () => new Response("", { status: 402 }));
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const { searchSerp, resetSerpCreditCooldown } = await import("@/lib/alerts/valueserp");
+    resetSerpCreditCooldown();
+
+    const first = await searchSerp({ query: "web development agencies", recency: "any" });
+    expect(first.ok).toBe(false);
+    expect(first.error).toContain("402");
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+
+    // A campaign tick fires ~21 searches; without the cooldown all of them
+    // would pay a round-trip to learn the same thing.
+    for (let i = 0; i < 20; i++) {
+      const again = await searchSerp({ query: `q${i}`, recency: "any" });
+      expect(again.ok).toBe(false);
+    }
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+
+  it("never bills a call it did not make", async () => {
+    globalThis.fetch = vi.fn(async () => new Response("", { status: 402 })) as unknown as typeof fetch;
+    const { searchSerp, resetSerpCreditCooldown } = await import("@/lib/alerts/valueserp");
+    resetSerpCreditCooldown();
+
+    expect((await searchSerp({ query: "a", recency: "any" })).calls).toBe(0);
+    expect((await searchSerp({ query: "b", recency: "any" })).calls).toBe(0);
+  });
+});

tests/lead-billing.test.ts

@@ -130,7 +130,14 @@ describe("the meter is actually connected", () => {
   it("gates discovery, the most expensive stage, on being able to pay", async () => {
     const src = await read("lib/outreach/runner.ts");
     const discovery = src.slice(src.indexOf("---- 4. Top the funnel up"));
-    expect(discovery.slice(0, 400)).toMatch(/canSpend\(\)/);
+    // Read the gate's own condition rather than a fixed byte window. The block
+    // picked up a discovery back-off check ahead of the gate, which pushed
+    // canSpend() past the old 400-character slice without weakening anything
+    // this test exists to protect.
+    const gateStart = discovery.indexOf("if (liveCount");
+    expect(gateStart).toBeGreaterThan(-1);
+    const gate = discovery.slice(gateStart, discovery.indexOf("{", gateStart));
+    expect(gate).toMatch(/canSpend\(\)/);
   });
 
   it("charges the manual finder", async () => {