From 4caae9c5334fe3622772133ed27a4749389ff97e Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Tue, 28 Jul 2026 09:55:03 +0000 Subject: [PATCH] fix(leads): count the ask as grounding, not as a fabrication to reject MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live campaign rejected every draft it produced, five per tick, for hours. Its ask read "download our free whitepaper at https://threatcrush.com/get-whitepaper"; the model included that URL exactly as instructed, and the guard compared the body against the facts list alone and scored following the instruction as an invented link. The grounding set was wrong, not the drafts. Everything the operator authored is grounded — the intro says who is writing, the ask says what to ask for, and an ask naming a URL is an instruction to include it. Checking output against a subset of the instructions makes correct output unrepresentable. Kept: a link or number appearing nowhere the operator wrote is still refused, which is the case the guard exists for. Co-Authored-By: Claude Opus 5 (1M context) --- lib/outreach/cold.ts | 16 +++++++++++----- lib/outreach/pipeline.ts | 8 +++++++- tests/custom-pitch.test.ts | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/lib/outreach/cold.ts b/lib/outreach/cold.ts index 4c946cf0..6f291300 100644 --- a/lib/outreach/cold.ts +++ b/lib/outreach/cold.ts @@ -414,12 +414,18 @@ export function unsupportedClaims(body: string, facts: ProspectFacts): string[] * facts, links to places the campaign never mentioned, and claimed prior * contact. Tone and phrasing are the prompt's job. */ -export function unsupportedCustomClaims(body: string, facts: string[]): string[] { +export function unsupportedCustomClaims(body: string, declaredText: string[]): string[] { const problems: string[] = []; const lower = body.toLowerCase(); - const declared = facts.join(" \n ").toLowerCase(); - - // A number in the body that appears in no declared fact is invented — + // The grounding set is everything the operator wrote, not the facts list + // alone. The intro says who is writing and the ask says what to ask for, + // and an ask like "link them to example.com/paper" is an instruction to + // include that URL — checking the output against a set that excludes it + // rejected every correct draft as a fabrication. + const declared = declaredText.join(" \n ").toLowerCase(); + + // A number in the body that appears nowhere the operator wrote is + // invented — // years of experience, team sizes, prices. Small integers are excluded // because they are usually prose ("one thing", "a couple of weeks"). for (const m of body.matchAll(/\b(\d[\d,]*(?:\.\d+)?)\s*(%|\+)?/g)) { @@ -435,7 +441,7 @@ export function unsupportedCustomClaims(body: string, facts: string[]): string[] for (const m of body.matchAll(/https?:\/\/[^\s)>\]]+/gi)) { const url = m[0].replace(/[.,]$/, ""); if (!declared.includes(url.toLowerCase())) { - problems.push(`links to ${url}, which is not in the campaign's declared facts`); + problems.push(`links to ${url}, which the campaign never mentions`); } } diff --git a/lib/outreach/pipeline.ts b/lib/outreach/pipeline.ts index 8a6cb889..21603028 100644 --- a/lib/outreach/pipeline.ts +++ b/lib/outreach/pipeline.ts @@ -776,7 +776,13 @@ async function draftCustomEmail(input: { return { ok: false, problems: [`generation failed: ${err instanceof Error ? err.message : "unknown"}`] }; } - const problems = unsupportedCustomClaims(output.body, facts); + // Facts, plus the intro and ask — everything the operator authored counts + // as grounded, or an ask that names a URL makes its own draft invalid. + const problems = unsupportedCustomClaims(output.body, [ + ...facts, + input.pitch.intro, + input.pitch.ask ?? "", + ]); if (problems.length) return { ok: false, problems }; const subject = output.subject?.trim(); diff --git a/tests/custom-pitch.test.ts b/tests/custom-pitch.test.ts index 028db46e..0197a433 100644 --- a/tests/custom-pitch.test.ts +++ b/tests/custom-pitch.test.ts @@ -105,3 +105,40 @@ describe("customDraftSystem", () => { expect(system).toMatch(/Never ask for a call in a first message/); }); }); + +describe("the ask is part of the grounding set", () => { + // A live campaign rejected every draft it produced. Its ask named a URL, + // the model included that URL as instructed, and the guard compared the + // body against the facts list alone — so following the instruction was + // scored as a fabrication. These are the real values from that campaign. + const INTRO = + "Anthony owner of threatcrush.com we want to generate leads for people to download our free whitepaper at https://threatcrush.com/get-whitepaper"; + const ASK = + "we want to generate leads for people to download our free whitepaper at https://threatcrush.com/get-whitepaper"; + const FACTS = ["30 years experience at small startups and enterprise companies as a software engineer."]; + const declared = [...FACTS, INTRO, ASK]; + + it("accepts a URL the ask told it to include", () => { + const body = + "You can grab the free whitepaper at https://threatcrush.com/get-whitepaper whenever it is useful."; + expect(unsupportedCustomClaims(body, declared)).toEqual([]); + }); + + it("rejected it when only the facts were checked, which was the bug", () => { + const body = "Grab it at https://threatcrush.com/get-whitepaper"; + expect(unsupportedCustomClaims(body, FACTS).join(" ")).toMatch(/threatcrush/); + }); + + it("still catches a URL nobody wrote anywhere", () => { + const body = "See https://not-ours.test/landing for details."; + expect(unsupportedCustomClaims(body, declared).join(" ")).toMatch(/not-ours\.test/); + }); + + it("accepts a number stated in the facts", () => { + expect(unsupportedCustomClaims("I have 30 years of experience.", declared)).toEqual([]); + }); + + it("still catches an invented number", () => { + expect(unsupportedCustomClaims("We have 450 customers.", declared).join(" ")).toMatch(/450/); + }); +});