diff --git a/lib/outreach/cold.ts b/lib/outreach/cold.ts index 4c946cf0..6f291300 100644 --- a/lib/outreach/cold.ts +++ b/lib/outreach/cold.ts @@ -414,12 +414,18 @@ export function unsupportedClaims(body: string, facts: ProspectFacts): string[] * facts, links to places the campaign never mentioned, and claimed prior * contact. Tone and phrasing are the prompt's job. */ -export function unsupportedCustomClaims(body: string, facts: string[]): string[] { +export function unsupportedCustomClaims(body: string, declaredText: string[]): string[] { const problems: string[] = []; const lower = body.toLowerCase(); - const declared = facts.join(" \n ").toLowerCase(); - - // A number in the body that appears in no declared fact is invented — + // The grounding set is everything the operator wrote, not the facts list + // alone. The intro says who is writing and the ask says what to ask for, + // and an ask like "link them to example.com/paper" is an instruction to + // include that URL — checking the output against a set that excludes it + // rejected every correct draft as a fabrication. + const declared = declaredText.join(" \n ").toLowerCase(); + + // A number in the body that appears nowhere the operator wrote is + // invented — // years of experience, team sizes, prices. Small integers are excluded // because they are usually prose ("one thing", "a couple of weeks"). for (const m of body.matchAll(/\b(\d[\d,]*(?:\.\d+)?)\s*(%|\+)?/g)) { @@ -435,7 +441,7 @@ export function unsupportedCustomClaims(body: string, facts: string[]): string[] for (const m of body.matchAll(/https?:\/\/[^\s)>\]]+/gi)) { const url = m[0].replace(/[.,]$/, ""); if (!declared.includes(url.toLowerCase())) { - problems.push(`links to ${url}, which is not in the campaign's declared facts`); + problems.push(`links to ${url}, which the campaign never mentions`); } } diff --git a/lib/outreach/pipeline.ts b/lib/outreach/pipeline.ts index 8a6cb889..21603028 100644 --- a/lib/outreach/pipeline.ts +++ b/lib/outreach/pipeline.ts @@ -776,7 +776,13 @@ async function draftCustomEmail(input: { return { ok: false, problems: [`generation failed: ${err instanceof Error ? err.message : "unknown"}`] }; } - const problems = unsupportedCustomClaims(output.body, facts); + // Facts, plus the intro and ask — everything the operator authored counts + // as grounded, or an ask that names a URL makes its own draft invalid. + const problems = unsupportedCustomClaims(output.body, [ + ...facts, + input.pitch.intro, + input.pitch.ask ?? "", + ]); if (problems.length) return { ok: false, problems }; const subject = output.subject?.trim(); diff --git a/tests/custom-pitch.test.ts b/tests/custom-pitch.test.ts index 028db46e..0197a433 100644 --- a/tests/custom-pitch.test.ts +++ b/tests/custom-pitch.test.ts @@ -105,3 +105,40 @@ describe("customDraftSystem", () => { expect(system).toMatch(/Never ask for a call in a first message/); }); }); + +describe("the ask is part of the grounding set", () => { + // A live campaign rejected every draft it produced. Its ask named a URL, + // the model included that URL as instructed, and the guard compared the + // body against the facts list alone — so following the instruction was + // scored as a fabrication. These are the real values from that campaign. + const INTRO = + "Anthony owner of threatcrush.com we want to generate leads for people to download our free whitepaper at https://threatcrush.com/get-whitepaper"; + const ASK = + "we want to generate leads for people to download our free whitepaper at https://threatcrush.com/get-whitepaper"; + const FACTS = ["30 years experience at small startups and enterprise companies as a software engineer."]; + const declared = [...FACTS, INTRO, ASK]; + + it("accepts a URL the ask told it to include", () => { + const body = + "You can grab the free whitepaper at https://threatcrush.com/get-whitepaper whenever it is useful."; + expect(unsupportedCustomClaims(body, declared)).toEqual([]); + }); + + it("rejected it when only the facts were checked, which was the bug", () => { + const body = "Grab it at https://threatcrush.com/get-whitepaper"; + expect(unsupportedCustomClaims(body, FACTS).join(" ")).toMatch(/threatcrush/); + }); + + it("still catches a URL nobody wrote anywhere", () => { + const body = "See https://not-ours.test/landing for details."; + expect(unsupportedCustomClaims(body, declared).join(" ")).toMatch(/not-ours\.test/); + }); + + it("accepts a number stated in the facts", () => { + expect(unsupportedCustomClaims("I have 30 years of experience.", declared)).toEqual([]); + }); + + it("still catches an invented number", () => { + expect(unsupportedCustomClaims("We have 450 customers.", declared).join(" ")).toMatch(/450/); + }); +});