Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions lib/outreach/cold.ts
Original file line number Diff line number Diff line change
Expand Up @@ -414,12 +414,18 @@ export function unsupportedClaims(body: string, facts: ProspectFacts): string[]
* facts, links to places the campaign never mentioned, and claimed prior
* contact. Tone and phrasing are the prompt's job.
*/
export function unsupportedCustomClaims(body: string, facts: string[]): string[] {
export function unsupportedCustomClaims(body: string, declaredText: string[]): string[] {
const problems: string[] = [];
const lower = body.toLowerCase();
const declared = facts.join(" \n ").toLowerCase();

// A number in the body that appears in no declared fact is invented —
// The grounding set is everything the operator wrote, not the facts list
// alone. The intro says who is writing and the ask says what to ask for,
// and an ask like "link them to example.com/paper" is an instruction to
// include that URL — checking the output against a set that excludes it
// rejected every correct draft as a fabrication.
const declared = declaredText.join(" \n ").toLowerCase();

// A number in the body that appears nowhere the operator wrote is
// invented —
// years of experience, team sizes, prices. Small integers are excluded
// because they are usually prose ("one thing", "a couple of weeks").
for (const m of body.matchAll(/\b(\d[\d,]*(?:\.\d+)?)\s*(%|\+)?/g)) {
Expand All @@ -435,7 +441,7 @@ export function unsupportedCustomClaims(body: string, facts: string[]): string[]
for (const m of body.matchAll(/https?:\/\/[^\s)>\]]+/gi)) {
const url = m[0].replace(/[.,]$/, "");
if (!declared.includes(url.toLowerCase())) {
problems.push(`links to ${url}, which is not in the campaign's declared facts`);
problems.push(`links to ${url}, which the campaign never mentions`);
}
}

Expand Down
8 changes: 7 additions & 1 deletion lib/outreach/pipeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -776,7 +776,13 @@ async function draftCustomEmail(input: {
return { ok: false, problems: [`generation failed: ${err instanceof Error ? err.message : "unknown"}`] };
}

const problems = unsupportedCustomClaims(output.body, facts);
// Facts, plus the intro and ask — everything the operator authored counts
// as grounded, or an ask that names a URL makes its own draft invalid.
const problems = unsupportedCustomClaims(output.body, [
...facts,
input.pitch.intro,
input.pitch.ask ?? "",
]);
if (problems.length) return { ok: false, problems };

const subject = output.subject?.trim();
Expand Down
37 changes: 37 additions & 0 deletions tests/custom-pitch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,3 +105,40 @@ describe("customDraftSystem", () => {
expect(system).toMatch(/Never ask for a call in a first message/);
});
});

describe("the ask is part of the grounding set", () => {
// A live campaign rejected every draft it produced. Its ask named a URL,
// the model included that URL as instructed, and the guard compared the
// body against the facts list alone — so following the instruction was
// scored as a fabrication. These are the real values from that campaign.
const INTRO =
"Anthony owner of threatcrush.com we want to generate leads for people to download our free whitepaper at https://threatcrush.com/get-whitepaper";
const ASK =
"we want to generate leads for people to download our free whitepaper at https://threatcrush.com/get-whitepaper";
const FACTS = ["30 years experience at small startups and enterprise companies as a software engineer."];
const declared = [...FACTS, INTRO, ASK];

it("accepts a URL the ask told it to include", () => {
const body =
"You can grab the free whitepaper at https://threatcrush.com/get-whitepaper whenever it is useful.";
expect(unsupportedCustomClaims(body, declared)).toEqual([]);
});

it("rejected it when only the facts were checked, which was the bug", () => {
const body = "Grab it at https://threatcrush.com/get-whitepaper";
expect(unsupportedCustomClaims(body, FACTS).join(" ")).toMatch(/threatcrush/);
});

it("still catches a URL nobody wrote anywhere", () => {
const body = "See https://not-ours.test/landing for details.";
expect(unsupportedCustomClaims(body, declared).join(" ")).toMatch(/not-ours\.test/);
});

it("accepts a number stated in the facts", () => {
expect(unsupportedCustomClaims("I have 30 years of experience.", declared)).toEqual([]);
});

it("still catches an invented number", () => {
expect(unsupportedCustomClaims("We have 450 customers.", declared).join(" ")).toMatch(/450/);
});
});
Loading