From 4194cb1d081e3e7491222c512489a10031dd1208 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Tue, 28 Jul 2026 02:12:28 +0000 Subject: [PATCH] feat(leads): sign in to seed directories that gate their listings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some directories only show their listings to a signed-in visitor. Detection could already tell that apart from an empty directory; this is the part that gets past it. Login is driven generically rather than per site. A per-directory login script is the thing worth avoiding: every gated directory would need one, and each would break on its own schedule. A login form is standardised enough to drive blind — one password field, one identifier field near it, one submit — and the password field is the anchor, so a page without one is refused rather than guessed at. The sign-in happens on the context that already hit the wall, because the redirect has put the form in front of us and the session cookies have to survive into the retry. After signing in the original URL is requested again and re-checked: signing in successfully and still being asked to log in means the account cannot see that page, which is a different problem and says so. A campaign parked on a gated directory is not failing, it is waiting on something only the user can supply. The gated hosts are recorded on the campaign, surfaced as waiting_for_auth, and each one carries the form that unblocks it — rather than leaving the reason buried in an error string nobody reads. Credentials are stored per host per organization, since a user has one account on a directory rather than one per search they paste in. Passwords exist in the database only as AES-256-GCM ciphertext, with the key in the app environment. They are deliberately not verified at save time: that would mean driving a browser through someone else's login form inside a server action, and a directory that is briefly slow would look like a bad password. The next tick tries them and records what happened, so the panel can show whether a login actually worked. A site that answers with a verification code or a challenge cannot be signed into from a server at all. That is reported as what it is instead of as a wrong password. Co-Authored-By: Claude Opus 5 (1M context) --- app/(app)/projects/[id]/leads/page.tsx | 22 ++ app/actions/seedCredentials.ts | 118 ++++++++++ components/leads/seed-logins.tsx | 206 ++++++++++++++++++ lib/outreach/discover.ts | 37 +++- lib/outreach/render.ts | 59 ++++- lib/outreach/runner.ts | 21 ++ lib/outreach/seedCredentials.ts | 103 +++++++++ lib/outreach/seedLogin.ts | 150 +++++++++++++ ...260728010000_outreach_seed_credentials.sql | 68 ++++++ 9 files changed, 766 insertions(+), 18 deletions(-) create mode 100644 app/actions/seedCredentials.ts create mode 100644 components/leads/seed-logins.tsx create mode 100644 lib/outreach/seedCredentials.ts create mode 100644 lib/outreach/seedLogin.ts create mode 100644 supabase/migrations/20260728010000_outreach_seed_credentials.sql diff --git a/app/(app)/projects/[id]/leads/page.tsx b/app/(app)/projects/[id]/leads/page.tsx index d77dff5e..ea08cc3e 100644 --- a/app/(app)/projects/[id]/leads/page.tsx +++ b/app/(app)/projects/[id]/leads/page.tsx @@ -8,6 +8,8 @@ import { LeadActions } from "@/components/leads/lead-actions"; import { CampaignPanel, type CampaignSummary } from "@/components/leads/campaign-panel"; import { SenderAddress } from "@/components/leads/sender-address"; import { MailboxConnect, type ConnectedMailbox } from "@/components/leads/mailbox-connect"; +import { SeedLogins } from "@/components/leads/seed-logins"; +import { listSeedCredentials, type StoredSeedCredential } from "@/lib/outreach/seedCredentials"; import { RefreshLeads } from "@/components/leads/refresh-leads"; import { loadAddressSettings } from "@/lib/outreach/postalAddress"; @@ -119,6 +121,20 @@ export default async function LeadsPage({ .eq("id", projectId) .maybeSingle(); const orgId = (projectRow?.organization_id as string | null) ?? null; + let seedCredentials: StoredSeedCredential[] = []; + if (orgId) seedCredentials = await listSeedCredentials(orgId); + + // Hosts any campaign in this project is parked on, waiting for a sign-in. + const waitingHosts = [ + ...new Set( + campaigns.flatMap((c) => + Array.isArray((c as { auth_required_hosts?: string[] }).auth_required_hosts) + ? ((c as { auth_required_hosts?: string[] }).auth_required_hosts as string[]) + : [], + ), + ), + ]; + let mailbox: ConnectedMailbox | null = null; if (orgId) { const { data: senderRow } = await supabase @@ -164,6 +180,12 @@ export default async function LeadsPage({ + +

Pipeline

diff --git a/app/actions/seedCredentials.ts b/app/actions/seedCredentials.ts new file mode 100644 index 00000000..e38b2dd3 --- /dev/null +++ b/app/actions/seedCredentials.ts @@ -0,0 +1,118 @@ +"use server"; + +import { revalidatePath } from "next/cache"; +import { serviceClient } from "@/lib/supabase/service"; +import { requireProjectAccess } from "@/lib/lx/currentSite"; +import { encryptSecret } from "@/lib/sp/vault"; +import { normalizeHost } from "@/lib/outreach/cold"; +import { seedHost } from "@/lib/outreach/seedCredentials"; + +type Ok> = { ok: true } & T; +type Err = { ok: false; error: string }; + +async function requireOrg( + projectId: string, +): Promise<{ ok: true; userId: string; organizationId: string } | Err> { + if (!projectId) return { ok: false, error: "Missing project." }; + const access = await requireProjectAccess(projectId); + if (!access.ok) return { ok: false, error: access.error }; + if (access.isViewer) return { ok: false, error: "Viewers can't manage seed logins." }; + + const { data: project } = await serviceClient() + .from("projects") + .select("organization_id") + .eq("id", projectId) + .maybeSingle(); + const organizationId = (project?.organization_id as string | null) ?? null; + if (!organizationId) { + return { ok: false, error: "This project isn't in an organization, so there's nowhere to store a login." }; + } + return { ok: true, userId: access.userId, organizationId }; +} + +/** + * Store a login for a gated seed directory. + * + * Unlike the mailbox, the credential is not verified at save time: doing so + * would mean driving a browser through someone else's login form inside a + * server action, and a directory that is slow or briefly down would look like + * a bad password. It is verified on the next tick instead, and the result is + * recorded on the row so the UI can show whether it actually worked. + */ +export async function saveSeedCredentialAction(input: { + projectId: string; + host: string; + username: string; + password: string; + loginUrl?: string; +}): Promise | Err> { + const access = await requireOrg(input.projectId); + if (!access.ok) return access; + + // Accept a full URL or a bare host — the value usually comes from a seed + // URL the user pasted somewhere else. + const host = input.host.includes("://") + ? seedHost(input.host) + : normalizeHost(input.host.trim()); + if (!host || !host.includes(".")) return { ok: false, error: "That doesn't look like a site." }; + if (!input.username.trim()) return { ok: false, error: "A username or email is required." }; + if (!input.password) return { ok: false, error: "A password is required." }; + + const { error } = await serviceClient() + .from("outreach_seed_credentials") + .upsert( + { + organization_id: access.organizationId, + created_by: access.userId, + host, + username: input.username.trim(), + enc_password: encryptSecret(input.password), + login_url: input.loginUrl?.trim() || null, + // A new password invalidates whatever the last attempt concluded. + verified_at: null, + last_error: null, + }, + { onConflict: "organization_id,host" }, + ); + if (error) return { ok: false, error: error.message }; + + // Any campaign parked on this host can stop waiting. + const { data: campaigns } = await serviceClient() + .from("outreach_campaigns") + .select("id, auth_required_hosts") + .eq("project_id", input.projectId); + for (const c of (campaigns as { id: string; auth_required_hosts: string[] | null }[] | null) ?? []) { + const waiting = Array.isArray(c.auth_required_hosts) ? c.auth_required_hosts : []; + const remaining = waiting.filter((u) => seedHost(u) !== host); + if (remaining.length !== waiting.length) { + await serviceClient() + .from("outreach_campaigns") + .update({ auth_required_hosts: remaining }) + .eq("id", c.id); + } + } + + revalidatePath(`/projects/${input.projectId}/leads`); + return { + ok: true, + note: `Saved a login for ${host}. The next campaign tick will try it and report whether it worked.`, + }; +} + +export async function deleteSeedCredentialAction(input: { + projectId: string; + host: string; +}): Promise<{ ok: true } | Err> { + const access = await requireOrg(input.projectId); + if (!access.ok) return access; + + const { error } = await serviceClient() + .from("outreach_seed_credentials") + .delete() + .eq("organization_id", access.organizationId) + .eq("host", normalizeHost(input.host)); + if (error) return { ok: false, error: error.message }; + + revalidatePath(`/projects/${input.projectId}/leads`); + return { ok: true }; +} diff --git a/components/leads/seed-logins.tsx b/components/leads/seed-logins.tsx new file mode 100644 index 00000000..21d207ce --- /dev/null +++ b/components/leads/seed-logins.tsx @@ -0,0 +1,206 @@ +"use client"; + +import { useState, useTransition } from "react"; +import { useRouter } from "next/navigation"; +import { + deleteSeedCredentialAction, + saveSeedCredentialAction, +} from "@/app/actions/seedCredentials"; +import type { StoredSeedCredential } from "@/lib/outreach/seedCredentials"; + +/** + * Seed directories that need a sign-in. + * + * The panel leads with what is actually blocked. A campaign parked on a gated + * directory isn't failing, it is waiting on something only the user can + * supply, so the hosts it is waiting for are named and each one carries the + * form that unblocks it. + */ +export function SeedLogins({ + projectId, + waitingHosts, + credentials, +}: { + projectId: string; + /** Seed URLs campaigns are parked on, with no stored credential. */ + waitingHosts: string[]; + credentials: StoredSeedCredential[]; +}) { + const router = useRouter(); + const [pending, start] = useTransition(); + const [open, setOpen] = useState(false); + const [host, setHost] = useState(""); + const [username, setUsername] = useState(""); + const [password, setPassword] = useState(""); + const [note, setNote] = useState(null); + const [error, setError] = useState(null); + + const save = (targetHost?: string) => + start(async () => { + setNote(null); + setError(null); + const res = await saveSeedCredentialAction({ + projectId, + host: targetHost ?? host, + username, + password, + }); + if (!res.ok) { + setError(res.error); + return; + } + setNote(res.note); + setHost(""); + setUsername(""); + setPassword(""); + setOpen(false); + router.refresh(); + }); + + const remove = (h: string) => + start(async () => { + setError(null); + const res = await deleteSeedCredentialAction({ projectId, host: h }); + if (!res.ok) setError(res.error); + else router.refresh(); + }); + + const hasAnything = waitingHosts.length > 0 || credentials.length > 0 || open; + if (!hasAnything) { + return ( +
+
+
+

Seed logins

+

+ Some directories only show their listings to a signed-in visitor. Store a login and + campaigns can seed from those too. +

+
+ +
+
+ ); + } + + return ( +
+
+

Seed logins

+ {!open && ( + + )} +
+ + {waitingHosts.length > 0 && ( +
+

+ Waiting for sign-in. A campaign is parked on{" "} + {waitingHosts.length === 1 ? "a directory that requires" : "directories that require"} a + login: +

+
    + {waitingHosts.map((h) => ( +
  • + {h} + +
    • + ))} +
+
+ )} + + {open && ( +
+ + + +

+ Encrypted with AES-256-GCM before storing; the key lives in the application + environment, not the database. Sign-in is attempted on the next campaign tick and the + result is shown here — a site that asks for a verification code can't be signed + into from a server, and will say so. +

+
+ + +
+
+ )} + + {credentials.length > 0 && ( +
    + {credentials.map((c) => ( +
  • +
    +

    {c.host}

    +

    + {c.username} + {c.verifiedAt ? ` · signed in ${c.verifiedAt.slice(0, 10)}` : " · not tried yet"} +

    + {c.lastError && ( +

    {c.lastError}

    + )} +
    + +
    • + ))} +
+ )} + + {note &&

{note}

} + {error &&

{error}

} +
+ ); +} diff --git a/lib/outreach/discover.ts b/lib/outreach/discover.ts index 73a0baad..48189e2a 100644 --- a/lib/outreach/discover.ts +++ b/lib/outreach/discover.ts @@ -21,7 +21,9 @@ import { searchSerp, hasValueSerpKey } from "@/lib/alerts/valueserp"; import { isThirdPartyHost } from "@/lib/leadCampaign"; import { businessSearch } from "./freeSearch"; import { normalizeHost } from "./cold"; +import { loadSeedCredential, recordSeedCredentialResult, seedHost } from "./seedCredentials"; import { looksLikeLoginWall } from "./loginWall"; +import type { SeedCredentials } from "./seedLogin"; export type DiscoveredProspect = { host: string; @@ -216,10 +218,12 @@ async function fetchHtml( async function loadSeedHtml( url: string, allowRender: boolean, + credentials?: SeedCredentials | null, ): Promise<{ html: string; rendered: boolean } | { error: string; loginRequired?: boolean }> { const direct = await fetchHtml(url); - // A login wall is settled: rendering it again only renders the login page. - if (!direct.ok && direct.loginRequired) { + // A login wall is settled unless we hold a credential — without one, + // rendering it again only renders the login page. + if (!direct.ok && direct.loginRequired && !credentials) { return { error: direct.error, loginRequired: true }; } if (direct.ok) { @@ -230,7 +234,7 @@ async function loadSeedHtml( } const { renderPage } = await import("./render"); - const rendered = await renderPage(url); + const rendered = await renderPage(url, { credentials }); if (rendered.ok) return { html: rendered.html, rendered: true }; if (rendered.loginRequired) return { error: rendered.error, loginRequired: true }; @@ -254,6 +258,8 @@ export async function discoverFromSeed(input: { depth?: 1 | 2; /** Cap on second-hop pages opened, since each is a full page load. */ maxDetailPages?: number; + /** Sign-in for a gated directory, when one is stored for this host. */ + credentials?: SeedCredentials | null; /** * Only take the second hop when the first found fewer than this many * businesses. Keeps ordinary directories at one cheap page load. @@ -270,7 +276,7 @@ export async function discoverFromSeed(input: { const allowRender = input.render !== false; const notes: string[] = []; - const seed = await loadSeedHtml(input.seedUrl, allowRender); + const seed = await loadSeedHtml(input.seedUrl, allowRender, input.credentials); if ("error" in seed) { return { prospects: [], @@ -302,7 +308,7 @@ export async function discoverFromSeed(input: { } for (const detailUrl of detailPages) { if (merged.size >= limit) break; - const detail = await loadSeedHtml(detailUrl, allowRender); + const detail = await loadSeedHtml(detailUrl, allowRender, input.credentials); if ("error" in detail) continue; for (const p of extractOutboundProspects({ html: detail.html, @@ -338,6 +344,8 @@ export async function discoverFromSearch(input: { query: string; limit?: number; source?: SearchSource; + /** Org whose stored seed logins apply. Omitted means none are used. */ + organizationId?: string | null; }): Promise<{ prospects: DiscoveredProspect[]; calls: number; error?: string }> { const limit = Math.min(input.limit ?? 30, 100); const source = input.source ?? "auto"; @@ -398,6 +406,8 @@ export async function discoverProspects(input: { seedUrls?: string[]; limit?: number; source?: SearchSource; + /** Org whose stored seed logins apply. Omitted means none are used. */ + organizationId?: string | null; }): Promise<{ prospects: DiscoveredProspect[]; serpCalls: number; @@ -428,9 +438,22 @@ export async function discoverProspects(input: { // Depth 2 by default: a platform directory keeps every listing on its own // domain, so depth 1 silently returns nothing for exactly the pages users // most often paste in. - const res = await discoverFromSeed({ seedUrl, limit, depth: 2 }); + const credentials = input.organizationId + ? await loadSeedCredential(input.organizationId, seedHost(seedUrl)) + : null; + const res = await discoverFromSeed({ seedUrl, limit, depth: 2, credentials }); if (res.error) errors.push(res.error); - if (res.loginRequired) loginRequiredSeeds.push(seedUrl); + // Only "waiting on the user" when we have nothing to try. A stored + // credential that failed is a different problem and reads as an error. + if (res.loginRequired && !credentials) loginRequiredSeeds.push(seedUrl); + if (input.organizationId && credentials) { + await recordSeedCredentialResult({ + organizationId: input.organizationId, + host: seedHost(seedUrl), + ok: !res.loginRequired, + error: res.error, + }); + } for (const p of res.prospects) if (!merged.has(p.host)) merged.set(p.host, p); } diff --git a/lib/outreach/render.ts b/lib/outreach/render.ts index 4ba6fc8a..80bf4609 100644 --- a/lib/outreach/render.ts +++ b/lib/outreach/render.ts @@ -26,6 +26,7 @@ import type { Browser, BrowserContext } from "playwright"; import { isPrivateAddress } from "./mailboxDiscovery"; import { looksLikeLoginWall } from "./loginWall"; +import type { SeedCredentials } from "./seedLogin"; import dns from "node:dns/promises"; import net from "node:net"; @@ -117,7 +118,13 @@ const CHALLENGE_RE = /just a moment|attention required|verifying you are human|c * Resolves rather than throws: discovery treats a failed render as "this seed * produced nothing", not as a reason to abort a campaign tick. */ -export async function renderPage(url: string): Promise { +export async function renderPage( + url: string, + opts?: { + /** Sign in and retry if the page turns out to be gated. */ + credentials?: SeedCredentials | null; + }, +): Promise { let parsed: URL; try { parsed = new URL(url); @@ -175,18 +182,46 @@ export async function renderPage(url: string): Promise { "the site served a bot-protection challenge instead of the page — rendering can't get past it", }; } - if (status >= 400) { - return { ok: false, status, error: `rendered with HTTP ${status}` }; - } - - const html = await page.content(); - const finalUrl = page.url(); + let html = await page.content(); + let finalUrl = page.url(); // A login wall answers 200 with a real page, so it has to be caught by - // what the page is rather than by the status. Reported as a failure - // because the caller asked for a directory and did not get one. - const wall = looksLikeLoginWall({ requestedUrl: url, finalUrl, html }); - if (wall.loginRequired) { + // what the page is rather than by the status. + let wall = looksLikeLoginWall({ requestedUrl: url, finalUrl, html }); + + if (wall.loginRequired && opts?.credentials) { + // We are already sitting on the login page — the redirect put us + // there — so the form is in front of us. Signing in on this same + // context keeps the session cookies for the retry. + const { submitLoginForm } = await import("./seedLogin"); + const login = await submitLoginForm(page, opts.credentials); + if (!login.ok) { + return { + ok: false, + status, + loginRequired: true, + error: `signing in failed — ${login.error}`, + }; + } + + await page.goto(url, { waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS }); + await page + .waitForLoadState("networkidle", { timeout: SETTLE_MS * 2 }) + .catch(() => page.waitForTimeout(SETTLE_MS)); + + html = await page.content(); + finalUrl = page.url(); + wall = looksLikeLoginWall({ requestedUrl: url, finalUrl, html }); + if (wall.loginRequired) { + return { + ok: false, + status, + loginRequired: true, + error: + "signed in, but the page still asked for a login — the account may not have access to it", + }; + } + } else if (wall.loginRequired) { return { ok: false, status, @@ -195,6 +230,8 @@ export async function renderPage(url: string): Promise { }; } + if (status >= 400) return { ok: false, status, error: `rendered with HTTP ${status}` }; + return { ok: true, status, diff --git a/lib/outreach/runner.ts b/lib/outreach/runner.ts index 9317b1e8..58102fb3 100644 --- a/lib/outreach/runner.ts +++ b/lib/outreach/runner.ts @@ -66,6 +66,8 @@ export type TickResult = { dryRuns: number; /** The campaign's actual auto_send setting, so the summary can't misreport it. */ autoSend: boolean; + /** Seed URLs sitting behind a sign-in with no stored credential. */ + awaitingAuth: string[]; skipped: string[]; errors: string[]; }; @@ -84,6 +86,7 @@ export async function runEmailCampaignTick(campaign: CampaignRow): Promise ["new", "researched", "drafted"].includes(p.status)).length; if (liveCount < campaign.target_pipeline) { const want = Math.min(campaign.target_pipeline - liveCount, MAX_DISCOVER_PER_TICK); + const { data: projectRow } = await sb + .from("projects") + .select("organization_id") + .eq("id", campaign.project_id) + .maybeSingle(); + const organizationId = (projectRow?.organization_id as string | null) ?? null; + const found = await discoverProspects({ queries: campaign.queries ?? [], seedUrls: campaign.seed_urls ?? [], limit: want * 3, // over-fetch: most candidates are already known or filtered + organizationId, }); result.errors.push(...found.errors); + result.awaitingAuth = found.loginRequiredSeeds; + + // Park the gated hosts on the campaign so the UI can say what it is + // waiting for, and offer the form that unblocks it, instead of leaving + // the reason buried in an error string. + await sb + .from("outreach_campaigns") + .update({ auth_required_hosts: found.loginRequiredSeeds }) + .eq("id", campaign.id); const known = new Set(prospects.map((p) => p.target_key)); let added = 0; @@ -361,6 +381,7 @@ export function summarize(r: TickResult): string { ? `${r.dryRuns} drafted, 0 sent` : `${r.dryRuns} drafted (auto_send off)`, ]; + if (r.awaitingAuth.length) parts.push(`${r.awaitingAuth.length} waiting_for_auth`); if (r.skipped.length) parts.push(`${r.skipped.length} skipped`); if (r.errors.length) parts.push(`${r.errors.length} errors`); return parts.join(", "); diff --git a/lib/outreach/seedCredentials.ts b/lib/outreach/seedCredentials.ts new file mode 100644 index 00000000..a8377d53 --- /dev/null +++ b/lib/outreach/seedCredentials.ts @@ -0,0 +1,103 @@ +// Stored logins for seed directories that gate their listings. +// +// Scoped per host and per organization: a user has one account on a +// directory, not one per search they paste in, and several campaigns +// seeding the same site should share it. +// +// The password only ever exists in the database as AES-256-GCM ciphertext +// (lib/sp/vault.ts), with the key in the app environment. A database dump +// yields nothing usable on its own — the same bar the mailbox credentials +// hold. + +import { serviceClient } from "@/lib/supabase/service"; +import { decryptSecret } from "@/lib/sp/vault"; +import { normalizeHost } from "./cold"; +import type { SeedCredentials } from "./seedLogin"; + +export type StoredSeedCredential = { + id: string; + host: string; + username: string; + loginUrl: string | null; + verifiedAt: string | null; + lastError: string | null; +}; + +/** Host key for a seed URL. Credentials are matched on this. */ +export function seedHost(url: string): string { + try { + return normalizeHost(new URL(url).hostname); + } catch { + return ""; + } +} + +/** + * The decrypted credential for a host, or null. + * + * Returns null rather than throwing when the ciphertext won't open: a + * rotated vault key should stall one seed, not take down a campaign tick. + */ +export async function loadSeedCredential( + organizationId: string, + host: string, +): Promise { + if (!organizationId || !host) return null; + const { data } = await serviceClient() + .from("outreach_seed_credentials") + .select("username, enc_password, login_url") + .eq("organization_id", organizationId) + .eq("host", normalizeHost(host)) + .maybeSingle(); + if (!data) return null; + + const row = data as Record; + const enc = row.enc_password; + if (!enc || !row.username) return null; + try { + return { + username: row.username, + password: decryptSecret(enc), + loginUrl: row.login_url ?? undefined, + }; + } catch { + return null; + } +} + +/** Every stored credential for an org, without the secrets. */ +export async function listSeedCredentials( + organizationId: string, +): Promise { + const { data } = await serviceClient() + .from("outreach_seed_credentials") + .select("id, host, username, login_url, verified_at, last_error") + .eq("organization_id", organizationId) + .order("host"); + return ((data as Record[] | null) ?? []).map((r) => ({ + id: r.id as string, + host: r.host as string, + username: r.username as string, + loginUrl: r.login_url, + verifiedAt: r.verified_at, + lastError: r.last_error, + })); +} + +/** Record whether a credential actually got us in, for the UI to show. */ +export async function recordSeedCredentialResult(input: { + organizationId: string; + host: string; + ok: boolean; + error?: string | null; +}): Promise { + await serviceClient() + .from("outreach_seed_credentials") + .update( + input.ok + ? { verified_at: new Date().toISOString(), last_error: null } + : { last_error: (input.error ?? "sign-in failed").slice(0, 300) }, + ) + .eq("organization_id", input.organizationId) + .eq("host", normalizeHost(input.host)); +} diff --git a/lib/outreach/seedLogin.ts b/lib/outreach/seedLogin.ts new file mode 100644 index 00000000..07716317 --- /dev/null +++ b/lib/outreach/seedLogin.ts @@ -0,0 +1,150 @@ +// Sign in to a seed directory, generically. +// +// The alternative is a per-site login script, which is the thing we are +// trying not to build: every directory would need one, and each would break +// on its own schedule. A login form is well enough standardised to drive +// blind — one password field, one identifier field near it, one submit — +// and a directory that gates its listings behind a sign-in is almost always +// using an ordinary form. +// +// Scope note: this is for ordinary gated directories. It is not going to get +// anyone into a site that fights automation — those answer a server-side +// login attempt with a checkpoint, a second factor, or a rate limit, none of +// which a form fill can satisfy. Detection stays useful there; this doesn't. + +import type { Page } from "playwright"; + +export type SeedCredentials = { + username: string; + password: string; + /** Where the login form lives, when discovery already found it. */ + loginUrl?: string; +}; + +export type LoginOutcome = { ok: true } | { ok: false; error: string }; + +const FIELD_TIMEOUT_MS = 8_000; +const SETTLE_MS = 6_000; + +// Ordered by how strongly each names an identifier field. `email` before +// `user` because a form with both usually wants the email. +const USERNAME_SELECTORS = [ + 'input[type="email"]:visible', + 'input[name*="email" i]:visible', + 'input[id*="email" i]:visible', + 'input[autocomplete="username"]:visible', + 'input[name*="user" i]:visible', + 'input[id*="user" i]:visible', + 'input[name*="login" i]:visible', + 'input[type="text"]:visible', +]; + +const SUBMIT_SELECTORS = [ + 'button[type="submit"]:visible', + 'input[type="submit"]:visible', + 'button:has-text("Log in"):visible', + 'button:has-text("Sign in"):visible', + 'button:has-text("Continue"):visible', +]; + +/** Text a site shows when it wants more than a password. */ +const EXTRA_STEP_RE = + /(two[- ]factor|verification code|security code|one[- ]time|authenticator|confirm your identity|unusual activity|suspicious|captcha|checkpoint|are you a robot)/i; + +const BAD_CREDENTIALS_RE = + /(incorrect|invalid|wrong password|didn'?t match|not recognised|not recognized|try again)/i; + +/** + * Fill and submit the login form on `page`, then confirm it took. + * + * Returns a plain outcome rather than throwing: a failed login means this + * seed produced nothing, not that the campaign should stop. + */ +export async function submitLoginForm( + page: Page, + creds: SeedCredentials, +): Promise { + // The password field is the anchor: if there isn't one, this isn't a login + // form and guessing at the rest would only produce noise. + const password = page.locator('input[type="password"]:visible').first(); + try { + await password.waitFor({ state: "visible", timeout: FIELD_TIMEOUT_MS }); + } catch { + return { ok: false, error: "no password field was found on the login page" }; + } + + let filledUsername = false; + for (const selector of USERNAME_SELECTORS) { + const field = page.locator(selector).first(); + if ((await field.count()) === 0) continue; + try { + await field.fill(creds.username, { timeout: 2_000 }); + filledUsername = true; + break; + } catch { + // Not editable, or covered by an overlay — try the next shape. + } + } + if (!filledUsername) { + return { ok: false, error: "no username or email field was found on the login page" }; + } + + try { + await password.fill(creds.password, { timeout: 2_000 }); + } catch { + return { ok: false, error: "the password field would not accept input" }; + } + + // Submitting: a click is preferred because some forms bind handlers to the + // button, but pressing Enter in the password field is the reliable fallback + // for forms whose button is a styled div. + let submitted = false; + for (const selector of SUBMIT_SELECTORS) { + const button = page.locator(selector).first(); + if ((await button.count()) === 0) continue; + try { + await button.click({ timeout: 3_000 }); + submitted = true; + break; + } catch { + // Fall through. + } + } + if (!submitted) { + try { + await password.press("Enter"); + submitted = true; + } catch { + return { ok: false, error: "the login form could not be submitted" }; + } + } + + await page + .waitForLoadState("networkidle", { timeout: SETTLE_MS }) + .catch(() => page.waitForTimeout(2_500)); + + const body = await page.content().catch(() => ""); + + // A second factor is a hard stop, and worth naming precisely: the + // credentials may be perfectly correct and still unusable from a server. + if (EXTRA_STEP_RE.test(body)) { + return { + ok: false, + error: + "the site asked for a second step (a verification code, or a challenge) that can't be answered from a server", + }; + } + + // Still showing a password field means the submit bounced. + const stillOnForm = (await page.locator('input[type="password"]:visible').count()) > 0; + if (stillOnForm) { + return { + ok: false, + error: BAD_CREDENTIALS_RE.test(body) + ? "the site rejected that username and password" + : "the login form was still showing after submitting", + }; + } + + return { ok: true }; +} diff --git a/supabase/migrations/20260728010000_outreach_seed_credentials.sql b/supabase/migrations/20260728010000_outreach_seed_credentials.sql new file mode 100644 index 00000000..0f02c0e2 --- /dev/null +++ b/supabase/migrations/20260728010000_outreach_seed_credentials.sql @@ -0,0 +1,68 @@ +-- Credentials for seed directories that sit behind a login. +-- +-- Some directories worth seeding — trade association rosters, members-only +-- marketplaces — only show their listings to a signed-in visitor. Detection +-- (lib/outreach/loginWall.ts) can now tell that apart from an empty +-- directory; this is where the credential to get past it lives. +-- +-- Scoped per host rather than per seed URL: a user has one account on a +-- directory, not one per search they paste in, and several seeds against the +-- same site should share it. +-- +-- The password is only ever written to enc_password, AES-256-GCM via +-- lib/sp/vault.ts, with the key held in the app environment and never in the +-- database. There is deliberately no plaintext column to write to. + +create table if not exists public.outreach_seed_credentials ( + id uuid primary key default gen_random_uuid(), + organization_id uuid not null references public.organizations(id) on delete cascade, + created_by uuid references public.profiles(id) on delete set null, + -- Normalized, no scheme or www: matched against a seed URL's host. + host text not null, + username text not null, + enc_password text not null, + -- The page the login form was found on, so a retry doesn't have to guess. + login_url text, + -- Last time these credentials actually got us past the wall. + verified_at timestamptz, + last_error text, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +-- One credential per host per org: a second one would be ambiguous at seed +-- time, and updating in place is what "change my password" should do. +create unique index if not exists outreach_seed_credentials_org_host_idx + on public.outreach_seed_credentials(organization_id, host); + +alter table public.outreach_seed_credentials enable row level security; + +-- Read paths run on the service client after an explicit access check, the +-- same as the rest of the outreach tables. Owning the org is what grants +-- management, so a member cannot read another team's stored logins. +drop policy if exists "outreach_seed_credentials owner all" + on public.outreach_seed_credentials; + +create policy "outreach_seed_credentials owner all" + on public.outreach_seed_credentials for all + using ((select public.is_org_owner(organization_id, auth.uid()))) + with check ((select public.is_org_owner(organization_id, auth.uid()))); + +drop trigger if exists outreach_seed_credentials_set_updated_at + on public.outreach_seed_credentials; +create trigger outreach_seed_credentials_set_updated_at + before update on public.outreach_seed_credentials + for each row execute function public.lx_set_updated_at(); + +comment on table public.outreach_seed_credentials is + 'Per-host logins for seed directories behind a sign-in wall. Passwords are AES-256-GCM (lib/sp/vault.ts); the key lives in the app environment, never here.'; + +-- A campaign whose seed is gated is not failing, it is waiting on something +-- only the user can supply. Recording which hosts are waiting lets the UI say +-- so plainly, and offer the form that unblocks it, instead of leaving the +-- reason buried in an error string nobody reads. +alter table public.outreach_campaigns + add column if not exists auth_required_hosts jsonb not null default '[]'::jsonb; + +comment on column public.outreach_campaigns.auth_required_hosts is + 'Seed hosts that returned a sign-in wall and have no stored credential. Empty means nothing is waiting on the user.';