diff --git a/app/(app)/projects/[id]/leads/page.tsx b/app/(app)/projects/[id]/leads/page.tsx index d77dff5e..ea08cc3e 100644 --- a/app/(app)/projects/[id]/leads/page.tsx +++ b/app/(app)/projects/[id]/leads/page.tsx @@ -8,6 +8,8 @@ import { LeadActions } from "@/components/leads/lead-actions"; import { CampaignPanel, type CampaignSummary } from "@/components/leads/campaign-panel"; import { SenderAddress } from "@/components/leads/sender-address"; import { MailboxConnect, type ConnectedMailbox } from "@/components/leads/mailbox-connect"; +import { SeedLogins } from "@/components/leads/seed-logins"; +import { listSeedCredentials, type StoredSeedCredential } from "@/lib/outreach/seedCredentials"; import { RefreshLeads } from "@/components/leads/refresh-leads"; import { loadAddressSettings } from "@/lib/outreach/postalAddress"; @@ -119,6 +121,20 @@ export default async function LeadsPage({ .eq("id", projectId) .maybeSingle(); const orgId = (projectRow?.organization_id as string | null) ?? null; + let seedCredentials: StoredSeedCredential[] = []; + if (orgId) seedCredentials = await listSeedCredentials(orgId); + + // Hosts any campaign in this project is parked on, waiting for a sign-in. + const waitingHosts = [ + ...new Set( + campaigns.flatMap((c) => + Array.isArray((c as { auth_required_hosts?: string[] }).auth_required_hosts) + ? ((c as { auth_required_hosts?: string[] }).auth_required_hosts as string[]) + : [], + ), + ), + ]; + let mailbox: ConnectedMailbox | null = null; if (orgId) { const { data: senderRow } = await supabase @@ -164,6 +180,12 @@ export default async function LeadsPage({ + +

Pipeline

diff --git a/app/actions/seedCredentials.ts b/app/actions/seedCredentials.ts new file mode 100644 index 00000000..e38b2dd3 --- /dev/null +++ b/app/actions/seedCredentials.ts @@ -0,0 +1,118 @@ +"use server"; + +import { revalidatePath } from "next/cache"; +import { serviceClient } from "@/lib/supabase/service"; +import { requireProjectAccess } from "@/lib/lx/currentSite"; +import { encryptSecret } from "@/lib/sp/vault"; +import { normalizeHost } from "@/lib/outreach/cold"; +import { seedHost } from "@/lib/outreach/seedCredentials"; + +type Ok> = { ok: true } & T; +type Err = { ok: false; error: string }; + +async function requireOrg( + projectId: string, +): Promise<{ ok: true; userId: string; organizationId: string } | Err> { + if (!projectId) return { ok: false, error: "Missing project." }; + const access = await requireProjectAccess(projectId); + if (!access.ok) return { ok: false, error: access.error }; + if (access.isViewer) return { ok: false, error: "Viewers can't manage seed logins." }; + + const { data: project } = await serviceClient() + .from("projects") + .select("organization_id") + .eq("id", projectId) + .maybeSingle(); + const organizationId = (project?.organization_id as string | null) ?? null; + if (!organizationId) { + return { ok: false, error: "This project isn't in an organization, so there's nowhere to store a login." }; + } + return { ok: true, userId: access.userId, organizationId }; +} + +/** + * Store a login for a gated seed directory. + * + * Unlike the mailbox, the credential is not verified at save time: doing so + * would mean driving a browser through someone else's login form inside a + * server action, and a directory that is slow or briefly down would look like + * a bad password. It is verified on the next tick instead, and the result is + * recorded on the row so the UI can show whether it actually worked. + */ +export async function saveSeedCredentialAction(input: { + projectId: string; + host: string; + username: string; + password: string; + loginUrl?: string; +}): Promise | Err> { + const access = await requireOrg(input.projectId); + if (!access.ok) return access; + + // Accept a full URL or a bare host — the value usually comes from a seed + // URL the user pasted somewhere else. + const host = input.host.includes("://") + ? seedHost(input.host) + : normalizeHost(input.host.trim()); + if (!host || !host.includes(".")) return { ok: false, error: "That doesn't look like a site." }; + if (!input.username.trim()) return { ok: false, error: "A username or email is required." }; + if (!input.password) return { ok: false, error: "A password is required." }; + + const { error } = await serviceClient() + .from("outreach_seed_credentials") + .upsert( + { + organization_id: access.organizationId, + created_by: access.userId, + host, + username: input.username.trim(), + enc_password: encryptSecret(input.password), + login_url: input.loginUrl?.trim() || null, + // A new password invalidates whatever the last attempt concluded. + verified_at: null, + last_error: null, + }, + { onConflict: "organization_id,host" }, + ); + if (error) return { ok: false, error: error.message }; + + // Any campaign parked on this host can stop waiting. + const { data: campaigns } = await serviceClient() + .from("outreach_campaigns") + .select("id, auth_required_hosts") + .eq("project_id", input.projectId); + for (const c of (campaigns as { id: string; auth_required_hosts: string[] | null }[] | null) ?? []) { + const waiting = Array.isArray(c.auth_required_hosts) ? c.auth_required_hosts : []; + const remaining = waiting.filter((u) => seedHost(u) !== host); + if (remaining.length !== waiting.length) { + await serviceClient() + .from("outreach_campaigns") + .update({ auth_required_hosts: remaining }) + .eq("id", c.id); + } + } + + revalidatePath(`/projects/${input.projectId}/leads`); + return { + ok: true, + note: `Saved a login for ${host}. The next campaign tick will try it and report whether it worked.`, + }; +} + +export async function deleteSeedCredentialAction(input: { + projectId: string; + host: string; +}): Promise<{ ok: true } | Err> { + const access = await requireOrg(input.projectId); + if (!access.ok) return access; + + const { error } = await serviceClient() + .from("outreach_seed_credentials") + .delete() + .eq("organization_id", access.organizationId) + .eq("host", normalizeHost(input.host)); + if (error) return { ok: false, error: error.message }; + + revalidatePath(`/projects/${input.projectId}/leads`); + return { ok: true }; +} diff --git a/components/leads/seed-logins.tsx b/components/leads/seed-logins.tsx new file mode 100644 index 00000000..21d207ce --- /dev/null +++ b/components/leads/seed-logins.tsx @@ -0,0 +1,206 @@ +"use client"; + +import { useState, useTransition } from "react"; +import { useRouter } from "next/navigation"; +import { + deleteSeedCredentialAction, + saveSeedCredentialAction, +} from "@/app/actions/seedCredentials"; +import type { StoredSeedCredential } from "@/lib/outreach/seedCredentials"; + +/** + * Seed directories that need a sign-in. + * + * The panel leads with what is actually blocked. A campaign parked on a gated + * directory isn't failing, it is waiting on something only the user can + * supply, so the hosts it is waiting for are named and each one carries the + * form that unblocks it. + */ +export function SeedLogins({ + projectId, + waitingHosts, + credentials, +}: { + projectId: string; + /** Seed URLs campaigns are parked on, with no stored credential. */ + waitingHosts: string[]; + credentials: StoredSeedCredential[]; +}) { + const router = useRouter(); + const [pending, start] = useTransition(); + const [open, setOpen] = useState(false); + const [host, setHost] = useState(""); + const [username, setUsername] = useState(""); + const [password, setPassword] = useState(""); + const [note, setNote] = useState(null); + const [error, setError] = useState(null); + + const save = (targetHost?: string) => + start(async () => { + setNote(null); + setError(null); + const res = await saveSeedCredentialAction({ + projectId, + host: targetHost ?? host, + username, + password, + }); + if (!res.ok) { + setError(res.error); + return; + } + setNote(res.note); + setHost(""); + setUsername(""); + setPassword(""); + setOpen(false); + router.refresh(); + }); + + const remove = (h: string) => + start(async () => { + setError(null); + const res = await deleteSeedCredentialAction({ projectId, host: h }); + if (!res.ok) setError(res.error); + else router.refresh(); + }); + + const hasAnything = waitingHosts.length > 0 || credentials.length > 0 || open; + if (!hasAnything) { + return ( +
+
+
+

Seed logins

+

+ Some directories only show their listings to a signed-in visitor. Store a login and + campaigns can seed from those too. +

+
+ +
+
+ ); + } + + return ( +
+
+

Seed logins

+ {!open && ( + + )} +
+ + {waitingHosts.length > 0 && ( +
+

+ Waiting for sign-in. A campaign is parked on{" "} + {waitingHosts.length === 1 ? "a directory that requires" : "directories that require"} a + login: +

+
    + {waitingHosts.map((h) => ( +
  • + {h} + +
    • + ))} +
+
+ )} + + {open && ( +
+ + + +

+ Encrypted with AES-256-GCM before storing; the key lives in the application + environment, not the database. Sign-in is attempted on the next campaign tick and the + result is shown here — a site that asks for a verification code can't be signed + into from a server, and will say so. +

+
+ + +
+
+ )} + + {credentials.length > 0 && ( +
    + {credentials.map((c) => ( +
  • +
    +

    {c.host}

    +

    + {c.username} + {c.verifiedAt ? ` · signed in ${c.verifiedAt.slice(0, 10)}` : " · not tried yet"} +

    + {c.lastError && ( +

    {c.lastError}

    + )} +
    + +
    • + ))} +
+ )} + + {note &&

{note}

} + {error &&

{error}

} +
+ ); +} diff --git a/lib/outreach/discover.ts b/lib/outreach/discover.ts index 73a0baad..48189e2a 100644 --- a/lib/outreach/discover.ts +++ b/lib/outreach/discover.ts @@ -21,7 +21,9 @@ import { searchSerp, hasValueSerpKey } from "@/lib/alerts/valueserp"; import { isThirdPartyHost } from "@/lib/leadCampaign"; import { businessSearch } from "./freeSearch"; import { normalizeHost } from "./cold"; +import { loadSeedCredential, recordSeedCredentialResult, seedHost } from "./seedCredentials"; import { looksLikeLoginWall } from "./loginWall"; +import type { SeedCredentials } from "./seedLogin"; export type DiscoveredProspect = { host: string; @@ -216,10 +218,12 @@ async function fetchHtml( async function loadSeedHtml( url: string, allowRender: boolean, + credentials?: SeedCredentials | null, ): Promise<{ html: string; rendered: boolean } | { error: string; loginRequired?: boolean }> { const direct = await fetchHtml(url); - // A login wall is settled: rendering it again only renders the login page. - if (!direct.ok && direct.loginRequired) { + // A login wall is settled unless we hold a credential — without one, + // rendering it again only renders the login page. + if (!direct.ok && direct.loginRequired && !credentials) { return { error: direct.error, loginRequired: true }; } if (direct.ok) { @@ -230,7 +234,7 @@ async function loadSeedHtml( } const { renderPage } = await import("./render"); - const rendered = await renderPage(url); + const rendered = await renderPage(url, { credentials }); if (rendered.ok) return { html: rendered.html, rendered: true }; if (rendered.loginRequired) return { error: rendered.error, loginRequired: true }; @@ -254,6 +258,8 @@ export async function discoverFromSeed(input: { depth?: 1 | 2; /** Cap on second-hop pages opened, since each is a full page load. */ maxDetailPages?: number; + /** Sign-in for a gated directory, when one is stored for this host. */ + credentials?: SeedCredentials | null; /** * Only take the second hop when the first found fewer than this many * businesses. Keeps ordinary directories at one cheap page load. @@ -270,7 +276,7 @@ export async function discoverFromSeed(input: { const allowRender = input.render !== false; const notes: string[] = []; - const seed = await loadSeedHtml(input.seedUrl, allowRender); + const seed = await loadSeedHtml(input.seedUrl, allowRender, input.credentials); if ("error" in seed) { return { prospects: [], @@ -302,7 +308,7 @@ export async function discoverFromSeed(input: { } for (const detailUrl of detailPages) { if (merged.size >= limit) break; - const detail = await loadSeedHtml(detailUrl, allowRender); + const detail = await loadSeedHtml(detailUrl, allowRender, input.credentials); if ("error" in detail) continue; for (const p of extractOutboundProspects({ html: detail.html, @@ -338,6 +344,8 @@ export async function discoverFromSearch(input: { query: string; limit?: number; source?: SearchSource; + /** Org whose stored seed logins apply. Omitted means none are used. */ + organizationId?: string | null; }): Promise<{ prospects: DiscoveredProspect[]; calls: number; error?: string }> { const limit = Math.min(input.limit ?? 30, 100); const source = input.source ?? "auto"; @@ -398,6 +406,8 @@ export async function discoverProspects(input: { seedUrls?: string[]; limit?: number; source?: SearchSource; + /** Org whose stored seed logins apply. Omitted means none are used. */ + organizationId?: string | null; }): Promise<{ prospects: DiscoveredProspect[]; serpCalls: number; @@ -428,9 +438,22 @@ export async function discoverProspects(input: { // Depth 2 by default: a platform directory keeps every listing on its own // domain, so depth 1 silently returns nothing for exactly the pages users // most often paste in. - const res = await discoverFromSeed({ seedUrl, limit, depth: 2 }); + const credentials = input.organizationId + ? await loadSeedCredential(input.organizationId, seedHost(seedUrl)) + : null; + const res = await discoverFromSeed({ seedUrl, limit, depth: 2, credentials }); if (res.error) errors.push(res.error); - if (res.loginRequired) loginRequiredSeeds.push(seedUrl); + // Only "waiting on the user" when we have nothing to try. A stored + // credential that failed is a different problem and reads as an error. + if (res.loginRequired && !credentials) loginRequiredSeeds.push(seedUrl); + if (input.organizationId && credentials) { + await recordSeedCredentialResult({ + organizationId: input.organizationId, + host: seedHost(seedUrl), + ok: !res.loginRequired, + error: res.error, + }); + } for (const p of res.prospects) if (!merged.has(p.host)) merged.set(p.host, p); } diff --git a/lib/outreach/render.ts b/lib/outreach/render.ts index 4ba6fc8a..80bf4609 100644 --- a/lib/outreach/render.ts +++ b/lib/outreach/render.ts @@ -26,6 +26,7 @@ import type { Browser, BrowserContext } from "playwright"; import { isPrivateAddress } from "./mailboxDiscovery"; import { looksLikeLoginWall } from "./loginWall"; +import type { SeedCredentials } from "./seedLogin"; import dns from "node:dns/promises"; import net from "node:net"; @@ -117,7 +118,13 @@ const CHALLENGE_RE = /just a moment|attention required|verifying you are human|c * Resolves rather than throws: discovery treats a failed render as "this seed * produced nothing", not as a reason to abort a campaign tick. */ -export async function renderPage(url: string): Promise { +export async function renderPage( + url: string, + opts?: { + /** Sign in and retry if the page turns out to be gated. */ + credentials?: SeedCredentials | null; + }, +): Promise { let parsed: URL; try { parsed = new URL(url); @@ -175,18 +182,46 @@ export async function renderPage(url: string): Promise { "the site served a bot-protection challenge instead of the page — rendering can't get past it", }; } - if (status >= 400) { - return { ok: false, status, error: `rendered with HTTP ${status}` }; - } - - const html = await page.content(); - const finalUrl = page.url(); + let html = await page.content(); + let finalUrl = page.url(); // A login wall answers 200 with a real page, so it has to be caught by - // what the page is rather than by the status. Reported as a failure - // because the caller asked for a directory and did not get one. - const wall = looksLikeLoginWall({ requestedUrl: url, finalUrl, html }); - if (wall.loginRequired) { + // what the page is rather than by the status. + let wall = looksLikeLoginWall({ requestedUrl: url, finalUrl, html }); + + if (wall.loginRequired && opts?.credentials) { + // We are already sitting on the login page — the redirect put us + // there — so the form is in front of us. Signing in on this same + // context keeps the session cookies for the retry. + const { submitLoginForm } = await import("./seedLogin"); + const login = await submitLoginForm(page, opts.credentials); + if (!login.ok) { + return { + ok: false, + status, + loginRequired: true, + error: `signing in failed — ${login.error}`, + }; + } + + await page.goto(url, { waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS }); + await page + .waitForLoadState("networkidle", { timeout: SETTLE_MS * 2 }) + .catch(() => page.waitForTimeout(SETTLE_MS)); + + html = await page.content(); + finalUrl = page.url(); + wall = looksLikeLoginWall({ requestedUrl: url, finalUrl, html }); + if (wall.loginRequired) { + return { + ok: false, + status, + loginRequired: true, + error: + "signed in, but the page still asked for a login — the account may not have access to it", + }; + } + } else if (wall.loginRequired) { return { ok: false, status, @@ -195,6 +230,8 @@ export async function renderPage(url: string): Promise { }; } + if (status >= 400) return { ok: false, status, error: `rendered with HTTP ${status}` }; + return { ok: true, status, diff --git a/lib/outreach/runner.ts b/lib/outreach/runner.ts index 9317b1e8..58102fb3 100644 --- a/lib/outreach/runner.ts +++ b/lib/outreach/runner.ts @@ -66,6 +66,8 @@ export type TickResult = { dryRuns: number; /** The campaign's actual auto_send setting, so the summary can't misreport it. */ autoSend: boolean; + /** Seed URLs sitting behind a sign-in with no stored credential. */ + awaitingAuth: string[]; skipped: string[]; errors: string[]; }; @@ -84,6 +86,7 @@ export async function runEmailCampaignTick(campaign: CampaignRow): Promise ["new", "researched", "drafted"].includes(p.status)).length; if (liveCount < campaign.target_pipeline) { const want = Math.min(campaign.target_pipeline - liveCount, MAX_DISCOVER_PER_TICK); + const { data: projectRow } = await sb + .from("projects") + .select("organization_id") + .eq("id", campaign.project_id) + .maybeSingle(); + const organizationId = (projectRow?.organization_id as string | null) ?? null; + const found = await discoverProspects({ queries: campaign.queries ?? [], seedUrls: campaign.seed_urls ?? [], limit: want * 3, // over-fetch: most candidates are already known or filtered + organizationId, }); result.errors.push(...found.errors); + result.awaitingAuth = found.loginRequiredSeeds; + + // Park the gated hosts on the campaign so the UI can say what it is + // waiting for, and offer the form that unblocks it, instead of leaving + // the reason buried in an error string. + await sb + .from("outreach_campaigns") + .update({ auth_required_hosts: found.loginRequiredSeeds }) + .eq("id", campaign.id); const known = new Set(prospects.map((p) => p.target_key)); let added = 0; @@ -361,6 +381,7 @@ export function summarize(r: TickResult): string { ? `${r.dryRuns} drafted, 0 sent` : `${r.dryRuns} drafted (auto_send off)`, ]; + if (r.awaitingAuth.length) parts.push(`${r.awaitingAuth.length} waiting_for_auth`); if (r.skipped.length) parts.push(`${r.skipped.length} skipped`); if (r.errors.length) parts.push(`${r.errors.length} errors`); return parts.join(", "); diff --git a/lib/outreach/seedCredentials.ts b/lib/outreach/seedCredentials.ts new file mode 100644 index 00000000..a8377d53 --- /dev/null +++ b/lib/outreach/seedCredentials.ts @@ -0,0 +1,103 @@ +// Stored logins for seed directories that gate their listings. +// +// Scoped per host and per organization: a user has one account on a +// directory, not one per search they paste in, and several campaigns +// seeding the same site should share it. +// +// The password only ever exists in the database as AES-256-GCM ciphertext +// (lib/sp/vault.ts), with the key in the app environment. A database dump +// yields nothing usable on its own — the same bar the mailbox credentials +// hold. + +import { serviceClient } from "@/lib/supabase/service"; +import { decryptSecret } from "@/lib/sp/vault"; +import { normalizeHost } from "./cold"; +import type { SeedCredentials } from "./seedLogin"; + +export type StoredSeedCredential = { + id: string; + host: string; + username: string; + loginUrl: string | null; + verifiedAt: string | null; + lastError: string | null; +}; + +/** Host key for a seed URL. Credentials are matched on this. */ +export function seedHost(url: string): string { + try { + return normalizeHost(new URL(url).hostname); + } catch { + return ""; + } +} + +/** + * The decrypted credential for a host, or null. + * + * Returns null rather than throwing when the ciphertext won't open: a + * rotated vault key should stall one seed, not take down a campaign tick. + */ +export async function loadSeedCredential( + organizationId: string, + host: string, +): Promise { + if (!organizationId || !host) return null; + const { data } = await serviceClient() + .from("outreach_seed_credentials") + .select("username, enc_password, login_url") + .eq("organization_id", organizationId) + .eq("host", normalizeHost(host)) + .maybeSingle(); + if (!data) return null; + + const row = data as Record; + const enc = row.enc_password; + if (!enc || !row.username) return null; + try { + return { + username: row.username, + password: decryptSecret(enc), + loginUrl: row.login_url ?? undefined, + }; + } catch { + return null; + } +} + +/** Every stored credential for an org, without the secrets. */ +export async function listSeedCredentials( + organizationId: string, +): Promise { + const { data } = await serviceClient() + .from("outreach_seed_credentials") + .select("id, host, username, login_url, verified_at, last_error") + .eq("organization_id", organizationId) + .order("host"); + return ((data as Record[] | null) ?? []).map((r) => ({ + id: r.id as string, + host: r.host as string, + username: r.username as string, + loginUrl: r.login_url, + verifiedAt: r.verified_at, + lastError: r.last_error, + })); +} + +/** Record whether a credential actually got us in, for the UI to show. */ +export async function recordSeedCredentialResult(input: { + organizationId: string; + host: string; + ok: boolean; + error?: string | null; +}): Promise { + await serviceClient() + .from("outreach_seed_credentials") + .update( + input.ok + ? { verified_at: new Date().toISOString(), last_error: null } + : { last_error: (input.error ?? "sign-in failed").slice(0, 300) }, + ) + .eq("organization_id", input.organizationId) + .eq("host", normalizeHost(input.host)); +} diff --git a/lib/outreach/seedLogin.ts b/lib/outreach/seedLogin.ts new file mode 100644 index 00000000..07716317 --- /dev/null +++ b/lib/outreach/seedLogin.ts @@ -0,0 +1,150 @@ +// Sign in to a seed directory, generically. +// +// The alternative is a per-site login script, which is the thing we are +// trying not to build: every directory would need one, and each would break +// on its own schedule. A login form is well enough standardised to drive +// blind — one password field, one identifier field near it, one submit — +// and a directory that gates its listings behind a sign-in is almost always +// using an ordinary form. +// +// Scope note: this is for ordinary gated directories. It is not going to get +// anyone into a site that fights automation — those answer a server-side +// login attempt with a checkpoint, a second factor, or a rate limit, none of +// which a form fill can satisfy. Detection stays useful there; this doesn't. + +import type { Page } from "playwright"; + +export type SeedCredentials = { + username: string; + password: string; + /** Where the login form lives, when discovery already found it. */ + loginUrl?: string; +}; + +export type LoginOutcome = { ok: true } | { ok: false; error: string }; + +const FIELD_TIMEOUT_MS = 8_000; +const SETTLE_MS = 6_000; + +// Ordered by how strongly each names an identifier field. `email` before +// `user` because a form with both usually wants the email. +const USERNAME_SELECTORS = [ + 'input[type="email"]:visible', + 'input[name*="email" i]:visible', + 'input[id*="email" i]:visible', + 'input[autocomplete="username"]:visible', + 'input[name*="user" i]:visible', + 'input[id*="user" i]:visible', + 'input[name*="login" i]:visible', + 'input[type="text"]:visible', +]; + +const SUBMIT_SELECTORS = [ + 'button[type="submit"]:visible', + 'input[type="submit"]:visible', + 'button:has-text("Log in"):visible', + 'button:has-text("Sign in"):visible', + 'button:has-text("Continue"):visible', +]; + +/** Text a site shows when it wants more than a password. */ +const EXTRA_STEP_RE = + /(two[- ]factor|verification code|security code|one[- ]time|authenticator|confirm your identity|unusual activity|suspicious|captcha|checkpoint|are you a robot)/i; + +const BAD_CREDENTIALS_RE = + /(incorrect|invalid|wrong password|didn'?t match|not recognised|not recognized|try again)/i; + +/** + * Fill and submit the login form on `page`, then confirm it took. + * + * Returns a plain outcome rather than throwing: a failed login means this + * seed produced nothing, not that the campaign should stop. + */ +export async function submitLoginForm( + page: Page, + creds: SeedCredentials, +): Promise { + // The password field is the anchor: if there isn't one, this isn't a login + // form and guessing at the rest would only produce noise. + const password = page.locator('input[type="password"]:visible').first(); + try { + await password.waitFor({ state: "visible", timeout: FIELD_TIMEOUT_MS }); + } catch { + return { ok: false, error: "no password field was found on the login page" }; + } + + let filledUsername = false; + for (const selector of USERNAME_SELECTORS) { + const field = page.locator(selector).first(); + if ((await field.count()) === 0) continue; + try { + await field.fill(creds.username, { timeout: 2_000 }); + filledUsername = true; + break; + } catch { + // Not editable, or covered by an overlay — try the next shape. + } + } + if (!filledUsername) { + return { ok: false, error: "no username or email field was found on the login page" }; + } + + try { + await password.fill(creds.password, { timeout: 2_000 }); + } catch { + return { ok: false, error: "the password field would not accept input" }; + } + + // Submitting: a click is preferred because some forms bind handlers to the + // button, but pressing Enter in the password field is the reliable fallback + // for forms whose button is a styled div. + let submitted = false; + for (const selector of SUBMIT_SELECTORS) { + const button = page.locator(selector).first(); + if ((await button.count()) === 0) continue; + try { + await button.click({ timeout: 3_000 }); + submitted = true; + break; + } catch { + // Fall through. + } + } + if (!submitted) { + try { + await password.press("Enter"); + submitted = true; + } catch { + return { ok: false, error: "the login form could not be submitted" }; + } + } + + await page + .waitForLoadState("networkidle", { timeout: SETTLE_MS }) + .catch(() => page.waitForTimeout(2_500)); + + const body = await page.content().catch(() => ""); + + // A second factor is a hard stop, and worth naming precisely: the + // credentials may be perfectly correct and still unusable from a server. + if (EXTRA_STEP_RE.test(body)) { + return { + ok: false, + error: + "the site asked for a second step (a verification code, or a challenge) that can't be answered from a server", + }; + } + + // Still showing a password field means the submit bounced. + const stillOnForm = (await page.locator('input[type="password"]:visible').count()) > 0; + if (stillOnForm) { + return { + ok: false, + error: BAD_CREDENTIALS_RE.test(body) + ? "the site rejected that username and password" + : "the login form was still showing after submitting", + }; + } + + return { ok: true }; +} diff --git a/supabase/migrations/20260728010000_outreach_seed_credentials.sql b/supabase/migrations/20260728010000_outreach_seed_credentials.sql new file mode 100644 index 00000000..0f02c0e2 --- /dev/null +++ b/supabase/migrations/20260728010000_outreach_seed_credentials.sql @@ -0,0 +1,68 @@ +-- Credentials for seed directories that sit behind a login. +-- +-- Some directories worth seeding — trade association rosters, members-only +-- marketplaces — only show their listings to a signed-in visitor. Detection +-- (lib/outreach/loginWall.ts) can now tell that apart from an empty +-- directory; this is where the credential to get past it lives. +-- +-- Scoped per host rather than per seed URL: a user has one account on a +-- directory, not one per search they paste in, and several seeds against the +-- same site should share it. +-- +-- The password is only ever written to enc_password, AES-256-GCM via +-- lib/sp/vault.ts, with the key held in the app environment and never in the +-- database. There is deliberately no plaintext column to write to. + +create table if not exists public.outreach_seed_credentials ( + id uuid primary key default gen_random_uuid(), + organization_id uuid not null references public.organizations(id) on delete cascade, + created_by uuid references public.profiles(id) on delete set null, + -- Normalized, no scheme or www: matched against a seed URL's host. + host text not null, + username text not null, + enc_password text not null, + -- The page the login form was found on, so a retry doesn't have to guess. + login_url text, + -- Last time these credentials actually got us past the wall. + verified_at timestamptz, + last_error text, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +-- One credential per host per org: a second one would be ambiguous at seed +-- time, and updating in place is what "change my password" should do. +create unique index if not exists outreach_seed_credentials_org_host_idx + on public.outreach_seed_credentials(organization_id, host); + +alter table public.outreach_seed_credentials enable row level security; + +-- Read paths run on the service client after an explicit access check, the +-- same as the rest of the outreach tables. Owning the org is what grants +-- management, so a member cannot read another team's stored logins. +drop policy if exists "outreach_seed_credentials owner all" + on public.outreach_seed_credentials; + +create policy "outreach_seed_credentials owner all" + on public.outreach_seed_credentials for all + using ((select public.is_org_owner(organization_id, auth.uid()))) + with check ((select public.is_org_owner(organization_id, auth.uid()))); + +drop trigger if exists outreach_seed_credentials_set_updated_at + on public.outreach_seed_credentials; +create trigger outreach_seed_credentials_set_updated_at + before update on public.outreach_seed_credentials + for each row execute function public.lx_set_updated_at(); + +comment on table public.outreach_seed_credentials is + 'Per-host logins for seed directories behind a sign-in wall. Passwords are AES-256-GCM (lib/sp/vault.ts); the key lives in the app environment, never here.'; + +-- A campaign whose seed is gated is not failing, it is waiting on something +-- only the user can supply. Recording which hosts are waiting lets the UI say +-- so plainly, and offer the form that unblocks it, instead of leaving the +-- reason buried in an error string nobody reads. +alter table public.outreach_campaigns + add column if not exists auth_required_hosts jsonb not null default '[]'::jsonb; + +comment on column public.outreach_campaigns.auth_required_hosts is + 'Seed hosts that returned a sign-in wall and have no stored credential. Empty means nothing is waiting on the user.';