From cabf691c263e319892db9e76c4bac74129f7b0c6 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Thu, 24 Sep 2026 06:31:44 +0000 Subject: [PATCH] Email tracking on every surface: API, CLI, TUI Until now a project's email tracking could only be seen or changed on the dashboard's Tracking tab. Now the same thing is reachable with a crp_ token: - API: GET /api/v1/email-tracking (every reachable project, role, on/off, tracking id, last day's human opens/clicks/unsubscribes, no secrets); GET /api/v1/email-tracking/[?secret=1] (the secret only on request and never to a viewer); POST .../ (owners and members only). Roles follow requireProjectAccess. - CLI, in-repo and the published @profullstack/crawlproof (0.3.0): crawlproof email-tracking list | show [--secret] | enable | disable | rotate. show --secret piped prints the secret alone, so it pipes into myna newsletter track set. - TUI: a sixth tab, Email, with its own loader; e turns the highlighted project on or off. Closes #267. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../[project]/[action]/route.ts | 39 +++++ app/api/v1/email-tracking/[project]/route.ts | 40 +++++ app/api/v1/email-tracking/route.ts | 37 ++++ cli/dashboard.ts | 148 +++++++++++++++- cli/index.ts | 8 + lib/emailTracking/access.ts | 159 ++++++++++++++++++ lib/emailTracking/cli.ts | 109 ++++++++++++ packages/cli/package.json | 2 +- packages/cli/src/cli.ts | 10 +- tests/email-tracking-api.test.ts | 144 ++++++++++++++++ tests/email-tracking-surfaces.test.ts | 101 +++++++++++ 11 files changed, 787 insertions(+), 10 deletions(-) create mode 100644 app/api/v1/email-tracking/[project]/[action]/route.ts create mode 100644 app/api/v1/email-tracking/[project]/route.ts create mode 100644 app/api/v1/email-tracking/route.ts create mode 100644 lib/emailTracking/access.ts create mode 100644 lib/emailTracking/cli.ts create mode 100644 tests/email-tracking-api.test.ts create mode 100644 tests/email-tracking-surfaces.test.ts diff --git a/app/api/v1/email-tracking/[project]/[action]/route.ts b/app/api/v1/email-tracking/[project]/[action]/route.ts new file mode 100644 index 0000000..9775fb9 --- /dev/null +++ b/app/api/v1/email-tracking/[project]/[action]/route.ts @@ -0,0 +1,39 @@ +// POST /api/v1/email-tracking// +// +// The dashboard tab's two buttons, for a bearer token. Both change what every +// future email does, so a read-only member is refused. Rotate answers with the +// new secret (the old one keeps verifying until the next rotation, so mail +// already sent still works); enable and disable answer without it. + +import { NextResponse, type NextRequest } from "next/server"; +import { serviceClient } from "@/lib/supabase/service"; +import { authenticateBearer } from "@/lib/sp/apiAuth"; +import { rotateSecret, setEnabled } from "@/lib/emailTracking/store"; +import { accessibleProjects, isAction, pickProject, shapeTracking } from "@/lib/emailTracking/access"; +import { env } from "@/lib/env"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +const siteBase = () => (env.siteUrl || "https://crawlproof.com").replace(/\/$/, ""); + +export async function POST(req: NextRequest, { params }: { params: Promise<{ project: string; action: string }> }) { + const auth = await authenticateBearer(req); + if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status }); + const { project: ref, action } = await params; + if (!isAction(action)) return NextResponse.json({ error: "The action is enable, disable or rotate." }, { status: 404 }); + const sb = serviceClient(); + try { + const project = pickProject(await accessibleProjects(sb, auth.userId), decodeURIComponent(ref)); + if (!project) return NextResponse.json({ error: "No such project, or more than one matches. Use the project id." }, { status: 404 }); + if (project.role === "viewer") { + return NextResponse.json({ error: "Read-only access: you can't change this project's tracking." }, { status: 403 }); + } + const row = action === "rotate" ? await rotateSecret(project.id) : await setEnabled(project.id, action === "enable"); + return NextResponse.json(shapeTracking(project, row, { siteBase: siteBase(), withSecret: action === "rotate" }), { + headers: { "cache-control": "no-store" }, + }); + } catch (e) { + return NextResponse.json({ error: e instanceof Error ? e.message : "Could not change email tracking." }, { status: 503 }); + } +} diff --git a/app/api/v1/email-tracking/[project]/route.ts b/app/api/v1/email-tracking/[project]/route.ts new file mode 100644 index 0000000..1c6ce10 --- /dev/null +++ b/app/api/v1/email-tracking/[project]/route.ts @@ -0,0 +1,40 @@ +// GET /api/v1/email-tracking/[?secret=1] +// +// One project's email tracking. The secret comes back only with ?secret=1 and +// only to someone who may change the project (never a read-only member): it +// signs every link in every email, so it is handed out on purpose, not by +// default. + +import { NextResponse, type NextRequest } from "next/server"; +import { serviceClient } from "@/lib/supabase/service"; +import { authenticateBearer } from "@/lib/sp/apiAuth"; +import { getOrCreateForProject } from "@/lib/emailTracking/store"; +import { accessibleProjects, eventCounts, pickProject, shapeTracking } from "@/lib/emailTracking/access"; +import { env } from "@/lib/env"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +const siteBase = () => (env.siteUrl || "https://crawlproof.com").replace(/\/$/, ""); + +export async function GET(req: NextRequest, { params }: { params: Promise<{ project: string }> }) { + const auth = await authenticateBearer(req); + if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status }); + const { project: ref } = await params; + const sb = serviceClient(); + try { + const project = pickProject(await accessibleProjects(sb, auth.userId), decodeURIComponent(ref)); + if (!project) return NextResponse.json({ error: "No such project, or more than one matches. Use the project id." }, { status: 404 }); + const wantsSecret = req.nextUrl.searchParams.get("secret") === "1"; + if (wantsSecret && project.role === "viewer") { + return NextResponse.json({ error: "Read-only access: the secret is for owners and members." }, { status: 403 }); + } + const row = await getOrCreateForProject(project.id); + const counts = await eventCounts(sb, [project.id], new Date(Date.now() - 86_400_000).toISOString()); + return NextResponse.json(shapeTracking(project, row, { siteBase: siteBase(), counts: counts[project.id], withSecret: wantsSecret }), { + headers: { "cache-control": "no-store" }, + }); + } catch (e) { + return NextResponse.json({ error: e instanceof Error ? e.message : "Could not read email tracking." }, { status: 503 }); + } +} diff --git a/app/api/v1/email-tracking/route.ts b/app/api/v1/email-tracking/route.ts new file mode 100644 index 0000000..e77dee8 --- /dev/null +++ b/app/api/v1/email-tracking/route.ts @@ -0,0 +1,37 @@ +// GET /api/v1/email-tracking — every project the token's user can reach, with +// its email tracking id, whether it is on, and the last day's human opens, +// clicks and unsubscribes. No secrets here; ask for one project with +// ?secret=1 for that. +// +// Same crp_ bearer as /api/ads/v1/*. This is what `crawlproof email-tracking +// list`, the TUI's Email tab and `myna newsletter track connect` read. + +import { NextResponse, type NextRequest } from "next/server"; +import { serviceClient } from "@/lib/supabase/service"; +import { authenticateBearer } from "@/lib/sp/apiAuth"; +import { getOrCreateForProject } from "@/lib/emailTracking/store"; +import { accessibleProjects, eventCounts, shapeTracking } from "@/lib/emailTracking/access"; +import { env } from "@/lib/env"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +const siteBase = () => (env.siteUrl || "https://crawlproof.com").replace(/\/$/, ""); + +export async function GET(req: NextRequest) { + const auth = await authenticateBearer(req); + if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status }); + const sb = serviceClient(); + try { + const projects = await accessibleProjects(sb, auth.userId); + const counts = await eventCounts(sb, projects.map((p) => p.id), new Date(Date.now() - 86_400_000).toISOString()); + const rows = []; + for (const project of projects) { + const row = await getOrCreateForProject(project.id); + rows.push(shapeTracking(project, row, { siteBase: siteBase(), counts: counts[project.id] })); + } + return NextResponse.json({ projects: rows }, { headers: { "cache-control": "no-store" } }); + } catch (e) { + return NextResponse.json({ error: e instanceof Error ? e.message : "Could not read email tracking." }, { status: 503 }); + } +} diff --git a/cli/dashboard.ts b/cli/dashboard.ts index 3eb392a..eace7f9 100644 --- a/cli/dashboard.ts +++ b/cli/dashboard.ts @@ -15,7 +15,22 @@ import { AD_TARGET_CTR, AD_TARGET_IMPRESSIONS, adTargets } from "../lib/dashboar import { buildSiteDetail, type SiteDetail } from "../lib/dashboard/site"; import type { Component } from "../lib/dashboard/score"; -export const TABS = ["ROI", "Traffic", "Ads", "Money", "Spend"] as const; +export const TABS = ["ROI", "Traffic", "Ads", "Money", "Spend", "Email"] as const; + +/** The Email tab's index: its data is not in the snapshot, so it draws without one. */ +export const EMAIL_TAB = 5; + +/** One project's email tracking, as GET /api/v1/email-tracking answers it. */ +export type EmailTrackingRow = { + project_id: string; + site: string; + role: string; + tracking_id: string; + enabled: boolean; + events_24h?: { open: number; click: number; unsubscribe: number }; +}; + +export type EmailState = { rows: EmailTrackingRow[]; loading: boolean; error: string | null; note: string | null }; export const RANGES = ["1h", "4h", "1d", "1w", "1m"] as const; /** How the Traffic list is ordered. `s` cycles it. */ @@ -108,6 +123,8 @@ export type State = { */ domain: string | null; sort: Sort; + /** The Email tab: every project's email tracking, read from its own endpoint. */ + email: EmailState; }; function pane(state: State, name: string, total: number): Pane { @@ -129,7 +146,7 @@ function scrollPane(p: Pane, delta: number, rows = 1): void { p.selected = Math.max(p.offset, Math.min(p.selected, max)); } -const TAB_PANE = ["vendors", "sites", "campaigns", "invoices", "ledger"]; +const TAB_PANE = ["vendors", "sites", "campaigns", "invoices", "ledger", "email"]; const signed = (theme: Theme, v: number) => (v >= 0 ? theme.success : theme.danger); @@ -997,7 +1014,57 @@ function spendScreen(ui: Container, state: State, theme: Theme): void { }); } -const SCREENS = [roiScreen, trafficScreen, adsScreen, moneyScreen, spendScreen]; +/** + * Email tracking, per project: on or off, the id a sender needs, and the last + * day's human opens, clicks and unsubscribes. `e` turns the highlighted one + * on or off; the secret stays out of the terminal (crawlproof email-tracking + * show --secret prints it on purpose). + */ +export function emailScreen(ui: Container, state: State, theme: Theme): void { + const email = state.email; + ui.panel( + { + title: "Email tracking", + subtitle: email.loading ? "reading…" : `${email.rows.filter((r) => r.enabled).length} of ${email.rows.length} on`, + footer: "e on/off · crawlproof email-tracking show --secret", + }, + (p) => { + if (email.error) p.text(`Could not load: ${email.error}`, { fg: theme.danger }); + if (email.note) p.text(email.note, { fg: theme.success }); + if (!email.rows.length) { + p.text(email.loading ? "Reading email tracking…" : "No projects.", { fg: theme.muted }); + return; + } + const view = pane(state, "email", email.rows.length); + p.table({ + columns: [ + { key: "on", title: "", width: 4 }, + { key: "site", title: "Site", width: 28 }, + { key: "id", title: "Tracking id", width: 26 }, + { key: "opens", title: "Opens", align: "right", width: 7 }, + { key: "clicks", title: "Clicks", align: "right", width: 7 }, + { key: "unsubs", title: "Unsubs", align: "right", width: 7 }, + { key: "role", title: "Role", width: 8 }, + ], + rows: email.rows.map((r) => ({ + on: r.enabled ? "on" : "off", + site: r.site, + id: r.tracking_id, + opens: count(r.events_24h?.open ?? 0), + clicks: count(r.events_24h?.click ?? 0), + unsubs: count(r.events_24h?.unsubscribe ?? 0), + role: r.role, + })), + offset: view.offset, + selected: view.selected, + scrollbar: true, + onScroll: (delta: number) => scrollPane(view, delta, 3), + }); + }, + ); +} + +const SCREENS = [roiScreen, trafficScreen, adsScreen, moneyScreen, spendScreen, emailScreen]; /** * Draw whichever screen the state is on. @@ -1076,6 +1143,10 @@ export function renderBody(ui: Container, state: State, theme: Theme): void { } function renderContent(ui: Container, state: State, theme: Theme): void { + if (state.tab === EMAIL_TAB) { + emailScreen(ui, state, theme); + return; + } if (!state.snapshot) { ui.panel({ title: "Spend & ROI" }, (p) => { if (state.error) { @@ -1120,6 +1191,7 @@ export function initialState(overrides: Partial = {}): State { targetCtr: AD_TARGET_CTR, domain: null, sort: "score", + email: { rows: [], loading: false, error: null, note: null }, ...overrides, }; } @@ -1140,7 +1212,11 @@ export type KeyLike = { name: string; shift?: boolean }; * without a terminal. Returns true when something changed and the frame is * worth redrawing. */ -export function handleKey(state: State, event: KeyLike, actions: { refresh: () => void }): boolean { +export function handleKey( + state: State, + event: KeyLike, + actions: { refresh: () => void; toggleEmail?: (row: EmailTrackingRow) => void }, +): boolean { if (state.showHelp) { state.showHelp = false; return true; @@ -1198,6 +1274,14 @@ export function handleKey(state: State, event: KeyLike, actions: { refresh: () = state.tab = (state.tab + TABS.length - 1) % TABS.length; return true; + case "e": { + if (state.tab !== EMAIL_TAB || !actions.toggleEmail) return false; + const row = state.email.rows[Math.min(view.selected, state.email.rows.length - 1)]; + if (!row) return false; + actions.toggleEmail(row); + return true; + } + case "s": { // Re-sorting keeps the highlight on the same property rather than on the // same row number, which is the only version of this that is not annoying. @@ -1295,6 +1379,51 @@ export type DashboardOptions = { }; /** Manual retries queue once during an active read, including range/filter changes. */ +/** + * The Email tab's own loader: GET /api/v1/email-tracking, and a toggle that + * POSTs enable or disable and reads the list again. Separate from the fleet + * refresh because it is one cheap call and does not wait on CoinPay. + */ +export function createEmailController( + state: State, + opts: Pick, + invalidate: () => void, + fetcher: typeof fetch = fetch, +): { load: () => Promise; toggle: (row: EmailTrackingRow) => Promise } { + const base = opts.baseUrl.replace(/\/$/, ""); + const headers = { authorization: `Bearer ${opts.token}`, accept: "application/json" }; + const load = async (): Promise => { + state.email.loading = true; + invalidate(); + try { + const res = await fetcher(`${base}/api/v1/email-tracking`, { headers }); + const json = (await res.json().catch(() => ({}))) as { projects?: EmailTrackingRow[]; error?: string }; + if (!res.ok) throw new Error(json.error ?? `HTTP ${res.status}`); + state.email.rows = json.projects ?? []; + state.email.error = null; + } catch (e) { + state.email.error = e instanceof Error ? e.message : String(e); + } finally { + state.email.loading = false; + invalidate(); + } + }; + const toggle = async (row: EmailTrackingRow): Promise => { + const action = row.enabled ? "disable" : "enable"; + try { + const res = await fetcher(`${base}/api/v1/email-tracking/${encodeURIComponent(row.project_id)}/${action}`, { method: "POST", headers }); + const json = (await res.json().catch(() => ({}))) as { error?: string }; + if (!res.ok) throw new Error(json.error ?? `HTTP ${res.status}`); + state.email.note = `${row.site}: email tracking ${action === "enable" ? "on" : "off"}.`; + state.email.error = null; + } catch (e) { + state.email.error = e instanceof Error ? e.message : String(e); + } + await load(); + }; + return { load, toggle }; +} + export function createRefreshController( state: State, opts: DashboardOptions, @@ -1379,7 +1508,9 @@ export async function runDashboard(opts: DashboardOptions): Promise { ...(opts.sort && SORTS.includes(opts.sort as never) ? { sort: opts.sort as Sort } : {}), }); - const refresh = createRefreshController(state, opts, () => app.invalidate()); + const refreshFleet = createRefreshController(state, opts, () => app.invalidate()); + const email = createEmailController(state, opts, () => app.invalidate()); + const refresh = (): Promise => Promise.all([refreshFleet(), email.load()]).then(() => undefined); const interval = Math.max(10, opts.interval ?? 60); const poll = setInterval(() => { @@ -1391,7 +1522,7 @@ export async function runDashboard(opts: DashboardOptions): Promise { tick.unref?.(); app.on("key", (event: KeyLike) => { - if (handleKey(state, event, { refresh })) app.invalidate(); + if (handleKey(state, event, { refresh, toggleEmail: (row) => void email.toggle(row) })) app.invalidate(); }); app.render(({ ui, theme, height }: RenderArgs) => { @@ -1424,7 +1555,8 @@ export async function runDashboard(opts: DashboardOptions): Promise { const onList = state.tab === 1 && !state.domain; ui.statusBar({ items: [ - { key: "1-5", label: "Screen" }, + { key: `1-${TABS.length}`, label: "Screen" }, + ...(state.tab === EMAIL_TAB ? [{ key: "e", label: "On/off" }] : []), ...(onList ? [{ key: "↵", label: "Open site" }] : []), ...(state.domain ? [{ key: "esc", label: "Back", active: true }] : []), ...(onList ? [{ key: "s", label: `Sort ${state.sort}` }] : []), @@ -1446,7 +1578,7 @@ export async function runDashboard(opts: DashboardOptions): Promise { width: 76, height: 28, message: - "1-5, ←/→ switch screens.\n" + + `1-${TABS.length}, ←/→ switch screens. On Email, e turns tracking on/off.\n` + "⧉ MD copies a summary; Tab focuses, Enter copies.\n" + `r refreshes now; it also refreshes every ${interval}s.\n` + "w cycles the window: 1h → 4h → 1d → 1w → 1m.\n" + diff --git a/cli/index.ts b/cli/index.ts index d0ebf9f..aeeeac9 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -14,6 +14,7 @@ // by share-token from the production API. import { readFileSync } from "node:fs"; +import { EMAIL_TRACKING_USAGE, runEmailTracking } from "../lib/emailTracking/cli"; import { isAllowedTargetUrl } from "../lib/rateLimit"; @@ -828,6 +829,7 @@ COMMANDS people are asking about right now, and earns 90 days during which every click is billed at $0.00 (the rate after that is $0.02/click). +${EMAIL_TRACKING_USAGE} ads trends [--window=7] [--limit=20] [--stale] [--json] What is trending, highest score first, with how many separate parties used each subject this window and last. --stale shows a @@ -964,6 +966,12 @@ async function main() { return await cmdTrack(args); case "ads": return await cmdAds(args); + case "email-tracking": + return await runEmailTracking(args.positional, args.flags as Record, (method, path) => apiCall(args, method, path), { + write: (line: string) => process.stdout.write(`${line}\n`), + error: (line: string) => console.error(line), + isTTY: Boolean(process.stdout.isTTY), + }); case "slots": return await cmdSlots(args); case "affiliate": diff --git a/lib/emailTracking/access.ts b/lib/emailTracking/access.ts new file mode 100644 index 0000000..5bd9771 --- /dev/null +++ b/lib/emailTracking/access.ts @@ -0,0 +1,159 @@ +// Email tracking for a bearer-token caller: which projects the token's user +// can reach, in what role, and what each project's tracking looks like from +// outside. The dashboard tab answers the same questions through a session +// (requireProjectAccess); this is the same rule for the API, the CLI and the +// TUI, which only have a crp_ token. +// +// Roles follow requireProjectAccess: the owner and org owners/members may +// change tracking, a project member with role "viewer" may only look, and a +// viewer never sees the secret. + +import type { SupabaseClient } from "@supabase/supabase-js"; +import type { EmailEventType } from "@/lib/emailTracking/core"; +import type { TrackingRow } from "@/lib/emailTracking/store"; + +export type ProjectRole = "owner" | "member" | "viewer"; + +export type AccessibleProject = { + id: string; + name: string; + url: string | null; + role: ProjectRole; +}; + +type ProjectRecord = { id: string; name: string | null; url: string | null }; + +const RANK: Record = { owner: 3, member: 2, viewer: 1 }; + +/** Every project the user can reach, the strongest role winning. */ +export async function accessibleProjects(sb: SupabaseClient, userId: string): Promise { + const found = new Map(); + const add = (rows: ProjectRecord[] | null, role: ProjectRole) => { + for (const row of rows ?? []) { + const current = found.get(row.id); + if (current && RANK[current.role] >= RANK[role]) continue; + found.set(row.id, { id: row.id, name: row.name ?? "", url: row.url ?? null, role }); + } + }; + + const owned = await sb.from("projects").select("id, name, url").eq("owner_id", userId); + add(owned.data as ProjectRecord[] | null, "owner"); + + const memberships = await sb.from("project_members").select("project_id, role").eq("user_id", userId); + const byRole = new Map(); + for (const m of (memberships.data as { project_id: string; role: string | null }[] | null) ?? []) { + const role: ProjectRole = m.role === "viewer" ? "viewer" : "member"; + byRole.set(role, [...(byRole.get(role) ?? []), m.project_id]); + } + for (const [role, ids] of byRole) { + if (!ids.length) continue; + const rows = await sb.from("projects").select("id, name, url").in("id", ids); + add(rows.data as ProjectRecord[] | null, role); + } + + const orgs = await sb.from("organization_members").select("organization_id").eq("user_id", userId).in("role", ["owner", "member"]); + const orgIds = ((orgs.data as { organization_id: string }[] | null) ?? []).map((o) => o.organization_id); + if (orgIds.length) { + const rows = await sb.from("projects").select("id, name, url").in("organization_id", orgIds); + add(rows.data as ProjectRecord[] | null, "member"); + } + + return [...found.values()].sort((a, b) => siteOf(a).localeCompare(siteOf(b))); +} + +/** The project's hostname without www, or its name when it has no URL. */ +export function siteOf(project: Pick): string { + if (project.url) { + try { + return new URL(project.url).hostname.toLowerCase().replace(/^www\./, ""); + } catch { + // fall through to the name + } + } + return project.name.trim().toLowerCase(); +} + +/** A hostname out of whatever was typed: "moshcode.sh", "https://www.moshcode.sh/x". */ +export function normalizeSiteRef(ref: string): string { + const raw = ref.trim().toLowerCase(); + if (!raw) return ""; + try { + return new URL(/^[a-z]+:\/\//.test(raw) ? raw : `https://${raw}`).hostname.replace(/^www\./, ""); + } catch { + return raw; + } +} + +/** A project by id, hostname or name. Null when none matches or the match is ambiguous. */ +export function pickProject(projects: AccessibleProject[], ref: string): AccessibleProject | null { + const byId = projects.find((p) => p.id === ref.trim()); + if (byId) return byId; + const host = normalizeSiteRef(ref); + const matches = projects.filter((p) => siteOf(p) === host || p.name.trim().toLowerCase() === ref.trim().toLowerCase()); + return matches.length === 1 ? (matches[0] as AccessibleProject) : null; +} + +export type EventCounts = Record; + +export const emptyCounts = (): EventCounts => ({ open: 0, click: 0, unsubscribe: 0 }); + +/** Events per project since `since`, counted by type. Machine opens and clicks are left out. */ +export async function eventCounts(sb: SupabaseClient, projectIds: string[], since: string): Promise> { + const out: Record = {}; + for (const id of projectIds) out[id] = emptyCounts(); + if (!projectIds.length) return out; + const { data } = await sb + .from("email_tracking_events") + .select("project_id, type, machine") + .in("project_id", projectIds) + .gte("at", since) + .limit(50_000); + for (const row of (data as { project_id: string; type: EmailEventType; machine: boolean }[] | null) ?? []) { + if (row.machine && row.type !== "unsubscribe") continue; + const counts = out[row.project_id]; + if (counts && row.type in counts) counts[row.type]++; + } + return out; +} + +export type PublicTracking = { + project_id: string; + site: string; + name: string; + role: ProjectRole; + tracking_id: string; + enabled: boolean; + enabled_at: string | null; + secret_rotated_at: string | null; + tracking_url: string; + events_url: string; + events_24h?: EventCounts; + /** Only when asked for, and never to a viewer. */ + secret?: string; +}; + +export function shapeTracking( + project: AccessibleProject, + row: TrackingRow, + options: { siteBase: string; counts?: EventCounts; withSecret?: boolean }, +): PublicTracking { + const base = options.siteBase.replace(/\/$/, ""); + return { + project_id: project.id, + site: siteOf(project), + name: project.name, + role: project.role, + tracking_id: row.tracking_id, + enabled: row.enabled, + enabled_at: row.enabled_at, + secret_rotated_at: row.secret_rotated_at, + tracking_url: `${base}/t/${row.tracking_id}`, + events_url: `${base}/api/v1/tracking/${row.tracking_id}/events`, + ...(options.counts ? { events_24h: options.counts } : {}), + ...(options.withSecret && project.role !== "viewer" ? { secret: row.secret } : {}), + }; +} + +export const ACTIONS = ["enable", "disable", "rotate"] as const; +export type TrackingAction = (typeof ACTIONS)[number]; +export const isAction = (v: string): v is TrackingAction => (ACTIONS as readonly string[]).includes(v); diff --git a/lib/emailTracking/cli.ts b/lib/emailTracking/cli.ts new file mode 100644 index 0000000..3cc839b --- /dev/null +++ b/lib/emailTracking/cli.ts @@ -0,0 +1,109 @@ +// `crawlproof email-tracking` — one implementation for both CLIs (the in-repo +// one and the published @profullstack/crawlproof). Each hands in its own +// token-authed HTTP call, so this file owns only the words. +// +// crawlproof email-tracking [list] [--json] +// crawlproof email-tracking show [--secret] [--json] +// crawlproof email-tracking enable|disable [--json] +// crawlproof email-tracking rotate [--json] +// +// `show --secret` piped (not a terminal) prints the secret alone, so +// crawlproof email-tracking show moshcode.sh --secret | myna newsletter track set +// works with nothing in between. + +export type ApiCall = ( + method: "GET" | "POST", + path: string, +) => Promise<{ status: number; json: Record }>; + +export type Out = { write(line: string): void; error(line: string): void; isTTY: boolean }; + +export const EMAIL_TRACKING_USAGE = ` email-tracking [list] [--json] + Every project's email tracking: id, on or off, and the last day's + opens, clicks and unsubscribes. Needs CRAWLPROOF_TOKEN. + email-tracking show [--secret] [--json] + One project. --secret adds the signing secret (owners and members + only); piped, it prints just the secret. + email-tracking enable|disable + Turn tracking on or off. Unsubscribe links keep working either way. + email-tracking rotate + A new secret. The old one keeps verifying mail already sent. +`; + +type Row = { + project_id: string; + site: string; + role: string; + tracking_id: string; + enabled: boolean; + enabled_at: string | null; + tracking_url: string; + events_url: string; + events_24h?: { open: number; click: number; unsubscribe: number }; + secret?: string; +}; + +const counts = (r: Row): string => (r.events_24h ? `${r.events_24h.open} opens, ${r.events_24h.click} clicks, ${r.events_24h.unsubscribe} unsubs (24h)` : ""); + +function describe(r: Row): string[] { + return [ + `${r.site} ${r.enabled ? "on" : "off"} (${r.role})`, + ` tracking id ${r.tracking_id}`, + ` tracking ${r.tracking_url}`, + ` events ${r.events_url}`, + ...(r.events_24h ? [` last day ${counts(r)}`] : []), + ...(r.secret ? [` secret ${r.secret}`] : []), + ]; +} + +export async function runEmailTracking(positional: string[], flags: Record, call: ApiCall, out: Out): Promise { + const [sub = "list", ref] = positional; + const fail = (what: string, status: number, json: Record): number => { + out.error(`email-tracking ${what} failed: ${status} ${String(json.error ?? "")}`.trim()); + return 1; + }; + const path = (r: string) => `/api/v1/email-tracking/${encodeURIComponent(r)}`; + + if (sub === "list") { + const { status, json } = await call("GET", "/api/v1/email-tracking"); + if (status >= 400) return fail("list", status, json); + const rows = (json.projects as Row[]) ?? []; + if (flags.json) { + out.write(JSON.stringify(rows, null, 2)); + return 0; + } + if (!rows.length) out.write("No projects."); + for (const r of rows) out.write(`${(r.enabled ? "on " : "off").padEnd(4)} ${r.site.padEnd(28)} ${r.tracking_id} ${counts(r)}`); + return 0; + } + + if (!ref) { + out.error(`usage: crawlproof email-tracking ${sub} `); + return 2; + } + + if (sub === "show") { + const { status, json } = await call("GET", `${path(ref)}${flags.secret ? "?secret=1" : ""}`); + if (status >= 400) return fail("show", status, json); + const row = json as unknown as Row; + if (flags.json) out.write(JSON.stringify(row, null, 2)); + else if (flags.secret && !out.isTTY) out.write(row.secret ?? ""); + else for (const line of describe(row)) out.write(line); + return 0; + } + + if (sub === "enable" || sub === "disable" || sub === "rotate") { + const { status, json } = await call("POST", `${path(ref)}/${sub}`); + if (status >= 400) return fail(sub, status, json); + const row = json as unknown as Row; + if (flags.json) out.write(JSON.stringify(row, null, 2)); + else if (sub === "rotate") { + out.write(`New secret for ${row.site}. Mail already sent keeps verifying with the old one until the next rotation.`); + out.write(out.isTTY ? ` secret ${row.secret}` : (row.secret ?? "")); + } else out.write(`${row.site}: email tracking ${row.enabled ? "on" : "off"} (${row.tracking_id})`); + return 0; + } + + out.error(`unknown: crawlproof email-tracking ${sub} (expected: list | show | enable | disable | rotate)`); + return 2; +} diff --git a/packages/cli/package.json b/packages/cli/package.json index f27afe8..8f2b6b1 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/crawlproof", - "version": "0.2.1", + "version": "0.3.0", "description": "What the fleet costs and what it returns: a live terminal dashboard over CrawlProof traffic, ad delivery and CoinPay banking.", "license": "MIT", "type": "module", diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index e2c9090..cbb0b31 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -14,8 +14,9 @@ import { readFileSync } from "node:fs"; import { collectDashboard } from "../../../lib/dashboard/collect"; import { renderStats } from "../../../lib/dashboard/stats-text"; import { FINANCE_DAYS, runDashboard } from "../../../cli/dashboard"; +import { EMAIL_TRACKING_USAGE, runEmailTracking } from "../../../lib/emailTracking/cli"; -export const VERSION = "0.2.1"; +export const VERSION = "0.3.0"; type Args = { command: string; @@ -145,6 +146,7 @@ COMMANDS ads delete --yes Look at and change what is running. A ref looks like crawlproof-ad-144. +${EMAIL_TRACKING_USAGE} help | version AUTH @@ -411,6 +413,12 @@ export async function main(argv: string[]): Promise { return await cmdAd(args); case "ads": return await cmdAds(args); + case "email-tracking": + return await runEmailTracking(args.positional, args.flags, (method, path) => apiCall(args, method, path), { + write: (line: string) => process.stdout.write(`${line}\n`), + error: (line: string) => console.error(line), + isTTY: Boolean(process.stdout.isTTY), + }); case "version": case "--version": case "-v": diff --git a/tests/email-tracking-api.test.ts b/tests/email-tracking-api.test.ts new file mode 100644 index 0000000..022c0a9 --- /dev/null +++ b/tests/email-tracking-api.test.ts @@ -0,0 +1,144 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +// A tiny stand-in for the Supabase query builder: from(table) filtered by the +// eq/in/gte calls the access module makes, over fixture rows. +type Row = Record; +const tables: Record = {}; +function query(table: string) { + let rows = [...(tables[table] ?? [])]; + const builder = { + select: () => builder, + eq: (col: string, v: unknown) => ((rows = rows.filter((r) => r[col] === v)), builder), + in: (col: string, vs: unknown[]) => ((rows = rows.filter((r) => vs.includes(r[col]))), builder), + gte: (col: string, v: string) => ((rows = rows.filter((r) => String(r[col]) >= v)), builder), + limit: () => builder, + then: (resolve: (v: { data: Row[]; error: null }) => void) => resolve({ data: rows, error: null }), + }; + return builder; +} + +const mocks = vi.hoisted(() => ({ auth: vi.fn(), setEnabled: vi.fn(), rotate: vi.fn(), getOrCreate: vi.fn() })); +vi.mock("@/lib/sp/apiAuth", () => ({ authenticateBearer: mocks.auth })); +vi.mock("@/lib/supabase/service", () => ({ serviceClient: () => ({ from: query }) })); +vi.mock("@/lib/emailTracking/store", () => ({ + getOrCreateForProject: mocks.getOrCreate, + setEnabled: mocks.setEnabled, + rotateSecret: mocks.rotate, +})); +vi.mock("@/lib/env", () => ({ env: { siteUrl: "https://crawlproof.com" } })); + +import { GET as list } from "@/app/api/v1/email-tracking/route"; +import { GET as one } from "@/app/api/v1/email-tracking/[project]/route"; +import { POST as act } from "@/app/api/v1/email-tracking/[project]/[action]/route"; +import { normalizeSiteRef, pickProject, siteOf } from "@/lib/emailTracking/access"; + +const row = (project_id: string, enabled = false) => ({ + project_id, + tracking_id: `trk${project_id}`, + secret: `secret-${project_id}`, + previous_secret: null, + secret_rotated_at: null, + enabled, + enabled_at: enabled ? "2026-09-24T00:00:00Z" : null, +}); + +beforeEach(() => { + mocks.auth.mockReset().mockResolvedValue({ ok: true, userId: "me" }); + mocks.getOrCreate.mockReset().mockImplementation(async (id: string) => row(id, id === "p-mosh")); + mocks.setEnabled.mockReset().mockImplementation(async (id: string, enabled: boolean) => row(id, enabled)); + mocks.rotate.mockReset().mockImplementation(async (id: string) => ({ ...row(id), secret: "fresh" })); + const recent = new Date(Date.now() - 60_000).toISOString(); + Object.assign(tables, { + projects: [ + { id: "p-mosh", name: "moshcode.sh", url: "https://moshcode.sh/", owner_id: "me", organization_id: null }, + { id: "p-org", name: "profullstack.com", url: "https://www.profullstack.com/", owner_id: "someone", organization_id: "org1" }, + { id: "p-view", name: "viewed.dev", url: "https://viewed.dev", owner_id: "someone", organization_id: null }, + { id: "p-other", name: "not-mine.com", url: "https://not-mine.com", owner_id: "stranger", organization_id: null }, + ], + project_members: [{ project_id: "p-view", user_id: "me", role: "viewer" }], + organization_members: [{ organization_id: "org1", user_id: "me", role: "member" }], + email_tracking_events: [ + { project_id: "p-mosh", type: "open", machine: false, at: recent }, + { project_id: "p-mosh", type: "open", machine: true, at: recent }, + { project_id: "p-mosh", type: "click", machine: false, at: recent }, + { project_id: "p-mosh", type: "unsubscribe", machine: false, at: recent }, + { project_id: "p-mosh", type: "open", machine: false, at: "2020-01-01T00:00:00Z" }, + ], + }); +}); + +const req = (path: string, init?: RequestInit) => new NextRequest(`https://crawlproof.com${path}`, init as never); +const ctx = >(params: T) => ({ params: Promise.resolve(params) }); + +describe("email tracking over the API", () => { + it("lists every reachable project with its role and a day of human events, and no secrets", async () => { + const r = await list(req("/api/v1/email-tracking")); + expect(r.status).toBe(200); + const { projects } = await r.json(); + expect(projects.map((p: { site: string; role: string }) => [p.site, p.role])).toEqual([ + ["moshcode.sh", "owner"], + ["profullstack.com", "member"], + ["viewed.dev", "viewer"], + ]); + const mosh = projects[0]; + expect(mosh).toMatchObject({ + tracking_id: "trkp-mosh", + enabled: true, + tracking_url: "https://crawlproof.com/t/trkp-mosh", + events_url: "https://crawlproof.com/api/v1/tracking/trkp-mosh/events", + events_24h: { open: 1, click: 1, unsubscribe: 1 }, + }); + expect(JSON.stringify(projects)).not.toContain("secret-"); + }); + + it("finds one project by hostname or URL, and hands out the secret only when asked, never to a viewer", async () => { + const plain = await (await one(req("/api/v1/email-tracking/moshcode.sh"), ctx({ project: "moshcode.sh" }))).json(); + expect(plain.secret).toBeUndefined(); + const withSecret = await one(req("/api/v1/email-tracking/x?secret=1"), ctx({ project: encodeURIComponent("https://www.profullstack.com/blog") })); + expect((await withSecret.json()).secret).toBe("secret-p-org"); + expect((await one(req("/api/v1/email-tracking/viewed.dev?secret=1"), ctx({ project: "viewed.dev" }))).status).toBe(403); + expect((await one(req("/api/v1/email-tracking/not-mine.com"), ctx({ project: "not-mine.com" }))).status).toBe(404); + }); + + it("enables, disables and rotates for owners and members, refuses viewers and unknown actions", async () => { + const on = await act(req("/api/v1/email-tracking/p-org/enable", { method: "POST" }), ctx({ project: "p-org", action: "enable" })); + expect(on.status).toBe(200); + expect(mocks.setEnabled).toHaveBeenCalledWith("p-org", true); + expect((await on.json()).secret).toBeUndefined(); + + await act(req("/x", { method: "POST" }), ctx({ project: "moshcode.sh", action: "disable" })); + expect(mocks.setEnabled).toHaveBeenLastCalledWith("p-mosh", false); + + const rotated = await act(req("/x", { method: "POST" }), ctx({ project: "moshcode.sh", action: "rotate" })); + expect((await rotated.json()).secret).toBe("fresh"); + + expect((await act(req("/x", { method: "POST" }), ctx({ project: "viewed.dev", action: "enable" }))).status).toBe(403); + expect((await act(req("/x", { method: "POST" }), ctx({ project: "p-mosh", action: "delete" }))).status).toBe(404); + expect(mocks.setEnabled).toHaveBeenCalledTimes(2); + }); + + it("refuses without a token before touching anything", async () => { + mocks.auth.mockResolvedValue({ ok: false, status: 401, error: "Missing bearer token." }); + expect((await list(req("/api/v1/email-tracking"))).status).toBe(401); + expect((await act(req("/x", { method: "POST" }), ctx({ project: "p-mosh", action: "enable" }))).status).toBe(401); + expect(mocks.getOrCreate).not.toHaveBeenCalled(); + expect(mocks.setEnabled).not.toHaveBeenCalled(); + }); +}); + +describe("site references", () => { + it("normalises what people type", () => { + expect(normalizeSiteRef("https://www.MoshCode.sh/x?y")).toBe("moshcode.sh"); + expect(normalizeSiteRef("moshcode.sh")).toBe("moshcode.sh"); + expect(siteOf({ name: "No URL", url: null })).toBe("no url"); + }); + it("refuses an ambiguous name", () => { + const projects = [ + { id: "a", name: "same", url: "https://same.dev", role: "owner" as const }, + { id: "b", name: "same", url: "https://same.dev", role: "owner" as const }, + ]; + expect(pickProject(projects, "same.dev")).toBeNull(); + expect(pickProject(projects, "a")?.id).toBe("a"); + }); +}); diff --git a/tests/email-tracking-surfaces.test.ts b/tests/email-tracking-surfaces.test.ts new file mode 100644 index 0000000..d72da74 --- /dev/null +++ b/tests/email-tracking-surfaces.test.ts @@ -0,0 +1,101 @@ +/** + * Email tracking outside the dashboard tab: the CLI command both CLIs share, + * and the TUI's Email tab with its loader and its on/off key. The API behind + * them is covered in email-tracking-api.test.ts. + */ +import { describe, expect, it, vi } from "vitest"; +import { renderToScreen } from "@profullstack/hqtui/testing"; + +import { runEmailTracking, type ApiCall } from "@/lib/emailTracking/cli"; +import { createEmailController, EMAIL_TAB, handleKey, initialState, renderBody, TABS } from "@/cli/dashboard"; + +const ROWS = [ + { project_id: "p-mosh", site: "moshcode.sh", role: "owner", tracking_id: "e960e0a69972a7f34ea197bb", enabled: true, enabled_at: null, tracking_url: "https://crawlproof.com/t/e960e0a69972a7f34ea197bb", events_url: "https://crawlproof.com/api/v1/tracking/e960e0a69972a7f34ea197bb/events", events_24h: { open: 12, click: 3, unsubscribe: 1 } }, + { project_id: "p-pfs", site: "profullstack.com", role: "member", tracking_id: "cf9378b423ec82ad8f896fb7", enabled: false, enabled_at: null, tracking_url: "https://crawlproof.com/t/cf9378b423ec82ad8f896fb7", events_url: "https://crawlproof.com/api/v1/tracking/cf9378b423ec82ad8f896fb7/events", events_24h: { open: 0, click: 0, unsubscribe: 0 } }, +]; + +function capture(isTTY = true) { + const lines: string[] = []; + const errors: string[] = []; + return { lines, errors, out: { write: (l: string) => lines.push(l), error: (l: string) => errors.push(l), isTTY } }; +} + +describe("crawlproof email-tracking", () => { + const call: ApiCall = vi.fn(async (method, path) => { + if (method === "GET" && path === "/api/v1/email-tracking") return { status: 200, json: { projects: ROWS } }; + if (method === "GET" && path.startsWith("/api/v1/email-tracking/moshcode.sh")) { + return { status: 200, json: { ...ROWS[0], ...(path.endsWith("?secret=1") ? { secret: "c17c" } : {}) } }; + } + if (method === "POST" && path === "/api/v1/email-tracking/profullstack.com/enable") return { status: 200, json: { ...ROWS[1], enabled: true } }; + if (method === "POST" && path === "/api/v1/email-tracking/moshcode.sh/rotate") return { status: 200, json: { ...ROWS[0], secret: "new-secret" } }; + return { status: 404, json: { error: "No such project." } }; + }); + + it("lists every project with its state and a day of events", async () => { + const c = capture(); + expect(await runEmailTracking([], {}, call, c.out)).toBe(0); + expect(c.lines[0]).toMatch(/^on\s+moshcode\.sh\s+e960e0a69972a7f34ea197bb\s+12 opens, 3 clicks, 1 unsubs \(24h\)$/); + expect(c.lines[1]).toMatch(/^off\s+profullstack\.com/); + }); + + it("shows one, with the secret only when asked, and alone when piped", async () => { + const tty = capture(true); + await runEmailTracking(["show", "moshcode.sh"], {}, call, tty.out); + expect(tty.lines.join("\n")).toContain("tracking id e960e0a69972a7f34ea197bb"); + expect(tty.lines.join("\n")).not.toContain("secret"); + const piped = capture(false); + await runEmailTracking(["show", "moshcode.sh"], { secret: true }, call, piped.out); + expect(piped.lines).toEqual(["c17c"]); + }); + + it("enables, rotates, and says what failed", async () => { + const on = capture(); + expect(await runEmailTracking(["enable", "profullstack.com"], {}, call, on.out)).toBe(0); + expect(on.lines).toEqual(["profullstack.com: email tracking on (cf9378b423ec82ad8f896fb7)"]); + const rotated = capture(false); + await runEmailTracking(["rotate", "moshcode.sh"], {}, call, rotated.out); + expect(rotated.lines[1]).toBe("new-secret"); + const missing = capture(); + expect(await runEmailTracking(["disable", "nope.dev"], {}, call, missing.out)).toBe(1); + expect(missing.errors[0]).toBe("email-tracking disable failed: 404 No such project."); + expect(await runEmailTracking(["enable"], {}, call, capture().out)).toBe(2); + expect(await runEmailTracking(["frobnicate", "x"], {}, call, capture().out)).toBe(2); + }); +}); + +describe("the TUI's Email tab", () => { + it("is a sixth tab that draws without the fleet snapshot", () => { + expect(TABS[EMAIL_TAB]).toBe("Email"); + const state = initialState({ tab: EMAIL_TAB }); + state.email.rows = ROWS; + const text = renderToScreen(({ ui, theme }) => renderBody(ui, state, theme), { width: 140, height: 20 }).text(); + for (const want of ["Email tracking", "1 of 2 on", "moshcode.sh", "e960e0a69972a7f34ea197bb", "profullstack.com", "owner"]) expect(text).toContain(want); + expect(text).not.toContain("Reading the fleet"); + }); + + it("loads from the API, and e flips the highlighted project then reloads", async () => { + const state = initialState({ tab: EMAIL_TAB }); + let enabled = false; + const fetcher = vi.fn(async (url: string, init?: RequestInit) => { + expect(new Headers(init?.headers as Record).get("authorization")).toBe("Bearer crp_test"); + if (url.endsWith("/api/v1/email-tracking")) return Response.json({ projects: [{ ...ROWS[1], enabled }] }); + if (url.endsWith("/api/v1/email-tracking/p-pfs/enable") && init?.method === "POST") { + enabled = true; + return Response.json({ ...ROWS[1], enabled: true }); + } + return Response.json({ error: "no" }, { status: 404 }); + }); + const email = createEmailController(state, { baseUrl: "https://crawlproof.test/", token: "crp_test" }, () => {}, fetcher as unknown as typeof fetch); + await email.load(); + expect(state.email.rows.map((r) => r.enabled)).toEqual([false]); + + let toggled: Promise | undefined; + expect(handleKey(state, { name: "e" }, { refresh: () => {}, toggleEmail: (row) => (toggled = email.toggle(row)) })).toBe(true); + await toggled; + expect(state.email.rows.map((r) => r.enabled)).toEqual([true]); + expect(state.email.note).toBe("profullstack.com: email tracking on."); + + state.tab = 0; + expect(handleKey(state, { name: "e" }, { refresh: () => {}, toggleEmail: () => {} })).toBe(false); + }); +});