From 733d4e06583f664190cb0cdba7a8f002bc578805 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 4 Oct 2026 13:35:48 +0000 Subject: [PATCH] feat(tracker): declared actors, opt-in human/agent identity Nothing on the wire separates a person from an agent driving a real browser, so visitors may now say who they are. An account registers actors (email, name, kind human|agent, optional human operator for an agent) and mints cpa_ tokens per browser or agent. The beacon carries the token by the Crawlproof-Actor header (headless agents) or the body, from localStorage set by a ?crp_actor= link or crawlproof('actor', token). A bare email is never accepted. Built for people gaming it: - one-way trust: a declared agent is believed (bot:declared, and the visitor rollup counts it as a bot); a declared human never overrides bot detection and a mismatch counts as a contradiction on the actor - tokens hashed with the API pepper under an "actor:" domain - a verified address can be claimed by one account only; the owner's login is verified on creation, anything else by an emailed link - names are private to the owner unless made public; other site owners see per-kind counts The tracker stays cookieless (credentials: 'omit' is pinned by a contract test), so there is no cookie channel; the dashboard's "Declare this browser" hands out one ?crp_actor= link per tracked project plus a bookmarklet for other sites. Surfaces: /api/tracker/v1/actors (+ tokens, verify), declared totals in /api/tracker/v1/stats, `crawlproof actors`, MCP list_actors / add_actor / mint_actor_token, Settings -> Declared actors, docs section. Migration 20261004120000_tracker_declared_actors.sql: apply by hand after merge. Until then the ingest treats every token as undeclared. Co-Authored-By: Claude Opus 5.5 --- app/(app)/dashboard/settings/actors/page.tsx | 53 +++ app/(app)/dashboard/settings/actors/panel.tsx | 295 +++++++++++++ app/(app)/dashboard/settings/page.tsx | 15 + app/(marketing)/docs/statistics/page.tsx | 34 +- app/api/mcp/route.ts | 2 + app/api/track/route.ts | 39 +- app/api/tracker/v1/actors/[id]/route.ts | 33 ++ .../tracker/v1/actors/[id]/tokens/route.ts | 39 ++ .../tracker/v1/actors/[id]/verify/route.ts | 28 ++ app/api/tracker/v1/actors/route.ts | 40 ++ app/api/tracker/v1/stats/route.ts | 11 +- app/stats.js/route.ts | 22 + cli/index.ts | 137 ++++++ lib/dashboard/stats-text.ts | 27 ++ lib/email.ts | 63 +++ lib/mcp/actors.ts | 94 ++++ lib/tracker/actorAuth.ts | 23 + lib/tracker/actorStore.ts | 407 ++++++++++++++++++ lib/tracker/actors.ts | 207 +++++++++ ...20261004120000_tracker_declared_actors.sql | 246 +++++++++++ tests/tracker-declared-actors.test.ts | 288 +++++++++++++ 21 files changed, 2094 insertions(+), 9 deletions(-) create mode 100644 app/(app)/dashboard/settings/actors/page.tsx create mode 100644 app/(app)/dashboard/settings/actors/panel.tsx create mode 100644 app/api/tracker/v1/actors/[id]/route.ts create mode 100644 app/api/tracker/v1/actors/[id]/tokens/route.ts create mode 100644 app/api/tracker/v1/actors/[id]/verify/route.ts create mode 100644 app/api/tracker/v1/actors/route.ts create mode 100644 lib/mcp/actors.ts create mode 100644 lib/tracker/actorAuth.ts create mode 100644 lib/tracker/actorStore.ts create mode 100644 lib/tracker/actors.ts create mode 100644 supabase/migrations/20261004120000_tracker_declared_actors.sql create mode 100644 tests/tracker-declared-actors.test.ts diff --git a/app/(app)/dashboard/settings/actors/page.tsx b/app/(app)/dashboard/settings/actors/page.tsx new file mode 100644 index 00000000..4ccda1d8 --- /dev/null +++ b/app/(app)/dashboard/settings/actors/page.tsx @@ -0,0 +1,53 @@ +import Link from "next/link"; +import { createClient } from "@/lib/supabase/server"; +import { serviceClient } from "@/lib/supabase/service"; +import { listActors } from "@/lib/tracker/actorStore"; +import { DECLARED_DEFINITION } from "@/lib/tracker/actors"; +import { ActorsPanel } from "./panel"; + +export const metadata = { title: "Declared actors" }; +export const dynamic = "force-dynamic"; + +export default async function ActorsPage() { + const supabase = await createClient(); + const { + data: { user }, + } = await supabase.auth.getUser(); + const sb = serviceClient(); + const [res, { data: projects }] = await Promise.all([ + listActors(sb, user!.id), + // The sites a browser declaration links to: yours, with the tracker on. + sb + .from("projects") + .select("id, name, url") + .eq("owner_id", user!.id) + .eq("tracker_enabled", true) + .order("name", { ascending: true }), + ]); + + return ( +
+
+ + ← Settings + +

Declared actors

+

+ Say who you are, and whether you are a person or an agent, on every site running the + CrawlProof tracker. {DECLARED_DEFINITION} Names are visible to you only unless you make an + actor public; other site owners see counts. +

+
+ {res.ok ? ( + p.url)} + /> + ) : ( +
+ Could not load actors: {res.error} +
+ )} +
+ ); +} diff --git a/app/(app)/dashboard/settings/actors/panel.tsx b/app/(app)/dashboard/settings/actors/panel.tsx new file mode 100644 index 00000000..65fd8f7f --- /dev/null +++ b/app/(app)/dashboard/settings/actors/panel.tsx @@ -0,0 +1,295 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState, useTransition } from "react"; +import type { Actor } from "@/lib/tracker/actorStore"; + +type Site = { id: string; name: string; url: string }; + +/** The site's address with ?crp_actor=, which stats.js stores and strips. */ +function declareLink(siteUrl: string, token: string) { + try { + const u = new URL(siteUrl); + u.searchParams.set("crp_actor", token); + return u.toString(); + } catch { + return null; + } +} + +/** For any other tracked site: one click stores the token for that site. */ +function bookmarklet(token: string) { + return `javascript:(function(){if(typeof window.crawlproof==='function'){window.crawlproof('actor','${token}');alert('CrawlProof: this browser is declared on '+location.hostname)}else{alert('No CrawlProof tracker on this page')}})()`; +} + +/** + * React 19 refuses a javascript: URL in href, so the bookmarklet's address is + * set on the element directly once it is mounted. + */ +function BookmarkletLink({ token }: { token: string }) { + const ref = useRef(null); + useEffect(() => { + ref.current?.setAttribute("href", bookmarklet(token)); + }, [token]); + return ( + e.preventDefault()}> + Declare me here + + ); +} + +async function call(method: string, path: string, body?: unknown) { + const res = await fetch(path, { + method, + headers: body ? { "content-type": "application/json" } : undefined, + body: body ? JSON.stringify(body) : undefined, + credentials: "same-origin", + }); + const json = (await res.json().catch(() => ({}))) as Record; + if (!res.ok) throw new Error(String(json.error ?? res.status)); + return json; +} + +function ago(iso: string | null) { + return iso ? iso.slice(0, 16).replace("T", " ") + " UTC" : "never"; +} + +export function ActorsPanel({ initial, sites }: { initial: Actor[]; sites: Site[] }) { + const [actors, setActors] = useState(initial); + const [browser, setBrowser] = useState<{ token: string; who: string } | null>(null); + const [error, setError] = useState(null); + const [notice, setNotice] = useState(null); + const [freshToken, setFreshToken] = useState(null); + const [pending, start] = useTransition(); + + const [email, setEmail] = useState(""); + const [name, setName] = useState(""); + const [kind, setKind] = useState<"human" | "agent">("human"); + const [operator, setOperator] = useState(""); + + const refresh = useCallback(async () => { + const list = await call("GET", "/api/tracker/v1/actors"); + setActors((list.actors as Actor[]) ?? []); + }, []); + + const run = (fn: () => Promise) => { + setError(null); + setNotice(null); + start(async () => { + try { + const msg = await fn(); + await refresh(); + if (msg) setNotice(msg); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } + }); + }; + + const humans = actors.filter((a) => a.kind === "human"); + + return ( +
+ {browser && ( +
+

Declare this browser as {browser.who}

+

+ Open each site once in this browser. The tracker stores the token for that site and + removes it from the address bar; every later visit is declared. The tracker stays + cookieless, so this is per site and per browser. +

+ {sites.length > 0 ? ( +
    + {sites.map((site) => { + const href = declareLink(site.url, browser.token); + return href ? ( +
  • + + {site.name || site.url} + +
    • + ) : null; + })} +
+ ) : ( +

None of your projects has the tracker on yet.

+ )} +

+ Any other site with the tracker: drag this to your bookmarks bar and click it there:{" "} + +

+ +
+ )} + + {freshToken && ( +
+

New token: copy it now, it is not shown again

+ {freshToken} +

+ Send it as the Crawlproof-Actor header (headless agents), open any tracked + site with ?crp_actor=<token>, or call{" "} + crawlproof('actor', '<token>'). +

+ +
+ )} + + {error &&

{error}

} + {notice &&

{notice}

} + +
+ {actors.length === 0 && ( +

No actors yet. Add yourself first.

+ )} + {actors.map((a) => { + const op = a.operator_actor_id ? actors.find((x) => x.id === a.operator_actor_id) : null; + return ( +
+
+
+

+ {a.name || a.email}{" "} + {a.kind} +

+

+ {a.email} · {a.email_verified ? "verified" : "unverified"} · {a.visibility} + {op ? ` · operated by ${op.name || op.email}` : ""} +

+

+ Last 30 days: {a.last30.pageviews} pageviews, {a.last30.events} events on{" "} + {a.last30.sites} site{a.last30.sites === 1 ? "" : "s"} + {a.last30.contradictions ? ( + + {" "}· {a.last30.contradictions} contradicted by bot detection + + ) : null} +

+
+
+ + + + +
+
+ {a.tokens.length > 0 && ( +
    + {a.tokens.map((t) => ( +
  • + {t.prefix}… + {t.label || "(no label)"} + last used {ago(t.last_used_at)} + +
    • + ))} +
+ )} +
+ ); + })} +
+ +
{ + e.preventDefault(); + run(async () => { + const r = await call("POST", "/api/tracker/v1/actors", { + email, + name, + kind, + operator: kind === "agent" && operator ? operator : undefined, + }); + setEmail(""); + setName(""); + setOperator(""); + const v = String(r.verification); + return v === "owner-login" + ? "Added and verified (it is your login address)." + : v === "sent" + ? "Added. A verification email is on its way; until it is clicked the address shows as unverified." + : `Added, unverified: the verification email could not be sent (${String(r.verificationError ?? "unknown")}).`; + }); + }} + > +

Add an actor

+
+ + + + {kind === "agent" && ( + + )} +
+ +
+
+ ); +} diff --git a/app/(app)/dashboard/settings/page.tsx b/app/(app)/dashboard/settings/page.tsx index 4a34b9df..14b11f50 100644 --- a/app/(app)/dashboard/settings/page.tsx +++ b/app/(app)/dashboard/settings/page.tsx @@ -55,6 +55,21 @@ export default async function SettingsPage() { Manage → +
  • +
    +

    Declared actors

    +

    + Tell tracked sites who you are, and whether you are a + person or an agent. Opt-in, self-reported. +

    +
    + + Manage → + +
    • `} +
    +

    Declared actors (opt-in)

    +

    + Nothing on the wire separates a person from an agent driving a real + browser, so visitors may say who they are. A CrawlProof account + registers actors (an email and a kind: human or agent) and mints a{" "} + cpa_ token for each browser or + agent. The token is the credential; an email alone claims nothing. +

    + 
    {`# an agent: send the header on every request
+Crawlproof-Actor: cpa_…
+
+# a person: open each site once (stored for that site, removed from the URL)
+https://example.com/?crp_actor=cpa_…
+
+# or from page code
+window.crawlproof?.("actor", "cpa_…");   // null forgets it`}
    +

    + It is self-reported, so the rule is one-way: a declared agent is + believed and counted as a bot; a declared human is recorded but never + overrides bot detection, and a mismatch is counted as a contradiction + against that actor. Names are visible only to the actor's owner + unless made public; other site owners see per-kind counts. Manage + actors under Settings → Declared actors, or with{" "} + crawlproof actors. +

    +
    +

    Careers widget

    @@ -217,10 +245,10 @@ window.crawlproof?.track("purchase", "pro_plan");`}

    Privacy model

    • No cookies.
      • -
    • No localStorage.
    • - Session ids use sessionStorage{" "} - when available and reset when the tab session ends. + A random visitor id and a session id (30 minutes of inactivity) + live in this site's localStorage, + plus a declared-actor token only for visitors who opt in.
    • No fingerprinting.
    • No auth tokens or API keys in the browser.
      • diff --git a/app/api/mcp/route.ts b/app/api/mcp/route.ts index d06fce37..cfdc6eb0 100644 --- a/app/api/mcp/route.ts +++ b/app/api/mcp/route.ts @@ -14,6 +14,7 @@ import { registerAuditTools } from "@/lib/mcp/audits"; import { registerLeadTools } from "@/lib/mcp/leads"; import { registerAutoblogTools } from "@/lib/mcp/autoblog"; import { registerAffiliateTools } from "@/lib/mcp/affiliate"; +import { registerActorTools } from "@/lib/mcp/actors"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; @@ -26,6 +27,7 @@ const handler = createMcpHandler( registerLeadTools(server); registerAutoblogTools(server); registerAffiliateTools(server); + registerActorTools(server); }, {}, // The route is mounted at /api/mcp, so mcp-handler must derive its endpoint diff --git a/app/api/track/route.ts b/app/api/track/route.ts index 694d12d0..e5e4a9bf 100644 --- a/app/api/track/route.ts +++ b/app/api/track/route.ts @@ -6,7 +6,6 @@ import { NextRequest, NextResponse } from "next/server"; import { z } from "zod"; import { serviceClient } from "@/lib/supabase/service"; import { categorize } from "@/lib/tracker/categorize"; -import { kindFromBucket } from "@/lib/tracker/humans"; import { SCRIPTED_CAP_EVENTS, SCRIPTED_CAP_PAGEVIEWS, @@ -20,6 +19,7 @@ import { enqueuePostHogEvent } from "@/lib/posthog/events"; import { pruneExitSessions, updateExitRollup } from "@/lib/tracker/exit"; import { gateAgent } from "@/lib/tracker/agent-gate"; import { refuse } from "@/lib/tracker/agent-response"; +import { actorTokenFrom, applyDeclaration, resolveActor, type ResolvedActor } from "@/lib/tracker/actors"; export const runtime = "nodejs"; @@ -39,6 +39,8 @@ const bodySchema = z.object({ timezone: z.string().max(128).optional(), visitorId: z.string().max(128).optional(), sessionId: z.string().max(128).optional(), + // Opt-in declared actor token (cpa_…), see lib/tracker/actors.ts. + actor: z.string().max(128).nullable().optional(), viewport: z .object({ width: z.number().int().nonnegative().optional(), @@ -190,6 +192,14 @@ async function ingest(request: NextRequest, parseBody: boolean) { if (gate.action !== "allow") return refuse(gate); const categorized = categorize({ referrer, userAgent, url: pageUrl }); + + // Declared actor, before the visitor rollup: a declared agent in a stock + // Chrome must not be counted as a human visitor. The rule only ever moves a + // hit toward the bot side (applyDeclaration). No token, or a failed lookup, + // is simply undeclared. + const actorToken = actorTokenFrom(request.headers, parsed.data.actor); + const actor: ResolvedActor | null = actorToken ? await resolveActor(sb, actorToken) : null; + const declared = applyDeclaration(categorized.bucket, actor?.kind ?? null); const today = new Date().toISOString().slice(0, 10); // YYYY-MM-DD UTC // Visitor rollup: one row per (project, day, visitor id), bumped per beacon. @@ -207,7 +217,7 @@ async function ingest(request: NextRequest, parseBody: boolean) { p_project: site, p_day: today, p_visitor: visitorId.slice(0, 128), - p_kind: kindFromBucket(categorized.bucket), + p_kind: declared.kind, p_pageview: event === "pageview", p_cap_events: SCRIPTED_CAP_EVENTS, p_cap_pageviews: SCRIPTED_CAP_PAGEVIEWS, @@ -217,14 +227,32 @@ async function ingest(request: NextRequest, parseBody: boolean) { // Silent — the beacon must never fail closed on a counter table. } } - const demotion = applyScriptedDemotion(categorized.bucket, touch); + const demotion = applyScriptedDemotion(declared.bucket, touch); const bucket = demotion.bucket; - const isAi = demotion.demoted ? false : categorized.isAi; + const isAi = demotion.demoted || declared.bucket !== categorized.bucket ? false : categorized.isAi; // Which side of the human / bot line this hit counts on. The bucket table // carries the whole bucket; the other rollups record only this, so the // stats page can split every breakdown, not just the headline. const kind = demotion.kind; + // Per-actor rollup. A declared human that detection (user agent or the + // scripted cap) still calls a bot is a contradiction: the token is being + // used by something that does not look like its owner. + if (actor) { + try { + await sb.rpc("tracker_touch_actor", { + p_project: site, + p_day: today, + p_actor: actor.actorId, + p_declared_kind: actor.kind, + p_pageview: event === "pageview", + p_contradiction: actor.kind === "human" && kind === "bot", + }); + } catch { + // Silent — the beacon must never fail closed on a counter table. + } + } + // UPSERT increment. Supabase JS doesn't expose a raw .increment() helper // so we read + write under the unique key. The PK protects against // dupes; a lost-update race here at worst undercounts by 1 per @@ -369,6 +397,9 @@ async function ingest(request: NextRequest, parseBody: boolean) { // bot came and never which one, which is the question every customer // actually asks. user_agent: gate.userAgent, + // Only when declared, so an undeclared beacon never names a column + // the database may not have yet (deploys run ahead of migrations). + ...(actor ? { actor_id: actor.actorId } : {}), }); // Prune stale rows (best-effort; skip on error). await sb diff --git a/app/api/tracker/v1/actors/[id]/route.ts b/app/api/tracker/v1/actors/[id]/route.ts new file mode 100644 index 00000000..e7b938ff --- /dev/null +++ b/app/api/tracker/v1/actors/[id]/route.ts @@ -0,0 +1,33 @@ +// /api/tracker/v1/actors/:id +// +// PATCH {name?, visibility?, operator?} edit +// DELETE revoke the actor and all its tokens + +import { NextResponse, type NextRequest } from "next/server"; +import { serviceClient } from "@/lib/supabase/service"; +import { actorOwner } from "@/lib/tracker/actorAuth"; +import { revokeActor, updateActor } from "@/lib/tracker/actorStore"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +type Ctx = { params: Promise<{ id: string }> }; + +export async function PATCH(req: NextRequest, ctx: Ctx) { + const owner = await actorOwner(req); + if (!owner.ok) return NextResponse.json({ error: owner.error }, { status: owner.status }); + const { id } = await ctx.params; + const body = (await req.json().catch(() => ({}))) as Record; + const res = await updateActor(serviceClient(), owner.userId, id, body); + if (!res.ok) return NextResponse.json({ error: res.error }, { status: res.status }); + return NextResponse.json({ ok: true }); +} + +export async function DELETE(req: NextRequest, ctx: Ctx) { + const owner = await actorOwner(req); + if (!owner.ok) return NextResponse.json({ error: owner.error }, { status: owner.status }); + const { id } = await ctx.params; + const res = await revokeActor(serviceClient(), owner.userId, id); + if (!res.ok) return NextResponse.json({ error: res.error }, { status: res.status }); + return NextResponse.json({ ok: true }); +} diff --git a/app/api/tracker/v1/actors/[id]/tokens/route.ts b/app/api/tracker/v1/actors/[id]/tokens/route.ts new file mode 100644 index 00000000..20710420 --- /dev/null +++ b/app/api/tracker/v1/actors/[id]/tokens/route.ts @@ -0,0 +1,39 @@ +// /api/tracker/v1/actors/:id/tokens +// +// POST {label?} mint a cpa_ token; returned ONCE in `token` +// (label "browser" = name it after the caller's UA) +// DELETE ?token= revoke one token, leaving the actor's others live + +import { NextResponse, type NextRequest } from "next/server"; +import { serviceClient } from "@/lib/supabase/service"; +import { actorOwner } from "@/lib/tracker/actorAuth"; +import { mintToken, revokeToken } from "@/lib/tracker/actorStore"; +import { browserLabel } from "@/lib/tracker/actors"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +type Ctx = { params: Promise<{ id: string }> }; + +export async function POST(req: NextRequest, ctx: Ctx) { + const owner = await actorOwner(req); + if (!owner.ok) return NextResponse.json({ error: owner.error }, { status: owner.status }); + const { id } = await ctx.params; + const body = (await req.json().catch(() => ({}))) as Record; + // "browser" asks for a label naming the calling browser, so the token list + // says which machine a dashboard-declared token lives on. + const label = body.label === "browser" ? browserLabel(req.headers.get("user-agent")) : body.label; + const res = await mintToken(serviceClient(), owner.userId, id, label); + if (!res.ok) return NextResponse.json({ error: res.error }, { status: res.status }); + return NextResponse.json(res.value, { status: 201 }); +} + +export async function DELETE(req: NextRequest, ctx: Ctx) { + const owner = await actorOwner(req); + if (!owner.ok) return NextResponse.json({ error: owner.error }, { status: owner.status }); + const { id } = await ctx.params; + const tokenId = req.nextUrl.searchParams.get("token") ?? ""; + const res = await revokeToken(serviceClient(), owner.userId, id, tokenId); + if (!res.ok) return NextResponse.json({ error: res.error }, { status: res.status }); + return NextResponse.json({ ok: true }); +} diff --git a/app/api/tracker/v1/actors/[id]/verify/route.ts b/app/api/tracker/v1/actors/[id]/verify/route.ts new file mode 100644 index 00000000..274fb85d --- /dev/null +++ b/app/api/tracker/v1/actors/[id]/verify/route.ts @@ -0,0 +1,28 @@ +// /api/tracker/v1/actors/:id/verify?t=… — the link in the verification email. +// Public on purpose: holding the emailed token is the proof. + +import { NextResponse, type NextRequest } from "next/server"; +import { serviceClient } from "@/lib/supabase/service"; +import { verifyActorEmail } from "@/lib/tracker/actorStore"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +const esc = (s: string) => + s.replace(/[&<>"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """ })[c]!); + +function page(title: string, body: string, status: number) { + return new NextResponse( + `${esc(title)}` + + `

      ${esc(title)}

      ${esc(body)}

      ` + + `

      Declared actors

      `, + { status, headers: { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" } }, + ); +} + +export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) { + const { id } = await ctx.params; + const res = await verifyActorEmail(serviceClient(), id, req.nextUrl.searchParams.get("t") ?? ""); + if (!res.ok) return page("Not verified", res.error, res.status); + return page("Address verified", `${res.value.email} is now a verified declared actor.`, 200); +} diff --git a/app/api/tracker/v1/actors/route.ts b/app/api/tracker/v1/actors/route.ts new file mode 100644 index 00000000..c58f776f --- /dev/null +++ b/app/api/tracker/v1/actors/route.ts @@ -0,0 +1,40 @@ +// /api/tracker/v1/actors — declared actors (lib/tracker/actors.ts). +// +// GET list this account's actors, tokens, 30-day use +// POST {email, kind, name?, operator?, visibility?, token_label?} +// register one; token_label also mints a token, +// returned ONCE in `token` +// +// Auth: Bearer crp_… or a dashboard session. + +import { NextResponse, type NextRequest } from "next/server"; +import { serviceClient } from "@/lib/supabase/service"; +import { actorOwner } from "@/lib/tracker/actorAuth"; +import { createActor, listActors } from "@/lib/tracker/actorStore"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +export async function GET(req: NextRequest) { + const owner = await actorOwner(req); + if (!owner.ok) return NextResponse.json({ error: owner.error }, { status: owner.status }); + const res = await listActors(serviceClient(), owner.userId); + if (!res.ok) return NextResponse.json({ error: res.error }, { status: res.status }); + return NextResponse.json({ actors: res.value }); +} + +export async function POST(req: NextRequest) { + const owner = await actorOwner(req); + if (!owner.ok) return NextResponse.json({ error: owner.error }, { status: owner.status }); + const body = (await req.json().catch(() => ({}))) as Record; + const res = await createActor(serviceClient(), owner.userId, { + email: body.email, + kind: body.kind, + name: body.name, + operator: body.operator, + visibility: body.visibility, + tokenLabel: body.token_label, + }); + if (!res.ok) return NextResponse.json({ error: res.error }, { status: res.status }); + return NextResponse.json(res.value, { status: 201 }); +} diff --git a/app/api/tracker/v1/stats/route.ts b/app/api/tracker/v1/stats/route.ts index 3ffed054..d053b353 100644 --- a/app/api/tracker/v1/stats/route.ts +++ b/app/api/tracker/v1/stats/route.ts @@ -13,6 +13,8 @@ import { serviceClient } from "@/lib/supabase/service"; import { authenticateBearer } from "@/lib/sp/apiAuth"; import { projectStats, resolveProject } from "@/lib/tracker/apiStats"; import { trackerRange } from "@/lib/tracker/ranges"; +import { DECLARED_DEFINITION } from "@/lib/tracker/actors"; +import { declaredSummary } from "@/lib/tracker/actorStore"; import { DEFAULT_WHO, parseWho, WHO_PARAM, whoToKind } from "@/lib/tracker/who"; export const runtime = "nodejs"; @@ -48,6 +50,11 @@ export async function GET(req: NextRequest) { const detail = parseDetail(sp.get("detail")); const range = trackerRange(sp.get("range")); - const stats = await projectStats(sb, resolved.project, range, whoToKind(who), who, detail); - return NextResponse.json(stats); + const [stats, declared] = await Promise.all([ + projectStats(sb, resolved.project, range, whoToKind(who), who, detail), + // Opt-in declared actors: per-kind totals, plus names only for the + // caller's own actors or ones made public. Null before the migration. + declaredSummary(sb, auth.userId, resolved.project.id, range, DECLARED_DEFINITION).catch(() => null), + ]); + return NextResponse.json({ ...stats, declared }); } diff --git a/app/stats.js/route.ts b/app/stats.js/route.ts index 70b95d01..6be79cea 100644 --- a/app/stats.js/route.ts +++ b/app/stats.js/route.ts @@ -80,6 +80,26 @@ ${VISITOR_SNIPPET} return fresh; } var visitorId = getVisitorId(); + + // Declared actor (opt-in, see lib/tracker/actors.ts). A link carrying + // ?crp_actor= stores the token for this site and is stripped + // from the address bar at once; ?crp_actor=off forgets it. The token rides + // every beacon. Nothing is stored or sent for a visitor who never opts in. + var ACTOR_KEY = 'crawlproof.actor'; + function setActor(token) { + try { + if (token && /^cpa_[A-Za-z0-9_-]{32,124}$/.test(token)) lsSet(ACTOR_KEY, token); + else localStorage.removeItem(ACTOR_KEY); + } catch (_) {} + } + try { + var u = new URL(location.href); + if (u.searchParams.has('crp_actor')) { + setActor(u.searchParams.get('crp_actor')); + u.searchParams.delete('crp_actor'); + history.replaceState(history.state, '', u.pathname + u.search + u.hash); + } + } catch (_) {} function labelFor(el) { try { return el.getAttribute('data-cp-label') @@ -105,6 +125,7 @@ ${VISITOR_SNIPPET} height: Math.max(0, window.innerHeight || 0) }, visitorId: visitorId, + actor: lsGet(ACTOR_KEY) || null, sessionId: getSessionId(), language: navigator.language || '', timezone: (window.Intl && Intl.DateTimeFormat) ? Intl.DateTimeFormat().resolvedOptions().timeZone : '', @@ -136,6 +157,7 @@ ${VISITOR_SNIPPET} var args = Array.prototype.slice.call(arguments, 1); try { if (method === 'track') return cpTrack(args[0], args[1]); + if (method === 'actor') return setActor(args[0]); return cpTrack(method, args[0]); } catch (_) {} } diff --git a/cli/index.ts b/cli/index.ts index 813b9863..e4cc7fa6 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -572,6 +572,126 @@ async function cmdStats(args: Args): Promise { return 0; } +/** + * `crawlproof actors` — declared actors: who you are when you visit a tracked + * site, and whether you are a person. Opt-in and self-reported; an agent is + * believed, a human never overrides bot detection (lib/tracker/actors.ts). + */ +async function cmdActors(args: Args): Promise { + const sub = args.positional[0] ?? "list"; + const json = Boolean(args.flags.json); + const fail = (what: string, r: { status: number; json: Record }) => { + console.error(`actors ${what} failed: ${r.status} ${String(r.json.error ?? "")}`); + return 1; + }; + type Row = { id: string; email: string; name: string; kind: string; email_verified: boolean; visibility: string; tokens: { id: string; prefix: string; label: string; last_used_at: string | null }[]; last30: { events: number; pageviews: number; contradictions: number; sites: number } }; + const list = async () => { + const r = await apiCall(args, "GET", "/api/tracker/v1/actors"); + return { r, actors: (r.json.actors as Row[] | undefined) ?? [] }; + }; + const find = (actors: Row[], key: string | undefined) => + key ? actors.find((a) => a.id === key || a.email === key.toLowerCase()) : undefined; + + if (sub === "list") { + const { r, actors } = await list(); + if (r.status >= 400) return fail("list", r); + if (json) { + process.stdout.write(`${JSON.stringify(actors, null, 2)}\n`); + return 0; + } + if (!actors.length) process.stdout.write("No actors yet. crawlproof actors add --kind=human|agent\n"); + for (const a of actors) { + const verified = a.email_verified ? "verified" : "unverified"; + const u = a.last30; + const flags = u.contradictions ? `, ${u.contradictions} contradicted` : ""; + process.stdout.write(`${a.kind.padEnd(5)} ${a.email}${a.name ? ` (${a.name})` : ""} ${verified}, ${a.visibility} ${a.id}\n`); + process.stdout.write(` 30d: ${u.pageviews} pv, ${u.events} ev on ${u.sites} site${u.sites === 1 ? "" : "s"}${flags}\n`); + for (const t of a.tokens) { + process.stdout.write(` token ${t.prefix}… ${t.label || "(no label)"} last used ${t.last_used_at?.slice(0, 16).replace("T", " ") ?? "never"} ${t.id}\n`); + } + } + return 0; + } + + if (sub === "add") { + const email = args.positional[1]; + const kind = args.flags.kind as string | undefined; + if (!email || (kind !== "human" && kind !== "agent")) { + console.error("usage: crawlproof actors add --kind=human|agent [--name=…] [--operator=] [--public] [--token-label=…] [--no-token] [--json]"); + return 2; + } + const body: Record = { email, kind, visibility: args.flags.public ? "public" : "private" }; + if (typeof args.flags.name === "string") body.name = args.flags.name; + if (typeof args.flags.operator === "string") body.operator = args.flags.operator; + if (!args.flags["no-token"]) body.token_label = typeof args.flags["token-label"] === "string" ? args.flags["token-label"] : "cli"; + const r = await apiCall(args, "POST", "/api/tracker/v1/actors", body); + if (r.status >= 400) return fail("add", r); + if (json) { + process.stdout.write(`${JSON.stringify(r.json, null, 2)}\n`); + return 0; + } + const actor = r.json.actor as { id: string; email: string; kind: string }; + const verification = { + "owner-login": "verified (it is your login)", + sent: "verification email sent", + "not-sent": `NOT verified: email could not be sent (${String(r.json.verificationError ?? "unknown")})`, + }[String(r.json.verification)] ?? ""; + process.stdout.write(`${actor.kind} ${actor.email} ${actor.id}\n${verification}\n`); + if (r.json.token) process.stdout.write(`\n${actorTokenHowTo(String(r.json.token))}`); + return 0; + } + + if (sub === "token") { + const { r, actors } = await list(); + if (r.status >= 400) return fail("token", r); + const actor = find(actors, args.positional[1]); + if (!actor) { + console.error("usage: crawlproof actors token [--label=…]"); + return 2; + } + const label = typeof args.flags.label === "string" ? args.flags.label : "cli"; + const m = await apiCall(args, "POST", `/api/tracker/v1/actors/${actor.id}/tokens`, { label }); + if (m.status >= 400) return fail("token", m); + if (json) process.stdout.write(`${JSON.stringify(m.json, null, 2)}\n`); + else process.stdout.write(actorTokenHowTo(String(m.json.token))); + return 0; + } + + if (sub === "revoke") { + const { r, actors } = await list(); + if (r.status >= 400) return fail("revoke", r); + const actor = find(actors, args.positional[1]); + if (!actor) { + console.error("usage: crawlproof actors revoke [--token=] (no --token revokes the actor and every token)"); + return 2; + } + const tokenId = args.flags.token as string | undefined; + const d = tokenId + ? await apiCall(args, "DELETE", `/api/tracker/v1/actors/${actor.id}/tokens?token=${encodeURIComponent(tokenId)}`) + : await apiCall(args, "DELETE", `/api/tracker/v1/actors/${actor.id}`); + if (d.status >= 400) return fail("revoke", d); + process.stdout.write(tokenId ? `revoked token ${tokenId} of ${actor.email}\n` : `revoked ${actor.email} and all its tokens\n`); + return 0; + } + + console.error(`unknown: crawlproof actors ${sub} (expected: list | add | token | revoke)`); + return 2; +} + +/** How to send a fresh actor token. Pure, for tests. */ +export function actorTokenHowTo(token: string): string { + return [ + `token (shown once): ${token}`, + "", + "Send it with your visits by any of:", + ` header Crawlproof-Actor: ${token} (Playwright extraHTTPHeaders, Puppeteer setExtraHTTPHeaders)`, + ` link https:///?crp_actor=${token} (stored for that site, stripped from the URL)`, + ` script crawlproof('actor', '${token}')`, + "Or, for a person: Dashboard → Settings → Declared actors → Declare this browser.", + "", + ].join("\n"); +} + async function cmdSlots(args: Args): Promise { const sub = args.positional[0]; if (sub === "create") { @@ -973,6 +1093,21 @@ ${EMAIL_TRACKING_USAGE} to the last day and humans only, because a launch is invisible inside a month of crawler traffic. The site is a hostname, a project id or a project name; with one project it can be left out. Needs an API token. + Lists declared actors seen on the site when there are any. + + actors [list] [--json] + actors add --kind=human|agent [--name=…] [--operator=] + [--public] [--token-label=…] [--no-token] [--json] + actors token [--label=…] + actors revoke [--token=] + Declared actors: say who you are, and whether you are a person, on every + site with the CrawlProof tracker. Opt-in and self-reported. A token + (cpa_…) is the credential, never the email; send it as the + Crawlproof-Actor header or open a site once with ?crp_actor=. An + agent is believed; a human never overrides bot detection and a mismatch + is counted as a contradiction. Names are visible to you only unless + --public. Your login address is verified on creation; any other gets a + verification email. dashboard [--range=1h|4h|1d|1w|1m] [--who=humans|bots|all] [--interval=60] [--sites=a.com,b.com] [--sort=score|visitors|pageviews] @@ -1050,6 +1185,8 @@ async function main() { return await cmdAffiliate(args); case "stats": return await cmdStats(args); + case "actors": + return await cmdActors(args); case "dashboard": case "roi": case "tui": diff --git a/lib/dashboard/stats-text.ts b/lib/dashboard/stats-text.ts index fe642b8e..6d7abd49 100644 --- a/lib/dashboard/stats-text.ts +++ b/lib/dashboard/stats-text.ts @@ -15,8 +15,14 @@ export type StatsAnswerish = { pages?: StatsItem[] | null; countries?: StatsItem[] | null; cities?: StatsItem[] | null; + declared?: { + totals?: Record; + actors?: (DeclaredLine & { name?: string; email?: string; kind?: string })[]; + } | null; }; +type DeclaredLine = { actors?: number; events?: number; pageviews?: number; contradictions?: number }; + const list = (v: StatsItem[] | null | undefined): StatsItem[] => (Array.isArray(v) ? v : []); export function renderStats( @@ -51,6 +57,27 @@ export function renderStats( section("Countries", list(answer.countries)); section("Cities", list(answer.cities)); + // Declared actors (opt-in). Printed only when someone declared, and kept + // apart from the numbers above: it is what visitors said, not what we saw. + const human = answer.declared?.totals?.human; + const agent = answer.declared?.totals?.agent; + if ((human?.events ?? 0) + (agent?.events ?? 0) > 0) { + out.push("", "Declared (self-reported)"); + const line = (label: string, t: DeclaredLine | undefined) => { + if (!t?.events) return; + const flags = t.contradictions ? `, ${t.contradictions} contradicted by detection` : ""; + const n = t.actors ?? 0; + out.push(` ${label.padEnd(7)} ${n} actor${n === 1 ? "" : "s"}, ${t.pageviews ?? 0} pageviews, ${t.events} events${flags}`); + }; + line("humans", human); + line("agents", agent); + for (const a of answer.declared?.actors ?? []) { + const who = a.name ? `${a.name} <${a.email}>` : (a.email ?? ""); + const flags = a.contradictions ? ` (${a.contradictions} contradicted)` : ""; + out.push(` ${String(a.kind).padEnd(5)} ${who} ${a.pageviews ?? 0} pv, ${a.events ?? 0} ev${flags}`); + } + } + // Nothing at all is a real answer, and the likeliest cause is worth naming. if (!(totals.pageviews ?? 0) && !(totals.events ?? 0) && !list(answer.sources).length) { out.push("", "Nothing in this window. Check the tag is on the page, or widen --range."); diff --git a/lib/email.ts b/lib/email.ts index 48a62d24..6074ff7d 100644 --- a/lib/email.ts +++ b/lib/email.ts @@ -743,6 +743,69 @@ export async function sendWatchConfirmEmail(input: { return { sent: true }; } +export function actorVerifyEmailHtml(input: { + kind: "human" | "agent"; + name: string; + verifyUrl: string; +}): string { + const who = input.kind === "agent" ? "an agent" : "a person"; + const label = input.name ? ` as ${escapeHtml(input.name)}` : ""; + const innerHtml = ` + +

      + Confirm this address as ${who} +

      +

      + A CrawlProof account registered this address${label}, so that + visits it declares are counted as ${who} on sites using the + CrawlProof tracker. Confirming marks the address verified; + until then it is shown as unverified. +

      + + + + + + Confirm this address → + + + + + +

      + Or copy this link into your browser:
      + ${input.verifyUrl} +

      + + `; + return emailShell({ + title: "Confirm a declared actor", + innerHtml, + footerNote: + "If you did not expect this, ignore it: an unverified address is never shown as verified, " + + "and no one can claim it as verified without this link.", + }); +} + +export async function sendActorVerifyEmail(input: { + to: string; + kind: "human" | "agent"; + name: string; + verifyUrl: string; +}): Promise<{ sent: boolean; error?: string }> { + const c = client(); + if (!c) return { sent: false, error: "RESEND_API_KEY not set" }; + const res = await c.send({ + from: env.resendFrom, + to: input.to, + subject: `Confirm: ${input.to} as ${input.kind === "agent" ? "an agent" : "a person"} on CrawlProof`, + html: actorVerifyEmailHtml(input), + }); + if (!res.sent) return { sent: false, error: res.error }; + return { sent: true }; +} + export function watchChangeEmailHtml(input: { host: string; label: string; diff --git a/lib/mcp/actors.ts b/lib/mcp/actors.ts new file mode 100644 index 00000000..30c679a3 --- /dev/null +++ b/lib/mcp/actors.ts @@ -0,0 +1,94 @@ +// Declared actors for the CrawlProof MCP server (lib/tracker/actors.ts): an +// agent can register itself, mint the token it sends as Crawlproof-Actor, and +// see its own footprint. Scoped to the authenticated user like every module. + +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { z } from "zod"; +import { serviceClient } from "@/lib/supabase/service"; +import { createActor, listActors, mintToken } from "@/lib/tracker/actorStore"; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function getUserId(extra: any): string { + const info = extra?.authInfo; + const uid = info?.extra?.userId ?? info?.clientId; + if (!uid || typeof uid !== "string") throw new Error("Unauthenticated."); + return uid; +} +function textResult(s: string) { + return { content: [{ type: "text" as const, text: s }] }; +} + +const HOW_TO = + "Send the token with visits as the `Crawlproof-Actor` request header (Playwright extraHTTPHeaders), " + + "a `?crp_actor=` link, or `crawlproof('actor', '')`. It is shown once."; + +export function registerActorTools(server: McpServer): void { + server.registerTool( + "list_actors", + { + description: + "The caller's declared actors (people and agents that say who they are to sites running the CrawlProof tracker), with tokens and last-30-day use.", + inputSchema: {}, + }, + async (_args, extra) => { + const res = await listActors(serviceClient(), getUserId(extra)); + if (!res.ok) return textResult(`Error: ${res.error}`); + if (!res.value.length) return textResult("No actors yet."); + return textResult( + res.value + .map((a) => { + const u = a.last30; + const flags = u.contradictions ? `, ${u.contradictions} contradicted by bot detection` : ""; + return `- ${a.kind} ${a.email}${a.name ? ` (${a.name})` : ""} ${a.email_verified ? "verified" : "unverified"}, ${a.visibility} (id: ${a.id})\n 30d: ${u.pageviews} pv, ${u.events} ev on ${u.sites} sites${flags}; ${a.tokens.length} live token(s)`; + }) + .join("\n"), + ); + }, + ); + + server.registerTool( + "add_actor", + { + description: + "Register a declared actor (self-reported, opt-in) and mint its first token. An agent is believed; a human never overrides bot detection.", + inputSchema: { + email: z.string().describe("The actor's email. Verified automatically if it is the caller's login, else a verification email is sent."), + kind: z.enum(["human", "agent"]), + name: z.string().optional(), + operator: z.string().optional().describe("For an agent: the email or id of the caller's human actor who runs it."), + public: z.boolean().optional().describe("Show the name to other site owners (default private)."), + }, + }, + async (args, extra) => { + const res = await createActor(serviceClient(), getUserId(extra), { + email: args.email, + kind: args.kind, + name: args.name, + operator: args.operator, + visibility: args.public ? "public" : "private", + tokenLabel: "mcp", + }); + if (!res.ok) return textResult(`Error: ${res.error}`); + const v = res.value; + return textResult( + `Added ${v.actor.kind} ${v.actor.email} (id: ${v.actor.id}); verification: ${v.verification}.\nToken: ${v.token}\n${HOW_TO}`, + ); + }, + ); + + server.registerTool( + "mint_actor_token", + { + description: "Mint another token for one of the caller's actors, e.g. one per agent run or browser.", + inputSchema: { + actor_id: z.string(), + label: z.string().optional(), + }, + }, + async (args, extra) => { + const res = await mintToken(serviceClient(), getUserId(extra), args.actor_id, args.label ?? "mcp"); + if (!res.ok) return textResult(`Error: ${res.error}`); + return textResult(`Token: ${res.value.token}\n${HOW_TO}`); + }, + ); +} diff --git a/lib/tracker/actorAuth.ts b/lib/tracker/actorAuth.ts new file mode 100644 index 00000000..9b4b008e --- /dev/null +++ b/lib/tracker/actorAuth.ts @@ -0,0 +1,23 @@ +// Who is managing actors: a `crp_` bearer token (CLI, MCP) or a dashboard +// session. Either way the answer is a user id every actor query scopes by. + +import type { NextRequest } from "next/server"; +import { authenticateBearer } from "@/lib/sp/apiAuth"; +import { createClient } from "@/lib/supabase/server"; + +export async function actorOwner( + req: NextRequest, +): Promise<{ ok: true; userId: string } | { ok: false; status: number; error: string }> { + if (req.headers.get("authorization")) { + const auth = await authenticateBearer(req); + return auth.ok ? { ok: true, userId: auth.userId } : auth; + } + try { + const supabase = await createClient(); + const { data } = await supabase.auth.getUser(); + if (data.user) return { ok: true, userId: data.user.id }; + } catch { + // fall through to 401 + } + return { ok: false, status: 401, error: "Sign in, or send Authorization: Bearer crp_…" }; +} diff --git a/lib/tracker/actorStore.ts b/lib/tracker/actorStore.ts new file mode 100644 index 00000000..fc8f65f8 --- /dev/null +++ b/lib/tracker/actorStore.ts @@ -0,0 +1,407 @@ +// Declared actors: the reads and writes behind the API, the dashboard and the +// CLI. Every function takes the service client and the owner's user id and +// scopes by it; callers authenticate first (lib/tracker/actorAuth.ts). +// +// Model and trust rule: lib/tracker/actors.ts. + +import crypto from "node:crypto"; +import type { SupabaseClient } from "@supabase/supabase-js"; +import { env } from "@/lib/env"; +import { sendActorVerifyEmail } from "@/lib/email"; +import type { TrackerRange } from "@/lib/tracker/ranges"; +import { + type DeclaredKind, + type DeclaredTotals, + hashActorToken, + mintActorToken, + normalizeEmail, + parseDeclaredKind, + toDeclaredTotals, +} from "@/lib/tracker/actors"; + +type Sb = SupabaseClient; + +export type ActorToken = { + id: string; + prefix: string; + label: string; + created_at: string; + last_used_at: string | null; +}; + +export type Actor = { + id: string; + email: string; + name: string; + kind: DeclaredKind; + operator_actor_id: string | null; + visibility: "private" | "public"; + email_verified: boolean; + created_at: string; + tokens: ActorToken[]; + /** Last 30 days across every project. */ + last30: { events: number; pageviews: number; contradictions: number; sites: number }; +}; + +export type StoreResult = { ok: true; value: T } | { ok: false; status: number; error: string }; + +const ACTOR_COLUMNS = "id, email, name, kind, operator_actor_id, visibility, email_verified_at, created_at"; + +function daysAgo(n: number): string { + return new Date(Date.now() - n * 86_400_000).toISOString().slice(0, 10); +} + +async function ownerEmail(sb: Sb, ownerId: string): Promise { + const { data } = await sb.from("profiles").select("email").eq("id", ownerId).maybeSingle(); + return normalizeEmail((data as { email?: string } | null)?.email); +} + +export async function listActors(sb: Sb, ownerId: string): Promise> { + const { data, error } = await sb + .from("tracker_actors") + .select(`${ACTOR_COLUMNS}, tracker_actor_tokens(id, prefix, label, created_at, last_used_at, revoked_at)`) + .eq("owner_id", ownerId) + .is("revoked_at", null) + .order("created_at", { ascending: true }); + if (error) return { ok: false, status: 500, error: error.message }; + const rows = (data ?? []) as Record[]; + + const ids = rows.map((r) => r.id as string); + const usage = new Map(); + if (ids.length) { + const { data: stats } = await sb + .from("tracker_actor_daily_stats") + .select("actor_id, project_id, events, pageviews, contradictions") + .in("actor_id", ids) + .gte("day", daysAgo(30)); + const sites = new Map>(); + for (const s of (stats ?? []) as Record[]) { + const id = s.actor_id as string; + const u = usage.get(id) ?? { events: 0, pageviews: 0, contradictions: 0, sites: 0 }; + u.events += Number(s.events) || 0; + u.pageviews += Number(s.pageviews) || 0; + u.contradictions += Number(s.contradictions) || 0; + usage.set(id, u); + const set = sites.get(id) ?? new Set(); + set.add(s.project_id as string); + sites.set(id, set); + } + for (const [id, set] of sites) usage.get(id)!.sites = set.size; + } + + return { + ok: true, + value: rows.map((r) => ({ + id: r.id as string, + email: r.email as string, + name: (r.name as string) ?? "", + kind: r.kind as DeclaredKind, + operator_actor_id: (r.operator_actor_id as string | null) ?? null, + visibility: r.visibility === "public" ? "public" : "private", + email_verified: !!r.email_verified_at, + created_at: r.created_at as string, + tokens: ((r.tracker_actor_tokens as Record[] | null) ?? []) + .filter((t) => !t.revoked_at) + .map((t) => ({ + id: t.id as string, + prefix: t.prefix as string, + label: (t.label as string) ?? "", + created_at: t.created_at as string, + last_used_at: (t.last_used_at as string | null) ?? null, + })), + last30: usage.get(r.id as string) ?? { events: 0, pageviews: 0, contradictions: 0, sites: 0 }, + })), + }; +} + +export type CreateActorInput = { + email: unknown; + kind: unknown; + name?: unknown; + /** Another of the owner's actors, by id or email: the human an agent acts for. */ + operator?: unknown; + visibility?: unknown; + /** Label for a first token, minted in the same call. Omit for none. */ + tokenLabel?: unknown; +}; + +export type CreatedActor = { + actor: { id: string; email: string; kind: DeclaredKind; email_verified: boolean }; + /** Shown once. Null when no token was asked for. */ + token: string | null; + verification: "owner-login" | "sent" | "not-sent"; + verificationError?: string; +}; + +async function resolveOperator(sb: Sb, ownerId: string, raw: unknown): Promise> { + if (raw === undefined || raw === null || raw === "") return { ok: true, value: null }; + if (typeof raw !== "string") return { ok: false, status: 400, error: "operator must be an actor id or email." }; + const email = normalizeEmail(raw); + let query = sb.from("tracker_actors").select("id, kind").eq("owner_id", ownerId).is("revoked_at", null); + query = email ? query.eq("email", email) : query.eq("id", raw); + const { data } = await query.limit(1).maybeSingle(); + const row = data as { id: string; kind: string } | null; + if (!row) return { ok: false, status: 400, error: `No actor "${raw}" on this account to act as operator.` }; + if (row.kind !== "human") return { ok: false, status: 400, error: "An operator must be a human actor." }; + return { ok: true, value: row.id }; +} + +export async function createActor(sb: Sb, ownerId: string, input: CreateActorInput): Promise> { + const email = normalizeEmail(input.email); + if (!email) return { ok: false, status: 400, error: "A valid email is required." }; + const kind = parseDeclaredKind(input.kind); + if (!kind) return { ok: false, status: 400, error: "kind must be human or agent." }; + const name = typeof input.name === "string" ? input.name.trim().slice(0, 120) : ""; + const visibility = input.visibility === "public" ? "public" : "private"; + + const operator = await resolveOperator(sb, ownerId, input.operator); + if (!operator.ok) return operator; + if (operator.value && kind !== "agent") { + return { ok: false, status: 400, error: "Only an agent has an operator." }; + } + + const { data: dupe } = await sb + .from("tracker_actors") + .select("id") + .eq("owner_id", ownerId) + .eq("email", email) + .is("revoked_at", null) + .limit(1) + .maybeSingle(); + if (dupe) return { ok: false, status: 409, error: `${email} is already an actor on this account.` }; + + // The owner's own login address is proven by the login itself. + const isOwnerLogin = (await ownerEmail(sb, ownerId)) === email; + let verifyToken: string | null = null; + if (isOwnerLogin) { + const { data: taken } = await sb + .from("tracker_actors") + .select("id") + .eq("email", email) + .not("email_verified_at", "is", null) + .is("revoked_at", null) + .limit(1) + .maybeSingle(); + if (taken) return { ok: false, status: 409, error: `${email} is already a verified actor on another account.` }; + } else { + verifyToken = crypto.randomBytes(24).toString("base64url"); + } + + const { data: inserted, error } = await sb + .from("tracker_actors") + .insert({ + owner_id: ownerId, + email, + name, + kind, + operator_actor_id: operator.value, + visibility, + email_verified_at: isOwnerLogin ? new Date().toISOString() : null, + verify_token_hash: verifyToken ? hashActorToken(`v_${verifyToken}`) : null, + }) + .select("id") + .single(); + if (error || !inserted) return { ok: false, status: 500, error: error?.message ?? "insert failed" }; + const actorId = (inserted as { id: string }).id; + + let token: string | null = null; + if (input.tokenLabel !== undefined && input.tokenLabel !== null && input.tokenLabel !== false) { + const minted = await mintToken(sb, ownerId, actorId, input.tokenLabel); + if (!minted.ok) return minted; + token = minted.value.token; + } + + let verification: CreatedActor["verification"] = isOwnerLogin ? "owner-login" : "not-sent"; + let verificationError: string | undefined; + if (verifyToken) { + const sent = await sendActorVerifyEmail({ + to: email, + kind, + name, + verifyUrl: `${env.siteUrl}/api/tracker/v1/actors/${actorId}/verify?t=${verifyToken}`, + }); + if (sent.sent) verification = "sent"; + else verificationError = sent.error; + } + + return { + ok: true, + value: { + actor: { id: actorId, email, kind, email_verified: isOwnerLogin }, + token, + verification, + ...(verificationError ? { verificationError } : {}), + }, + }; +} + +async function ownedActor(sb: Sb, ownerId: string, actorId: string) { + if (!/^[0-9a-f-]{36}$/i.test(actorId)) return null; + const { data } = await sb + .from("tracker_actors") + .select("id, email, kind") + .eq("id", actorId) + .eq("owner_id", ownerId) + .is("revoked_at", null) + .maybeSingle(); + return data as { id: string; email: string; kind: DeclaredKind } | null; +} + +export async function mintToken( + sb: Sb, + ownerId: string, + actorId: string, + label: unknown, +): Promise> { + if (!(await ownedActor(sb, ownerId, actorId))) return { ok: false, status: 404, error: "No such actor." }; + const minted = mintActorToken(); + const { data, error } = await sb + .from("tracker_actor_tokens") + .insert({ + actor_id: actorId, + prefix: minted.prefix, + token_hash: minted.hash, + label: typeof label === "string" ? label.trim().slice(0, 200) : "", + }) + .select("id") + .single(); + if (error || !data) return { ok: false, status: 500, error: error?.message ?? "insert failed" }; + return { ok: true, value: { id: (data as { id: string }).id, token: minted.plaintext, prefix: minted.prefix } }; +} + +export async function revokeToken(sb: Sb, ownerId: string, actorId: string, tokenId: string): Promise> { + if (!(await ownedActor(sb, ownerId, actorId))) return { ok: false, status: 404, error: "No such actor." }; + const { data, error } = await sb + .from("tracker_actor_tokens") + .update({ revoked_at: new Date().toISOString() }) + .eq("id", tokenId) + .eq("actor_id", actorId) + .is("revoked_at", null) + .select("id"); + if (error) return { ok: false, status: 500, error: error.message }; + if (!data?.length) return { ok: false, status: 404, error: "No such live token." }; + return { ok: true, value: null }; +} + +export async function revokeActor(sb: Sb, ownerId: string, actorId: string): Promise> { + if (!(await ownedActor(sb, ownerId, actorId))) return { ok: false, status: 404, error: "No such actor." }; + const now = new Date().toISOString(); + await sb.from("tracker_actor_tokens").update({ revoked_at: now }).eq("actor_id", actorId).is("revoked_at", null); + const { error } = await sb.from("tracker_actors").update({ revoked_at: now }).eq("id", actorId); + if (error) return { ok: false, status: 500, error: error.message }; + return { ok: true, value: null }; +} + +export async function updateActor( + sb: Sb, + ownerId: string, + actorId: string, + patch: { name?: unknown; visibility?: unknown; operator?: unknown }, +): Promise> { + const actor = await ownedActor(sb, ownerId, actorId); + if (!actor) return { ok: false, status: 404, error: "No such actor." }; + const update: Record = {}; + if (typeof patch.name === "string") update.name = patch.name.trim().slice(0, 120); + if (patch.visibility === "public" || patch.visibility === "private") update.visibility = patch.visibility; + if (patch.operator !== undefined) { + if (actor.kind !== "agent" && patch.operator) return { ok: false, status: 400, error: "Only an agent has an operator." }; + const op = await resolveOperator(sb, ownerId, patch.operator); + if (!op.ok) return op; + update.operator_actor_id = op.value; + } + if (!Object.keys(update).length) return { ok: false, status: 400, error: "Nothing to update." }; + const { error } = await sb.from("tracker_actors").update(update).eq("id", actorId); + if (error) return { ok: false, status: 500, error: error.message }; + return { ok: true, value: null }; +} + +/** The link in the verification email. Public: the token is the proof. */ +export async function verifyActorEmail(sb: Sb, actorId: string, token: string): Promise> { + if (!/^[0-9a-f-]{36}$/i.test(actorId) || !/^[A-Za-z0-9_-]{16,64}$/.test(token)) { + return { ok: false, status: 400, error: "Malformed link." }; + } + const { data } = await sb + .from("tracker_actors") + .select("id, email, verify_token_hash, email_verified_at, revoked_at") + .eq("id", actorId) + .maybeSingle(); + const row = data as { email: string; verify_token_hash: string | null; email_verified_at: string | null; revoked_at: string | null } | null; + if (!row || row.revoked_at) return { ok: false, status: 404, error: "This actor no longer exists." }; + if (row.email_verified_at) return { ok: true, value: { email: row.email } }; + const expected = row.verify_token_hash ?? ""; + const got = hashActorToken(`v_${token}`); + if (expected.length !== got.length || !crypto.timingSafeEqual(Buffer.from(expected), Buffer.from(got))) { + return { ok: false, status: 400, error: "This link is not valid." }; + } + const { error } = await sb + .from("tracker_actors") + .update({ email_verified_at: new Date().toISOString(), verify_token_hash: null }) + .eq("id", actorId); + // The partial unique index refuses a second verified claim on an address. + if (error) return { ok: false, status: 409, error: `${row.email} is already a verified actor on another account.` }; + return { ok: true, value: { email: row.email } }; +} + +/** First UTC day a range covers, for the daily actor rollup. */ +export function rangeSinceDay(range: TrackerRange): string { + if (range.days === 0) return "1970-01-01"; + if (range.days) return daysAgo(range.days - 1); + return daysAgo(0); +} + +export type DeclaredSummary = { + definition: string; + totals: DeclaredTotals; + /** + * Named actors on this site: only the viewer's own, or actors their owners + * made public. Everyone else is in the totals and nowhere else. + */ + actors: { name: string; email: string; kind: DeclaredKind; events: number; pageviews: number; contradictions: number; mine: boolean }[]; +}; + +export async function declaredSummary( + sb: Sb, + viewerId: string, + projectId: string, + range: TrackerRange, + definition: string, +): Promise { + const since = rangeSinceDay(range); + const { data: totals, error } = await sb.rpc("tracker_declared_totals", { p_project: projectId, p_since: since }); + // Before the migration is applied the RPC is missing; say nothing rather + // than print zeros that read as "nobody declared". + if (error) return null; + + const { data: rows } = await sb + .from("tracker_actor_daily_stats") + .select("actor_id, events, pageviews, contradictions, actor:tracker_actors!inner(name, email, kind, owner_id, visibility)") + .eq("project_id", projectId) + .gte("day", since); + const byActor = new Map(); + for (const r of (rows ?? []) as Record[]) { + const a = r.actor as { name: string; email: string; kind: string; owner_id: string; visibility: string } | null; + if (!a) continue; + const mine = a.owner_id === viewerId; + if (!mine && a.visibility !== "public") continue; + const key = r.actor_id as string; + const cur = byActor.get(key) ?? { + name: a.name, + email: a.email, + kind: (a.kind === "agent" ? "agent" : "human") as DeclaredKind, + events: 0, + pageviews: 0, + contradictions: 0, + mine, + }; + cur.events += Number(r.events) || 0; + cur.pageviews += Number(r.pageviews) || 0; + cur.contradictions += Number(r.contradictions) || 0; + byActor.set(key, cur); + } + + return { + definition, + totals: toDeclaredTotals(totals), + actors: [...byActor.values()].sort((x, y) => y.events - x.events), + }; +} diff --git a/lib/tracker/actors.ts b/lib/tracker/actors.ts new file mode 100644 index 00000000..b68ce946 --- /dev/null +++ b/lib/tracker/actors.ts @@ -0,0 +1,207 @@ +// Declared actors: a visitor saying who it is, and whether it is a person. +// +// Nothing on the wire tells a person from an agent driving a real browser, so +// this is opt-in and on the honor system. An account registers actors (email, +// name, kind) and mints a `cpa_` token per browser or agent; the visitor sends +// the token with the beacon by one of two channels: +// +// 1. the `Crawlproof-Actor` request header — headless agents (Playwright +// extraHTTPHeaders, Puppeteer setExtraHTTPHeaders) +// 2. the `actor` field of the beacon body — stats.js, from the site's +// localStorage, set by opening the site once with `?crp_actor=` +// or by `crawlproof('actor', token)` (the dashboard bookmarklet) +// +// No cookie channel, on purpose: the tracker is documented as cookieless and +// the beacon sends `credentials: 'omit'` (tests/contract/stats-js.test.ts). +// The token sits in localStorage next to the visitor id it already keeps. +// +// The token is the credential; a bare email is never accepted, so typing +// someone else's address claims nothing. What a token cannot stop is an owner +// lying about the KIND, which is why applyDeclaration below is asymmetric. +// +// Schema: supabase/migrations/20261004120000_tracker_declared_actors.sql. + +import crypto from "node:crypto"; +import type { SupabaseClient } from "@supabase/supabase-js"; +import { env } from "@/lib/env"; +import { kindFromBucket, type TrackerKind } from "@/lib/tracker/humans"; + +export type DeclaredKind = "human" | "agent"; + +export const ACTOR_TOKEN_PREFIX = "cpa_"; +export const ACTOR_HEADER = "crawlproof-actor"; +/** The URL parameter stats.js lifts into localStorage and strips. */ +export const ACTOR_PARAM = "crp_actor"; + +/** The bucket a declared agent's hit is counted under. Renders "Bot · declared". */ +export const DECLARED_AGENT_BUCKET = "bot:declared"; + +export const DECLARED_DEFINITION = + "Visitors who opted in and said who they are. Self-declared: an agent is believed, a human is not taken on trust and never overrides bot detection."; + +const PREFIX_DISPLAY_LEN = 8; + +export type MintedActorToken = { plaintext: string; prefix: string; hash: string }; + +export function mintActorToken(): MintedActorToken { + const plaintext = `${ACTOR_TOKEN_PREFIX}${crypto.randomBytes(32).toString("base64url")}`; + return { + plaintext, + prefix: plaintext.slice(0, PREFIX_DISPLAY_LEN), + hash: hashActorToken(plaintext), + }; +} + +/** + * sha256(domain || token || pepper). Same reasoning as lib/sp/apiToken.ts: 256 + * bits of entropy make a fast hash safe. The "actor:" domain keeps an actor + * token hash from ever colliding with an API token hash under the same pepper. + */ +export function hashActorToken(plaintext: string): string { + if (!env.spTokenPepper) throw new Error("SP_TOKEN_PEPPER not set."); + return crypto + .createHash("sha256") + .update(`actor:${plaintext}${env.spTokenPepper}`, "utf8") + .digest("hex"); +} + +export function isActorTokenShape(s: string | null | undefined): s is string { + return !!s && s.startsWith(ACTOR_TOKEN_PREFIX) && s.length >= 36 && s.length <= 128 && /^[A-Za-z0-9_-]+$/.test(s); +} + +/** + * The declared token on a beacon: the header if it carries one (set per agent + * run, so the most deliberate), else the beacon body. + */ +export function actorTokenFrom(headers: Headers, bodyActor: string | null | undefined): string | null { + const candidates = [headers.get(ACTOR_HEADER), bodyActor ?? null]; + for (const c of candidates) { + const t = c?.trim(); + if (isActorTokenShape(t)) return t; + } + return null; +} + +/** + * The bucket and kind a hit is counted under once its declaration has had its + * say. The only rule that matters, and it is asymmetric on purpose: + * + * agent -> believed. A hit detection called human moves to bot:declared. + * Nobody gains by claiming to be a bot; the worst a liar does is + * shrink their own human count. + * human -> recorded, never trusted. The bucket stands. If detection (user + * agent or scripted cap) already called it a bot, it stays a bot + * and counts as a contradiction against the actor, which is the + * signal that a token is being used by something it should not be. + * + * So a declaration can only ever move traffic toward the bot side. + */ +export function applyDeclaration( + bucket: string, + declared: DeclaredKind | null, +): { bucket: string; kind: TrackerKind; contradiction: boolean } { + const kind = kindFromBucket(bucket); + if (declared === "agent" && kind === "human") { + return { bucket: DECLARED_AGENT_BUCKET, kind: "bot", contradiction: false }; + } + return { bucket, kind, contradiction: declared === "human" && kind === "bot" }; +} + +export type ResolvedActor = { actorId: string; tokenId: string; kind: DeclaredKind }; + +// A beacon fires several times per page view; a token is looked up once a +// minute per process, not once per scroll. Revocation therefore takes up to +// TTL to bite, which is fine for analytics. +const CACHE_TTL_MS = 60_000; +const CACHE_MAX = 5_000; +const cache = new Map(); + +/** Test hook. */ +export function clearActorCache() { + cache.clear(); +} + +/** + * The live actor behind a token, or null for unknown, revoked, or an actor + * whose owner revoked it. Never throws: a failed lookup is "undeclared". + */ +export async function resolveActor(sb: SupabaseClient, token: string): Promise { + let hash: string; + try { + hash = hashActorToken(token); + } catch { + return null; + } + const hit = cache.get(hash); + if (hit && Date.now() - hit.at < CACHE_TTL_MS) return hit.value; + + let value: ResolvedActor | null = null; + try { + const { data } = await sb + .from("tracker_actor_tokens") + .select("id, revoked_at, actor:tracker_actors!inner(id, kind, revoked_at)") + .eq("token_hash", hash) + .maybeSingle(); + const row = data as + | { id: string; revoked_at: string | null; actor: { id: string; kind: string; revoked_at: string | null } | null } + | null; + if (row && !row.revoked_at && row.actor && !row.actor.revoked_at) { + const kind = row.actor.kind === "agent" ? "agent" : row.actor.kind === "human" ? "human" : null; + if (kind) value = { actorId: row.actor.id, tokenId: row.id, kind }; + } + if (value) { + void sb + .from("tracker_actor_tokens") + .update({ last_used_at: new Date().toISOString() }) + .eq("id", value.tokenId) + .then(() => undefined); + } + } catch { + return null; + } + + if (cache.size >= CACHE_MAX) cache.clear(); + cache.set(hash, { at: Date.now(), value }); + return value; +} + +/** A short, human label for the token, so the list says which browser it is. */ +export function browserLabel(ua: string | null, now = new Date()): string { + const u = ua ?? ""; + const browser = /Edg\//.test(u) ? "Edge" : /Firefox\//.test(u) ? "Firefox" : /Chrome\//.test(u) ? "Chrome" : /Safari\//.test(u) ? "Safari" : "Browser"; + const os = /Android/.test(u) ? "Android" : /iPhone|iPad/.test(u) ? "iOS" : /Mac OS X/.test(u) ? "macOS" : /Windows/.test(u) ? "Windows" : /Linux/.test(u) ? "Linux" : ""; + return `Browser: ${browser}${os ? ` on ${os}` : ""}, ${now.toISOString().slice(0, 10)}`; +} + +// ---------------------------------------------------------------- validation + +export function normalizeEmail(raw: unknown): string | null { + if (typeof raw !== "string") return null; + const email = raw.trim().toLowerCase(); + if (email.length < 3 || email.length > 320) return null; + return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email) ? email : null; +} + +export function parseDeclaredKind(raw: unknown): DeclaredKind | null { + return raw === "human" || raw === "agent" ? raw : null; +} + +/** Per-kind totals as tracker_declared_totals returns them, coerced. */ +export type DeclaredTotals = Record; + +export function toDeclaredTotals(rows: unknown): DeclaredTotals { + const out: DeclaredTotals = { + human: { actors: 0, events: 0, pageviews: 0, contradictions: 0 }, + agent: { actors: 0, events: 0, pageviews: 0, contradictions: 0 }, + }; + if (!Array.isArray(rows)) return out; + for (const r of rows as Record[]) { + const kind = parseDeclaredKind(r.declared_kind); + if (!kind) continue; + for (const key of ["actors", "events", "pageviews", "contradictions"] as const) { + const n = Number(r[key]); + out[kind][key] = Number.isFinite(n) ? n : 0; + } + } + return out; +} diff --git a/supabase/migrations/20261004120000_tracker_declared_actors.sql b/supabase/migrations/20261004120000_tracker_declared_actors.sql new file mode 100644 index 00000000..ffcb65ce --- /dev/null +++ b/supabase/migrations/20261004120000_tracker_declared_actors.sql @@ -0,0 +1,246 @@ +-- Declared actors: a visitor may say who it is, and whether it is a person. +-- +-- WHY: nothing on the wire separates a person from an agent driving a real +-- browser. The tracker guesses from the user agent and the scripted cap, and +-- an undeclared agent in Chrome reads as a human visit. This adds the honest +-- half: an opt-in declaration. A signed-in account registers ACTORS (an +-- email, a name, and a kind: human or agent; an agent may name the human who +-- runs it), mints a secret token per browser or agent, and the visitor sends +-- that token with the beacon. The route never accepts a bare email. +-- +-- TRUST RULE (lib/tracker/actors.ts, applyDeclaration): a declaration can +-- only move a hit toward the bot side. "agent" is believed and counts the hit +-- as bot:declared. "human" is recorded but never overrides detection: a +-- declared human the user agent or the scripted cap calls a bot stays a bot, +-- and the hit is counted as a contradiction against that actor. +-- +-- PRIVACY: an actor's identity is visible to its owner only (visibility +-- 'private', the default). Another site's owner sees declared counts per +-- kind, never the email. The ingest route and API read with the service role; +-- RLS here keeps the tables owner-scoped for anything that reads as a user. +-- +-- WHAT: +-- 1. tracker_actors — who: email, name, kind, operator, visibility +-- 2. tracker_actor_tokens — credentials, hashed like sp_api_token +-- 3. tracker_actor_daily_stats — per (project, day, actor) counts +-- 4. tracker_events.actor_id — the live view can say which actor +-- 5. tracker_touch_actor — the per-beacon upsert +-- 6. tracker_declared_totals — per-kind totals for a project and window +-- +-- Idempotent. Deploys do not run migrations: apply by hand after merge. + +-- --------------------------------------------------------------------------- +-- 1. Actors +-- --------------------------------------------------------------------------- +create table if not exists public.tracker_actors ( + id uuid primary key default gen_random_uuid(), + owner_id uuid not null references auth.users(id) on delete cascade, + email text not null check (length(email) between 3 and 320 and position('@' in email) > 1), + name text not null default '' check (length(name) <= 120), + kind text not null check (kind in ('human', 'agent')), + -- The human an agent acts for. Must be another actor of the same owner; + -- enforced in the API (a check constraint cannot see another row). + operator_actor_id uuid references public.tracker_actors(id) on delete set null, + visibility text not null default 'private' check (visibility in ('private', 'public')), + -- Set when the address proved it receives mail (or is the owner's login). + email_verified_at timestamptz, + verify_token_hash text, + created_at timestamptz not null default now(), + revoked_at timestamptz +); + +-- One live, verified claim per address across every account, so nobody can +-- register a verified anthony@ after the real one has. Unverified rows are +-- labels only and may repeat. +create unique index if not exists tracker_actors_verified_email_uidx + on public.tracker_actors (lower(email)) + where email_verified_at is not null and revoked_at is null; + +create index if not exists tracker_actors_owner_idx + on public.tracker_actors (owner_id) where revoked_at is null; + +alter table public.tracker_actors enable row level security; + +do $$ +begin + if not exists ( + select 1 from pg_policies + where schemaname = 'public' and tablename = 'tracker_actors' + and policyname = 'tracker_actors owner select' + ) then + create policy "tracker_actors owner select" + on public.tracker_actors for select + using (owner_id = auth.uid()); + end if; +end $$; + +grant select on public.tracker_actors to authenticated; +grant select, insert, update, delete on public.tracker_actors to service_role; + +-- --------------------------------------------------------------------------- +-- 2. Tokens (one per browser or agent; revoke one without the rest) +-- --------------------------------------------------------------------------- +create table if not exists public.tracker_actor_tokens ( + id uuid primary key default gen_random_uuid(), + actor_id uuid not null references public.tracker_actors(id) on delete cascade, + prefix text not null, + token_hash text not null unique, + label text not null default '' check (length(label) <= 200), + created_at timestamptz not null default now(), + last_used_at timestamptz, + revoked_at timestamptz +); + +create index if not exists tracker_actor_tokens_actor_idx + on public.tracker_actor_tokens (actor_id); + +alter table public.tracker_actor_tokens enable row level security; + +do $$ +begin + if not exists ( + select 1 from pg_policies + where schemaname = 'public' and tablename = 'tracker_actor_tokens' + and policyname = 'tracker_actor_tokens owner select' + ) then + create policy "tracker_actor_tokens owner select" + on public.tracker_actor_tokens for select + using (actor_id in (select id from public.tracker_actors where owner_id = auth.uid())); + end if; +end $$; + +grant select on public.tracker_actor_tokens to authenticated; +grant select, insert, update, delete on public.tracker_actor_tokens to service_role; + +-- --------------------------------------------------------------------------- +-- 3. Per (project, day, actor) rollup +-- --------------------------------------------------------------------------- +create table if not exists public.tracker_actor_daily_stats ( + project_id uuid not null references public.projects(id) on delete cascade, + day date not null, + actor_id uuid not null references public.tracker_actors(id) on delete cascade, + -- Copied from the actor at write time so per-kind totals need no join + -- (and so a project owner can count kinds without reading tracker_actors). + declared_kind text not null check (declared_kind in ('human', 'agent')), + events integer not null default 0, + pageviews integer not null default 0, + -- Hits where the declaration said human and detection said bot. + contradictions integer not null default 0, + first_seen timestamptz not null default now(), + last_seen timestamptz not null default now(), + primary key (project_id, day, actor_id) +); + +create index if not exists tracker_actor_daily_stats_actor_idx + on public.tracker_actor_daily_stats (actor_id, day); + +alter table public.tracker_actor_daily_stats enable row level security; + +do $$ +begin + if not exists ( + select 1 from pg_policies + where schemaname = 'public' and tablename = 'tracker_actor_daily_stats' + and policyname = 'tracker_actor_daily_stats project owner select' + ) then + create policy "tracker_actor_daily_stats project owner select" + on public.tracker_actor_daily_stats for select + using (project_id in (select id from public.projects where owner_id = auth.uid())); + end if; + + if not exists ( + select 1 from pg_policies + where schemaname = 'public' and tablename = 'tracker_actor_daily_stats' + and policyname = 'tracker_actor_daily_stats member select' + ) then + create policy "tracker_actor_daily_stats member select" + on public.tracker_actor_daily_stats for select + using (public.is_project_member(project_id, auth.uid())); + end if; + + if not exists ( + select 1 from pg_policies + where schemaname = 'public' and tablename = 'tracker_actor_daily_stats' + and policyname = 'tracker_actor_daily_stats actor owner select' + ) then + create policy "tracker_actor_daily_stats actor owner select" + on public.tracker_actor_daily_stats for select + using (actor_id in (select id from public.tracker_actors where owner_id = auth.uid())); + end if; +end $$; + +grant select on public.tracker_actor_daily_stats to authenticated; +grant select, insert, update, delete on public.tracker_actor_daily_stats to service_role; + +-- --------------------------------------------------------------------------- +-- 4. Raw events learn the actor +-- --------------------------------------------------------------------------- +alter table public.tracker_events + add column if not exists actor_id uuid references public.tracker_actors(id) on delete set null; + +-- --------------------------------------------------------------------------- +-- 5. Per-beacon touch +-- --------------------------------------------------------------------------- +create or replace function public.tracker_touch_actor( + p_project uuid, + p_day date, + p_actor uuid, + p_declared_kind text, + p_pageview boolean, + p_contradiction boolean +) +returns void +language sql +volatile +security invoker +set search_path = public +as $$ + insert into public.tracker_actor_daily_stats as s + (project_id, day, actor_id, declared_kind, events, pageviews, contradictions, first_seen, last_seen) + values + (p_project, p_day, p_actor, p_declared_kind, 1, + case when p_pageview then 1 else 0 end, + case when p_contradiction then 1 else 0 end, + now(), now()) + on conflict (project_id, day, actor_id) do update set + declared_kind = excluded.declared_kind, + events = s.events + 1, + pageviews = s.pageviews + excluded.pageviews, + contradictions = s.contradictions + excluded.contradictions, + last_seen = now(); +$$; + +revoke all on function public.tracker_touch_actor(uuid, date, uuid, text, boolean, boolean) from public, anon, authenticated; +grant execute on function public.tracker_touch_actor(uuid, date, uuid, text, boolean, boolean) to service_role; + +-- --------------------------------------------------------------------------- +-- 6. Per-kind totals for a project over [p_since, today] +-- --------------------------------------------------------------------------- +create or replace function public.tracker_declared_totals( + p_project uuid, + p_since date +) +returns table ( + declared_kind text, + actors bigint, + events bigint, + pageviews bigint, + contradictions bigint +) +language sql +stable +security invoker +set search_path = public +as $$ + select + s.declared_kind, + count(distinct s.actor_id) as actors, + coalesce(sum(s.events), 0) as events, + coalesce(sum(s.pageviews), 0) as pageviews, + coalesce(sum(s.contradictions), 0) as contradictions + from public.tracker_actor_daily_stats s + where s.project_id = p_project and s.day >= p_since + group by s.declared_kind; +$$; + +grant execute on function public.tracker_declared_totals(uuid, date) to authenticated, service_role; diff --git a/tests/tracker-declared-actors.test.ts b/tests/tracker-declared-actors.test.ts new file mode 100644 index 00000000..9245395e --- /dev/null +++ b/tests/tracker-declared-actors.test.ts @@ -0,0 +1,288 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { NextRequest } from "next/server"; +import { + ACTOR_HEADER, + DECLARED_AGENT_BUCKET, + actorTokenFrom, + applyDeclaration, + browserLabel, + clearActorCache, + hashActorToken, + isActorTokenShape, + mintActorToken, + normalizeEmail, + resolveActor, + toDeclaredTotals, +} from "@/lib/tracker/actors"; +import { hashApiToken } from "@/lib/sp/apiToken"; +import { rangeSinceDay } from "@/lib/tracker/actorStore"; +import { trackerRange } from "@/lib/tracker/ranges"; +import { renderStats } from "@/lib/dashboard/stats-text"; +import { actorTokenHowTo } from "@/cli/index"; + +// Declared actors are on the honor system, and others will try to game it. +// These pin the parts that make lying cheap to spot and useless to profit +// from: a declaration only ever moves traffic toward the bot side, a bare +// email claims nothing, and names stay private to their owner. + +const TOKEN = mintActorToken().plaintext; + +describe("applyDeclaration (the asymmetric trust rule)", () => { + it("believes an agent: a human-looking hit moves to bot:declared", () => { + expect(applyDeclaration("human:direct", "agent")).toEqual({ + bucket: DECLARED_AGENT_BUCKET, + kind: "bot", + contradiction: false, + }); + expect(applyDeclaration("search:google", "agent").kind).toBe("bot"); + }); + + it("leaves an already-detected bot's bucket alone when it declares agent", () => { + expect(applyDeclaration("bot:gptbot", "agent")).toEqual({ bucket: "bot:gptbot", kind: "bot", contradiction: false }); + }); + + it("never lets a declared human override bot detection, and flags it", () => { + expect(applyDeclaration("bot:gptbot", "human")).toEqual({ bucket: "bot:gptbot", kind: "bot", contradiction: true }); + expect(applyDeclaration("bot:scripted", "human").contradiction).toBe(true); + }); + + it("records a consistent human without changing anything", () => { + expect(applyDeclaration("human:direct", "human")).toEqual({ bucket: "human:direct", kind: "human", contradiction: false }); + }); + + it("is a no-op when undeclared", () => { + expect(applyDeclaration("referral:x.com", null)).toEqual({ bucket: "referral:x.com", kind: "human", contradiction: false }); + }); +}); + +describe("tokens", () => { + it("mints cpa_ tokens that pass the shape check and hash deterministically", () => { + const t = mintActorToken(); + expect(t.plaintext.startsWith("cpa_")).toBe(true); + expect(isActorTokenShape(t.plaintext)).toBe(true); + expect(t.prefix).toBe(t.plaintext.slice(0, 8)); + expect(hashActorToken(t.plaintext)).toBe(t.hash); + }); + + it("keeps actor and API token hashes apart under the same pepper", () => { + expect(hashActorToken(TOKEN)).not.toBe(hashApiToken(TOKEN)); + }); + + it("rejects an email, an API token and junk as an actor token", () => { + expect(isActorTokenShape("anthony@profullstack.com")).toBe(false); + expect(isActorTokenShape("crp_" + "a".repeat(43))).toBe(false); + expect(isActorTokenShape("cpa_short")).toBe(false); + expect(isActorTokenShape("cpa_" + "a".repeat(40) + "