From 5005f4a82e49bcdcb393d21ecb385a6fe95068ea Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 21 Sep 2026 18:31:25 +0000 Subject: [PATCH] =?UTF-8?q?tracker:=20count=20people,=20not=20beacons=20?= =?UTF-8?q?=E2=80=94=20visitor=20rollup=20+=20scripted=20cap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Human visits" was every beacon from a non-crawler user agent: the page view plus four scroll depths, every click and every form submit, three to four per page view. On four properties it read 51,531 in a week while the raw event table held 420 distinct visitor ids in a day, which is also what datafa.st reported for the same sites. Nothing outside the 24h raw table kept a visitor id, so weekly uniques could not be stated at all. - tracker_visitor_daily_stats: one row per (project, UTC day, visitor id) with event and pageview counts and which side of the human/bot line the visitor ended the day on. tracker_touch_visitor upserts it per beacon in one round trip and applies the scripted cap: more than 500 events or 200 page views from one visitor in a day flips it to bot for the day, and the ingest route counts every later beacon from it under bot:scripted. A stock-UA headless browser was the largest "human" source and nothing in the classifier could see it. - tracker_visitor_totals / tracker_visitor_daily_series: exact distinct visitors over a window (and the window before) and per day. - Stats page, /dashboard cards and /dashboard/analytics lead with "Human visitors" and "Page views" from the rollup; the old figure stays as "Human events" with a definition that says what it is. A failed rollup read leaves the tile out rather than showing 0. - /api/tracker/v1/stats totals: visitors and pageviews come from the rollup (null when unreadable, never 0), events is the old number. The CLI line prints all three. totalsFromSeries no longer returns beacons as visitors, so `crawlproof dashboard` cost-per-visitor is finally per visitor. Migration applied to production 2026-09-21 via the Supabase MCP; rows begin that day and the UI captions any window that reaches further back. Co-Authored-By: Claude Fable 5.1 --- app/(app)/dashboard/analytics/page.tsx | 77 ++++++- app/(app)/dashboard/page.tsx | 46 +++- .../dashboard/projects/[id]/stats/page.tsx | 48 ++-- app/api/track/route.ts | 42 +++- lib/dashboard/stats-text.ts | 10 +- lib/tracker/apiStats.ts | 55 ++++- lib/tracker/humans.ts | 22 +- lib/tracker/scripted.ts | 74 ++++++ lib/tracker/visitors.ts | 139 ++++++++++++ lib/tracker/who.ts | 63 +++++- .../20260921120000_tracker_visitor_rollup.sql | 213 +++++++++++++++++ tests/tracker-api-stats.test.ts | 18 +- tests/tracker-visitor-rollup.test.ts | 214 ++++++++++++++++++ 13 files changed, 951 insertions(+), 70 deletions(-) create mode 100644 lib/tracker/scripted.ts create mode 100644 lib/tracker/visitors.ts create mode 100644 supabase/migrations/20260921120000_tracker_visitor_rollup.sql create mode 100644 tests/tracker-visitor-rollup.test.ts diff --git a/app/(app)/dashboard/analytics/page.tsx b/app/(app)/dashboard/analytics/page.tsx index 4c8edad..7b7a649 100644 --- a/app/(app)/dashboard/analytics/page.tsx +++ b/app/(app)/dashboard/analytics/page.tsx @@ -30,8 +30,17 @@ import { BOTS_LABEL, HUMANS_DEFINITION, HUMANS_LABEL, + VISITORS_DEFINITION, + VISITORS_LABEL, humansFrom, } from "@/lib/tracker/humans"; +import { + VISITORS_CAPTION, + fetchVisitorTotals, + sumVisitorTotals, + visitorsPartial, + type VisitorTotals, +} from "@/lib/tracker/visitors"; import { TrackerAnalytics, type TrackerListItem, @@ -71,14 +80,18 @@ type PortfolioProject = { organization_id?: string | null; }; -// Every figure on this page leads with humans (bucket not `bot:`, AI referrals -// included) and shows bot crawls apart. See lib/tracker/humans.ts. +// Every figure on this page leads with people — distinct visitors from the +// visitor rollup — then human EVENTS (bucket not `bot:`, AI referrals +// included; several beacons per page view) and shows bot crawls apart. See +// lib/tracker/humans.ts and lib/tracker/visitors.ts. type ProjectRow = { project: PortfolioProject; totals: ProjectTotals; - /** Human visits, this window vs the one before. */ + /** Distinct human visitors this window vs the one before; null when the rollup is unavailable. */ + visitors: VisitorTotals | null; + /** Human events, this window vs the one before. */ trend: Trend; - /** Daily human visits for the sparkline; null when over the row budget. */ + /** Daily human events for the sparkline; null when over the row budget. */ samples: number[] | null; }; @@ -217,6 +230,22 @@ export default async function PortfolioAnalyticsPage({ const portfolio = sumTotals(totalsByProject.values()); const trends = totalsTrends(portfolio); + // People. One more RPC, kept apart from the eleven above because its + // failure means "no visitor figure", not "no traffic": the tile and the + // column are left out rather than zeroed. + const visitorsByProject = await fetchVisitorTotals(supabase, projectIds, days, "human"); + const portfolioVisitors = visitorsByProject + ? sumVisitorTotals(visitorsByProject.values()) + : null; + const visitorsTrend = portfolioVisitors + ? computeTrend(portfolioVisitors.visitors, portfolioVisitors.prevVisitors) + : null; + const visitorsCaption = !visitorsByProject + ? "Visitor counts are unavailable right now; the figures below count events." + : visitorsPartial(days) + ? VISITORS_CAPTION + : null; + // Ranked by current-window HUMAN volume: the biggest properties get the // chart bands, and the daily-detail budget is spent on them first. Ranking // by events would hand the top band to whichever site a crawler is hitting. @@ -264,6 +293,9 @@ export default async function PortfolioAnalyticsPage({ return { project, totals, + visitors: visitorsByProject + ? (visitorsByProject.get(project.id) ?? { visitors: 0, prevVisitors: 0, pageviews: 0, prevPageviews: 0 }) + : null, trend: computeTrend(totals.humans, totals.prevHumans), samples: byDay ? axis.map((day) => byDay.get(day) ?? 0) : null, }; @@ -444,11 +476,19 @@ export default async function PortfolioAnalyticsPage({

-
+
+ {visitorsTrend && ( + + )}
+ {visitorsCaption && ( +

{visitorsCaption}

+ )} {portfolio.events === 0 && portfolio.prevEvents === 0 ? (
@@ -485,13 +528,21 @@ export default async function PortfolioAnalyticsPage({

Portfolio trend

- Daily human visits, stacked by property. Bot crawls across + Daily human events, stacked by property. Bot crawls across every property are the dashed line, kept out of the stack.

+ {portfolioVisitors && ( + <> + + {portfolioVisitors.visitors.toLocaleString()} human visitors + + {" · "} + + )} - {portfolio.humans.toLocaleString()} human visits + {portfolio.humans.toLocaleString()} human events {" · "} @@ -633,8 +684,11 @@ function ProjectTrendTable({ rows }: { rows: ProjectRow[] }) { Property + + Visitors + - Humans + Events Previous @@ -650,7 +704,7 @@ function ProjectTrendTable({ rows }: { rows: ProjectRow[] }) { - {rows.map(({ project, totals, trend, samples }) => ( + {rows.map(({ project, totals, visitors, trend, samples }) => ( + + {visitors ? visitors.visitors.toLocaleString() : "—"} + {totals.humans.toLocaleString()} diff --git a/app/(app)/dashboard/page.tsx b/app/(app)/dashboard/page.tsx index 5bd7b40..2a541e4 100644 --- a/app/(app)/dashboard/page.tsx +++ b/app/(app)/dashboard/page.tsx @@ -3,7 +3,8 @@ import { createClient } from "@/lib/supabase/server"; import { rpcFailed, type Loaded } from "@/lib/loaded"; import { ScoreBadge } from "@/components/score-badge"; import { FontSparkline } from "@/components/font-sparkline"; -import { BOTS_DEFINITION, HUMANS_DEFINITION } from "@/lib/tracker/humans"; +import { BOTS_DEFINITION, HUMANS_DEFINITION, VISITORS_DEFINITION } from "@/lib/tracker/humans"; +import { fetchVisitorDailySeries } from "@/lib/tracker/visitors"; import { ProjectLogo } from "@/components/project-logo"; import { StatsUnavailable } from "@/components/stats-unavailable"; import { backfillProjectLogo } from "@/app/actions/createProject"; @@ -121,11 +122,12 @@ export default async function DashboardPage({ // project has an lx_site row in status=active; social is "on" when at // least one social account is linked at the project level. const projectIds = (projects ?? []).map((p) => p.id); - const [autoblogIds, socialIds, latestPosts, traffic] = await Promise.all([ + const [autoblogIds, socialIds, latestPosts, traffic, visitorsByProject] = await Promise.all([ fetchEnabledProjectIds(supabase, "lx_site", projectIds, { status: "active" }), fetchEnabledProjectIds(supabase, "sp_site_account", projectIds), fetchLatestBlogPostByProject(supabase, projectIds), fetchSevenDayTraffic(supabase, projectIds), + fetchSevenDayVisitors(supabase, projectIds), ]); const trafficByProject = traffic.data; const trafficFailed = traffic.failed; @@ -266,23 +268,27 @@ export default async function DashboardPage({
{trafficFailed ? "Traffic unavailable" - : `${totalHumans(trafficByProject.get(p.id) ?? []).toLocaleString()} human visits`} + : visitorsByProject + ? `${sum(visitorsByProject.get(p.id)).toLocaleString()} human visitors` + : `${totalHumans(trafficByProject.get(p.id) ?? []).toLocaleString()} human events`}
{trafficFailed ? "Query failed \u2014 not zero" - : `${totalBots(trafficByProject.get(p.id) ?? []).toLocaleString()} bot hits \u00b7 Past 7 days`} + : `${visitorsByProject ? `${totalHumans(trafficByProject.get(p.id) ?? []).toLocaleString()} human events \u00b7 ` : ""}${totalBots(trafficByProject.get(p.id) ?? []).toLocaleString()} bot hits \u00b7 Past 7 days`}
{!trafficFailed && ( - + )}
{orgSchemaReady && ( @@ -436,6 +442,32 @@ async function fetchSevenDayTraffic( return { data: out, failed: false }; } +// Distinct human visitors per day for the past seven UTC days, per project: +// the number the card leads with. The visitor rollup only began on +// lib/tracker/visitors.ts VISITORS_SINCE, and it counts people where +// dashboard_project_traffic counts beacons (several per page view), which +// is why the two figures on a card differ by a hundredfold and both are +// shown. null when the RPC failed or is not deployed: the card then leads +// with events under their real name rather than showing "0 visitors". +async function fetchSevenDayVisitors( + supabase: Awaited>, + projectIds: string[], +): Promise | null> { + const days = lastSevenDays(); + const byProject = await fetchVisitorDailySeries(supabase, projectIds, 7, "human"); + if (!byProject) return null; + const out = new Map(); + for (const projectId of projectIds) { + const byDay = byProject.get(projectId); + out.set(projectId, days.map((day) => byDay?.get(day) ?? 0)); + } + return out; +} + +function sum(samples: number[] | undefined) { + return (samples ?? []).reduce((total, n) => total + n, 0); +} + function lastSevenDays() { return Array.from({ length: 7 }, (_, index) => { const date = new Date(); diff --git a/app/(app)/dashboard/projects/[id]/stats/page.tsx b/app/(app)/dashboard/projects/[id]/stats/page.tsx index c9eb177..8472824 100644 --- a/app/(app)/dashboard/projects/[id]/stats/page.tsx +++ b/app/(app)/dashboard/projects/[id]/stats/page.tsx @@ -8,9 +8,14 @@ import { TrackerAnalytics, type TrackerPanels, } from "@/components/charts/tracker-analytics"; -import { fetchPanels, PANEL_KEYS } from "@/lib/tracker/panels"; +import { fetchPanels, PANEL_KEYS, resolveDays, rollupDays } from "@/lib/tracker/panels"; import { DEFAULT_TRACKER_RANGE, trackerRange } from "@/lib/tracker/ranges"; import { headlineTiles, whoOrDefault, whoToKind } from "@/lib/tracker/who"; +import { + VISITORS_CAPTION, + fetchVisitorTotals, + visitorsPartial, +} from "@/lib/tracker/visitors"; import { InstallSnippet } from "./install-snippet"; import { TrackerToggle } from "./tracker-toggle"; import { CareersToggle } from "./careers-toggle"; @@ -65,31 +70,41 @@ export default async function ProjectStatsPage({ // Every panel is fetched at the toggle's `kind`, so a card and the tiles // above it never disagree on who they are counting. const range = trackerRange(DEFAULT_TRACKER_RANGE); - const panels = (await fetchPanels( - supabase, - id, - PANEL_KEYS, - range, - kind, - )) as unknown as TrackerPanels; + // The visitor rollup is day-resolution, so a sub-day range reads today's + // rollup — the same rule the device and exit-page cards apply. + const visitorDays = rollupDays(range, await resolveDays(supabase, id, range)); + const [panels, visitorTotals] = await Promise.all([ + fetchPanels(supabase, id, PANEL_KEYS, range, kind) as unknown as Promise, + fetchVisitorTotals(supabase, [id], visitorDays, kind), + ]); - // Headline metrics come straight from the series so they stay exact even - // though Top sources below is truncated to the top 10 buckets. The page - // leads with humans (every bucket that is not `bot:`, AI referrals - // included) and shows bot crawls apart — see lib/tracker/humans.ts for why - // the old bot-inclusive total is no longer a headline. Under Humans or - // Bots only that side's tiles are shown (lib/tracker/who.ts). + // Headline metrics: the people first. Distinct visitors and their page + // views come from the visitor rollup (lib/tracker/visitors.ts); the event + // figures come straight from the series so they stay exact even though Top + // sources below is truncated to the top 10 buckets. The event count used to + // be labelled "Human visits" and ran ~100x above the number of people — it + // is stats.js firing several beacons per page view — so it now sits after + // the visitor tiles under its real name. Under Humans or Bots only that + // side's tiles are shown (lib/tracker/who.ts). const points = panels.series.points; const totalAi = points.reduce((s, p) => s + p.ai, 0); const totalBot = points.reduce((s, p) => s + p.bots, 0); const totalHuman = points.reduce((s, p) => s + p.humans, 0); const grandTotal = points.reduce((s, p) => s + p.events, 0); const eventTotal = points.reduce((s, p) => s + p.pageviews + p.interactions, 0); + const visitors = visitorTotals?.get(id) ?? (visitorTotals ? { visitors: 0, pageviews: 0 } : null); const tiles = headlineTiles(who, { + visitors: visitors ? visitors.visitors : null, + pageviews: visitors ? visitors.pageviews : null, humans: totalHuman, ai: totalAi, bots: totalBot, }); + const visitorsCaption = visitors === null + ? "Visitor counts are unavailable right now; the figures below count events." + : visitorsPartial(visitorDays) + ? VISITORS_CAPTION + : null; // Older projects have rollup rows in tracker_daily_stats but nothing in // tracker_event_daily_stats, which would leave Event mix empty on a page @@ -265,7 +280,7 @@ export default async function ProjectStatsPage({ -
+
{tiles.map((tile) => ( ))}
+ {visitorsCaption && ( +

{visitorsCaption}

+ )} {grandTotal === 0 && eventTotal === 0 ? (
diff --git a/app/api/track/route.ts b/app/api/track/route.ts index 0b104a9..694d12d 100644 --- a/app/api/track/route.ts +++ b/app/api/track/route.ts @@ -7,6 +7,13 @@ import { z } from "zod"; import { serviceClient } from "@/lib/supabase/service"; import { categorize } from "@/lib/tracker/categorize"; import { kindFromBucket } from "@/lib/tracker/humans"; +import { + SCRIPTED_CAP_EVENTS, + SCRIPTED_CAP_PAGEVIEWS, + applyScriptedDemotion, + parseVisitorTouch, + type VisitorTouch, +} from "@/lib/tracker/scripted"; import { parseDevice } from "@/lib/tracker/device"; import { clientIpFromHeaders, lookupGeo } from "@/lib/tracker/geo"; import { enqueuePostHogEvent } from "@/lib/posthog/events"; @@ -182,12 +189,41 @@ async function ingest(request: NextRequest, parseBody: boolean) { const gate = await gateAgent(sb, site, userAgent); if (gate.action !== "allow") return refuse(gate); - const { bucket, isAi } = categorize({ referrer, userAgent, url: pageUrl }); + const categorized = categorize({ referrer, userAgent, url: pageUrl }); + const today = new Date().toISOString().slice(0, 10); // YYYY-MM-DD UTC + + // Visitor rollup: one row per (project, day, visitor id), bumped per beacon. + // It is the only place the tracker counts people rather than events, and + // its answer can demote this hit: a visitor past the scripted cap for the + // day is a driven browser, not a reader, whatever its user agent claims + // (lib/tracker/scripted.ts). Read before the first counter write so every + // rollup below lands on the same side. Best-effort: no visitor id, or a + // failed RPC, leaves the user-agent verdict standing. + const visitorId = textOrNull(parsed.data.visitorId); + let touch: VisitorTouch | null = null; + if (visitorId) { + try { + const { data } = await sb.rpc("tracker_touch_visitor", { + p_project: site, + p_day: today, + p_visitor: visitorId.slice(0, 128), + p_kind: kindFromBucket(categorized.bucket), + p_pageview: event === "pageview", + p_cap_events: SCRIPTED_CAP_EVENTS, + p_cap_pageviews: SCRIPTED_CAP_PAGEVIEWS, + }); + touch = parseVisitorTouch(data); + } catch { + // Silent — the beacon must never fail closed on a counter table. + } + } + const demotion = applyScriptedDemotion(categorized.bucket, touch); + const bucket = demotion.bucket; + const isAi = demotion.demoted ? false : categorized.isAi; // Which side of the human / bot line this hit counts on. The bucket table // carries the whole bucket; the other rollups record only this, so the // stats page can split every breakdown, not just the headline. - const kind = kindFromBucket(bucket); - const today = new Date().toISOString().slice(0, 10); // YYYY-MM-DD UTC + const kind = demotion.kind; // UPSERT increment. Supabase JS doesn't expose a raw .increment() helper // so we read + write under the unique key. The PK protects against diff --git a/lib/dashboard/stats-text.ts b/lib/dashboard/stats-text.ts index 2fdbfbd..fb90f04 100644 --- a/lib/dashboard/stats-text.ts +++ b/lib/dashboard/stats-text.ts @@ -9,7 +9,7 @@ export type StatsItem = { label: string; value: number }; export type StatsAnswerish = { project?: { name?: string; url?: string } | null; - totals?: { visitors?: number; pageviews?: number } | null; + totals?: { visitors?: number | null; pageviews?: number | null; events?: number } | null; sources?: StatsItem[] | null; referrers?: StatsItem[] | null; pages?: StatsItem[] | null; @@ -24,7 +24,11 @@ export function renderStats( const totals = answer.totals ?? {}; const out: string[] = [ `${answer.project?.name ?? "project"} ${range} ${who}`, - `${totals.visitors ?? 0} visitors, ${totals.pageviews ?? 0} pageviews`, + // People first. A null visitor count is the rollup being unreadable, not + // an empty site, and is printed as such rather than as 0. + totals.visitors === null || totals.visitors === undefined + ? `visitors unavailable, ${totals.events ?? 0} events` + : `${totals.visitors} visitors, ${totals.pageviews ?? 0} pageviews, ${totals.events ?? 0} events`, ]; const section = (title: string, items: StatsItem[]) => { @@ -40,7 +44,7 @@ export function renderStats( section("Pages", list(answer.pages)); // Nothing at all is a real answer, and the likeliest cause is worth naming. - if (!(totals.pageviews ?? 0) && !list(answer.sources).length) { + if (!(totals.pageviews ?? 0) && !(totals.events ?? 0) && !list(answer.sources).length) { out.push("", "Nothing in this window. Check the tag is on the page, or widen --range."); } return `${out.join("\n")}\n`; diff --git a/lib/tracker/apiStats.ts b/lib/tracker/apiStats.ts index b475ed4..5403365 100644 --- a/lib/tracker/apiStats.ts +++ b/lib/tracker/apiStats.ts @@ -20,6 +20,8 @@ import { } from "@/lib/tracker/panels"; import type { TrackerRange } from "@/lib/tracker/ranges"; import type { TrackerKind } from "@/lib/tracker/humans"; +import { fetchVisitorTotals } from "@/lib/tracker/visitors"; +import { rollupDays } from "@/lib/tracker/panels"; type Sb = SupabaseClient; @@ -108,7 +110,16 @@ export type StatsAnswer = { project: { id: string; name: string; url: string }; range: string; who: string; - totals: { visitors: number; pageviews: number }; + /** + * `visitors` is distinct visitor ids over the window from the visitor + * rollup (people); `pageviews` is the page views those visitors produced; + * `events` is every beacon on the requested side, which is what `visitors` + * used to be filled with before the rollup existed (and read ~100x too + * high). `visitors` and `pageviews` are null when the rollup could not be + * read, never 0: a caller computing cost per visitor must not divide by a + * failure. + */ + totals: { visitors: number | null; pageviews: number | null; events: number }; sources: ListItem[]; referrers: ListItem[]; pages: ListItem[]; @@ -167,17 +178,35 @@ export function mixFromSeries(payload: PanelPayload | undefined): { return mix; } -/** Sum a series payload's points into the two numbers a summary line needs. */ -export function totalsFromSeries(payload: PanelPayload | undefined): { visitors: number; pageviews: number } { - if (!payload || Array.isArray(payload)) return { visitors: 0, pageviews: 0 }; +/** + * Sum a series payload's points into the event-side numbers. `events` is the + * human (or requested-side) beacon count; `pageviews` here is the + * bot-inclusive pageview leg of the series and is only a fallback for a + * caller with no visitor rollup. Never called `visitors`: a series point has + * no visitor field and the old fallback to `humans` is the 100x bug. + */ +export function totalsFromSeries(payload: PanelPayload | undefined): { events: number; pageviews: number } { + if (!payload || Array.isArray(payload)) return { events: 0, pageviews: 0 }; const points = (payload as { points?: Record[] }).points ?? []; - let visitors = 0; + let events = 0; let pageviews = 0; for (const point of points) { - visitors += Number(point.visitors ?? point.humans ?? 0) || 0; + events += Number(point.humans ?? 0) || 0; pageviews += Number(point.pageviews ?? 0) || 0; } - return { visitors, pageviews }; + return { events, pageviews }; +} + +/** The totals block: people from the rollup, events from the series. */ +export function mergeTotals( + fromSeries: { events: number; pageviews: number }, + visitors: { visitors: number; pageviews: number } | null | undefined, +): StatsAnswer["totals"] { + return { + visitors: visitors ? visitors.visitors : null, + pageviews: visitors ? visitors.pageviews : null, + events: fromSeries.events, + }; } export async function projectStats( @@ -189,20 +218,26 @@ export async function projectStats( /** Add the series and the unfiltered human / bot mix. One extra RPC at most. */ detail = false, ): Promise { - const [panels, mixSeries] = await Promise.all([ + const days = await resolveDays(sb, project.id, range); + const [panels, mixSeries, visitorTotals] = await Promise.all([ fetchPanels(sb, project.id, STATS_PANELS, range, kind), // Only when the answer is filtered: at kind null the main series already is // the unfiltered one, and a second identical query would be a second query. detail && kind !== null - ? fetchPanel(sb, project.id, "series", range, await resolveDays(sb, project.id, range), null) + ? fetchPanel(sb, project.id, "series", range, days, null) : Promise.resolve(undefined), + // People. The rollup is day-resolution, so a sub-day range reads today's. + fetchVisitorTotals(sb, [project.id], rollupDays(range, days), kind), ]); const answer: StatsAnswer = { project: { id: project.id, name: project.name, url: project.url }, range: range.key, who, - totals: totalsFromSeries(panels.series), + totals: mergeTotals( + totalsFromSeries(panels.series), + visitorTotals ? (visitorTotals.get(project.id) ?? { visitors: 0, pageviews: 0 }) : null, + ), sources: asList(panels.sources), referrers: asList(panels.referrers), pages: asList(panels.pages), diff --git a/lib/tracker/humans.ts b/lib/tracker/humans.ts index 0bb58e2..a9554e0 100644 --- a/lib/tracker/humans.ts +++ b/lib/tracker/humans.ts @@ -13,14 +13,30 @@ // bot = bucket starts with "bot:" (named AI crawlers + bot:other) // so humans + bots = events, exactly. -export const HUMANS_LABEL = "Human visits"; +// Two human figures, and the names keep them apart: +// VISITORS — distinct visitor ids per day (tracker_visitor_daily_stats), +// the number of PEOPLE. This is the headline. +// HUMANS — every beacon from a non-crawler (tracker_daily_stats), i.e. +// EVENTS: the page view plus each scroll depth, click and form +// submit stats.js fires. Three to four per page view. It used to +// be labelled "Human visits" and read 100x above the number of +// people; it keeps its name in code and loses it in the UI. +export const VISITORS_LABEL = "Human visitors"; +export const HUMANS_LABEL = "Human events"; export const BOTS_LABEL = "Bot crawls"; +export const PAGEVIEWS_LABEL = "Page views"; + +export const VISITORS_DEFINITION = + "Distinct people, counted once per site per day by the visitor id the tracker stores in the browser. Crawlers and scripted browsers are excluded."; export const HUMANS_DEFINITION = - "Everything not identified as a crawler, including visits referred by AI assistants such as ChatGPT or Perplexity."; + "Every tracked event from a visitor not identified as a crawler: the page view plus each scroll depth, click and form submit. Several per page view, so this is not a count of people. Includes visits referred by AI assistants such as ChatGPT or Perplexity."; + +export const PAGEVIEWS_DEFINITION = + "Pages rendered for people: one per page load from a visitor not identified as a crawler or a scripted browser."; export const BOTS_DEFINITION = - "Hits from user agents identified as crawlers: AI training and retrieval bots, search engine bots, and other automated clients. Not people."; + "Hits from user agents identified as crawlers (AI training and retrieval bots, search engine bots, other automated clients) plus scripted browsers caught by volume. Not people."; export const AI_REFERRALS_DEFINITION = "People who arrived from an AI assistant. These are already counted inside human visits."; diff --git a/lib/tracker/scripted.ts b/lib/tracker/scripted.ts new file mode 100644 index 0000000..1d9a20d --- /dev/null +++ b/lib/tracker/scripted.ts @@ -0,0 +1,74 @@ +// The scripted-visitor cap: the one rule that turns a browser being driven +// into a bot, applied per beacon by the ingest route. +// +// WHY: lib/tracker/categorize.ts can only call a hit a bot when the user +// agent says so. A headless Chrome with a stock UA is "human" by definition, +// and one was measured minting a fresh visitor id per page view (19k +// "visitors" on 19k page views in a day), another produced 398 events on 4 +// page views, another 126 page views from one id. No person does that. The +// tracker_visitor_daily_stats rollup keeps per-visitor, per-day counts, so +// volume is a signal the classifier finally has. +// +// The caps are per visitor per UTC day and deliberately generous: a binge +// reader on a search-heavy site can reach 100 page views, and the cost of a +// false demotion (one real person counted as a bot for a day) is the same +// size as the cost of a miss (one script counted as a person for a day). The +// point is the tail, not the median. +// +// Mirrored by the defaults of tracker_touch_visitor in +// supabase/migrations/20260921120000_tracker_visitor_rollup.sql. + +import { kindFromBucket, type TrackerKind } from "@/lib/tracker/humans"; + +/** More beacons than this from one visitor in one UTC day is a script. */ +export const SCRIPTED_CAP_EVENTS = 500; +/** More page views than this from one visitor in one UTC day is a script. */ +export const SCRIPTED_CAP_PAGEVIEWS = 200; + +/** The bucket a demoted hit is counted under. Renders as "Bot · scripted". */ +export const SCRIPTED_BUCKET = "bot:scripted"; + +/** What tracker_touch_visitor hands back after the increment. */ +export type VisitorTouch = { + kind: TrackerKind; + events: number; + pageviews: number; +}; + +/** + * The bucket and kind a hit should be counted under once the visitor rollup + * has had its say. A hit the user agent already made a bot is left alone. A + * hit from a visitor the rollup has flipped to `bot` — by this beacon or an + * earlier one today — is counted under bot:scripted, so every rollup the + * route writes next lands on the bot side. Without a touch (no visitor id, + * or the RPC failed) the bucket stands: the beacon must never fail closed + * on a counter table hiccup. + */ +export function applyScriptedDemotion( + bucket: string, + touch: VisitorTouch | null, +): { bucket: string; kind: TrackerKind; demoted: boolean } { + const kind = kindFromBucket(bucket); + if (kind === "bot" || !touch || touch.kind !== "bot") { + return { bucket, kind, demoted: false }; + } + return { bucket: SCRIPTED_BUCKET, kind: "bot", demoted: true }; +} + +/** Coerce one RPC row; null when the shape is not what the migration returns. */ +export function parseVisitorTouch(row: unknown): VisitorTouch | null { + const r = (Array.isArray(row) ? row[0] : row) as + | { kind?: unknown; events?: unknown; pageviews?: unknown } + | null + | undefined; + if (!r || typeof r !== "object") return null; + const kind = r.kind === "bot" ? "bot" : r.kind === "human" ? "human" : null; + if (!kind) return null; + const events = Number(r.events); + const pageviews = Number(r.pageviews); + return { + kind, + events: Number.isFinite(events) ? events : 0, + pageviews: Number.isFinite(pageviews) ? pageviews : 0, + }; +} diff --git a/lib/tracker/visitors.ts b/lib/tracker/visitors.ts new file mode 100644 index 0000000..22cb434 --- /dev/null +++ b/lib/tracker/visitors.ts @@ -0,0 +1,139 @@ +// Readers for the visitor rollup (tracker_visitor_daily_stats), the one place +// the tracker counts PEOPLE rather than beacons. +// +// WHY: the bucket rollup is bumped per event and stats.js fires several +// events per page view, so "Human visits" ran 100x above the number of +// people. Every headline now leads with distinct visitors from this rollup +// and shows the event count beside it under its real name. +// +// Every reader returns null when the RPC is missing or fails, and the pages +// render "unavailable" for that figure rather than 0: a zero here would be +// read as a dead site, which is the misreading the whole split exists to +// prevent. Rows begin on VISITORS_SINCE (there is no history to backfill: +// tracker_events keeps 24h), so a window that reaches further back is +// partial and the UI says so. + +import type { SupabaseClient } from "@supabase/supabase-js"; +import type { TrackerKind } from "@/lib/tracker/humans"; + +type Sb = SupabaseClient; + +/** The UTC day the rollup migration was applied to production. */ +export const VISITORS_SINCE = "2026-09-21"; +export const VISITORS_SINCE_LABEL = "21 Sep 2026"; +export const VISITORS_CAPTION = `Visitors counted from ${VISITORS_SINCE_LABEL}; earlier traffic has no visitor count.`; + +export type VisitorTotals = { + /** Distinct visitor ids in the window. */ + visitors: number; + prevVisitors: number; + /** Page views those visitors produced. */ + pageviews: number; + prevPageviews: number; +}; + +export function emptyVisitorTotals(): VisitorTotals { + return { visitors: 0, prevVisitors: 0, pageviews: 0, prevPageviews: 0 }; +} + +const num = (v: unknown) => { + const n = Number(v); + return Number.isFinite(n) ? n : 0; +}; + +/** One row of tracker_visitor_totals; bigint columns arrive as strings. */ +export type VisitorTotalsRow = { + project_id: string; + visitors: number | string; + prev_visitors: number | string; + pageviews: number | string; + prev_pageviews: number | string; +}; + +export function toVisitorTotals(row: VisitorTotalsRow): VisitorTotals { + return { + visitors: num(row.visitors), + prevVisitors: num(row.prev_visitors), + pageviews: num(row.pageviews), + prevPageviews: num(row.prev_pageviews), + }; +} + +export function sumVisitorTotals(all: Iterable): VisitorTotals { + const out = emptyVisitorTotals(); + for (const t of all) { + out.visitors += t.visitors; + out.prevVisitors += t.prevVisitors; + out.pageviews += t.pageviews; + out.prevPageviews += t.prevPageviews; + } + return out; +} + +/** + * Distinct visitors per project over the last `days` UTC days and the equal + * window before. Projects with no rows are absent from the map (0 visitors). + * null when the RPC failed or does not exist yet. + */ +export async function fetchVisitorTotals( + sb: Sb, + projectIds: string[], + days: number, + kind: TrackerKind | null = "human", +): Promise | null> { + const out = new Map(); + if (projectIds.length === 0) return out; + const { data, error } = await sb.rpc("tracker_visitor_totals", { + p_projects: projectIds, + days: Math.max(1, days), + p_kind: kind, + }); + if (error) return null; + for (const row of (data ?? []) as VisitorTotalsRow[]) { + out.set(row.project_id, toVisitorTotals(row)); + } + return out; +} + +export type VisitorDayRow = { + project_id: string; + day: string; + visitors: number | string; + pageviews: number | string; +}; + +/** + * Visitors per (project, UTC day) over the last `days` days, as + * project -> day -> visitors. Days with no rows are absent. null on failure. + */ +export async function fetchVisitorDailySeries( + sb: Sb, + projectIds: string[], + days: number, + kind: TrackerKind | null = "human", +): Promise> | null> { + const out = new Map>(); + for (const id of projectIds) out.set(id, new Map()); + if (projectIds.length === 0) return out; + const { data, error } = await sb.rpc("tracker_visitor_daily_series", { + p_projects: projectIds, + days: Math.max(1, days), + p_kind: kind, + }); + if (error) return null; + for (const row of (data ?? []) as VisitorDayRow[]) { + out.get(row.project_id)?.set(row.day, num(row.visitors)); + } + return out; +} + +/** + * Whether a window of `days` days ending today reaches back before the rollup + * existed, in which case its visitor figure is partial and needs the caption. + */ +export function visitorsPartial(days: number, now = new Date()): boolean { + const start = new Date(now); + start.setUTCHours(0, 0, 0, 0); + start.setUTCDate(start.getUTCDate() - Math.max(1, days) + 1); + return start.toISOString().slice(0, 10) < VISITORS_SINCE; +} diff --git a/lib/tracker/who.ts b/lib/tracker/who.ts index 63d44e9..1833170 100644 --- a/lib/tracker/who.ts +++ b/lib/tracker/who.ts @@ -17,6 +17,10 @@ import { BOTS_LABEL, HUMANS_DEFINITION, HUMANS_LABEL, + PAGEVIEWS_DEFINITION, + PAGEVIEWS_LABEL, + VISITORS_DEFINITION, + VISITORS_LABEL, type TrackerKind, } from "@/lib/tracker/humans"; @@ -78,31 +82,66 @@ export function whoCaption(who: Who): string | null { } export type HeadlineTotals = { + /** + * Distinct visitors in the window from the visitor rollup, on the side + * `who` asks for. null when the rollup could not be read: the tile is then + * left out rather than shown as 0, because 0 reads as a dead site. + */ + visitors?: number | null; + /** Page views those visitors produced; drawn only beside `visitors`. */ + pageviews?: number | null; + /** Events, not people: every beacon on the human side. */ humans: number; ai: number; bots: number; }; export type HeadlineTile = { - key: "humans" | "ai" | "bots"; + key: "visitors" | "pageviews" | "humans" | "ai" | "bots"; label: string; value: number; - tone: "accent" | "pass" | "warn"; + tone: "accent" | "pass" | "warn" | "muted"; hint: string; }; /** - * Which headline tiles the page shows for a given toggle. Humans and Bots - * show their own side's figures; All keeps the three-tile layout the page - * shipped with. The figures come from the series the page fetched at that - * same `who`, so a filtered view never mixes in the other side's count. + * Which headline tiles the page shows for a given toggle. Humans leads with + * people (distinct visitors, then their page views) and shows the event + * count under its real name after them; Bots shows its own side; All keeps + * both sides. The figures come from the series and the visitor rollup the + * page fetched at that same `who`, so a filtered view never mixes in the + * other side's count. */ export function headlineTiles(who: Who, t: HeadlineTotals): HeadlineTile[] { + const people: HeadlineTile[] = + t.visitors === null || t.visitors === undefined + ? [] + : [ + { + key: "visitors", + label: who === "bots" ? "Bot visitors" : VISITORS_LABEL, + value: t.visitors, + tone: who === "bots" ? "warn" : "accent", + hint: + who === "bots" + ? "Distinct visitor ids on the bot side: crawlers that ran the script, and scripted browsers caught by volume." + : VISITORS_DEFINITION, + }, + { + key: "pageviews", + label: PAGEVIEWS_LABEL, + value: t.pageviews ?? 0, + tone: who === "bots" ? "warn" : "accent", + hint: PAGEVIEWS_DEFINITION, + }, + ]; const humans: HeadlineTile = { key: "humans", label: HUMANS_LABEL, value: t.humans, - tone: "accent", + // Leads only when the rollup has nothing to say; otherwise it is the + // number people used to mistake for readers, kept but demoted. + tone: people.length ? "muted" : "accent", hint: HUMANS_DEFINITION, }; const ai: HeadlineTile = { @@ -121,11 +160,11 @@ export function headlineTiles(who: Who, t: HeadlineTotals): HeadlineTile[] { }; switch (who) { case "humans": - return [humans, ai]; + return [...people, humans, ai]; case "bots": - return [bots]; + return [...people, bots]; default: - return [humans, ai, bots]; + return [...people, humans, ai, bots]; } } @@ -141,7 +180,7 @@ export type PulseLayer = { const HUMANS_LAYER: PulseLayer = { dataKey: "humans", - name: "Human visits", + name: "Human events", stackId: "1", color: "var(--color-accent)", fillOpacity: 0.28, @@ -198,7 +237,7 @@ export function pulseHeadline( } return { total: totals.humans, - unit: ["human visit", "human visits"], + unit: ["human event", "human events"], hint: HUMANS_DEFINITION, }; } diff --git a/supabase/migrations/20260921120000_tracker_visitor_rollup.sql b/supabase/migrations/20260921120000_tracker_visitor_rollup.sql new file mode 100644 index 0000000..57c079e --- /dev/null +++ b/supabase/migrations/20260921120000_tracker_visitor_rollup.sql @@ -0,0 +1,213 @@ +-- Visitors: the tracker learns to count people. +-- +-- WHY: every headline the dashboards led with was a count of EVENTS. The +-- bucket rollup (tracker_daily_stats) is bumped once per beacon, and stats.js +-- fires a beacon for the pageview, four scroll depths, every click and every +-- form submit, so "Human visits" was ~3-4 events per page view from anything +-- whose user agent did not say bot. On four properties that read as 51,531 +-- "human visits" in a week while the raw event table held 420 distinct +-- visitor ids in a day, which is also what an independent analytics tool +-- reported. Nothing outside the 24h raw table kept a visitor id, so the +-- product could not state weekly unique visitors at all. +-- +-- WHAT: +-- 1. tracker_visitor_daily_stats — one row per (project, UTC day, visitor), +-- with the event and pageview counts for that visitor that day and which +-- side of the human / bot line the visitor ended the day on. Row count is +-- visitors x days, not events, so it keeps. +-- 2. tracker_touch_visitor — the per-beacon upsert. Returns the visitor's +-- counts after the increment so the ingest route can make one round +-- trip. It also applies the SCRIPTED cap: a "visitor" that produces more +-- than p_cap_events events or p_cap_pageviews page views in one UTC day +-- is not a person reading a website, it is a browser being driven +-- (the ones seen so far: 398 events on 4 page views, 126 page views from +-- one id, ~1 fresh id per page view from a headless Chrome). The row is +-- flipped to kind = 'bot' and stays there for the day; the route then +-- counts every later beacon from it under bot:scripted. +-- 3. tracker_visitor_totals — distinct visitors and their page views over a +-- window and the equal-length window before it, per project. Exact +-- uniques, because the ids are here. +-- 4. tracker_visitor_daily_series — visitors per (project, day), for the +-- dashboard sparklines and the stats page. +-- +-- ROWS BEGIN THE DAY THIS IS APPLIED. There is no history to backfill from: +-- tracker_events keeps 24h. The UI says so beside every visitor figure. +-- +-- security invoker throughout; RLS mirrors tracker_daily_stats (owner or +-- member may read). The ingest route writes with the service role. +-- Idempotent. Apply one file at a time via the Supabase MCP, not `db push`. + +-- --------------------------------------------------------------------------- +-- 1. Table +-- --------------------------------------------------------------------------- +create table if not exists public.tracker_visitor_daily_stats ( + project_id uuid not null references public.projects(id) on delete cascade, + day date not null, + visitor_id text not null, + kind text not null default 'human' check (kind in ('human', 'bot')), + events integer not null default 0, + pageviews integer not null default 0, + first_seen timestamptz not null default now(), + last_seen timestamptz not null default now(), + primary key (project_id, day, visitor_id) +); + +-- The read RPCs count rows per (project, day, kind); the PK prefix covers +-- (project, day) but the filter on kind and the pageviews sum need this. +create index if not exists tracker_visitor_daily_stats_project_day_kind_idx + on public.tracker_visitor_daily_stats (project_id, day, kind) + include (pageviews); + +alter table public.tracker_visitor_daily_stats enable row level security; + +do $$ +begin + if not exists ( + select 1 from pg_policies + where schemaname = 'public' + and tablename = 'tracker_visitor_daily_stats' + and policyname = 'tracker_visitor_daily_stats owner select' + ) then + create policy "tracker_visitor_daily_stats owner select" + on public.tracker_visitor_daily_stats + for select + using ( + project_id in (select id from public.projects where owner_id = auth.uid()) + ); + end if; + + if not exists ( + select 1 from pg_policies + where schemaname = 'public' + and tablename = 'tracker_visitor_daily_stats' + and policyname = 'tracker_visitor_daily_stats member select' + ) then + create policy "tracker_visitor_daily_stats member select" + on public.tracker_visitor_daily_stats + for select + using (public.is_project_member(project_id, auth.uid())); + end if; +end $$; + +grant select on public.tracker_visitor_daily_stats to authenticated; +grant select, insert, update, delete on public.tracker_visitor_daily_stats to service_role; + +-- --------------------------------------------------------------------------- +-- 2. Per-beacon touch (+ scripted cap) +-- --------------------------------------------------------------------------- +create or replace function public.tracker_touch_visitor( + p_project uuid, + p_day date, + p_visitor text, + p_kind text, + p_pageview boolean, + p_cap_events integer default 500, + p_cap_pageviews integer default 200 +) +returns table (kind text, events integer, pageviews integer) +language sql +volatile +security invoker +set search_path = public +as $$ + insert into public.tracker_visitor_daily_stats as s + (project_id, day, visitor_id, kind, events, pageviews, first_seen, last_seen) + values + (p_project, p_day, p_visitor, + -- A single beacon can already blow the cap when the cap is 0; keep the + -- rule in one place by evaluating it on the insert too. + case + when p_kind = 'bot' then 'bot' + when 1 > coalesce(p_cap_events, 500) then 'bot' + when (case when p_pageview then 1 else 0 end) > coalesce(p_cap_pageviews, 200) then 'bot' + else 'human' + end, + 1, (case when p_pageview then 1 else 0 end), now(), now()) + on conflict (project_id, day, visitor_id) do update + set events = s.events + 1, + pageviews = s.pageviews + (case when p_pageview then 1 else 0 end), + last_seen = now(), + -- Sticky: once a bot (by user agent or by volume), a bot for the day. + kind = case + when s.kind = 'bot' or p_kind = 'bot' then 'bot' + when s.events + 1 > coalesce(p_cap_events, 500) then 'bot' + when s.pageviews + (case when p_pageview then 1 else 0 end) > coalesce(p_cap_pageviews, 200) then 'bot' + else 'human' + end + returning s.kind, s.events, s.pageviews; +$$; + +revoke all on function public.tracker_touch_visitor(uuid, date, text, text, boolean, integer, integer) from public; +grant execute on function public.tracker_touch_visitor(uuid, date, text, text, boolean, integer, integer) to service_role; + +-- --------------------------------------------------------------------------- +-- 3. Window totals: distinct visitors + their page views, current and previous +-- --------------------------------------------------------------------------- +create or replace function public.tracker_visitor_totals( + p_projects uuid[], + days integer default 30, + p_kind text default null +) +returns table ( + project_id uuid, + visitors bigint, + prev_visitors bigint, + pageviews bigint, + prev_pageviews bigint +) +language sql +stable +security invoker +set search_path = public +as $$ + with bounds as ( + select (today - (n - 1)) as cur_start, + (today - (2 * n - 1)) as prev_start, + (today - n) as prev_end + from ( + select greatest(coalesce(days, 30), 1) as n, + (now() at time zone 'UTC')::date as today + ) win + ) + select s.project_id, + count(distinct s.visitor_id) filter (where s.day >= b.cur_start)::bigint as visitors, + count(distinct s.visitor_id) filter (where s.day <= b.prev_end)::bigint as prev_visitors, + coalesce(sum(s.pageviews) filter (where s.day >= b.cur_start), 0)::bigint as pageviews, + coalesce(sum(s.pageviews) filter (where s.day <= b.prev_end), 0)::bigint as prev_pageviews + from public.tracker_visitor_daily_stats s + cross join bounds b + where s.project_id = any(p_projects) + and s.day >= b.prev_start + and (p_kind is null or s.kind = p_kind) + group by s.project_id; +$$; + +grant execute on function public.tracker_visitor_totals(uuid[], integer, text) to authenticated, service_role; + +-- --------------------------------------------------------------------------- +-- 4. Daily series: visitors per (project, day) +-- --------------------------------------------------------------------------- +create or replace function public.tracker_visitor_daily_series( + p_projects uuid[], + days integer default 30, + p_kind text default null +) +returns table (project_id uuid, day date, visitors bigint, pageviews bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select s.project_id, + s.day, + count(*)::bigint as visitors, + coalesce(sum(s.pageviews), 0)::bigint as pageviews + from public.tracker_visitor_daily_stats s + where s.project_id = any(p_projects) + and s.day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or s.kind = p_kind) + group by s.project_id, s.day + order by s.project_id, s.day; +$$; + +grant execute on function public.tracker_visitor_daily_series(uuid[], integer, text) to authenticated, service_role; diff --git a/tests/tracker-api-stats.test.ts b/tests/tracker-api-stats.test.ts index b8985dc..f1b4cbf 100644 --- a/tests/tracker-api-stats.test.ts +++ b/tests/tracker-api-stats.test.ts @@ -85,17 +85,21 @@ describe("resolveProject", () => { }); describe("totalsFromSeries", () => { - it("sums the points, counting humans when visitors is absent", () => { + // A series point has no visitor field. The old fallback counted `humans` + // (beacons) as visitors, which is how four sites read 51k "visitors" in a + // week against 420 people a day. People now come from the visitor rollup + // (tests/tracker-visitor-rollup.test.ts); the series only yields events. + it("sums the human beacons as events, never as visitors", () => { expect( - totalsFromSeries({ points: [{ visitors: 3, pageviews: 9 }, { visitors: 1, pageviews: 2 }] } as never), - ).toEqual({ visitors: 4, pageviews: 11 }); - expect(totalsFromSeries({ points: [{ humans: 2, pageviews: 5 }] } as never)).toEqual({ visitors: 2, pageviews: 5 }); + totalsFromSeries({ points: [{ humans: 3, pageviews: 9 }, { humans: 1, pageviews: 2 }] } as never), + ).toEqual({ events: 4, pageviews: 11 }); + expect(totalsFromSeries({ points: [{ visitors: 3, humans: 2, pageviews: 5 }] } as never)).toEqual({ events: 2, pageviews: 5 }); }); it("is zero for a list payload or nothing at all, rather than NaN", () => { - expect(totalsFromSeries(undefined)).toEqual({ visitors: 0, pageviews: 0 }); - expect(totalsFromSeries([] as never)).toEqual({ visitors: 0, pageviews: 0 }); - expect(totalsFromSeries({ points: [{ pageviews: "4" }] } as never)).toEqual({ visitors: 0, pageviews: 4 }); + expect(totalsFromSeries(undefined)).toEqual({ events: 0, pageviews: 0 }); + expect(totalsFromSeries([] as never)).toEqual({ events: 0, pageviews: 0 }); + expect(totalsFromSeries({ points: [{ pageviews: "4" }] } as never)).toEqual({ events: 0, pageviews: 4 }); }); }); diff --git a/tests/tracker-visitor-rollup.test.ts b/tests/tracker-visitor-rollup.test.ts new file mode 100644 index 0000000..78e0de5 --- /dev/null +++ b/tests/tracker-visitor-rollup.test.ts @@ -0,0 +1,214 @@ +import { describe, expect, it } from "vitest"; +import { + SCRIPTED_BUCKET, + SCRIPTED_CAP_EVENTS, + SCRIPTED_CAP_PAGEVIEWS, + applyScriptedDemotion, + parseVisitorTouch, +} from "@/lib/tracker/scripted"; +import { + HUMANS_LABEL, + PAGEVIEWS_LABEL, + VISITORS_LABEL, + kindFromBucket, +} from "@/lib/tracker/humans"; +import { headlineTiles, pulseHeadline } from "@/lib/tracker/who"; +import { + VISITORS_SINCE, + sumVisitorTotals, + toVisitorTotals, + visitorsPartial, +} from "@/lib/tracker/visitors"; +import { mergeTotals, totalsFromSeries } from "@/lib/tracker/apiStats"; +import { renderStats } from "@/lib/dashboard/stats-text"; + +// The tracker used to lead with "Human visits", which was every beacon from a +// non-crawler user agent: the page view plus four scroll depths, every click +// and every form submit. On four properties that read 51,531 in a week +// while the raw table held 420 distinct visitor ids in a day — the number an +// independent analytics tool also reported. These pin the pieces that keep +// people and events apart: the scripted cap, the tile order, the API totals +// and the CLI line. + +describe("applyScriptedDemotion", () => { + it("leaves a user-agent bot alone, whatever the rollup says", () => { + const out = applyScriptedDemotion("bot:gptbot", { kind: "bot", events: 1, pageviews: 1 }); + expect(out).toEqual({ bucket: "bot:gptbot", kind: "bot", demoted: false }); + }); + + it("keeps a human hit human while the rollup still calls the visitor human", () => { + const out = applyScriptedDemotion("search:google", { kind: "human", events: 40, pageviews: 12 }); + expect(out).toEqual({ bucket: "search:google", kind: "human", demoted: false }); + }); + + it("demotes a human hit once the rollup has flipped the visitor to bot", () => { + const out = applyScriptedDemotion("referral:bittorrented.com", { + kind: "bot", + events: SCRIPTED_CAP_EVENTS + 1, + pageviews: 4, + }); + expect(out).toEqual({ bucket: SCRIPTED_BUCKET, kind: "bot", demoted: true }); + expect(kindFromBucket(out.bucket)).toBe("bot"); + }); + + it("fails open: no visitor id or a failed RPC leaves the verdict standing", () => { + expect(applyScriptedDemotion("human:direct", null)).toEqual({ + bucket: "human:direct", + kind: "human", + demoted: false, + }); + }); + + it("caps are per visitor per day and generous enough for a binge reader", () => { + expect(SCRIPTED_CAP_PAGEVIEWS).toBeGreaterThanOrEqual(150); + expect(SCRIPTED_CAP_EVENTS).toBeGreaterThan(SCRIPTED_CAP_PAGEVIEWS); + }); +}); + +describe("parseVisitorTouch", () => { + it("reads the one row the RPC returns, as an array or bare, with bigint strings", () => { + expect(parseVisitorTouch([{ kind: "human", events: "3", pageviews: "1" }])).toEqual({ + kind: "human", + events: 3, + pageviews: 1, + }); + expect(parseVisitorTouch({ kind: "bot", events: 501, pageviews: 4 })).toEqual({ + kind: "bot", + events: 501, + pageviews: 4, + }); + }); + + it("is null for anything that is not a kind", () => { + expect(parseVisitorTouch(null)).toBeNull(); + expect(parseVisitorTouch([])).toBeNull(); + expect(parseVisitorTouch({ kind: "unknown", events: 1 })).toBeNull(); + expect(parseVisitorTouch("bot")).toBeNull(); + }); +}); + +describe("headlineTiles with the visitor rollup", () => { + const totals = { visitors: 420, pageviews: 1585, humans: 5796, ai: 12, bots: 90 }; + + it("leads with people, then their page views, then events under their real name", () => { + const tiles = headlineTiles("humans", totals); + expect(tiles.map((t) => [t.key, t.value])).toEqual([ + ["visitors", 420], + ["pageviews", 1585], + ["humans", 5796], + ["ai", 12], + ]); + expect(tiles[0].label).toBe(VISITORS_LABEL); + expect(tiles[1].label).toBe(PAGEVIEWS_LABEL); + expect(tiles[2].label).toBe(HUMANS_LABEL); + expect(HUMANS_LABEL).not.toMatch(/visit/i); + // The event count is no longer the accent figure. + expect(tiles[0].tone).toBe("accent"); + expect(tiles[2].tone).toBe("muted"); + }); + + it("shows the bot side's own visitors under Bots", () => { + const tiles = headlineTiles("bots", { ...totals, visitors: 7, pageviews: 30 }); + expect(tiles.map((t) => t.key)).toEqual(["visitors", "pageviews", "bots"]); + expect(tiles[0].label).toBe("Bot visitors"); + }); + + it("keeps both sides under All", () => { + expect(headlineTiles("all", totals).map((t) => t.key)).toEqual([ + "visitors", + "pageviews", + "humans", + "ai", + "bots", + ]); + }); + + it("leaves the people tiles out — never 0 — when the rollup is unavailable", () => { + const tiles = headlineTiles("humans", { ...totals, visitors: null, pageviews: null }); + expect(tiles.map((t) => t.key)).toEqual(["humans", "ai"]); + expect(tiles[0].tone).toBe("accent"); + // And the pre-rollup call shape is unchanged. + expect(headlineTiles("humans", { humans: 3, ai: 1, bots: 9 }).map((t) => t.key)).toEqual([ + "humans", + "ai", + ]); + }); + + it("names the pulse figure as events, not visits", () => { + expect(pulseHeadline("humans", { humans: 3, bots: 9 }).unit).toEqual([ + "human event", + "human events", + ]); + }); +}); + +describe("visitor totals", () => { + it("coerces PostgREST strings and sums across projects", () => { + const a = toVisitorTotals({ project_id: "a", visitors: "372", prev_visitors: "0", pageviews: "1469", prev_pageviews: "0" }); + const b = toVisitorTotals({ project_id: "b", visitors: 30, prev_visitors: 4, pageviews: 73, prev_pageviews: 9 }); + expect(sumVisitorTotals([a, b])).toEqual({ + visitors: 402, + prevVisitors: 4, + pageviews: 1542, + prevPageviews: 9, + }); + }); + + it("knows when a window reaches back before the rollup existed", () => { + const since = new Date(`${VISITORS_SINCE}T12:00:00Z`); + expect(visitorsPartial(1, since)).toBe(false); + expect(visitorsPartial(2, since)).toBe(true); + const later = new Date(since.getTime() + 30 * 86_400_000); + expect(visitorsPartial(30, later)).toBe(false); + expect(visitorsPartial(31, later)).toBe(false); + expect(visitorsPartial(32, later)).toBe(true); + }); +}); + +describe("API totals", () => { + it("never reports events as visitors", () => { + const fromSeries = totalsFromSeries({ + points: [ + { humans: 2130, pageviews: 700 }, + { humans: 3666, pageviews: 885 }, + ], + } as never); + expect(fromSeries).toEqual({ events: 5796, pageviews: 1585 }); + expect("visitors" in fromSeries).toBe(false); + }); + + it("takes people from the rollup and events from the series", () => { + expect(mergeTotals({ events: 5796, pageviews: 1585 }, { visitors: 420, pageviews: 1469 })).toEqual({ + visitors: 420, + pageviews: 1469, + events: 5796, + }); + }); + + it("reports an unreadable rollup as null, not 0", () => { + expect(mergeTotals({ events: 5796, pageviews: 1585 }, null)).toEqual({ + visitors: null, + pageviews: null, + events: 5796, + }); + }); +}); + +describe("CLI stats line", () => { + it("prints people, page views and events, in that order", () => { + const text = renderStats( + { project: { name: "bittorrented.com" }, totals: { visitors: 420, pageviews: 1469, events: 5796 } }, + { range: "1d", who: "humans" }, + ); + expect(text).toContain("420 visitors, 1469 pageviews, 5796 events"); + }); + + it("says visitors are unavailable rather than printing 0", () => { + const text = renderStats( + { project: { name: "x" }, totals: { visitors: null, pageviews: null, events: 12 } }, + { range: "1d", who: "humans" }, + ); + expect(text).toContain("visitors unavailable, 12 events"); + expect(text).not.toContain("Nothing in this window"); + }); +});