diff --git a/app/(app)/dashboard/analytics/page.tsx b/app/(app)/dashboard/analytics/page.tsx index 4c8edad..7b7a649 100644 --- a/app/(app)/dashboard/analytics/page.tsx +++ b/app/(app)/dashboard/analytics/page.tsx @@ -30,8 +30,17 @@ import { BOTS_LABEL, HUMANS_DEFINITION, HUMANS_LABEL, + VISITORS_DEFINITION, + VISITORS_LABEL, humansFrom, } from "@/lib/tracker/humans"; +import { + VISITORS_CAPTION, + fetchVisitorTotals, + sumVisitorTotals, + visitorsPartial, + type VisitorTotals, +} from "@/lib/tracker/visitors"; import { TrackerAnalytics, type TrackerListItem, @@ -71,14 +80,18 @@ type PortfolioProject = { organization_id?: string | null; }; -// Every figure on this page leads with humans (bucket not `bot:`, AI referrals -// included) and shows bot crawls apart. See lib/tracker/humans.ts. +// Every figure on this page leads with people — distinct visitors from the +// visitor rollup — then human EVENTS (bucket not `bot:`, AI referrals +// included; several beacons per page view) and shows bot crawls apart. See +// lib/tracker/humans.ts and lib/tracker/visitors.ts. type ProjectRow = { project: PortfolioProject; totals: ProjectTotals; - /** Human visits, this window vs the one before. */ + /** Distinct human visitors this window vs the one before; null when the rollup is unavailable. */ + visitors: VisitorTotals | null; + /** Human events, this window vs the one before. */ trend: Trend; - /** Daily human visits for the sparkline; null when over the row budget. */ + /** Daily human events for the sparkline; null when over the row budget. */ samples: number[] | null; }; @@ -217,6 +230,22 @@ export default async function PortfolioAnalyticsPage({ const portfolio = sumTotals(totalsByProject.values()); const trends = totalsTrends(portfolio); + // People. One more RPC, kept apart from the eleven above because its + // failure means "no visitor figure", not "no traffic": the tile and the + // column are left out rather than zeroed. + const visitorsByProject = await fetchVisitorTotals(supabase, projectIds, days, "human"); + const portfolioVisitors = visitorsByProject + ? sumVisitorTotals(visitorsByProject.values()) + : null; + const visitorsTrend = portfolioVisitors + ? computeTrend(portfolioVisitors.visitors, portfolioVisitors.prevVisitors) + : null; + const visitorsCaption = !visitorsByProject + ? "Visitor counts are unavailable right now; the figures below count events." + : visitorsPartial(days) + ? VISITORS_CAPTION + : null; + // Ranked by current-window HUMAN volume: the biggest properties get the // chart bands, and the daily-detail budget is spent on them first. Ranking // by events would hand the top band to whichever site a crawler is hitting. @@ -264,6 +293,9 @@ export default async function PortfolioAnalyticsPage({ return { project, totals, + visitors: visitorsByProject + ? (visitorsByProject.get(project.id) ?? { visitors: 0, prevVisitors: 0, pageviews: 0, prevPageviews: 0 }) + : null, trend: computeTrend(totals.humans, totals.prevHumans), samples: byDay ? axis.map((day) => byDay.get(day) ?? 0) : null, }; @@ -444,11 +476,19 @@ export default async function PortfolioAnalyticsPage({

-
+
+ {visitorsTrend && ( + + )}
+ {visitorsCaption && ( +

{visitorsCaption}

+ )} {portfolio.events === 0 && portfolio.prevEvents === 0 ? (
@@ -485,13 +528,21 @@ export default async function PortfolioAnalyticsPage({

Portfolio trend

- Daily human visits, stacked by property. Bot crawls across + Daily human events, stacked by property. Bot crawls across every property are the dashed line, kept out of the stack.

+ {portfolioVisitors && ( + <> + + {portfolioVisitors.visitors.toLocaleString()} human visitors + + {" · "} + + )} - {portfolio.humans.toLocaleString()} human visits + {portfolio.humans.toLocaleString()} human events {" · "} @@ -633,8 +684,11 @@ function ProjectTrendTable({ rows }: { rows: ProjectRow[] }) { Property + + Visitors + - Humans + Events Previous @@ -650,7 +704,7 @@ function ProjectTrendTable({ rows }: { rows: ProjectRow[] }) { - {rows.map(({ project, totals, trend, samples }) => ( + {rows.map(({ project, totals, visitors, trend, samples }) => ( + + {visitors ? visitors.visitors.toLocaleString() : "—"} + {totals.humans.toLocaleString()} diff --git a/app/(app)/dashboard/page.tsx b/app/(app)/dashboard/page.tsx index 5bd7b40..2a541e4 100644 --- a/app/(app)/dashboard/page.tsx +++ b/app/(app)/dashboard/page.tsx @@ -3,7 +3,8 @@ import { createClient } from "@/lib/supabase/server"; import { rpcFailed, type Loaded } from "@/lib/loaded"; import { ScoreBadge } from "@/components/score-badge"; import { FontSparkline } from "@/components/font-sparkline"; -import { BOTS_DEFINITION, HUMANS_DEFINITION } from "@/lib/tracker/humans"; +import { BOTS_DEFINITION, HUMANS_DEFINITION, VISITORS_DEFINITION } from "@/lib/tracker/humans"; +import { fetchVisitorDailySeries } from "@/lib/tracker/visitors"; import { ProjectLogo } from "@/components/project-logo"; import { StatsUnavailable } from "@/components/stats-unavailable"; import { backfillProjectLogo } from "@/app/actions/createProject"; @@ -121,11 +122,12 @@ export default async function DashboardPage({ // project has an lx_site row in status=active; social is "on" when at // least one social account is linked at the project level. const projectIds = (projects ?? []).map((p) => p.id); - const [autoblogIds, socialIds, latestPosts, traffic] = await Promise.all([ + const [autoblogIds, socialIds, latestPosts, traffic, visitorsByProject] = await Promise.all([ fetchEnabledProjectIds(supabase, "lx_site", projectIds, { status: "active" }), fetchEnabledProjectIds(supabase, "sp_site_account", projectIds), fetchLatestBlogPostByProject(supabase, projectIds), fetchSevenDayTraffic(supabase, projectIds), + fetchSevenDayVisitors(supabase, projectIds), ]); const trafficByProject = traffic.data; const trafficFailed = traffic.failed; @@ -266,23 +268,27 @@ export default async function DashboardPage({
{trafficFailed ? "Traffic unavailable" - : `${totalHumans(trafficByProject.get(p.id) ?? []).toLocaleString()} human visits`} + : visitorsByProject + ? `${sum(visitorsByProject.get(p.id)).toLocaleString()} human visitors` + : `${totalHumans(trafficByProject.get(p.id) ?? []).toLocaleString()} human events`}
{trafficFailed ? "Query failed \u2014 not zero" - : `${totalBots(trafficByProject.get(p.id) ?? []).toLocaleString()} bot hits \u00b7 Past 7 days`} + : `${visitorsByProject ? `${totalHumans(trafficByProject.get(p.id) ?? []).toLocaleString()} human events \u00b7 ` : ""}${totalBots(trafficByProject.get(p.id) ?? []).toLocaleString()} bot hits \u00b7 Past 7 days`}
{!trafficFailed && ( - + )}
{orgSchemaReady && ( @@ -436,6 +442,32 @@ async function fetchSevenDayTraffic( return { data: out, failed: false }; } +// Distinct human visitors per day for the past seven UTC days, per project: +// the number the card leads with. The visitor rollup only began on +// lib/tracker/visitors.ts VISITORS_SINCE, and it counts people where +// dashboard_project_traffic counts beacons (several per page view), which +// is why the two figures on a card differ by a hundredfold and both are +// shown. null when the RPC failed or is not deployed: the card then leads +// with events under their real name rather than showing "0 visitors". +async function fetchSevenDayVisitors( + supabase: Awaited>, + projectIds: string[], +): Promise | null> { + const days = lastSevenDays(); + const byProject = await fetchVisitorDailySeries(supabase, projectIds, 7, "human"); + if (!byProject) return null; + const out = new Map(); + for (const projectId of projectIds) { + const byDay = byProject.get(projectId); + out.set(projectId, days.map((day) => byDay?.get(day) ?? 0)); + } + return out; +} + +function sum(samples: number[] | undefined) { + return (samples ?? []).reduce((total, n) => total + n, 0); +} + function lastSevenDays() { return Array.from({ length: 7 }, (_, index) => { const date = new Date(); diff --git a/app/(app)/dashboard/projects/[id]/stats/page.tsx b/app/(app)/dashboard/projects/[id]/stats/page.tsx index c9eb177..8472824 100644 --- a/app/(app)/dashboard/projects/[id]/stats/page.tsx +++ b/app/(app)/dashboard/projects/[id]/stats/page.tsx @@ -8,9 +8,14 @@ import { TrackerAnalytics, type TrackerPanels, } from "@/components/charts/tracker-analytics"; -import { fetchPanels, PANEL_KEYS } from "@/lib/tracker/panels"; +import { fetchPanels, PANEL_KEYS, resolveDays, rollupDays } from "@/lib/tracker/panels"; import { DEFAULT_TRACKER_RANGE, trackerRange } from "@/lib/tracker/ranges"; import { headlineTiles, whoOrDefault, whoToKind } from "@/lib/tracker/who"; +import { + VISITORS_CAPTION, + fetchVisitorTotals, + visitorsPartial, +} from "@/lib/tracker/visitors"; import { InstallSnippet } from "./install-snippet"; import { TrackerToggle } from "./tracker-toggle"; import { CareersToggle } from "./careers-toggle"; @@ -65,31 +70,41 @@ export default async function ProjectStatsPage({ // Every panel is fetched at the toggle's `kind`, so a card and the tiles // above it never disagree on who they are counting. const range = trackerRange(DEFAULT_TRACKER_RANGE); - const panels = (await fetchPanels( - supabase, - id, - PANEL_KEYS, - range, - kind, - )) as unknown as TrackerPanels; + // The visitor rollup is day-resolution, so a sub-day range reads today's + // rollup — the same rule the device and exit-page cards apply. + const visitorDays = rollupDays(range, await resolveDays(supabase, id, range)); + const [panels, visitorTotals] = await Promise.all([ + fetchPanels(supabase, id, PANEL_KEYS, range, kind) as unknown as Promise, + fetchVisitorTotals(supabase, [id], visitorDays, kind), + ]); - // Headline metrics come straight from the series so they stay exact even - // though Top sources below is truncated to the top 10 buckets. The page - // leads with humans (every bucket that is not `bot:`, AI referrals - // included) and shows bot crawls apart — see lib/tracker/humans.ts for why - // the old bot-inclusive total is no longer a headline. Under Humans or - // Bots only that side's tiles are shown (lib/tracker/who.ts). + // Headline metrics: the people first. Distinct visitors and their page + // views come from the visitor rollup (lib/tracker/visitors.ts); the event + // figures come straight from the series so they stay exact even though Top + // sources below is truncated to the top 10 buckets. The event count used to + // be labelled "Human visits" and ran ~100x above the number of people — it + // is stats.js firing several beacons per page view — so it now sits after + // the visitor tiles under its real name. Under Humans or Bots only that + // side's tiles are shown (lib/tracker/who.ts). const points = panels.series.points; const totalAi = points.reduce((s, p) => s + p.ai, 0); const totalBot = points.reduce((s, p) => s + p.bots, 0); const totalHuman = points.reduce((s, p) => s + p.humans, 0); const grandTotal = points.reduce((s, p) => s + p.events, 0); const eventTotal = points.reduce((s, p) => s + p.pageviews + p.interactions, 0); + const visitors = visitorTotals?.get(id) ?? (visitorTotals ? { visitors: 0, pageviews: 0 } : null); const tiles = headlineTiles(who, { + visitors: visitors ? visitors.visitors : null, + pageviews: visitors ? visitors.pageviews : null, humans: totalHuman, ai: totalAi, bots: totalBot, }); + const visitorsCaption = visitors === null + ? "Visitor counts are unavailable right now; the figures below count events." + : visitorsPartial(visitorDays) + ? VISITORS_CAPTION + : null; // Older projects have rollup rows in tracker_daily_stats but nothing in // tracker_event_daily_stats, which would leave Event mix empty on a page @@ -265,7 +280,7 @@ export default async function ProjectStatsPage({ -
+
{tiles.map((tile) => ( ))}
+ {visitorsCaption && ( +

{visitorsCaption}

+ )} {grandTotal === 0 && eventTotal === 0 ? (
diff --git a/app/api/track/route.ts b/app/api/track/route.ts index 0b104a9..694d12d 100644 --- a/app/api/track/route.ts +++ b/app/api/track/route.ts @@ -7,6 +7,13 @@ import { z } from "zod"; import { serviceClient } from "@/lib/supabase/service"; import { categorize } from "@/lib/tracker/categorize"; import { kindFromBucket } from "@/lib/tracker/humans"; +import { + SCRIPTED_CAP_EVENTS, + SCRIPTED_CAP_PAGEVIEWS, + applyScriptedDemotion, + parseVisitorTouch, + type VisitorTouch, +} from "@/lib/tracker/scripted"; import { parseDevice } from "@/lib/tracker/device"; import { clientIpFromHeaders, lookupGeo } from "@/lib/tracker/geo"; import { enqueuePostHogEvent } from "@/lib/posthog/events"; @@ -182,12 +189,41 @@ async function ingest(request: NextRequest, parseBody: boolean) { const gate = await gateAgent(sb, site, userAgent); if (gate.action !== "allow") return refuse(gate); - const { bucket, isAi } = categorize({ referrer, userAgent, url: pageUrl }); + const categorized = categorize({ referrer, userAgent, url: pageUrl }); + const today = new Date().toISOString().slice(0, 10); // YYYY-MM-DD UTC + + // Visitor rollup: one row per (project, day, visitor id), bumped per beacon. + // It is the only place the tracker counts people rather than events, and + // its answer can demote this hit: a visitor past the scripted cap for the + // day is a driven browser, not a reader, whatever its user agent claims + // (lib/tracker/scripted.ts). Read before the first counter write so every + // rollup below lands on the same side. Best-effort: no visitor id, or a + // failed RPC, leaves the user-agent verdict standing. + const visitorId = textOrNull(parsed.data.visitorId); + let touch: VisitorTouch | null = null; + if (visitorId) { + try { + const { data } = await sb.rpc("tracker_touch_visitor", { + p_project: site, + p_day: today, + p_visitor: visitorId.slice(0, 128), + p_kind: kindFromBucket(categorized.bucket), + p_pageview: event === "pageview", + p_cap_events: SCRIPTED_CAP_EVENTS, + p_cap_pageviews: SCRIPTED_CAP_PAGEVIEWS, + }); + touch = parseVisitorTouch(data); + } catch { + // Silent — the beacon must never fail closed on a counter table. + } + } + const demotion = applyScriptedDemotion(categorized.bucket, touch); + const bucket = demotion.bucket; + const isAi = demotion.demoted ? false : categorized.isAi; // Which side of the human / bot line this hit counts on. The bucket table // carries the whole bucket; the other rollups record only this, so the // stats page can split every breakdown, not just the headline. - const kind = kindFromBucket(bucket); - const today = new Date().toISOString().slice(0, 10); // YYYY-MM-DD UTC + const kind = demotion.kind; // UPSERT increment. Supabase JS doesn't expose a raw .increment() helper // so we read + write under the unique key. The PK protects against diff --git a/lib/dashboard/stats-text.ts b/lib/dashboard/stats-text.ts index 2fdbfbd..fb90f04 100644 --- a/lib/dashboard/stats-text.ts +++ b/lib/dashboard/stats-text.ts @@ -9,7 +9,7 @@ export type StatsItem = { label: string; value: number }; export type StatsAnswerish = { project?: { name?: string; url?: string } | null; - totals?: { visitors?: number; pageviews?: number } | null; + totals?: { visitors?: number | null; pageviews?: number | null; events?: number } | null; sources?: StatsItem[] | null; referrers?: StatsItem[] | null; pages?: StatsItem[] | null; @@ -24,7 +24,11 @@ export function renderStats( const totals = answer.totals ?? {}; const out: string[] = [ `${answer.project?.name ?? "project"} ${range} ${who}`, - `${totals.visitors ?? 0} visitors, ${totals.pageviews ?? 0} pageviews`, + // People first. A null visitor count is the rollup being unreadable, not + // an empty site, and is printed as such rather than as 0. + totals.visitors === null || totals.visitors === undefined + ? `visitors unavailable, ${totals.events ?? 0} events` + : `${totals.visitors} visitors, ${totals.pageviews ?? 0} pageviews, ${totals.events ?? 0} events`, ]; const section = (title: string, items: StatsItem[]) => { @@ -40,7 +44,7 @@ export function renderStats( section("Pages", list(answer.pages)); // Nothing at all is a real answer, and the likeliest cause is worth naming. - if (!(totals.pageviews ?? 0) && !list(answer.sources).length) { + if (!(totals.pageviews ?? 0) && !(totals.events ?? 0) && !list(answer.sources).length) { out.push("", "Nothing in this window. Check the tag is on the page, or widen --range."); } return `${out.join("\n")}\n`; diff --git a/lib/tracker/apiStats.ts b/lib/tracker/apiStats.ts index b475ed4..5403365 100644 --- a/lib/tracker/apiStats.ts +++ b/lib/tracker/apiStats.ts @@ -20,6 +20,8 @@ import { } from "@/lib/tracker/panels"; import type { TrackerRange } from "@/lib/tracker/ranges"; import type { TrackerKind } from "@/lib/tracker/humans"; +import { fetchVisitorTotals } from "@/lib/tracker/visitors"; +import { rollupDays } from "@/lib/tracker/panels"; type Sb = SupabaseClient; @@ -108,7 +110,16 @@ export type StatsAnswer = { project: { id: string; name: string; url: string }; range: string; who: string; - totals: { visitors: number; pageviews: number }; + /** + * `visitors` is distinct visitor ids over the window from the visitor + * rollup (people); `pageviews` is the page views those visitors produced; + * `events` is every beacon on the requested side, which is what `visitors` + * used to be filled with before the rollup existed (and read ~100x too + * high). `visitors` and `pageviews` are null when the rollup could not be + * read, never 0: a caller computing cost per visitor must not divide by a + * failure. + */ + totals: { visitors: number | null; pageviews: number | null; events: number }; sources: ListItem[]; referrers: ListItem[]; pages: ListItem[]; @@ -167,17 +178,35 @@ export function mixFromSeries(payload: PanelPayload | undefined): { return mix; } -/** Sum a series payload's points into the two numbers a summary line needs. */ -export function totalsFromSeries(payload: PanelPayload | undefined): { visitors: number; pageviews: number } { - if (!payload || Array.isArray(payload)) return { visitors: 0, pageviews: 0 }; +/** + * Sum a series payload's points into the event-side numbers. `events` is the + * human (or requested-side) beacon count; `pageviews` here is the + * bot-inclusive pageview leg of the series and is only a fallback for a + * caller with no visitor rollup. Never called `visitors`: a series point has + * no visitor field and the old fallback to `humans` is the 100x bug. + */ +export function totalsFromSeries(payload: PanelPayload | undefined): { events: number; pageviews: number } { + if (!payload || Array.isArray(payload)) return { events: 0, pageviews: 0 }; const points = (payload as { points?: Record[] }).points ?? []; - let visitors = 0; + let events = 0; let pageviews = 0; for (const point of points) { - visitors += Number(point.visitors ?? point.humans ?? 0) || 0; + events += Number(point.humans ?? 0) || 0; pageviews += Number(point.pageviews ?? 0) || 0; } - return { visitors, pageviews }; + return { events, pageviews }; +} + +/** The totals block: people from the rollup, events from the series. */ +export function mergeTotals( + fromSeries: { events: number; pageviews: number }, + visitors: { visitors: number; pageviews: number } | null | undefined, +): StatsAnswer["totals"] { + return { + visitors: visitors ? visitors.visitors : null, + pageviews: visitors ? visitors.pageviews : null, + events: fromSeries.events, + }; } export async function projectStats( @@ -189,20 +218,26 @@ export async function projectStats( /** Add the series and the unfiltered human / bot mix. One extra RPC at most. */ detail = false, ): Promise { - const [panels, mixSeries] = await Promise.all([ + const days = await resolveDays(sb, project.id, range); + const [panels, mixSeries, visitorTotals] = await Promise.all([ fetchPanels(sb, project.id, STATS_PANELS, range, kind), // Only when the answer is filtered: at kind null the main series already is // the unfiltered one, and a second identical query would be a second query. detail && kind !== null - ? fetchPanel(sb, project.id, "series", range, await resolveDays(sb, project.id, range), null) + ? fetchPanel(sb, project.id, "series", range, days, null) : Promise.resolve(undefined), + // People. The rollup is day-resolution, so a sub-day range reads today's. + fetchVisitorTotals(sb, [project.id], rollupDays(range, days), kind), ]); const answer: StatsAnswer = { project: { id: project.id, name: project.name, url: project.url }, range: range.key, who, - totals: totalsFromSeries(panels.series), + totals: mergeTotals( + totalsFromSeries(panels.series), + visitorTotals ? (visitorTotals.get(project.id) ?? { visitors: 0, pageviews: 0 }) : null, + ), sources: asList(panels.sources), referrers: asList(panels.referrers), pages: asList(panels.pages), diff --git a/lib/tracker/humans.ts b/lib/tracker/humans.ts index 0bb58e2..a9554e0 100644 --- a/lib/tracker/humans.ts +++ b/lib/tracker/humans.ts @@ -13,14 +13,30 @@ // bot = bucket starts with "bot:" (named AI crawlers + bot:other) // so humans + bots = events, exactly. -export const HUMANS_LABEL = "Human visits"; +// Two human figures, and the names keep them apart: +// VISITORS — distinct visitor ids per day (tracker_visitor_daily_stats), +// the number of PEOPLE. This is the headline. +// HUMANS — every beacon from a non-crawler (tracker_daily_stats), i.e. +// EVENTS: the page view plus each scroll depth, click and form +// submit stats.js fires. Three to four per page view. It used to +// be labelled "Human visits" and read 100x above the number of +// people; it keeps its name in code and loses it in the UI. +export const VISITORS_LABEL = "Human visitors"; +export const HUMANS_LABEL = "Human events"; export const BOTS_LABEL = "Bot crawls"; +export const PAGEVIEWS_LABEL = "Page views"; + +export const VISITORS_DEFINITION = + "Distinct people, counted once per site per day by the visitor id the tracker stores in the browser. Crawlers and scripted browsers are excluded."; export const HUMANS_DEFINITION = - "Everything not identified as a crawler, including visits referred by AI assistants such as ChatGPT or Perplexity."; + "Every tracked event from a visitor not identified as a crawler: the page view plus each scroll depth, click and form submit. Several per page view, so this is not a count of people. Includes visits referred by AI assistants such as ChatGPT or Perplexity."; + +export const PAGEVIEWS_DEFINITION = + "Pages rendered for people: one per page load from a visitor not identified as a crawler or a scripted browser."; export const BOTS_DEFINITION = - "Hits from user agents identified as crawlers: AI training and retrieval bots, search engine bots, and other automated clients. Not people."; + "Hits from user agents identified as crawlers (AI training and retrieval bots, search engine bots, other automated clients) plus scripted browsers caught by volume. Not people."; export const AI_REFERRALS_DEFINITION = "People who arrived from an AI assistant. These are already counted inside human visits."; diff --git a/lib/tracker/scripted.ts b/lib/tracker/scripted.ts new file mode 100644 index 0000000..1d9a20d --- /dev/null +++ b/lib/tracker/scripted.ts @@ -0,0 +1,74 @@ +// The scripted-visitor cap: the one rule that turns a browser being driven +// into a bot, applied per beacon by the ingest route. +// +// WHY: lib/tracker/categorize.ts can only call a hit a bot when the user +// agent says so. A headless Chrome with a stock UA is "human" by definition, +// and one was measured minting a fresh visitor id per page view (19k +// "visitors" on 19k page views in a day), another produced 398 events on 4 +// page views, another 126 page views from one id. No person does that. The +// tracker_visitor_daily_stats rollup keeps per-visitor, per-day counts, so +// volume is a signal the classifier finally has. +// +// The caps are per visitor per UTC day and deliberately generous: a binge +// reader on a search-heavy site can reach 100 page views, and the cost of a +// false demotion (one real person counted as a bot for a day) is the same +// size as the cost of a miss (one script counted as a person for a day). The +// point is the tail, not the median. +// +// Mirrored by the defaults of tracker_touch_visitor in +// supabase/migrations/20260921120000_tracker_visitor_rollup.sql. + +import { kindFromBucket, type TrackerKind } from "@/lib/tracker/humans"; + +/** More beacons than this from one visitor in one UTC day is a script. */ +export const SCRIPTED_CAP_EVENTS = 500; +/** More page views than this from one visitor in one UTC day is a script. */ +export const SCRIPTED_CAP_PAGEVIEWS = 200; + +/** The bucket a demoted hit is counted under. Renders as "Bot · scripted". */ +export const SCRIPTED_BUCKET = "bot:scripted"; + +/** What tracker_touch_visitor hands back after the increment. */ +export type VisitorTouch = { + kind: TrackerKind; + events: number; + pageviews: number; +}; + +/** + * The bucket and kind a hit should be counted under once the visitor rollup + * has had its say. A hit the user agent already made a bot is left alone. A + * hit from a visitor the rollup has flipped to `bot` — by this beacon or an + * earlier one today — is counted under bot:scripted, so every rollup the + * route writes next lands on the bot side. Without a touch (no visitor id, + * or the RPC failed) the bucket stands: the beacon must never fail closed + * on a counter table hiccup. + */ +export function applyScriptedDemotion( + bucket: string, + touch: VisitorTouch | null, +): { bucket: string; kind: TrackerKind; demoted: boolean } { + const kind = kindFromBucket(bucket); + if (kind === "bot" || !touch || touch.kind !== "bot") { + return { bucket, kind, demoted: false }; + } + return { bucket: SCRIPTED_BUCKET, kind: "bot", demoted: true }; +} + +/** Coerce one RPC row; null when the shape is not what the migration returns. */ +export function parseVisitorTouch(row: unknown): VisitorTouch | null { + const r = (Array.isArray(row) ? row[0] : row) as + | { kind?: unknown; events?: unknown; pageviews?: unknown } + | null + | undefined; + if (!r || typeof r !== "object") return null; + const kind = r.kind === "bot" ? "bot" : r.kind === "human" ? "human" : null; + if (!kind) return null; + const events = Number(r.events); + const pageviews = Number(r.pageviews); + return { + kind, + events: Number.isFinite(events) ? events : 0, + pageviews: Number.isFinite(pageviews) ? pageviews : 0, + }; +} diff --git a/lib/tracker/visitors.ts b/lib/tracker/visitors.ts new file mode 100644 index 0000000..22cb434 --- /dev/null +++ b/lib/tracker/visitors.ts @@ -0,0 +1,139 @@ +// Readers for the visitor rollup (tracker_visitor_daily_stats), the one place +// the tracker counts PEOPLE rather than beacons. +// +// WHY: the bucket rollup is bumped per event and stats.js fires several +// events per page view, so "Human visits" ran 100x above the number of +// people. Every headline now leads with distinct visitors from this rollup +// and shows the event count beside it under its real name. +// +// Every reader returns null when the RPC is missing or fails, and the pages +// render "unavailable" for that figure rather than 0: a zero here would be +// read as a dead site, which is the misreading the whole split exists to +// prevent. Rows begin on VISITORS_SINCE (there is no history to backfill: +// tracker_events keeps 24h), so a window that reaches further back is +// partial and the UI says so. + +import type { SupabaseClient } from "@supabase/supabase-js"; +import type { TrackerKind } from "@/lib/tracker/humans"; + +type Sb = SupabaseClient; + +/** The UTC day the rollup migration was applied to production. */ +export const VISITORS_SINCE = "2026-09-21"; +export const VISITORS_SINCE_LABEL = "21 Sep 2026"; +export const VISITORS_CAPTION = `Visitors counted from ${VISITORS_SINCE_LABEL}; earlier traffic has no visitor count.`; + +export type VisitorTotals = { + /** Distinct visitor ids in the window. */ + visitors: number; + prevVisitors: number; + /** Page views those visitors produced. */ + pageviews: number; + prevPageviews: number; +}; + +export function emptyVisitorTotals(): VisitorTotals { + return { visitors: 0, prevVisitors: 0, pageviews: 0, prevPageviews: 0 }; +} + +const num = (v: unknown) => { + const n = Number(v); + return Number.isFinite(n) ? n : 0; +}; + +/** One row of tracker_visitor_totals; bigint columns arrive as strings. */ +export type VisitorTotalsRow = { + project_id: string; + visitors: number | string; + prev_visitors: number | string; + pageviews: number | string; + prev_pageviews: number | string; +}; + +export function toVisitorTotals(row: VisitorTotalsRow): VisitorTotals { + return { + visitors: num(row.visitors), + prevVisitors: num(row.prev_visitors), + pageviews: num(row.pageviews), + prevPageviews: num(row.prev_pageviews), + }; +} + +export function sumVisitorTotals(all: Iterable): VisitorTotals { + const out = emptyVisitorTotals(); + for (const t of all) { + out.visitors += t.visitors; + out.prevVisitors += t.prevVisitors; + out.pageviews += t.pageviews; + out.prevPageviews += t.prevPageviews; + } + return out; +} + +/** + * Distinct visitors per project over the last `days` UTC days and the equal + * window before. Projects with no rows are absent from the map (0 visitors). + * null when the RPC failed or does not exist yet. + */ +export async function fetchVisitorTotals( + sb: Sb, + projectIds: string[], + days: number, + kind: TrackerKind | null = "human", +): Promise | null> { + const out = new Map(); + if (projectIds.length === 0) return out; + const { data, error } = await sb.rpc("tracker_visitor_totals", { + p_projects: projectIds, + days: Math.max(1, days), + p_kind: kind, + }); + if (error) return null; + for (const row of (data ?? []) as VisitorTotalsRow[]) { + out.set(row.project_id, toVisitorTotals(row)); + } + return out; +} + +export type VisitorDayRow = { + project_id: string; + day: string; + visitors: number | string; + pageviews: number | string; +}; + +/** + * Visitors per (project, UTC day) over the last `days` days, as + * project -> day -> visitors. Days with no rows are absent. null on failure. + */ +export async function fetchVisitorDailySeries( + sb: Sb, + projectIds: string[], + days: number, + kind: TrackerKind | null = "human", +): Promise> | null> { + const out = new Map>(); + for (const id of projectIds) out.set(id, new Map()); + if (projectIds.length === 0) return out; + const { data, error } = await sb.rpc("tracker_visitor_daily_series", { + p_projects: projectIds, + days: Math.max(1, days), + p_kind: kind, + }); + if (error) return null; + for (const row of (data ?? []) as VisitorDayRow[]) { + out.get(row.project_id)?.set(row.day, num(row.visitors)); + } + return out; +} + +/** + * Whether a window of `days` days ending today reaches back before the rollup + * existed, in which case its visitor figure is partial and needs the caption. + */ +export function visitorsPartial(days: number, now = new Date()): boolean { + const start = new Date(now); + start.setUTCHours(0, 0, 0, 0); + start.setUTCDate(start.getUTCDate() - Math.max(1, days) + 1); + return start.toISOString().slice(0, 10) < VISITORS_SINCE; +} diff --git a/lib/tracker/who.ts b/lib/tracker/who.ts index 63d44e9..1833170 100644 --- a/lib/tracker/who.ts +++ b/lib/tracker/who.ts @@ -17,6 +17,10 @@ import { BOTS_LABEL, HUMANS_DEFINITION, HUMANS_LABEL, + PAGEVIEWS_DEFINITION, + PAGEVIEWS_LABEL, + VISITORS_DEFINITION, + VISITORS_LABEL, type TrackerKind, } from "@/lib/tracker/humans"; @@ -78,31 +82,66 @@ export function whoCaption(who: Who): string | null { } export type HeadlineTotals = { + /** + * Distinct visitors in the window from the visitor rollup, on the side + * `who` asks for. null when the rollup could not be read: the tile is then + * left out rather than shown as 0, because 0 reads as a dead site. + */ + visitors?: number | null; + /** Page views those visitors produced; drawn only beside `visitors`. */ + pageviews?: number | null; + /** Events, not people: every beacon on the human side. */ humans: number; ai: number; bots: number; }; export type HeadlineTile = { - key: "humans" | "ai" | "bots"; + key: "visitors" | "pageviews" | "humans" | "ai" | "bots"; label: string; value: number; - tone: "accent" | "pass" | "warn"; + tone: "accent" | "pass" | "warn" | "muted"; hint: string; }; /** - * Which headline tiles the page shows for a given toggle. Humans and Bots - * show their own side's figures; All keeps the three-tile layout the page - * shipped with. The figures come from the series the page fetched at that - * same `who`, so a filtered view never mixes in the other side's count. + * Which headline tiles the page shows for a given toggle. Humans leads with + * people (distinct visitors, then their page views) and shows the event + * count under its real name after them; Bots shows its own side; All keeps + * both sides. The figures come from the series and the visitor rollup the + * page fetched at that same `who`, so a filtered view never mixes in the + * other side's count. */ export function headlineTiles(who: Who, t: HeadlineTotals): HeadlineTile[] { + const people: HeadlineTile[] = + t.visitors === null || t.visitors === undefined + ? [] + : [ + { + key: "visitors", + label: who === "bots" ? "Bot visitors" : VISITORS_LABEL, + value: t.visitors, + tone: who === "bots" ? "warn" : "accent", + hint: + who === "bots" + ? "Distinct visitor ids on the bot side: crawlers that ran the script, and scripted browsers caught by volume." + : VISITORS_DEFINITION, + }, + { + key: "pageviews", + label: PAGEVIEWS_LABEL, + value: t.pageviews ?? 0, + tone: who === "bots" ? "warn" : "accent", + hint: PAGEVIEWS_DEFINITION, + }, + ]; const humans: HeadlineTile = { key: "humans", label: HUMANS_LABEL, value: t.humans, - tone: "accent", + // Leads only when the rollup has nothing to say; otherwise it is the + // number people used to mistake for readers, kept but demoted. + tone: people.length ? "muted" : "accent", hint: HUMANS_DEFINITION, }; const ai: HeadlineTile = { @@ -121,11 +160,11 @@ export function headlineTiles(who: Who, t: HeadlineTotals): HeadlineTile[] { }; switch (who) { case "humans": - return [humans, ai]; + return [...people, humans, ai]; case "bots": - return [bots]; + return [...people, bots]; default: - return [humans, ai, bots]; + return [...people, humans, ai, bots]; } } @@ -141,7 +180,7 @@ export type PulseLayer = { const HUMANS_LAYER: PulseLayer = { dataKey: "humans", - name: "Human visits", + name: "Human events", stackId: "1", color: "var(--color-accent)", fillOpacity: 0.28, @@ -198,7 +237,7 @@ export function pulseHeadline( } return { total: totals.humans, - unit: ["human visit", "human visits"], + unit: ["human event", "human events"], hint: HUMANS_DEFINITION, }; } diff --git a/supabase/migrations/20260921120000_tracker_visitor_rollup.sql b/supabase/migrations/20260921120000_tracker_visitor_rollup.sql new file mode 100644 index 0000000..57c079e --- /dev/null +++ b/supabase/migrations/20260921120000_tracker_visitor_rollup.sql @@ -0,0 +1,213 @@ +-- Visitors: the tracker learns to count people. +-- +-- WHY: every headline the dashboards led with was a count of EVENTS. The +-- bucket rollup (tracker_daily_stats) is bumped once per beacon, and stats.js +-- fires a beacon for the pageview, four scroll depths, every click and every +-- form submit, so "Human visits" was ~3-4 events per page view from anything +-- whose user agent did not say bot. On four properties that read as 51,531 +-- "human visits" in a week while the raw event table held 420 distinct +-- visitor ids in a day, which is also what an independent analytics tool +-- reported. Nothing outside the 24h raw table kept a visitor id, so the +-- product could not state weekly unique visitors at all. +-- +-- WHAT: +-- 1. tracker_visitor_daily_stats — one row per (project, UTC day, visitor), +-- with the event and pageview counts for that visitor that day and which +-- side of the human / bot line the visitor ended the day on. Row count is +-- visitors x days, not events, so it keeps. +-- 2. tracker_touch_visitor — the per-beacon upsert. Returns the visitor's +-- counts after the increment so the ingest route can make one round +-- trip. It also applies the SCRIPTED cap: a "visitor" that produces more +-- than p_cap_events events or p_cap_pageviews page views in one UTC day +-- is not a person reading a website, it is a browser being driven +-- (the ones seen so far: 398 events on 4 page views, 126 page views from +-- one id, ~1 fresh id per page view from a headless Chrome). The row is +-- flipped to kind = 'bot' and stays there for the day; the route then +-- counts every later beacon from it under bot:scripted. +-- 3. tracker_visitor_totals — distinct visitors and their page views over a +-- window and the equal-length window before it, per project. Exact +-- uniques, because the ids are here. +-- 4. tracker_visitor_daily_series — visitors per (project, day), for the +-- dashboard sparklines and the stats page. +-- +-- ROWS BEGIN THE DAY THIS IS APPLIED. There is no history to backfill from: +-- tracker_events keeps 24h. The UI says so beside every visitor figure. +-- +-- security invoker throughout; RLS mirrors tracker_daily_stats (owner or +-- member may read). The ingest route writes with the service role. +-- Idempotent. Apply one file at a time via the Supabase MCP, not `db push`. + +-- --------------------------------------------------------------------------- +-- 1. Table +-- --------------------------------------------------------------------------- +create table if not exists public.tracker_visitor_daily_stats ( + project_id uuid not null references public.projects(id) on delete cascade, + day date not null, + visitor_id text not null, + kind text not null default 'human' check (kind in ('human', 'bot')), + events integer not null default 0, + pageviews integer not null default 0, + first_seen timestamptz not null default now(), + last_seen timestamptz not null default now(), + primary key (project_id, day, visitor_id) +); + +-- The read RPCs count rows per (project, day, kind); the PK prefix covers +-- (project, day) but the filter on kind and the pageviews sum need this. +create index if not exists tracker_visitor_daily_stats_project_day_kind_idx + on public.tracker_visitor_daily_stats (project_id, day, kind) + include (pageviews); + +alter table public.tracker_visitor_daily_stats enable row level security; + +do $$ +begin + if not exists ( + select 1 from pg_policies + where schemaname = 'public' + and tablename = 'tracker_visitor_daily_stats' + and policyname = 'tracker_visitor_daily_stats owner select' + ) then + create policy "tracker_visitor_daily_stats owner select" + on public.tracker_visitor_daily_stats + for select + using ( + project_id in (select id from public.projects where owner_id = auth.uid()) + ); + end if; + + if not exists ( + select 1 from pg_policies + where schemaname = 'public' + and tablename = 'tracker_visitor_daily_stats' + and policyname = 'tracker_visitor_daily_stats member select' + ) then + create policy "tracker_visitor_daily_stats member select" + on public.tracker_visitor_daily_stats + for select + using (public.is_project_member(project_id, auth.uid())); + end if; +end $$; + +grant select on public.tracker_visitor_daily_stats to authenticated; +grant select, insert, update, delete on public.tracker_visitor_daily_stats to service_role; + +-- --------------------------------------------------------------------------- +-- 2. Per-beacon touch (+ scripted cap) +-- --------------------------------------------------------------------------- +create or replace function public.tracker_touch_visitor( + p_project uuid, + p_day date, + p_visitor text, + p_kind text, + p_pageview boolean, + p_cap_events integer default 500, + p_cap_pageviews integer default 200 +) +returns table (kind text, events integer, pageviews integer) +language sql +volatile +security invoker +set search_path = public +as $$ + insert into public.tracker_visitor_daily_stats as s + (project_id, day, visitor_id, kind, events, pageviews, first_seen, last_seen) + values + (p_project, p_day, p_visitor, + -- A single beacon can already blow the cap when the cap is 0; keep the + -- rule in one place by evaluating it on the insert too. + case + when p_kind = 'bot' then 'bot' + when 1 > coalesce(p_cap_events, 500) then 'bot' + when (case when p_pageview then 1 else 0 end) > coalesce(p_cap_pageviews, 200) then 'bot' + else 'human' + end, + 1, (case when p_pageview then 1 else 0 end), now(), now()) + on conflict (project_id, day, visitor_id) do update + set events = s.events + 1, + pageviews = s.pageviews + (case when p_pageview then 1 else 0 end), + last_seen = now(), + -- Sticky: once a bot (by user agent or by volume), a bot for the day. + kind = case + when s.kind = 'bot' or p_kind = 'bot' then 'bot' + when s.events + 1 > coalesce(p_cap_events, 500) then 'bot' + when s.pageviews + (case when p_pageview then 1 else 0 end) > coalesce(p_cap_pageviews, 200) then 'bot' + else 'human' + end + returning s.kind, s.events, s.pageviews; +$$; + +revoke all on function public.tracker_touch_visitor(uuid, date, text, text, boolean, integer, integer) from public; +grant execute on function public.tracker_touch_visitor(uuid, date, text, text, boolean, integer, integer) to service_role; + +-- --------------------------------------------------------------------------- +-- 3. Window totals: distinct visitors + their page views, current and previous +-- --------------------------------------------------------------------------- +create or replace function public.tracker_visitor_totals( + p_projects uuid[], + days integer default 30, + p_kind text default null +) +returns table ( + project_id uuid, + visitors bigint, + prev_visitors bigint, + pageviews bigint, + prev_pageviews bigint +) +language sql +stable +security invoker +set search_path = public +as $$ + with bounds as ( + select (today - (n - 1)) as cur_start, + (today - (2 * n - 1)) as prev_start, + (today - n) as prev_end + from ( + select greatest(coalesce(days, 30), 1) as n, + (now() at time zone 'UTC')::date as today + ) win + ) + select s.project_id, + count(distinct s.visitor_id) filter (where s.day >= b.cur_start)::bigint as visitors, + count(distinct s.visitor_id) filter (where s.day <= b.prev_end)::bigint as prev_visitors, + coalesce(sum(s.pageviews) filter (where s.day >= b.cur_start), 0)::bigint as pageviews, + coalesce(sum(s.pageviews) filter (where s.day <= b.prev_end), 0)::bigint as prev_pageviews + from public.tracker_visitor_daily_stats s + cross join bounds b + where s.project_id = any(p_projects) + and s.day >= b.prev_start + and (p_kind is null or s.kind = p_kind) + group by s.project_id; +$$; + +grant execute on function public.tracker_visitor_totals(uuid[], integer, text) to authenticated, service_role; + +-- --------------------------------------------------------------------------- +-- 4. Daily series: visitors per (project, day) +-- --------------------------------------------------------------------------- +create or replace function public.tracker_visitor_daily_series( + p_projects uuid[], + days integer default 30, + p_kind text default null +) +returns table (project_id uuid, day date, visitors bigint, pageviews bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select s.project_id, + s.day, + count(*)::bigint as visitors, + coalesce(sum(s.pageviews), 0)::bigint as pageviews + from public.tracker_visitor_daily_stats s + where s.project_id = any(p_projects) + and s.day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or s.kind = p_kind) + group by s.project_id, s.day + order by s.project_id, s.day; +$$; + +grant execute on function public.tracker_visitor_daily_series(uuid[], integer, text) to authenticated, service_role; diff --git a/tests/tracker-api-stats.test.ts b/tests/tracker-api-stats.test.ts index b8985dc..f1b4cbf 100644 --- a/tests/tracker-api-stats.test.ts +++ b/tests/tracker-api-stats.test.ts @@ -85,17 +85,21 @@ describe("resolveProject", () => { }); describe("totalsFromSeries", () => { - it("sums the points, counting humans when visitors is absent", () => { + // A series point has no visitor field. The old fallback counted `humans` + // (beacons) as visitors, which is how four sites read 51k "visitors" in a + // week against 420 people a day. People now come from the visitor rollup + // (tests/tracker-visitor-rollup.test.ts); the series only yields events. + it("sums the human beacons as events, never as visitors", () => { expect( - totalsFromSeries({ points: [{ visitors: 3, pageviews: 9 }, { visitors: 1, pageviews: 2 }] } as never), - ).toEqual({ visitors: 4, pageviews: 11 }); - expect(totalsFromSeries({ points: [{ humans: 2, pageviews: 5 }] } as never)).toEqual({ visitors: 2, pageviews: 5 }); + totalsFromSeries({ points: [{ humans: 3, pageviews: 9 }, { humans: 1, pageviews: 2 }] } as never), + ).toEqual({ events: 4, pageviews: 11 }); + expect(totalsFromSeries({ points: [{ visitors: 3, humans: 2, pageviews: 5 }] } as never)).toEqual({ events: 2, pageviews: 5 }); }); it("is zero for a list payload or nothing at all, rather than NaN", () => { - expect(totalsFromSeries(undefined)).toEqual({ visitors: 0, pageviews: 0 }); - expect(totalsFromSeries([] as never)).toEqual({ visitors: 0, pageviews: 0 }); - expect(totalsFromSeries({ points: [{ pageviews: "4" }] } as never)).toEqual({ visitors: 0, pageviews: 4 }); + expect(totalsFromSeries(undefined)).toEqual({ events: 0, pageviews: 0 }); + expect(totalsFromSeries([] as never)).toEqual({ events: 0, pageviews: 0 }); + expect(totalsFromSeries({ points: [{ pageviews: "4" }] } as never)).toEqual({ events: 0, pageviews: 4 }); }); }); diff --git a/tests/tracker-visitor-rollup.test.ts b/tests/tracker-visitor-rollup.test.ts new file mode 100644 index 0000000..78e0de5 --- /dev/null +++ b/tests/tracker-visitor-rollup.test.ts @@ -0,0 +1,214 @@ +import { describe, expect, it } from "vitest"; +import { + SCRIPTED_BUCKET, + SCRIPTED_CAP_EVENTS, + SCRIPTED_CAP_PAGEVIEWS, + applyScriptedDemotion, + parseVisitorTouch, +} from "@/lib/tracker/scripted"; +import { + HUMANS_LABEL, + PAGEVIEWS_LABEL, + VISITORS_LABEL, + kindFromBucket, +} from "@/lib/tracker/humans"; +import { headlineTiles, pulseHeadline } from "@/lib/tracker/who"; +import { + VISITORS_SINCE, + sumVisitorTotals, + toVisitorTotals, + visitorsPartial, +} from "@/lib/tracker/visitors"; +import { mergeTotals, totalsFromSeries } from "@/lib/tracker/apiStats"; +import { renderStats } from "@/lib/dashboard/stats-text"; + +// The tracker used to lead with "Human visits", which was every beacon from a +// non-crawler user agent: the page view plus four scroll depths, every click +// and every form submit. On four properties that read 51,531 in a week +// while the raw table held 420 distinct visitor ids in a day — the number an +// independent analytics tool also reported. These pin the pieces that keep +// people and events apart: the scripted cap, the tile order, the API totals +// and the CLI line. + +describe("applyScriptedDemotion", () => { + it("leaves a user-agent bot alone, whatever the rollup says", () => { + const out = applyScriptedDemotion("bot:gptbot", { kind: "bot", events: 1, pageviews: 1 }); + expect(out).toEqual({ bucket: "bot:gptbot", kind: "bot", demoted: false }); + }); + + it("keeps a human hit human while the rollup still calls the visitor human", () => { + const out = applyScriptedDemotion("search:google", { kind: "human", events: 40, pageviews: 12 }); + expect(out).toEqual({ bucket: "search:google", kind: "human", demoted: false }); + }); + + it("demotes a human hit once the rollup has flipped the visitor to bot", () => { + const out = applyScriptedDemotion("referral:bittorrented.com", { + kind: "bot", + events: SCRIPTED_CAP_EVENTS + 1, + pageviews: 4, + }); + expect(out).toEqual({ bucket: SCRIPTED_BUCKET, kind: "bot", demoted: true }); + expect(kindFromBucket(out.bucket)).toBe("bot"); + }); + + it("fails open: no visitor id or a failed RPC leaves the verdict standing", () => { + expect(applyScriptedDemotion("human:direct", null)).toEqual({ + bucket: "human:direct", + kind: "human", + demoted: false, + }); + }); + + it("caps are per visitor per day and generous enough for a binge reader", () => { + expect(SCRIPTED_CAP_PAGEVIEWS).toBeGreaterThanOrEqual(150); + expect(SCRIPTED_CAP_EVENTS).toBeGreaterThan(SCRIPTED_CAP_PAGEVIEWS); + }); +}); + +describe("parseVisitorTouch", () => { + it("reads the one row the RPC returns, as an array or bare, with bigint strings", () => { + expect(parseVisitorTouch([{ kind: "human", events: "3", pageviews: "1" }])).toEqual({ + kind: "human", + events: 3, + pageviews: 1, + }); + expect(parseVisitorTouch({ kind: "bot", events: 501, pageviews: 4 })).toEqual({ + kind: "bot", + events: 501, + pageviews: 4, + }); + }); + + it("is null for anything that is not a kind", () => { + expect(parseVisitorTouch(null)).toBeNull(); + expect(parseVisitorTouch([])).toBeNull(); + expect(parseVisitorTouch({ kind: "unknown", events: 1 })).toBeNull(); + expect(parseVisitorTouch("bot")).toBeNull(); + }); +}); + +describe("headlineTiles with the visitor rollup", () => { + const totals = { visitors: 420, pageviews: 1585, humans: 5796, ai: 12, bots: 90 }; + + it("leads with people, then their page views, then events under their real name", () => { + const tiles = headlineTiles("humans", totals); + expect(tiles.map((t) => [t.key, t.value])).toEqual([ + ["visitors", 420], + ["pageviews", 1585], + ["humans", 5796], + ["ai", 12], + ]); + expect(tiles[0].label).toBe(VISITORS_LABEL); + expect(tiles[1].label).toBe(PAGEVIEWS_LABEL); + expect(tiles[2].label).toBe(HUMANS_LABEL); + expect(HUMANS_LABEL).not.toMatch(/visit/i); + // The event count is no longer the accent figure. + expect(tiles[0].tone).toBe("accent"); + expect(tiles[2].tone).toBe("muted"); + }); + + it("shows the bot side's own visitors under Bots", () => { + const tiles = headlineTiles("bots", { ...totals, visitors: 7, pageviews: 30 }); + expect(tiles.map((t) => t.key)).toEqual(["visitors", "pageviews", "bots"]); + expect(tiles[0].label).toBe("Bot visitors"); + }); + + it("keeps both sides under All", () => { + expect(headlineTiles("all", totals).map((t) => t.key)).toEqual([ + "visitors", + "pageviews", + "humans", + "ai", + "bots", + ]); + }); + + it("leaves the people tiles out — never 0 — when the rollup is unavailable", () => { + const tiles = headlineTiles("humans", { ...totals, visitors: null, pageviews: null }); + expect(tiles.map((t) => t.key)).toEqual(["humans", "ai"]); + expect(tiles[0].tone).toBe("accent"); + // And the pre-rollup call shape is unchanged. + expect(headlineTiles("humans", { humans: 3, ai: 1, bots: 9 }).map((t) => t.key)).toEqual([ + "humans", + "ai", + ]); + }); + + it("names the pulse figure as events, not visits", () => { + expect(pulseHeadline("humans", { humans: 3, bots: 9 }).unit).toEqual([ + "human event", + "human events", + ]); + }); +}); + +describe("visitor totals", () => { + it("coerces PostgREST strings and sums across projects", () => { + const a = toVisitorTotals({ project_id: "a", visitors: "372", prev_visitors: "0", pageviews: "1469", prev_pageviews: "0" }); + const b = toVisitorTotals({ project_id: "b", visitors: 30, prev_visitors: 4, pageviews: 73, prev_pageviews: 9 }); + expect(sumVisitorTotals([a, b])).toEqual({ + visitors: 402, + prevVisitors: 4, + pageviews: 1542, + prevPageviews: 9, + }); + }); + + it("knows when a window reaches back before the rollup existed", () => { + const since = new Date(`${VISITORS_SINCE}T12:00:00Z`); + expect(visitorsPartial(1, since)).toBe(false); + expect(visitorsPartial(2, since)).toBe(true); + const later = new Date(since.getTime() + 30 * 86_400_000); + expect(visitorsPartial(30, later)).toBe(false); + expect(visitorsPartial(31, later)).toBe(false); + expect(visitorsPartial(32, later)).toBe(true); + }); +}); + +describe("API totals", () => { + it("never reports events as visitors", () => { + const fromSeries = totalsFromSeries({ + points: [ + { humans: 2130, pageviews: 700 }, + { humans: 3666, pageviews: 885 }, + ], + } as never); + expect(fromSeries).toEqual({ events: 5796, pageviews: 1585 }); + expect("visitors" in fromSeries).toBe(false); + }); + + it("takes people from the rollup and events from the series", () => { + expect(mergeTotals({ events: 5796, pageviews: 1585 }, { visitors: 420, pageviews: 1469 })).toEqual({ + visitors: 420, + pageviews: 1469, + events: 5796, + }); + }); + + it("reports an unreadable rollup as null, not 0", () => { + expect(mergeTotals({ events: 5796, pageviews: 1585 }, null)).toEqual({ + visitors: null, + pageviews: null, + events: 5796, + }); + }); +}); + +describe("CLI stats line", () => { + it("prints people, page views and events, in that order", () => { + const text = renderStats( + { project: { name: "bittorrented.com" }, totals: { visitors: 420, pageviews: 1469, events: 5796 } }, + { range: "1d", who: "humans" }, + ); + expect(text).toContain("420 visitors, 1469 pageviews, 5796 events"); + }); + + it("says visitors are unavailable rather than printing 0", () => { + const text = renderStats( + { project: { name: "x" }, totals: { visitors: null, pageviews: null, events: 12 } }, + { range: "1d", who: "humans" }, + ); + expect(text).toContain("visitors unavailable, 12 events"); + expect(text).not.toContain("Nothing in this window"); + }); +});