diff --git a/app/(app)/dashboard/projects/[id]/stats/live-visitors.tsx b/app/(app)/dashboard/projects/[id]/stats/live-visitors.tsx index 3ff85b1f..f77219ca 100644 --- a/app/(app)/dashboard/projects/[id]/stats/live-visitors.tsx +++ b/app/(app)/dashboard/projects/[id]/stats/live-visitors.tsx @@ -3,6 +3,7 @@ import { useEffect, useState, useCallback } from "react"; import { VisitorGlobe, type GlobePoint } from "./visitor-globe"; import { LiveChart } from "./live-chart"; +import { DEFAULT_WHO, WHO_PARAM, type Who } from "@/lib/tracker/who"; type LiveEvent = { id: number; @@ -48,7 +49,14 @@ function flagEmoji(code: string) { ); } -export function LiveVisitors({ projectId }: { projectId: string }) { +export function LiveVisitors({ + projectId, + who = DEFAULT_WHO, +}: { + projectId: string; + /** The page-wide Humans / Bots / All toggle; the feed is filtered to it. */ + who?: Who; +}) { const [data, setData] = useState(null); const [loading, setLoading] = useState(true); const [lastUpdated, setLastUpdated] = useState(null); @@ -68,7 +76,7 @@ export function LiveVisitors({ projectId }: { projectId: string }) { const fetchData = useCallback(async () => { try { const res = await fetch( - `/api/projects/${projectId}/live-events?minutes=${minutes}`, + `/api/projects/${projectId}/live-events?minutes=${minutes}&${WHO_PARAM}=${who}`, { cache: "no-store" }, ); if (!res.ok) return; @@ -80,7 +88,7 @@ export function LiveVisitors({ projectId }: { projectId: string }) { } finally { setLoading(false); } - }, [projectId, minutes]); + }, [projectId, minutes, who]); useEffect(() => { fetchData(); @@ -102,6 +110,11 @@ export function LiveVisitors({ projectId }: { projectId: string }) {

Live

+ {who !== "all" && ( + + {who} + + )} {lastUpdated && ( · refreshes every 30s · {relTime(lastUpdated.toISOString())} @@ -148,7 +161,10 @@ export function LiveVisitors({ projectId }: { projectId: string }) {

Loading…

) : totalEvents === 0 ? (

- No events in the last {minutes} min. Traffic will appear once the tracker script fires. + No {who === "all" ? "" : who === "humans" ? "human " : "bot "}events in the last {minutes} min.{" "} + {who === "all" + ? "Traffic will appear once the tracker script fires." + : "Switch to All to see everything the tracker has recorded."}

) : ( <> diff --git a/app/(app)/dashboard/projects/[id]/stats/page.tsx b/app/(app)/dashboard/projects/[id]/stats/page.tsx index 65c0c8cb..c9eb1771 100644 --- a/app/(app)/dashboard/projects/[id]/stats/page.tsx +++ b/app/(app)/dashboard/projects/[id]/stats/page.tsx @@ -9,20 +9,15 @@ import { type TrackerPanels, } from "@/components/charts/tracker-analytics"; import { fetchPanels, PANEL_KEYS } from "@/lib/tracker/panels"; -import { - AI_REFERRALS_DEFINITION, - BOTS_DEFINITION, - BOTS_LABEL, - HUMANS_DEFINITION, - HUMANS_LABEL, -} from "@/lib/tracker/humans"; import { DEFAULT_TRACKER_RANGE, trackerRange } from "@/lib/tracker/ranges"; +import { headlineTiles, whoOrDefault, whoToKind } from "@/lib/tracker/who"; import { InstallSnippet } from "./install-snippet"; import { TrackerToggle } from "./tracker-toggle"; import { CareersToggle } from "./careers-toggle"; import { AutoInstall } from "./auto-install"; import { LiveVisitors } from "./live-visitors"; import { StatsSubnav } from "./stats-subnav"; +import { WhoToggle } from "./who-toggle"; import { getOrMintInstallationToken } from "@/lib/github/installations"; import { listInstallationRepos } from "@/lib/github/app"; @@ -36,10 +31,17 @@ type BoundRepo = { export default async function ProjectStatsPage({ params, + searchParams, }: { params: Promise<{ id: string }>; + searchParams: Promise<{ who?: string }>; }) { const { id } = await params; + // The page-wide Humans / Bots / All toggle. Lives in the URL so a link to + // the page carries the reader's choice; junk falls back to humans, the + // side the page leads with. See lib/tracker/who.ts. + const who = whoOrDefault((await searchParams).who); + const kind = whoToKind(who); const supabase = await createClient(); const { data: project } = await supabase .from("projects") @@ -59,25 +61,35 @@ export default async function ProjectStatsPage({ // timeframe tabs and re-fetches just itself from // /api/projects/:id/tracker-stats, so narrowing one chart to the last hour // does not re-run the other eleven. + // + // Every panel is fetched at the toggle's `kind`, so a card and the tiles + // above it never disagree on who they are counting. const range = trackerRange(DEFAULT_TRACKER_RANGE); const panels = (await fetchPanels( supabase, id, PANEL_KEYS, range, + kind, )) as unknown as TrackerPanels; // Headline metrics come straight from the series so they stay exact even // though Top sources below is truncated to the top 10 buckets. The page // leads with humans (every bucket that is not `bot:`, AI referrals // included) and shows bot crawls apart — see lib/tracker/humans.ts for why - // the old bot-inclusive total is no longer a headline. + // the old bot-inclusive total is no longer a headline. Under Humans or + // Bots only that side's tiles are shown (lib/tracker/who.ts). const points = panels.series.points; const totalAi = points.reduce((s, p) => s + p.ai, 0); const totalBot = points.reduce((s, p) => s + p.bots, 0); const totalHuman = points.reduce((s, p) => s + p.humans, 0); const grandTotal = points.reduce((s, p) => s + p.events, 0); const eventTotal = points.reduce((s, p) => s + p.pageviews + p.interactions, 0); + const tiles = headlineTiles(who, { + humans: totalHuman, + ai: totalAi, + bots: totalBot, + }); // Older projects have rollup rows in tracker_daily_stats but nothing in // tracker_event_daily_stats, which would leave Event mix empty on a page @@ -160,7 +172,7 @@ export default async function ProjectStatsPage({
- +
@@ -237,40 +249,54 @@ export default async function ProjectStatsPage({
)} +
+
+

Traffic

+

+ {range.description}. Every figure below counts{" "} + {who === "humans" + ? "people only" + : who === "bots" + ? "crawlers only" + : "people and crawlers together"} + . +

+
+ +
+
- - - + {tiles.map((tile) => ( + + ))}
{grandTotal === 0 && eventTotal === 0 ? (

{trackerEnabled - ? "No events yet. Once the snippet is installed and your site gets traffic, sources will appear here." + ? who === "all" + ? "No events yet. Once the snippet is installed and your site gets traffic, sources will appear here." + : `No ${who === "humans" ? "human" : "bot"} events in this window. Switch to All to see everything the tracker has recorded.` : "Enable the tracker to start collecting events."}

) : ( + /* Keyed on the toggle: a change re-renders this page with panels + fetched at the new filter, and the remount hands every card that + fresh initial data instead of a stale cache. */ )}
diff --git a/app/(app)/dashboard/projects/[id]/stats/who-toggle.tsx b/app/(app)/dashboard/projects/[id]/stats/who-toggle.tsx new file mode 100644 index 00000000..42964c9d --- /dev/null +++ b/app/(app)/dashboard/projects/[id]/stats/who-toggle.tsx @@ -0,0 +1,102 @@ +"use client"; + +import { useRef, useTransition } from "react"; +import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { + DEFAULT_WHO, + WHO_OPTIONS, + WHO_PARAM, + whoCaption, + type Who, +} from "@/lib/tracker/who"; + +// The page-wide Humans / Bots / All control on the project stats page. +// +// The selection lives in the URL (?who=), not in component state: the page +// is a server component that reads it, fetches every panel and the headline +// tiles at that filter, and keys the chart cards on it so they remount on +// fresh data. This control only navigates. Replacing rather than pushing +// keeps the back button for leaving the page, not for undoing a toggle, and +// the transition dims the strip while the new numbers are on their way so a +// click never looks ignored. +// +// Same visual language as the timeframe tabs in each card header, with +// arrow-key roving so keyboard users are not stranded on the selected option. +export function WhoToggle({ value }: { value: Who }) { + const router = useRouter(); + const pathname = usePathname(); + const searchParams = useSearchParams(); + const [pending, start] = useTransition(); + const refs = useRef>([]); + + function select(next: Who) { + if (next === value) return; + const params = new URLSearchParams(searchParams?.toString()); + if (next === DEFAULT_WHO) params.delete(WHO_PARAM); + else params.set(WHO_PARAM, next); + const qs = params.toString(); + start(() => { + router.replace(qs ? `${pathname}?${qs}` : pathname, { scroll: false }); + }); + } + + function onKeyDown(event: React.KeyboardEvent, index: number) { + const delta = + event.key === "ArrowRight" ? 1 : event.key === "ArrowLeft" ? -1 : 0; + if (!delta) return; + event.preventDefault(); + const next = (index + delta + WHO_OPTIONS.length) % WHO_OPTIONS.length; + refs.current[next]?.focus(); + select(WHO_OPTIONS[next].key); + } + + const caption = whoCaption(value); + + return ( +
+
+ {WHO_OPTIONS.map((option, index) => { + const selected = option.key === value; + return ( + + ); + })} +
+ {caption && ( +

+ {caption} +

+ )} +
+ ); +} diff --git a/app/api/projects/[id]/live-events/route.ts b/app/api/projects/[id]/live-events/route.ts index d04c91ae..4d48c8df 100644 --- a/app/api/projects/[id]/live-events/route.ts +++ b/app/api/projects/[id]/live-events/route.ts @@ -1,11 +1,14 @@ -// GET /api/projects/[id]/live-events?minutes=30 -// Returns recent raw tracker events for the real-time stats panel. +// GET /api/projects/[id]/live-events?minutes=30&who=humans +// Returns recent raw tracker events for the real-time stats panel. `who` is +// the page-wide Humans / Bots / All toggle (lib/tracker/who.ts) and filters +// on the bucket prefix, which the raw row has always carried; junk is a 400. // Requires project owner or member auth. import { NextRequest, NextResponse } from "next/server"; import { requireProjectAccess } from "@/lib/lx/currentSite"; import { serviceClient } from "@/lib/supabase/service"; import { bucketLabel } from "@/lib/tracker/categorize"; +import { DEFAULT_WHO, parseWho, WHO_PARAM, whoToKind } from "@/lib/tracker/who"; // Approximate country centroids — fallback when precise lat/lng is missing. const COUNTRY_CENTROIDS: Record = { @@ -47,14 +50,28 @@ export async function GET( ); const since = new Date(Date.now() - minutes * 60 * 1000).toISOString(); + const whoParam = request.nextUrl.searchParams.get(WHO_PARAM); + const who = whoParam === null ? DEFAULT_WHO : parseWho(whoParam); + if (!who) { + return NextResponse.json( + { error: "Unknown who. Expected humans, bots or all." }, + { status: 400 }, + ); + } + const kind = whoToKind(who); + const svc = serviceClient(); - const { data: rows, error } = await svc + let query = svc .from("tracker_events") .select( "id, occurred_at, event, page_path, referrer_host, event_target, bucket, country_code, country_name, city, lat, lng, visitor_id, session_id", ) .eq("project_id", projectId) - .gte("occurred_at", since) + .gte("occurred_at", since); + // Same rule as the RPCs' p_kind: bot iff the bucket starts with "bot:". + if (kind === "bot") query = query.like("bucket", "bot:%"); + if (kind === "human") query = query.not("bucket", "like", "bot:%"); + const { data: rows, error } = await query .order("occurred_at", { ascending: false }) .limit(500); @@ -131,6 +148,7 @@ export async function GET( return NextResponse.json({ minutes, + who, total_events: events.length, unique_sessions: sessionsSeen.size, // Up to 300 so the live chart can bucket a fuller slice of the window; diff --git a/app/api/projects/[id]/tracker-stats/route.ts b/app/api/projects/[id]/tracker-stats/route.ts index ae226fde..2e7dee8e 100644 --- a/app/api/projects/[id]/tracker-stats/route.ts +++ b/app/api/projects/[id]/tracker-stats/route.ts @@ -1,4 +1,4 @@ -// GET /api/projects/[id]/tracker-stats?range=1h&panel=pages +// GET /api/projects/[id]/tracker-stats?range=1h&panel=pages&who=humans // // Re-renders one stats panel at a different timeframe. The stats page // server-renders every panel at the default range; each card's timeframe tabs @@ -6,6 +6,9 @@ // aggregate instead of a full page reload. // // `panel` may be repeated (or comma-separated) to fetch several at once. +// `who` is the page-wide Humans / Bots / All toggle (lib/tracker/who.ts); +// absent means the page default, anything else than the three values is a +// 400 rather than a silent fall-back to a different answer. // Requires project owner or member auth, same as the other project routes. import { NextRequest, NextResponse } from "next/server"; @@ -13,6 +16,7 @@ import { requireProjectAccess } from "@/lib/lx/currentSite"; import { serviceClient } from "@/lib/supabase/service"; import { trackerRange, rangesForPanel } from "@/lib/tracker/ranges"; import { fetchPanels, PANEL_KEYS, type PanelKey } from "@/lib/tracker/panels"; +import { DEFAULT_WHO, parseWho, WHO_PARAM, whoToKind } from "@/lib/tracker/who"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; @@ -32,6 +36,15 @@ export async function GET( const sp = request.nextUrl.searchParams; const range = trackerRange(sp.get("range")); + const whoParam = sp.get(WHO_PARAM); + const who = whoParam === null ? DEFAULT_WHO : parseWho(whoParam); + if (!who) { + return NextResponse.json( + { error: "Unknown who. Expected humans, bots or all." }, + { status: 400 }, + ); + } + const requested = sp .getAll("panel") .flatMap((v) => v.split(",")) @@ -61,8 +74,14 @@ export async function GET( } try { - const data = await fetchPanels(serviceClient(), projectId, panels, range); - return NextResponse.json({ range: range.key, panels: data }); + const data = await fetchPanels( + serviceClient(), + projectId, + panels, + range, + whoToKind(who), + ); + return NextResponse.json({ range: range.key, who, panels: data }); } catch (err) { return NextResponse.json( { error: err instanceof Error ? err.message : "Stats query failed." }, diff --git a/app/api/track/route.ts b/app/api/track/route.ts index c94b49cf..d43c5665 100644 --- a/app/api/track/route.ts +++ b/app/api/track/route.ts @@ -6,6 +6,7 @@ import { NextRequest, NextResponse } from "next/server"; import { z } from "zod"; import { serviceClient } from "@/lib/supabase/service"; import { categorize } from "@/lib/tracker/categorize"; +import { kindFromBucket } from "@/lib/tracker/humans"; import { parseDevice } from "@/lib/tracker/device"; import { clientIpFromHeaders, lookupGeo } from "@/lib/tracker/geo"; import { enqueuePostHogEvent } from "@/lib/posthog/events"; @@ -182,6 +183,10 @@ async function ingest(request: NextRequest, parseBody: boolean) { if (gate.action !== "allow") return refuse(gate); const { bucket, isAi } = categorize({ referrer, userAgent }); + // Which side of the human / bot line this hit counts on. The bucket table + // carries the whole bucket; the other rollups record only this, so the + // stats page can split every breakdown, not just the headline. + const kind = kindFromBucket(bucket); const today = new Date().toISOString().slice(0, 10); // YYYY-MM-DD UTC // UPSERT increment. Supabase JS doesn't expose a raw .increment() helper @@ -221,6 +226,7 @@ async function ingest(request: NextRequest, parseBody: boolean) { .eq("page_path", pagePath) .eq("referrer_host", referrerHost) .eq("event_target", eventTarget) + .eq("kind", kind) .maybeSingle(); if (eventExisting) { @@ -235,7 +241,8 @@ async function ingest(request: NextRequest, parseBody: boolean) { .eq("event", event) .eq("page_path", pagePath) .eq("referrer_host", referrerHost) - .eq("event_target", eventTarget); + .eq("event_target", eventTarget) + .eq("kind", kind); } else { await sb.from("tracker_event_daily_stats").insert({ project_id: site, @@ -244,6 +251,7 @@ async function ingest(request: NextRequest, parseBody: boolean) { page_path: pagePath, referrer_host: referrerHost, event_target: eventTarget, + kind, count: 1, }); } @@ -252,7 +260,7 @@ async function ingest(request: NextRequest, parseBody: boolean) { // Best-effort — analytics writes must never break the beacon response. if (event === "pageview" && parsed.data.sessionId) { try { - await updateExitRollup(sb, site, parsed.data.sessionId, pagePath, today); + await updateExitRollup(sb, site, parsed.data.sessionId, pagePath, today, kind); await pruneExitSessions(sb, site); } catch { // Silent — never affect the 204. @@ -271,6 +279,7 @@ async function ingest(request: NextRequest, parseBody: boolean) { .eq("device_type", device.deviceType) .eq("browser", device.browser) .eq("os", device.os) + .eq("kind", kind) .maybeSingle(); if (deviceExisting) { @@ -284,7 +293,8 @@ async function ingest(request: NextRequest, parseBody: boolean) { .eq("day", today) .eq("device_type", device.deviceType) .eq("browser", device.browser) - .eq("os", device.os); + .eq("os", device.os) + .eq("kind", kind); } else { await sb.from("tracker_device_daily_stats").insert({ project_id: site, @@ -292,6 +302,7 @@ async function ingest(request: NextRequest, parseBody: boolean) { device_type: device.deviceType, browser: device.browser, os: device.os, + kind, count: 1, }); } @@ -344,6 +355,7 @@ async function ingest(request: NextRequest, parseBody: boolean) { .eq("region_code", geo.regionCode) .eq("city", geo.city) .eq("timezone", geo.timezone) + .eq("kind", kind) .maybeSingle(); if (geoExisting) { @@ -360,7 +372,8 @@ async function ingest(request: NextRequest, parseBody: boolean) { .eq("country_code", geo.countryCode) .eq("region_code", geo.regionCode) .eq("city", geo.city) - .eq("timezone", geo.timezone); + .eq("timezone", geo.timezone) + .eq("kind", kind); } else { await sb.from("tracker_geo_daily_stats").insert({ project_id: site, @@ -371,6 +384,7 @@ async function ingest(request: NextRequest, parseBody: boolean) { region_name: geo.regionName, city: geo.city, timezone: geo.timezone, + kind, count: 1, }); } diff --git a/components/charts/tracker-analytics.tsx b/components/charts/tracker-analytics.tsx index d1d23fdd..bf8f7b31 100644 --- a/components/charts/tracker-analytics.tsx +++ b/components/charts/tracker-analytics.tsx @@ -29,6 +29,12 @@ import { BOTS_DEFINITION, HUMANS_DEFINITION, } from "@/lib/tracker/humans"; +import { + DEFAULT_WHO, + pulseHeadline, + pulseLayers, + type Who, +} from "@/lib/tracker/who"; export type TrackerDailyPoint = { date: string; @@ -73,13 +79,20 @@ export function TrackerAnalytics({ projectId, initial, initialRange = DEFAULT_TRACKER_RANGE, + who = DEFAULT_WHO, }: { /** Omitted by the portfolio page, which has no single-project endpoint. */ projectId?: string; initial: TrackerPanels; initialRange?: TrackerRangeKey; + /** + * The page-wide Humans / Bots / All toggle `initial` was rendered at. Every + * card sends it with its range requests; the page keys this component on + * it so a toggle remounts the cards on fresh initial data. + */ + who?: Who; }) { - const common = { projectId, initialRange }; + const common = { projectId, initialRange, who }; return (
@@ -265,10 +278,12 @@ function TrafficPulse({ projectId, initialData, initialRange, + who, }: { projectId?: string; initialData: SeriesPayload; initialRange: TrackerRangeKey; + who: Who; }) { const { ranges, range, setRange, data, loading, error, showTabs } = usePanelRange( @@ -276,24 +291,27 @@ function TrafficPulse({ "series", initialData, initialRange, + who, ); const points = data?.points ?? []; // Lead with people. Older rollup days predate the bucket table and carry // humans: 0, so the frame total is the human figure, not the old // bot-inclusive event count — which is exactly the number this chart used - // to mislead with. + // to mislead with. Under the Bots toggle the frame leads with bots instead. const humans = points.reduce((sum, point) => sum + point.humans, 0); const bots = points.reduce((sum, point) => sum + point.bots, 0); const byTime = data?.granularity === "time"; + const headline = pulseHeadline(who, { humans, bots }); + const layers = pulseLayers(who); return ( - {/* The stack is humans + bots = every event, split honestly. - The bot-inclusive pageview series is gone: on a crawled site it + {/* Under All the stack is humans + bots = every event, split + honestly; Humans and Bots draw their own band alone. The + bot-inclusive pageview series is gone: on a crawled site it was indistinguishable from the bot band and double-counted it. AI referrals (a subset of humans) and interactions (clicks and - forms, whoever fired them) are unstacked overlays. */} - - - - + forms, on the same side as the band) are unstacked overlays. + Which layers appear is decided in lib/tracker/who.ts. */} + {layers.map((layer) => ( + + ))}
-
-
Human visits:
-
{HUMANS_DEFINITION}
-
-
-
Bot crawls:
-
{BOTS_DEFINITION}
-
-
-
AI referrals:
-
{AI_REFERRALS_DEFINITION}
-
+ {who !== "bots" && ( +
+
Human visits:
+
{HUMANS_DEFINITION}
+
+ )} + {who !== "humans" && ( +
+
Bot crawls:
+
{BOTS_DEFINITION}
+
+ )} + {who !== "bots" && ( +
+
AI referrals:
+
{AI_REFERRALS_DEFINITION}
+
+ )}
); } +function pulseSubtitle(who: Who, bots: number) { + switch (who) { + case "humans": + return "Human visits only. AI referrals are the share arriving from an AI assistant; interactions are clicks and form events by people."; + case "bots": + return "Bot crawls only: AI training and retrieval bots, search engine bots and other automated clients. Interactions are the rare click or form event a bot fires."; + default: + return `Human visits and bot crawls, stacked. AI referrals are the share of humans arriving from an AI assistant; interactions are clicks and form events. ${bots.toLocaleString()} bot ${bots === 1 ? "hit" : "hits"} in this window.`; + } +} + function BreakdownPanel({ projectId, panel, title, initialData, initialRange, + who, }: { projectId?: string; panel: string; title: string; initialData: ListItem[]; initialRange: TrackerRangeKey; + who: Who; }) { const { ranges, range, setRange, data, loading, error, showTabs } = - usePanelRange(projectId, panel, initialData, initialRange); + usePanelRange(projectId, panel, initialData, initialRange, who); const rows = data ?? []; const total = rows.reduce((sum, row) => sum + row.value, 0); @@ -478,6 +492,7 @@ function RankedPanel({ empty, initialData, initialRange, + who, }: { projectId?: string; panel: string; @@ -485,9 +500,10 @@ function RankedPanel({ empty: string; initialData: ListItem[]; initialRange: TrackerRangeKey; + who: Who; }) { const { ranges, range, setRange, data, loading, error, showTabs } = - usePanelRange(projectId, panel, initialData, initialRange); + usePanelRange(projectId, panel, initialData, initialRange, who); const rows = data ?? []; const total = rows.reduce((sum, row) => sum + row.value, 0); diff --git a/components/charts/use-panel-range.ts b/components/charts/use-panel-range.ts index 54da23d4..46d5b958 100644 --- a/components/charts/use-panel-range.ts +++ b/components/charts/use-panel-range.ts @@ -7,6 +7,7 @@ import { type TrackerRangeKey, } from "@/lib/tracker/ranges"; import type { PanelPayload } from "@/lib/tracker/panels"; +import { DEFAULT_WHO, WHO_PARAM, type Who } from "@/lib/tracker/who"; // Range state + fetching for one stats card. // @@ -15,6 +16,14 @@ import type { PanelPayload } from "@/lib/tracker/panels"; // the life of the card — flipping back and forth between 1H and 1M is a // common way to read these charts, and re-querying on each flip makes the // comparison feel slower than the data it is showing. +// +// `who` is the page-wide Humans / Bots / All toggle. It is part of the cache +// key and of the request, so a card asked for 1H under Humans and then under +// Bots holds both and never shows one side's shape under the other's label. +// `initialData` is only trusted for the (`initialRange`, `who`) it was +// rendered at; the page re-renders with fresh initial data when the toggle +// changes, and remounts the cards so the cache starts over from it. +// // `projectId` is undefined on the portfolio analytics page, which aggregates // across every project and drives its own page-wide range control — there is // no single-project endpoint to ask, so those panels stay on what the server @@ -24,6 +33,7 @@ export function usePanelRange( panel: string, initialData: T, initialRange: TrackerRangeKey = DEFAULT_TRACKER_RANGE, + who: Who = DEFAULT_WHO, ) { const ranges = rangesForPanel(panel); const [range, setRange] = useState(initialRange); @@ -31,7 +41,9 @@ export function usePanelRange( const [loading, setLoading] = useState(false); const [error, setError] = useState(null); - const cache = useRef(new Map([[initialRange, initialData]])); + const cache = useRef( + new Map([[cacheKey(who, initialRange), initialData]]), + ); // Guards against a slow early request landing after a later one and // repainting the card with the range the reader already moved off. const latest = useRef(0); @@ -39,7 +51,7 @@ export function usePanelRange( useEffect(() => { if (!projectId) return; - const cached = cache.current.get(range); + const cached = cache.current.get(cacheKey(who, range)); if (cached) { setData(cached); setError(null); @@ -54,15 +66,14 @@ export function usePanelRange( (async () => { try { - const res = await fetch( - `/api/projects/${projectId}/tracker-stats?range=${range}&panel=${panel}`, - { signal: controller.signal }, - ); + const res = await fetch(panelUrl(projectId, panel, range, who), { + signal: controller.signal, + }); const body = await res.json(); if (!res.ok) throw new Error(body?.error ?? `HTTP ${res.status}`); const payload = body?.panels?.[panel] as T | undefined; if (payload === undefined) throw new Error("Panel missing from response."); - cache.current.set(range, payload); + cache.current.set(cacheKey(who, range), payload); if (seq === latest.current) { setData(payload); setLoading(false); @@ -76,7 +87,7 @@ export function usePanelRange( })(); return () => controller.abort(); - }, [projectId, panel, range]); + }, [projectId, panel, range, who]); const changeRange = useCallback((key: TrackerRangeKey) => setRange(key), []); @@ -91,3 +102,18 @@ export function usePanelRange( showTabs: !!projectId, }; } + +function cacheKey(who: Who, range: TrackerRangeKey) { + return `${who}:${range}`; +} + +/** The request one card makes for one (range, who). Exported for tests. */ +export function panelUrl( + projectId: string, + panel: string, + range: TrackerRangeKey, + who: Who, +) { + const params = new URLSearchParams({ range, panel, [WHO_PARAM]: who }); + return `/api/projects/${projectId}/tracker-stats?${params.toString()}`; +} diff --git a/lib/tracker/exit.ts b/lib/tracker/exit.ts index 00b225bb..11aac64a 100644 --- a/lib/tracker/exit.ts +++ b/lib/tracker/exit.ts @@ -8,16 +8,27 @@ // increment the new one. tracker_exit_daily_stats therefore always reflects, // for each session, exactly one exit page — its latest pageview so far. // +// Each exit row is keyed by `kind` (human / bot, lib/tracker/humans.ts) as +// well as day and path, so the stats page can show human exits apart from +// crawler exits. The session record remembers the kind it was counted under, +// and the decrement uses that, never the current hit's: a session that began +// before the kind-split migration sits on an `unknown` row and must be taken +// off that row, not off a `human` row that was never incremented. +// // Lives here (rather than inline in /api/track) so it's unit-testable without // dragging in the full ingest route, mirroring lib/tracker/device.ts & geo.ts. import type { SupabaseClient } from "@supabase/supabase-js"; +import type { TrackerKind } from "@/lib/tracker/humans"; // The service client is typed permissively (DB = any) in this repo, so accept // any Supabase client here too. // eslint-disable-next-line @typescript-eslint/no-explicit-any type Sb = SupabaseClient; +/** What the rollup row records; `unknown` only on rows from before the split. */ +export type ExitKind = TrackerKind | "unknown"; + // Bump a single exit-page counter by `delta` using the same read-modify-write // upsert pattern as the other tracker rollups. Counts are clamped at 0 so a // decrement racing ahead of its matching increment can never go negative. @@ -27,6 +38,7 @@ export async function bumpExit( day: string, pagePath: string, delta: number, + kind: ExitKind, ) { const { data: existing } = await sb .from("tracker_exit_daily_stats") @@ -34,6 +46,7 @@ export async function bumpExit( .eq("project_id", project) .eq("day", day) .eq("page_path", pagePath) + .eq("kind", kind) .maybeSingle(); if (existing) { @@ -45,11 +58,12 @@ export async function bumpExit( }) .eq("project_id", project) .eq("day", day) - .eq("page_path", pagePath); + .eq("page_path", pagePath) + .eq("kind", kind); } else if (delta > 0) { await sb .from("tracker_exit_daily_stats") - .insert({ project_id: project, day, page_path: pagePath, count: delta }); + .insert({ project_id: project, day, page_path: pagePath, kind, count: delta }); } } @@ -61,30 +75,41 @@ export async function updateExitRollup( sessionId: string, pagePath: string, today: string, + kind: TrackerKind, ) { const { data: session } = await sb .from("tracker_exit_sessions") - .select("last_page_path, last_day") + .select("last_page_path, last_day, kind") .eq("project_id", project) .eq("session_id", sessionId) .maybeSingle(); if (session) { - // Already counted on the same page and day — nothing to move. - if (session.last_page_path === pagePath && session.last_day === today) return; - await bumpExit(sb, project, session.last_day, session.last_page_path, -1); - await bumpExit(sb, project, today, pagePath, 1); + // The row this session is currently counted on. Sessions written before + // the split carry no kind and sit on an `unknown` row. + const previousKind: ExitKind = session.kind ?? "unknown"; + // Already counted on the same page, day and kind — nothing to move. + if ( + session.last_page_path === pagePath && + session.last_day === today && + previousKind === kind + ) { + return; + } + await bumpExit(sb, project, session.last_day, session.last_page_path, -1, previousKind); + await bumpExit(sb, project, today, pagePath, 1, kind); await sb .from("tracker_exit_sessions") .update({ last_page_path: pagePath, last_day: today, + kind, updated_at: new Date().toISOString(), }) .eq("project_id", project) .eq("session_id", sessionId); } else { - await bumpExit(sb, project, today, pagePath, 1); + await bumpExit(sb, project, today, pagePath, 1, kind); await sb .from("tracker_exit_sessions") .insert({ @@ -92,6 +117,7 @@ export async function updateExitRollup( session_id: sessionId, last_page_path: pagePath, last_day: today, + kind, }); } } diff --git a/lib/tracker/humans.ts b/lib/tracker/humans.ts index 20420e4d..0bb58e2e 100644 --- a/lib/tracker/humans.ts +++ b/lib/tracker/humans.ts @@ -28,6 +28,19 @@ export const AI_REFERRALS_DEFINITION = export const ALL_EVENTS_DEFINITION = "Every tracked event, human and bot together. This is the old headline number; bots can dominate it."; +/** + * The side of the line a single hit falls on, as recorded in the `kind` + * column of the bucket-less rollups (tracker_event/device/geo/exit_daily_stats) + * by 20260905190000_tracker_kind_split.sql. Rows written before that + * migration are 'unknown' in the database and are never produced here. + */ +export type TrackerKind = "human" | "bot"; + +/** Same rule as the SQL: bot iff the bucket starts with "bot:". */ +export function kindFromBucket(bucket: string): TrackerKind { + return bucket.startsWith("bot:") ? "bot" : "human"; +} + type Countable = number | string | null | undefined; /** Coerce a PostgREST count (bigint columns arrive as strings). null when absent. */ diff --git a/lib/tracker/panels.ts b/lib/tracker/panels.ts index a25f066f..d33f0a9c 100644 --- a/lib/tracker/panels.ts +++ b/lib/tracker/panels.ts @@ -10,12 +10,14 @@ // client and the route handler's, both RLS-scoped to the signed-in user). // // Each panel picks its RPC from the range: raw tracker_events twins under a -// day, *_daily_stats rollups above it. See lib/tracker/ranges.ts. +// day, *_daily_stats rollups above it. See lib/tracker/ranges.ts. Every RPC +// also takes `p_kind` (null / "human" / "bot"), which is how the page-wide +// Humans / Bots / All toggle reaches each card; see lib/tracker/who.ts. import type { SupabaseClient } from "@supabase/supabase-js"; import { bucketLabel } from "@/lib/tracker/categorize"; import { countryNameFromCode } from "@/lib/tracker/country"; -import { humansFrom } from "@/lib/tracker/humans"; +import { humansFrom, type TrackerKind } from "@/lib/tracker/humans"; import { buildDailyAxis, toSeriesRow } from "@/lib/tracker/series"; import { isRawRange, @@ -115,6 +117,8 @@ export async function fetchPanel( panel: PanelKey, range: TrackerRange, days: number, + /** null = every row; "human" / "bot" = that side only. See lib/tracker/who.ts. */ + kind: TrackerKind | null = null, ): Promise { const raw = isRawRange(range); const minutes = range.minutes ?? 1440; @@ -124,6 +128,7 @@ export async function fetchPanel( if (raw) { const { data } = await sb.rpc("tracker_recent_series", { p_project: projectId, + p_kind: kind, p_minutes: minutes, p_bucket_seconds: range.bucketSeconds ?? 300, }); @@ -138,23 +143,34 @@ export async function fetchPanel( } const { data } = await sb.rpc("tracker_daily_series", { p_project: projectId, + p_kind: kind, days, }); const series = ((data ?? []) as Parameters[0][]).map( toSeriesRow, ); - return { points: buildDailyAxis(series, days), granularity: "day" }; + return { + // The legacy events backfill is for an unfiltered series only: on a + // filtered one a zero day is the honest answer, and the event-table + // legs it would fill from may hold the other side's rows. + points: buildDailyAxis(series, days, new Date(), { + legacyEventsBackfill: kind === null, + }), + granularity: "day", + }; } case "sources": { const { data } = raw ? await sb.rpc("tracker_recent_bucket_totals", { p_project: projectId, + p_kind: kind, p_minutes: minutes, lim: TOP_N, }) : await sb.rpc("tracker_bucket_totals", { p_project: projectId, + p_kind: kind, days, lim: TOP_N, }); @@ -167,9 +183,14 @@ export async function fetchPanel( const { data } = raw ? await sb.rpc("tracker_recent_event_mix", { p_project: projectId, + p_kind: kind, p_minutes: minutes, }) - : await sb.rpc("tracker_event_mix", { p_project: projectId, days }); + : await sb.rpc("tracker_event_mix", { + p_project: projectId, + days, + p_kind: kind, + }); return ((data ?? []) as Array<{ event: string; total: number | string }>) .map((r) => ({ label: eventLabel(r.event), value: Number(r.total) })) .sort((a, b) => b.value - a.value) @@ -180,11 +201,13 @@ export async function fetchPanel( const { data } = raw ? await sb.rpc("tracker_recent_top_pages", { p_project: projectId, + p_kind: kind, p_minutes: minutes, lim: TOP_N, }) : await sb.rpc("tracker_top_pages", { p_project: projectId, + p_kind: kind, days, lim: TOP_N, }); @@ -197,6 +220,7 @@ export async function fetchPanel( // Rollup-only: tracker_exit_sessions keys on a date, not a timestamp. const { data } = await sb.rpc("tracker_top_exit_pages", { p_project: projectId, + p_kind: kind, days: rollupDays(range, days), lim: TOP_N, }); @@ -209,11 +233,13 @@ export async function fetchPanel( const { data } = raw ? await sb.rpc("tracker_recent_top_referrers", { p_project: projectId, + p_kind: kind, p_minutes: minutes, lim: TOP_N, }) : await sb.rpc("tracker_top_referrers", { p_project: projectId, + p_kind: kind, days, lim: TOP_N, }); @@ -226,11 +252,13 @@ export async function fetchPanel( const { data } = raw ? await sb.rpc("tracker_recent_top_actions", { p_project: projectId, + p_kind: kind, p_minutes: minutes, lim: TOP_N, }) : await sb.rpc("tracker_top_actions", { p_project: projectId, + p_kind: kind, days, lim: TOP_N, }); @@ -250,11 +278,13 @@ export async function fetchPanel( const { data } = raw ? await sb.rpc("tracker_recent_top_countries", { p_project: projectId, + p_kind: kind, p_minutes: minutes, lim: TOP_N, }) : await sb.rpc("tracker_top_countries", { p_project: projectId, + p_kind: kind, days, lim: TOP_N, }); @@ -279,11 +309,13 @@ export async function fetchPanel( const { data } = raw ? await sb.rpc("tracker_recent_top_cities", { p_project: projectId, + p_kind: kind, p_minutes: minutes, lim: TOP_N, }) : await sb.rpc("tracker_top_cities", { p_project: projectId, + p_kind: kind, days, lim: TOP_N, }); @@ -315,6 +347,7 @@ export async function fetchPanel( // device_type / browser / os columns. const { data } = await sb.rpc("tracker_device_totals", { p_project: projectId, + p_kind: kind, days: rollupDays(range, days), }); const rows = ( @@ -345,10 +378,11 @@ export async function fetchPanels( projectId: string, panels: PanelKey[], range: TrackerRange, + kind: TrackerKind | null = null, ): Promise> { const days = await resolveDays(sb, projectId, range); const results = await Promise.all( - panels.map((p) => fetchPanel(sb, projectId, p, range, days)), + panels.map((p) => fetchPanel(sb, projectId, p, range, days, kind)), ); return Object.fromEntries(panels.map((p, i) => [p, results[i]])); } diff --git a/lib/tracker/series.ts b/lib/tracker/series.ts index d55c6c9d..5d0504e4 100644 --- a/lib/tracker/series.ts +++ b/lib/tracker/series.ts @@ -65,10 +65,22 @@ export function utcDayAxis(days: number, now = new Date()): string[] { // Map the per-day series onto a zero-filled `days`-wide axis. Rows outside the // window (should be none — the RPC applies the same window) are ignored. +export type BuildDailyAxisOptions = { + /** + * Fill a zero `events` day from pageviews + interactions. Right for an + * unfiltered series (old rollups predate the bucket table); wrong for one + * filtered to humans or bots, where a zero is the honest answer and the + * event-table legs may be the other side's, or pre-split `unknown`, rows. + * Defaults to on so existing callers keep their behaviour. + */ + legacyEventsBackfill?: boolean; +}; + export function buildDailyAxis( series: TrackerSeriesRow[], days: number, now = new Date(), + { legacyEventsBackfill = true }: BuildDailyAxisOptions = {}, ): TrackerDailyPointShape[] { const byDay = new Map(); for (const date of utcDayAxis(days, now)) { @@ -98,8 +110,10 @@ export function buildDailyAxis( // clearly had traffic. Fall back to the event table's own totals. `humans` // is deliberately left alone: those event-table totals are bot-inclusive, // so there is no honest human figure for such a day. - for (const point of byDay.values()) { - if (point.events === 0) point.events = point.pageviews + point.interactions; + if (legacyEventsBackfill) { + for (const point of byDay.values()) { + if (point.events === 0) point.events = point.pageviews + point.interactions; + } } return Array.from(byDay.values()); diff --git a/lib/tracker/who.ts b/lib/tracker/who.ts new file mode 100644 index 00000000..63d44e90 --- /dev/null +++ b/lib/tracker/who.ts @@ -0,0 +1,204 @@ +// The page-wide Humans / Bots / All toggle on /projects/:id/stats. +// +// One value, carried in the URL as ?who=, drives every number on the page: +// the headline tiles, the Traffic pulse series, each breakdown card and the +// live panel. It maps onto the `p_kind` argument every tracker RPC accepts +// (see supabase/migrations/20260905190000_tracker_kind_split.sql) and onto +// the bucket filter the raw live-events route applies. The mapping lives +// here so the server page, the API route and the client hook cannot drift +// on what "humans" means. +// +// Default is humans: the page leads with people (lib/tracker/humans.ts). + +import { + AI_REFERRALS_DEFINITION, + ALL_EVENTS_DEFINITION, + BOTS_DEFINITION, + BOTS_LABEL, + HUMANS_DEFINITION, + HUMANS_LABEL, + type TrackerKind, +} from "@/lib/tracker/humans"; + +export type Who = "humans" | "bots" | "all"; + +export const WHO_VALUES: Who[] = ["humans", "bots", "all"]; + +export const DEFAULT_WHO: Who = "humans"; + +export const WHO_PARAM = "who"; + +/** Strict parse: null for anything that is not exactly one of the three. */ +export function parseWho(value: string | null | undefined): Who | null { + return (WHO_VALUES as string[]).includes(value ?? "") ? (value as Who) : null; +} + +/** Lenient parse for the page: junk falls back to the default. */ +export function whoOrDefault(value: string | null | undefined): Who { + return parseWho(value) ?? DEFAULT_WHO; +} + +/** The RPC argument: null means "no filter", which is what All asks for. */ +export function whoToKind(who: Who): TrackerKind | null { + switch (who) { + case "humans": + return "human"; + case "bots": + return "bot"; + default: + return null; + } +} + +export type WhoOption = { + key: Who; + label: string; + /** Tooltip; the same definition every labelled figure carries. */ + description: string; +}; + +export const WHO_OPTIONS: WhoOption[] = [ + { key: "humans", label: "Humans", description: HUMANS_DEFINITION }, + { key: "bots", label: "Bots", description: BOTS_DEFINITION }, + { key: "all", label: "All", description: ALL_EVENTS_DEFINITION }, +]; + +// The bucket-less rollups (pages, referrers, actions, exit pages, countries, +// cities, devices) only record the human / bot side of a hit from the day +// the kind-split migration was applied. Rows before it are `unknown` and +// answer only under All, so any filtered view has to say where its history +// starts. +export const KIND_SPLIT_SINCE = "2026-09-05"; +export const KIND_SPLIT_SINCE_LABEL = "5 Sep 2026"; +export const KIND_SPLIT_CAPTION = `Split recorded from ${KIND_SPLIT_SINCE_LABEL}; earlier traffic appears under All.`; + +/** The caption under the toggle, or null when nothing is filtered out. */ +export function whoCaption(who: Who): string | null { + return who === "all" ? null : KIND_SPLIT_CAPTION; +} + +export type HeadlineTotals = { + humans: number; + ai: number; + bots: number; +}; + +export type HeadlineTile = { + key: "humans" | "ai" | "bots"; + label: string; + value: number; + tone: "accent" | "pass" | "warn"; + hint: string; +}; + +/** + * Which headline tiles the page shows for a given toggle. Humans and Bots + * show their own side's figures; All keeps the three-tile layout the page + * shipped with. The figures come from the series the page fetched at that + * same `who`, so a filtered view never mixes in the other side's count. + */ +export function headlineTiles(who: Who, t: HeadlineTotals): HeadlineTile[] { + const humans: HeadlineTile = { + key: "humans", + label: HUMANS_LABEL, + value: t.humans, + tone: "accent", + hint: HUMANS_DEFINITION, + }; + const ai: HeadlineTile = { + key: "ai", + label: "AI referrals", + value: t.ai, + tone: "pass", + hint: AI_REFERRALS_DEFINITION, + }; + const bots: HeadlineTile = { + key: "bots", + label: BOTS_LABEL, + value: t.bots, + tone: "warn", + hint: BOTS_DEFINITION, + }; + switch (who) { + case "humans": + return [humans, ai]; + case "bots": + return [bots]; + default: + return [humans, ai, bots]; + } +} + +export type PulseLayer = { + dataKey: "humans" | "bots" | "ai" | "interactions"; + name: string; + /** Stacked layers share a stack id; overlays have none. */ + stackId?: string; + color: string; + fillOpacity: number; + dashed?: boolean; +}; + +const HUMANS_LAYER: PulseLayer = { + dataKey: "humans", + name: "Human visits", + stackId: "1", + color: "var(--color-accent)", + fillOpacity: 0.28, +}; +const BOTS_LAYER: PulseLayer = { + dataKey: "bots", + name: "Bot crawls", + stackId: "1", + color: "var(--color-warn)", + fillOpacity: 0.14, +}; +const AI_LAYER: PulseLayer = { + dataKey: "ai", + name: "AI referrals (within humans)", + color: "var(--color-pass)", + fillOpacity: 0.16, +}; +const INTERACTIONS_LAYER: PulseLayer = { + dataKey: "interactions", + name: "Interactions", + color: "#60a5fa", + fillOpacity: 0, + dashed: true, +}; + +/** + * Which series the Traffic pulse draws. Humans: the human band with AI + * referrals inside it; Bots: the bot band alone; All: both stacked. The + * interactions overlay (clicks and forms) is on every view, filtered to the + * same side by the RPC. + */ +export function pulseLayers(who: Who): PulseLayer[] { + switch (who) { + case "humans": + return [HUMANS_LAYER, AI_LAYER, INTERACTIONS_LAYER]; + case "bots": + return [BOTS_LAYER, INTERACTIONS_LAYER]; + default: + return [HUMANS_LAYER, BOTS_LAYER, AI_LAYER, INTERACTIONS_LAYER]; + } +} + +/** The figure the Traffic pulse frame leads with, and the noun for it. */ +export function pulseHeadline( + who: Who, + totals: { humans: number; bots: number }, +): { total: number; unit: [string, string]; hint: string } { + if (who === "bots") { + return { + total: totals.bots, + unit: ["bot crawl", "bot crawls"], + hint: BOTS_DEFINITION, + }; + } + return { + total: totals.humans, + unit: ["human visit", "human visits"], + hint: HUMANS_DEFINITION, + }; +} diff --git a/supabase/migrations/20260905190000_tracker_kind_split.sql b/supabase/migrations/20260905190000_tracker_kind_split.sql new file mode 100644 index 00000000..30a64094 --- /dev/null +++ b/supabase/migrations/20260905190000_tracker_kind_split.sql @@ -0,0 +1,1026 @@ +-- Human / bot split for every tracker breakdown, not just the headline. +-- +-- WHY: 20260905120000_tracker_human_split gave the headline tiles and the +-- series chart a human figure, but it could only do that from +-- tracker_daily_stats, the one rollup that records a bucket. Top pages, +-- referrers, actions, exit pages, countries, cities and devices all read +-- rollups that have no idea whether the hit was a person or a crawler, so a +-- reader who narrows the page to Humans still saw a Top pages list that was +-- 99% GPTBot. The stats page now carries a page-wide Humans / Bots / All +-- toggle, and for it to mean the same thing on every card each rollup has +-- to record which side of the line it counted. +-- +-- WHAT: a `kind` column on the four bucket-less rollups -- +-- tracker_event_daily_stats, tracker_device_daily_stats, +-- tracker_geo_daily_stats, tracker_exit_daily_stats -- plus +-- tracker_exit_sessions, so the exit marker moves within one kind. `kind` is +-- 'human' or 'bot' by the same definition as everywhere else (see +-- lib/tracker/humans.ts: bot = bucket starts with 'bot:', human = anything +-- else, AI referrals included), and joins each table's primary key so a +-- human and a bot hit on the same path in the same day are two rows. +-- +-- ROWS BEFORE THIS MIGRATION ARE `unknown`. Nothing in the old rollups can +-- say what they were, so they are not guessed at: they carry kind = 'unknown' +-- and appear only under All (p_kind null). Under Humans or Bots the panels +-- read from these four tables begin on the day this is applied; the UI says +-- so under the toggle. The bucket-based figures (headline tiles, the series +-- chart, Top sources) and the raw tracker_events table have always recorded +-- the bucket, so those are split across all history. +-- +-- RPCs: every panel function -- singles, _multi and the tracker_recent_* raw +-- twins -- gains a trailing `p_kind text default null`. null = all rows; +-- 'human' / 'bot' filter on `kind` for the rollups above and on the bucket +-- prefix for tracker_daily_stats and tracker_events. Adding a defaulted +-- parameter changes the signature, so each is dropped by its exact old +-- signature and re-created with the old arguments first; callers that never +-- pass p_kind keep working unchanged. Dropping a function drops its grants, +-- hence the re-grants. +-- +-- work_mem: tracker_top_pages_multi, tracker_top_actions_multi and +-- tracker_top_exit_pages_multi carry `set work_mem = '16MB'` from the applied +-- tracker_reporting_indexes migration (their HashAggregate spilled at the +-- 3.5MB instance default). Re-created here WITH that setting; it is +-- per-function on purpose, because /dashboard/analytics fires eleven of these +-- concurrently. +-- +-- Indexes: the two covering indexes from 20260902120000 gain `kind` in their +-- INCLUDE list so the filtered aggregates stay index-only. `page_path` stays +-- in the include -- without it tracker_top_pages_multi heap-fetches every +-- matching row (measured at 111k buffers). New names, then drop the old, so +-- coverage never lapses. +-- +-- LOCKING: swapping a primary key rebuilds its index under an ACCESS +-- EXCLUSIVE lock. tracker_event_daily_stats is ~1.2M rows / 326MB, so expect +-- ingest writes to queue for the seconds the build takes. The check +-- constraints are added NOT VALID and validated separately so they never +-- take more than SHARE UPDATE EXCLUSIVE. +-- +-- security invoker throughout, matching every other tracker_* RPC: these take +-- project ids straight from the caller and rely on RLS. Never definer. +-- Idempotent. Apply one file at a time via the Supabase MCP, not `db push`. + +-- --------------------------------------------------------------------------- +-- 1. `kind` column + check on each bucket-less rollup and the exit sessions. +-- --------------------------------------------------------------------------- +alter table public.tracker_event_daily_stats + add column if not exists kind text not null default 'unknown'; +alter table public.tracker_device_daily_stats + add column if not exists kind text not null default 'unknown'; +alter table public.tracker_geo_daily_stats + add column if not exists kind text not null default 'unknown'; +alter table public.tracker_exit_daily_stats + add column if not exists kind text not null default 'unknown'; +alter table public.tracker_exit_sessions + add column if not exists kind text not null default 'unknown'; + +do $$ +declare + t text; +begin + foreach t in array array[ + 'tracker_event_daily_stats', + 'tracker_device_daily_stats', + 'tracker_geo_daily_stats', + 'tracker_exit_daily_stats', + 'tracker_exit_sessions' + ] loop + if not exists ( + select 1 from pg_constraint + where conrelid = ('public.' || t)::regclass + and conname = t || '_kind_check' + ) then + execute format( + 'alter table public.%I add constraint %I check (kind in (''human'', ''bot'', ''unknown'')) not valid', + t, t || '_kind_check' + ); + execute format('alter table public.%I validate constraint %I', t, t || '_kind_check'); + end if; + end loop; +end $$; + +-- --------------------------------------------------------------------------- +-- 2. `kind` joins each primary key. The ingest route reads and writes these +-- rows by the full key (select ... eq(each column) then insert/update), so +-- the key is the conflict target. Guarded on whether the current PK already +-- names `kind`, so a replay is a no-op rather than a second rebuild. +-- --------------------------------------------------------------------------- +do $$ +declare + spec record; +begin + for spec in + select * from (values + ('tracker_event_daily_stats', + 'project_id, day, event, page_path, referrer_host, event_target, kind'), + ('tracker_device_daily_stats', + 'project_id, day, device_type, browser, os, kind'), + ('tracker_geo_daily_stats', + 'project_id, day, country_code, region_code, city, timezone, kind'), + ('tracker_exit_daily_stats', + 'project_id, day, page_path, kind') + ) as v(tbl, cols) + loop + if not exists ( + select 1 + from pg_constraint c + join pg_attribute a + on a.attrelid = c.conrelid and a.attnum = any(c.conkey) + where c.conrelid = ('public.' || spec.tbl)::regclass + and c.contype = 'p' + and a.attname = 'kind' + ) then + execute format('alter table public.%I drop constraint if exists %I', spec.tbl, spec.tbl || '_pkey'); + execute format('alter table public.%I add constraint %I primary key (%s)', spec.tbl, spec.tbl || '_pkey', spec.cols); + end if; + end loop; +end $$; + +-- --------------------------------------------------------------------------- +-- 3. Covering indexes: same keys, `kind` added to INCLUDE, page_path kept. +-- --------------------------------------------------------------------------- +create index if not exists tracker_event_daily_stats_project_event_day_kind_idx + on public.tracker_event_daily_stats (project_id, event, day desc) + include (page_path, count, kind); + +create index if not exists tracker_event_daily_stats_project_day_kind_cover_idx + on public.tracker_event_daily_stats (project_id, day desc) + include (event, page_path, referrer_host, event_target, count, kind); + +drop index if exists public.tracker_event_daily_stats_project_event_day_idx; +drop index if exists public.tracker_event_daily_stats_project_day_cover_idx; + +-- --------------------------------------------------------------------------- +-- 4. Single-project RPCs (+ p_kind). +-- --------------------------------------------------------------------------- +drop function if exists public.tracker_daily_series(uuid, integer); + +create function public.tracker_daily_series( + p_project uuid, + days integer default 30, + p_kind text default null +) +returns table ( + day date, + pageviews bigint, + interactions bigint, + ai bigint, + bots bigint, + events bigint, + humans bigint +) +language sql +stable +security invoker +set search_path = public +as $$ + with since as ( + select ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) as d + ), + ev as ( + select day, + sum(count) filter (where event = 'pageview')::bigint as pageviews, + sum(count) filter (where event <> 'pageview')::bigint as interactions + from public.tracker_event_daily_stats + where project_id = p_project + and day >= (select d from since) + and (p_kind is null or kind = p_kind) + group by day + ), + bk as ( + select day, + sum(count) filter (where bucket like 'ai_referral:%')::bigint as ai, + sum(count) filter (where bucket like 'bot:%')::bigint as bots, + sum(count)::bigint as events, + sum(count) filter (where bucket not like 'bot:%')::bigint as humans + from public.tracker_daily_stats + where project_id = p_project + and day >= (select d from since) + and (p_kind is null + or (p_kind = 'bot' and bucket like 'bot:%') + or (p_kind = 'human' and bucket not like 'bot:%')) + group by day + ) + select coalesce(ev.day, bk.day) as day, + coalesce(ev.pageviews, 0) as pageviews, + coalesce(ev.interactions, 0) as interactions, + coalesce(bk.ai, 0) as ai, + coalesce(bk.bots, 0) as bots, + coalesce(bk.events, 0) as events, + coalesce(bk.humans, 0) as humans + from ev + full outer join bk on ev.day = bk.day; +$$; + +drop function if exists public.tracker_bucket_totals(uuid, integer, integer); + +create function public.tracker_bucket_totals( + p_project uuid, + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table (bucket text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select bucket, sum(count)::bigint as total + from public.tracker_daily_stats + where project_id = p_project + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null + or (p_kind = 'bot' and bucket like 'bot:%') + or (p_kind = 'human' and bucket not like 'bot:%')) + group by bucket + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_event_mix(uuid, integer); + +create function public.tracker_event_mix( + p_project uuid, + days integer default 30, + p_kind text default null +) +returns table (event text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select event, sum(count)::bigint as total + from public.tracker_event_daily_stats + where project_id = p_project + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by event + order by total desc; +$$; + +drop function if exists public.tracker_top_pages(uuid, integer, integer); + +create function public.tracker_top_pages( + p_project uuid, + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table (page_path text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select coalesce(nullif(page_path, ''), '/') as page_path, sum(count)::bigint as total + from public.tracker_event_daily_stats + where project_id = p_project + and event = 'pageview' + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by 1 + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_top_referrers(uuid, integer, integer); + +create function public.tracker_top_referrers( + p_project uuid, + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table (referrer_host text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select referrer_host, sum(count)::bigint as total + from public.tracker_event_daily_stats + where project_id = p_project + and coalesce(referrer_host, '') <> '' + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by referrer_host + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_top_actions(uuid, integer, integer); + +create function public.tracker_top_actions( + p_project uuid, + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table (event text, event_target text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select event, event_target, sum(count)::bigint as total + from public.tracker_event_daily_stats + where project_id = p_project + and event <> 'pageview' + and coalesce(event_target, '') <> '' + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by event, event_target + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_top_exit_pages(uuid, integer, integer); + +create function public.tracker_top_exit_pages( + p_project uuid, + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table (page_path text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select coalesce(nullif(page_path, ''), '/') as page_path, sum(count)::bigint as total + from public.tracker_exit_daily_stats + where project_id = p_project + and count > 0 + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by 1 + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_top_countries(uuid, integer, integer); + +create function public.tracker_top_countries( + p_project uuid, + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table (country_code text, country_name text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select country_code, max(country_name) as country_name, sum(count)::bigint as total + from public.tracker_geo_daily_stats + where project_id = p_project + and (coalesce(country_code, '') <> '' or coalesce(country_name, '') <> '') + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by country_code + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_top_cities(uuid, integer, integer); + +create function public.tracker_top_cities( + p_project uuid, + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table ( + city text, + region_code text, + region_name text, + country_code text, + country_name text, + total bigint +) +language sql +stable +security invoker +set search_path = public +as $$ + select city, region_code, region_name, country_code, country_name, + sum(count)::bigint as total + from public.tracker_geo_daily_stats + where project_id = p_project + and coalesce(city, '') <> '' + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by city, region_code, region_name, country_code, country_name + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_device_totals(uuid, integer); + +create function public.tracker_device_totals( + p_project uuid, + days integer default 30, + p_kind text default null +) +returns table (device_type text, browser text, os text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select device_type, browser, os, sum(count)::bigint as total + from public.tracker_device_daily_stats + where project_id = p_project + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by device_type, browser, os; +$$; + +grant execute on function public.tracker_daily_series(uuid, integer, text) to authenticated, service_role; +grant execute on function public.tracker_bucket_totals(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_event_mix(uuid, integer, text) to authenticated, service_role; +grant execute on function public.tracker_top_pages(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_top_referrers(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_top_actions(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_top_exit_pages(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_top_countries(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_top_cities(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_device_totals(uuid, integer, text) to authenticated, service_role; + +-- --------------------------------------------------------------------------- +-- 5. Portfolio (_multi) RPCs (+ p_kind). work_mem kept on the three that +-- spill. +-- --------------------------------------------------------------------------- +drop function if exists public.tracker_daily_series_multi(uuid[], integer); + +create function public.tracker_daily_series_multi( + p_projects uuid[], + days integer default 30, + p_kind text default null +) +returns table ( + day date, + pageviews bigint, + interactions bigint, + ai bigint, + bots bigint, + events bigint, + humans bigint +) +language sql +stable +security invoker +set search_path = public +as $$ + with since as ( + select ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) as d + ), + ev as ( + select day, + sum(count) filter (where event = 'pageview')::bigint as pageviews, + sum(count) filter (where event <> 'pageview')::bigint as interactions + from public.tracker_event_daily_stats + where project_id = any(p_projects) + and day >= (select d from since) + and (p_kind is null or kind = p_kind) + group by day + ), + bk as ( + select day, + sum(count) filter (where bucket like 'ai_referral:%')::bigint as ai, + sum(count) filter (where bucket like 'bot:%')::bigint as bots, + sum(count)::bigint as events, + sum(count) filter (where bucket not like 'bot:%')::bigint as humans + from public.tracker_daily_stats + where project_id = any(p_projects) + and day >= (select d from since) + and (p_kind is null + or (p_kind = 'bot' and bucket like 'bot:%') + or (p_kind = 'human' and bucket not like 'bot:%')) + group by day + ) + select coalesce(ev.day, bk.day) as day, + coalesce(ev.pageviews, 0) as pageviews, + coalesce(ev.interactions, 0) as interactions, + coalesce(bk.ai, 0) as ai, + coalesce(bk.bots, 0) as bots, + coalesce(bk.events, 0) as events, + coalesce(bk.humans, 0) as humans + from ev + full outer join bk on ev.day = bk.day; +$$; + +drop function if exists public.tracker_bucket_totals_multi(uuid[], integer, integer); + +create function public.tracker_bucket_totals_multi( + p_projects uuid[], + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table (bucket text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select bucket, sum(count)::bigint as total + from public.tracker_daily_stats + where project_id = any(p_projects) + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null + or (p_kind = 'bot' and bucket like 'bot:%') + or (p_kind = 'human' and bucket not like 'bot:%')) + group by bucket + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_event_mix_multi(uuid[], integer); + +create function public.tracker_event_mix_multi( + p_projects uuid[], + days integer default 30, + p_kind text default null +) +returns table (event text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select event, sum(count)::bigint as total + from public.tracker_event_daily_stats + where project_id = any(p_projects) + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by event + order by total desc; +$$; + +drop function if exists public.tracker_top_pages_multi(uuid[], integer, integer); + +create function public.tracker_top_pages_multi( + p_projects uuid[], + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table (project_id uuid, page_path text, total bigint) +language sql +stable +security invoker +set search_path = public +set work_mem = '16MB' +as $$ + select project_id, + coalesce(nullif(page_path, ''), '/') as page_path, + sum(count)::bigint as total + from public.tracker_event_daily_stats + where project_id = any(p_projects) + and event = 'pageview' + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by project_id, 2 + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_top_exit_pages_multi(uuid[], integer, integer); + +create function public.tracker_top_exit_pages_multi( + p_projects uuid[], + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table (project_id uuid, page_path text, total bigint) +language sql +stable +security invoker +set search_path = public +set work_mem = '16MB' +as $$ + select project_id, + coalesce(nullif(page_path, ''), '/') as page_path, + sum(count)::bigint as total + from public.tracker_exit_daily_stats + where project_id = any(p_projects) + and count > 0 + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by project_id, 2 + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_top_referrers_multi(uuid[], integer, integer); + +create function public.tracker_top_referrers_multi( + p_projects uuid[], + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table (referrer_host text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select referrer_host, sum(count)::bigint as total + from public.tracker_event_daily_stats + where project_id = any(p_projects) + and coalesce(referrer_host, '') <> '' + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by referrer_host + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_top_actions_multi(uuid[], integer, integer); + +create function public.tracker_top_actions_multi( + p_projects uuid[], + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table (project_id uuid, event text, event_target text, total bigint) +language sql +stable +security invoker +set search_path = public +set work_mem = '16MB' +as $$ + select project_id, event, event_target, sum(count)::bigint as total + from public.tracker_event_daily_stats + where project_id = any(p_projects) + and event <> 'pageview' + and coalesce(event_target, '') <> '' + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by project_id, event, event_target + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_top_countries_multi(uuid[], integer, integer); + +create function public.tracker_top_countries_multi( + p_projects uuid[], + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table (country_code text, country_name text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select country_code, max(country_name) as country_name, sum(count)::bigint as total + from public.tracker_geo_daily_stats + where project_id = any(p_projects) + and (coalesce(country_code, '') <> '' or coalesce(country_name, '') <> '') + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by country_code + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_top_cities_multi(uuid[], integer, integer); + +create function public.tracker_top_cities_multi( + p_projects uuid[], + days integer default 30, + lim integer default 10, + p_kind text default null +) +returns table ( + city text, + region_code text, + region_name text, + country_code text, + country_name text, + total bigint +) +language sql +stable +security invoker +set search_path = public +as $$ + select city, region_code, region_name, country_code, country_name, + sum(count)::bigint as total + from public.tracker_geo_daily_stats + where project_id = any(p_projects) + and coalesce(city, '') <> '' + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by city, region_code, region_name, country_code, country_name + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_device_totals_multi(uuid[], integer); + +create function public.tracker_device_totals_multi( + p_projects uuid[], + days integer default 30, + p_kind text default null +) +returns table (device_type text, browser text, os text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select device_type, browser, os, sum(count)::bigint as total + from public.tracker_device_daily_stats + where project_id = any(p_projects) + and day >= ((now() at time zone 'UTC')::date - (greatest(coalesce(days, 30), 1) - 1)) + and (p_kind is null or kind = p_kind) + group by device_type, browser, os; +$$; + +grant execute on function public.tracker_daily_series_multi(uuid[], integer, text) to authenticated, service_role; +grant execute on function public.tracker_bucket_totals_multi(uuid[], integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_event_mix_multi(uuid[], integer, text) to authenticated, service_role; +grant execute on function public.tracker_top_pages_multi(uuid[], integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_top_exit_pages_multi(uuid[], integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_top_referrers_multi(uuid[], integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_top_actions_multi(uuid[], integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_top_countries_multi(uuid[], integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_top_cities_multi(uuid[], integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_device_totals_multi(uuid[], integer, text) to authenticated, service_role; + +-- --------------------------------------------------------------------------- +-- 6. Raw tracker_events twins (+ p_kind on the bucket prefix). The raw row +-- has always carried its bucket, so these are split across the whole 24h +-- window from the moment this is applied. +-- --------------------------------------------------------------------------- +drop function if exists public.tracker_recent_series(uuid, integer, integer); + +create function public.tracker_recent_series( + p_project uuid, + p_minutes integer default 60, + p_bucket_seconds integer default 300, + p_kind text default null +) +returns table ( + ts timestamptz, + pageviews bigint, + interactions bigint, + ai bigint, + bots bigint, + events bigint, + humans bigint +) +language sql +stable +security invoker +set search_path = public +as $$ + with args as ( + select least(greatest(coalesce(p_minutes, 60), 1), 1440) as mins, + least(greatest(coalesce(p_bucket_seconds, 300), 60), 86400) as secs + ) + select to_timestamp( + floor(extract(epoch from e.occurred_at) / (select secs from args)) + * (select secs from args) + ) as ts, + count(*) filter (where e.event = 'pageview')::bigint as pageviews, + count(*) filter (where e.event <> 'pageview')::bigint as interactions, + count(*) filter (where e.bucket like 'ai_referral:%')::bigint as ai, + count(*) filter (where e.bucket like 'bot:%')::bigint as bots, + count(*)::bigint as events, + count(*) filter (where coalesce(e.bucket, '') not like 'bot:%')::bigint as humans + from public.tracker_events e + where e.project_id = p_project + and e.occurred_at >= now() - ((select mins from args) || ' minutes')::interval + and (p_kind is null + or (p_kind = 'bot' and e.bucket like 'bot:%') + or (p_kind = 'human' and coalesce(e.bucket, '') not like 'bot:%')) + group by 1 + order by 1; +$$; + +drop function if exists public.tracker_recent_bucket_totals(uuid, integer, integer); + +create function public.tracker_recent_bucket_totals( + p_project uuid, + p_minutes integer default 60, + lim integer default 10, + p_kind text default null +) +returns table (bucket text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select bucket, count(*)::bigint as total + from public.tracker_events + where project_id = p_project + and coalesce(bucket, '') <> '' + and occurred_at >= now() + - (least(greatest(coalesce(p_minutes, 60), 1), 1440) || ' minutes')::interval + and (p_kind is null + or (p_kind = 'bot' and bucket like 'bot:%') + or (p_kind = 'human' and bucket not like 'bot:%')) + group by bucket + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_recent_event_mix(uuid, integer); + +create function public.tracker_recent_event_mix( + p_project uuid, + p_minutes integer default 60, + p_kind text default null +) +returns table (event text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select event, count(*)::bigint as total + from public.tracker_events + where project_id = p_project + and occurred_at >= now() + - (least(greatest(coalesce(p_minutes, 60), 1), 1440) || ' minutes')::interval + and (p_kind is null + or (p_kind = 'bot' and bucket like 'bot:%') + or (p_kind = 'human' and coalesce(bucket, '') not like 'bot:%')) + group by event + order by total desc; +$$; + +drop function if exists public.tracker_recent_top_pages(uuid, integer, integer); + +create function public.tracker_recent_top_pages( + p_project uuid, + p_minutes integer default 60, + lim integer default 10, + p_kind text default null +) +returns table (page_path text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select coalesce(nullif(page_path, ''), '/') as page_path, count(*)::bigint as total + from public.tracker_events + where project_id = p_project + and event = 'pageview' + and occurred_at >= now() + - (least(greatest(coalesce(p_minutes, 60), 1), 1440) || ' minutes')::interval + and (p_kind is null + or (p_kind = 'bot' and bucket like 'bot:%') + or (p_kind = 'human' and coalesce(bucket, '') not like 'bot:%')) + group by 1 + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_recent_top_referrers(uuid, integer, integer); + +create function public.tracker_recent_top_referrers( + p_project uuid, + p_minutes integer default 60, + lim integer default 10, + p_kind text default null +) +returns table (referrer_host text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select referrer_host, count(*)::bigint as total + from public.tracker_events + where project_id = p_project + and coalesce(referrer_host, '') <> '' + and occurred_at >= now() + - (least(greatest(coalesce(p_minutes, 60), 1), 1440) || ' minutes')::interval + and (p_kind is null + or (p_kind = 'bot' and bucket like 'bot:%') + or (p_kind = 'human' and coalesce(bucket, '') not like 'bot:%')) + group by referrer_host + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_recent_top_actions(uuid, integer, integer); + +create function public.tracker_recent_top_actions( + p_project uuid, + p_minutes integer default 60, + lim integer default 10, + p_kind text default null +) +returns table (event text, event_target text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select event, event_target, count(*)::bigint as total + from public.tracker_events + where project_id = p_project + and event <> 'pageview' + and coalesce(event_target, '') <> '' + and occurred_at >= now() + - (least(greatest(coalesce(p_minutes, 60), 1), 1440) || ' minutes')::interval + and (p_kind is null + or (p_kind = 'bot' and bucket like 'bot:%') + or (p_kind = 'human' and coalesce(bucket, '') not like 'bot:%')) + group by event, event_target + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_recent_top_countries(uuid, integer, integer); + +create function public.tracker_recent_top_countries( + p_project uuid, + p_minutes integer default 60, + lim integer default 10, + p_kind text default null +) +returns table (country_code text, country_name text, total bigint) +language sql +stable +security invoker +set search_path = public +as $$ + select country_code, max(country_name) as country_name, count(*)::bigint as total + from public.tracker_events + where project_id = p_project + and coalesce(country_code, '') <> '' + and occurred_at >= now() + - (least(greatest(coalesce(p_minutes, 60), 1), 1440) || ' minutes')::interval + and (p_kind is null + or (p_kind = 'bot' and bucket like 'bot:%') + or (p_kind = 'human' and coalesce(bucket, '') not like 'bot:%')) + group by country_code + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +drop function if exists public.tracker_recent_top_cities(uuid, integer, integer); + +create function public.tracker_recent_top_cities( + p_project uuid, + p_minutes integer default 60, + lim integer default 10, + p_kind text default null +) +returns table ( + city text, + region_code text, + region_name text, + country_code text, + country_name text, + total bigint +) +language sql +stable +security invoker +set search_path = public +as $$ + select city, + ''::text as region_code, + ''::text as region_name, + max(country_code) as country_code, + max(country_name) as country_name, + count(*)::bigint as total + from public.tracker_events + where project_id = p_project + and coalesce(city, '') <> '' + and occurred_at >= now() + - (least(greatest(coalesce(p_minutes, 60), 1), 1440) || ' minutes')::interval + and (p_kind is null + or (p_kind = 'bot' and bucket like 'bot:%') + or (p_kind = 'human' and coalesce(bucket, '') not like 'bot:%')) + group by city + order by total desc + limit greatest(coalesce(lim, 10), 1); +$$; + +grant execute on function public.tracker_recent_series(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_recent_bucket_totals(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_recent_event_mix(uuid, integer, text) to authenticated, service_role; +grant execute on function public.tracker_recent_top_pages(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_recent_top_referrers(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_recent_top_actions(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_recent_top_countries(uuid, integer, integer, text) to authenticated, service_role; +grant execute on function public.tracker_recent_top_cities(uuid, integer, integer, text) to authenticated, service_role; diff --git a/tests/contract/tracker-exit.test.ts b/tests/contract/tracker-exit.test.ts index 5e64300a..c36c8644 100644 --- a/tests/contract/tracker-exit.test.ts +++ b/tests/contract/tracker-exit.test.ts @@ -4,10 +4,12 @@ import { updateExitRollup } from "@/lib/tracker/exit"; // Minimal in-memory fake of the two tables the exit rollup touches, exposing // just the subset of the Supabase query builder the module uses. It records // exit counts and per-session last-page state so we can assert the "exit -// marker moves as the session advances" behavior end to end. +// marker moves as the session advances" behavior end to end. Exit rows are +// keyed by kind as well, as the real primary key is since the kind-split +// migration; a session row may lack `kind` (written before the split). function makeFakeSb() { - const exits = new Map(); - const sessions = new Map(); + const exits = new Map(); + const sessions = new Map(); const key = (...parts: string[]) => parts.join("|"); function table(name: string) { @@ -29,11 +31,14 @@ function makeFakeSb() { lt() { return chain; }, async maybeSingle() { if (name === "tracker_exit_daily_stats") { - const row = exits.get(key(filters.project_id, filters.day, filters.page_path)); + const row = exits.get(key(filters.project_id, filters.day, filters.page_path, filters.kind)); return { data: row ? { count: row.count } : null, error: null }; } const s = sessions.get(key(filters.project_id, filters.session_id)); - return { data: s ? { last_page_path: s.last_page_path, last_day: s.last_day } : null, error: null }; + return { + data: s ? { last_page_path: s.last_page_path, last_day: s.last_day, kind: s.kind ?? null } : null, + error: null, + }; }, then(resolve: (v: unknown) => unknown) { return Promise.resolve(exec()).then(resolve); @@ -43,24 +48,31 @@ function makeFakeSb() { function exec() { if (op === "insert" && pending) { if (name === "tracker_exit_daily_stats") { - exits.set(key(pending.project_id as string, pending.day as string, pending.page_path as string), { - project: pending.project_id as string, day: pending.day as string, page: pending.page_path as string, count: pending.count as number, + // Insert without a kind (the pre-split shape) lands on `unknown`. + const kind = (pending.kind as string | undefined) ?? "unknown"; + exits.set(key(pending.project_id as string, pending.day as string, pending.page_path as string, kind), { + project: pending.project_id as string, day: pending.day as string, page: pending.page_path as string, kind, count: pending.count as number, }); } else { sessions.set(key(pending.project_id as string, pending.session_id as string), { project: pending.project_id as string, session: pending.session_id as string, last_page_path: pending.last_page_path as string, last_day: pending.last_day as string, + kind: pending.kind as string | undefined, }); } } else if (op === "update" && pending) { if (name === "tracker_exit_daily_stats") { - const k = key(filters.project_id, filters.day, filters.page_path); + const k = key(filters.project_id, filters.day, filters.page_path, filters.kind); const row = exits.get(k); if (row) row.count = pending.count as number; } else { const k = key(filters.project_id, filters.session_id); const s = sessions.get(k); - if (s) { s.last_page_path = pending.last_page_path as string; s.last_day = pending.last_day as string; } + if (s) { + s.last_page_path = pending.last_page_path as string; + s.last_day = pending.last_day as string; + if (pending.kind !== undefined) s.kind = pending.kind as string; + } } } return { data: null, error: null }; @@ -72,8 +84,13 @@ function makeFakeSb() { return { // eslint-disable-next-line @typescript-eslint/no-explicit-any sb: { from: (name: string) => table(name) } as any, - exitCount: (project: string, day: string, page: string) => - exits.get(key(project, day, page))?.count ?? 0, + exitCount: (project: string, day: string, page: string, kind = "human") => + exits.get(key(project, day, page, kind))?.count ?? 0, + // Seed the pre-split state: an exit row and a session with no kind. + seedUnknown: (project: string, session: string, day: string, page: string) => { + exits.set(key(project, day, page, "unknown"), { project, day, page, kind: "unknown", count: 1 }); + sessions.set(key(project, session), { project, session, last_page_path: page, last_day: day }); + }, }; } @@ -83,30 +100,30 @@ const DAY = "2026-07-04"; describe("updateExitRollup", () => { it("counts the first pageview of a session as its exit page", async () => { const { sb, exitCount } = makeFakeSb(); - await updateExitRollup(sb, P, "s1", "/pricing", DAY); + await updateExitRollup(sb, P, "s1", "/pricing", DAY, "human"); expect(exitCount(P, DAY, "/pricing")).toBe(1); }); it("moves the exit marker to the session's latest page", async () => { const { sb, exitCount } = makeFakeSb(); - await updateExitRollup(sb, P, "s1", "/pricing", DAY); - await updateExitRollup(sb, P, "s1", "/checkout", DAY); + await updateExitRollup(sb, P, "s1", "/pricing", DAY, "human"); + await updateExitRollup(sb, P, "s1", "/checkout", DAY, "human"); expect(exitCount(P, DAY, "/pricing")).toBe(0); // no longer the exit expect(exitCount(P, DAY, "/checkout")).toBe(1); }); it("does not change counts when the same page repeats within a session", async () => { const { sb, exitCount } = makeFakeSb(); - await updateExitRollup(sb, P, "s1", "/pricing", DAY); - await updateExitRollup(sb, P, "s1", "/pricing", DAY); // reload + await updateExitRollup(sb, P, "s1", "/pricing", DAY, "human"); + await updateExitRollup(sb, P, "s1", "/pricing", DAY, "human"); // reload expect(exitCount(P, DAY, "/pricing")).toBe(1); }); it("counts distinct sessions independently", async () => { const { sb, exitCount } = makeFakeSb(); - await updateExitRollup(sb, P, "s1", "/pricing", DAY); - await updateExitRollup(sb, P, "s2", "/pricing", DAY); - await updateExitRollup(sb, P, "s2", "/checkout", DAY); + await updateExitRollup(sb, P, "s1", "/pricing", DAY, "human"); + await updateExitRollup(sb, P, "s2", "/pricing", DAY, "human"); + await updateExitRollup(sb, P, "s2", "/checkout", DAY, "human"); expect(exitCount(P, DAY, "/pricing")).toBe(1); // s1 still exits here expect(exitCount(P, DAY, "/checkout")).toBe(1); // s2 moved here }); @@ -114,11 +131,46 @@ describe("updateExitRollup", () => { it("never lets a decrement drive a count below zero", async () => { const { sb, exitCount } = makeFakeSb(); // Two sessions land on /a, then both move to /b. - await updateExitRollup(sb, P, "s1", "/a", DAY); - await updateExitRollup(sb, P, "s2", "/a", DAY); - await updateExitRollup(sb, P, "s1", "/b", DAY); - await updateExitRollup(sb, P, "s2", "/b", DAY); + await updateExitRollup(sb, P, "s1", "/a", DAY, "human"); + await updateExitRollup(sb, P, "s2", "/a", DAY, "human"); + await updateExitRollup(sb, P, "s1", "/b", DAY, "human"); + await updateExitRollup(sb, P, "s2", "/b", DAY, "human"); expect(exitCount(P, DAY, "/a")).toBe(0); expect(exitCount(P, DAY, "/b")).toBe(2); }); + + it("keeps human and bot exits on the same page as separate rows", async () => { + const { sb, exitCount } = makeFakeSb(); + await updateExitRollup(sb, P, "person", "/pricing", DAY, "human"); + await updateExitRollup(sb, P, "crawler", "/pricing", DAY, "bot"); + expect(exitCount(P, DAY, "/pricing", "human")).toBe(1); + expect(exitCount(P, DAY, "/pricing", "bot")).toBe(1); + }); + + it("takes a pre-split session off its unknown row, not off a human row", async () => { + const { sb, exitCount, seedUnknown } = makeFakeSb(); + seedUnknown(P, "s-old", DAY, "/pricing"); + expect(exitCount(P, DAY, "/pricing", "unknown")).toBe(1); + + await updateExitRollup(sb, P, "s-old", "/checkout", DAY, "human"); + + expect(exitCount(P, DAY, "/pricing", "unknown")).toBe(0); + expect(exitCount(P, DAY, "/pricing", "human")).toBe(0); + expect(exitCount(P, DAY, "/checkout", "human")).toBe(1); + }); + + it("moves a pre-split session that reloads the same page onto its kind row", async () => { + const { sb, exitCount, seedUnknown } = makeFakeSb(); + seedUnknown(P, "s-old", DAY, "/pricing"); + + await updateExitRollup(sb, P, "s-old", "/pricing", DAY, "human"); + + // Same page and day, but a different kind than the row it sat on: the + // marker moves so Humans can see this exit from now on. + expect(exitCount(P, DAY, "/pricing", "unknown")).toBe(0); + expect(exitCount(P, DAY, "/pricing", "human")).toBe(1); + // And a further reload is a no-op. + await updateExitRollup(sb, P, "s-old", "/pricing", DAY, "human"); + expect(exitCount(P, DAY, "/pricing", "human")).toBe(1); + }); }); diff --git a/tests/contract/tracker-kind-split.test.ts b/tests/contract/tracker-kind-split.test.ts new file mode 100644 index 00000000..a1087b8e --- /dev/null +++ b/tests/contract/tracker-kind-split.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +// The kind-split migration is applied by hand, so this is the only place its +// shape is checked before it reaches prod. It pins the things that were easy +// to get wrong: every panel RPC has to accept p_kind or the page 404s on the +// missing signature; the three portfolio functions that spill their +// HashAggregate must keep their work_mem or /dashboard/analytics times out +// again; and no tracker RPC may ever become security definer, because they +// take project ids from the caller and lean on RLS. + +const migration = readFileSync( + join( + process.cwd(), + "supabase/migrations/20260905190000_tracker_kind_split.sql", + ), + "utf8", +); + +const KIND_TABLES = [ + "tracker_event_daily_stats", + "tracker_device_daily_stats", + "tracker_geo_daily_stats", + "tracker_exit_daily_stats", +]; + +const SINGLE_RPCS: Array<[string, string]> = [ + ["tracker_daily_series", "uuid, integer"], + ["tracker_bucket_totals", "uuid, integer, integer"], + ["tracker_event_mix", "uuid, integer"], + ["tracker_top_pages", "uuid, integer, integer"], + ["tracker_top_referrers", "uuid, integer, integer"], + ["tracker_top_actions", "uuid, integer, integer"], + ["tracker_top_exit_pages", "uuid, integer, integer"], + ["tracker_top_countries", "uuid, integer, integer"], + ["tracker_top_cities", "uuid, integer, integer"], + ["tracker_device_totals", "uuid, integer"], +]; + +const MULTI_RPCS: Array<[string, string]> = [ + ["tracker_daily_series_multi", "uuid[], integer"], + ["tracker_bucket_totals_multi", "uuid[], integer, integer"], + ["tracker_event_mix_multi", "uuid[], integer"], + ["tracker_top_pages_multi", "uuid[], integer, integer"], + ["tracker_top_exit_pages_multi", "uuid[], integer, integer"], + ["tracker_top_referrers_multi", "uuid[], integer, integer"], + ["tracker_top_actions_multi", "uuid[], integer, integer"], + ["tracker_top_countries_multi", "uuid[], integer, integer"], + ["tracker_top_cities_multi", "uuid[], integer, integer"], + ["tracker_device_totals_multi", "uuid[], integer"], +]; + +const RECENT_RPCS: Array<[string, string]> = [ + ["tracker_recent_series", "uuid, integer, integer"], + ["tracker_recent_bucket_totals", "uuid, integer, integer"], + ["tracker_recent_event_mix", "uuid, integer"], + ["tracker_recent_top_pages", "uuid, integer, integer"], + ["tracker_recent_top_referrers", "uuid, integer, integer"], + ["tracker_recent_top_actions", "uuid, integer, integer"], + ["tracker_recent_top_countries", "uuid, integer, integer"], + ["tracker_recent_top_cities", "uuid, integer, integer"], +]; + +const ALL_RPCS = [...SINGLE_RPCS, ...MULTI_RPCS, ...RECENT_RPCS]; + +const WORK_MEM_RPCS = [ + "tracker_top_pages_multi", + "tracker_top_actions_multi", + "tracker_top_exit_pages_multi", +]; + +function esc(s: string) { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** The body of `create function public.(` up to its closing `$$;`. */ +function definition(name: string): string { + const m = migration.match( + new RegExp(`create function public\\.${esc(name)}\\(([\\s\\S]*?)\\$\\$;`), + ); + if (!m) throw new Error(`no create function for ${name}`); + return m[0]; +} + +describe("tracker kind split migration", () => { + it("adds kind to each bucket-less rollup and the exit session record, unknown by default", () => { + for (const table of [...KIND_TABLES, "tracker_exit_sessions"]) { + expect(migration).toMatch( + new RegExp( + `alter table public\\.${table}\\s+add column if not exists kind text not null default 'unknown'`, + ), + ); + } + expect(migration).toContain("check (kind in (''human'', ''bot'', ''unknown''))"); + }); + + it("puts kind last in each rollup's primary key, matching the ingest's key columns", () => { + expect(migration).toContain( + "'project_id, day, event, page_path, referrer_host, event_target, kind'", + ); + expect(migration).toContain("'project_id, day, device_type, browser, os, kind'"); + expect(migration).toContain( + "'project_id, day, country_code, region_code, city, timezone, kind'", + ); + expect(migration).toContain("'project_id, day, page_path, kind'"); + // Guarded on the current key, so a replay does not rebuild. + expect(migration).toMatch(/c\.contype = 'p'\s+and a\.attname = 'kind'/); + }); + + it("keeps page_path in the covering indexes and adds kind to the include", () => { + expect(migration).toMatch( + /on public\.tracker_event_daily_stats \(project_id, event, day desc\)\s+include \(page_path, count, kind\)/, + ); + expect(migration).toMatch( + /on public\.tracker_event_daily_stats \(project_id, day desc\)\s+include \(event, page_path, referrer_host, event_target, count, kind\)/, + ); + // New names first, then the old ones go, so coverage never lapses. + const created = migration.indexOf("tracker_event_daily_stats_project_event_day_kind_idx"); + const dropped = migration.indexOf( + "drop index if exists public.tracker_event_daily_stats_project_event_day_idx", + ); + expect(created).toBeGreaterThan(-1); + expect(dropped).toBeGreaterThan(created); + }); + + it("drops each RPC by its exact old signature and re-creates it with p_kind last", () => { + for (const [name, oldArgs] of ALL_RPCS) { + expect(migration, name).toContain( + `drop function if exists public.${name}(${oldArgs});`, + ); + const def = definition(name); + expect(def, name).toMatch(/p_kind text default null\n\)/); + expect(migration, name).toContain( + `grant execute on function public.${name}(${oldArgs}, text) to authenticated, service_role;`, + ); + } + }); + + it("filters every RPC by p_kind, on kind for rollups and on the bucket prefix elsewhere", () => { + for (const [name] of ALL_RPCS) { + const def = definition(name); + const onKind = def.includes("(p_kind is null or kind = p_kind)"); + const onBucket = + def.includes("(p_kind = 'bot' and bucket like 'bot:%')") || + def.includes("(p_kind = 'bot' and e.bucket like 'bot:%')"); + expect(onKind || onBucket, name).toBe(true); + } + // The bucket table and the raw table have no kind column. + for (const name of [ + "tracker_bucket_totals", + "tracker_bucket_totals_multi", + ...RECENT_RPCS.map(([n]) => n), + ]) { + expect(definition(name), name).not.toContain("kind = p_kind"); + } + // The series RPCs filter both legs. + for (const name of ["tracker_daily_series", "tracker_daily_series_multi"]) { + const def = definition(name); + expect(def, name).toContain("(p_kind is null or kind = p_kind)"); + expect(def, name).toContain("(p_kind = 'human' and bucket not like 'bot:%')"); + } + }); + + it("keeps work_mem on the three portfolio functions that spill, and only those", () => { + for (const [name] of ALL_RPCS) { + const hasWorkMem = /set work_mem = '16MB'/.test(definition(name)); + expect(hasWorkMem, name).toBe(WORK_MEM_RPCS.includes(name)); + } + }); + + it("never makes a tracker RPC security definer", () => { + expect(migration).not.toMatch(/security definer/i); + for (const [name] of ALL_RPCS) { + expect(definition(name), name).toContain("security invoker"); + expect(definition(name), name).toContain("set search_path = public"); + } + }); + + it("says that rows before it are unknown and only appear under All", () => { + expect(migration).toMatch(/ROWS BEFORE THIS MIGRATION ARE `unknown`/); + expect(migration).toMatch(/appear only under All/); + }); +}); diff --git a/tests/contract/tracker-stats-who.test.ts b/tests/contract/tracker-stats-who.test.ts new file mode 100644 index 00000000..a5bd3b24 --- /dev/null +++ b/tests/contract/tracker-stats-who.test.ts @@ -0,0 +1,140 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { NextRequest } from "next/server"; + +// The tracker-stats and live-events routes carry the page-wide Humans / Bots / +// All toggle. A missing `who` is the page default; a junk one is a 400, never +// a quiet fall-back to a different answer than the card asked for. + +const rpc = vi.hoisted(() => + vi.fn(async () => ({ data: [], error: null })), +); + +const liveQuery = vi.hoisted(() => { + const calls: Array<[string, unknown[]]> = []; + const chain: Record = {}; + for (const m of ["select", "eq", "gte", "like", "not", "order"]) { + chain[m] = (...args: unknown[]) => { + calls.push([m, args]); + return chain; + }; + } + chain.limit = async () => ({ data: [], error: null }); + return { chain, calls }; +}); + +vi.mock("@/lib/lx/currentSite", () => ({ + requireProjectAccess: async () => ({ ok: true }), +})); + +vi.mock("@/lib/supabase/service", () => ({ + serviceClient: () => ({ + rpc, + from: () => liveQuery.chain, + }), +})); + +import { GET as statsGET } from "@/app/api/projects/[id]/tracker-stats/route"; +import { GET as liveGET } from "@/app/api/projects/[id]/live-events/route"; + +function req(url: string) { + const request = new Request(url) as unknown as NextRequest; + Object.defineProperty(request, "nextUrl", { value: new URL(url) }); + return request; +} + +const params = Promise.resolve({ id: "p1" }); + +describe("GET /api/projects/:id/tracker-stats ?who=", () => { + beforeEach(() => { + rpc.mockClear(); + }); + + it("defaults to humans and passes p_kind = human", async () => { + const res = await statsGET( + req("https://crawlproof.com/api/projects/p1/tracker-stats?range=1m&panel=pages"), + { params }, + ); + expect(res.status).toBe(200); + expect((await res.json()).who).toBe("humans"); + expect(rpc).toHaveBeenCalledWith( + "tracker_top_pages", + expect.objectContaining({ p_kind: "human" }), + ); + }); + + it("maps bots and all onto p_kind", async () => { + await statsGET( + req("https://crawlproof.com/api/projects/p1/tracker-stats?range=1h&panel=pages&who=bots"), + { params }, + ); + expect(rpc).toHaveBeenCalledWith( + "tracker_recent_top_pages", + expect.objectContaining({ p_kind: "bot" }), + ); + + rpc.mockClear(); + await statsGET( + req("https://crawlproof.com/api/projects/p1/tracker-stats?range=1m&panel=devices&who=all"), + { params }, + ); + expect(rpc).toHaveBeenCalledWith( + "tracker_device_totals", + expect.objectContaining({ p_kind: null }), + ); + }); + + it("rejects junk with a 400 instead of answering a different question", async () => { + for (const bad of ["Humans", "human", "everyone", ""]) { + const res = await statsGET( + req( + `https://crawlproof.com/api/projects/p1/tracker-stats?range=1m&panel=pages&who=${bad}`, + ), + { params }, + ); + expect(res.status, bad).toBe(400); + expect((await res.json()).error).toMatch(/who/i); + } + expect(rpc).not.toHaveBeenCalled(); + }); +}); + +describe("GET /api/projects/:id/live-events ?who=", () => { + beforeEach(() => { + liveQuery.calls.length = 0; + }); + + it("filters humans by excluding bot: buckets", async () => { + const res = await liveGET( + req("https://crawlproof.com/api/projects/p1/live-events?minutes=30"), + { params }, + ); + expect(res.status).toBe(200); + expect((await res.json()).who).toBe("humans"); + expect(liveQuery.calls).toContainEqual(["not", ["bucket", "like", "bot:%"]]); + expect(liveQuery.calls.find(([m]) => m === "like")).toBeUndefined(); + }); + + it("filters bots by the bucket prefix and All by nothing", async () => { + await liveGET( + req("https://crawlproof.com/api/projects/p1/live-events?minutes=30&who=bots"), + { params }, + ); + expect(liveQuery.calls).toContainEqual(["like", ["bucket", "bot:%"]]); + + liveQuery.calls.length = 0; + await liveGET( + req("https://crawlproof.com/api/projects/p1/live-events?minutes=30&who=all"), + { params }, + ); + expect(liveQuery.calls.find(([m]) => m === "like" || m === "not")).toBeUndefined(); + }); + + it("rejects junk with a 400", async () => { + const res = await liveGET( + req("https://crawlproof.com/api/projects/p1/live-events?minutes=30&who=robots"), + { params }, + ); + expect(res.status).toBe(400); + expect(liveQuery.calls).toHaveLength(0); + }); +}); diff --git a/tests/tracker-rollup-ranges.test.ts b/tests/tracker-rollup-ranges.test.ts index dde1818b..4f7665bb 100644 --- a/tests/tracker-rollup-ranges.test.ts +++ b/tests/tracker-rollup-ranges.test.ts @@ -58,6 +58,8 @@ describe("rollup-only panels at the 1D tab", () => { await fetchPanel(sb as never, "p1", panel as never, trackerRange("1d"), 412); expect(sb.calls).toHaveLength(1); expect(sb.calls[0].args.days, `${panel} at 1D`).toBe(1); + // The Humans / Bots / All toggle reaches these too; unset means All. + expect(sb.calls[0].args, `${panel} at 1D`).toHaveProperty("p_kind", null); } }); @@ -69,6 +71,15 @@ describe("rollup-only panels at the 1D tab", () => { } }); + it("collapses 1D to one day under a kind filter as well", async () => { + for (const panel of rollupOnly) { + const sb = spySb(); + await fetchPanel(sb as never, "p1", panel as never, trackerRange("1d"), 412, "human"); + expect(sb.calls[0].args.days, `${panel} at 1D humans`).toBe(1); + expect(sb.calls[0].args.p_kind, `${panel} at 1D humans`).toBe("human"); + } + }); + it("labels the 1D tab as the UTC day it actually reads", () => { for (const panel of rollupOnly) { const ranges = rangesForPanel(panel); diff --git a/tests/tracker-who.test.ts b/tests/tracker-who.test.ts new file mode 100644 index 00000000..411863d2 --- /dev/null +++ b/tests/tracker-who.test.ts @@ -0,0 +1,267 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_WHO, + KIND_SPLIT_CAPTION, + WHO_OPTIONS, + WHO_VALUES, + headlineTiles, + parseWho, + pulseHeadline, + pulseLayers, + whoCaption, + whoOrDefault, + whoToKind, +} from "@/lib/tracker/who"; +import { kindFromBucket } from "@/lib/tracker/humans"; +import { fetchPanel, PANEL_KEYS } from "@/lib/tracker/panels"; +import { trackerRange } from "@/lib/tracker/ranges"; +import { buildDailyAxis } from "@/lib/tracker/series"; +import { panelUrl } from "@/components/charts/use-panel-range"; + +// The Humans / Bots / All toggle on the project stats page is one URL value +// that has to reach every number on the page through the same mapping. These +// pin that mapping: URL -> Who -> p_kind on every RPC, and Who -> which tiles +// and which chart layers are drawn. + +describe("parseWho / whoOrDefault", () => { + it("accepts exactly the three values", () => { + for (const v of WHO_VALUES) expect(parseWho(v)).toBe(v); + expect(WHO_VALUES).toEqual(["humans", "bots", "all"]); + }); + + it("rejects junk strictly, so the API can 400 it", () => { + expect(parseWho("Humans")).toBeNull(); + expect(parseWho("human")).toBeNull(); + expect(parseWho("")).toBeNull(); + expect(parseWho("bots;drop")).toBeNull(); + expect(parseWho(null)).toBeNull(); + expect(parseWho(undefined)).toBeNull(); + }); + + it("defaults the page to humans", () => { + expect(DEFAULT_WHO).toBe("humans"); + expect(whoOrDefault(undefined)).toBe("humans"); + expect(whoOrDefault("nope")).toBe("humans"); + expect(whoOrDefault("bots")).toBe("bots"); + expect(whoOrDefault("all")).toBe("all"); + }); +}); + +describe("whoToKind", () => { + it("maps onto the RPC argument, with All meaning no filter", () => { + expect(whoToKind("humans")).toBe("human"); + expect(whoToKind("bots")).toBe("bot"); + expect(whoToKind("all")).toBeNull(); + }); +}); + +describe("kindFromBucket", () => { + it("uses the one definition: bot iff the bucket starts with bot:", () => { + expect(kindFromBucket("bot:gptbot")).toBe("bot"); + expect(kindFromBucket("bot:other")).toBe("bot"); + // AI referrals are people arriving from an assistant. + expect(kindFromBucket("ai_referral:chatgpt")).toBe("human"); + expect(kindFromBucket("search:google")).toBe("human"); + expect(kindFromBucket("social:x")).toBe("human"); + expect(kindFromBucket("referral:example.com")).toBe("human"); + expect(kindFromBucket("human:direct")).toBe("human"); + // Only the prefix counts; "robots" is not a bot bucket. + expect(kindFromBucket("referral:robots.example")).toBe("human"); + }); +}); + +describe("whoCaption", () => { + it("says where the split starts on every filtered view, and nothing on All", () => { + expect(whoCaption("humans")).toBe(KIND_SPLIT_CAPTION); + expect(whoCaption("bots")).toBe(KIND_SPLIT_CAPTION); + expect(whoCaption("all")).toBeNull(); + expect(KIND_SPLIT_CAPTION).toBe( + "Split recorded from 5 Sep 2026; earlier traffic appears under All.", + ); + }); + + it("keeps a definition tooltip on every option", () => { + for (const option of WHO_OPTIONS) { + expect(option.description.length).toBeGreaterThan(20); + } + }); +}); + +describe("headlineTiles", () => { + const totals = { humans: 2570, ai: 40, bots: 254430 }; + + it("shows only that side's figures under Humans and Bots", () => { + expect(headlineTiles("humans", totals).map((t) => [t.key, t.value])).toEqual([ + ["humans", 2570], + ["ai", 40], + ]); + expect(headlineTiles("bots", totals).map((t) => [t.key, t.value])).toEqual([ + ["bots", 254430], + ]); + }); + + it("keeps the three-tile layout under All", () => { + expect(headlineTiles("all", totals).map((t) => t.key)).toEqual([ + "humans", + "ai", + "bots", + ]); + }); + + it("carries a definition on every tile", () => { + for (const who of WHO_VALUES) { + for (const tile of headlineTiles(who, totals)) { + expect(tile.hint.length).toBeGreaterThan(20); + expect(tile.label.length).toBeGreaterThan(0); + } + } + }); +}); + +describe("pulseLayers / pulseHeadline", () => { + it("draws the human band alone under Humans, the bot band alone under Bots", () => { + expect(pulseLayers("humans").map((l) => l.dataKey)).toEqual([ + "humans", + "ai", + "interactions", + ]); + expect(pulseLayers("bots").map((l) => l.dataKey)).toEqual([ + "bots", + "interactions", + ]); + }); + + it("stacks humans and bots under All", () => { + const layers = pulseLayers("all"); + expect(layers.map((l) => l.dataKey)).toEqual([ + "humans", + "bots", + "ai", + "interactions", + ]); + expect(layers.find((l) => l.dataKey === "humans")?.stackId).toBe("1"); + expect(layers.find((l) => l.dataKey === "bots")?.stackId).toBe("1"); + // Overlays, not part of the stack, or they would inflate the total. + expect(layers.find((l) => l.dataKey === "ai")?.stackId).toBeUndefined(); + expect(layers.find((l) => l.dataKey === "interactions")?.stackId).toBeUndefined(); + }); + + it("leads the frame with bots only under Bots", () => { + const totals = { humans: 3, bots: 997 }; + expect(pulseHeadline("humans", totals).total).toBe(3); + expect(pulseHeadline("all", totals).total).toBe(3); + expect(pulseHeadline("bots", totals).total).toBe(997); + expect(pulseHeadline("bots", totals).unit).toEqual(["bot crawl", "bot crawls"]); + }); +}); + +describe("panelUrl", () => { + it("sends who with every range request", () => { + expect(panelUrl("p1", "pages", "1h", "bots")).toBe( + "/api/projects/p1/tracker-stats?range=1h&panel=pages&who=bots", + ); + }); +}); + +// Records the arguments each RPC is called with. +function spySb() { + const calls: Array<{ fn: string; args: Record }> = []; + return { + calls, + rpc(fn: string, args: Record) { + calls.push({ fn, args }); + return Promise.resolve({ data: [], error: null }); + }, + }; +} + +describe("fetchPanel passes p_kind to every RPC", () => { + it("on the rollup ranges", async () => { + for (const panel of PANEL_KEYS) { + for (const kind of ["human", "bot", null] as const) { + const sb = spySb(); + await fetchPanel(sb as never, "p1", panel, trackerRange("1m"), 30, kind); + expect(sb.calls, `${panel} ${kind}`).toHaveLength(1); + expect(sb.calls[0].args, `${panel} ${kind}`).toHaveProperty("p_kind", kind); + expect(sb.calls[0].fn).not.toMatch(/^tracker_recent_/); + } + } + }); + + it("on the raw ranges too, including the rollup-only panels at 1D", async () => { + for (const panel of PANEL_KEYS) { + const sb = spySb(); + await fetchPanel(sb as never, "p1", panel, trackerRange("1d"), 412, "bot"); + expect(sb.calls, panel).toHaveLength(1); + expect(sb.calls[0].args, panel).toHaveProperty("p_kind", "bot"); + } + }); + + it("defaults to no filter so older callers keep their numbers", async () => { + const sb = spySb(); + await fetchPanel(sb as never, "p1", "pages", trackerRange("1m"), 30); + expect(sb.calls[0].args.p_kind).toBeNull(); + }); +}); + +describe("filtered series and the legacy events backfill", () => { + const NOW = new Date("2026-09-05T12:00:00Z"); + // A pre-split day: the event table has rows (kind = unknown, so excluded + // under a filter) while the bucket leg reports 0 for this side. + const rows = [ + { + day: "2026-09-04", + pageviews: 50, + interactions: 5, + ai: 0, + bots: 0, + events: 0, + humans: 0, + }, + ]; + + it("fills events from the event table only on the unfiltered series", () => { + const all = buildDailyAxis(rows, 2, NOW); + expect(all.find((p) => p.date === "2026-09-04")?.events).toBe(55); + }); + + it("leaves a filtered zero alone rather than borrowing the other side's rows", () => { + const filtered = buildDailyAxis(rows, 2, NOW, { legacyEventsBackfill: false }); + expect(filtered.find((p) => p.date === "2026-09-04")?.events).toBe(0); + }); + + it("is wired that way through fetchPanel", async () => { + const calls: Array> = []; + const sb = { + rpc(fn: string, args: Record) { + calls.push({ fn, ...args }); + return Promise.resolve({ + data: fn === "tracker_daily_series" ? [{ ...rows[0], day: today() }] : [], + error: null, + }); + }, + }; + const human = (await fetchPanel( + sb as never, + "p1", + "series", + trackerRange("1w"), + 7, + "human", + )) as { points: Array<{ events: number }> }; + const all = (await fetchPanel( + sb as never, + "p1", + "series", + trackerRange("1w"), + 7, + null, + )) as { points: Array<{ events: number }> }; + expect(human.points.at(-1)?.events).toBe(0); + expect(all.points.at(-1)?.events).toBe(55); + }); +}); + +function today() { + return new Date().toISOString().slice(0, 10); +}