Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions app/(app)/projects/[id]/stats/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,12 @@ type GeoRow = {
timezone: string;
count: number;
};
type DeviceRow = {
device_type: string;
browser: string;
os: string;
count: number;
};

const WINDOW_DAYS = 30;

Expand Down Expand Up @@ -90,6 +96,14 @@ export default async function ProjectStatsPage({

const geoRows = (geoStats ?? []) as GeoRow[];

const { data: deviceStats } = await supabase
.from("tracker_device_daily_stats")
.select("device_type, browser, os, count")
.eq("project_id", id)
.gte("day", since);

const deviceRows = (deviceStats ?? []) as DeviceRow[];

// Roll up by bucket for the table view; sort by total desc.
const byBucket = new Map<string, number>();
let totalAi = 0;
Expand Down Expand Up @@ -136,6 +150,12 @@ export default async function ProjectStatsPage({
return [row.city, region, country].filter(Boolean).join(", ");
});

const topDevices = topDeviceItems(deviceRows, (row) =>
deviceTypeLabel(row.device_type),
);
const topBrowsers = topDeviceItems(deviceRows, (row) => row.browser);
const topOperatingSystems = topDeviceItems(deviceRows, (row) => row.os);

const trackerEnabled = !!(project as { tracker_enabled?: boolean })
.tracker_enabled;

Expand Down Expand Up @@ -283,6 +303,9 @@ export default async function ProjectStatsPage({
actions={topActions}
countries={topCountries}
cities={topCities}
devices={topDevices}
browsers={topBrowsers}
operatingSystems={topOperatingSystems}
/>
)}
</div>
Expand Down Expand Up @@ -431,6 +454,37 @@ function topGeoItems(
.slice(0, 10);
}

function topDeviceItems(
rows: DeviceRow[],
labelFor: (row: DeviceRow) => string,
): TrackerListItem[] {
const map = new Map<string, number>();
for (const row of rows) {
const label = labelFor(row);
if (!label) continue;
map.set(label, (map.get(label) ?? 0) + row.count);
}
return Array.from(map.entries())
.map(([label, value]) => ({ label, value }))
.sort((a, b) => b.value - a.value)
.slice(0, 10);
}

function deviceTypeLabel(deviceType: string) {
switch (deviceType) {
case "mobile":
return "Mobile";
case "tablet":
return "Tablet";
case "desktop":
return "Desktop";
case "bot":
return "Bot";
default:
return "";
}
}

function eventLabel(event: string) {
return event
.split("_")
Expand Down
4 changes: 3 additions & 1 deletion app/(marketing)/docs/statistics/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,9 @@ window.crawlproof?.track("purchase", "pro_plan");`}</pre>
</ul>
<p className="text-sm leading-relaxed">
CrawlProof stores daily aggregate rollups by source, event, page,
referrer host, target label, and location. The network request is
referrer host, target label, location, and device (device type,
browser, and OS, derived server-side from the User-Agent — raw
User-Agent strings are never stored). The network request is
best-effort and never blocks the host page.
</p>
</section>
Expand Down
39 changes: 39 additions & 0 deletions app/api/track/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { NextRequest, NextResponse } from "next/server";
import { z } from "zod";
import { serviceClient } from "@/lib/supabase/service";
import { categorize } from "@/lib/tracker/categorize";
import { parseDevice } from "@/lib/tracker/device";
import { clientIpFromHeaders, lookupGeo } from "@/lib/tracker/geo";
import { enqueuePostHogEvent } from "@/lib/posthog/events";
import { AUDIENCE_BROWSER_EVENTS, ingestAudienceEvent } from "@/lib/audience/hub";
Expand Down Expand Up @@ -330,6 +331,44 @@ async function ingest(request: NextRequest, parseBody: boolean) {
});
}

// Device / browser / OS rollup, derived from the request User-Agent. Like
// the geo rollup we store only aggregate counts, never the raw UA string.
const device = parseDevice(userAgent);
if (device.deviceType || device.browser || device.os) {
const { data: deviceExisting } = await sb
.from("tracker_device_daily_stats")
.select("count")
.eq("project_id", site)
.eq("day", today)
.eq("device_type", device.deviceType)
.eq("browser", device.browser)
.eq("os", device.os)
.maybeSingle();

if (deviceExisting) {
await sb
.from("tracker_device_daily_stats")
.update({
count: (deviceExisting.count ?? 0) + 1,
updated_at: new Date().toISOString(),
})
.eq("project_id", site)
.eq("day", today)
.eq("device_type", device.deviceType)
.eq("browser", device.browser)
.eq("os", device.os);
} else {
await sb.from("tracker_device_daily_stats").insert({
project_id: site,
day: today,
device_type: device.deviceType,
browser: device.browser,
os: device.os,
count: 1,
});
}
}

const geo = await lookupGeo(clientIpFromHeaders(request.headers));

// Write a raw event row for the real-time "active in last 30 min" view.
Expand Down
20 changes: 20 additions & 0 deletions components/charts/tracker-analytics.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ export function TrackerAnalytics({
actions,
countries,
cities,
devices,
browsers,
operatingSystems,
}: {
daily: TrackerDailyPoint[];
events: TrackerListItem[];
Expand All @@ -57,6 +60,9 @@ export function TrackerAnalytics({
actions: TrackerListItem[];
countries: TrackerListItem[];
cities: TrackerListItem[];
devices: TrackerListItem[];
browsers: TrackerListItem[];
operatingSystems: TrackerListItem[];
}) {
const total = daily.reduce((sum, point) => sum + point.events, 0);

Expand Down Expand Up @@ -176,6 +182,20 @@ export function TrackerAnalytics({
data={cities}
empty="No city data yet."
/>
<Breakdown title="Devices" data={devices} />
</div>

<div className="grid gap-4 lg:grid-cols-2">
<RankedList
title="Browsers"
data={browsers}
empty="No browser data yet."
/>
<RankedList
title="Operating systems"
data={operatingSystems}
empty="No OS data yet."
/>
</div>
</div>
);
Expand Down
76 changes: 76 additions & 0 deletions lib/tracker/device.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
// Lightweight, dependency-free User-Agent parsing for the drop-in tracker.
// We only need coarse buckets for analytics — device type, browser family,
// and OS family — not a full UA database. Anything we can't confidently
// classify falls back to "" (matching the empty-string defaults used by the
// geo rollups) so the dashboard can simply skip unknowns.

export interface DeviceInfo {
/** "mobile" | "tablet" | "desktop" | "bot" | "" */
deviceType: string;
/** Browser family, e.g. "Chrome", "Safari", "Firefox", "Edge". */
browser: string;
/** OS family, e.g. "Windows", "macOS", "iOS", "Android", "Linux". */
os: string;
}

const EMPTY: DeviceInfo = { deviceType: "", browser: "", os: "" };

// Substring needles that mark a non-human client. Kept loose on purpose —
// categorize() already does the authoritative bot bucketing; this is just so
// device stats don't get polluted by obvious crawlers.
const BOT_RE = /bot\b|crawler|spider|scraper|headless|preview|fetch|monitor|http-client|axios|curl|wget|python-requests|go-http/i;

function detectOs(ua: string): string {
// Order matters: iOS/iPadOS report "like Mac OS X", and Android UAs also
// contain "Linux", so the more specific tokens must be checked first.
if (/windows phone/i.test(ua)) return "Windows Phone";
if (/windows nt|win64|win32|windows/i.test(ua)) return "Windows";
if (/android/i.test(ua)) return "Android";
if (/(iphone|ipad|ipod)/i.test(ua)) return "iOS";
if (/cros/i.test(ua)) return "ChromeOS";
if (/mac os x|macintosh/i.test(ua)) return "macOS";
if (/linux/i.test(ua)) return "Linux";
return "";
}

function detectBrowser(ua: string): string {
// Edge / Opera / Samsung masquerade as Chrome, and Chrome masquerades as
// Safari, so check the more specific tokens before the generic ones.
if (/edg(a|ios|e)?\//i.test(ua)) return "Edge";
if (/opr\/|opera/i.test(ua)) return "Opera";
if (/samsungbrowser/i.test(ua)) return "Samsung Internet";
if (/ucbrowser/i.test(ua)) return "UC Browser";
if (/firefox|fxios/i.test(ua)) return "Firefox";
if (/chrome|crios|chromium/i.test(ua)) return "Chrome";
if (/safari/i.test(ua)) return "Safari";
if (/msie|trident/i.test(ua)) return "Internet Explorer";
return "";
}

function detectDeviceType(ua: string): string {
if (/ipad|tablet|playbook|silk|kindle/i.test(ua)) return "tablet";
// Android phones say "Mobile"; Android tablets omit it.
if (/android/i.test(ua)) return /mobile/i.test(ua) ? "mobile" : "tablet";
if (/iphone|ipod/i.test(ua)) return "mobile";
if (/mobi|mobile|phone|iemobile|blackberry|bb10|opera mini/i.test(ua)) {
return "mobile";
}
return "desktop";
}

/**
* Parse a raw User-Agent header into coarse analytics buckets. Returns empty
* strings for fields we can't classify; never throws.
*/
export function parseDevice(userAgent: string | null | undefined): DeviceInfo {
const ua = (userAgent ?? "").trim();
if (!ua) return EMPTY;
if (BOT_RE.test(ua)) {
return { deviceType: "bot", browser: "", os: "" };
}
return {
deviceType: detectDeviceType(ua),
browser: detectBrowser(ua),
os: detectOs(ua),
};
}
39 changes: 39 additions & 0 deletions supabase/migrations/20260617120000_tracker_device_rollups.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
-- Daily device rollups for the drop-in stats tracker. /api/track derives
-- coarse device type / browser / OS buckets from the request User-Agent and
-- writes only aggregate counts — no raw UA strings are stored, keeping this in
-- line with the cookie-free, PII-light design of the other tracker tables.
create table if not exists public.tracker_device_daily_stats (
project_id uuid not null references public.projects(id) on delete cascade,
day date not null,
device_type text not null default '',
browser text not null default '',
os text not null default '',
count int not null default 0,
updated_at timestamptz not null default now(),
primary key (project_id, day, device_type, browser, os)
);

create index if not exists tracker_device_daily_stats_project_day_idx
on public.tracker_device_daily_stats(project_id, day desc);

alter table public.tracker_device_daily_stats enable row level security;

create policy "tracker_device_daily_stats owner select"
on public.tracker_device_daily_stats for select
using (
project_id in (select id from public.projects where owner_id = auth.uid())
);

-- Members of a project may read its device stats (mirrors the member-select
-- policies on the other tracker_*_daily_stats tables).
create policy "tracker_device_daily_stats member select"
on public.tracker_device_daily_stats for select
using (
exists(
select 1 from public.project_members
where project_id = tracker_device_daily_stats.project_id
and user_id = auth.uid()
)
);

-- Service role writes from /api/track; end users only read their own rollups.
70 changes: 70 additions & 0 deletions tests/contract/tracker-device.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import { parseDevice } from "@/lib/tracker/device";

describe("parseDevice", () => {
it("returns empty fields for missing UA", () => {
expect(parseDevice(null)).toEqual({ deviceType: "", browser: "", os: "" });
expect(parseDevice("")).toEqual({ deviceType: "", browser: "", os: "" });
});

it("classifies desktop Chrome on Windows", () => {
const ua =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
expect(parseDevice(ua)).toEqual({
deviceType: "desktop",
browser: "Chrome",
os: "Windows",
});
});

it("classifies iPhone Safari as mobile iOS", () => {
const ua =
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1";
expect(parseDevice(ua)).toEqual({
deviceType: "mobile",
browser: "Safari",
os: "iOS",
});
});

it("classifies iPad as a tablet", () => {
const ua =
"Mozilla/5.0 (iPad; CPU OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/604.1";
expect(parseDevice(ua).deviceType).toBe("tablet");
expect(parseDevice(ua).os).toBe("iOS");
});

it("distinguishes Android phones from tablets", () => {
const phone =
"Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36";
const tablet =
"Mozilla/5.0 (Linux; Android 13; SM-X710) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
expect(parseDevice(phone)).toMatchObject({
deviceType: "mobile",
browser: "Chrome",
os: "Android",
});
expect(parseDevice(tablet).deviceType).toBe("tablet");
});

it("detects Edge over Chrome and macOS Safari", () => {
const edge =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0";
expect(parseDevice(edge).browser).toBe("Edge");

const mac =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15";
expect(parseDevice(mac)).toEqual({
deviceType: "desktop",
browser: "Safari",
os: "macOS",
});
});

it("flags obvious bots as device type bot", () => {
expect(parseDevice("Mozilla/5.0 (compatible; GPTBot/1.2)").deviceType).toBe(
"bot",
);
expect(parseDevice("curl/8.4.0").deviceType).toBe("bot");
});
});
Loading