From b53b1c7f689afd7bec2577b219c2e83304feff20 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Tue, 17 Nov 2026 23:58:49 +0200 Subject: [PATCH 1/3] feat(analytics): record where downloads come from and GitHub's totals - The first page view of a visit records the referring site's host (never its full address) and the link's utm_source, utm_medium and utm_campaign tags. - The site's download links carry that source and the page they were clicked on; a download link on another site is credited to that site's host. - Web sightings now keep the real referring site instead of this website. - The events Worker saves GitHub's per-file release totals once a day, so downloads made straight from GitHub are counted. Windows and Linux totals include updates that fell back to GitHub. - The analytics report shows 30-day downloads by source, campaign, place, network and GitHub's totals. Co-Authored-By: Claude Opus 5.5 --- README.md | 11 ++- functions/_lib/analyticsPeople.ts | 6 +- functions/_lib/events.ts | 10 ++- functions/_lib/visitSource.test.ts | 41 ++++++++++ functions/_lib/visitSource.ts | 55 +++++++++++++ functions/api/download/[asset].test.ts | 40 ++++++++++ functions/api/download/[asset].ts | 33 +++++++- functions/api/events.test.ts | 40 ++++++++++ functions/api/events.ts | 3 + migrations/0009_github_release_downloads.sql | 16 ++++ scripts/analytics-report.mjs | 82 +++++++++++++++++++- scripts/analytics-report.test.mjs | 47 +++++++++++ src/layouts/Layout.astro | 79 ++++++++++++++++++- workers/events/src/githubDownloads.test.ts | 60 ++++++++++++++ workers/events/src/githubDownloads.ts | 82 ++++++++++++++++++++ workers/events/src/index.ts | 22 +++++- 16 files changed, 611 insertions(+), 16 deletions(-) create mode 100644 functions/_lib/visitSource.test.ts create mode 100644 functions/_lib/visitSource.ts create mode 100644 migrations/0009_github_release_downloads.sql create mode 100644 workers/events/src/githubDownloads.test.ts create mode 100644 workers/events/src/githubDownloads.ts diff --git a/README.md b/README.md index 2b94ff0..153af6d 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,8 @@ The source of truth for [scientfactory.com](https://scientfactory.com), includin - [`ScientFactory/scient-desktop`](https://github.com/ScientFactory/scient-desktop) owns the desktop application and its releases. - [`ScientFactory/scient-agent`](https://github.com/ScientFactory/scient-agent) - is the planned home for future native Scient-agent work. Its current - OpenCode-derived starting repository is not yet an implemented native-agent - foundation. + owns Scient Agent, the native research agent built from Oh My Pi that the + desktop application installs and runs. - [`ScientFactory/ScientFactory-website`](https://github.com/ScientFactory/ScientFactory-website) owns this website and download experience. @@ -118,13 +117,17 @@ Cloudflare D1 stores four website event types: Before a visitor chooses analytics, and after choosing **Essential only**, every event receives a new event-specific identifier. Those counts represent events rather than unique people. After explicit **Allow analytics** consent, the site sets a random first-party visitor identifier and creates a session identifier so visits, downloads, sessions, and return behavior can be measured. The identity is not derived from an IP address, browser fingerprint, email address, advertising identifier, referrer, or third-party account. `download_failed` is limited to a failure in ScientFactory's redirect service; the website cannot observe a transfer failure after GitHub begins serving an installer. +Each visit also records where it came from: the referring site's host (never its full address) and the link's `utm_source`, `utm_medium` and `utm_campaign` tags. The browser keeps them for the tab and sends them once on the visit's first page view, and the site's download links carry them with the page the download was clicked on. A download link on another site is credited to that site's host. Values that look like an address or an email are dropped. + +Downloads made directly from GitHub never reach this site. Once a day the events Worker saves GitHub's running total for each release file in `github_release_downloads`. GitHub counts every fetch: Mac `.dmg` files are installs only and Mac updates fetch the `.zip`, but Windows `.exe` and Linux `.AppImage` totals also include updates that fell back to GitHub from the update mirror. A token in the optional `GITHUB_TOKEN` Worker secret only raises GitHub's rate limit. + The production binding is `DOWNLOAD_DB`, backed by the `scientfactory-downloads` D1 database. New events use the shared `analytics_events` table; the earlier `site_events` table remains as read-only historical data. Apply new migrations before deploying code that depends on them: ```sh bun run db:migrate ``` -To view the lifetime event summary plus 30-day identity, consent, session, download, outbound-link, and failure breakdowns: +To view the lifetime event summary plus 30-day identity, consent, session, download (by source, campaign, place and network, and GitHub's totals), outbound-link, and failure breakdowns: ```sh bun run analytics:report diff --git a/functions/_lib/analyticsPeople.ts b/functions/_lib/analyticsPeople.ts index 07e63ab..e416e2d 100644 --- a/functions/_lib/analyticsPeople.ts +++ b/functions/_lib/analyticsPeople.ts @@ -180,12 +180,14 @@ export function webObservationStatement( occurredAt: string, visitorId: string, request: Request, + /** The site the visit came from; the request's own Referer is usually this website. */ + sourceHost: string | null = null, ): D1PreparedStatement | null { const network = networkObservation(request); if (!network) return null; - let referrer: string | null = null; + let referrer: string | null = sourceHost; try { - referrer = new URL(request.headers.get("Referer") ?? "").hostname; + referrer ??= new URL(request.headers.get("Referer") ?? "").hostname; } catch { /* optional */ } diff --git a/functions/_lib/events.ts b/functions/_lib/events.ts index e99b289..c49e747 100644 --- a/functions/_lib/events.ts +++ b/functions/_lib/events.ts @@ -4,6 +4,7 @@ import { webObservationStatement } from "./analyticsPeople"; // Layer: Cloudflare Pages Function utility import { cleanSessionId, websiteIdentity, websiteSessionId } from "./identity"; +import type { VisitSource } from "./visitSource"; export const SITE_EVENT_NAMES = [ "page_viewed", @@ -14,7 +15,8 @@ export const SITE_EVENT_NAMES = [ export type SiteEventName = (typeof SITE_EVENT_NAMES)[number]; -export interface SiteEvent { +/** Page views on landing and downloads also carry where the visit came from. */ +export interface SiteEvent extends VisitSource { readonly eventName: SiteEventName; readonly pagePath?: string | null; readonly assetKey?: string | null; @@ -123,6 +125,11 @@ export async function insertSiteEvent( upstream_elapsed_ms: event.upstreamElapsedMs ?? null, upstream_attempts: event.upstreamAttempts ?? null, fallback: event.fallback ?? null, + referrer_host: limited(event.referrerHost, 253), + utm_source: limited(event.utmSource, 100), + utm_medium: limited(event.utmMedium, 100), + utm_campaign: limited(event.utmCampaign, 100), + landing_path: cleanPath(event.landingPath), }).filter((entry): entry is [string, string | number] => entry[1] !== null), ); @@ -165,6 +172,7 @@ export async function insertSiteEvent( occurredAt, distinctId, request, + event.referrerHost ?? null, ); if (observation) statements.push(observation); } diff --git a/functions/_lib/visitSource.test.ts b/functions/_lib/visitSource.test.ts new file mode 100644 index 0000000..6561c11 --- /dev/null +++ b/functions/_lib/visitSource.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; + +import { cleanCampaignTag, cleanReferrerHost, visitSourceFrom } from "./visitSource"; + +describe("visit source", () => { + it("keeps a referring site's host and drops this website", () => { + expect(cleanReferrerHost("WWW.Google.com", "scientfactory.com")).toBe("google.com"); + expect(cleanReferrerHost("com.google.android.gm", "scientfactory.com")).toBe( + "com.google.android.gm", + ); + expect(cleanReferrerHost("www.scientfactory.com", "scientfactory.com")).toBeNull(); + expect(cleanReferrerHost("https://google.com/search?q=x", "scientfactory.com")).toBeNull(); + expect(cleanReferrerHost(42, "scientfactory.com")).toBeNull(); + }); + + it("keeps short campaign labels and drops addresses and emails", () => { + expect(cleanCampaignTag(" spring-launch_2026 ")).toBe("spring-launch_2026"); + expect(cleanCampaignTag("LinkedIn | paid")).toBe("LinkedIn | paid"); + expect(cleanCampaignTag("someone@example.com")).toBeNull(); + expect(cleanCampaignTag("https://example.com")).toBeNull(); + expect(cleanCampaignTag("/private/path")).toBeNull(); + expect(cleanCampaignTag("x".repeat(101))).toBeNull(); + }); + + it("reads every field through the same checks", () => { + const values: Record = { + referrer_host: "x.com", + utm_source: "twitter", + utm_medium: "social", + utm_campaign: "bad