diff --git a/README.md b/README.md index 2b94ff0..153af6d 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,8 @@ The source of truth for [scientfactory.com](https://scientfactory.com), includin - [`ScientFactory/scient-desktop`](https://github.com/ScientFactory/scient-desktop) owns the desktop application and its releases. - [`ScientFactory/scient-agent`](https://github.com/ScientFactory/scient-agent) - is the planned home for future native Scient-agent work. Its current - OpenCode-derived starting repository is not yet an implemented native-agent - foundation. + owns Scient Agent, the native research agent built from Oh My Pi that the + desktop application installs and runs. - [`ScientFactory/ScientFactory-website`](https://github.com/ScientFactory/ScientFactory-website) owns this website and download experience. @@ -118,13 +117,17 @@ Cloudflare D1 stores four website event types: Before a visitor chooses analytics, and after choosing **Essential only**, every event receives a new event-specific identifier. Those counts represent events rather than unique people. After explicit **Allow analytics** consent, the site sets a random first-party visitor identifier and creates a session identifier so visits, downloads, sessions, and return behavior can be measured. The identity is not derived from an IP address, browser fingerprint, email address, advertising identifier, referrer, or third-party account. `download_failed` is limited to a failure in ScientFactory's redirect service; the website cannot observe a transfer failure after GitHub begins serving an installer. +Each visit also records where it came from: the referring site's host (never its full address) and the link's `utm_source`, `utm_medium` and `utm_campaign` tags. The browser keeps them for the tab and sends them once on the visit's first page view, and the site's download links carry them with the page the download was clicked on. A download link on another site is credited to that site's host. Values that look like an address or an email are dropped. + +Downloads made directly from GitHub never reach this site. Once a day the events Worker saves GitHub's running total for each release file in `github_release_downloads`. GitHub counts every fetch: Mac `.dmg` files are installs only and Mac updates fetch the `.zip`, but Windows `.exe` and Linux `.AppImage` totals also include updates that fell back to GitHub from the update mirror. A token in the optional `GITHUB_TOKEN` Worker secret only raises GitHub's rate limit. + The production binding is `DOWNLOAD_DB`, backed by the `scientfactory-downloads` D1 database. New events use the shared `analytics_events` table; the earlier `site_events` table remains as read-only historical data. Apply new migrations before deploying code that depends on them: ```sh bun run db:migrate ``` -To view the lifetime event summary plus 30-day identity, consent, session, download, outbound-link, and failure breakdowns: +To view the lifetime event summary plus 30-day identity, consent, session, download (by source, campaign, place and network, and GitHub's totals), outbound-link, and failure breakdowns: ```sh bun run analytics:report diff --git a/functions/_lib/analyticsPeople.ts b/functions/_lib/analyticsPeople.ts index 07e63ab..335915b 100644 --- a/functions/_lib/analyticsPeople.ts +++ b/functions/_lib/analyticsPeople.ts @@ -180,15 +180,14 @@ export function webObservationStatement( occurredAt: string, visitorId: string, request: Request, + /** The site the visit came from; the request's own Referer is usually this website. */ + sourceHost: string | null = null, ): D1PreparedStatement | null { const network = networkObservation(request); if (!network) return null; - let referrer: string | null = null; - try { - referrer = new URL(request.headers.get("Referer") ?? "").hostname; - } catch { - /* optional */ - } + // The request's own Referer is this website (the page that sent the event), so only the + // visit's source, or the other site whose download link was followed, is recorded. + const referrer = sourceHost; return db .prepare(`INSERT OR IGNORE INTO analytics_web_sightings (event_id, event_name, occurred_at, visitor_id, ip_address, referrer_host, country, region, city, timezone, asn, network_name) diff --git a/functions/_lib/events.ts b/functions/_lib/events.ts index e99b289..41f9952 100644 --- a/functions/_lib/events.ts +++ b/functions/_lib/events.ts @@ -4,6 +4,7 @@ import { webObservationStatement } from "./analyticsPeople"; // Layer: Cloudflare Pages Function utility import { cleanSessionId, websiteIdentity, websiteSessionId } from "./identity"; +import type { VisitSource } from "./visitSource"; export const SITE_EVENT_NAMES = [ "page_viewed", @@ -14,7 +15,8 @@ export const SITE_EVENT_NAMES = [ export type SiteEventName = (typeof SITE_EVENT_NAMES)[number]; -export interface SiteEvent { +/** Page views on landing and downloads also carry where the visit came from. */ +export interface SiteEvent extends VisitSource { readonly eventName: SiteEventName; readonly pagePath?: string | null; readonly assetKey?: string | null; @@ -64,9 +66,11 @@ function limited(value: string | null | undefined, maximum: number): string | nu return value.slice(0, maximum); } +/** A path on this site: never another host (`//host/...`) or anything with an email in it. */ export function cleanPath(value: string | null | undefined): string | null { - if (!value?.startsWith("/")) return null; + if (!value?.startsWith("/") || value.startsWith("//") || value.startsWith("/\\")) return null; const path = value.split(/[?#]/, 1)[0] ?? "/"; + if (path.includes("@")) return null; return limited(path, 512); } @@ -123,6 +127,11 @@ export async function insertSiteEvent( upstream_elapsed_ms: event.upstreamElapsedMs ?? null, upstream_attempts: event.upstreamAttempts ?? null, fallback: event.fallback ?? null, + referrer_host: limited(event.referrerHost, 253), + utm_source: limited(event.utmSource, 100), + utm_medium: limited(event.utmMedium, 100), + utm_campaign: limited(event.utmCampaign, 100), + landing_path: cleanPath(event.landingPath), }).filter((entry): entry is [string, string | number] => entry[1] !== null), ); @@ -165,6 +174,7 @@ export async function insertSiteEvent( occurredAt, distinctId, request, + event.referrerHost ?? null, ); if (observation) statements.push(observation); } diff --git a/functions/_lib/visitSource.test.ts b/functions/_lib/visitSource.test.ts new file mode 100644 index 0000000..ffda6a9 --- /dev/null +++ b/functions/_lib/visitSource.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; + +import { cleanCampaignTag, cleanReferrerHost, visitSourceFrom } from "./visitSource"; + +describe("visit source", () => { + it("keeps a referring site's host and drops this website", () => { + expect(cleanReferrerHost("WWW.Google.com", "scientfactory.com")).toBe("google.com"); + expect(cleanReferrerHost("com.google.android.gm", "scientfactory.com")).toBe( + "com.google.android.gm", + ); + expect(cleanReferrerHost("www.scientfactory.com", "scientfactory.com")).toBeNull(); + expect(cleanReferrerHost("https://google.com/search?q=x", "scientfactory.com")).toBeNull(); + expect(cleanReferrerHost(42, "scientfactory.com")).toBeNull(); + }); + + it("keeps short campaign labels and drops addresses and emails", () => { + expect(cleanCampaignTag(" spring-launch_2026 ")).toBe("spring-launch_2026"); + expect(cleanCampaignTag("LinkedIn | paid")).toBe("LinkedIn | paid"); + expect(cleanCampaignTag("someone@example.com")).toBeNull(); + expect(cleanCampaignTag("https://example.com")).toBeNull(); + expect(cleanCampaignTag("/private/path")).toBeNull(); + expect(cleanCampaignTag("x".repeat(101))).toBeNull(); + expect(cleanCampaignTag("example.com/private/path")).toBeNull(); + expect(cleanCampaignTag("google.com")).toBe("google.com"); + }); + + it("keeps only paths on this site", () => { + const read = (landing: string) => + visitSourceFrom((name) => (name === "landing_path" ? landing : null), "scientfactory.com") + .landingPath; + expect(read("/docs/install?x=1")).toBe("/docs/install"); + expect(read("//example.com/private")).toBeNull(); + expect(read("/\\example.com")).toBeNull(); + expect(read("/people/person@example.com")).toBeNull(); + }); + + it("reads every field through the same checks", () => { + const values: Record = { + referrer_host: "x.com", + utm_source: "twitter", + utm_medium: "social", + utm_campaign: "bad