From 348e140a45ad2e8fcc4c8b1075038cd4dda65e89 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 10 Aug 2026 10:26:34 +0000 Subject: [PATCH 1/2] feat(ads): share the visitor id with /ad.js, salt + rotate IP hashes Three changes to make ad metering identify visitors it previously couldn't, and to stop the fallback identifier from being reversible. 1. Shared visitor id. /ad.js only ever *read* localStorage['crawlproof.visitor']; nothing but /stats.js wrote it, so a publisher running the ad tag without the analytics tag sent an empty visitor on every impression -- ~69% of production impressions carried no visitor id. The minting logic moves to lib/tracker/visitorSnippet.ts and is now inlined verbatim by both routes, so the ad tag mints the id when it is the first CrawlProof script on the page. Still localStorage, still no cookie. 2. Salted, rotating IP hashes. lib/ads/serve.ts hashed IPs as bare sha256(ip) truncated to 32 hex chars. IPv4 is 2^32 addresses, so that is a reversible encoding of the IP, not a pseudonym -- and ip_hash is the only identifier the ad network has for terminal traffic, which has no localStorage at all. Both implementations (here and lib/rateLimit.ts, which used a hardcoded constant prefix) collapse into lib/ipHash.ts. Two exported variants over one implementation, because the callers need opposite properties: abuse caps look back 24h+ and must not reset at the rotation boundary, so they keep a stable salt; ad metering only needs to recognise an IP for hours, so it rotates daily and the ability to correlate expires on its own. Click dedupe queries every salt window inside its 6h lookback, so the check doesn't silently miss after each rotation. IP_HASH_SALT unset reproduces the legacy digest byte for byte, so deploying this without the env var set changes nothing. 3. Documented ?v= for terminal publishers. A terminal has no cookies and no localStorage, so unlike the web tag we cannot mint an id for the caller -- without it every fetch looks like a new person, which is why a scheduled curl loop reads as a spike of unique visitors. The slot manager now ships a snippet that generates one opaque id per machine at install time. Also replaces the raw control bytes embedded in lib/rateLimit.ts's CONTROL_OR_WS regex with \u escapes. They made the file read as binary to grep and to git (which diffed it as Bin), so searches over it silently returned nothing. Behaviour is identical and now covered by tests. Co-Authored-By: Claude Opus 5 (1M context) --- .env.example | 5 + app/ad.js/route.ts | 14 ++- app/api/ads/motd/route.ts | 8 ++ app/stats.js/route.ts | 30 +----- components/ads/slot-manager.tsx | 11 ++ lib/ads/fraud.ts | 35 +++++-- lib/ads/serve.ts | 19 ++-- lib/env.ts | 6 ++ lib/ipHash.ts | 127 +++++++++++++++++++++++ lib/rateLimit.ts | Bin 7045 -> 7305 bytes lib/tracker/visitorSnippet.ts | 53 ++++++++++ tests/contract/ad-visitor-id.test.ts | 55 ++++++++++ tests/contract/ip-hash.test.ts | 117 +++++++++++++++++++++ tests/contract/url-control-chars.test.ts | 48 +++++++++ 14 files changed, 476 insertions(+), 52 deletions(-) create mode 100644 lib/ipHash.ts create mode 100644 lib/tracker/visitorSnippet.ts create mode 100644 tests/contract/ad-visitor-id.test.ts create mode 100644 tests/contract/ip-hash.test.ts create mode 100644 tests/contract/url-control-chars.test.ts diff --git a/.env.example b/.env.example index c89143a5..e124c00a 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,11 @@ MAXMIND_GEOLITE2_CITY_DB_PATH=data/GeoLite2-City.mmdb # /api/ads/motd). Without it, slotless requests can only return the house ad. ADS_DEFAULT_SLOT_ID= +# Secret salt for every client-IP hash (ad metering + abuse caps). Unset falls +# back to the legacy unsalted digest, which is brute-forceable across the whole +# IPv4 space. Generate with `openssl rand -base64 32`. +IP_HASH_SALT= + # CoinPay credit purchases COINPAY_MERCHANT_ID=merchant_xxx COINPAY_API_KEY=cp_xxx diff --git a/app/ad.js/route.ts b/app/ad.js/route.ts index 84829621..ea7dbfb2 100644 --- a/app/ad.js/route.ts +++ b/app/ad.js/route.ts @@ -6,6 +6,7 @@ // leaves the container empty and never breaks the host page. import { env } from "@/lib/env"; +import { VISITOR_SNIPPET } from "@/lib/tracker/visitorSnippet"; const FORMATS = { banner_300x250: [300, 250], @@ -18,13 +19,7 @@ const snippet = `(function(){ try { var ORIGIN = ${JSON.stringify(env.siteUrl)}; var SIZES = ${JSON.stringify(FORMATS)}; - function visitorId() { - try { - var v = localStorage.getItem('crawlproof.visitor'); - if (v) return v; - } catch (_) {} - return ''; - } +${VISITOR_SNIPPET} function pickFormat(el, w) { var f = el.getAttribute('data-format'); if (f && SIZES[f]) return f; @@ -41,7 +36,10 @@ const snippet = `(function(){ var format = pickFormat(el, w); var dims = SIZES[format] || SIZES.banner_300x250; var q = '?slot=' + encodeURIComponent(slot) + '&format=' + encodeURIComponent(format); - var v = visitorId(); + // Mints the id if this is the first CrawlProof script on the page. It + // used to only read one stats.js had already written, so an ad-tag-only + // publisher reported every impression as an anonymous visitor. + var v = getVisitorId(); if (v) q += '&v=' + encodeURIComponent(v); el.setAttribute('data-cp-filled', '1'); fetch(ORIGIN + '/api/ads/serve' + q, { mode: 'cors', credentials: 'omit', cache: 'no-store' }) diff --git a/app/api/ads/motd/route.ts b/app/api/ads/motd/route.ts index 44112369..bcb1e579 100644 --- a/app/api/ads/motd/route.ts +++ b/app/api/ads/motd/route.ts @@ -2,6 +2,14 @@ // // curl -s "https://crawlproof.com/api/ads/motd?slot=" // curl -s "https://crawlproof.com/api/ads/motd?slot=&cols=64&color=1" +// curl -s "https://crawlproof.com/api/ads/motd?slot=&v=" +// +// `v` is the visitor id. On the web, /ad.js mints and persists one in +// localStorage; a terminal has neither cookies nor localStorage, so the caller +// has to supply it or every fetch counts as a new person — which is exactly why +// a scheduled curl loop shows up as a spike of unique visitors. Publishers +// should generate one opaque random id per machine at install time and pass it +// on every request. See the snippet in the slot manager. // // Returns an ASCII box (text/plain), sized to `cols`, with optional ANSI // colour. Meant for shell MOTDs, SSH login banners, BBS screens, and CLI tools diff --git a/app/stats.js/route.ts b/app/stats.js/route.ts index 8e9b08d4..70b95d01 100644 --- a/app/stats.js/route.ts +++ b/app/stats.js/route.ts @@ -5,6 +5,7 @@ // must never break the host page. import { env } from "@/lib/env"; +import { VISITOR_SNIPPET } from "@/lib/tracker/visitorSnippet"; const snippet = `(function(){ try { @@ -51,31 +52,10 @@ const snippet = `(function(){ })(); function pageUrl() { return location.origin + location.pathname + location.search; } - function uuid(prefix) { - try { - if (crypto && crypto.randomUUID) return prefix + crypto.randomUUID(); - } catch (_) {} - return prefix + Math.random().toString(16).slice(2) + '-' + Date.now().toString(16); - } - function lsGet(k) { try { return localStorage.getItem(k); } catch (_) { return null; } } - function lsSet(k, v) { try { localStorage.setItem(k, v); } catch (_) {} } - // In-memory fallbacks for when localStorage is blocked (private mode, etc.) - // so a single page load still reports one stable visitor/session. - var memVisitor = null, memSession = null, memSessionTs = 0; - // Persistent visitor id — stored in localStorage so it survives tab close, - // reload, and browser restart. localStorage is partitioned per site origin, - // making this a stable per-site visitor identifier (not per-tab, which would - // count every new tab as a new visitor). - function getVisitorId() { - var k = 'crawlproof.visitor'; - var id = lsGet(k); - if (id) return id; - if (memVisitor) return memVisitor; - id = uuid('v'); - lsSet(k, id); - memVisitor = id; - return id; - } +${VISITOR_SNIPPET} + // Session-only companion state. The visitor equivalents live in the shared + // snippet above, which /ad.js also inlines so both agree on the same id. + var memSession = null, memSessionTs = 0; // Session id with a 30-minute inactivity window, shared across tabs. The // window slides on every event, so an active visit stays one session and // reopening within 30 min reuses it instead of minting a new one. diff --git a/components/ads/slot-manager.tsx b/components/ads/slot-manager.tsx index 564114cd..f7925600 100644 --- a/components/ads/slot-manager.tsx +++ b/components/ads/slot-manager.tsx @@ -33,6 +33,17 @@ function embedFor(slotId: string, format: AdFormatId, origin: string): string { "# Options: &color=1 for ANSI colour, &cols=44..120 for width,", "# &src= to tell surfaces apart (rides through to the click URL).", "", + "# Repeat visitors: &v=. A terminal has no cookies and no localStorage,", + "# so unlike the web tag we can't mint this for you — without it every", + "# fetch looks like a brand new person. Generate one stable random id per", + "# machine at install time and pass it every time:", + "# id=$(cat /etc/crawlproof-visitor 2>/dev/null) || {", + "# id=$(head -c16 /dev/urandom | od -An -tx1 | tr -d ' \\n')", + "# printf '%s' \"$id\" >/etc/crawlproof-visitor", + "# }", + `# curl -fsS "${origin}/api/ads/motd?slot=${slotId}&cols=72&v=$id"`, + "# Use an opaque random value — never a hostname, username, or IP.", + "", "# Rendering a template server-side? Leave a token where the ad goes —", "# {{ads}} {{ads:64}} {{ads:terminal:64}}", "# — and swap it for the fetched text before you send the response.", diff --git a/lib/ads/fraud.ts b/lib/ads/fraud.ts index 4ce9608f..3b0e1cd4 100644 --- a/lib/ads/fraud.ts +++ b/lib/ads/fraud.ts @@ -4,8 +4,10 @@ import { serviceClient } from "@/lib/supabase/service"; // accrues to the publisher. Cheap, best-effort, and conservative: when in // doubt we still redirect the user, we just don't charge for the click. -// A visitor is counted at most once per campaign within this window. -const DEDUPE_WINDOW_MS = 6 * 60 * 60 * 1000; // 6h +// A visitor is counted at most once per campaign within this window. Exported +// so callers can ask lib/ipHash for every rotating hash an IP could have been +// stored under across the same span. +export const CLICK_DEDUPE_WINDOW_MS = 6 * 60 * 60 * 1000; // 6h export function isBotDevice(device?: string | null): boolean { return device === "bot"; @@ -26,7 +28,12 @@ export async function assessClickValidity(input: { slotId?: string | null; impressionId?: string | null; visitorId?: string | null; - ipHash?: string | null; + /** + * Every rotating IP hash to match against — today's plus any earlier salt + * window still inside CLICK_DEDUPE_WINDOW_MS. A single hash would stop + * matching yesterday's rows the moment the salt rotates. + */ + ipHashes?: string[] | null; device?: string | null; }): Promise { // 1. Bots never bill. @@ -49,11 +56,13 @@ export async function assessClickValidity(input: { // 3. Dedupe on this campaign by visitor id or ip hash within the window. const visitor = safeId(input.visitorId); - const ipHash = safeId(input.ipHash); - if (!visitor && !ipHash) return { valid: true }; // nothing to dedupe on + const ipHashes = (input.ipHashes ?? []) + .map((h) => safeId(h)) + .filter((h): h is string => h !== null); + if (!visitor && ipHashes.length === 0) return { valid: true }; // nothing to dedupe on - const since = new Date(Date.now() - DEDUPE_WINDOW_MS).toISOString(); - let q = sb + const since = new Date(Date.now() - CLICK_DEDUPE_WINDOW_MS).toISOString(); + const q = sb .from("ad_clicks") .select("id") .eq("campaign_id", input.campaignId) @@ -61,11 +70,15 @@ export async function assessClickValidity(input: { .gte("ts", since) .limit(1); - if (visitor && ipHash) q = q.or(`visitor_id.eq.${visitor},ip_hash.eq.${ipHash}`); - else if (visitor) q = q.eq("visitor_id", visitor); - else q = q.eq("ip_hash", ipHash!); + // One .or() covering every identifier: the visitor id plus each salt window's + // hash. Every term has been through safeId, so nothing unescaped reaches + // PostgREST's filter syntax. + const terms = [ + ...(visitor ? [`visitor_id.eq.${visitor}`] : []), + ...ipHashes.map((h) => `ip_hash.eq.${h}`), + ]; - const { data: dupe } = await q; + const { data: dupe } = await q.or(terms.join(",")); if (dupe && dupe.length > 0) return { valid: false, reason: "duplicate" }; return { valid: true }; diff --git a/lib/ads/serve.ts b/lib/ads/serve.ts index fa512c8c..e7aa62b8 100644 --- a/lib/ads/serve.ts +++ b/lib/ads/serve.ts @@ -12,7 +12,8 @@ import { import { TERMINAL_FORMAT_ID } from "./formats"; import { houseFill, HOUSE_AD_ROTATION_RATE } from "./house"; import { CREDIT_CENTS, DEFAULT_BID_CREDITS, PLATFORM_RATE } from "./pricing"; -import { assessClickValidity, isBotDevice } from "./fraud"; +import { assessClickValidity, isBotDevice, CLICK_DEDUPE_WINDOW_MS } from "./fraud"; +import { hashIpRotating, rotatingIpHashCandidates } from "@/lib/ipHash"; import { runAuction } from "./auction"; import { generateShortCode } from "./shortcode"; @@ -50,10 +51,9 @@ export function isAdFormat(v: string | null | undefined): v is AdFormatId { return !!v && (AD_FORMAT_IDS as string[]).includes(v); } -export function hashIp(ip: string | null): string | null { - if (!ip) return null; - return crypto.createHash("sha256").update(ip).digest("hex").slice(0, 32); -} +// hashIp used to live here as a bare sha256(ip). It now comes from lib/ipHash, +// salted and rotating daily — see that module for why the ad path wants the +// rotating variant and the abuse caps want the stable one. type CreativeRow = { id: string; @@ -240,7 +240,7 @@ export async function serveAd( campaign_id: campaign.id, creative_id: pick.id, visitor_id: ctx.visitorId ?? null, - ip_hash: hashIp(ctx.ip ?? null), + ip_hash: hashIpRotating(ctx.ip ?? null), geo_country: ctx.country ?? null, device: ctx.device ?? null, billable: false, @@ -316,13 +316,16 @@ export async function resolveClick(input: { // slot_id must be present (ad_clicks.slot_id NOT NULL) to record a click. if (input.slotId) { - const ipHash = hashIp(input.ctx?.ip ?? null); + // The row is stored under today's salt; the dedupe lookup has to consider + // yesterday's too, or every check silently misses for the first hours after + // the salt rotates. + const ipHash = hashIpRotating(input.ctx?.ip ?? null); const validity = await assessClickValidity({ campaignId: campaign.id, slotId: input.slotId, impressionId: input.impressionId, visitorId: input.ctx?.visitorId, - ipHash, + ipHashes: rotatingIpHashCandidates(input.ctx?.ip ?? null, CLICK_DEDUPE_WINDOW_MS), device: input.ctx?.device, }); diff --git a/lib/env.ts b/lib/env.ts index 1a859417..a587a426 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -21,6 +21,12 @@ export const env = { // network's own publisher slot: anonymous terminal traffic is CrawlProof's // own inventory, so its impressions/clicks accrue there. adsDefaultSlotId: process.env.ADS_DEFAULT_SLOT_ID ?? "", + // Server-side secret salting every client-IP hash (ad metering + abuse caps). + // Unset falls back to the legacy unsalted digest, which is brute-forceable + // across the whole IPv4 space in minutes — set it in production. Generate + // with `openssl rand -base64 32`. Changing it resets abuse counters once and + // breaks in-flight ad click dedupe for up to 6h; both self-heal. + ipHashSalt: process.env.IP_HASH_SALT ?? "", // CoinPay — crypto credit purchases. coinpayMerchantId: process.env.COINPAY_MERCHANT_ID ?? "", coinpayApiKey: process.env.COINPAY_API_KEY ?? "", diff --git a/lib/ipHash.ts b/lib/ipHash.ts new file mode 100644 index 00000000..954aceae --- /dev/null +++ b/lib/ipHash.ts @@ -0,0 +1,127 @@ +import crypto from "node:crypto"; +import { env } from "./env"; + +// Single source of truth for turning a client IP into a storable identifier. +// +// This replaces two divergent implementations that had drifted apart: +// * lib/rateLimit.ts — sha256("crawlproof:" + ip), a hardcoded constant +// prefix, which is a pepper in name only since it lives in the source. +// * lib/ads/serve.ts — sha256(ip), no prefix at all. +// +// Neither was anonymisation. IPv4 is 2^32 addresses, so the entire space can be +// enumerated against a truncated SHA-256 in minutes on commodity hardware; an +// unsalted digest of an IP is a reversible encoding of that IP, not a +// pseudonym. A server-side secret salt is the thing that makes it one-way in +// practice, which matters here because ip_hash is the only identifier the ad +// network has for terminal traffic (curl has no localStorage) and for the ~69% +// of web impressions that historically arrived with no visitor id. +// +// Two exported variants over one implementation, because the callers need +// opposite properties from the same primitive: +// +// hashIp() Stable over time. Abuse caps look back 24h and longer +// (checkAnonymousLimit), so a rotating salt would silently +// refill every anonymous quota at the rotation boundary — +// a quota bypass, not a privacy win. +// hashIpRotating() Salt changes daily. Ad metering only needs to recognise +// an IP within hours (6h click dedupe, frequency capping). +// Past that window, being *able* to re-identify a visitor +// is a liability rather than a feature, so the capability +// is designed to expire on its own. +// +// Rotation makes long-term correlation impossible even against a full database +// leak, while leaving the short-window behaviour the ad network actually uses +// intact. + +const LEGACY_PREFIX = "crawlproof:"; +const DAY_MS = 86_400_000; + +function sha(input: string): string { + return crypto.createHash("sha256").update(input).digest("hex").slice(0, 32); +} + +let warned = false; +function salt(): string { + const s = env.ipHashSalt; + if (s) return s; + // Deliberately does not throw. An unsalted hash is a weakness; a hard failure + // on every request that touches an IP is an outage of both ad serving and + // rate limiting. Warn once — enough to be visible in logs, quiet enough not + // to flood them at request volume. + if (!warned) { + warned = true; + console.warn( + "[ipHash] IP_HASH_SALT is unset — falling back to the legacy unsalted digest. Set it in production.", + ); + } + return ""; +} + +// NUL-joined so a salt ending in digits can't collide with an IP starting with +// them; no component of the input can bleed into the next. +function join(parts: string[]): string { + return parts.join("\u0000"); +} + +/** + * Stable, salted hash of a client IP. Use for abuse caps and anything that + * looks back more than a day. + * + * A missing IP hashes to a shared "unknown" bucket on purpose: for rate + * limiting, lumping unattributable requests together is the conservative + * choice. + */ +export function hashIp(ip: string | null | undefined): string { + const v = ip ?? "unknown"; + const s = salt(); + // With no salt configured this reproduces the historical digest byte for + // byte, so an environment that hasn't set IP_HASH_SALT yet doesn't invalidate + // every stored hash and hand each rate-limited visitor a fresh quota. + return s ? sha(join([s, LEGACY_PREFIX + v])) : sha(LEGACY_PREFIX + v); +} + +function dayIndex(at: Date): number { + return Math.floor(at.getTime() / DAY_MS); +} + +function rotatingFor(ip: string, day: number): string { + return sha(join([salt(), `day:${day}`, ip])); +} + +/** + * Daily-rotating salted hash of a client IP, for ad metering. + * + * Returns null for a missing IP rather than bucketing to a shared constant — + * the opposite of hashIp, and load-bearing: every unattributable ad request + * sharing one hash would make them all look like duplicates of each other and + * invalidate legitimate clicks wholesale. + */ +export function hashIpRotating( + ip: string | null | undefined, + at: Date = new Date(), +): string | null { + if (!ip) return null; + return rotatingFor(ip, dayIndex(at)); +} + +/** + * Every rotating hash an IP could have been stored under across `lookbackMs`. + * + * Dedupe windows don't respect the rotation boundary: a 6h click window that + * starts at 23:00 has to match rows written under yesterday's salt. Querying + * only today's hash would make every dedupe check silently miss for the first + * hours of each day — precisely when a duplicate click is most likely to be + * someone probing the boundary. Newest first. + */ +export function rotatingIpHashCandidates( + ip: string | null | undefined, + lookbackMs: number, + at: Date = new Date(), +): string[] { + if (!ip) return []; + const today = dayIndex(at); + const earliest = dayIndex(new Date(at.getTime() - Math.max(0, lookbackMs))); + const out: string[] = []; + for (let day = today; day >= earliest; day--) out.push(rotatingFor(ip, day)); + return out; +} diff --git a/lib/rateLimit.ts b/lib/rateLimit.ts index a873c338a4c84f96f5f2d9ab7badb72c50bfe63c..aeb4b6b3f5db2f15fb93969b207c9e00a94f8be4 100644 GIT binary patch delta 506 zcmZXRv2GMG5Qc>!nzSe-(ywcfqTC&zNKv4HE>WmLOOe_2Y;2v^v)HqDIU#fv1rI`z z66Ha71s;X*-gUUJWozd9|M~sn;^*b>+2hfRFXL-l^Lo7L>VDBZ{W*I1Y`0wEmgoF1 zNLF^}vh;|dS%%??ia*1~ht?w$B-ubcql9!H8KEB{_nee8NJ1wWji^61o_Z_q#f?@* z$$=btR?gPAzzq)yDj+24+3m2GhUaW-nI!lj6t(IKyE7rPhHvoZ=X${Ry~o*yWwn_} z`BUhF&hF4sjAl={z)5;#GIsa0W~i8-H7_dg);erLYcU1v60<9l>S(89G1~^Dm+SkzlAe~zGFw3`*F*(ZH#qlAp|MFNIzHIjX>DTkWC;tF88nD0s delta 244 zcmX|*u}T9$6h)PY#rBqx$6zrFx`c>|8zim%MV6Vj*}=_w!<(5T5<@BrKP8=z){pU1 z98|0>oO|Hj_xray { + it("mints the visitor id itself instead of only reading one stats.js wrote", async () => { + const script = await (await adJs()).text(); + + // The regression this guards: ad.js used to do a bare localStorage.getItem + // and give up. A publisher running the ad tag without the analytics tag + // therefore sent an empty visitor on every impression — in production that + // was ~69% of impressions with no visitor id at all. + expect(script).toContain("function getVisitorId()"); + expect(script).toContain("var v = getVisitorId();"); + expect(script).toContain("lsSet(k, id)"); + expect(script).toContain("&v="); + }); + + it("persists in localStorage, not a cookie or sessionStorage", async () => { + const script = await (await adJs()).text(); + expect(script).toContain("localStorage"); + expect(script).toContain("crawlproof.visitor"); + expect(script).not.toContain("sessionStorage"); + expect(script).not.toContain("document.cookie"); + }); + + it("still sends no credentials with the fill request", async () => { + const script = await (await adJs()).text(); + expect(script).toContain("credentials: 'omit'"); + }); +}); + +describe("shared visitor snippet", () => { + it("is inlined verbatim by both /ad.js and /stats.js", async () => { + const ad = await (await adJs()).text(); + const stats = await (await statsJs()).text(); + + // Both tags must agree on the id or an impression can't be tied back to the + // same person the stats dashboard counted. + expect(ad).toContain(VISITOR_SNIPPET.trim()); + expect(stats).toContain(VISITOR_SNIPPET.trim()); + }); + + it("keeps stats.js session handling working alongside the shared helpers", async () => { + const stats = await (await statsJs()).text(); + // getSessionId builds on lsGet/lsSet/uuid from the shared snippet; make sure + // extracting them didn't strand the session code. + expect(stats).toContain("function getSessionId()"); + expect(stats).toContain("crawlproof.session"); + expect(stats).toContain("var memSession = null, memSessionTs = 0;"); + // memVisitor now lives in the shared snippet — exactly one declaration. + expect(stats.match(/var memVisitor = null/g) ?? []).toHaveLength(1); + }); +}); diff --git a/tests/contract/ip-hash.test.ts b/tests/contract/ip-hash.test.ts new file mode 100644 index 00000000..8c12b6eb --- /dev/null +++ b/tests/contract/ip-hash.test.ts @@ -0,0 +1,117 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +// The salt is read through lib/env at call time, so each test sets the env var +// and re-imports the module with a fresh registry. +async function loadIpHash(salt: string | undefined) { + vi.resetModules(); + if (salt === undefined) delete process.env.IP_HASH_SALT; + else process.env.IP_HASH_SALT = salt; + return import("@/lib/ipHash"); +} + +const ORIGINAL = process.env.IP_HASH_SALT; + +afterEach(() => { + if (ORIGINAL === undefined) delete process.env.IP_HASH_SALT; + else process.env.IP_HASH_SALT = ORIGINAL; + vi.resetModules(); +}); + +describe("hashIp (stable)", () => { + it("changes the digest when the salt changes", async () => { + const a = (await loadIpHash("salt-one")).hashIp("203.0.113.7"); + const b = (await loadIpHash("salt-two")).hashIp("203.0.113.7"); + expect(a).not.toEqual(b); + }); + + it("is stable across calls and across days for a given salt", async () => { + const { hashIp } = await loadIpHash("salt-one"); + // Abuse caps look back 24h+; the stable variant must not drift with time, + // or every anonymous quota would refill at the rotation boundary. + expect(hashIp("203.0.113.7")).toEqual(hashIp("203.0.113.7")); + }); + + it("buckets a missing IP rather than returning null", async () => { + const { hashIp } = await loadIpHash("salt-one"); + // Conservative for rate limiting: unattributable requests share a bucket. + expect(hashIp(null)).toEqual(hashIp(undefined)); + expect(typeof hashIp(null)).toBe("string"); + }); + + it("reproduces the legacy unsalted digest when no salt is configured", async () => { + // An env that hasn't set IP_HASH_SALT yet must not silently invalidate every + // stored hash and hand each rate-limited visitor a fresh quota. + const crypto = await import("node:crypto"); + const legacy = crypto + .createHash("sha256") + .update("crawlproof:203.0.113.7") + .digest("hex") + .slice(0, 32); + const { hashIp } = await loadIpHash(undefined); + expect(hashIp("203.0.113.7")).toEqual(legacy); + }); +}); + +describe("hashIpRotating", () => { + it("returns null for a missing IP instead of a shared bucket", async () => { + const { hashIpRotating } = await loadIpHash("salt-one"); + // Load-bearing: one shared hash for every unattributable ad request would + // make them all look like duplicates and invalidate legitimate clicks. + expect(hashIpRotating(null)).toBeNull(); + expect(hashIpRotating("")).toBeNull(); + }); + + it("gives the same IP a different hash on a different day", async () => { + const { hashIpRotating } = await loadIpHash("salt-one"); + const day1 = hashIpRotating("203.0.113.7", new Date("2026-08-10T12:00:00Z")); + const day2 = hashIpRotating("203.0.113.7", new Date("2026-08-11T12:00:00Z")); + expect(day1).not.toEqual(day2); + }); + + it("is stable within the same day", async () => { + const { hashIpRotating } = await loadIpHash("salt-one"); + const morning = hashIpRotating("203.0.113.7", new Date("2026-08-10T01:00:00Z")); + const evening = hashIpRotating("203.0.113.7", new Date("2026-08-10T23:00:00Z")); + expect(morning).toEqual(evening); + }); + + it("differs from the stable hash, so the two can't be cross-correlated", async () => { + const { hashIp, hashIpRotating } = await loadIpHash("salt-one"); + expect(hashIpRotating("203.0.113.7")).not.toEqual(hashIp("203.0.113.7")); + }); +}); + +describe("rotatingIpHashCandidates", () => { + const SIX_HOURS = 6 * 60 * 60 * 1000; + + it("returns only today's hash when the window doesn't cross a rotation", async () => { + const { rotatingIpHashCandidates } = await loadIpHash("salt-one"); + const at = new Date("2026-08-10T12:00:00Z"); + expect(rotatingIpHashCandidates("203.0.113.7", SIX_HOURS, at)).toHaveLength(1); + }); + + it("includes yesterday's hash when the window crosses the boundary", async () => { + const { hashIpRotating, rotatingIpHashCandidates } = await loadIpHash("salt-one"); + // 02:00 looking back 6h reaches into the previous salt window. Without this + // the dedupe check silently misses for the first hours of every day. + const at = new Date("2026-08-10T02:00:00Z"); + const candidates = rotatingIpHashCandidates("203.0.113.7", SIX_HOURS, at); + expect(candidates).toHaveLength(2); + expect(candidates[0]).toEqual(hashIpRotating("203.0.113.7", at)); + expect(candidates).toContain( + hashIpRotating("203.0.113.7", new Date("2026-08-09T22:00:00Z")), + ); + }); + + it("returns nothing for a missing IP", async () => { + const { rotatingIpHashCandidates } = await loadIpHash("salt-one"); + expect(rotatingIpHashCandidates(null, SIX_HOURS)).toEqual([]); + }); + + it("yields hex-only values safe to interpolate into a PostgREST filter", async () => { + const { rotatingIpHashCandidates } = await loadIpHash("salt-one"); + for (const h of rotatingIpHashCandidates("203.0.113.7", SIX_HOURS)) { + expect(h).toMatch(/^[0-9a-f]{32}$/); + } + }); +}); diff --git a/tests/contract/url-control-chars.test.ts b/tests/contract/url-control-chars.test.ts new file mode 100644 index 00000000..f5299efc --- /dev/null +++ b/tests/contract/url-control-chars.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { isAllowedTargetUrl } from "@/lib/rateLimit"; + +// CONTROL_OR_WS used to be written with raw control bytes embedded directly in +// the regex literal, which made the whole file read as binary to grep and other +// tooling (searches for anything in lib/rateLimit.ts silently returned nothing). +// It is now spelled with \u escapes. These tests pin the behaviour so the +// rewrite is provably equivalent rather than merely plausible. +describe("isAllowedTargetUrl control-character handling", () => { + // Built with String.fromCharCode so this source file stays plain ASCII — + // embedding the raw bytes is what made lib/rateLimit.ts unsearchable. + const CONTROL_CASES: Array<[string, string]> = [ + ["NUL", String.fromCharCode(0)], + ["TAB", String.fromCharCode(9)], + ["LF", String.fromCharCode(10)], + ["CR", String.fromCharCode(13)], + ["unit separator", String.fromCharCode(31)], + ["DEL", String.fromCharCode(127)], + ["space", " "], + ]; + + for (const [name, ch] of CONTROL_CASES) { + it(`rejects a URL containing ${name}`, () => { + const result = isAllowedTargetUrl(`https://example.com/a${ch}b`); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.reason).toMatch(/control characters or whitespace/); + } + }); + } + + it("still accepts an ordinary URL", () => { + const result = isAllowedTargetUrl("https://example.com/path?a=1"); + expect(result.ok).toBe(true); + }); + + it("still accepts URL punctuation adjacent to the rejected range", () => { + // '!' (0x21) sits just above the control range; nothing in normal URL + // syntax should have been caught by the widened escape form. + const result = isAllowedTargetUrl("https://example.com/a!b~c$d"); + expect(result.ok).toBe(true); + }); + + it("trims surrounding whitespace rather than rejecting it", () => { + const result = isAllowedTargetUrl(" https://example.com/ok "); + expect(result.ok).toBe(true); + }); +}); From 33ec0c9afac5de61ad15d2fbd67e4f5aea67ecf8 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 10 Aug 2026 10:27:39 +0000 Subject: [PATCH 2/2] test(ads): assert /ad.js and /stats.js output actually parses Both tags are string-templated into third-party pages, so a stray brace or a bad interpolation would ship a script that throws on every publisher site. Containment assertions cannot catch that; parsing can. Co-Authored-By: Claude Opus 5 (1M context) --- tests/contract/ad-visitor-id.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/contract/ad-visitor-id.test.ts b/tests/contract/ad-visitor-id.test.ts index 90956551..93bcba91 100644 --- a/tests/contract/ad-visitor-id.test.ts +++ b/tests/contract/ad-visitor-id.test.ts @@ -42,6 +42,16 @@ describe("shared visitor snippet", () => { expect(stats).toContain(VISITOR_SNIPPET.trim()); }); + it("produces syntactically valid JavaScript in both tags", async () => { + // These are string-templated into third-party pages, so a stray brace or a + // bad interpolation ships a script that throws on every publisher site. + // Containment assertions can't catch that; parsing can. + const ad = await (await adJs()).text(); + const stats = await (await statsJs()).text(); + expect(() => new Function(ad)).not.toThrow(); + expect(() => new Function(stats)).not.toThrow(); + }); + it("keeps stats.js session handling working alongside the shared helpers", async () => { const stats = await (await statsJs()).text(); // getSessionId builds on lsGet/lsSet/uuid from the shared snippet; make sure