diff --git a/.env.example b/.env.example index c89143a5..e124c00a 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,11 @@ MAXMIND_GEOLITE2_CITY_DB_PATH=data/GeoLite2-City.mmdb # /api/ads/motd). Without it, slotless requests can only return the house ad. ADS_DEFAULT_SLOT_ID= +# Secret salt for every client-IP hash (ad metering + abuse caps). Unset falls +# back to the legacy unsalted digest, which is brute-forceable across the whole +# IPv4 space. Generate with `openssl rand -base64 32`. +IP_HASH_SALT= + # CoinPay credit purchases COINPAY_MERCHANT_ID=merchant_xxx COINPAY_API_KEY=cp_xxx diff --git a/app/ad.js/route.ts b/app/ad.js/route.ts index 84829621..ea7dbfb2 100644 --- a/app/ad.js/route.ts +++ b/app/ad.js/route.ts @@ -6,6 +6,7 @@ // leaves the container empty and never breaks the host page. import { env } from "@/lib/env"; +import { VISITOR_SNIPPET } from "@/lib/tracker/visitorSnippet"; const FORMATS = { banner_300x250: [300, 250], @@ -18,13 +19,7 @@ const snippet = `(function(){ try { var ORIGIN = ${JSON.stringify(env.siteUrl)}; var SIZES = ${JSON.stringify(FORMATS)}; - function visitorId() { - try { - var v = localStorage.getItem('crawlproof.visitor'); - if (v) return v; - } catch (_) {} - return ''; - } +${VISITOR_SNIPPET} function pickFormat(el, w) { var f = el.getAttribute('data-format'); if (f && SIZES[f]) return f; @@ -41,7 +36,10 @@ const snippet = `(function(){ var format = pickFormat(el, w); var dims = SIZES[format] || SIZES.banner_300x250; var q = '?slot=' + encodeURIComponent(slot) + '&format=' + encodeURIComponent(format); - var v = visitorId(); + // Mints the id if this is the first CrawlProof script on the page. It + // used to only read one stats.js had already written, so an ad-tag-only + // publisher reported every impression as an anonymous visitor. + var v = getVisitorId(); if (v) q += '&v=' + encodeURIComponent(v); el.setAttribute('data-cp-filled', '1'); fetch(ORIGIN + '/api/ads/serve' + q, { mode: 'cors', credentials: 'omit', cache: 'no-store' }) diff --git a/app/api/ads/motd/route.ts b/app/api/ads/motd/route.ts index 44112369..bcb1e579 100644 --- a/app/api/ads/motd/route.ts +++ b/app/api/ads/motd/route.ts @@ -2,6 +2,14 @@ // // curl -s "https://crawlproof.com/api/ads/motd?slot=" // curl -s "https://crawlproof.com/api/ads/motd?slot=&cols=64&color=1" +// curl -s "https://crawlproof.com/api/ads/motd?slot=&v=" +// +// `v` is the visitor id. On the web, /ad.js mints and persists one in +// localStorage; a terminal has neither cookies nor localStorage, so the caller +// has to supply it or every fetch counts as a new person — which is exactly why +// a scheduled curl loop shows up as a spike of unique visitors. Publishers +// should generate one opaque random id per machine at install time and pass it +// on every request. See the snippet in the slot manager. // // Returns an ASCII box (text/plain), sized to `cols`, with optional ANSI // colour. Meant for shell MOTDs, SSH login banners, BBS screens, and CLI tools diff --git a/app/stats.js/route.ts b/app/stats.js/route.ts index 8e9b08d4..70b95d01 100644 --- a/app/stats.js/route.ts +++ b/app/stats.js/route.ts @@ -5,6 +5,7 @@ // must never break the host page. import { env } from "@/lib/env"; +import { VISITOR_SNIPPET } from "@/lib/tracker/visitorSnippet"; const snippet = `(function(){ try { @@ -51,31 +52,10 @@ const snippet = `(function(){ })(); function pageUrl() { return location.origin + location.pathname + location.search; } - function uuid(prefix) { - try { - if (crypto && crypto.randomUUID) return prefix + crypto.randomUUID(); - } catch (_) {} - return prefix + Math.random().toString(16).slice(2) + '-' + Date.now().toString(16); - } - function lsGet(k) { try { return localStorage.getItem(k); } catch (_) { return null; } } - function lsSet(k, v) { try { localStorage.setItem(k, v); } catch (_) {} } - // In-memory fallbacks for when localStorage is blocked (private mode, etc.) - // so a single page load still reports one stable visitor/session. - var memVisitor = null, memSession = null, memSessionTs = 0; - // Persistent visitor id — stored in localStorage so it survives tab close, - // reload, and browser restart. localStorage is partitioned per site origin, - // making this a stable per-site visitor identifier (not per-tab, which would - // count every new tab as a new visitor). - function getVisitorId() { - var k = 'crawlproof.visitor'; - var id = lsGet(k); - if (id) return id; - if (memVisitor) return memVisitor; - id = uuid('v'); - lsSet(k, id); - memVisitor = id; - return id; - } +${VISITOR_SNIPPET} + // Session-only companion state. The visitor equivalents live in the shared + // snippet above, which /ad.js also inlines so both agree on the same id. + var memSession = null, memSessionTs = 0; // Session id with a 30-minute inactivity window, shared across tabs. The // window slides on every event, so an active visit stays one session and // reopening within 30 min reuses it instead of minting a new one. diff --git a/components/ads/slot-manager.tsx b/components/ads/slot-manager.tsx index 564114cd..f7925600 100644 --- a/components/ads/slot-manager.tsx +++ b/components/ads/slot-manager.tsx @@ -33,6 +33,17 @@ function embedFor(slotId: string, format: AdFormatId, origin: string): string { "# Options: &color=1 for ANSI colour, &cols=44..120 for width,", "# &src= to tell surfaces apart (rides through to the click URL).", "", + "# Repeat visitors: &v=. A terminal has no cookies and no localStorage,", + "# so unlike the web tag we can't mint this for you — without it every", + "# fetch looks like a brand new person. Generate one stable random id per", + "# machine at install time and pass it every time:", + "# id=$(cat /etc/crawlproof-visitor 2>/dev/null) || {", + "# id=$(head -c16 /dev/urandom | od -An -tx1 | tr -d ' \\n')", + "# printf '%s' \"$id\" >/etc/crawlproof-visitor", + "# }", + `# curl -fsS "${origin}/api/ads/motd?slot=${slotId}&cols=72&v=$id"`, + "# Use an opaque random value — never a hostname, username, or IP.", + "", "# Rendering a template server-side? Leave a token where the ad goes —", "# {{ads}} {{ads:64}} {{ads:terminal:64}}", "# — and swap it for the fetched text before you send the response.", diff --git a/lib/ads/fraud.ts b/lib/ads/fraud.ts index 4ce9608f..3b0e1cd4 100644 --- a/lib/ads/fraud.ts +++ b/lib/ads/fraud.ts @@ -4,8 +4,10 @@ import { serviceClient } from "@/lib/supabase/service"; // accrues to the publisher. Cheap, best-effort, and conservative: when in // doubt we still redirect the user, we just don't charge for the click. -// A visitor is counted at most once per campaign within this window. -const DEDUPE_WINDOW_MS = 6 * 60 * 60 * 1000; // 6h +// A visitor is counted at most once per campaign within this window. Exported +// so callers can ask lib/ipHash for every rotating hash an IP could have been +// stored under across the same span. +export const CLICK_DEDUPE_WINDOW_MS = 6 * 60 * 60 * 1000; // 6h export function isBotDevice(device?: string | null): boolean { return device === "bot"; @@ -26,7 +28,12 @@ export async function assessClickValidity(input: { slotId?: string | null; impressionId?: string | null; visitorId?: string | null; - ipHash?: string | null; + /** + * Every rotating IP hash to match against — today's plus any earlier salt + * window still inside CLICK_DEDUPE_WINDOW_MS. A single hash would stop + * matching yesterday's rows the moment the salt rotates. + */ + ipHashes?: string[] | null; device?: string | null; }): Promise { // 1. Bots never bill. @@ -49,11 +56,13 @@ export async function assessClickValidity(input: { // 3. Dedupe on this campaign by visitor id or ip hash within the window. const visitor = safeId(input.visitorId); - const ipHash = safeId(input.ipHash); - if (!visitor && !ipHash) return { valid: true }; // nothing to dedupe on + const ipHashes = (input.ipHashes ?? []) + .map((h) => safeId(h)) + .filter((h): h is string => h !== null); + if (!visitor && ipHashes.length === 0) return { valid: true }; // nothing to dedupe on - const since = new Date(Date.now() - DEDUPE_WINDOW_MS).toISOString(); - let q = sb + const since = new Date(Date.now() - CLICK_DEDUPE_WINDOW_MS).toISOString(); + const q = sb .from("ad_clicks") .select("id") .eq("campaign_id", input.campaignId) @@ -61,11 +70,15 @@ export async function assessClickValidity(input: { .gte("ts", since) .limit(1); - if (visitor && ipHash) q = q.or(`visitor_id.eq.${visitor},ip_hash.eq.${ipHash}`); - else if (visitor) q = q.eq("visitor_id", visitor); - else q = q.eq("ip_hash", ipHash!); + // One .or() covering every identifier: the visitor id plus each salt window's + // hash. Every term has been through safeId, so nothing unescaped reaches + // PostgREST's filter syntax. + const terms = [ + ...(visitor ? [`visitor_id.eq.${visitor}`] : []), + ...ipHashes.map((h) => `ip_hash.eq.${h}`), + ]; - const { data: dupe } = await q; + const { data: dupe } = await q.or(terms.join(",")); if (dupe && dupe.length > 0) return { valid: false, reason: "duplicate" }; return { valid: true }; diff --git a/lib/ads/serve.ts b/lib/ads/serve.ts index fa512c8c..e7aa62b8 100644 --- a/lib/ads/serve.ts +++ b/lib/ads/serve.ts @@ -12,7 +12,8 @@ import { import { TERMINAL_FORMAT_ID } from "./formats"; import { houseFill, HOUSE_AD_ROTATION_RATE } from "./house"; import { CREDIT_CENTS, DEFAULT_BID_CREDITS, PLATFORM_RATE } from "./pricing"; -import { assessClickValidity, isBotDevice } from "./fraud"; +import { assessClickValidity, isBotDevice, CLICK_DEDUPE_WINDOW_MS } from "./fraud"; +import { hashIpRotating, rotatingIpHashCandidates } from "@/lib/ipHash"; import { runAuction } from "./auction"; import { generateShortCode } from "./shortcode"; @@ -50,10 +51,9 @@ export function isAdFormat(v: string | null | undefined): v is AdFormatId { return !!v && (AD_FORMAT_IDS as string[]).includes(v); } -export function hashIp(ip: string | null): string | null { - if (!ip) return null; - return crypto.createHash("sha256").update(ip).digest("hex").slice(0, 32); -} +// hashIp used to live here as a bare sha256(ip). It now comes from lib/ipHash, +// salted and rotating daily — see that module for why the ad path wants the +// rotating variant and the abuse caps want the stable one. type CreativeRow = { id: string; @@ -240,7 +240,7 @@ export async function serveAd( campaign_id: campaign.id, creative_id: pick.id, visitor_id: ctx.visitorId ?? null, - ip_hash: hashIp(ctx.ip ?? null), + ip_hash: hashIpRotating(ctx.ip ?? null), geo_country: ctx.country ?? null, device: ctx.device ?? null, billable: false, @@ -316,13 +316,16 @@ export async function resolveClick(input: { // slot_id must be present (ad_clicks.slot_id NOT NULL) to record a click. if (input.slotId) { - const ipHash = hashIp(input.ctx?.ip ?? null); + // The row is stored under today's salt; the dedupe lookup has to consider + // yesterday's too, or every check silently misses for the first hours after + // the salt rotates. + const ipHash = hashIpRotating(input.ctx?.ip ?? null); const validity = await assessClickValidity({ campaignId: campaign.id, slotId: input.slotId, impressionId: input.impressionId, visitorId: input.ctx?.visitorId, - ipHash, + ipHashes: rotatingIpHashCandidates(input.ctx?.ip ?? null, CLICK_DEDUPE_WINDOW_MS), device: input.ctx?.device, }); diff --git a/lib/env.ts b/lib/env.ts index 1a859417..a587a426 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -21,6 +21,12 @@ export const env = { // network's own publisher slot: anonymous terminal traffic is CrawlProof's // own inventory, so its impressions/clicks accrue there. adsDefaultSlotId: process.env.ADS_DEFAULT_SLOT_ID ?? "", + // Server-side secret salting every client-IP hash (ad metering + abuse caps). + // Unset falls back to the legacy unsalted digest, which is brute-forceable + // across the whole IPv4 space in minutes — set it in production. Generate + // with `openssl rand -base64 32`. Changing it resets abuse counters once and + // breaks in-flight ad click dedupe for up to 6h; both self-heal. + ipHashSalt: process.env.IP_HASH_SALT ?? "", // CoinPay — crypto credit purchases. coinpayMerchantId: process.env.COINPAY_MERCHANT_ID ?? "", coinpayApiKey: process.env.COINPAY_API_KEY ?? "", diff --git a/lib/ipHash.ts b/lib/ipHash.ts new file mode 100644 index 00000000..954aceae --- /dev/null +++ b/lib/ipHash.ts @@ -0,0 +1,127 @@ +import crypto from "node:crypto"; +import { env } from "./env"; + +// Single source of truth for turning a client IP into a storable identifier. +// +// This replaces two divergent implementations that had drifted apart: +// * lib/rateLimit.ts — sha256("crawlproof:" + ip), a hardcoded constant +// prefix, which is a pepper in name only since it lives in the source. +// * lib/ads/serve.ts — sha256(ip), no prefix at all. +// +// Neither was anonymisation. IPv4 is 2^32 addresses, so the entire space can be +// enumerated against a truncated SHA-256 in minutes on commodity hardware; an +// unsalted digest of an IP is a reversible encoding of that IP, not a +// pseudonym. A server-side secret salt is the thing that makes it one-way in +// practice, which matters here because ip_hash is the only identifier the ad +// network has for terminal traffic (curl has no localStorage) and for the ~69% +// of web impressions that historically arrived with no visitor id. +// +// Two exported variants over one implementation, because the callers need +// opposite properties from the same primitive: +// +// hashIp() Stable over time. Abuse caps look back 24h and longer +// (checkAnonymousLimit), so a rotating salt would silently +// refill every anonymous quota at the rotation boundary — +// a quota bypass, not a privacy win. +// hashIpRotating() Salt changes daily. Ad metering only needs to recognise +// an IP within hours (6h click dedupe, frequency capping). +// Past that window, being *able* to re-identify a visitor +// is a liability rather than a feature, so the capability +// is designed to expire on its own. +// +// Rotation makes long-term correlation impossible even against a full database +// leak, while leaving the short-window behaviour the ad network actually uses +// intact. + +const LEGACY_PREFIX = "crawlproof:"; +const DAY_MS = 86_400_000; + +function sha(input: string): string { + return crypto.createHash("sha256").update(input).digest("hex").slice(0, 32); +} + +let warned = false; +function salt(): string { + const s = env.ipHashSalt; + if (s) return s; + // Deliberately does not throw. An unsalted hash is a weakness; a hard failure + // on every request that touches an IP is an outage of both ad serving and + // rate limiting. Warn once — enough to be visible in logs, quiet enough not + // to flood them at request volume. + if (!warned) { + warned = true; + console.warn( + "[ipHash] IP_HASH_SALT is unset — falling back to the legacy unsalted digest. Set it in production.", + ); + } + return ""; +} + +// NUL-joined so a salt ending in digits can't collide with an IP starting with +// them; no component of the input can bleed into the next. +function join(parts: string[]): string { + return parts.join("\u0000"); +} + +/** + * Stable, salted hash of a client IP. Use for abuse caps and anything that + * looks back more than a day. + * + * A missing IP hashes to a shared "unknown" bucket on purpose: for rate + * limiting, lumping unattributable requests together is the conservative + * choice. + */ +export function hashIp(ip: string | null | undefined): string { + const v = ip ?? "unknown"; + const s = salt(); + // With no salt configured this reproduces the historical digest byte for + // byte, so an environment that hasn't set IP_HASH_SALT yet doesn't invalidate + // every stored hash and hand each rate-limited visitor a fresh quota. + return s ? sha(join([s, LEGACY_PREFIX + v])) : sha(LEGACY_PREFIX + v); +} + +function dayIndex(at: Date): number { + return Math.floor(at.getTime() / DAY_MS); +} + +function rotatingFor(ip: string, day: number): string { + return sha(join([salt(), `day:${day}`, ip])); +} + +/** + * Daily-rotating salted hash of a client IP, for ad metering. + * + * Returns null for a missing IP rather than bucketing to a shared constant — + * the opposite of hashIp, and load-bearing: every unattributable ad request + * sharing one hash would make them all look like duplicates of each other and + * invalidate legitimate clicks wholesale. + */ +export function hashIpRotating( + ip: string | null | undefined, + at: Date = new Date(), +): string | null { + if (!ip) return null; + return rotatingFor(ip, dayIndex(at)); +} + +/** + * Every rotating hash an IP could have been stored under across `lookbackMs`. + * + * Dedupe windows don't respect the rotation boundary: a 6h click window that + * starts at 23:00 has to match rows written under yesterday's salt. Querying + * only today's hash would make every dedupe check silently miss for the first + * hours of each day — precisely when a duplicate click is most likely to be + * someone probing the boundary. Newest first. + */ +export function rotatingIpHashCandidates( + ip: string | null | undefined, + lookbackMs: number, + at: Date = new Date(), +): string[] { + if (!ip) return []; + const today = dayIndex(at); + const earliest = dayIndex(new Date(at.getTime() - Math.max(0, lookbackMs))); + const out: string[] = []; + for (let day = today; day >= earliest; day--) out.push(rotatingFor(ip, day)); + return out; +} diff --git a/lib/rateLimit.ts b/lib/rateLimit.ts index a873c338..aeb4b6b3 100644 Binary files a/lib/rateLimit.ts and b/lib/rateLimit.ts differ diff --git a/lib/tracker/visitorSnippet.ts b/lib/tracker/visitorSnippet.ts new file mode 100644 index 00000000..ec843de7 --- /dev/null +++ b/lib/tracker/visitorSnippet.ts @@ -0,0 +1,53 @@ +// Browser-side visitor identity, shared verbatim by /stats.js and /ad.js. +// +// One definition, two consumers. These two scripts have to agree on the +// visitor id or the ad network can't tie an impression back to the same person +// the stats dashboard counted — and they used to disagree in the worst +// possible way: stats.js minted the id, ad.js only ever *read* it. A publisher +// running the ad tag without the analytics tag therefore sent an empty visitor +// on every single impression. In production that was ~69% of ad impressions +// carrying no visitor at all, which in turn left ip_hash as the only signal +// available for click dedupe and frequency capping. +// +// Emitted as a string rather than a real module because both routes hand-serve +// dependency-free ES5 to arbitrary third-party pages. There is no bundler in +// that path, so sharing has to happen at the source level. +// +// Declares into the caller's scope: uuid(), lsGet(), lsSet(), getVisitorId(). +// Callers that need their own storage helpers (stats.js builds a session id on +// top of these) can rely on all four being present. +// +// Scope note: this runs inside the *publisher's* page, so localStorage is +// partitioned to the publisher's origin — not CrawlProof's. The id is a stable +// per-site visitor identifier and deliberately cannot follow anyone across +// publishers; that would need third-party storage, which browsers no longer +// grant. Frequency capping and returning-visitor counts work per site. Cross- +// publisher reach does not, by design of the platform rather than of this code. +export const VISITOR_SNIPPET = ` + function uuid(prefix) { + try { + if (crypto && crypto.randomUUID) return prefix + crypto.randomUUID(); + } catch (_) {} + return prefix + Math.random().toString(16).slice(2) + '-' + Date.now().toString(16); + } + function lsGet(k) { try { return localStorage.getItem(k); } catch (_) { return null; } } + function lsSet(k, v) { try { localStorage.setItem(k, v); } catch (_) {} } + // In-memory fallback for when localStorage is blocked (private mode, ITP, + // an embedded webview) so a single page load still reports one stable + // visitor instead of a fresh id per call. + var memVisitor = null; + // Persistent visitor id — localStorage so it survives tab close, reload, + // and browser restart. Deliberately not the per-tab storage API, which + // counted every new tab as a new visitor, and deliberately not a cookie: + // nothing here needs to ride on every HTTP request, and a cookie would be + // sent to the publisher's own backend on every request too. + function getVisitorId() { + var k = 'crawlproof.visitor'; + var id = lsGet(k); + if (id) return id; + if (memVisitor) return memVisitor; + id = uuid('v'); + lsSet(k, id); + memVisitor = id; + return id; + }`; diff --git a/tests/contract/ad-visitor-id.test.ts b/tests/contract/ad-visitor-id.test.ts new file mode 100644 index 00000000..93bcba91 --- /dev/null +++ b/tests/contract/ad-visitor-id.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; +import { GET as adJs } from "@/app/ad.js/route"; +import { GET as statsJs } from "@/app/stats.js/route"; +import { VISITOR_SNIPPET } from "@/lib/tracker/visitorSnippet"; + +describe("/ad.js visitor identity", () => { + it("mints the visitor id itself instead of only reading one stats.js wrote", async () => { + const script = await (await adJs()).text(); + + // The regression this guards: ad.js used to do a bare localStorage.getItem + // and give up. A publisher running the ad tag without the analytics tag + // therefore sent an empty visitor on every impression — in production that + // was ~69% of impressions with no visitor id at all. + expect(script).toContain("function getVisitorId()"); + expect(script).toContain("var v = getVisitorId();"); + expect(script).toContain("lsSet(k, id)"); + expect(script).toContain("&v="); + }); + + it("persists in localStorage, not a cookie or sessionStorage", async () => { + const script = await (await adJs()).text(); + expect(script).toContain("localStorage"); + expect(script).toContain("crawlproof.visitor"); + expect(script).not.toContain("sessionStorage"); + expect(script).not.toContain("document.cookie"); + }); + + it("still sends no credentials with the fill request", async () => { + const script = await (await adJs()).text(); + expect(script).toContain("credentials: 'omit'"); + }); +}); + +describe("shared visitor snippet", () => { + it("is inlined verbatim by both /ad.js and /stats.js", async () => { + const ad = await (await adJs()).text(); + const stats = await (await statsJs()).text(); + + // Both tags must agree on the id or an impression can't be tied back to the + // same person the stats dashboard counted. + expect(ad).toContain(VISITOR_SNIPPET.trim()); + expect(stats).toContain(VISITOR_SNIPPET.trim()); + }); + + it("produces syntactically valid JavaScript in both tags", async () => { + // These are string-templated into third-party pages, so a stray brace or a + // bad interpolation ships a script that throws on every publisher site. + // Containment assertions can't catch that; parsing can. + const ad = await (await adJs()).text(); + const stats = await (await statsJs()).text(); + expect(() => new Function(ad)).not.toThrow(); + expect(() => new Function(stats)).not.toThrow(); + }); + + it("keeps stats.js session handling working alongside the shared helpers", async () => { + const stats = await (await statsJs()).text(); + // getSessionId builds on lsGet/lsSet/uuid from the shared snippet; make sure + // extracting them didn't strand the session code. + expect(stats).toContain("function getSessionId()"); + expect(stats).toContain("crawlproof.session"); + expect(stats).toContain("var memSession = null, memSessionTs = 0;"); + // memVisitor now lives in the shared snippet — exactly one declaration. + expect(stats.match(/var memVisitor = null/g) ?? []).toHaveLength(1); + }); +}); diff --git a/tests/contract/ip-hash.test.ts b/tests/contract/ip-hash.test.ts new file mode 100644 index 00000000..8c12b6eb --- /dev/null +++ b/tests/contract/ip-hash.test.ts @@ -0,0 +1,117 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +// The salt is read through lib/env at call time, so each test sets the env var +// and re-imports the module with a fresh registry. +async function loadIpHash(salt: string | undefined) { + vi.resetModules(); + if (salt === undefined) delete process.env.IP_HASH_SALT; + else process.env.IP_HASH_SALT = salt; + return import("@/lib/ipHash"); +} + +const ORIGINAL = process.env.IP_HASH_SALT; + +afterEach(() => { + if (ORIGINAL === undefined) delete process.env.IP_HASH_SALT; + else process.env.IP_HASH_SALT = ORIGINAL; + vi.resetModules(); +}); + +describe("hashIp (stable)", () => { + it("changes the digest when the salt changes", async () => { + const a = (await loadIpHash("salt-one")).hashIp("203.0.113.7"); + const b = (await loadIpHash("salt-two")).hashIp("203.0.113.7"); + expect(a).not.toEqual(b); + }); + + it("is stable across calls and across days for a given salt", async () => { + const { hashIp } = await loadIpHash("salt-one"); + // Abuse caps look back 24h+; the stable variant must not drift with time, + // or every anonymous quota would refill at the rotation boundary. + expect(hashIp("203.0.113.7")).toEqual(hashIp("203.0.113.7")); + }); + + it("buckets a missing IP rather than returning null", async () => { + const { hashIp } = await loadIpHash("salt-one"); + // Conservative for rate limiting: unattributable requests share a bucket. + expect(hashIp(null)).toEqual(hashIp(undefined)); + expect(typeof hashIp(null)).toBe("string"); + }); + + it("reproduces the legacy unsalted digest when no salt is configured", async () => { + // An env that hasn't set IP_HASH_SALT yet must not silently invalidate every + // stored hash and hand each rate-limited visitor a fresh quota. + const crypto = await import("node:crypto"); + const legacy = crypto + .createHash("sha256") + .update("crawlproof:203.0.113.7") + .digest("hex") + .slice(0, 32); + const { hashIp } = await loadIpHash(undefined); + expect(hashIp("203.0.113.7")).toEqual(legacy); + }); +}); + +describe("hashIpRotating", () => { + it("returns null for a missing IP instead of a shared bucket", async () => { + const { hashIpRotating } = await loadIpHash("salt-one"); + // Load-bearing: one shared hash for every unattributable ad request would + // make them all look like duplicates and invalidate legitimate clicks. + expect(hashIpRotating(null)).toBeNull(); + expect(hashIpRotating("")).toBeNull(); + }); + + it("gives the same IP a different hash on a different day", async () => { + const { hashIpRotating } = await loadIpHash("salt-one"); + const day1 = hashIpRotating("203.0.113.7", new Date("2026-08-10T12:00:00Z")); + const day2 = hashIpRotating("203.0.113.7", new Date("2026-08-11T12:00:00Z")); + expect(day1).not.toEqual(day2); + }); + + it("is stable within the same day", async () => { + const { hashIpRotating } = await loadIpHash("salt-one"); + const morning = hashIpRotating("203.0.113.7", new Date("2026-08-10T01:00:00Z")); + const evening = hashIpRotating("203.0.113.7", new Date("2026-08-10T23:00:00Z")); + expect(morning).toEqual(evening); + }); + + it("differs from the stable hash, so the two can't be cross-correlated", async () => { + const { hashIp, hashIpRotating } = await loadIpHash("salt-one"); + expect(hashIpRotating("203.0.113.7")).not.toEqual(hashIp("203.0.113.7")); + }); +}); + +describe("rotatingIpHashCandidates", () => { + const SIX_HOURS = 6 * 60 * 60 * 1000; + + it("returns only today's hash when the window doesn't cross a rotation", async () => { + const { rotatingIpHashCandidates } = await loadIpHash("salt-one"); + const at = new Date("2026-08-10T12:00:00Z"); + expect(rotatingIpHashCandidates("203.0.113.7", SIX_HOURS, at)).toHaveLength(1); + }); + + it("includes yesterday's hash when the window crosses the boundary", async () => { + const { hashIpRotating, rotatingIpHashCandidates } = await loadIpHash("salt-one"); + // 02:00 looking back 6h reaches into the previous salt window. Without this + // the dedupe check silently misses for the first hours of every day. + const at = new Date("2026-08-10T02:00:00Z"); + const candidates = rotatingIpHashCandidates("203.0.113.7", SIX_HOURS, at); + expect(candidates).toHaveLength(2); + expect(candidates[0]).toEqual(hashIpRotating("203.0.113.7", at)); + expect(candidates).toContain( + hashIpRotating("203.0.113.7", new Date("2026-08-09T22:00:00Z")), + ); + }); + + it("returns nothing for a missing IP", async () => { + const { rotatingIpHashCandidates } = await loadIpHash("salt-one"); + expect(rotatingIpHashCandidates(null, SIX_HOURS)).toEqual([]); + }); + + it("yields hex-only values safe to interpolate into a PostgREST filter", async () => { + const { rotatingIpHashCandidates } = await loadIpHash("salt-one"); + for (const h of rotatingIpHashCandidates("203.0.113.7", SIX_HOURS)) { + expect(h).toMatch(/^[0-9a-f]{32}$/); + } + }); +}); diff --git a/tests/contract/url-control-chars.test.ts b/tests/contract/url-control-chars.test.ts new file mode 100644 index 00000000..f5299efc --- /dev/null +++ b/tests/contract/url-control-chars.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { isAllowedTargetUrl } from "@/lib/rateLimit"; + +// CONTROL_OR_WS used to be written with raw control bytes embedded directly in +// the regex literal, which made the whole file read as binary to grep and other +// tooling (searches for anything in lib/rateLimit.ts silently returned nothing). +// It is now spelled with \u escapes. These tests pin the behaviour so the +// rewrite is provably equivalent rather than merely plausible. +describe("isAllowedTargetUrl control-character handling", () => { + // Built with String.fromCharCode so this source file stays plain ASCII — + // embedding the raw bytes is what made lib/rateLimit.ts unsearchable. + const CONTROL_CASES: Array<[string, string]> = [ + ["NUL", String.fromCharCode(0)], + ["TAB", String.fromCharCode(9)], + ["LF", String.fromCharCode(10)], + ["CR", String.fromCharCode(13)], + ["unit separator", String.fromCharCode(31)], + ["DEL", String.fromCharCode(127)], + ["space", " "], + ]; + + for (const [name, ch] of CONTROL_CASES) { + it(`rejects a URL containing ${name}`, () => { + const result = isAllowedTargetUrl(`https://example.com/a${ch}b`); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.reason).toMatch(/control characters or whitespace/); + } + }); + } + + it("still accepts an ordinary URL", () => { + const result = isAllowedTargetUrl("https://example.com/path?a=1"); + expect(result.ok).toBe(true); + }); + + it("still accepts URL punctuation adjacent to the rejected range", () => { + // '!' (0x21) sits just above the control range; nothing in normal URL + // syntax should have been caught by the widened escape form. + const result = isAllowedTargetUrl("https://example.com/a!b~c$d"); + expect(result.ok).toBe(true); + }); + + it("trims surrounding whitespace rather than rejecting it", () => { + const result = isAllowedTargetUrl(" https://example.com/ok "); + expect(result.ok).toBe(true); + }); +});