feat(ads): share the visitor id with /ad.js, salt + rotate IP hashes

.env.example

@@ -17,6 +17,11 @@ MAXMIND_GEOLITE2_CITY_DB_PATH=data/GeoLite2-City.mmdb
 # /api/ads/motd). Without it, slotless requests can only return the house ad.
 ADS_DEFAULT_SLOT_ID=
 
+# Secret salt for every client-IP hash (ad metering + abuse caps). Unset falls
+# back to the legacy unsalted digest, which is brute-forceable across the whole
+# IPv4 space. Generate with `openssl rand -base64 32`.
+IP_HASH_SALT=
+
 # CoinPay credit purchases
 COINPAY_MERCHANT_ID=merchant_xxx
 COINPAY_API_KEY=cp_xxx

app/ad.js/route.ts

@@ -6,6 +6,7 @@
 // leaves the container empty and never breaks the host page.
 
 import { env } from "@/lib/env";
+import { VISITOR_SNIPPET } from "@/lib/tracker/visitorSnippet";
 
 const FORMATS = {
   banner_300x250: [300, 250],
@@ -18,13 +19,7 @@ const snippet = `(function(){
   try {
     var ORIGIN = ${JSON.stringify(env.siteUrl)};
     var SIZES = ${JSON.stringify(FORMATS)};
-    function visitorId() {
-      try {
-        var v = localStorage.getItem('crawlproof.visitor');
-        if (v) return v;
-      } catch (_) {}
-      return '';
-    }
+${VISITOR_SNIPPET}
     function pickFormat(el, w) {
       var f = el.getAttribute('data-format');
       if (f && SIZES[f]) return f;
@@ -41,7 +36,10 @@ const snippet = `(function(){
       var format = pickFormat(el, w);
       var dims = SIZES[format] || SIZES.banner_300x250;
       var q = '?slot=' + encodeURIComponent(slot) + '&format=' + encodeURIComponent(format);
-      var v = visitorId();
+      // Mints the id if this is the first CrawlProof script on the page. It
+      // used to only read one stats.js had already written, so an ad-tag-only
+      // publisher reported every impression as an anonymous visitor.
+      var v = getVisitorId();
       if (v) q += '&v=' + encodeURIComponent(v);
       el.setAttribute('data-cp-filled', '1');
       fetch(ORIGIN + '/api/ads/serve' + q, { mode: 'cors', credentials: 'omit', cache: 'no-store' })

app/api/ads/motd/route.ts

@@ -2,6 +2,14 @@
 //
 //   curl -s "https://crawlproof.com/api/ads/motd?slot=<slot_id>"
 //   curl -s "https://crawlproof.com/api/ads/motd?slot=<slot_id>&cols=64&color=1"
+//   curl -s "https://crawlproof.com/api/ads/motd?slot=<slot_id>&v=<visitor_id>"
+//
+// `v` is the visitor id. On the web, /ad.js mints and persists one in
+// localStorage; a terminal has neither cookies nor localStorage, so the caller
+// has to supply it or every fetch counts as a new person — which is exactly why
+// a scheduled curl loop shows up as a spike of unique visitors. Publishers
+// should generate one opaque random id per machine at install time and pass it
+// on every request. See the snippet in the slot manager.
 //
 // Returns an ASCII box (text/plain), sized to `cols`, with optional ANSI
 // colour. Meant for shell MOTDs, SSH login banners, BBS screens, and CLI tools

app/stats.js/route.ts

@@ -5,6 +5,7 @@
 // must never break the host page.
 
 import { env } from "@/lib/env";
+import { VISITOR_SNIPPET } from "@/lib/tracker/visitorSnippet";
 
 const snippet = `(function(){
   try {
@@ -51,31 +52,10 @@ const snippet = `(function(){
     })();
 
     function pageUrl() { return location.origin + location.pathname + location.search; }
-    function uuid(prefix) {
-      try {
-        if (crypto && crypto.randomUUID) return prefix + crypto.randomUUID();
-      } catch (_) {}
-      return prefix + Math.random().toString(16).slice(2) + '-' + Date.now().toString(16);
-    }
-    function lsGet(k) { try { return localStorage.getItem(k); } catch (_) { return null; } }
-    function lsSet(k, v) { try { localStorage.setItem(k, v); } catch (_) {} }
-    // In-memory fallbacks for when localStorage is blocked (private mode, etc.)
-    // so a single page load still reports one stable visitor/session.
-    var memVisitor = null, memSession = null, memSessionTs = 0;
-    // Persistent visitor id — stored in localStorage so it survives tab close,
-    // reload, and browser restart. localStorage is partitioned per site origin,
-    // making this a stable per-site visitor identifier (not per-tab, which would
-    // count every new tab as a new visitor).
-    function getVisitorId() {
-      var k = 'crawlproof.visitor';
-      var id = lsGet(k);
-      if (id) return id;
-      if (memVisitor) return memVisitor;
-      id = uuid('v');
-      lsSet(k, id);
-      memVisitor = id;
-      return id;
-    }
+${VISITOR_SNIPPET}
+    // Session-only companion state. The visitor equivalents live in the shared
+    // snippet above, which /ad.js also inlines so both agree on the same id.
+    var memSession = null, memSessionTs = 0;
     // Session id with a 30-minute inactivity window, shared across tabs. The
     // window slides on every event, so an active visit stays one session and
     // reopening within 30 min reuses it instead of minting a new one.

components/ads/slot-manager.tsx

@@ -33,6 +33,17 @@ function embedFor(slotId: string, format: AdFormatId, origin: string): string {
       "# Options: &color=1 for ANSI colour, &cols=44..120 for width,",
       "# &src=<tag> to tell surfaces apart (rides through to the click URL).",
       "",
+      "# Repeat visitors: &v=<id>. A terminal has no cookies and no localStorage,",
+      "# so unlike the web tag we can't mint this for you — without it every",
+      "# fetch looks like a brand new person. Generate one stable random id per",
+      "# machine at install time and pass it every time:",
+      "#   id=$(cat /etc/crawlproof-visitor 2>/dev/null) || {",
+      "#     id=$(head -c16 /dev/urandom | od -An -tx1 | tr -d ' \\n')",
+      "#     printf '%s' \"$id\" >/etc/crawlproof-visitor",
+      "#   }",
+      `#   curl -fsS "${origin}/api/ads/motd?slot=${slotId}&cols=72&v=$id"`,
+      "# Use an opaque random value — never a hostname, username, or IP.",
+      "",
       "# Rendering a template server-side? Leave a token where the ad goes —",
       "#   {{ads}}  {{ads:64}}  {{ads:terminal:64}}",
       "# — and swap it for the fetched text before you send the response.",

lib/ads/fraud.ts

@@ -4,8 +4,10 @@ import { serviceClient } from "@/lib/supabase/service";
 // accrues to the publisher. Cheap, best-effort, and conservative: when in
 // doubt we still redirect the user, we just don't charge for the click.
 
-// A visitor is counted at most once per campaign within this window.
-const DEDUPE_WINDOW_MS = 6 * 60 * 60 * 1000; // 6h
+// A visitor is counted at most once per campaign within this window. Exported
+// so callers can ask lib/ipHash for every rotating hash an IP could have been
+// stored under across the same span.
+export const CLICK_DEDUPE_WINDOW_MS = 6 * 60 * 60 * 1000; // 6h
 
 export function isBotDevice(device?: string | null): boolean {
   return device === "bot";
@@ -26,7 +28,12 @@ export async function assessClickValidity(input: {
   slotId?: string | null;
   impressionId?: string | null;
   visitorId?: string | null;
-  ipHash?: string | null;
+  /**
+   * Every rotating IP hash to match against — today's plus any earlier salt
+   * window still inside CLICK_DEDUPE_WINDOW_MS. A single hash would stop
+   * matching yesterday's rows the moment the salt rotates.
+   */
+  ipHashes?: string[] | null;
   device?: string | null;
 }): Promise<ClickValidity> {
   // 1. Bots never bill.
@@ -49,23 +56,29 @@ export async function assessClickValidity(input: {
 
   // 3. Dedupe on this campaign by visitor id or ip hash within the window.
   const visitor = safeId(input.visitorId);
-  const ipHash = safeId(input.ipHash);
-  if (!visitor && !ipHash) return { valid: true }; // nothing to dedupe on
+  const ipHashes = (input.ipHashes ?? [])
+    .map((h) => safeId(h))
+    .filter((h): h is string => h !== null);
+  if (!visitor && ipHashes.length === 0) return { valid: true }; // nothing to dedupe on
 
-  const since = new Date(Date.now() - DEDUPE_WINDOW_MS).toISOString();
-  let q = sb
+  const since = new Date(Date.now() - CLICK_DEDUPE_WINDOW_MS).toISOString();
+  const q = sb
     .from("ad_clicks")
     .select("id")
     .eq("campaign_id", input.campaignId)
     .eq("valid", true)
     .gte("ts", since)
     .limit(1);
 
-  if (visitor && ipHash) q = q.or(`visitor_id.eq.${visitor},ip_hash.eq.${ipHash}`);
-  else if (visitor) q = q.eq("visitor_id", visitor);
-  else q = q.eq("ip_hash", ipHash!);
+  // One .or() covering every identifier: the visitor id plus each salt window's
+  // hash. Every term has been through safeId, so nothing unescaped reaches
+  // PostgREST's filter syntax.
+  const terms = [
+    ...(visitor ? [`visitor_id.eq.${visitor}`] : []),
+    ...ipHashes.map((h) => `ip_hash.eq.${h}`),
+  ];
 
-  const { data: dupe } = await q;
+  const { data: dupe } = await q.or(terms.join(","));
   if (dupe && dupe.length > 0) return { valid: false, reason: "duplicate" };
 
   return { valid: true };

lib/ads/serve.ts

@@ -12,7 +12,8 @@ import {
 import { TERMINAL_FORMAT_ID } from "./formats";
 import { houseFill, HOUSE_AD_ROTATION_RATE } from "./house";
 import { CREDIT_CENTS, DEFAULT_BID_CREDITS, PLATFORM_RATE } from "./pricing";
-import { assessClickValidity, isBotDevice } from "./fraud";
+import { assessClickValidity, isBotDevice, CLICK_DEDUPE_WINDOW_MS } from "./fraud";
+import { hashIpRotating, rotatingIpHashCandidates } from "@/lib/ipHash";
 import { runAuction } from "./auction";
 import { generateShortCode } from "./shortcode";
 
@@ -50,10 +51,9 @@ export function isAdFormat(v: string | null | undefined): v is AdFormatId {
   return !!v && (AD_FORMAT_IDS as string[]).includes(v);
 }
 
-export function hashIp(ip: string | null): string | null {
-  if (!ip) return null;
-  return crypto.createHash("sha256").update(ip).digest("hex").slice(0, 32);
-}
+// hashIp used to live here as a bare sha256(ip). It now comes from lib/ipHash,
+// salted and rotating daily — see that module for why the ad path wants the
+// rotating variant and the abuse caps want the stable one.
 
 type CreativeRow = {
   id: string;
@@ -240,7 +240,7 @@ export async function serveAd(
     campaign_id: campaign.id,
     creative_id: pick.id,
     visitor_id: ctx.visitorId ?? null,
-    ip_hash: hashIp(ctx.ip ?? null),
+    ip_hash: hashIpRotating(ctx.ip ?? null),
     geo_country: ctx.country ?? null,
     device: ctx.device ?? null,
     billable: false,
@@ -316,13 +316,16 @@ export async function resolveClick(input: {
 
   // slot_id must be present (ad_clicks.slot_id NOT NULL) to record a click.
   if (input.slotId) {
-    const ipHash = hashIp(input.ctx?.ip ?? null);
+    // The row is stored under today's salt; the dedupe lookup has to consider
+    // yesterday's too, or every check silently misses for the first hours after
+    // the salt rotates.
+    const ipHash = hashIpRotating(input.ctx?.ip ?? null);
     const validity = await assessClickValidity({
       campaignId: campaign.id,
       slotId: input.slotId,
       impressionId: input.impressionId,
       visitorId: input.ctx?.visitorId,
-      ipHash,
+      ipHashes: rotatingIpHashCandidates(input.ctx?.ip ?? null, CLICK_DEDUPE_WINDOW_MS),
       device: input.ctx?.device,
     });
 

lib/env.ts

@@ -21,6 +21,12 @@ export const env = {
   // network's own publisher slot: anonymous terminal traffic is CrawlProof's
   // own inventory, so its impressions/clicks accrue there.
   adsDefaultSlotId: process.env.ADS_DEFAULT_SLOT_ID ?? "",
+  // Server-side secret salting every client-IP hash (ad metering + abuse caps).
+  // Unset falls back to the legacy unsalted digest, which is brute-forceable
+  // across the whole IPv4 space in minutes — set it in production. Generate
+  // with `openssl rand -base64 32`. Changing it resets abuse counters once and
+  // breaks in-flight ad click dedupe for up to 6h; both self-heal.
+  ipHashSalt: process.env.IP_HASH_SALT ?? "",
   // CoinPay — crypto credit purchases.
   coinpayMerchantId: process.env.COINPAY_MERCHANT_ID ?? "",
   coinpayApiKey: process.env.COINPAY_API_KEY ?? "",

lib/ipHash.ts

@@ -0,0 +1,127 @@
+import crypto from "node:crypto";
+import { env } from "./env";
+
+// Single source of truth for turning a client IP into a storable identifier.
+//
+// This replaces two divergent implementations that had drifted apart:
+//   * lib/rateLimit.ts — sha256("crawlproof:" + ip), a hardcoded constant
+//     prefix, which is a pepper in name only since it lives in the source.
+//   * lib/ads/serve.ts — sha256(ip), no prefix at all.
+//
+// Neither was anonymisation. IPv4 is 2^32 addresses, so the entire space can be
+// enumerated against a truncated SHA-256 in minutes on commodity hardware; an
+// unsalted digest of an IP is a reversible encoding of that IP, not a
+// pseudonym. A server-side secret salt is the thing that makes it one-way in
+// practice, which matters here because ip_hash is the only identifier the ad
+// network has for terminal traffic (curl has no localStorage) and for the ~69%
+// of web impressions that historically arrived with no visitor id.
+//
+// Two exported variants over one implementation, because the callers need
+// opposite properties from the same primitive:
+//
+//   hashIp()          Stable over time. Abuse caps look back 24h and longer
+//                     (checkAnonymousLimit), so a rotating salt would silently
+//                     refill every anonymous quota at the rotation boundary —
+//                     a quota bypass, not a privacy win.
+//   hashIpRotating()  Salt changes daily. Ad metering only needs to recognise
+//                     an IP within hours (6h click dedupe, frequency capping).
+//                     Past that window, being *able* to re-identify a visitor
+//                     is a liability rather than a feature, so the capability
+//                     is designed to expire on its own.
+//
+// Rotation makes long-term correlation impossible even against a full database
+// leak, while leaving the short-window behaviour the ad network actually uses
+// intact.
+
+const LEGACY_PREFIX = "crawlproof:";
+const DAY_MS = 86_400_000;
+
+function sha(input: string): string {
+  return crypto.createHash("sha256").update(input).digest("hex").slice(0, 32);
+}
+
+let warned = false;
+function salt(): string {
+  const s = env.ipHashSalt;
+  if (s) return s;
+  // Deliberately does not throw. An unsalted hash is a weakness; a hard failure
+  // on every request that touches an IP is an outage of both ad serving and
+  // rate limiting. Warn once — enough to be visible in logs, quiet enough not
+  // to flood them at request volume.
+  if (!warned) {
+    warned = true;
+    console.warn(
+      "[ipHash] IP_HASH_SALT is unset — falling back to the legacy unsalted digest. Set it in production.",
+    );
+  }
+  return "";
+}
+
+// NUL-joined so a salt ending in digits can't collide with an IP starting with
+// them; no component of the input can bleed into the next.
+function join(parts: string[]): string {
+  return parts.join("\u0000");
+}
+
+/**
+ * Stable, salted hash of a client IP. Use for abuse caps and anything that
+ * looks back more than a day.
+ *
+ * A missing IP hashes to a shared "unknown" bucket on purpose: for rate
+ * limiting, lumping unattributable requests together is the conservative
+ * choice.
+ */
+export function hashIp(ip: string | null | undefined): string {
+  const v = ip ?? "unknown";
+  const s = salt();
+  // With no salt configured this reproduces the historical digest byte for
+  // byte, so an environment that hasn't set IP_HASH_SALT yet doesn't invalidate
+  // every stored hash and hand each rate-limited visitor a fresh quota.
+  return s ? sha(join([s, LEGACY_PREFIX + v])) : sha(LEGACY_PREFIX + v);
+}
+
+function dayIndex(at: Date): number {
+  return Math.floor(at.getTime() / DAY_MS);
+}
+
+function rotatingFor(ip: string, day: number): string {
+  return sha(join([salt(), `day:${day}`, ip]));
+}
+
+/**
+ * Daily-rotating salted hash of a client IP, for ad metering.
+ *
+ * Returns null for a missing IP rather than bucketing to a shared constant —
+ * the opposite of hashIp, and load-bearing: every unattributable ad request
+ * sharing one hash would make them all look like duplicates of each other and
+ * invalidate legitimate clicks wholesale.
+ */
+export function hashIpRotating(
+  ip: string | null | undefined,
+  at: Date = new Date(),
+): string | null {
+  if (!ip) return null;
+  return rotatingFor(ip, dayIndex(at));
+}
+
+/**
+ * Every rotating hash an IP could have been stored under across `lookbackMs`.
+ *
+ * Dedupe windows don't respect the rotation boundary: a 6h click window that
+ * starts at 23:00 has to match rows written under yesterday's salt. Querying
+ * only today's hash would make every dedupe check silently miss for the first
+ * hours of each day — precisely when a duplicate click is most likely to be
+ * someone probing the boundary. Newest first.
+ */
+export function rotatingIpHashCandidates(
+  ip: string | null | undefined,
+  lookbackMs: number,
+  at: Date = new Date(),
+): string[] {
+  if (!ip) return [];
+  const today = dayIndex(at);
+  const earliest = dayIndex(new Date(at.getTime() - Math.max(0, lookbackMs)));
+  const out: string[] = [];
+  for (let day = today; day >= earliest; day--) out.push(rotatingFor(ip, day));
+  return out;
+}