Enable the x402 gateway's edge controls (0.2.2): OVH denylist, spoofed-browser toll, signed-in exemption

package.json

@@ -40,7 +40,7 @@
     "@profullstack/player": "0.3.1",
     "@profullstack/referrals": "^0.1.0",
     "@profullstack/stack": "^0.1.3",
-    "@profullstack/x402-gateway": "0.1.0",
+    "@profullstack/x402-gateway": "0.2.2",
     "@radix-ui/react-dialog": "^1.1.23",
     "@radix-ui/react-dropdown-menu": "^2.1.24",
     "@radix-ui/react-icons": "^1.3.2",

src/lib/crawl-gateway.ts

@@ -6,6 +6,18 @@
  * HTML sales page at /crawl. A paid pass in the `x-crawl-pass` header lets them
  * through. People, Googlebot and retrieval crawlers are untouched.
  *
+ * Two edge controls catch crawlers that do not say who they are:
+ * - `denyCidrs`: hosting ranges that serve no readers get a tiny 403 first.
+ * - `chargeSpoofedBrowsers`: a "Chrome/..." user agent with no Sec-Fetch-Mode
+ *   header is an HTTP client wearing a copied string (every Chromium since 76
+ *   sends it and nothing can strip it), so it is charged like GPTBot. Anything
+ *   that declares itself (Googlebot's evergreen string, Bingbot, any "bot")
+ *   is judged by the lists instead, and Firefox/Safari are never judged.
+ *
+ * This site is mostly real people, so `exempt` matters most: a request that
+ * carries a valid-looking Supabase session or a valid-looking `btr_` API
+ * bearer token is never charged, whatever else it looks like.
+ *
  * Used by src/proxy.ts (the gate) and src/app/robots.txt/route.ts (the lists),
  * so robots.txt and the gate never disagree about who is who.
  *
@@ -14,9 +26,76 @@
 
 import { createGateway } from '@profullstack/x402-gateway';
 
+/**
+ * OVH VPS fleet ranges, measured 2026-08-28 on rssamplifier: vps-*.vps.ovh.net
+ * hosts spoofing "Chrome/148" across these /16s. Hosting ranges serve no
+ * readers, so they are refused outright rather than offered a pass.
+ */
+const OVH_VPS_FLEET_CIDRS = [
+  '51.38.0.0/16',
+  '54.38.0.0/16',
+  '141.94.0.0/16',
+  '145.239.0.0/16',
+  '149.202.0.0/16',
+  '151.80.0.0/16',
+  '57.129.0.0/16',
+  '213.32.0.0/16',
+];
+
+/** The Supabase session cookie src/proxy.ts refreshes and src/lib/auth reads. */
+const SESSION_COOKIE_NAME = 'sb-auth-token';
+
+/** A v1 API token from src/lib/api-tokens: `btr_` + 32 random bytes as hex. */
+const API_BEARER_RE = /^Bearer\s+btr_[0-9a-f]{64}$/i;
+
+/** Three base64url segments: the shape of the Supabase access token. */
+const JWT_RE = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/;
+
+function cookieValue(request: Request, name: string): string | null {
+  const header = request.headers.get('cookie');
+  if (!header) return null;
+  for (const part of header.split(';')) {
+    const eq = part.indexOf('=');
+    if (eq === -1) continue;
+    if (part.slice(0, eq).trim() === name) return part.slice(eq + 1).trim();
+  }
+  return null;
+}
+
+/**
+ * Whether the request carries a valid-looking Supabase session: the cookie
+ * src/proxy.ts refreshes, holding JSON with a JWT-shaped access_token and a
+ * refresh_token. Shape only, no verification -- the gate runs before anything
+ * that could ask Supabase, and the point is that a junk cookie named
+ * sb-auth-token does not buy a spoofing fleet a way past the toll.
+ */
+export function hasSessionCookie(request: Request): boolean {
+  const raw = cookieValue(request, SESSION_COOKIE_NAME);
+  if (!raw) return false;
+  try {
+    const session = JSON.parse(decodeURIComponent(raw)) as { access_token?: unknown; refresh_token?: unknown };
+    return (
+      typeof session.access_token === 'string' &&
+      JWT_RE.test(session.access_token) &&
+      typeof session.refresh_token === 'string' &&
+      session.refresh_token.length > 0
+    );
+  } catch {
+    return false;
+  }
+}
+
+/** Whether the request carries a valid-looking v1 API bearer token. */
+export function hasApiBearer(request: Request): boolean {
+  return API_BEARER_RE.test(request.headers.get('authorization') ?? '');
+}
+
 export const gateway = createGateway({
   siteUrl: process.env.NEXT_PUBLIC_APP_URL ?? 'https://bittorrented.com',
   siteName: 'bittorrented',
   coinpay: { apiKey: process.env.COINPAY_X402_KEY },
   payTo: process.env.CRAWL_PAY_TO,
+  denyCidrs: OVH_VPS_FLEET_CIDRS,
+  chargeSpoofedBrowsers: true,
+  exempt: (request) => hasSessionCookie(request) || hasApiBearer(request),
 });

src/proxy.test.ts

@@ -29,6 +29,7 @@ describe('Bot Handling Middleware', () => {
     const req = new NextRequest(url, {
       headers: {
         ...(userAgent ? { 'user-agent': userAgent } : {}),
+        'sec-fetch-mode': 'navigate', // what every real Chromium sends; see the edge-control tests for its absence
         'x-forwarded-for': `${Math.random().toString(36).slice(2)}.1.1.1`, // unique IP per call to avoid rate limit state
       },
     });
@@ -112,6 +113,7 @@ describe('Crawl Gateway (x402)', () => {
     const req = new NextRequest(url, {
       headers: {
         ...(userAgent ? { 'user-agent': userAgent } : {}),
+        'sec-fetch-mode': 'navigate',
         'x-forwarded-for': `${Math.random().toString(36).slice(2)}.1.1.1`,
         ...headers,
       },
@@ -189,6 +191,7 @@ describe('Supabase session refresh and referral cookie', () => {
     const req = new NextRequest(new URL(`http://localhost${pathname}`), {
       headers: {
         'user-agent': ua,
+        'sec-fetch-mode': 'navigate',
         'x-forwarded-for': `${Math.random().toString(36).slice(2)}.1.1.1`,
         ...(cookie ? { cookie } : {}),
         ...headers,
@@ -281,13 +284,23 @@ describe('Supabase session refresh and referral cookie', () => {
   });
 
   it('answers a training crawler with 402 without touching Supabase', async () => {
-    const res = await call('/browse', { ua: META_UA, cookies: { 'sb-auth-token': authCookie(30) } });
+    const res = await call('/browse', { ua: META_UA });
     expect(res).toBeDefined();
     expect(res!.status).toBe(402);
     expect(fetchMock).not.toHaveBeenCalled();
     expect(res!.headers.get('set-cookie')).toBeNull();
   });
 
+  it('a signed-in session is never charged, whatever user agent carries it', async () => {
+    // The gateway's `exempt` runs before the agent lists: this site is mostly
+    // people, and a request that presents a real session is treated as one of
+    // them. It then goes through the ordinary session refresh like any browser.
+    const res = await call('/browse', { ua: META_UA, cookies: { 'sb-auth-token': authCookie(30), 'x-profile-id': 'p1' } });
+    expectPassThrough(res);
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    expect(JSON.parse(decodeURIComponent(res.cookies.get('sb-auth-token')!.value)).refresh_token).toBe('new-refresh');
+  });
+
   it('stores a valid ?ref= code in the referral_code cookie', async () => {
     const res = await call('/browse?ref=ABC-123_x');
     expectPassThrough(res);
@@ -309,3 +322,136 @@ describe('Supabase session refresh and referral cookie', () => {
     expect(JSON.parse(decodeURIComponent(res.cookies.get('sb-auth-token')!.value)).refresh_token).toBe('new-refresh');
   });
 });
+
+describe('Edge controls: hosting ranges and spoofed browsers', () => {
+  const CHROME_UA =
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
+  const FIREFOX_UA = 'Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0';
+  const GOOGLEBOT_EVERGREEN_UA =
+    'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/148.0.0.0 Safari/537.36';
+  const BINGBOT_UA =
+    'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/148.0.0.0 Safari/537.36';
+
+  /**
+   * A Supabase session cookie of the shape src/proxy.ts refreshes. Fixture,
+   * not a credential: the access token is an unsigned JWT (`alg: none`) that
+   * expired in 1970 and the refresh token is one letter. The gate checks shape
+   * only and never verifies either, which is exactly what these tests pin.
+   */
+  // threatcrush-disable-next-line secret-jwt
+  const FIXTURE_UNSIGNED_EXPIRED_JWT = 'eyJhbGciOiJub25lIn0.eyJleHAiOjB9.sig';
+  // threatcrush-disable-next-line secret-generic-credential
+  const FIXTURE_REFRESH = 'r';
+  const SESSION_COOKIE = `sb-auth-token=${encodeURIComponent(
+    JSON.stringify({ access_token: FIXTURE_UNSIGNED_EXPIRED_JWT, refresh_token: FIXTURE_REFRESH })
+  )}`;
+  const API_BEARER = `Bearer btr_${'ab'.repeat(32)}`;
+
+  /** Exactly the headers given, nothing implied: these tests are about what is missing. */
+  function raw(pathname: string, headers: Record<string, string>) {
+    const h: Record<string, string> = { 'x-forwarded-for': `${Math.random().toString(36).slice(2)}.1.1.1`, ...headers };
+    return middleware(new NextRequest(new URL(`http://localhost${pathname}`), { headers: h }));
+  }
+
+  describe('denyCidrs (OVH VPS fleet)', () => {
+    it('refuses a request whose last x-forwarded-for hop is in an OVH range, even a well-formed browser', async () => {
+      const res = await raw('/browse', {
+        'user-agent': CHROME_UA,
+        'sec-fetch-mode': 'navigate',
+        'x-forwarded-for': '203.0.113.9, 51.38.12.34',
+      });
+      expect(res).toBeDefined();
+      expect(res!.status).toBe(403);
+      expect(await res!.text()).toContain('Not available from this network');
+    });
+
+    it('refuses by x-real-ip too', async () => {
+      const res = await raw('/browse', { 'user-agent': CHROME_UA, 'sec-fetch-mode': 'navigate', 'x-real-ip': '145.239.200.1' });
+      expect(res!.status).toBe(403);
+    });
+
+    it('judges the LAST hop only: a client-seeded first hop cannot get anyone refused', async () => {
+      const res = await raw('/browse', {
+        'user-agent': CHROME_UA,
+        'sec-fetch-mode': 'navigate',
+        'x-forwarded-for': '51.38.12.34, 203.0.113.9',
+      });
+      expectPassThrough(res);
+    });
+
+    it('covers every listed range', async () => {
+      for (const ip of ['51.38.1.1', '54.38.1.1', '141.94.1.1', '145.239.1.1', '149.202.1.1', '151.80.1.1', '57.129.1.1', '213.32.1.1']) {
+        const res = await raw('/', { 'user-agent': CHROME_UA, 'sec-fetch-mode': 'navigate', 'x-forwarded-for': ip });
+        expect(res!.status, ip).toBe(403);
+      }
+    });
+  });
+
+  describe('chargeSpoofedBrowsers', () => {
+    it('charges a Chrome user agent that sends no Sec-Fetch-Mode', async () => {
+      const res = await raw('/browse', { 'user-agent': CHROME_UA });
+      expect(res).toBeDefined();
+      expect(res!.status).toBe(402);
+      expect(res!.headers.get('content-type')).toContain('application/json');
+    });
+
+    it('passes the same Chrome user agent with Sec-Fetch-Mode to the existing behaviour', async () => {
+      const res = await raw('/browse', { 'user-agent': CHROME_UA, 'sec-fetch-mode': 'navigate' });
+      expectPassThrough(res);
+    });
+
+    it('never judges Googlebot\'s evergreen Chrome string', async () => {
+      const res = await raw('/browse', { 'user-agent': GOOGLEBOT_EVERGREEN_UA });
+      expectPassThrough(res);
+    });
+
+    it('never judges Bingbot\'s evergreen Chrome string', async () => {
+      const res = await raw('/browse', { 'user-agent': BINGBOT_UA });
+      expectPassThrough(res);
+    });
+
+    it('never judges Firefox, which older builds send without Sec-Fetch', async () => {
+      const res = await raw('/browse', { 'user-agent': FIREFOX_UA });
+      expectPassThrough(res);
+    });
+
+    it('still lets a spoofed browser read robots.txt and the sales page', async () => {
+      expectPassThrough(await raw('/robots.txt', { 'user-agent': CHROME_UA }));
+      const sales = await raw('/crawl', { 'user-agent': CHROME_UA, accept: 'text/html' });
+      expect(sales!.status).toBe(402);
+      expect(sales!.headers.get('content-type')).toContain('text/html');
+    });
+  });
+
+  describe('exempt: signed-in people and API clients are never charged', () => {
+    it('passes a Chrome request without Sec-Fetch when it carries a Supabase session cookie', async () => {
+      const res = await raw('/browse', { 'user-agent': CHROME_UA, cookie: `${SESSION_COOKIE}; x-profile-id=p1` });
+      expectPassThrough(res);
+    });
+
+    it('does not accept a junk cookie merely named sb-auth-token', async () => {
+      const res = await raw('/browse', { 'user-agent': CHROME_UA, cookie: 'sb-auth-token=not-a-session' });
+      expect(res!.status).toBe(402);
+    });
+
+    it('passes a Chrome request without Sec-Fetch when it carries a valid-looking btr_ API bearer token', async () => {
+      const res = await raw('/api/v1/me', { 'user-agent': CHROME_UA, authorization: API_BEARER });
+      expectPassThrough(res);
+    });
+
+    it('does not accept a bearer token of the wrong shape', async () => {
+      const res = await raw('/api/v1/me', { 'user-agent': CHROME_UA, authorization: 'Bearer btr_short' });
+      expect(res!.status).toBe(402);
+    });
+
+    it('a session does not get a hosting range past the 403', async () => {
+      const res = await raw('/browse', {
+        'user-agent': CHROME_UA,
+        'sec-fetch-mode': 'navigate',
+        cookie: SESSION_COOKIE,
+        'x-forwarded-for': '149.202.3.4',
+      });
+      expect(res!.status).toBe(403);
+    });
+  });
+});