0.2.0: denyCidrs, chargeSpoofedBrowsers, exempt

README.md

@@ -93,8 +93,16 @@ export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] });
 | `contact` | | mailto: or URL for bulk deals |
 | `onSale` | | `({ payer, ref, token, expiresAt, userAgent, priceCents, currency }) => …`, for accounting |
 
+| `denyCidrs` | `[]` | IPv4 ranges answered with a tiny `403` before anything else. For a VPS fleet that spoofs a browser: hosting ranges serve no readers. |
+| `chargeSpoofedBrowsers` | `false` | charge a request that claims `Chrome/…` but sends no `Sec-Fetch-Mode`. Every Chromium since 76, headless included, sends it on every request and no script or extension can remove it, so its absence means an HTTP client with a copied string. Firefox and Safari are not judged. |
+| `exempt` | | `(request) => boolean`, never charged: e.g. a request carrying your signed-in cookie |
+
 Without `coinpay.apiKey` and `payTo` the gateway still answers training crawlers with 402 and the page says payments are off. Nothing is sold, but nothing is given away either.
 
+## Crawlers that do not say who they are
+
+The lists catch crawlers that name themselves. Two do not: a VPS fleet wearing a browser string, and a residential-proxy rotation cycling a few Chrome strings across hundreds of addresses. `denyCidrs` handles the first (`['51.38.0.0/16', '54.38.0.0/16', …]` for one provider's ranges); `chargeSpoofedBrowsers` handles both by asking a question only a browser can answer. A request that answers it is left alone. One that cannot gets the same 402 as GPTBot, which costs the site a hash instead of a render.
+
 ## How the money moves
 
 The offer is x402 v2 in CoinPay's dialect: USDC under the `exact` scheme on Base, Polygon or Ethereum, EIP-3009 `transferWithAuthorization`. The buyer signs, the gateway sends the proof to CoinPay's `/api/x402/verify` and `/api/x402/settle`, and CoinPay's relayer broadcasts the transfer, paying the gas. The USDC goes straight to `payTo`.

index.d.ts

@@ -49,6 +49,12 @@ export interface GatewayOptions {
   retrieval?: string[];
   /** Who is charged. Default: the training list, substring-matched on the user agent. */
   isPaidAgent?: (userAgent: string) => boolean;
+  /** IPv4 CIDRs answered 403 before anything else (a VPS fleet's provider ranges). */
+  denyCidrs?: string[];
+  /** Charge a request that claims "Chrome/…" but lacks the Sec-Fetch-Mode header every Chromium sends. Default false. */
+  chargeSpoofedBrowsers?: boolean;
+  /** Requests never charged, e.g. ones carrying a signed-in cookie. */
+  exempt?: (request: Request) => boolean;
   /** Pass signing secret. Defaults to the CoinPay key. */
   secret?: string;
   page?: (ctx: PageContext) => string;
@@ -109,6 +115,14 @@ export const RETRIEVAL_AGENTS: string[];
 export function isTrainingAgent(userAgent?: string | null, agents?: string[]): boolean;
 
 export function robotsTxt(options: RobotsOptions & { siteUrl: string }): string;
+
+/** ./edge (also re-exported from the root) */
+export interface Cidr { base: number; mask: number; text: string }
+export function parseCidr(cidr: string): Cidr | null;
+export function compileCidrs(list?: string[]): Cidr[];
+export function inCidrs(ip: string, compiled: Cidr[]): boolean;
+export function clientIp(request: Request): string;
+export function isSpoofedBrowser(request: Request): boolean;
 export function renderPage(ctx: PageContext): string;
 
 export function mintPass(args: { secret: string; ref: string | null; expiresAt: number; now?: number }): Promise<{ token: string; expiresAt: number; ref: string | null }>;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/x402-gateway",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "type": "module",
   "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, and a robots.txt that keeps search crawlers welcome.",
   "keywords": [
@@ -30,7 +30,8 @@
     "./hono": { "types": "./index.d.ts", "import": "./src/hono.js" },
     "./next": { "types": "./index.d.ts", "import": "./src/next.js" },
     "./robots": { "types": "./index.d.ts", "import": "./src/robots.js" },
-    "./agents": { "types": "./index.d.ts", "import": "./src/agents.js" }
+    "./agents": { "types": "./index.d.ts", "import": "./src/agents.js" },
+    "./edge": { "types": "./index.d.ts", "import": "./src/edge.js" }
   },
   "types": "./index.d.ts",
   "files": ["src", "index.d.ts", "README.md", "LICENSE"],

src/edge.js

@@ -0,0 +1,92 @@
+/**
+ * The two checks that do not need a user agent to be honest.
+ *
+ * A crawler that names itself is charged by the lists in ./agents. The ones
+ * that do not -- a VPS fleet wearing "Chrome/148", a residential-proxy
+ * rotation cycling three Chrome strings across five hundred addresses -- need
+ * something the request cannot help giving away. Two things qualify:
+ *
+ *   1. Where it came from. A hosting provider's address range serves no
+ *      readers, only machines. A CIDR denylist answers those with a tiny 403
+ *      before anything else runs.
+ *
+ *   2. Whether it is the browser it claims to be. Every Chromium since 76,
+ *      headless included, sends `Sec-Fetch-Mode` on every request; it is a
+ *      forbidden header, so no page script and no extension can remove it.
+ *      A request that says "Chrome/145" and does not send it is an HTTP client
+ *      with a copied string. That is not a person, and it is charged like any
+ *      other crawler.
+ */
+
+/* ------------------------------------------------------------------ CIDRs -- */
+
+function ipv4ToInt(ip) {
+  const parts = ip.split('.');
+  if (parts.length !== 4) return null;
+  let n = 0;
+  for (const p of parts) {
+    if (!/^\d{1,3}$/.test(p)) return null;
+    const v = Number(p);
+    if (v > 255) return null;
+    n = n * 256 + v;
+  }
+  return n;
+}
+
+/** Parse "a.b.c.d/len" (or a bare address) into a matcher. Null if unreadable. */
+export function parseCidr(cidr) {
+  const [ip, lenRaw] = String(cidr).trim().split('/');
+  const base = ipv4ToInt(ip);
+  if (base === null) return null;
+  const len = lenRaw === undefined ? 32 : Number(lenRaw);
+  if (!Number.isInteger(len) || len < 0 || len > 32) return null;
+  const mask = len === 0 ? 0 : (0xffffffff << (32 - len)) >>> 0;
+  return { base: (base & mask) >>> 0, mask, text: `${ip}/${len}` };
+}
+
+/** Compile a denylist once. Unreadable entries are dropped, not guessed at. */
+export function compileCidrs(list = []) {
+  return list.map(parseCidr).filter(Boolean);
+}
+
+/** Whether an IPv4 address falls inside any compiled range. */
+export function inCidrs(ip, compiled) {
+  const n = ipv4ToInt(String(ip ?? '').trim());
+  if (n === null) return false;
+  return compiled.some((c) => ((n & c.mask) >>> 0) === c.base);
+}
+
+/**
+ * The caller's address, as the edge reported it.
+ *
+ * `x-forwarded-for` is a list the client can seed; the LAST hop appended by
+ * our own edge is trustworthy and the first is not, but every platform in
+ * front of these sites (Railway, a bare droplet behind nginx) puts the real
+ * client first and nothing else, so first is what is used. `x-real-ip` is the
+ * nginx spelling of the same thing.
+ */
+export function clientIp(request) {
+  const xff = request.headers.get('x-forwarded-for');
+  if (xff) return xff.split(',')[0].trim();
+  return request.headers.get('x-real-ip')?.trim() ?? '';
+}
+
+/* -------------------------------------------------------------- spoofing -- */
+
+const CLAIMS_CHROMIUM = /\bChrome\/\d+/;
+
+/**
+ * A request that claims a Chromium user agent but carries none of the
+ * fetch-metadata headers Chromium cannot omit.
+ *
+ * Only Chromium is judged: Firefox and Safari added Sec-Fetch later and
+ * older builds of both are still out there, so their absence proves nothing.
+ * `sec-fetch-mode` is the one checked because it is present on every request
+ * kind -- navigation, subresource, fetch -- unlike `sec-ch-ua`, which a
+ * privacy proxy may strip.
+ */
+export function isSpoofedBrowser(request) {
+  const ua = request.headers.get('user-agent') ?? '';
+  if (!CLAIMS_CHROMIUM.test(ua)) return false;
+  return !request.headers.has('sec-fetch-mode');
+}

src/index.js

@@ -1,4 +1,5 @@
 import { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from './agents.js';
+import { clientIp, compileCidrs, inCidrs, isSpoofedBrowser } from './edge.js';
 import { renderPage } from './page.js';
 import { mintPass, readPass } from './pass.js';
 import { robotsTxt } from './robots.js';
@@ -14,6 +15,7 @@ import {
 } from './x402.js';
 
 export { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from './agents.js';
+export { clientIp, compileCidrs, inCidrs, isSpoofedBrowser, parseCidr } from './edge.js';
 export { renderPage } from './page.js';
 export { mintPass, readPass } from './pass.js';
 export { robotsTxt } from './robots.js';
@@ -45,6 +47,9 @@ export { buildOffer, decodePayment, expectedFor, METHODS, verifyAndSettle } from
  * @param {string} [options.path='/crawl']         the sales page
  * @param {string[]} [options.openPaths]           extra paths a refused crawler may read
  * @param {(ua: string) => boolean} [options.isPaidAgent]
+ * @param {string[]} [options.denyCidrs]           IPv4 ranges answered 403 before anything else, e.g. a VPS fleet's provider
+ * @param {boolean} [options.chargeSpoofedBrowsers=false]  charge a "Chrome/…" request that lacks the Sec-Fetch-Mode header every Chromium sends
+ * @param {(request: Request) => boolean} [options.exempt]  requests never charged, e.g. ones carrying a signed-in cookie
  * @param {string} [options.secret]                pass signing secret; defaults to the CoinPay key
  * @param {(ctx: object) => string} [options.page] custom sales page renderer
  * @param {string} [options.contact]               mailto: or URL for bulk deals
@@ -57,6 +62,7 @@ export function createGateway(options = {}) {
   const secret = o.secret || o.coinpay.apiKey || null;
 
   const openPaths = ['/robots.txt', o.path, '/security.txt', '/.well-known/', ...o.openPaths];
+  const denied = compileCidrs(o.denyCidrs);
   const isOpen = (path) => openPaths.some((p) => (p.endsWith('/') ? path.startsWith(p) : path === p));
 
   const price = `${(o.priceCents / 100).toFixed(2)} ${o.currency}`;
@@ -204,9 +210,26 @@ export function createGateway(options = {}) {
    * starts from the user agent.
    */
   async function handle(request) {
+    /*
+     * Addresses that serve no readers are refused before anything else, with
+     * a body small enough that refusing costs nothing. Not 402: there is no
+     * pass on sale to a hosting range that spoofs a browser, because whoever
+     * runs it has already declined to say who they are.
+     */
+    if (denied.length && inCidrs(clientIp(request), denied)) {
+      return new Response('Not available from this network.\n', {
+        status: 403,
+        headers: { 'content-type': 'text/plain; charset=utf-8', ...noStore },
+      });
+    }
+
     const path = new URL(request.url).pathname;
     if (path === o.path) return sell(request);
-    if (!o.isPaidAgent(request.headers.get('user-agent') ?? '')) return null;
+    if (o.exempt && o.exempt(request)) return null;
+    const pays =
+      o.isPaidAgent(request.headers.get('user-agent') ?? '') ||
+      (o.chargeSpoofedBrowsers && isSpoofedBrowser(request));
+    if (!pays) return null;
     if (isOpen(path)) return null;
 
     const token = passFrom(request);
@@ -248,6 +271,9 @@ function normalise(options) {
     header: String(options.header ?? 'x-crawl-pass').toLowerCase(),
     path: options.path ?? '/crawl',
     openPaths: options.openPaths ?? [],
+    denyCidrs: options.denyCidrs ?? [],
+    chargeSpoofedBrowsers: Boolean(options.chargeSpoofedBrowsers),
+    exempt: options.exempt ?? null,
     training,
     retrieval: options.retrieval ?? RETRIEVAL_AGENTS,
     isPaidAgent: options.isPaidAgent ?? ((ua) => isTrainingAgent(ua, training)),

test/gateway.test.js

@@ -1,6 +1,7 @@
 import assert from 'node:assert/strict';
 import { describe, it } from 'node:test';
 import { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from '../src/agents.js';
+import { clientIp, compileCidrs, inCidrs, isSpoofedBrowser, parseCidr } from '../src/edge.js';
 import { createGateway, wantsHtml } from '../src/index.js';
 import { x402Gateway } from '../src/hono.js';
 import { robotsRoute, x402Proxy } from '../src/next.js';
@@ -343,6 +344,78 @@ describe('the gate', () => {
   });
 });
 
+describe('crawlers that do not say who they are', () => {
+  const OVH = ['51.38.0.0/16', '54.38.0.0/16', '141.94.0.0/16'];
+  const from = (ip, extra = {}) =>
+    req('/topics/x', { ua: CHROME, ...extra, headers: { 'x-forwarded-for': `${ip}, 10.0.0.1`, ...(extra.headers ?? {}) } });
+
+  it('parses CIDRs and matches addresses, and drops what it cannot read', () => {
+    const c = compileCidrs([...OVH, 'garbage', '1.2.3.4', '300.1.1.1/8', '10.0.0.0/33']);
+    assert.equal(c.length, 4);
+    assert.equal(inCidrs('51.38.200.7', c), true);
+    assert.equal(inCidrs('51.39.0.1', c), false);
+    assert.equal(inCidrs('1.2.3.4', c), true);
+    assert.equal(inCidrs('1.2.3.5', c), false);
+    assert.equal(inCidrs('not an ip', c), false);
+    assert.equal(inCidrs('', c), false);
+    assert.equal(parseCidr('0.0.0.0/0').mask, 0);
+    assert.equal(inCidrs('9.9.9.9', compileCidrs(['0.0.0.0/0'])), true);
+  });
+
+  it('reads the client address the way the edge writes it', () => {
+    assert.equal(clientIp(req('/', { headers: { 'x-forwarded-for': '203.0.113.9, 10.1.1.1' } })), '203.0.113.9');
+    assert.equal(clientIp(req('/', { headers: { 'x-real-ip': '203.0.113.10' } })), '203.0.113.10');
+    assert.equal(clientIp(req('/')), '');
+  });
+
+  it('refuses a denied range with a tiny 403 before anything else, even a paying pass or the sales page', async () => {
+    const { gateway, cp } = gatewayFor({ denyCidrs: OVH });
+    const res = await gateway.handle(from('54.38.1.2'));
+    assert.equal(res.status, 403);
+    assert.equal(res.headers.get('cache-control'), 'no-store');
+    assert.equal((await gateway.handle(from('54.38.1.2', { headers: { 'x-payment': proof() } }))).status, 403);
+    assert.equal(cp.calls.length, 0);
+    assert.equal((await gateway.handle(req('/crawl', { headers: { 'x-forwarded-for': '141.94.9.9' } }))).status, 403);
+    // A neighbour outside the range, same UA, is a person.
+    assert.equal(await gateway.handle(from('51.39.0.1')), null);
+  });
+
+  it('knows a copied Chrome string from Chrome', () => {
+    assert.equal(isSpoofedBrowser(req('/', { ua: CHROME })), true, 'no Sec-Fetch-Mode at all');
+    assert.equal(isSpoofedBrowser(req('/', { ua: CHROME, headers: { 'sec-fetch-mode': 'navigate' } })), false);
+    assert.equal(isSpoofedBrowser(req('/', { ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0' })), false, 'Firefox is not judged');
+    assert.equal(isSpoofedBrowser(req('/', { ua: 'curl/8.0' })), false, 'an honest client is not judged either');
+    assert.equal(isSpoofedBrowser(req('/', { ua: META })), false, 'a declared crawler is charged by name, not by this');
+  });
+
+  it('charges a spoofed browser only when asked to, and never one that answers the question', async () => {
+    const quiet = gatewayFor().gateway;
+    assert.equal(await quiet.handle(req('/topics/x', { ua: CHROME })), null, 'off by default');
+
+    const { gateway } = gatewayFor({ chargeSpoofedBrowsers: true });
+    const res = await gateway.handle(req('/topics/x', { ua: CHROME, accept: 'application/json' }));
+    assert.equal(res.status, 402);
+    assert.equal((await res.json()).accepts.length, 3);
+    const real = req('/topics/x', { ua: CHROME, headers: { 'sec-fetch-mode': 'navigate', 'sec-fetch-site': 'none' } });
+    assert.equal(await gateway.handle(real), null);
+    // A spoofed browser that pays gets a pass like anyone else.
+    const paid = await gateway.handle(req('/topics/x', { ua: CHROME, headers: { 'x-payment': proof({ nonce: '0xs1' }) } }));
+    assert.equal(paid.status, 200);
+    const { pass } = await paid.json();
+    assert.equal(await gateway.handle(req('/topics/y', { ua: CHROME, headers: { 'x-crawl-pass': pass } })), null);
+  });
+
+  it('exempts what the site says to exempt, before any charge', async () => {
+    const { gateway } = gatewayFor({
+      chargeSpoofedBrowsers: true,
+      exempt: (r) => (r.headers.get('cookie') ?? '').includes('signed_in=1'),
+    });
+    assert.equal(await gateway.handle(req('/topics/x', { ua: CHROME, headers: { cookie: 'signed_in=1' } })), null);
+    assert.equal(await gateway.handle(req('/topics/x', { ua: META, headers: { cookie: 'signed_in=1' } })), null, 'even a named crawler with the cookie');
+    assert.equal((await gateway.handle(req('/topics/x', { ua: META }))).status, 402);
+  });
+});
+
 describe('adapters', () => {
   it('Hono: returns the gateway response or calls next', async () => {
     const { gateway } = gatewayFor();