diff --git a/README.md b/README.md index 6b6122f..1dd2485 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,16 @@ export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] }); | `contact` | | mailto: or URL for bulk deals | | `onSale` | | `({ payer, ref, token, expiresAt, userAgent, priceCents, currency }) => …`, for accounting | +| `denyCidrs` | `[]` | IPv4 ranges answered with a tiny `403` before anything else. For a VPS fleet that spoofs a browser: hosting ranges serve no readers. | +| `chargeSpoofedBrowsers` | `false` | charge a request that claims `Chrome/…` but sends no `Sec-Fetch-Mode`. Every Chromium since 76, headless included, sends it on every request and no script or extension can remove it, so its absence means an HTTP client with a copied string. Firefox and Safari are not judged. | +| `exempt` | | `(request) => boolean`, never charged: e.g. a request carrying your signed-in cookie | + Without `coinpay.apiKey` and `payTo` the gateway still answers training crawlers with 402 and the page says payments are off. Nothing is sold, but nothing is given away either. +## Crawlers that do not say who they are + +The lists catch crawlers that name themselves. Two do not: a VPS fleet wearing a browser string, and a residential-proxy rotation cycling a few Chrome strings across hundreds of addresses. `denyCidrs` handles the first (`['51.38.0.0/16', '54.38.0.0/16', …]` for one provider's ranges); `chargeSpoofedBrowsers` handles both by asking a question only a browser can answer. A request that answers it is left alone. One that cannot gets the same 402 as GPTBot, which costs the site a hash instead of a render. + ## How the money moves The offer is x402 v2 in CoinPay's dialect: USDC under the `exact` scheme on Base, Polygon or Ethereum, EIP-3009 `transferWithAuthorization`. The buyer signs, the gateway sends the proof to CoinPay's `/api/x402/verify` and `/api/x402/settle`, and CoinPay's relayer broadcasts the transfer, paying the gas. The USDC goes straight to `payTo`. diff --git a/index.d.ts b/index.d.ts index 9051812..e8d2bdf 100644 --- a/index.d.ts +++ b/index.d.ts @@ -49,6 +49,12 @@ export interface GatewayOptions { retrieval?: string[]; /** Who is charged. Default: the training list, substring-matched on the user agent. */ isPaidAgent?: (userAgent: string) => boolean; + /** IPv4 CIDRs answered 403 before anything else (a VPS fleet's provider ranges). */ + denyCidrs?: string[]; + /** Charge a request that claims "Chrome/…" but lacks the Sec-Fetch-Mode header every Chromium sends. Default false. */ + chargeSpoofedBrowsers?: boolean; + /** Requests never charged, e.g. ones carrying a signed-in cookie. */ + exempt?: (request: Request) => boolean; /** Pass signing secret. Defaults to the CoinPay key. */ secret?: string; page?: (ctx: PageContext) => string; @@ -109,6 +115,14 @@ export const RETRIEVAL_AGENTS: string[]; export function isTrainingAgent(userAgent?: string | null, agents?: string[]): boolean; export function robotsTxt(options: RobotsOptions & { siteUrl: string }): string; + +/** ./edge (also re-exported from the root) */ +export interface Cidr { base: number; mask: number; text: string } +export function parseCidr(cidr: string): Cidr | null; +export function compileCidrs(list?: string[]): Cidr[]; +export function inCidrs(ip: string, compiled: Cidr[]): boolean; +export function clientIp(request: Request): string; +export function isSpoofedBrowser(request: Request): boolean; export function renderPage(ctx: PageContext): string; export function mintPass(args: { secret: string; ref: string | null; expiresAt: number; now?: number }): Promise<{ token: string; expiresAt: number; ref: string | null }>; diff --git a/package.json b/package.json index b1e5a59..88fc2c6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/x402-gateway", - "version": "0.1.0", + "version": "0.2.0", "type": "module", "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, and a robots.txt that keeps search crawlers welcome.", "keywords": [ @@ -30,7 +30,8 @@ "./hono": { "types": "./index.d.ts", "import": "./src/hono.js" }, "./next": { "types": "./index.d.ts", "import": "./src/next.js" }, "./robots": { "types": "./index.d.ts", "import": "./src/robots.js" }, - "./agents": { "types": "./index.d.ts", "import": "./src/agents.js" } + "./agents": { "types": "./index.d.ts", "import": "./src/agents.js" }, + "./edge": { "types": "./index.d.ts", "import": "./src/edge.js" } }, "types": "./index.d.ts", "files": ["src", "index.d.ts", "README.md", "LICENSE"], diff --git a/src/edge.js b/src/edge.js new file mode 100644 index 0000000..bf29727 --- /dev/null +++ b/src/edge.js @@ -0,0 +1,92 @@ +/** + * The two checks that do not need a user agent to be honest. + * + * A crawler that names itself is charged by the lists in ./agents. The ones + * that do not -- a VPS fleet wearing "Chrome/148", a residential-proxy + * rotation cycling three Chrome strings across five hundred addresses -- need + * something the request cannot help giving away. Two things qualify: + * + * 1. Where it came from. A hosting provider's address range serves no + * readers, only machines. A CIDR denylist answers those with a tiny 403 + * before anything else runs. + * + * 2. Whether it is the browser it claims to be. Every Chromium since 76, + * headless included, sends `Sec-Fetch-Mode` on every request; it is a + * forbidden header, so no page script and no extension can remove it. + * A request that says "Chrome/145" and does not send it is an HTTP client + * with a copied string. That is not a person, and it is charged like any + * other crawler. + */ + +/* ------------------------------------------------------------------ CIDRs -- */ + +function ipv4ToInt(ip) { + const parts = ip.split('.'); + if (parts.length !== 4) return null; + let n = 0; + for (const p of parts) { + if (!/^\d{1,3}$/.test(p)) return null; + const v = Number(p); + if (v > 255) return null; + n = n * 256 + v; + } + return n; +} + +/** Parse "a.b.c.d/len" (or a bare address) into a matcher. Null if unreadable. */ +export function parseCidr(cidr) { + const [ip, lenRaw] = String(cidr).trim().split('/'); + const base = ipv4ToInt(ip); + if (base === null) return null; + const len = lenRaw === undefined ? 32 : Number(lenRaw); + if (!Number.isInteger(len) || len < 0 || len > 32) return null; + const mask = len === 0 ? 0 : (0xffffffff << (32 - len)) >>> 0; + return { base: (base & mask) >>> 0, mask, text: `${ip}/${len}` }; +} + +/** Compile a denylist once. Unreadable entries are dropped, not guessed at. */ +export function compileCidrs(list = []) { + return list.map(parseCidr).filter(Boolean); +} + +/** Whether an IPv4 address falls inside any compiled range. */ +export function inCidrs(ip, compiled) { + const n = ipv4ToInt(String(ip ?? '').trim()); + if (n === null) return false; + return compiled.some((c) => ((n & c.mask) >>> 0) === c.base); +} + +/** + * The caller's address, as the edge reported it. + * + * `x-forwarded-for` is a list the client can seed; the LAST hop appended by + * our own edge is trustworthy and the first is not, but every platform in + * front of these sites (Railway, a bare droplet behind nginx) puts the real + * client first and nothing else, so first is what is used. `x-real-ip` is the + * nginx spelling of the same thing. + */ +export function clientIp(request) { + const xff = request.headers.get('x-forwarded-for'); + if (xff) return xff.split(',')[0].trim(); + return request.headers.get('x-real-ip')?.trim() ?? ''; +} + +/* -------------------------------------------------------------- spoofing -- */ + +const CLAIMS_CHROMIUM = /\bChrome\/\d+/; + +/** + * A request that claims a Chromium user agent but carries none of the + * fetch-metadata headers Chromium cannot omit. + * + * Only Chromium is judged: Firefox and Safari added Sec-Fetch later and + * older builds of both are still out there, so their absence proves nothing. + * `sec-fetch-mode` is the one checked because it is present on every request + * kind -- navigation, subresource, fetch -- unlike `sec-ch-ua`, which a + * privacy proxy may strip. + */ +export function isSpoofedBrowser(request) { + const ua = request.headers.get('user-agent') ?? ''; + if (!CLAIMS_CHROMIUM.test(ua)) return false; + return !request.headers.has('sec-fetch-mode'); +} diff --git a/src/index.js b/src/index.js index dcc2792..c1efcf5 100644 --- a/src/index.js +++ b/src/index.js @@ -1,4 +1,5 @@ import { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from './agents.js'; +import { clientIp, compileCidrs, inCidrs, isSpoofedBrowser } from './edge.js'; import { renderPage } from './page.js'; import { mintPass, readPass } from './pass.js'; import { robotsTxt } from './robots.js'; @@ -14,6 +15,7 @@ import { } from './x402.js'; export { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from './agents.js'; +export { clientIp, compileCidrs, inCidrs, isSpoofedBrowser, parseCidr } from './edge.js'; export { renderPage } from './page.js'; export { mintPass, readPass } from './pass.js'; export { robotsTxt } from './robots.js'; @@ -45,6 +47,9 @@ export { buildOffer, decodePayment, expectedFor, METHODS, verifyAndSettle } from * @param {string} [options.path='/crawl'] the sales page * @param {string[]} [options.openPaths] extra paths a refused crawler may read * @param {(ua: string) => boolean} [options.isPaidAgent] + * @param {string[]} [options.denyCidrs] IPv4 ranges answered 403 before anything else, e.g. a VPS fleet's provider + * @param {boolean} [options.chargeSpoofedBrowsers=false] charge a "Chrome/…" request that lacks the Sec-Fetch-Mode header every Chromium sends + * @param {(request: Request) => boolean} [options.exempt] requests never charged, e.g. ones carrying a signed-in cookie * @param {string} [options.secret] pass signing secret; defaults to the CoinPay key * @param {(ctx: object) => string} [options.page] custom sales page renderer * @param {string} [options.contact] mailto: or URL for bulk deals @@ -57,6 +62,7 @@ export function createGateway(options = {}) { const secret = o.secret || o.coinpay.apiKey || null; const openPaths = ['/robots.txt', o.path, '/security.txt', '/.well-known/', ...o.openPaths]; + const denied = compileCidrs(o.denyCidrs); const isOpen = (path) => openPaths.some((p) => (p.endsWith('/') ? path.startsWith(p) : path === p)); const price = `${(o.priceCents / 100).toFixed(2)} ${o.currency}`; @@ -204,9 +210,26 @@ export function createGateway(options = {}) { * starts from the user agent. */ async function handle(request) { + /* + * Addresses that serve no readers are refused before anything else, with + * a body small enough that refusing costs nothing. Not 402: there is no + * pass on sale to a hosting range that spoofs a browser, because whoever + * runs it has already declined to say who they are. + */ + if (denied.length && inCidrs(clientIp(request), denied)) { + return new Response('Not available from this network.\n', { + status: 403, + headers: { 'content-type': 'text/plain; charset=utf-8', ...noStore }, + }); + } + const path = new URL(request.url).pathname; if (path === o.path) return sell(request); - if (!o.isPaidAgent(request.headers.get('user-agent') ?? '')) return null; + if (o.exempt && o.exempt(request)) return null; + const pays = + o.isPaidAgent(request.headers.get('user-agent') ?? '') || + (o.chargeSpoofedBrowsers && isSpoofedBrowser(request)); + if (!pays) return null; if (isOpen(path)) return null; const token = passFrom(request); @@ -248,6 +271,9 @@ function normalise(options) { header: String(options.header ?? 'x-crawl-pass').toLowerCase(), path: options.path ?? '/crawl', openPaths: options.openPaths ?? [], + denyCidrs: options.denyCidrs ?? [], + chargeSpoofedBrowsers: Boolean(options.chargeSpoofedBrowsers), + exempt: options.exempt ?? null, training, retrieval: options.retrieval ?? RETRIEVAL_AGENTS, isPaidAgent: options.isPaidAgent ?? ((ua) => isTrainingAgent(ua, training)), diff --git a/test/gateway.test.js b/test/gateway.test.js index 29ac568..b78e18e 100644 --- a/test/gateway.test.js +++ b/test/gateway.test.js @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; import { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from '../src/agents.js'; +import { clientIp, compileCidrs, inCidrs, isSpoofedBrowser, parseCidr } from '../src/edge.js'; import { createGateway, wantsHtml } from '../src/index.js'; import { x402Gateway } from '../src/hono.js'; import { robotsRoute, x402Proxy } from '../src/next.js'; @@ -343,6 +344,78 @@ describe('the gate', () => { }); }); +describe('crawlers that do not say who they are', () => { + const OVH = ['51.38.0.0/16', '54.38.0.0/16', '141.94.0.0/16']; + const from = (ip, extra = {}) => + req('/topics/x', { ua: CHROME, ...extra, headers: { 'x-forwarded-for': `${ip}, 10.0.0.1`, ...(extra.headers ?? {}) } }); + + it('parses CIDRs and matches addresses, and drops what it cannot read', () => { + const c = compileCidrs([...OVH, 'garbage', '1.2.3.4', '300.1.1.1/8', '10.0.0.0/33']); + assert.equal(c.length, 4); + assert.equal(inCidrs('51.38.200.7', c), true); + assert.equal(inCidrs('51.39.0.1', c), false); + assert.equal(inCidrs('1.2.3.4', c), true); + assert.equal(inCidrs('1.2.3.5', c), false); + assert.equal(inCidrs('not an ip', c), false); + assert.equal(inCidrs('', c), false); + assert.equal(parseCidr('0.0.0.0/0').mask, 0); + assert.equal(inCidrs('9.9.9.9', compileCidrs(['0.0.0.0/0'])), true); + }); + + it('reads the client address the way the edge writes it', () => { + assert.equal(clientIp(req('/', { headers: { 'x-forwarded-for': '203.0.113.9, 10.1.1.1' } })), '203.0.113.9'); + assert.equal(clientIp(req('/', { headers: { 'x-real-ip': '203.0.113.10' } })), '203.0.113.10'); + assert.equal(clientIp(req('/')), ''); + }); + + it('refuses a denied range with a tiny 403 before anything else, even a paying pass or the sales page', async () => { + const { gateway, cp } = gatewayFor({ denyCidrs: OVH }); + const res = await gateway.handle(from('54.38.1.2')); + assert.equal(res.status, 403); + assert.equal(res.headers.get('cache-control'), 'no-store'); + assert.equal((await gateway.handle(from('54.38.1.2', { headers: { 'x-payment': proof() } }))).status, 403); + assert.equal(cp.calls.length, 0); + assert.equal((await gateway.handle(req('/crawl', { headers: { 'x-forwarded-for': '141.94.9.9' } }))).status, 403); + // A neighbour outside the range, same UA, is a person. + assert.equal(await gateway.handle(from('51.39.0.1')), null); + }); + + it('knows a copied Chrome string from Chrome', () => { + assert.equal(isSpoofedBrowser(req('/', { ua: CHROME })), true, 'no Sec-Fetch-Mode at all'); + assert.equal(isSpoofedBrowser(req('/', { ua: CHROME, headers: { 'sec-fetch-mode': 'navigate' } })), false); + assert.equal(isSpoofedBrowser(req('/', { ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0' })), false, 'Firefox is not judged'); + assert.equal(isSpoofedBrowser(req('/', { ua: 'curl/8.0' })), false, 'an honest client is not judged either'); + assert.equal(isSpoofedBrowser(req('/', { ua: META })), false, 'a declared crawler is charged by name, not by this'); + }); + + it('charges a spoofed browser only when asked to, and never one that answers the question', async () => { + const quiet = gatewayFor().gateway; + assert.equal(await quiet.handle(req('/topics/x', { ua: CHROME })), null, 'off by default'); + + const { gateway } = gatewayFor({ chargeSpoofedBrowsers: true }); + const res = await gateway.handle(req('/topics/x', { ua: CHROME, accept: 'application/json' })); + assert.equal(res.status, 402); + assert.equal((await res.json()).accepts.length, 3); + const real = req('/topics/x', { ua: CHROME, headers: { 'sec-fetch-mode': 'navigate', 'sec-fetch-site': 'none' } }); + assert.equal(await gateway.handle(real), null); + // A spoofed browser that pays gets a pass like anyone else. + const paid = await gateway.handle(req('/topics/x', { ua: CHROME, headers: { 'x-payment': proof({ nonce: '0xs1' }) } })); + assert.equal(paid.status, 200); + const { pass } = await paid.json(); + assert.equal(await gateway.handle(req('/topics/y', { ua: CHROME, headers: { 'x-crawl-pass': pass } })), null); + }); + + it('exempts what the site says to exempt, before any charge', async () => { + const { gateway } = gatewayFor({ + chargeSpoofedBrowsers: true, + exempt: (r) => (r.headers.get('cookie') ?? '').includes('signed_in=1'), + }); + assert.equal(await gateway.handle(req('/topics/x', { ua: CHROME, headers: { cookie: 'signed_in=1' } })), null); + assert.equal(await gateway.handle(req('/topics/x', { ua: META, headers: { cookie: 'signed_in=1' } })), null, 'even a named crawler with the cookie'); + assert.equal((await gateway.handle(req('/topics/x', { ua: META }))).status, 402); + }); +}); + describe('adapters', () => { it('Hono: returns the gateway response or calls next', async () => { const { gateway } = gatewayFor();