diff --git a/README.md b/README.md index 0c533fe..f7a23cc 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,61 @@ export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] }); Without `coinpay.apiKey` and `payTo` the gateway still answers training crawlers with 402 and the page says payments are off. Nothing is sold, but nothing is given away either. +## A free allowance that sells instead of refusing + +Named crawlers are one problem. The other is an ordinary client hammering the +site far past what a reader would. The usual answer is `429 Too Many Requests`, +which tells them to go away. That is the right answer when you have nothing to +sell. Here you do. + +```js +createGateway({ + siteUrl: 'https://your-site.com', + coinpay: { apiKey: process.env.COINPAY_X402_KEY }, + payTo: process.env.CRAWL_PAY_TO, + + freeQuota: 100, // 100 requests a minute, free, no account + benefits: [ + 'No rate limit', + 'Bulk export endpoint', + 'Every field, not just the summary', + ], +}); +``` + +Past the allowance the gateway answers `402` with the offer, the RateLimit +headers, and a body that says what ran out, when it comes back, and what a pass +costs. The moment a caller runs out of free requests is the best sales pitch the +site will ever get: it has just demonstrated it wants more than the free tier +and is still holding the request. + +`freeQuota` takes `{ requests, windowSeconds, paths, identify, store }`. A pass +is checked before the allowance, so a paying caller is never metered. The sales +page, `robots.txt` and `.well-known/` stay reachable when the allowance is gone, +because being unable to reach the page that sells the fix would be the worst +possible failure of a throttle that exists to sell something. + +### On rotating addresses to get around it + +The default identity is the caller's address, and a proxy rotation defeats it. +That is not a hole to be patched, because the arithmetic already argues for +paying: + +- Residential proxy bandwidth is sold **by the gigabyte**, and a crawl big + enough to be worth rotating for passes a dollar on the first day. +- A rotation still fetches every page one at a time. It buys no speed. +- A pass is a flat price with nothing to maintain and nothing to keep working. + +So the throttled page makes that case in as many words rather than pretending to +be undefeatable. Detection is a race you re-run every time someone changes +tactics. Price is not: the better your free tier and the clearer your paid one, +the less anyone bothers. Set `benefits` to the things a rotation genuinely +cannot get, a bulk endpoint above all, and evasion stops being worth the effort +rather than being blocked. + +Where you *can* identify a caller properly, do: pass `identify` and key on an API +key or an account, and the allowance becomes exact. + ## Crawlers that do not say who they are The lists catch crawlers that name themselves. Two do not: a VPS fleet wearing a browser string, and a residential-proxy rotation cycling a few Chrome strings across hundreds of addresses. `denyCidrs` handles the first (`['51.38.0.0/16', '54.38.0.0/16', …]` for one provider's ranges); `chargeSpoofedBrowsers` handles both by asking a question only a browser can answer. A request that answers it is left alone. One that cannot gets the same 402 as GPTBot, which costs the site a hash instead of a render. diff --git a/index.d.ts b/index.d.ts index 8e3eeac..d2d11d4 100644 --- a/index.d.ts +++ b/index.d.ts @@ -74,9 +74,51 @@ export interface GatewayOptions { contact?: string; /** Awaited before the buyer's receipt is sent, and its errors are swallowed: a sale is recorded, and a recording failure never costs the buyer the pass. */ onSale?: (sale: Sale) => void | Promise; + /** + * A free allowance for ordinary callers, after which the gateway answers 402 + * with the offer instead of letting the request through. A bare number means + * that many requests per minute. A valid pass is never metered. + */ + freeQuota?: number | FreeQuotaOptions; + /** What a pass unlocks, listed on the sales page and in the 402 body. */ + benefits?: string[]; fetch?: typeof fetch; } +export interface QuotaHit { + /** Requests in the current window, including this one. */ + count: number; + /** Seconds until the window rolls over. */ + resetSeconds: number; +} + +/** Somewhere to count requests. Supply one to share an allowance across a fleet. */ +export interface QuotaStore { + hit(key: string, windowSeconds: number): QuotaHit | Promise; +} + +export interface FreeQuotaOptions { + /** Free requests per window. */ + requests: number; + /** Window length. Default 60. */ + windowSeconds?: number; + /** + * What to count against. Defaults to the caller's address, which a rotation + * defeats; that is answered with price rather than detection, see the README. + */ + identify?: (request: Request) => string | null; + /** Default: an in-process counter, so each instance grants its own allowance. */ + store?: QuotaStore; + /** Only meter these paths or prefixes. Default: everything the gate sees. */ + paths?: string[]; +} + +/** An in-process fixed-window counter. */ +export function memoryQuotaStore(options?: { + now?: () => number; + sweepEvery?: number; +}): QuotaStore & { sweep(): void; readonly size: number }; + export interface AcceptEntry { scheme: 'exact'; network: string; diff --git a/package.json b/package.json index 1a322ce..ec0b04c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/x402-gateway", - "version": "0.3.1", + "version": "0.4.0", "type": "module", "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, and a robots.txt that keeps search crawlers welcome.", "keywords": [ diff --git a/src/index.js b/src/index.js index 6c8864e..687a8a2 100644 --- a/src/index.js +++ b/src/index.js @@ -1,5 +1,6 @@ import { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from './agents.js'; import { clientIp, compileCidrs, inCidrs, isSpoofedBrowser } from './edge.js'; +import { memoryQuotaStore, meters, normaliseQuota, quotaHeaders, spend } from './quota.js'; import { renderPage } from './page.js'; import { mintPass, readPass } from './pass.js'; import { robotsTxt } from './robots.js'; @@ -20,6 +21,7 @@ export { renderPage } from './page.js'; export { mintPass, readPass } from './pass.js'; export { robotsTxt } from './robots.js'; export { buildOffer, decodePayment, expectedFor, METHODS, verifyAndSettle } from './x402.js'; +export { memoryQuotaStore, normaliseQuota, quotaHeaders, spend } from './quota.js'; /** * A gateway that sells crawl access to training crawlers, by the day, over x402. @@ -119,10 +121,23 @@ export function createGateway(options = {}) { status, headers: { 'content-type': 'application/json; charset=utf-8', ...noStore, ...headers }, }); - const html = (body, status) => - new Response(body, { status, headers: { 'content-type': 'text/html; charset=utf-8', ...noStore } }); + const html = (body, status, headers = {}) => + new Response(body, { + status, + headers: { 'content-type': 'text/html; charset=utf-8', ...noStore, ...headers }, + }); - const pageCtx = (days = 1) => ({ + const pageCtx = (days = 1, usage = null) => ({ + quota: o.freeQuota + ? { + requests: o.freeQuota.requests, + windowSeconds: o.freeQuota.windowSeconds, + used: usage?.count ?? null, + resetSeconds: usage?.resetSeconds ?? null, + exceeded: Boolean(usage?.overLimit), + } + : null, + benefits: o.benefits, days, total: money(o.priceCents * days), siteName: o.siteName, @@ -155,10 +170,14 @@ export function createGateway(options = {}) { * body and not the headers, and a crawler that wanted the page can fetch it * again a moment later with the pass. */ - async function sell(request) { + async function sell(request, context = {}) { const ua = request.headers.get('user-agent') ?? ''; const proofHeader = request.headers.get('x-payment'); const asked = daysFrom(request); + // Present when the free allowance is what stopped this request, rather than + // the crawler lists. It changes what the 402 says, not what it costs. + const usage = context.usage ?? null; + const rateHeaders = usage ? quotaHeaders(o.freeQuota, usage) : {}; if (proofHeader) { if (!enabled) return json(receipt(asked, { error: 'Payments are not switched on here.' }), 402); @@ -252,7 +271,33 @@ export function createGateway(options = {}) { ); } - if (wantsHtml(request.headers.get('accept'))) return html(o.page(pageCtx(asked)), 402); + if (wantsHtml(request.headers.get('accept'))) { + return html(o.page(pageCtx(asked, usage)), 402, rateHeaders); + } + + if (usage) { + // Say what ran out, when it comes back, and what a pass costs, in that + // order. A caller reading this is deciding between waiting, rotating + // addresses, and paying, and the numbers are the argument. + return json( + receipt(asked, { + error: + `Free allowance used: ${o.freeQuota.requests} requests per ` + + `${o.freeQuota.windowSeconds}s. It resets in ${usage.resetSeconds}s. ` + + `A pass removes the limit for ${price} a day.`, + quota: { + requests: o.freeQuota.requests, + windowSeconds: o.freeQuota.windowSeconds, + used: usage.count, + resetSeconds: usage.resetSeconds, + }, + ...(o.benefits ? { unlocks: o.benefits } : {}), + }), + 402, + rateHeaders, + ); + } + return json(receipt(asked, { error: `Payment required for training crawlers. Read ${buyUrl} for how.` }), 402); } @@ -280,15 +325,41 @@ export function createGateway(options = {}) { const path = new URL(request.url).pathname; if (path === o.path) return sell(request); if (o.exempt && o.exempt(request)) return null; + + /* + * A valid pass is checked before anything else that could refuse, because + * a pass is the thing being sold: whoever holds one is neither charged as + * a crawler nor metered against the free allowance. It used to be read + * only after the crawler lists matched, which was fine while the lists + * were the only reason to refuse and is not now that a quota exists. + */ + const token = passFrom(request); + const paid = Boolean(token && (await readPass(token, { secret }))); + if (paid) return null; + const pays = o.isPaidAgent(request.headers.get('user-agent') ?? '') || (o.chargeSpoofedBrowsers && isSpoofedBrowser(request)); - if (!pays) return null; + if (pays && !isOpen(path)) return sell(request); if (isOpen(path)) return null; - const token = passFrom(request); - if (token && (await readPass(token, { secret }))) return null; - return sell(request); + /* + * Everyone else gets the free allowance. Running out is answered with a + * price rather than a 429: the caller has just shown it wants more than + * the free tier and is still holding the request, which is the best moment + * this site will ever get to sell it a pass. + */ + if (o.freeQuota && meters(o.freeQuota, path)) { + const key = o.freeQuota.identify ? o.freeQuota.identify(request) : clientIp(request); + const usage = await spend(o.freeQuota, key); + // Under the limit the request carries on untouched. `handle` answers with + // a Response or nothing at all, and quietly growing that contract to + // smuggle headers out would break every adapter that checks it for truth. + // The allowance is advertised on the 402, which is where it is read. + if (usage?.overLimit) return sell(request, { usage }); + } + + return null; } return { @@ -372,6 +443,8 @@ function normalise(options) { page: options.page ?? renderPage, contact: options.contact ?? '', onSale: options.onSale ?? null, + freeQuota: normaliseQuota(options.freeQuota), + benefits: Array.isArray(options.benefits) ? options.benefits : null, fetch: options.fetch ?? globalThis.fetch, }; } diff --git a/src/page.js b/src/page.js index 0017006..0888174 100644 --- a/src/page.js +++ b/src/page.js @@ -58,7 +58,43 @@ export function renderPage(ctx) { days = 1, total = price, maxDays = 30, + quota = null, + benefits = null, } = ctx; + + /* + * Two audiences reach this page and they need different first sentences. A + * training crawler is here because it is on a list. A heavy reader is here + * because it ran out of the free allowance, and telling that one it is a + * training crawler is both wrong and insulting. The price is the same; the + * argument is not. + */ + const throttled = Boolean(quota?.exceeded); + const headline = throttled + ? 'You have used up the free allowance.' + : 'Training crawlers pay for access here.'; + const opening = throttled + ? `

${esc(String(quota.requests))} requests every ${esc(String(quota.windowSeconds))} seconds are free, no key and no account, and that is not changing. ` + + `You have gone past it${quota.resetSeconds ? `, and it resets in ${esc(String(quota.resetSeconds))} seconds` : ''}. ` + + `A pass lifts the limit rather than waiting it out.

` + : `

People read ${esc(siteName)} free. So do search engines and the retrieval crawlers behind AI answers, because they send readers back. A crawler that copies pages into a training corpus sends nobody back, so it pays for the time it spends.

`; + + /* + * Said plainly because it is the honest argument and the one that actually + * lands. Rotating addresses to dodge a free-tier limit is not free: the + * bandwidth is metered and billed by the gigabyte, and a crawl big enough to + * be worth rotating for costs more in proxies than the pass does. Anyone + * weighing the two should be able to see that from here. + */ + const arithmetic = throttled + ? `

Before you reach for a proxy pool

+

Spreading the same crawl over rotating addresses works, and it is the expensive way to do this. Residential bandwidth is sold by the gigabyte, you still fetch every page one at a time, and the bill starts on the first day. A pass is ${esc(price)} a day, flat, with no rotation to maintain and nothing to keep working. We would rather sell you access than play that game, which is why the limit answers with a price instead of a refusal.

` + : ''; + + const unlocks = + benefits && benefits.length + ? `

What a pass gets you

\n` + : ''; const window = minutes === 1440 ? 'one day' @@ -82,8 +118,8 @@ export function renderPage(ctx) {
-

Training crawlers pay for access here.

-

People read ${esc(siteName)} free. So do search engines and the retrieval crawlers behind AI answers, because they send readers back. A crawler that copies pages into a training corpus sends nobody back, so it pays for the time it spends.

+

${esc(headline)}

+${opening}
${esc(days > 1 ? total : price)} for ${esc(days > 1 ? `${days} × ${window}` : window)} of requests
${ @@ -97,6 +133,9 @@ ${ : '

Payments are not switched on here yet. The offer below is empty until the operator configures a payout address, so for now this crawler is simply refused.

' } +${unlocks} +${arithmetic} +

How it works

  1. Any page you fetch answers 402 Payment Required. This page, fetched with Accept: application/json, returns the x402 offer: USDC, exact scheme, on ${esc(networks || 'Base, Polygon or Ethereum')}.
    2. diff --git a/src/quota.js b/src/quota.js new file mode 100644 index 0000000..e106825 --- /dev/null +++ b/src/quota.js @@ -0,0 +1,149 @@ +/** + * A free allowance, and what happens when it runs out. + * + * The usual answer to "too many requests" is 429, which tells a caller to go + * away and try later. That is the right answer when you have nothing to sell. + * Here we do: the moment the allowance runs out is the best sales pitch the + * site will ever get, because the caller has just demonstrated it wants more + * than the free tier and is still holding the request. So the gateway answers + * 402 with a price instead. + * + * ON EVADING THIS BY ROTATING ADDRESSES. + * + * The default identity is the caller's address, and rotating addresses defeats + * it. That is not a flaw to be patched, because the arithmetic already argues + * for paying: residential proxy bandwidth is sold by the gigabyte at prices + * that pass a dollar inside the first day of any serious crawl, and a rotation + * still has to fetch every page one at a time. Someone who spends more on + * proxies than the pass costs, in order to avoid the pass, has not been + * defeated by cleverness. They have made an arithmetic mistake, and the 402 + * body is where we point it out. Detection is a race. Price is not. + * + * The store is pluggable so a fleet can share one counter across sites. The + * built-in one counts per process, so several instances each grant their own + * allowance; that errs toward generosity, which is the right direction to be + * wrong in for a free tier. + */ + +/** + * @typedef {object} QuotaHit + * @property {number} count requests in the current window, including this one + * @property {number} resetSeconds seconds until the window rolls over + */ + +/** + * @typedef {object} QuotaStore + * @property {(key: string, windowSeconds: number) => QuotaHit | Promise} hit + */ + +/** + * A counter in this process's memory. + * + * Fixed windows rather than a sliding log: a sliding window is more accurate + * and has to keep every timestamp, and for deciding whether to show someone a + * price that accuracy is not worth the memory. Expired entries are swept + * occasionally on write, so an idle key does not cost anything for long. + */ +export function memoryQuotaStore({ now = () => Date.now(), sweepEvery = 1000 } = {}) { + /** @type {Map} */ + const windows = new Map(); + let writes = 0; + + const sweep = (at) => { + for (const [key, entry] of windows) { + if (entry.expiresAt <= at) windows.delete(key); + } + }; + + return { + hit(key, windowSeconds) { + const at = now(); + writes += 1; + if (writes % sweepEvery === 0) sweep(at); + + const entry = windows.get(key); + if (!entry || entry.expiresAt <= at) { + windows.set(key, { count: 1, expiresAt: at + windowSeconds * 1000 }); + return { count: 1, resetSeconds: windowSeconds }; + } + entry.count += 1; + return { + count: entry.count, + resetSeconds: Math.max(1, Math.ceil((entry.expiresAt - at) / 1000)), + }; + }, + /** Test seam, and a way for a long-lived process to reclaim memory on demand. */ + sweep() { + sweep(now()); + }, + get size() { + return windows.size; + }, + }; +} + +/** + * Normalise what a site passes as `freeQuota`. + * + * A bare number means "that many per minute", because per minute is how + * everyone states a rate limit out loud. + */ +export function normaliseQuota(input) { + if (!input) return null; + const config = typeof input === 'number' ? { requests: input } : input; + const requests = Number(config.requests); + if (!Number.isFinite(requests) || requests < 1) return null; + + return { + requests: Math.floor(requests), + windowSeconds: + Number.isFinite(config.windowSeconds) && config.windowSeconds > 0 + ? Math.floor(config.windowSeconds) + : 60, + identify: typeof config.identify === 'function' ? config.identify : null, + store: + config.store && typeof config.store.hit === 'function' ? config.store : memoryQuotaStore(), + /** Only meter these paths or prefixes. Null meters everything the gate sees. */ + paths: Array.isArray(config.paths) && config.paths.length ? config.paths : null, + }; +} + +/** Whether this path is metered at all. */ +export function meters(quota, path) { + if (!quota.paths) return true; + return quota.paths.some((p) => (p.endsWith('/') ? path.startsWith(p) : path === p)); +} + +/** + * Spend one request against the allowance. + * + * Null when the caller cannot be identified, which counts as within the + * allowance. A caller we cannot count is not evidence of abuse, and charging + * one because our own edge did not hand us an address would be charging for our + * own gap. + */ +export async function spend(quota, key) { + if (!key) return null; + const hit = await quota.store.hit(key, quota.windowSeconds); + return { + count: hit.count, + remaining: Math.max(0, quota.requests - hit.count), + resetSeconds: hit.resetSeconds, + overLimit: hit.count > quota.requests, + }; +} + +/** + * Headers describing the allowance, in the shape the draft IETF RateLimit + * fields use. Sent whether or not the caller is over, because a client that can + * see it is approaching a wall can buy a pass before it hits one, which is a + * better outcome for both sides than a surprise. + */ +export function quotaHeaders(quota, usage) { + if (!usage) return {}; + return { + 'ratelimit-limit': String(quota.requests), + 'ratelimit-remaining': String(usage.remaining), + 'ratelimit-reset': String(usage.resetSeconds), + }; +} diff --git a/test/quota.test.js b/test/quota.test.js new file mode 100644 index 0000000..fa154b9 --- /dev/null +++ b/test/quota.test.js @@ -0,0 +1,268 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { createGateway, mintPass } from '../src/index.js'; +import { memoryQuotaStore, normaliseQuota, spend } from '../src/quota.js'; + +const SITE = 'https://example.com'; +const SECRET = 'cp_live_test_secret_0123456789'; +const PAY_TO = '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5'; +const CHROME = + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'; + +const reader = (ip, extra = {}) => + new Request(`${SITE}/topics/anything`, { + headers: { 'user-agent': CHROME, 'sec-fetch-mode': 'navigate', 'x-real-ip': ip, ...extra }, + }); + +const gateway = (freeQuota, extra = {}) => + createGateway({ + siteUrl: SITE, + coinpay: { apiKey: SECRET }, + payTo: PAY_TO, + freeQuota, + ...extra, + }); + +describe('normaliseQuota', () => { + it('reads a bare number as requests per minute', () => { + const quota = normaliseQuota(100); + assert.equal(quota.requests, 100); + assert.equal(quota.windowSeconds, 60); + }); + + it('is off for anything that is not a usable allowance', () => { + for (const input of [undefined, null, 0, -5, {}, { requests: 'lots' }]) { + assert.equal(normaliseQuota(input), null, JSON.stringify(input)); + } + }); +}); + +describe('memoryQuotaStore', () => { + it('counts within a window and rolls over after it', () => { + let clock = 0; + const store = memoryQuotaStore({ now: () => clock }); + assert.equal(store.hit('a', 60).count, 1); + assert.equal(store.hit('a', 60).count, 2); + clock += 60_000; + assert.equal(store.hit('a', 60).count, 1, 'a new window starts at one'); + }); + + it('counts each key separately', () => { + const store = memoryQuotaStore(); + store.hit('a', 60); + assert.equal(store.hit('b', 60).count, 1); + }); + + it('reports a reset that shrinks as the window runs down', () => { + let clock = 0; + const store = memoryQuotaStore({ now: () => clock }); + assert.equal(store.hit('a', 60).resetSeconds, 60); + clock += 30_000; + assert.equal(store.hit('a', 60).resetSeconds, 30); + }); + + it('forgets expired keys rather than growing without bound', () => { + let clock = 0; + const store = memoryQuotaStore({ now: () => clock }); + for (let i = 0; i < 50; i += 1) store.hit(`key-${i}`, 60); + assert.equal(store.size, 50); + clock += 61_000; + store.sweep(); + assert.equal(store.size, 0); + }); +}); + +describe('spend', () => { + it('treats an unidentifiable caller as within the allowance', async () => { + // Refusing here would charge someone for our own edge not giving us an + // address, which is our gap and not their abuse. + assert.equal(await spend(normaliseQuota(1), ''), null); + assert.equal(await spend(normaliseQuota(1), null), null); + }); + + it('goes over only once the allowance is actually exceeded', async () => { + const quota = normaliseQuota({ requests: 2, windowSeconds: 60 }); + assert.equal((await spend(quota, 'ip')).overLimit, false); + assert.equal((await spend(quota, 'ip')).overLimit, false, 'the last free one is still free'); + assert.equal((await spend(quota, 'ip')).overLimit, true); + }); + + it('reports what is left, never going below zero', async () => { + const quota = normaliseQuota({ requests: 1, windowSeconds: 60 }); + assert.equal((await spend(quota, 'ip')).remaining, 0); + assert.equal((await spend(quota, 'ip')).remaining, 0); + }); +}); + +describe('the gate with a free allowance', () => { + it('lets an ordinary reader through until the allowance runs out', async () => { + const gw = gateway({ requests: 3, windowSeconds: 60 }); + for (let i = 0; i < 3; i += 1) { + assert.equal(await gw.handle(reader('1.2.3.4')), null, `request ${i + 1} should pass`); + } + const refused = await gw.handle(reader('1.2.3.4')); + assert.equal(refused.status, 402, 'the fourth is answered with a price'); + }); + + it('answers 402 and not 429, because there is something to sell', async () => { + const gw = gateway(1); + await gw.handle(reader('1.2.3.4')); + const answer = await gw.handle(reader('1.2.3.4')); + assert.equal(answer.status, 402); + const body = await answer.json(); + assert.equal(body.x402Version, 2); + assert.ok(body.accepts.length > 0, 'the offer is right there in the refusal'); + }); + + it('says what ran out, when it returns, and what it costs', async () => { + const gw = gateway({ requests: 1, windowSeconds: 60 }); + await gw.handle(reader('1.2.3.4')); + const body = await (await gw.handle(reader('1.2.3.4'))).json(); + assert.equal(body.quota.requests, 1); + assert.equal(body.quota.windowSeconds, 60); + assert.equal(body.quota.used, 2); + assert.match(body.error, /Free allowance used/); + assert.match(body.error, /1\.00 USD a day/); + }); + + it('sends the RateLimit headers on the refusal', async () => { + const gw = gateway(1); + await gw.handle(reader('1.2.3.4')); + const answer = await gw.handle(reader('1.2.3.4')); + assert.equal(answer.headers.get('ratelimit-limit'), '1'); + assert.equal(answer.headers.get('ratelimit-remaining'), '0'); + assert.ok(Number(answer.headers.get('ratelimit-reset')) > 0); + }); + + it('counts each address separately, which is the part a rotation exploits', async () => { + // Stated as a test because it is the known limit of the mechanism, not a + // bug: the answer to rotation is the price, not a cleverer counter. + const gw = gateway(1); + await gw.handle(reader('1.1.1.1')); + assert.equal(await gw.handle(reader('2.2.2.2')), null); + }); + + it('lists what a pass unlocks when the site says so', async () => { + const gw = gateway(1, { benefits: ['No rate limit', 'Bulk export endpoint'] }); + await gw.handle(reader('1.2.3.4')); + const body = await (await gw.handle(reader('1.2.3.4'))).json(); + assert.deepEqual(body.unlocks, ['No rate limit', 'Bulk export endpoint']); + }); +}); + +describe('a pass lifts the allowance', () => { + it('never throttles a request holding a valid pass', async () => { + const gw = gateway(1); + const now = Math.floor(Date.now() / 1000); + const { token } = await mintPass({ secret: SECRET, ref: null, expiresAt: now + 3600, now }); + + // Far past the free allowance, and every one of them is let through. + for (let i = 0; i < 25; i += 1) { + const answer = await gw.handle(reader('1.2.3.4', { 'x-crawl-pass': token })); + assert.equal(answer, null, `paid request ${i + 1} should pass`); + } + }); + + it('still throttles once the pass is not presented', async () => { + const gw = gateway(1); + const now = Math.floor(Date.now() / 1000); + const { token } = await mintPass({ secret: SECRET, ref: null, expiresAt: now + 3600, now }); + await gw.handle(reader('9.9.9.9', { 'x-crawl-pass': token })); + await gw.handle(reader('9.9.9.9')); + assert.equal((await gw.handle(reader('9.9.9.9'))).status, 402); + }); +}); + +describe('what the allowance does not touch', () => { + it('leaves the sales page reachable when the allowance is spent', async () => { + // Being unable to reach the page that sells the fix would be the worst + // possible failure of a throttle that exists to sell something. + const gw = gateway(1); + await gw.handle(reader('1.2.3.4')); + await gw.handle(reader('1.2.3.4')); + const page = await gw.handle( + new Request(`${SITE}/crawl`, { headers: { 'x-real-ip': '1.2.3.4', accept: 'text/html' } }), + ); + assert.equal(page.status, 402); + assert.match(await page.text(), /Crawl access/); + }); + + it('leaves robots.txt readable', async () => { + const gw = gateway(1); + await gw.handle(reader('1.2.3.4')); + await gw.handle(reader('1.2.3.4')); + const answer = await gw.handle( + new Request(`${SITE}/robots.txt`, { headers: { 'x-real-ip': '1.2.3.4' } }), + ); + assert.equal(answer, null); + }); + + it('meters only the paths a site names', async () => { + const gw = gateway({ requests: 1, paths: ['/api/'] }); + await gw.handle(new Request(`${SITE}/api/x`, { headers: { 'x-real-ip': '5.5.5.5' } })); + const metered = await gw.handle( + new Request(`${SITE}/api/y`, { headers: { 'x-real-ip': '5.5.5.5' } }), + ); + assert.equal(metered.status, 402); + const unmetered = await gw.handle( + new Request(`${SITE}/about`, { headers: { 'x-real-ip': '5.5.5.5' } }), + ); + assert.equal(unmetered, null); + }); + + it('is off entirely when no allowance is configured', async () => { + const gw = gateway(undefined); + for (let i = 0; i < 50; i += 1) { + assert.equal(await gw.handle(reader('1.2.3.4')), null); + } + }); + + it('still charges a training crawler on its first request', async () => { + // The allowance is for readers. A named training crawler is charged by the + // list, and a generous free tier must not become a loophole for it. + const gw = gateway(1000); + const answer = await gw.handle( + new Request(`${SITE}/`, { headers: { 'user-agent': 'GPTBot/1.2', 'x-real-ip': '1.2.3.4' } }), + ); + assert.equal(answer.status, 402); + }); +}); + +describe('the page a throttled reader sees', () => { + it('does not call a heavy reader a training crawler', async () => { + const gw = gateway(1); + const html = () => + gw.handle( + new Request(`${SITE}/topics/x`, { + headers: { + 'user-agent': CHROME, + 'sec-fetch-mode': 'navigate', + 'x-real-ip': '1.2.3.4', + accept: 'text/html', + }, + }), + ); + await html(); + const page = await (await html()).text(); + assert.match(page, /used up the free allowance/); + assert.doesNotMatch(page, /Training crawlers pay for access here/); + }); + + it('makes the case against a proxy pool in plain arithmetic', async () => { + const gw = gateway(1); + const html = () => + gw.handle( + new Request(`${SITE}/topics/x`, { + headers: { + 'user-agent': CHROME, + 'sec-fetch-mode': 'navigate', + 'x-real-ip': '1.2.3.4', + accept: 'text/html', + }, + }), + ); + await html(); + const page = await (await html()).text(); + assert.match(page, /sold by the gigabyte/); + }); +});