From 4f7355586e51717f285b195c1bf2278d8a49cf90 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 7 Sep 2026 02:17:28 +0000 Subject: [PATCH 1/2] Ask the callers with no name to do some arithmetic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every limit this site had is keyed on who is asking. crawlThrottle meters a caller by address and user-agent token, tiers.js sorts them onto rungs, and the gateway's spoof check asks whether a thing claiming Chrome sends the headers Chrome cannot suppress. All three are beaten by the same move, and it is the move the traffic makes: ask from somewhere else, every time. Measured during the 2026-09-07 outage: 500 requests in 3.5 seconds from 500 distinct addresses, no address twice, two Chrome strings between them, and all 500 paths different across 329 topic slugs. A per-caller limit sees five hundred callers with one request each and refuses nobody; a cache is never asked the same question twice and hits nothing; the fleet already sends Sec-Fetch-Mode, so the spoof check waves it through. So this rung is keyed on what the caller can do. Before an expensive page is rendered, an anonymous caller is asked to find a number whose SHA-256 opens with 18 zero bits. Verifying is one hash and costs us nothing. The answer is bound to the address that solved it, which is the part the rotation cannot help with: five hundred addresses now means five hundred solves. Honest about what it is worth: not a wall. A real JavaScript engine can pay this, so against headless browsers it is a tax rather than a refusal. What it stops outright is the larger population that replays headers with no engine behind them. workers.js makes the same point about capacity; the two together are a slope, not a gate. Who is never asked, because each of these would be a bug rather than a policy: - crawl-gateway's exempt() is reused rather than copied, so a reader with a session, a program with an API key, and a named retrieval or search crawler pass exactly as they do at the 402. - The API is never challenged. apiguard.js argues that case already and it is right. A scraper pushed onto /api/topics/* reads 5 KB of JSON instead of rendering 47 KB of HTML, which is the trade we want. - Feed exports are never challenged. A challenge on a .rss breaks every subscriber silently and permanently. - Only /topics/*, /authors/* and */read, which was 400 of the 500 requests. Off unless switched on. CHALLENGE_ENABLED defaults to off and without CHALLENGE_SECRET it stays off however it is set, because a forgeable token costs every reader a second and costs the fleet nothing. This is the one limit here that a reader can see, so it arrives when someone decides the traffic is worth interrupting people for, not with a deploy. Two things the measurements changed: The hash is written out rather than called through crypto.subtle. A promise per hash is the dominant cost — Web Crypto managed 67k hashes/s end to end, which would have made 18 bits an eight-second wait. The first hand-written version was worse still at 17k/s, because applying SHA-256's padding through a function call per byte costs a thousand calls a block; the solver now owns the padding and rebuilds it on the ten occasions the nonce gains a digit. That reads 610k hashes/s, so 18 bits is about 0.43s, and it solved in 106ms against the real server. The benchmark itself had to leave the vm sandbox. Typed arrays across a vm boundary defeat the JIT: the same text does 610k hashes/s under new Function and 10k inside vm.runInNewContext, so a sandboxed benchmark would have condemned a solver that is fast in the browser it is written for. Verified end to end against a real server: 503 with a 5.7 KB interstitial in place of a 35.7 KB render, solved in 106ms, the token admits, the same token from another address is refused, and one solve carries across pages. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KPvk8mVEpxWwTRFby9m8VT --- apps/web/src/lib/challenge.js | 564 ++++++++++++++++++++++++++++++++ apps/web/src/proxy.js | 22 +- apps/web/test/challenge.test.js | 356 ++++++++++++++++++++ 3 files changed, 941 insertions(+), 1 deletion(-) create mode 100644 apps/web/src/lib/challenge.js create mode 100644 apps/web/test/challenge.test.js diff --git a/apps/web/src/lib/challenge.js b/apps/web/src/lib/challenge.js new file mode 100644 index 0000000..c1bd329 --- /dev/null +++ b/apps/web/src/lib/challenge.js @@ -0,0 +1,564 @@ +/** + * A question only something running JavaScript can answer. + * + * ## Why the rungs below this one do not reach + * + * Every limit this site had before is keyed on *who is asking*. + * `crawlThrottle.js` meters a caller by address and user-agent token; + * `tiers.js` sorts them onto rungs; the gateway's `chargeSpoofedBrowsers` asks + * whether a thing claiming Chrome sends the headers Chrome cannot suppress. + * All three are defeated by the same move, and it is the move the traffic + * actually makes: ask from somewhere else, every time. + * + * Measured on 2026-09-07: 500 requests in 3.5 seconds from **500 distinct + * addresses**, no address twice, two Chrome user-agent strings between them, + * and every one of the 500 paths different — 329 distinct topic slugs. A + * per-caller limit sees five hundred callers with one request each and refuses + * nobody. A cache is never asked the same question twice and hits nothing. The + * fleet already sends `Sec-Fetch-Mode`, so the spoof check waves it through + * (see `rssamplifier-residential-proxy-fleet`). Rotating the address is cheap + * and there is no rung that charges for it. + * + * So this one is keyed on *what the caller can do*. Before an expensive page is + * rendered, the caller is asked to find a number whose hash starts with a run + * of zero bits. Verifying the answer is one hash and costs us nothing. Finding + * it costs the caller a fraction of a second of real CPU, and — this is the + * part the address rotation cannot help with — it has to be paid **per + * address**, because the answer is bound to the address that solved it. + * Rotating through five hundred addresses now means solving five hundred times. + * + * ## What it is honestly worth + * + * Not a wall. A client that runs a real JavaScript engine can pay this; the + * fleet may well be headless browsers, in which case what it buys is a tax + * rather than a refusal, and the tax is one solve per address per hour rather + * than per request. What it stops outright is the much larger population that + * replays headers without an engine behind them. Both outcomes are worth + * having and neither is a victory: `workers.js` makes the same point about + * capacity, and the two together are a slope, not a gate. + * + * ## Who is never asked + * + * `crawl-gateway.js` already names the callers who have said who they are — a + * reader with a session, a program with an API key, a retrieval crawler or + * search engine on the lists — and `exempt()` is reused here unchanged rather + * than copied, so the two can never drift into disagreeing about who is + * welcome. Training crawlers never reach this code: the gate answers them 402 + * first. + * + * Three more exclusions, each of which would be a bug rather than a policy: + * + * - **The API is never challenged.** `apiguard.js` carries a written decision + * against gating it, and that decision is right: a metered-but-open API can + * grow a paid tier by raising a number, and a gated one has already broken + * every agent that reads this directory today. A scraper pushed onto + * `/api/topics/*` is a scraper reading 5 KB of JSON instead of rendering + * 47 KB of HTML, which is the trade we want anyway. + * - **Feed exports are never challenged.** `.rss`, `.atom`, `.json`, `.opml`, + * `.m3u`, `.pls` are what feed readers subscribe to. A challenge there + * breaks every subscriber silently and permanently. + * - **Only HTML page routes under /topics and /authors, and the reader.** + * That was 400 of the 500 requests in the sample. The rest of the site is + * cheap enough not to be worth the interruption. + * + * ## Off unless switched on + * + * `CHALLENGE_ENABLED` defaults to **off**, and without `CHALLENGE_SECRET` it + * stays off however it is set. This is the one limit on the site that a real + * reader can *see*, so it does not arrive with a deploy — it arrives when + * someone decides the traffic is worth interrupting people for, and it leaves + * again by unsetting one variable. + * + * Edge-clean, like `crawl-gateway.js`: Web Crypto only, no `node:` import, and + * every environment variable read through a non-literal key because Next + * inlines `process.env.NAME` at build time and the image is built without + * these values. + * + * It answers with a plain `Response` and reads the path off `request.url` + * rather than reaching for `NextResponse` and `nextUrl`. Next accepts either, + * and `next/server` has no export map plain Node can resolve — importing it is + * why proxy.js has to be tested by reading its own source back as text. This + * module is the one with the arithmetic in it, so it is the one that has to be + * exercised for real rather than grepped. + */ + +import { clientIp } from '@profullstack/x402-gateway/edge'; + +import { exempt } from './crawl-gateway.js'; + +const env = process.env; + +/** The cookie carrying a solved challenge. */ +export const CHALLENGE_COOKIE = 'rsa_pow'; + +/** How many leading zero bits the hash must have. */ +const DEFAULT_BITS = 18; + +/** How long one solution is good for. */ +const DEFAULT_TTL_MINUTES = 60; + +/** + * The most a solution may be worth, in minutes. + * + * A long-lived token is a long-lived pass for whoever solved it once, and the + * whole cost model here is "per address, per period". Beyond a day the tax + * rounds to nothing. + */ +const MAX_TTL_MINUTES = 1440; + +/** + * The routes worth interrupting someone for. + * + * The three the fleet actually walks. `/read` is matched at the end of any path + * because the reader lives at `/{slug}/read`, and it is the most expensive + * thing on the site — see `pageGate.js`. + */ +const EXPENSIVE = /^\/(?:topics|authors)\/|\/read$/; + +/** + * Anything a machine subscribes to, which must never be challenged. + * + * Matched on the extension rather than the route because the same topic is + * served at `/topics/x` and `/topics/x.rss`, and only the first is a page. + */ +const EXPORT_EXTENSION = /\.(?:rss|atom|json|opml|m3u|pls|xml|txt|csv)$/; + +/** + * An integer environment variable inside its bounds, or the fallback. + * + * Junk falls back rather than to nothing, for the reason `pageGate.js` gives — + * except that here "nothing" would mean a difficulty of zero, which is a + * challenge every caller passes without doing any work at all. A typo must not + * quietly turn the defence off while leaving the interruption in place. + * + * @param {string} name + * @param {number} fallback + * @param {number} min + * @param {number} max + * @returns {number} + */ +function envInt(name, fallback, min, max) { + const raw = Number(env[name]); + return Number.isInteger(raw) && raw >= min && raw <= max ? raw : fallback; +} + +/** How many leading zero bits an answer must have. @returns {number} */ +export function bits() { + return envInt('CHALLENGE_BITS', DEFAULT_BITS, 1, 32); +} + +/** How long a solution stays good, in milliseconds. @returns {number} */ +export function ttlMs() { + return envInt('CHALLENGE_TTL_MINUTES', DEFAULT_TTL_MINUTES, 1, MAX_TTL_MINUTES) * 60_000; +} + +/** + * The signing secret, or empty when there is none. + * + * There is deliberately no default. A shared constant would let anyone compute + * a valid token offline, which is worse than not running this at all: it would + * cost every real reader a second and cost the fleet nothing. + * + * @returns {string} + */ +function secret() { + return (env['CHALLENGE_SECRET'] ?? '').trim(); +} + +/** + * Whether the challenge is switched on. + * + * Two conditions, and the secret is the one that cannot be overridden: without + * it the tokens are forgeable, so being "enabled" would be a costume. Off is + * the safe direction for a check that stands between readers and the site. + * + * @returns {boolean} + */ +export function enabled() { + return env['CHALLENGE_ENABLED'] === '1' && secret().length > 0; +} + +/** + * Who this challenge is bound to. + * + * The address, and the user-agent alongside it so one solved token cannot be + * handed round a fleet sharing an exit address. Deliberately *not* the path: + * the reader is meant to solve once and then browse, and a per-path challenge + * would ask again on every link. + * + * The address comes from `clientIp`, which reads `X-Real-IP` and otherwise the + * LAST `X-Forwarded-For` hop — never the first, which the client writes. + * + * @param {Request} request + * @returns {string} + */ +function subject(request) { + return `${clientIp(request)}|${request.headers.get('user-agent') ?? ''}`; +} + +/** Bytes as lower-case hex. @param {ArrayBuffer} buf @returns {string} */ +function hex(buf) { + return [...new Uint8Array(buf)].map((b) => b.toString(16).padStart(2, '0')).join(''); +} + +/** + * The puzzle this caller has to solve at this moment. + * + * An HMAC over the subject and the issue time, so the server keeps no state: + * a token carries the time it was issued, and the seed it was solved against + * can always be recomputed from that. Nothing is stored, nothing expires from + * memory, and sixteen workers agree without talking to each other — which + * matters, because since 2026-09-07 there are sixteen of them. + * + * @param {Request} request + * @param {number} issuedAt + * @returns {Promise} + */ +export async function seedFor(request, issuedAt) { + const key = await crypto.subtle.importKey( + 'raw', + new TextEncoder().encode(secret()), + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign'], + ); + const mac = await crypto.subtle.sign( + 'HMAC', + key, + new TextEncoder().encode(`${subject(request)}|${issuedAt}`), + ); + return hex(mac); +} + +/** + * Whether a hash opens with `n` zero bits. + * + * Bit-wise rather than counting hex characters, so the difficulty is a smooth + * dial: every extra bit doubles the work, where every extra hex digit would + * multiply it by sixteen and leave nothing usable between "instant" and + * "unbearable". + * + * @param {Uint8Array} digest + * @param {number} n + * @returns {boolean} + */ +export function hasLeadingZeroBits(digest, n) { + const whole = n >> 3; + for (let i = 0; i < whole; i += 1) if (digest[i] !== 0) return false; + const rest = n & 7; + return rest === 0 || (digest[whole] >> (8 - rest)) === 0; +} + +/** + * Whether this request carries a solution that is this caller's, current, and + * hard enough. + * + * All three have to be checked. Age alone lets a token outlive its cost; the + * subject alone lets a solved token be passed around; the work alone lets any + * pair of numbers through. The difficulty is read now rather than taken from + * the token, so raising `CHALLENGE_BITS` re-challenges everyone rather than + * honouring answers to an easier question. + * + * @param {Request} request + * @returns {Promise} + */ +export async function solved(request) { + const cookie = request.headers.get('cookie') ?? ''; + const found = /(?:^|;\s*)rsa_pow=([^;]+)/.exec(cookie); + if (!found) return false; + + const [issued, nonce] = decodeURIComponent(found[1]).split('.'); + const issuedAt = Number(issued); + if (!Number.isInteger(issuedAt) || !nonce) return false; + + const age = Date.now() - issuedAt; + if (age < 0 || age > ttlMs()) return false; + + const seed = await seedFor(request, issuedAt); + const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(seed + nonce)); + return hasLeadingZeroBits(new Uint8Array(digest), bits()); +} + +/** + * Whether this request is one we would interrupt, before asking whether it + * has already been answered. + * + * Order matters for cost: the cheap string tests run before `exempt()` reads + * headers, and `solved()` — the only part that hashes — runs last and only for + * a request that would otherwise be refused. + * + * @param {Request} request + * @param {string} pathname + * @returns {boolean} + */ +export function wouldChallenge(request, pathname) { + if (!enabled()) return false; + if (request.method !== 'GET' && request.method !== 'HEAD') return false; + if (!EXPENSIVE.test(pathname)) return false; + if (EXPORT_EXTENSION.test(pathname)) return false; + if (pathname.startsWith('/api/')) return false; + return !exempt(request); +} + +/** + * The challenge, or nothing when this caller may pass. + * + * Returns a response only when one is owed, so the proxy reads it the same way + * it reads the gateway: an answer stops the request, silence lets it through. + * + * @param {Request} request + * @returns {Promise} + */ +export async function challenge(request) { + const url = new URL(request.url); + if (!wouldChallenge(request, url.pathname)) return null; + if (await solved(request)) return null; + + const issuedAt = Date.now(); + const seed = await seedFor(request, issuedAt); + return page(seed, issuedAt, bits(), url.pathname + url.search); +} + +/** + * The interstitial. + * + * Served 503 rather than 200, with `Retry-After`, so that anything reading + * status codes — a monitor, a search crawler that slipped past the lists, a + * feed reader following a link — treats it as "come back", not as the page. A + * 200 here would let a crawler index the interstitial as the article. + * + * Its own tight Content-Security-Policy rather than the site's: this page loads + * nothing, so it may as well say so. The site's policy allows inline script for + * the reason next.config.mjs gives, and this page needs that much and no more. + * + * @param {string} seed + * @param {number} issuedAt + * @param {number} difficulty + * @param {string} target + * @returns {Response} + */ +function page(seed, issuedAt, difficulty, target) { + return new Response(html(seed, issuedAt, difficulty, target), { + status: 503, + headers: { + 'content-type': 'text/html; charset=utf-8', + 'cache-control': 'no-store', + 'retry-after': '5', + 'x-robots-tag': 'noindex', + 'content-security-policy': + "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'", + }, + }); +} + +/** + * SHA-256, as the page computes it. + * + * Kept as source text rather than written as a function here, because it is + * shipped to the browser inside the interstitial and there is nowhere else for + * it to live: the page loads no scripts (its own Content-Security-Policy says + * `default-src 'none'`, and next.config.mjs would not let it reach a CDN + * anyway). Exported so that test/challenge.test.js can run this exact text and + * check it against Web Crypto — a hash that disagreed with the server's by one + * bit would produce a challenge nobody can ever solve, and the only symptom + * would be readers stuck on an interstitial forever while the logs said nothing. + * + * ## Why not `crypto.subtle.digest` + * + * Because it is asynchronous, and a promise per hash costs far more than the + * hash. Measured end to end against this server: Web Crypto managed about + * 67,000 hashes a second, so the 18-bit default — 262,000 hashes on average — + * would have taken a reader **eight seconds**. That difference is the whole + * design. The difficulty has to be high enough to be a real tax on a fleet + * solving once per address and low enough that a person barely notices, and + * with an async hash there is no such number. + * + * ## Why the caller owns the padding + * + * `sha256Head` takes a buffer that is *already* padded to a whole number of + * blocks and reads its words straight out of it. The first version of this + * computed each byte through a function that applied the padding on the fly, + * which was correct and managed 17,000 hashes a second — slower than the Web + * Crypto it replaced — because it made a thousand function calls per block. + * The message only changes in its last few digits between attempts, so the + * solver writes those in place and rebuilds the padding on the ten occasions + * the nonce gains a digit. + * + * Only the first word of the digest is returned. The difficulty is capped at + * 32 bits, so nothing past it is ever looked at. + */ +export const SHA256_JS = ` +var K = [ + 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +]; +var W = new Int32Array(64); + +/* The first word of SHA-256(b), where b is already padded to \`blocks\` bytes. */ +function sha256Head(b, blocks) { + var h0=0x6a09e667,h1=0xbb67ae85,h2=0x3c6ef372,h3=0xa54ff53a, + h4=0x510e527f,h5=0x9b05688c,h6=0x1f83d9ab,h7=0x5be0cd19; + for (var off = 0; off < blocks; off += 64) { + for (var i = 0; i < 16; i++) { + var p = off + (i << 2); + W[i] = (b[p] << 24) | (b[p+1] << 16) | (b[p+2] << 8) | b[p+3]; + } + for (i = 16; i < 64; i++) { + var x = W[i-15], y = W[i-2]; + var s0 = ((x>>>7)|(x<<25)) ^ ((x>>>18)|(x<<14)) ^ (x>>>3); + var s1 = ((y>>>17)|(y<<15)) ^ ((y>>>19)|(y<<13)) ^ (y>>>10); + W[i] = (((W[i-16] + s0) | 0) + ((W[i-7] + s1) | 0)) | 0; + } + var a=h0,b0=h1,c0=h2,d=h3,e=h4,f=h5,g=h6,h=h7; + for (i = 0; i < 64; i++) { + var S1 = ((e>>>6)|(e<<26)) ^ ((e>>>11)|(e<<21)) ^ ((e>>>25)|(e<<7)); + var t1 = (((((h + S1) | 0) + (((e & f) ^ (~e & g)) | 0)) | 0) + ((K[i] + W[i]) | 0)) | 0; + var S0 = ((a>>>2)|(a<<30)) ^ ((a>>>13)|(a<<19)) ^ ((a>>>22)|(a<<10)); + var t2 = (S0 + (((a & b0) ^ (a & c0) ^ (b0 & c0)) | 0)) | 0; + h=g; g=f; f=e; e=(d+t1)|0; d=c0; c0=b0; b0=a; a=(t1+t2)|0; + } + h0=(h0+a)|0; h1=(h1+b0)|0; h2=(h2+c0)|0; h3=(h3+d)|0; + h4=(h4+e)|0; h5=(h5+f)|0; h6=(h6+g)|0; h7=(h7+h)|0; + } + return h0 >>> 0; +} + +/* A buffer holding "" padded for SHA-256, rebuilt only when the + nonce gains a digit and its length changes. */ +function padded(seedBytes, digits) { + var len = seedBytes.length + digits; + var blocks = (((len + 8) >> 6) + 1) << 6; + var b = new Uint8Array(blocks); + b.set(seedBytes); + b[len] = 0x80; + var bits = len * 8; + b[blocks-4] = (bits >>> 24) & 0xff; + b[blocks-3] = (bits >>> 16) & 0xff; + b[blocks-2] = (bits >>> 8) & 0xff; + b[blocks-1] = bits & 0xff; + return { buf: b, blocks: blocks }; +} +`; + +/** + * The page's markup, and the loop that solves the challenge. + * + * The loop runs synchronously in bursts of about sixty milliseconds and yields + * between them, so the tab stays responsive and the progress bar moves. It is + * not trying to be unnoticeable — the reader is told what is happening and why + * — but a solver that froze the page would read as a broken site rather than as + * a wait. + * + * Both escape hatches are named in the markup, because a page that interrupts + * someone owes them a way past that does not depend on the thing being + * interrupted. A person signs in — the session exempts them from then on. A + * program that cannot run this is told where the open endpoints are, which is + * the same answer the 429 gives. + * + * @param {string} seed + * @param {number} issuedAt + * @param {number} difficulty + * @param {string} target + * @returns {string} + */ +function html(seed, issuedAt, difficulty, target) { + const json = JSON.stringify({ seed, issuedAt, difficulty, target }).replace(/ + + + + + +One moment · RSS Amplifier + + + +
+

One moment

+

Checking your browser…

+ +

The directory is being scraped hard right now, so a page costs +a moment of your computer's time before it loads. This happens about once an +hour, not once a page. Nothing about you is stored beyond the answer.

+

Sign in and you will not be asked again. +If you are a program: /llms.txt, +/api/feeds, /opml and +/mcp are open and never challenged.

+ +
+ + + +`; +} diff --git a/apps/web/src/proxy.js b/apps/web/src/proxy.js index b7e362b..0a44fcd 100644 --- a/apps/web/src/proxy.js +++ b/apps/web/src/proxy.js @@ -1,6 +1,7 @@ import { NextResponse } from 'next/server'; import { SIGNED_IN_HINT_COOKIE, hintToRestore } from './lib/session-hint.js'; +import { challenge } from './lib/challenge.js'; import { gate, hasValidPass } from './lib/crawl-gateway.js'; import { attempt, callerIdentity } from './lib/crawlThrottle.js'; import { countRequest } from './lib/trafficCounter.js'; @@ -9,7 +10,7 @@ import { TIERS, tierFor } from './lib/tiers.js'; /** * The one thing that runs in front of every request. * - * Four jobs, and they want different surfaces, which is the only reason this + * Five jobs, and they want different surfaces, which is the only reason this * file is more than it was: * * 0. Charge training crawlers. Reasoning in lib/crawl-gateway.js. First, @@ -18,6 +19,12 @@ import { TIERS, tierFor } from './lib/tiers.js'; * and because the sales page at /crawl is answered here for everyone, * whatever they wear. Same surface as the throttle: the pages and feeds * are what a corpus crawl is after. + * 0.5. Ask a caller with no name to do some arithmetic. Reasoning in + * lib/challenge.js, and it is off unless CHALLENGE_ENABLED and + * CHALLENGE_SECRET are both set. After the gate, because a training + * crawler is owed a 402 and an offer rather than a puzzle, and before the + * throttle, because the whole point is that the throttle cannot see this + * traffic: it arrives one request per address. * 1. Shape crawl traffic. Reasoning in lib/crawlThrottle.js. This wants to * see *everything* an expensive caller can ask for — the API, the feed * files, the framing proxy — because those are where the load actually is. @@ -75,6 +82,19 @@ export async function proxy(request) { * questions. Reading it again is one HMAC, and only for a request that * actually presents a token. */ + /* + * The arithmetic. Answers only a caller that is anonymous, asking for one of + * the three expensive page routes, and has not already solved one this hour; + * everyone else falls straight through, and with the feature off this is a + * single string comparison. Counted as a refusal for the same reason the gate + * and the 429 are: a limit that hides what it turned away cannot be tuned. + */ + const dare = await challenge(request); + if (dare) { + countRequest(request, tierFor(request).name, true); + return dare; + } + const tier = (await hasValidPass(request)) ? TIERS.pass : tierFor(request); const verdict = attempt(callerIdentity(request), Date.now(), tier); diff --git a/apps/web/test/challenge.test.js b/apps/web/test/challenge.test.js new file mode 100644 index 0000000..cb95062 --- /dev/null +++ b/apps/web/test/challenge.test.js @@ -0,0 +1,356 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +/** + * The proof-of-work challenge. + * + * Two failures are worth more than the rest, and both are silent. + * + * The first is challenging someone we promised not to. A feed reader that gets + * an interstitial where its `.rss` used to be does not complain; it simply + * stops updating, for as long as nobody notices. Same for a search crawler, an + * agent on `/api/*`, and a signed-in reader. So most of what follows is about + * who is *not* asked. + * + * The second is a check that costs a real reader a second and costs the fleet + * nothing — a forgeable token, a token that outlives its cost, a token solved + * on one address and spent on another, or a difficulty that quietly fell to + * zero because someone typed a variable wrong. Those are the acceptance tests + * at the bottom. + */ + +const SECRET = 'test-secret-not-a-real-one'; + +/** The environment the feature is actually on in. @param {Record} [extra] */ +function on(extra = {}) { + process.env.CHALLENGE_ENABLED = '1'; + process.env.CHALLENGE_SECRET = SECRET; + process.env.CHALLENGE_BITS = '8'; + delete process.env.CHALLENGE_TTL_MINUTES; + for (const [k, v] of Object.entries(extra)) process.env[k] = v; +} + +function off() { + delete process.env.CHALLENGE_ENABLED; + delete process.env.CHALLENGE_SECRET; + delete process.env.CHALLENGE_BITS; + delete process.env.CHALLENGE_TTL_MINUTES; +} + +/** A request as the edge presents it. */ +function req(path, { ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/145.0.0.0', ip = '203.0.113.7', cookie, auth, method = 'GET' } = {}) { + const headers = new Headers({ 'user-agent': ua, 'x-real-ip': ip }); + if (cookie) headers.set('cookie', cookie); + if (auth) headers.set('authorization', auth); + return new Request(`https://rssamplifier.com${path}`, { method, headers }); +} + +const { SHA256_JS, bits, challenge, enabled, hasLeadingZeroBits, seedFor, solved, wouldChallenge } = + await import('../src/lib/challenge.js'); + +test.beforeEach(off); +test.after(off); + +/* ------------------------------------------------------------------ off -- */ + +test('off by default, so a deploy cannot start interrupting readers on its own', () => { + assert.equal(enabled(), false); + assert.equal(wouldChallenge(req('/topics/bing-ai'), '/topics/bing-ai'), false); +}); + +test('enabled without a secret stays off, because forgeable tokens are worse than none', () => { + process.env.CHALLENGE_ENABLED = '1'; + assert.equal(enabled(), false, 'a costume of a defence costs readers and stops nobody'); + + process.env.CHALLENGE_SECRET = ' '; + assert.equal(enabled(), false, 'whitespace is not a secret'); +}); + +/* ------------------------------------------------------- who is not asked -- */ + +test('the API is never challenged', () => { + on(); + for (const p of ['/api/topics/nfl-season', '/api/authors/zawn-villines', '/api/feeds', '/api/mcp']) { + assert.equal(wouldChallenge(req(p), p), false, `${p} must stay open to agents`); + } +}); + +test('feed exports are never challenged, whatever route they hang off', () => { + on(); + for (const p of [ + '/topics/bing-ai.rss', + '/topics/bing-ai.atom', + '/topics/otaku-jump/audio.pls', + '/topics/parallel-computer.m3u', + '/joereg4-com.json', + '/opml', + ]) { + assert.equal(wouldChallenge(req(p), p), false, `${p} is what a subscriber fetches`); + } +}); + +test('only the three expensive page routes are challenged', () => { + on(); + for (const p of ['/topics/bing-ai', '/authors/mourjo-sen', '/dusty-phillips-codes/read']) { + assert.equal(wouldChallenge(req(p), p), true, `${p} is what the fleet walks`); + } + for (const p of ['/', '/login', '/crawl', '/leaderboard', '/search', '/videos']) { + assert.equal(wouldChallenge(req(p), p), false, `${p} is not worth interrupting for`); + } +}); + +test('a caller who has already said who they are is never challenged', () => { + on(); + const p = '/topics/bing-ai'; + + assert.equal(wouldChallenge(req(p, { cookie: 'rsa_session=abc' }), p), false, 'signed in'); + assert.equal( + wouldChallenge(req(p, { auth: 'Bearer rsa_1234abcd_aaaaaaaaaaaaaaaaaaaaaa' }), p), + false, + 'carries an API key', + ); + assert.equal( + wouldChallenge( + req(p, { ua: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' }), + p, + ), + false, + 'a search engine that names itself', + ); +}); + +test('only GET and HEAD, so a form post is never answered with a puzzle', () => { + on(); + assert.equal(wouldChallenge(req('/topics/x', { method: 'POST' }), '/topics/x'), false); + assert.equal(wouldChallenge(req('/topics/x', { method: 'HEAD' }), '/topics/x'), true); +}); + +/* ----------------------------------------------------------- the numbers -- */ + +test('leading zero bits are counted bit-wise, not by hex digit', () => { + assert.equal(hasLeadingZeroBits(new Uint8Array([0x00, 0x00, 0xff]), 16), true); + assert.equal(hasLeadingZeroBits(new Uint8Array([0x00, 0x01]), 16), false); + assert.equal(hasLeadingZeroBits(new Uint8Array([0x0f]), 4), true, 'half a byte counts'); + assert.equal(hasLeadingZeroBits(new Uint8Array([0x1f]), 4), false); + assert.equal(hasLeadingZeroBits(new Uint8Array([0xff]), 0), true, 'nothing asked, nothing owed'); +}); + +test('junk in the difficulty falls back to the default, never to zero', () => { + on({ CHALLENGE_BITS: 'lots' }); + assert.equal(bits(), 18); + + on({ CHALLENGE_BITS: '0' }); + assert.equal(bits(), 18, 'zero bits is a challenge every caller passes for free'); + + on({ CHALLENGE_BITS: '-4' }); + assert.equal(bits(), 18); + + on({ CHALLENGE_BITS: '999' }); + assert.equal(bits(), 18, 'a difficulty nobody can solve is an outage'); +}); + +/* ------------------------------------------------------------ acceptance -- */ + +/** Solve one, the way the browser does. */ +async function solve(request, issuedAt, difficulty) { + const seed = await seedFor(request, issuedAt); + const enc = new TextEncoder(); + for (let n = 0; n < 5_000_000; n += 1) { + const d = await crypto.subtle.digest('SHA-256', enc.encode(seed + n)); + if (hasLeadingZeroBits(new Uint8Array(d), difficulty)) return n; + } + throw new Error('unsolvable'); +} + +test('a genuine solution is accepted', async () => { + on(); + const r = req('/topics/bing-ai'); + const issuedAt = Date.now(); + const nonce = await solve(r, issuedAt, 8); + + const withToken = req('/topics/bing-ai', { cookie: `rsa_pow=${issuedAt}.${nonce}` }); + assert.equal(await solved(withToken), true); +}); + +test('a made-up answer is refused', async () => { + on(); + const r = req('/topics/bing-ai', { cookie: `rsa_pow=${Date.now()}.999999999` }); + assert.equal(await solved(r), false, 'any two numbers must not open the door'); +}); + +test('a solution is bound to the address that solved it', async () => { + on(); + const mine = req('/topics/bing-ai', { ip: '203.0.113.7' }); + const issuedAt = Date.now(); + const nonce = await solve(mine, issuedAt, 8); + const cookie = `rsa_pow=${issuedAt}.${nonce}`; + + assert.equal(await solved(req('/topics/bing-ai', { ip: '203.0.113.7', cookie })), true); + assert.equal( + await solved(req('/topics/bing-ai', { ip: '198.51.100.9', cookie })), + false, + 'this is the whole cost model: rotating the address means solving again', + ); +}); + +test('a solution expires, so the tax is paid again', async () => { + on({ CHALLENGE_TTL_MINUTES: '1' }); + const r = req('/topics/bing-ai'); + const issuedAt = Date.now() - 5 * 60_000; + const nonce = await solve(r, issuedAt, 8); + + assert.equal( + await solved(req('/topics/bing-ai', { cookie: `rsa_pow=${issuedAt}.${nonce}` })), + false, + ); +}); + +test('a solution to an easier question stops working when the difficulty rises', async () => { + on({ CHALLENGE_BITS: '4' }); + const r = req('/topics/bing-ai'); + const issuedAt = Date.now(); + const nonce = await solve(r, issuedAt, 4); + const cookie = `rsa_pow=${issuedAt}.${nonce}`; + + assert.equal(await solved(req('/topics/bing-ai', { cookie })), true); + + // Ten bits is 64x the work; the four-bit answer will almost never satisfy it, + // and re-solving is exactly what raising the dial is meant to force. + on({ CHALLENGE_BITS: '10' }); + const stillGood = await solved(req('/topics/bing-ai', { cookie })); + assert.equal(stillGood, false, 'the difficulty is read now, not taken from the token'); +}); + +test('a token that claims the future is refused', async () => { + on(); + const ahead = Date.now() + 60 * 60_000; + assert.equal(await solved(req('/topics/x', { cookie: `rsa_pow=${ahead}.1` })), false); +}); + +test('junk in the cookie is refused rather than thrown', async () => { + on(); + for (const v of ['', 'x', '.', 'abc.def', '123', `${Date.now()}.`]) { + assert.equal(await solved(req('/topics/x', { cookie: `rsa_pow=${v}` })), false, JSON.stringify(v)); + } +}); + +/* ----------------------------------------------------------- the response -- */ + +test('the interstitial is a 503 that no crawler will index as the article', async () => { + on(); + const request = req('/topics/bing-ai'); + request.nextUrl = new URL('https://rssamplifier.com/topics/bing-ai'); + + const res = await challenge(request); + assert.ok(res, 'a challenge is owed'); + assert.equal(res.status, 503, 'a 200 here would be indexed in place of the page'); + assert.equal(res.headers.get('x-robots-tag'), 'noindex'); + assert.equal(res.headers.get('cache-control'), 'no-store'); + assert.ok(res.headers.get('retry-after')); + + const body = await res.text(); + assert.match(body, /llms\.txt/, 'a program is told where the open doors are'); + assert.match(body, /\/login/, 'a person is told how to stop being asked'); + assert.match(body, /noscript/, 'and what to do without JavaScript'); +}); + +test('nothing is owed once it has been solved', async () => { + on(); + const plain = req('/topics/bing-ai'); + const issuedAt = Date.now(); + const nonce = await solve(plain, issuedAt, 8); + + const request = req('/topics/bing-ai', { cookie: `rsa_pow=${issuedAt}.${nonce}` }); + assert.equal(await challenge(request), null); +}); + +/* ------------------------------------------------------------ the solver -- */ + +/** + * The hash the page ships, checked against the one the server verifies with. + * + * This is the failure with no symptom worth naming: a hand-written SHA-256 that + * is wrong in any bit produces a challenge that can never be solved, and the + * only sign of it is readers sitting on an interstitial forever while the logs + * show nothing at all. So the exact text that goes into the page is run here + * and compared against Web Crypto over inputs that cross both padding + * boundaries — a message that fits in one block, one that spills into a second, + * and the 55/56/64-byte edges where the length field moves. + */ +/* + * Compiled with `new Function` rather than run in a `vm` context, and the + * difference is not a detail: typed arrays reached across a vm boundary defeat + * the JIT, and the same text that does 610,000 hashes a second here managed + * 10,000 inside `vm.runInNewContext`. A benchmark run in a sandbox would have + * condemned a solver that is perfectly fast in the browser it is written for. + */ +const solver = new Function(`${SHA256_JS}; return { sha256Head, padded };`)(); + +test('the shipped hash agrees with Web Crypto, including at the padding edges', async () => { + const enc = new TextEncoder(); + const cases = ['', 'a', 'abc', 'x'.repeat(55), 'x'.repeat(56), 'x'.repeat(63), 'x'.repeat(64), 'x'.repeat(65), 'x'.repeat(200), 'deadbeef1234567890']; + + for (const s of cases) { + const bytes = enc.encode(s); + const real = new Uint8Array(await crypto.subtle.digest('SHA-256', bytes)); + const expected = ((real[0] << 24) | (real[1] << 16) | (real[2] << 8) | real[3]) >>> 0; + + const { buf, blocks } = solver.padded(bytes, 0); + assert.equal(solver.sha256Head(buf, blocks), expected, `SHA-256 of ${s.length} bytes must match`); + } +}); + +test('the solver is fast enough that the default difficulty is bearable', () => { + const bytes = new TextEncoder().encode('benchmark-seed-0123456789012345678901234567890123456789012345678901'); + const { buf, blocks } = solver.padded(bytes, 0); + + const started = Date.now(); + const N = 50_000; + for (let i = 0; i < N; i += 1) solver.sha256Head(buf, blocks); + const perSecond = N / ((Date.now() - started) / 1000 || 0.001); + + // The 18-bit default averages 262,144 hashes. Web Crypto managed ~67k/s, + // which would have made a reader wait eight seconds, and the first version of + // this hash managed 17k/s — worse than what it replaced. Well under a second + // at the default is the bar, so this has to clear a few hundred thousand. + assert.ok(perSecond > 300_000, `only ${Math.round(perSecond)} hashes/s — too slow to ask a reader for`); +}); + +/* --------------------------------------------------------------- wiring -- */ + +/** + * Where the challenge sits in the proxy, read out of proxy.js itself. + * + * proxy.js cannot be imported here — it pulls in `next/server`, which has no + * export map plain Node can resolve — so its shape is read back from the + * source, the way test/crawl-gateway.test.js reads the gate's position. The + * order is the policy: a training crawler is owed a 402 and an offer rather + * than a puzzle, and the throttle must not get to refuse a caller we were about + * to challenge, because the whole premise here is that the throttle cannot see + * this traffic. + */ +function proxySource() { + return readFileSync(fileURLToPath(new URL('../src/proxy.js', import.meta.url)), 'utf8'); +} + +test('the challenge runs after the gate and before the throttle', () => { + const src = proxySource(); + const gateAt = src.indexOf('await gate(request)'); + const challengeAt = src.indexOf('await challenge(request)'); + const throttleAt = src.indexOf('attempt(callerIdentity(request)'); + + assert.ok(gateAt >= 0 && challengeAt >= 0 && throttleAt >= 0, 'all three still run'); + assert.ok(gateAt < challengeAt, 'a training crawler is owed the offer, not a puzzle'); + assert.ok(challengeAt < throttleAt, 'the throttle must not answer a caller we were about to challenge'); +}); + +test('a challenged request is still counted, or the ledger hides what it turned away', () => { + const src = proxySource(); + const challengeAt = src.indexOf('const dare = await challenge(request)'); + const counted = src.indexOf('countRequest', challengeAt); + const returned = src.indexOf('return dare', challengeAt); + assert.ok(counted >= 0 && counted < returned, 'counted before it is returned'); +}); From e31f9e2c83cb487898c40343326eb263b91a9e10 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 7 Sep 2026 02:20:24 +0000 Subject: [PATCH 2/2] Judge the solver by the wait, not by this machine's clock rate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI failed at 277,778 hashes/s against a threshold of 300,000 — a threshold set from the development box's own 610,000, which is the mistake. The number that matters is what a reader waits: 262,144 hashes at 18 bits is 0.43s here and 0.95s on the runner, and both are fine. So the assertion is now the projected wait with a bar far below either. It still catches the two regressions that each happened once — Web Crypto at 67k/s, an eight-second wait, and the first hand-written hash at 17k/s — and it no longer fails on hardware that is merely slower than mine. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KPvk8mVEpxWwTRFby9m8VT --- apps/web/test/challenge.test.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/apps/web/test/challenge.test.js b/apps/web/test/challenge.test.js index cb95062..35913b0 100644 --- a/apps/web/test/challenge.test.js +++ b/apps/web/test/challenge.test.js @@ -312,11 +312,21 @@ test('the solver is fast enough that the default difficulty is bearable', () => for (let i = 0; i < N; i += 1) solver.sha256Head(buf, blocks); const perSecond = N / ((Date.now() - started) / 1000 || 0.001); - // The 18-bit default averages 262,144 hashes. Web Crypto managed ~67k/s, - // which would have made a reader wait eight seconds, and the first version of - // this hash managed 17k/s — worse than what it replaced. Well under a second - // at the default is the bar, so this has to clear a few hundred thousand. - assert.ok(perSecond > 300_000, `only ${Math.round(perSecond)} hashes/s — too slow to ask a reader for`); + // Stated as the wait rather than as a rate, because the rate is the machine + // and the wait is the thing a reader feels. The 18-bit default averages + // 262,144 hashes: this box does it in 0.43s, a CI runner in about 0.95s. + // + // The bar is deliberately far below both. It is here to catch the two + // regressions that already happened once each — Web Crypto at ~67k/s, an + // eight-second wait, and a first hand-written hash at ~17k/s, slower than + // what it replaced — not to measure whatever hardware this runs on. A + // threshold set near the development machine's own number fails on every + // slower runner and teaches people to ignore it. + const seconds = 2 ** 18 / perSecond; + assert.ok( + seconds < 2.5, + `${Math.round(perSecond)} hashes/s puts the default difficulty at ${seconds.toFixed(1)}s, which is too long to ask a reader for`, + ); }); /* --------------------------------------------------------------- wiring -- */