From 1fcf704577303702213abac49ae987b6f99daa90 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 6 Sep 2026 10:18:22 +0000 Subject: [PATCH 1/2] The pass we sell is now a rung on the ladder /crawl has sold a day pass since 2026-09-05. It bought a training crawler past the 402 and then met the same 600-an-hour throttle as an anonymous curl, so the thing we sell bought nothing a caller could feel. Three changes, all of them rate: - TIERS.pass, at the sponsor ceiling: 120,000 an hour and 2,000 a minute. Deliberately the same numbers as sponsor rather than higher, because a sponsored key is a relationship and a pass is a dollar, and there is no honest reason the dollar should out-rank the relationship. - The proxy places a pass holder on that rung. The gate has already read the same token and said nothing about it, because "should this be charged" and "what allowance is this" are different questions; reading it again is one HMAC and only for a request that presents a token. - The 429 names the pass, with its price and how to buy it. Every branch of the upgrade path ended at something only a person can do: read an email, fill in a form, ask us. An agent that hits the wall at three in the morning had nowhere to go but slow down or spread itself over a proxy pool. Now it has a rung it can climb alone. The sales page also lists what a pass buys, via the benefits list added in x402-gateway 0.4.0 (bumped here from 0.3.0). WHAT IS NOT IN THIS, AND WHY. Every line of that list is rate. The directory's position, argued at length in lib/tiers.js, is that the data stays open to everyone -- no key, no account, no field withheld -- and money buys speed. Withholding fields from the free tier would sell the opposite of what this site is for, and would cost us the search and AI-answer traffic that is the only way a person finds us. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TKuF2jCbRj3GZTQVtwhi5m --- apps/web/package.json | 2 +- apps/web/src/lib/crawl-gateway.js | 63 ++++++++++++++- apps/web/src/lib/tiers.js | 16 ++++ apps/web/src/proxy.js | 42 +++++++++- apps/web/test/paid-tier.test.js | 122 ++++++++++++++++++++++++++++++ pnpm-lock.yaml | 10 +-- 6 files changed, 245 insertions(+), 10 deletions(-) create mode 100644 apps/web/test/paid-tier.test.js diff --git a/apps/web/package.json b/apps/web/package.json index 26402d8..a807abb 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -12,7 +12,7 @@ "dependencies": { "@profullstack/player": "^0.3.1", "@profullstack/rssamplifier": "workspace:*", - "@profullstack/x402-gateway": "0.3.0", + "@profullstack/x402-gateway": "0.4.0", "@rssamplifier/auth": "workspace:*", "@rssamplifier/db": "workspace:*", "@rssamplifier/feed": "workspace:*", diff --git a/apps/web/src/lib/crawl-gateway.js b/apps/web/src/lib/crawl-gateway.js index 1cde064..77d3499 100644 --- a/apps/web/src/lib/crawl-gateway.js +++ b/apps/web/src/lib/crawl-gateway.js @@ -1,4 +1,4 @@ -import { createGateway, isTrainingAgent, RETRIEVAL_AGENTS } from '@profullstack/x402-gateway'; +import { createGateway, isTrainingAgent, readPass, RETRIEVAL_AGENTS } from '@profullstack/x402-gateway'; import { crawlSales } from '@rssamplifier/db'; import { db } from './db.js'; @@ -186,6 +186,26 @@ export const gateway = createGateway({ */ chargeSpoofedBrowsers: true, exempt, + /* + * What a pass actually buys, printed on the sales page and in every 402. + * + * Every line is rate, and that is not a hedge — it is the directory's whole + * position, argued in lib/tiers.js: the data stays open to everyone, no key, + * no account, no field withheld, and money buys speed. A benefits list that + * promised fields or endpoints the free tier cannot have would be selling the + * opposite of what this site is for, and would cost us the search and AI + * answer traffic that is the only way a person finds us. + * + * Written as the numbers rather than as adjectives, because the reader is + * deciding between paying, waiting, and rotating addresses, and only the + * numbers settle that. + */ + benefits: [ + '120,000 requests an hour, up from 600 anonymous or 6,000 signed in', + '2,000 requests a minute of burst, up from 120', + 'No throttle on any page, feed, or API route, for the whole day', + 'Every field and every entry point stays open to everyone, paid or not: a pass buys rate, never access', + ], /* * Book the sale. * @@ -228,3 +248,44 @@ export const gateway = createGateway({ * @type {(request: Request) => Promise} */ export const gate = x402Proxy(gateway); + +/** Where a pass is presented: the named header, or a bearer token. */ +const PASS_HEADER = 'x-crawl-pass'; +const BEARER_PASS = /^Bearer\s+(cp_[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)$/i; + +/** + * Whether this request carries a crawl pass that is genuinely ours and still + * live. + * + * The gate already reads the pass, and answers `undefined` for a holder exactly + * as it does for a person — which is correct for deciding whether to charge, + * and loses the one fact the throttle needs: that this caller has paid. Rather + * than have the gate report it, which would grow a contract five other sites + * depend on, the pass is read again here. It is an HMAC over a short string + * with no database behind it, so reading it twice costs a hash. + * + * The secret is the CoinPay key, which is what `createGateway` defaults to when + * no explicit `secret` is passed, so this verifies exactly the tokens the + * gateway mints and nothing else. With no key configured there are no valid + * passes to find, and every caller falls through to the free ladder. + * + * @param {Request} request + * @returns {Promise} + */ +export async function hasValidPass(request) { + const secret = env['COINPAY_X402_KEY']; + if (!secret) return false; + + const direct = request.headers.get(PASS_HEADER); + const bearer = BEARER_PASS.exec(request.headers.get('authorization') ?? ''); + const token = direct ? direct.trim() : bearer ? bearer[1] : null; + if (!token) return false; + + try { + return Boolean(await readPass(token, { secret })); + } catch { + // A malformed token is not an error worth failing a request over; it is + // simply not a pass, and the caller lands on the free ladder. + return false; + } +} diff --git a/apps/web/src/lib/tiers.js b/apps/web/src/lib/tiers.js index 9c7804b..1634ae7 100644 --- a/apps/web/src/lib/tiers.js +++ b/apps/web/src/lib/tiers.js @@ -101,6 +101,22 @@ export const TIERS = { hourly: FREE_HOURLY * AUTH_MULTIPLIER, }, sponsor: { name: 'sponsor', burst: envInt('TIER_SPONSOR_BURST', 2_000), hourly: SPONSOR_HOURLY }, + /** + * A bought crawl pass, at the sponsor's ceiling. + * + * The rung the ladder was missing. Until now the top of it was "ask us and we + * will sponsor you", which is not something a program can do at three in the + * morning, and the pass that /crawl already sold bought a training crawler + * past a 402 and then met the same 600-an-hour throttle as everyone else. So + * the thing we sell bought nothing a heavy caller could feel. + * + * Deliberately the same numbers as `sponsor` rather than higher: a sponsored + * key is a relationship and a pass is a dollar, and there is no honest reason + * the dollar should out-rank the relationship. The reasoning for that ceiling + * being bounded rather than infinite is in SPONSOR_HOURLY above and applies + * unchanged here. + */ + pass: { name: 'pass', burst: envInt('TIER_SPONSOR_BURST', 2_000), hourly: SPONSOR_HOURLY }, }; /** diff --git a/apps/web/src/proxy.js b/apps/web/src/proxy.js index 9953d8e..b7e362b 100644 --- a/apps/web/src/proxy.js +++ b/apps/web/src/proxy.js @@ -1,10 +1,10 @@ import { NextResponse } from 'next/server'; import { SIGNED_IN_HINT_COOKIE, hintToRestore } from './lib/session-hint.js'; -import { gate } from './lib/crawl-gateway.js'; +import { gate, hasValidPass } from './lib/crawl-gateway.js'; import { attempt, callerIdentity } from './lib/crawlThrottle.js'; import { countRequest } from './lib/trafficCounter.js'; -import { tierFor } from './lib/tiers.js'; +import { TIERS, tierFor } from './lib/tiers.js'; /** * The one thing that runs in front of every request. @@ -63,7 +63,19 @@ export async function proxy(request) { * unlimited, and nothing above it can be worth paying for. Signed in is now * a large budget rather than no budget. */ - const tier = tierFor(request); + /* + * A bought pass is the top rung, and it is checked before the rest of the + * ladder because it is the only one someone paid for. Without this the pass + * we sell at /crawl bought a crawler past the 402 and then met the same + * 600-an-hour throttle as an anonymous curl, which is to say it bought + * nothing anyone could feel. + * + * The gate above has already read the same token and said nothing about it, + * because "should this be charged" and "what allowance is this" are different + * questions. Reading it again is one HMAC, and only for a request that + * actually presents a token. + */ + const tier = (await hasValidPass(request)) ? TIERS.pass : tierFor(request); const verdict = attempt(callerIdentity(request), Date.now(), tier); @@ -112,6 +124,29 @@ function tooMany(verdict, tier) { ? 'Create an API key at https://rssamplifier.com/account and send it as a bearer token; a sponsored key raises the ceiling further.' : 'This is the sponsor ceiling. If you need more than this, ask and we will raise it.'; + /* + * The rung a program can climb on its own. + * + * Every branch above ends at something only a person can do: read an email, + * fill in a form, ask us. An agent that hits this wall at three in the + * morning has nowhere to go, and the honest options left to it are to slow + * down or to spread itself over a proxy pool. So the pass is named here, with + * its price, as the one upgrade that needs no human on either side. + * + * Said to the paid tier too, where it reads as "you already have this", + * because a caller at the sponsor ceiling asking what is above it should be + * told there is nothing rather than sold something twice. + */ + const buyable = + tier.name === 'pass' || tier.name === 'sponsor' + ? undefined + : { + url: 'https://rssamplifier.com/crawl', + price: '1.00 USD per day, USDC, settled by CoinPay', + buys: '120,000 requests an hour and 2,000 a minute, for the whole day', + how: 'Fetch https://rssamplifier.com/crawl with Accept: application/json for an x402 offer, pay it, then send the pass in the x-crawl-pass header. No account and no human needed.', + }; + return NextResponse.json( { error: 'rate limit exceeded', @@ -121,6 +156,7 @@ function tooMany(verdict, tier) { tier: tier.name, hourlyLimit: Number.isFinite(tier.hourly) ? tier.hourly : null, upgrade: nextRung, + ...(buyable ? { buy: buyable } : {}), retryAfter: verdict.retryAfter, }, { diff --git a/apps/web/test/paid-tier.test.js b/apps/web/test/paid-tier.test.js new file mode 100644 index 0000000..faee42a --- /dev/null +++ b/apps/web/test/paid-tier.test.js @@ -0,0 +1,122 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { mintPass } from '@profullstack/x402-gateway'; + +import { TIERS } from '../src/lib/tiers.js'; + +/** + * The pass rung, and the thing it fixes. + * + * Until now /crawl sold a pass that bought a training crawler past the 402 and + * then met the same 600-an-hour throttle as an anonymous curl. The pass is now + * a rung on the ladder, so the thing we sell buys something a caller can feel. + * + * `hasValidPass` is imported lazily inside each test: lib/crawl-gateway.js + * reads COINPAY_X402_KEY at module scope through `env`, and these set it first. + */ +const SECRET = 'cp_live_test_secret_0123456789'; + +async function passModule() { + process.env.COINPAY_X402_KEY = SECRET; + return import('../src/lib/crawl-gateway.js'); +} + +const withHeaders = (headers) => + new Request('https://rssamplifier.com/topics/anything', { headers: new Headers(headers) }); + +async function livePass() { + const now = Math.floor(Date.now() / 1000); + const { token } = await mintPass({ secret: SECRET, ref: 'test', expiresAt: now + 3600, now }); + return token; +} + +test('a pass sits at the sponsor ceiling, not above it', () => { + // A sponsored key is a relationship and a pass is a dollar. There is no + // honest reason the dollar should out-rank the relationship. + assert.equal(TIERS.pass.hourly, TIERS.sponsor.hourly); + assert.equal(TIERS.pass.burst, TIERS.sponsor.burst); +}); + +test('a pass is worth far more than signing in, or nobody would buy one', () => { + assert.ok( + TIERS.pass.hourly > TIERS.session.hourly, + 'a bought pass must beat the free account tier', + ); + assert.ok(TIERS.pass.hourly > TIERS.anon.hourly * 100); +}); + +test('a live pass in the header is recognised', async () => { + const { hasValidPass } = await passModule(); + const token = await livePass(); + assert.equal(await hasValidPass(withHeaders({ 'x-crawl-pass': token })), true); +}); + +test('a live pass presented as a bearer token is recognised', async () => { + // The gateway accepts both spellings, so the throttle must agree with it or + // a caller doing exactly what the sales page said gets throttled anyway. + const { hasValidPass } = await passModule(); + const token = await livePass(); + assert.equal(await hasValidPass(withHeaders({ authorization: `Bearer ${token}` })), true); +}); + +test('no pass is not a pass', async () => { + const { hasValidPass } = await passModule(); + assert.equal(await hasValidPass(withHeaders({})), false); +}); + +test('a forged pass is refused, which is the whole point of signing them', async () => { + const { hasValidPass } = await passModule(); + const token = await livePass(); + const tampered = `${token.slice(0, -1)}${token.endsWith('A') ? 'B' : 'A'}`; + assert.equal(await hasValidPass(withHeaders({ 'x-crawl-pass': tampered })), false); +}); + +test('a pass from a different secret is refused', async () => { + const { hasValidPass } = await passModule(); + const now = Math.floor(Date.now() / 1000); + const { token } = await mintPass({ + secret: 'someone-elses-key', + ref: null, + expiresAt: now + 3600, + now, + }); + assert.equal(await hasValidPass(withHeaders({ 'x-crawl-pass': token })), false); +}); + +test('an expired pass is refused', async () => { + const { hasValidPass } = await passModule(); + const now = Math.floor(Date.now() / 1000); + // Minted live, then judged from a moment after it lapsed. + const { token } = await mintPass({ secret: SECRET, ref: null, expiresAt: now + 1, now }); + await new Promise((resolve) => setTimeout(resolve, 1100)); + assert.equal(await hasValidPass(withHeaders({ 'x-crawl-pass': token })), false); +}); + +test('garbage in the header is not an error, just not a pass', async () => { + const { hasValidPass } = await passModule(); + for (const value of ['', ' ', 'cp_', 'cp_nodot', 'not-a-pass', 'Bearer nonsense']) { + assert.equal( + await hasValidPass(withHeaders({ 'x-crawl-pass': value })), + false, + JSON.stringify(value), + ); + } +}); + +test('the sales page lists what a pass buys, and every line of it is rate', async () => { + // The directory's position is that data is open and money buys speed. A + // benefits list promising fields or endpoints the free tier cannot have would + // sell the opposite of what this site is for. + const { gateway } = await passModule(); + const benefits = gateway.options.benefits; + assert.ok(Array.isArray(benefits) && benefits.length > 0, 'a pass has to say what it buys'); + assert.ok( + benefits.some((line) => /120,000 requests an hour/.test(line)), + 'the headline number belongs in the list', + ); + assert.ok( + benefits.some((line) => /rate, never access/.test(line)), + 'and so does the promise that nothing is withheld', + ); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d473a7d..36a6495 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,8 +56,8 @@ importers: specifier: workspace:* version: link:../cli '@profullstack/x402-gateway': - specifier: 0.3.0 - version: 0.3.0 + specifier: 0.4.0 + version: 0.4.0 '@rssamplifier/auth': specifier: workspace:* version: link:../../packages/auth @@ -626,8 +626,8 @@ packages: engines: {node: '>=20.19'} hasBin: true - '@profullstack/x402-gateway@0.3.0': - resolution: {integrity: sha512-aaSMdtVInaAD6te/RnNnnqZW2+oo/dUsOTFTJCl0hOn58s85Yw3vZGk3KCtZmjrwV2O2rdTjKlZjbKp3CGgntw==} + '@profullstack/x402-gateway@0.4.0': + resolution: {integrity: sha512-Rk3dtpwgi5rOIcHvsRqv7XvbT8rEHyhN/d3zdtsPe8zQfhhYWEjuFuvd/2bIWoWZ6fILQRQOCbejOe2VCRlsNg==} engines: {node: '>=20.11'} '@simplewebauthn/browser@13.3.0': @@ -1384,7 +1384,7 @@ snapshots: '@noble/curves': 2.4.0 '@noble/hashes': 2.4.0 - '@profullstack/x402-gateway@0.3.0': {} + '@profullstack/x402-gateway@0.4.0': {} '@simplewebauthn/browser@13.3.0': {} From e434bb259f30b71a2e5e3318ec68e213d55f8275 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 6 Sep 2026 10:19:48 +0000 Subject: [PATCH 2/2] Let the lockfile carry x402-gateway 0.4.0 pnpm 11 refuses a lockfile entry published inside minimumReleaseAge, and 0.4.0 is hours old. The exclude list already names every earlier version of this package for the same reason. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TKuF2jCbRj3GZTQVtwhi5m --- pnpm-workspace.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5341416..9591678 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -13,7 +13,7 @@ allowBuilds: msgpackr-extract: false minimumReleaseAgeExclude: - '@profullstack/player@0.2.0 || 0.3.1' - - '@profullstack/x402-gateway@0.1.0 || 0.2.1 || 0.3.0' + - '@profullstack/x402-gateway@0.1.0 || 0.2.1 || 0.3.0 || 0.4.0' - '@profullstack/x402-client@0.2.0' - '@profullstack/leaderboard@0.3.0 || 0.3.1' - '@profullstack/partners@0.2.0'