From 15d487c6be1f1012a36c2b6ac662859f0fdc5349 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 9 Sep 2026 16:57:05 +0000 Subject: [PATCH 1/2] Meter every route, and pay partners what a pass was actually worth Two things, and the second one was found on the way to the first. **The throttle.** The gate charges crawlers that say who they are. Nothing charged the ones that do not, and nothing counted a page route at all. That is the shape that failed on coinpayportal on 2026-09-08: a headless browser found a route nobody had listed and walked 19,000 of its URLs a day for two days, declaring nothing, tripping no list. So @profullstack/throttle now meters every route at 100 requests a minute per caller, answered 402 with this buyer's own offer rather than 429. The price here is the buyer's own -- a dollar a day at list, less the more it has spent -- so the gateway is chosen per request, and a refusal has to quote the price that buyer would actually pay. One throttle per gateway, sharing ONE store: if the counting split along with the price, a caller crossing a discount threshold mid-window would be handed a fresh hundred requests for the privilege. Both halves are tested. **The bug.** apps/web asked for x402-gateway ^0.1.0 and got 0.1.0, whose onSale payload has no totalCents and no days; both arrived in 0.3.0 with multi-day passes. pricing.js reads them anyway, and everything downstream treats a missing figure as nothing: partners.js:95 !Number(undefined) is true, so splitSale returns 0 and the publishers whose writing was crawled are paid nothing attribution.js:47 totalCents 0, so no niche revenue is booked and no operator is owed a share queries.js:1006 the crawl_sales row stores a null total_cents So the seller side of the marketplace has been recording sales it never paid out on, for as long as passes have been sold. Bumping to ^0.6.0 fixes it going forward; the rows already written are still wrong, and crawl_sales wants a look for null total_cents before anyone trusts that ledger. A caret range on a 0.x version only matches patches, which is why ^0.1.0 sat five minors behind without anything ever saying so. 510 tests pass, seven of them new. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YDGCxTmEPs3ecwjjLJDQXh --- apps/web/package.json | 3 +- apps/web/src/app.js | 12 +++++ apps/web/src/lib/throttle.js | 58 ++++++++++++++++++++ bun.lock | 37 ++++++------- test/throttle.test.js | 100 +++++++++++++++++++++++++++++++++++ 5 files changed, 191 insertions(+), 19 deletions(-) create mode 100644 apps/web/src/lib/throttle.js create mode 100644 test/throttle.test.js diff --git a/apps/web/package.json b/apps/web/package.json index 5746f0c..9ba677a 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -21,7 +21,8 @@ "@profullstack/leaderboard": "^0.3.0", "@profullstack/nichedb": "workspace:*", "@profullstack/partners": "0.2.0", - "@profullstack/x402-gateway": "^0.1.0", + "@profullstack/throttle": "^0.2.2", + "@profullstack/x402-gateway": "^0.6.0", "@simplewebauthn/browser": "^13.2.0", "hono": "^4.10.3" } diff --git a/apps/web/src/app.js b/apps/web/src/app.js index 156b5b6..1267f00 100644 --- a/apps/web/src/app.js +++ b/apps/web/src/app.js @@ -6,6 +6,7 @@ import { modulesFor, withModules } from './lib/modules.js'; import { partners } from './lib/partners.js'; import { gateway, gatewayFor } from './lib/pricing.js'; import { Denied } from './lib/service.js'; +import { meter } from './lib/throttle.js'; import { registerAgents } from './routes/agents.js'; import { registerApi } from './routes/api.js'; import { registerAuth } from './routes/auth.js'; @@ -46,6 +47,17 @@ app.use('*', async (c, next) => { const { gateway: chosen } = await gatewayFor(c.req.raw); const answer = await chosen.handle(c.req.raw); if (answer) return answer; + + /* + * Then the site-wide allowance (lib/throttle.js), which meters every route: + * 100 requests a minute per caller, answered 402 at this buyer's own price + * rather than 429. The gate above sells to crawlers that say who they are; + * this sells to the ones that do not, and nothing counted a page route + * before it. + */ + const overLimit = await meter(chosen, c.req.raw); + if (overLimit) return overLimit; + await next(); }); diff --git a/apps/web/src/lib/throttle.js b/apps/web/src/lib/throttle.js new file mode 100644 index 0000000..ee41eea --- /dev/null +++ b/apps/web/src/lib/throttle.js @@ -0,0 +1,58 @@ +/** + * The site-wide allowance: a hundred requests a minute, per caller, on every + * route. Going over is answered 402 with the caller's own crawl offer. + * + * WHY. The gate above charges crawlers that say who they are. Nothing charged + * the ones that do not, and nothing counted a page route at all. That is the + * shape that failed on coinpayportal on 2026-09-08: a headless browser found a + * route nobody had listed and walked 19,000 of its URLs a day for two days, + * declaring nothing, tripping no list, every hit served from the database. + * + * WHY A THROTTLE PER GATEWAY. The price here is the buyer's own -- a dollar a + * day at list, less the more it has spent (lib/pricing.js) -- so the gateway is + * chosen per request and a refusal has to quote the price that buyer would + * actually pay. The counting must NOT split along with it, or a caller whose + * price changed mid-window would be handed a fresh allowance for crossing a + * discount threshold. One store, shared by every gateway's throttle. + */ + +import { createThrottle, memoryStore } from '@profullstack/throttle'; + +/** One counter for the whole site, whatever price the caller is being quoted. */ +const store = memoryStore(); + +const throttles = new Map(); + +/** The throttle that refuses at `gateway`'s price. Built once per gateway. */ +function throttleFor(gateway) { + let throttle = throttles.get(gateway); + if (!throttle) { + throttle = createThrottle({ + gateway, + store, + /* + * A signed-in reader and an API caller get the larger budget, keyed on + * the credential rather than the address so two of them never share a + * bucket. Not an exemption: an unmetered site for anyone willing to sign + * up first is a worse trade than metering a member generously. + */ + credential: { limit: 600, ceiling: 1200 }, + rules: [ + /* Sign-in stays address-bucketed, or a guess buys the member budget. */ + { path: '/auth/', limit: 10, credential: false }, + { path: '/healthz', open: true }, + /* + * The surfaces an agent needs in order to USE the data rather than + * copy it stay generous, for the same reason they are outside the + * gate: they are the point of the index, not the cost of it. + */ + { path: '/mcp', limit: 600 }, + ], + }); + throttles.set(gateway, throttle); + } + return throttle; +} + +/** Resolves to a Response for a caller over the allowance, or undefined. */ +export const meter = (gateway, request) => throttleFor(gateway).handle(request); diff --git a/bun.lock b/bun.lock index 557cf1c..4009fb7 100644 --- a/bun.lock +++ b/bun.lock @@ -12,14 +12,14 @@ }, "apps/cli": { "name": "@profullstack/nichedb", - "version": "0.1.0", + "version": "0.3.1", "bin": { "nichedb": "./bin/nichedb.js", }, }, "apps/web": { "name": "@nichedb/web", - "version": "0.1.0", + "version": "0.3.1", "dependencies": { "@nichedb/auth": "workspace:*", "@nichedb/config": "workspace:*", @@ -33,14 +33,15 @@ "@profullstack/leaderboard": "^0.3.0", "@profullstack/nichedb": "workspace:*", "@profullstack/partners": "0.2.0", - "@profullstack/x402-gateway": "^0.1.0", + "@profullstack/throttle": "^0.2.2", + "@profullstack/x402-gateway": "^0.6.0", "@simplewebauthn/browser": "^13.2.0", "hono": "^4.10.3", }, }, "apps/worker": { "name": "@nichedb/worker", - "version": "0.1.0", + "version": "0.3.1", "dependencies": { "@nichedb/config": "workspace:*", "@nichedb/core": "workspace:*", @@ -51,14 +52,14 @@ }, "packages/adapters": { "name": "@nichedb/adapters", - "version": "0.1.0", + "version": "0.3.1", "dependencies": { "@nichedb/core": "workspace:*", }, }, "packages/auth": { "name": "@nichedb/auth", - "version": "0.1.0", + "version": "0.3.1", "dependencies": { "@nichedb/config": "workspace:*", "@nichedb/db": "workspace:*", @@ -68,11 +69,11 @@ }, "packages/config": { "name": "@nichedb/config", - "version": "0.1.0", + "version": "0.3.1", }, "packages/core": { "name": "@nichedb/core", - "version": "0.1.0", + "version": "0.3.1", "dependencies": { "@nichedb/adapters": "workspace:*", "@nichedb/config": "workspace:*", @@ -82,7 +83,7 @@ }, "packages/db": { "name": "@nichedb/db", - "version": "0.1.0", + "version": "0.3.1", "dependencies": { "@nichedb/config": "workspace:*", "@nichedb/knowledge": "workspace:*", @@ -90,18 +91,18 @@ }, "packages/enrichers": { "name": "@nichedb/enrichers", - "version": "0.1.0", + "version": "0.3.1", "dependencies": { "@nichedb/core": "workspace:*", }, }, "packages/knowledge": { "name": "@nichedb/knowledge", - "version": "0.1.0", + "version": "0.3.1", }, "packages/notify": { "name": "@nichedb/notify", - "version": "0.1.0", + "version": "0.3.1", "dependencies": { "@nichedb/config": "workspace:*", "@nichedb/db": "workspace:*", @@ -111,7 +112,7 @@ }, "packages/payments": { "name": "@nichedb/payments", - "version": "0.1.0", + "version": "0.3.1", "dependencies": { "@profullstack/coinpay": "^0.8.0", "@profullstack/referrals": "^0.1.0", @@ -119,7 +120,7 @@ }, "packages/queue": { "name": "@nichedb/queue", - "version": "0.1.0", + "version": "0.3.1", "dependencies": { "@nichedb/config": "workspace:*", "@nichedb/core": "workspace:*", @@ -306,7 +307,7 @@ "@profullstack/favicon-generator": ["@profullstack/favicon-generator@1.2.1", "", { "dependencies": { "inquirer": "^10.2.2", "sharp": "^0.33.5" }, "bin": { "fav": "bin/cli.js" } }, "sha512-1w+EcoEmi60TXXQd97Brs7kN/BmlaoOcNvE39ZombUWSCBFBNe6fFm2mA7HORXb7TM39mHklNTIoG20glLsgVA=="], - "@profullstack/leaderboard": ["@profullstack/leaderboard@0.3.0", "", {}, "sha512-SKPmIqhmrAhFQqHl9wJHV1NzJHUCNBsqWc1F6M5AeFfphXWGfKSmgeCC3QN37JH/cQ1NxS8kYC/DqbjQqNBoaQ=="], + "@profullstack/leaderboard": ["@profullstack/leaderboard@0.3.1", "", {}, "sha512-LbxLwy+RvT/Qez0mfWCk4o9hvcq0kzwBZ0vF+CYp+oGvyqJWUaSTLhCHWGtKKAI3c0ZUXj6ll0iuRjbuZUE2Sg=="], "@profullstack/nichedb": ["@profullstack/nichedb@workspace:apps/cli"], @@ -314,7 +315,9 @@ "@profullstack/referrals": ["@profullstack/referrals@0.1.0", "", { "peerDependencies": { "react": ">=18" }, "optionalPeers": ["react"] }, "sha512-u66SdBVpsv3kc0N+NWISPoYD5vjCERyv5wfD07iSkZwQeC2IA+ihX5jNA4e7Xr+Y4AUvxLycG+3b4VaROqzgRg=="], - "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.1.0", "", {}, "sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ=="], + "@profullstack/throttle": ["@profullstack/throttle@0.2.2", "", { "dependencies": { "@profullstack/x402-gateway": "^0.6.0" } }, "sha512-qpb9yKE7frwTP29YILx+QePqyEPWGVZxPYve7ITqSfARzOl0ZKhfjeizxPKZUgYhFQfNk6kWCa6/53RVoj304g=="], + + "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.6.0", "", {}, "sha512-n9/7NixIM9wVOMGWDc+3RtFsGABG+HAYfkZ11JxoEIlF/FDRGEXbvHFlNL/M97RNy01964FhAbQp34OMJoxqZg=="], "@redis/bloom": ["@redis/bloom@1.2.0", "", { "peerDependencies": { "@redis/client": "^1.0.0" } }, "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg=="], @@ -544,8 +547,6 @@ "@inquirer/select/@inquirer/type": ["@inquirer/type@4.1.1", "", { "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-yJoHYrMnxIsJZCY+0Vb66Dy3he3kL3e2wOBKhoSwWWAzZAY82emlxwgprCtp6yRixvNRNq9ztfRWQYPNr3Go7A=="], - "@profullstack/partners/@profullstack/leaderboard": ["@profullstack/leaderboard@0.3.1", "", {}, "sha512-LbxLwy+RvT/Qez0mfWCk4o9hvcq0kzwBZ0vF+CYp+oGvyqJWUaSTLhCHWGtKKAI3c0ZUXj6ll0iuRjbuZUE2Sg=="], - "@redis/client/cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="], "@types/mute-stream/@types/node": ["@types/node@26.4.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-k97ENvZWtvA6yqz5/FS6a7duDgOPEeOQOc2iKS/nY6mX6qJUKtLnWzQS+Xj6tXweyj6ZcTAK2Qecetnvi9nCLA=="], diff --git a/test/throttle.test.js b/test/throttle.test.js new file mode 100644 index 0000000..f9fb1ac --- /dev/null +++ b/test/throttle.test.js @@ -0,0 +1,100 @@ +/** + * The site-wide allowance. + * + * The gate charges crawlers that say who they are. Nothing charged the ones + * that do not, and nothing counted a page route at all -- which is the shape + * that failed on coinpayportal on 2026-09-08, where a headless browser found a + * route nobody had listed and walked 19,000 of its URLs a day for two days. + */ +import { describe, expect, test } from 'bun:test'; + +// The config reads the environment once at import. A gateway needs a key and a +// payTo to make a real offer; without them a refusal is a 429, which is correct +// behaviour but not what these tests are about. +const PAY_TO = '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5'; +process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test'; +process.env.SITE_URL ??= 'https://nichedb.test'; +process.env.COINPAY_X402_KEY ??= 'cp_live_test_secret_0123456789'; +process.env.CRAWL_PAY_TO ??= PAY_TO; + +// The site's own gateway factory, not a stand-in: the price a refusal quotes is +// the buyer's own, and that is exactly the part worth testing. +const { gatewayAt } = await import('../apps/web/src/lib/pricing.js'); +const { meter } = await import('../apps/web/src/lib/throttle.js'); +const BROWSER = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'; + +/** The gateway that charges this much a day -- the same one the app would pick. */ +const paid = (priceCents) => gatewayAt(priceCents); + +const request = (path, ip, headers = {}) => + new Request(`https://nichedb.test${path}`, { + headers: { 'user-agent': BROWSER, 'sec-fetch-mode': 'navigate', 'x-real-ip': ip, ...headers }, + }); + +/** How many land before the throttle refuses. Each case needs its own address. */ +async function countUntilLimited(gateway, path, ip, attempts, headers = {}) { + let allowed = 0; + for (let i = 0; i < attempts; i++) { + if (await meter(gateway, request(path, ip, headers))) break; + allowed++; + } + return allowed; +} + +describe('the site-wide allowance', () => { + const gateway = paid(100); + + test('meters a page route, which nothing here did before', async () => { + expect(await countUntilLimited(gateway, '/niches/housing', '10.5.0.1', 140)).toBe(100); + }); + + test('gives each caller its own allowance', async () => { + expect(await countUntilLimited(gateway, '/niches/housing', '10.5.0.2', 5)).toBe(5); + expect(await countUntilLimited(gateway, '/niches/housing', '10.5.0.3', 5)).toBe(5); + }); + + test('keeps sign-in address-bucketed however it is credentialed', async () => { + // Or a brute-force bolts on an Authorization header and buys the member budget. + const allowed = await countUntilLimited(gateway, '/auth/verify', '10.5.0.4', 40, { + authorization: 'Bearer anything', + }); + expect(allowed).toBe(10); + }); + + test('never meters the health check', async () => { + expect(await countUntilLimited(gateway, '/healthz', '10.5.0.5', 150)).toBe(150); + }); + + test('refuses with 402 and an offer, not 429', async () => { + for (let i = 0; i < 100; i++) await meter(gateway, request('/niches/crime', '10.5.0.6')); + const answer = await meter(gateway, request('/niches/crime', '10.5.0.6')); + expect(answer?.status).toBe(402); + const body = await answer.json(); + expect(body.accepts.length).toBeGreaterThan(0); + expect(body.error).toMatch(/100 requests per 60s/); + }); +}); + +describe('the price a refusal quotes', () => { + // The price here is the buyer's own: a dollar a day at list, less the more it + // has spent. A refusal has to quote the price THAT buyer would pay. + test('follows the gateway the request was priced with', async () => { + const discounted = paid(40); + for (let i = 0; i < 100; i++) await meter(discounted, request('/niches/markets', '10.5.0.7')); + const answer = await meter(discounted, request('/niches/markets', '10.5.0.7')); + expect(answer?.status).toBe(402); + expect((await answer.json()).pass.price).toBe('0.40 USD'); + }); + + test('but the counting does not split along with it', async () => { + // Or a caller crossing a discount threshold mid-window would be handed a + // fresh hundred requests for the privilege. + const list = paid(100); + const discounted = paid(40); + for (let i = 0; i < 60; i++) await meter(list, request('/niches/news', '10.5.0.8')); + for (let i = 0; i < 40; i++) await meter(discounted, request('/niches/news', '10.5.0.8')); + const answer = await meter(discounted, request('/niches/news', '10.5.0.8')); + expect(answer?.status).toBe(402); + }); +}); From d32601b1900dd12bc59280a3b20e468b149efb22 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 9 Sep 2026 17:04:04 +0000 Subject: [PATCH 2/2] Ask for a price nothing else has cached The 402 test passed here and failed in CI. gatewayAt memoises one gateway per price, and pricing.js builds the list-price one at import -- so when pricing.test.js loaded first, the cached list-price gateway had been built before this file set COINPAY_X402_KEY, could not take money, and every refusal was a 429. Locally the files happened to load the other way round. Asking for a price nothing else uses means the gateway is always constructed after the environment is set. Verified in both file orders. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YDGCxTmEPs3ecwjjLJDQXh --- test/throttle.test.js | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/test/throttle.test.js b/test/throttle.test.js index f9fb1ac..c8979e3 100644 --- a/test/throttle.test.js +++ b/test/throttle.test.js @@ -24,7 +24,16 @@ const { meter } = await import('../apps/web/src/lib/throttle.js'); const BROWSER = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'; -/** The gateway that charges this much a day -- the same one the app would pick. */ +/** + * The gateway that charges this much a day -- the same one the app would pick. + * + * Always at a price nothing else uses. `gatewayAt` memoises one gateway per + * price, and pricing.js builds the LIST-price one at import; if that import + * happened before this file set the environment above, the cached gateway has + * no key, cannot take money, and every refusal here is a 429. That is exactly + * what happened in CI while these tests asked for the list price and passed + * locally, where the files happened to load the other way round. + */ const paid = (priceCents) => gatewayAt(priceCents); const request = (path, ip, headers = {}) => @@ -43,7 +52,7 @@ async function countUntilLimited(gateway, path, ip, attempts, headers = {}) { } describe('the site-wide allowance', () => { - const gateway = paid(100); + const gateway = paid(250); test('meters a page route, which nothing here did before', async () => { expect(await countUntilLimited(gateway, '/niches/housing', '10.5.0.1', 140)).toBe(100); @@ -80,18 +89,18 @@ describe('the price a refusal quotes', () => { // The price here is the buyer's own: a dollar a day at list, less the more it // has spent. A refusal has to quote the price THAT buyer would pay. test('follows the gateway the request was priced with', async () => { - const discounted = paid(40); + const discounted = paid(37); for (let i = 0; i < 100; i++) await meter(discounted, request('/niches/markets', '10.5.0.7')); const answer = await meter(discounted, request('/niches/markets', '10.5.0.7')); expect(answer?.status).toBe(402); - expect((await answer.json()).pass.price).toBe('0.40 USD'); + expect((await answer.json()).pass.price).toBe('0.37 USD'); }); test('but the counting does not split along with it', async () => { // Or a caller crossing a discount threshold mid-window would be handed a // fresh hundred requests for the privilege. - const list = paid(100); - const discounted = paid(40); + const list = paid(250); + const discounted = paid(37); for (let i = 0; i < 60; i++) await meter(list, request('/niches/news', '10.5.0.8')); for (let i = 0; i < 40; i++) await meter(discounted, request('/niches/news', '10.5.0.8')); const answer = await meter(discounted, request('/niches/news', '10.5.0.8'));