Meter every route, and pay partners what a pass was actually worth

apps/web/package.json

@@ -21,7 +21,8 @@
     "@profullstack/leaderboard": "^0.3.0",
     "@profullstack/nichedb": "workspace:*",
     "@profullstack/partners": "0.2.0",
-    "@profullstack/x402-gateway": "^0.1.0",
+    "@profullstack/throttle": "^0.2.2",
+    "@profullstack/x402-gateway": "^0.6.0",
     "@simplewebauthn/browser": "^13.2.0",
     "hono": "^4.10.3"
   }

apps/web/src/app.js

@@ -6,6 +6,7 @@ import { modulesFor, withModules } from './lib/modules.js';
 import { partners } from './lib/partners.js';
 import { gateway, gatewayFor } from './lib/pricing.js';
 import { Denied } from './lib/service.js';
+import { meter } from './lib/throttle.js';
 import { registerAgents } from './routes/agents.js';
 import { registerApi } from './routes/api.js';
 import { registerAuth } from './routes/auth.js';
@@ -46,6 +47,17 @@ app.use('*', async (c, next) => {
   const { gateway: chosen } = await gatewayFor(c.req.raw);
   const answer = await chosen.handle(c.req.raw);
   if (answer) return answer;
+
+  /*
+   * Then the site-wide allowance (lib/throttle.js), which meters every route:
+   * 100 requests a minute per caller, answered 402 at this buyer's own price
+   * rather than 429. The gate above sells to crawlers that say who they are;
+   * this sells to the ones that do not, and nothing counted a page route
+   * before it.
+   */
+  const overLimit = await meter(chosen, c.req.raw);
+  if (overLimit) return overLimit;
+
   await next();
 });
 

apps/web/src/lib/throttle.js

@@ -0,0 +1,58 @@
+/**
+ * The site-wide allowance: a hundred requests a minute, per caller, on every
+ * route. Going over is answered 402 with the caller's own crawl offer.
+ *
+ * WHY. The gate above charges crawlers that say who they are. Nothing charged
+ * the ones that do not, and nothing counted a page route at all. That is the
+ * shape that failed on coinpayportal on 2026-09-08: a headless browser found a
+ * route nobody had listed and walked 19,000 of its URLs a day for two days,
+ * declaring nothing, tripping no list, every hit served from the database.
+ *
+ * WHY A THROTTLE PER GATEWAY. The price here is the buyer's own -- a dollar a
+ * day at list, less the more it has spent (lib/pricing.js) -- so the gateway is
+ * chosen per request and a refusal has to quote the price that buyer would
+ * actually pay. The counting must NOT split along with it, or a caller whose
+ * price changed mid-window would be handed a fresh allowance for crossing a
+ * discount threshold. One store, shared by every gateway's throttle.
+ */
+
+import { createThrottle, memoryStore } from '@profullstack/throttle';
+
+/** One counter for the whole site, whatever price the caller is being quoted. */
+const store = memoryStore();
+
+const throttles = new Map();
+
+/** The throttle that refuses at `gateway`'s price. Built once per gateway. */
+function throttleFor(gateway) {
+  let throttle = throttles.get(gateway);
+  if (!throttle) {
+    throttle = createThrottle({
+      gateway,
+      store,
+      /*
+       * A signed-in reader and an API caller get the larger budget, keyed on
+       * the credential rather than the address so two of them never share a
+       * bucket. Not an exemption: an unmetered site for anyone willing to sign
+       * up first is a worse trade than metering a member generously.
+       */
+      credential: { limit: 600, ceiling: 1200 },
+      rules: [
+        /* Sign-in stays address-bucketed, or a guess buys the member budget. */
+        { path: '/auth/', limit: 10, credential: false },
+        { path: '/healthz', open: true },
+        /*
+         * The surfaces an agent needs in order to USE the data rather than
+         * copy it stay generous, for the same reason they are outside the
+         * gate: they are the point of the index, not the cost of it.
+         */
+        { path: '/mcp', limit: 600 },
+      ],
+    });
+    throttles.set(gateway, throttle);
+  }
+  return throttle;
+}
+
+/** Resolves to a Response for a caller over the allowance, or undefined. */
+export const meter = (gateway, request) => throttleFor(gateway).handle(request);

test/throttle.test.js

@@ -0,0 +1,109 @@
+/**
+ * The site-wide allowance.
+ *
+ * The gate charges crawlers that say who they are. Nothing charged the ones
+ * that do not, and nothing counted a page route at all -- which is the shape
+ * that failed on coinpayportal on 2026-09-08, where a headless browser found a
+ * route nobody had listed and walked 19,000 of its URLs a day for two days.
+ */
+import { describe, expect, test } from 'bun:test';
+
+// The config reads the environment once at import. A gateway needs a key and a
+// payTo to make a real offer; without them a refusal is a 429, which is correct
+// behaviour but not what these tests are about.
+const PAY_TO = '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5';
+process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test';
+process.env.SITE_URL ??= 'https://nichedb.test';
+process.env.COINPAY_X402_KEY ??= 'cp_live_test_secret_0123456789';
+process.env.CRAWL_PAY_TO ??= PAY_TO;
+
+// The site's own gateway factory, not a stand-in: the price a refusal quotes is
+// the buyer's own, and that is exactly the part worth testing.
+const { gatewayAt } = await import('../apps/web/src/lib/pricing.js');
+const { meter } = await import('../apps/web/src/lib/throttle.js');
+const BROWSER =
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36';
+
+/**
+ * The gateway that charges this much a day -- the same one the app would pick.
+ *
+ * Always at a price nothing else uses. `gatewayAt` memoises one gateway per
+ * price, and pricing.js builds the LIST-price one at import; if that import
+ * happened before this file set the environment above, the cached gateway has
+ * no key, cannot take money, and every refusal here is a 429. That is exactly
+ * what happened in CI while these tests asked for the list price and passed
+ * locally, where the files happened to load the other way round.
+ */
+const paid = (priceCents) => gatewayAt(priceCents);
+
+const request = (path, ip, headers = {}) =>
+  new Request(`https://nichedb.test${path}`, {
+    headers: { 'user-agent': BROWSER, 'sec-fetch-mode': 'navigate', 'x-real-ip': ip, ...headers },
+  });
+
+/** How many land before the throttle refuses. Each case needs its own address. */
+async function countUntilLimited(gateway, path, ip, attempts, headers = {}) {
+  let allowed = 0;
+  for (let i = 0; i < attempts; i++) {
+    if (await meter(gateway, request(path, ip, headers))) break;
+    allowed++;
+  }
+  return allowed;
+}
+
+describe('the site-wide allowance', () => {
+  const gateway = paid(250);
+
+  test('meters a page route, which nothing here did before', async () => {
+    expect(await countUntilLimited(gateway, '/niches/housing', '10.5.0.1', 140)).toBe(100);
+  });
+
+  test('gives each caller its own allowance', async () => {
+    expect(await countUntilLimited(gateway, '/niches/housing', '10.5.0.2', 5)).toBe(5);
+    expect(await countUntilLimited(gateway, '/niches/housing', '10.5.0.3', 5)).toBe(5);
+  });
+
+  test('keeps sign-in address-bucketed however it is credentialed', async () => {
+    // Or a brute-force bolts on an Authorization header and buys the member budget.
+    const allowed = await countUntilLimited(gateway, '/auth/verify', '10.5.0.4', 40, {
+      authorization: 'Bearer anything',
+    });
+    expect(allowed).toBe(10);
+  });
+
+  test('never meters the health check', async () => {
+    expect(await countUntilLimited(gateway, '/healthz', '10.5.0.5', 150)).toBe(150);
+  });
+
+  test('refuses with 402 and an offer, not 429', async () => {
+    for (let i = 0; i < 100; i++) await meter(gateway, request('/niches/crime', '10.5.0.6'));
+    const answer = await meter(gateway, request('/niches/crime', '10.5.0.6'));
+    expect(answer?.status).toBe(402);
+    const body = await answer.json();
+    expect(body.accepts.length).toBeGreaterThan(0);
+    expect(body.error).toMatch(/100 requests per 60s/);
+  });
+});
+
+describe('the price a refusal quotes', () => {
+  // The price here is the buyer's own: a dollar a day at list, less the more it
+  // has spent. A refusal has to quote the price THAT buyer would pay.
+  test('follows the gateway the request was priced with', async () => {
+    const discounted = paid(37);
+    for (let i = 0; i < 100; i++) await meter(discounted, request('/niches/markets', '10.5.0.7'));
+    const answer = await meter(discounted, request('/niches/markets', '10.5.0.7'));
+    expect(answer?.status).toBe(402);
+    expect((await answer.json()).pass.price).toBe('0.37 USD');
+  });
+
+  test('but the counting does not split along with it', async () => {
+    // Or a caller crossing a discount threshold mid-window would be handed a
+    // fresh hundred requests for the privilege.
+    const list = paid(250);
+    const discounted = paid(37);
+    for (let i = 0; i < 60; i++) await meter(list, request('/niches/news', '10.5.0.8'));
+    for (let i = 0; i < 40; i++) await meter(discounted, request('/niches/news', '10.5.0.8'));
+    const answer = await meter(discounted, request('/niches/news', '10.5.0.8'));
+    expect(answer?.status).toBe(402);
+  });
+});