From 7781d73f912a40c640145d5a92cfab6af378f943 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 6 Sep 2026 02:19:13 +0000 Subject: [PATCH] A public board: who pays for the directory, and who just asks traffic_hourly has counted who asks and how many we turn away since 2026-09-02. The half it could never answer is the half with money in it: the gateway has been selling day passes and writing none of it down, so a sale existed only for as long as the response took to send. crawl_sales is that half. One row per sale, unique on the payment ref so a settlement delivered twice books once. The hook returns its promise rather than dropping it, because the gateway awaits onSale before the receipt goes out (its JSDoc said otherwise; profullstack/x402-gateway#5), and a rejection is swallowed there, so a database failure still sells the pass it was paid for. The board projects both tables instead of keeping a third copy that could disagree with either. Only badges get a table, being awarded rather than derived. Sales are tagged with classifyAgent, so a family reads the same on both sides. The two sides never share a list. Agents paying are ranked by money; agents asking are ranked by requests and by refusals. 'Paid us $3' and 'asked 209,000 times' are not the same fact. /leaderboard is in OPEN_PATHS with and without the trailing slash: the gateway prefix-matches only entries ending in a slash, so the bare path would open the index and still charge for every board on it. Verified against a local libSQL file: the migration applies, a duplicate ref books once, a sale with no payer still lands, the route handler serves both sides off real traffic_hourly rows, RSS names the side, a share card renders and an unknown board 404s. The existing 382 web tests still pass. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0144uEVbZK3jdaQkwcLYTXPE --- apps/web/package.json | 3 +- .../src/app/leaderboard/[[...path]]/route.js | 16 +++ apps/web/src/lib/crawl-gateway.js | 47 +++++++- apps/web/src/lib/leaderboard.js | 114 ++++++++++++++++++ packages/db/index.js | 1 + .../20260906020000_crawl_sales_and_board.sql | 37 ++++++ packages/db/src/crawl-sales.js | 65 ++++++++++ pnpm-lock.yaml | 9 ++ pnpm-workspace.yaml | 1 + 9 files changed, 291 insertions(+), 2 deletions(-) create mode 100644 apps/web/src/app/leaderboard/[[...path]]/route.js create mode 100644 apps/web/src/lib/leaderboard.js create mode 100644 packages/db/migrations/20260906020000_crawl_sales_and_board.sql create mode 100644 packages/db/src/crawl-sales.js diff --git a/apps/web/package.json b/apps/web/package.json index d0a1570..1230473 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -26,6 +26,7 @@ "@swc/helpers": "^0.5.23", "next": "^16.2.4", "react": "^19.2.0", - "react-dom": "^19.2.0" + "react-dom": "^19.2.0", + "@profullstack/leaderboard": "^0.3.0" } } diff --git a/apps/web/src/app/leaderboard/[[...path]]/route.js b/apps/web/src/app/leaderboard/[[...path]]/route.js new file mode 100644 index 0000000..ba6803d --- /dev/null +++ b/apps/web/src/app/leaderboard/[[...path]]/route.js @@ -0,0 +1,16 @@ +import { leaderboard } from '../../../lib/leaderboard.js'; + +/** + * The public board: who pays for the directory, and who just asks. + * + * One catch-all route rather than a page plus a pile of API endpoints, because + * the board serves its own HTML, JSON, RSS, per-agent share cards and the + * embed widget, all under this path. + */ +export const dynamic = 'force-dynamic'; + +export async function GET(request) { + return (await leaderboard().handle(request)) ?? new Response('Not found', { status: 404 }); +} + +export const HEAD = GET; diff --git a/apps/web/src/lib/crawl-gateway.js b/apps/web/src/lib/crawl-gateway.js index 2330279..aa889b3 100644 --- a/apps/web/src/lib/crawl-gateway.js +++ b/apps/web/src/lib/crawl-gateway.js @@ -1,4 +1,8 @@ import { createGateway, isTrainingAgent, RETRIEVAL_AGENTS } from '@profullstack/x402-gateway'; +import { crawlSales } from '@rssamplifier/db'; + +import { db } from './db.js'; +import { classifyAgent } from './traffic.js'; import { x402Proxy } from '@profullstack/x402-gateway/next'; import { SIGNED_IN_HINT_COOKIE } from './session-hint.js'; @@ -57,7 +61,21 @@ function siteUrl() { * before the rewrite to /api/mcp) and at its long one, so a client that read * the API docs is not charged for calling the same thing by its other name. */ -export const OPEN_PATHS = ['/llms.txt', '/skill.md', '/opml', '/mcp', '/api/mcp', '/api/feeds']; +export const OPEN_PATHS = [ + '/llms.txt', + '/skill.md', + '/opml', + '/mcp', + '/api/mcp', + '/api/feeds', + // The board is open on purpose, and both spellings are needed: the gateway + // prefix-matches only entries ending in a slash, so '/leaderboard' alone + // would open the index and still charge for every board on it. An agent that + // hits a 402 on the page ranking its own spend cannot read the case for + // buying a pass. + '/leaderboard', + '/leaderboard/', +]; /** * Addresses that serve no readers: the OVH VPS fleet. @@ -162,6 +180,33 @@ export const gateway = createGateway({ */ chargeSpoofedBrowsers: true, exempt, + /* + * Book the sale. + * + * traffic_hourly has counted who asks and who is refused since 2026-09-02, + * and said nothing about who paid: a pass existed only for as long as the + * response took to send. The promise is returned rather than dropped + * because the gateway awaits this hook before the receipt goes out, which + * is what makes the row land before the buyer is told it worked. A + * rejection is swallowed there, so a database failure still sells the pass + * it was paid for. + */ + onSale: (sale) => + crawlSales + .recordCrawlSale(db(), { + payer: sale.payer, + ref: sale.ref, + days: sale.days, + priceCents: sale.priceCents, + totalCents: sale.totalCents, + currency: sale.currency, + userAgent: sale.userAgent, + // The same vocabulary traffic_hourly keys on, so a family reads the + // same on both sides of the board. + agent: classifyAgent(sale.userAgent), + expiresAt: sale.expiresAt, + }) + .catch((err) => console.error('[x402] could not record the sale', err)), }); /** diff --git a/apps/web/src/lib/leaderboard.js b/apps/web/src/lib/leaderboard.js new file mode 100644 index 0000000..a573128 --- /dev/null +++ b/apps/web/src/lib/leaderboard.js @@ -0,0 +1,114 @@ +import { createLeaderboard, projectionStore } from '@profullstack/leaderboard'; +import { crawlSales } from '@rssamplifier/db'; + +import { db, siteUrl } from './db.js'; + +/** + * The public board over the crawler paywall. + * + * Two sides, kept apart. Agents that paid are ranked by money; agents that + * were counted or turned away are ranked by volume. Putting them in one list + * would say those are the same kind of fact, and the difference between them + * is the whole argument for having a paywall at all. + * + * Nothing is written here. `traffic_hourly` is already the record of who asked + * and who was refused, and `crawl_sales` is the record of who paid, so the + * board projects both rather than keeping a third copy that could disagree + * with either. Badges are the exception: they are awarded at a moment rather + * than derived from a sum. + */ + +/** 'YYYY-MM-DDTHH' in UTC, the shape traffic_hourly keys on. */ +const hourToMs = (hour) => Date.parse(`${hour}:00:00Z`); + +const badges = { + async awardBadge(player, badge) { + const { rowsAffected } = await db().execute({ + sql: `insert into leaderboard_badges (player, badge, awarded_at) + values (?, ?, ?) on conflict (player, badge) do nothing`, + args: [player, badge, new Date().toISOString()], + }); + return Number(rowsAffected) > 0; + }, + async badges() { + const { rows } = await db().execute('select player, badge, awarded_at from leaderboard_badges'); + /** @type {Record>} */ + const out = {}; + for (const r of rows) { + const player = String(r.player); + out[player] ??= {}; + out[player][String(r.badge)] = Date.parse(String(r.awarded_at)); + } + return out; + }, +}; + +/** + * A wallet address is long and all of it is public, so show the ends. The + * display name for a sale is the agent family, because "ai-openai" tells a + * reader something that `0x46E9…6C79` does not. + */ +const shortWallet = (p) => (p.length > 14 ? `${p.slice(0, 6)}…${p.slice(-4)}` : p); + +async function events({ since }) { + const sinceIso = new Date(since || 0).toISOString(); + const client = db(); + const [sales, traffic] = await Promise.all([ + crawlSales.crawlSalesSince(client, sinceIso), + client.execute({ + sql: `select hour, agent, sum(hits) as hits, sum(refused) as refused + from traffic_hourly where hour >= ? group by hour, agent`, + // traffic_hourly.hour is 'YYYY-MM-DDTHH', which compares correctly as text. + args: [sinceIso.slice(0, 13)], + }), + ]); + + const out = []; + for (const s of sales) { + const player = s.payer ? String(s.payer) : s.agent ? `ua:${s.agent}` : null; + if (!player) continue; + const at = Date.parse(String(s.created_at)); + const name = s.agent ? String(s.agent) : shortWallet(String(s.payer)); + const each = (metric, delta) => out.push({ player, name, metric, delta, at }); + each('spent', Number(s.total_cents) || 0); + each('passes', 1); + each('days', Number(s.days) || 1); + } + for (const r of traffic.rows) { + const at = hourToMs(String(r.hour)); + if (!Number.isFinite(at)) continue; + const player = `ua:${r.agent}`; + const name = String(r.agent); + out.push({ player, name, metric: 'hits', delta: Number(r.hits) || 0, at }); + out.push({ player, name, metric: 'refused', delta: Number(r.refused) || 0, at }); + } + return out; +} + +/** @type {ReturnType | null} */ +let board = null; + +/** + * Built lazily: the module is imported by a route, and `siteUrl()` reads an + * environment variable Next would otherwise bake in at build time. + */ +export function leaderboard() { + board ??= createLeaderboard({ + siteName: 'RSS Amplifier', + siteUrl: siteUrl(), + basePath: '/leaderboard', + store: projectionStore({ events, badges }), + sides: { buy: 'Agents paying', use: 'Agents asking' }, + boards: { + spenders: { label: 'Biggest spenders', metric: 'spent', format: 'usd', unit: 'Spent', side: 'buy', actor: 'Agent' }, + passes: { label: 'Most passes bought', metric: 'passes', format: 'integer', unit: 'Passes', side: 'buy', actor: 'Agent' }, + days: { label: 'Most days of access', metric: 'days', format: 'integer', unit: 'Days', side: 'buy', actor: 'Agent' }, + busiest: { label: 'Most requests', metric: 'hits', format: 'integer', unit: 'Requests', side: 'use', actor: 'Agent' }, + refused: { label: 'Most requests refused', metric: 'refused', format: 'integer', unit: 'Refused', side: 'use', actor: 'Agent' }, + }, + // Nobody earns here: the directory sells access to its own index. + ladder: null, + cacheMs: 60_000, + }); + return board; +} diff --git a/packages/db/index.js b/packages/db/index.js index 3f9084b..3e85441 100644 --- a/packages/db/index.js +++ b/packages/db/index.js @@ -22,3 +22,4 @@ export * as social from './src/social.js'; export * as dataset from './src/dataset.js'; export * as traffic from './src/traffic.js'; export * as removals from './src/removals.js'; +export * as crawlSales from './src/crawl-sales.js'; diff --git a/packages/db/migrations/20260906020000_crawl_sales_and_board.sql b/packages/db/migrations/20260906020000_crawl_sales_and_board.sql new file mode 100644 index 0000000..973124a --- /dev/null +++ b/packages/db/migrations/20260906020000_crawl_sales_and_board.sql @@ -0,0 +1,37 @@ +-- What the crawler paywall earned, and who paid it. +-- +-- traffic_hourly already answers "who is asking and how often", including how +-- many of those we turned away. The half it cannot answer is the half with +-- money in it: the gateway has been selling day passes and writing none of it +-- down, so a sale existed only for as long as the response took to send. +-- +-- One row per sale. `ref` is the payment reference and is unique, so a +-- settlement delivered twice books once rather than doubling the day's +-- takings. Per-request rows are fine here, unlike the traffic rollup: a sale +-- is rare, and the write path on this database is the scarce thing. +create table if not exists crawl_sales ( + id integer primary key autoincrement, + payer text, + ref text unique, + days integer not null default 1, + price_cents integer not null default 0, + total_cents integer not null default 0, + currency text not null default 'USD', + user_agent text, + agent text, + expires_at text, + created_at text not null +); + +create index if not exists crawl_sales_payer on crawl_sales (payer, created_at); +create index if not exists crawl_sales_created on crawl_sales (created_at); + +-- Badges for the public board. Everything else it shows is projected out of +-- traffic_hourly and crawl_sales; a badge is awarded at a moment and then +-- kept, and that fact lives nowhere else. +create table if not exists leaderboard_badges ( + player text not null, + badge text not null, + awarded_at text not null, + primary key (player, badge) +); diff --git a/packages/db/src/crawl-sales.js b/packages/db/src/crawl-sales.js new file mode 100644 index 0000000..0bd8181 --- /dev/null +++ b/packages/db/src/crawl-sales.js @@ -0,0 +1,65 @@ +import { nowIso } from './client.js'; + +/** + * The crawler paywall's books. + * + * traffic_hourly counts who asked and who was refused. This is the other half: + * who paid. Kept as one row per sale rather than a rollup, because a sale is + * rare enough that the write costs nothing, and because the payment reference + * has to be unique for the deduplication below to mean anything. + * + * @typedef {import('@libsql/client').Client} Client + */ + +/** + * Book a sale. + * + * `on conflict (ref) do nothing` is the whole reason `ref` is unique: a + * settlement delivered twice must book once. A pass bought without a + * reference still records, it just cannot be deduplicated. + * + * Nothing here binds `undefined`: the remote libSQL client throws on it while + * a local file binds it as null, so an undefined slips through every local + * test and fails only in production. + * + * @param {Client} db + * @param {{ payer?: string|null, ref?: string|null, days?: number, priceCents?: number, + * totalCents?: number, currency?: string, userAgent?: string|null, + * agent?: string|null, expiresAt?: string|null }} sale + * @returns {Promise} + */ +export async function recordCrawlSale(db, sale) { + await db.execute({ + sql: `insert into crawl_sales + (payer, ref, days, price_cents, total_cents, currency, user_agent, agent, expires_at, created_at) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + on conflict (ref) do nothing`, + args: [ + sale.payer ?? null, + sale.ref ?? null, + Number(sale.days ?? 1), + Number(sale.priceCents ?? 0), + Number(sale.totalCents ?? 0), + String(sale.currency ?? 'USD'), + sale.userAgent ?? null, + sale.agent ?? null, + sale.expiresAt ?? null, + nowIso(), + ], + }); +} + +/** + * Every sale since `sinceIso`, oldest first. + * + * @param {Client} db + * @param {string} sinceIso + */ +export async function crawlSalesSince(db, sinceIso) { + const { rows } = await db.execute({ + sql: `select payer, agent, user_agent, total_cents, days, created_at + from crawl_sales where created_at >= ? order by created_at`, + args: [String(sinceIso)], + }); + return rows; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1ecd8b6..b48a2bb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,6 +43,9 @@ importers: apps/web: dependencies: + '@profullstack/leaderboard': + specifier: ^0.3.0 + version: 0.3.0 '@profullstack/player': specifier: ^0.3.1 version: 0.3.1(react@19.2.8) @@ -594,6 +597,10 @@ packages: resolution: {integrity: sha512-C2Xj8FZ0uHWeCXXqX5B4/gVFQmtSkiuOolzAgutjTfseNOHT3pUjljDZsTSxXFGgio54bCzVFqmEOUrIVk8RDA==} engines: {node: '>=20.0.0'} + '@profullstack/leaderboard@0.3.0': + resolution: {integrity: sha512-SKPmIqhmrAhFQqHl9wJHV1NzJHUCNBsqWc1F6M5AeFfphXWGfKSmgeCC3QN37JH/cQ1NxS8kYC/DqbjQqNBoaQ==} + engines: {node: '>=20.11'} + '@profullstack/player@0.3.1': resolution: {integrity: sha512-/BvjRREIQ+WBw+MJXuSUDONreRnivB36wZDiflnzqrTtDhBUZEbkAgsfKF/m67U7GdSYl1uRJMbQoPEw5I9nMA==} engines: {node: '>=20'} @@ -1346,6 +1353,8 @@ snapshots: tslib: 2.8.1 tsyringe: 4.10.0 + '@profullstack/leaderboard@0.3.0': {} + '@profullstack/player@0.3.1(react@19.2.8)': dependencies: hls.js: 1.7.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index effa365..e595706 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -15,3 +15,4 @@ minimumReleaseAgeExclude: - '@profullstack/player@0.2.0 || 0.3.1' - '@profullstack/x402-gateway@0.1.0 || 0.2.1 || 0.3.0' - '@profullstack/x402-client@0.2.0' + - '@profullstack/leaderboard@0.3.0'