diff --git a/index.d.ts b/index.d.ts index d2d11d4..d9a002d 100644 --- a/index.d.ts +++ b/index.d.ts @@ -149,9 +149,24 @@ export interface RobotsOptions { comments?: string[]; } +/** What the caller spent against an allowance, as `spend()` reports it. */ +export interface QuotaUsage { + count: number; + remaining: number; + resetSeconds: number; + overLimit: boolean; +} + +/** The allowance a 402 should quote, when the caller kept the count itself. */ +export interface SellContext { + usage?: QuotaUsage | null; + /** Defaults to the gateway's own `freeQuota`. */ + quota?: { requests: number; windowSeconds: number } | null; +} + export interface Gateway { handle: Handle; - sell: (request: Request) => Promise; + sell: (request: Request, context?: SellContext) => Promise; enabled: boolean; options: Required> & { onSale: GatewayOptions['onSale'] | null; @@ -161,6 +176,10 @@ export interface Gateway { }; robotsTxt: (extra?: RobotsOptions) => string; page: () => string; + /** The pass this request presents, from the gateway's header or a bearer token. */ + passFrom: (request: Request) => string | null; + /** Whether that token is a live pass this gateway minted. */ + verifyPass: (token: string | null) => Promise; } export function createGateway(options: GatewayOptions): Gateway; diff --git a/package.json b/package.json index ec0b04c..a347bb5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/x402-gateway", - "version": "0.4.0", + "version": "0.5.0", "type": "module", "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, and a robots.txt that keeps search crawlers welcome.", "keywords": [ @@ -20,22 +20,52 @@ "middleware", "paywall" ], - "repository": { "type": "git", "url": "git+https://github.com/profullstack/x402-gateway.git" }, + "repository": { + "type": "git", + "url": "git+https://github.com/profullstack/x402-gateway.git" + }, "homepage": "https://github.com/profullstack/x402-gateway#readme", - "bugs": { "url": "https://github.com/profullstack/x402-gateway/issues" }, + "bugs": { + "url": "https://github.com/profullstack/x402-gateway/issues" + }, "license": "MIT", "author": "Profullstack, LLC", "exports": { - ".": { "types": "./index.d.ts", "import": "./src/index.js" }, - "./hono": { "types": "./index.d.ts", "import": "./src/hono.js" }, - "./next": { "types": "./index.d.ts", "import": "./src/next.js" }, - "./robots": { "types": "./index.d.ts", "import": "./src/robots.js" }, - "./agents": { "types": "./index.d.ts", "import": "./src/agents.js" }, - "./edge": { "types": "./index.d.ts", "import": "./src/edge.js" } + ".": { + "types": "./index.d.ts", + "import": "./src/index.js" + }, + "./hono": { + "types": "./index.d.ts", + "import": "./src/hono.js" + }, + "./next": { + "types": "./index.d.ts", + "import": "./src/next.js" + }, + "./robots": { + "types": "./index.d.ts", + "import": "./src/robots.js" + }, + "./agents": { + "types": "./index.d.ts", + "import": "./src/agents.js" + }, + "./edge": { + "types": "./index.d.ts", + "import": "./src/edge.js" + } }, "types": "./index.d.ts", - "files": ["src", "index.d.ts", "README.md", "LICENSE"], - "engines": { "node": ">=20.11" }, + "files": [ + "src", + "index.d.ts", + "README.md", + "LICENSE" + ], + "engines": { + "node": ">=20.11" + }, "sideEffects": false, "scripts": { "test": "node --test" diff --git a/src/index.js b/src/index.js index 687a8a2..754d47b 100644 --- a/src/index.js +++ b/src/index.js @@ -127,11 +127,11 @@ export function createGateway(options = {}) { headers: { 'content-type': 'text/html; charset=utf-8', ...noStore, ...headers }, }); - const pageCtx = (days = 1, usage = null) => ({ - quota: o.freeQuota + const pageCtx = (days = 1, usage = null, quota = o.freeQuota) => ({ + quota: quota ? { - requests: o.freeQuota.requests, - windowSeconds: o.freeQuota.windowSeconds, + requests: quota.requests, + windowSeconds: quota.windowSeconds, used: usage?.count ?? null, resetSeconds: usage?.resetSeconds ?? null, exceeded: Boolean(usage?.overLimit), @@ -177,7 +177,15 @@ export function createGateway(options = {}) { // Present when the free allowance is what stopped this request, rather than // the crawler lists. It changes what the 402 says, not what it costs. const usage = context.usage ?? null; - const rateHeaders = usage ? quotaHeaders(o.freeQuota, usage) : {}; + /* + * The allowance the caller was measured against. Defaults to this + * gateway's own, but a caller that keeps its own counter -- an app-wide + * throttle metering every route, not just the crawler lists -- hands its + * own in, so the 402 quotes the limit that actually stopped the request + * rather than one the gateway happens to hold. + */ + const quota = context.quota ?? o.freeQuota; + const rateHeaders = usage && quota ? quotaHeaders(quota, usage) : {}; if (proofHeader) { if (!enabled) return json(receipt(asked, { error: 'Payments are not switched on here.' }), 402); @@ -272,22 +280,22 @@ export function createGateway(options = {}) { } if (wantsHtml(request.headers.get('accept'))) { - return html(o.page(pageCtx(asked, usage)), 402, rateHeaders); + return html(o.page(pageCtx(asked, usage, quota)), 402, rateHeaders); } - if (usage) { + if (usage && quota) { // Say what ran out, when it comes back, and what a pass costs, in that // order. A caller reading this is deciding between waiting, rotating // addresses, and paying, and the numbers are the argument. return json( receipt(asked, { error: - `Free allowance used: ${o.freeQuota.requests} requests per ` + - `${o.freeQuota.windowSeconds}s. It resets in ${usage.resetSeconds}s. ` + + `Free allowance used: ${quota.requests} requests per ` + + `${quota.windowSeconds}s. It resets in ${usage.resetSeconds}s. ` + `A pass removes the limit for ${price} a day.`, quota: { - requests: o.freeQuota.requests, - windowSeconds: o.freeQuota.windowSeconds, + requests: quota.requests, + windowSeconds: quota.windowSeconds, used: usage.count, resetSeconds: usage.resetSeconds, }, @@ -367,6 +375,19 @@ export function createGateway(options = {}) { sell, enabled, options: o, + /** + * The pass a request presents, and whether it is one this gateway minted + * and still honours. + * + * Exposed because the pass has to be honoured by everything that could + * refuse a request, not only by `handle`. An app-wide throttle that meters + * every route has to skip whoever already paid, and recomputing the + * signing-secret fallback (`secret || coinpay.apiKey`) on its side is + * exactly the kind of duplicate rule that drifts and starts charging + * paying crawlers twice. + */ + passFrom, + verifyPass: async (token) => Boolean(token && (await readPass(token, { secret }))), /** robots.txt with this gateway's lists and sales path. */ robotsTxt: (extra = {}) => robotsTxt({ siteUrl: o.siteUrl, path: o.path, training: o.training, retrieval: o.retrieval, ...extra }), diff --git a/test/quota.test.js b/test/quota.test.js index fa154b9..38dcfb2 100644 --- a/test/quota.test.js +++ b/test/quota.test.js @@ -266,3 +266,78 @@ describe('the page a throttled reader sees', () => { assert.match(page, /sold by the gigabyte/); }); }); + +describe("an allowance the caller counted itself", () => { + // The app-wide throttle in @profullstack/throttle meters every route, not + // just the crawler lists, and then asks the gateway to sell. The 402 has to + // quote the limit that actually stopped the request. + const usage = { count: 101, remaining: 0, resetSeconds: 42, overLimit: true }; + + it("quotes the caller's numbers, not the gateway's", async () => { + const gate = gateway(null); // no freeQuota of its own + const answer = await gate.sell(reader("203.0.113.9"), { + usage, + quota: { requests: 100, windowSeconds: 60 }, + }); + assert.equal(answer.status, 402); + const body = await answer.json(); + assert.match(body.error, /100 requests per 60s/); + assert.match(body.error, /resets in 42s/); + assert.equal(body.quota.requests, 100); + assert.equal(body.quota.used, 101); + assert.equal(answer.headers.get("ratelimit-limit"), "100"); + assert.equal(answer.headers.get("ratelimit-reset"), "42"); + }); + + it("still falls back to the gateway's own allowance", async () => { + const gate = gateway(25); + const answer = await gate.sell(reader("203.0.113.9"), { usage }); + const body = await answer.json(); + assert.match(body.error, /25 requests per 60s/); + }); + + // Regression: `sell` used to read o.freeQuota unconditionally whenever a + // usage was passed, so a gateway without one threw on the throttle's path. + it("does not throw when neither side has an allowance", async () => { + const gate = gateway(null); + const answer = await gate.sell(reader("203.0.113.9"), { usage }); + assert.equal(answer.status, 402); + const body = await answer.json(); + assert.match(body.error, /Payment required/); + }); + + it("renders the caller's allowance on the HTML page too", async () => { + const gate = gateway(null); + const answer = await gate.sell( + reader("203.0.113.9", { accept: "text/html" }), + { usage, quota: { requests: 100, windowSeconds: 60 } }, + ); + assert.equal(answer.status, 402); + assert.match(await answer.text(), /100/); + }); +}); + +describe("verifyPass", () => { + it("honours a pass this gateway minted, and nothing else", async () => { + const gate = gateway(100); + const now = Math.floor(Date.now() / 1000); + const good = await mintPass({ secret: SECRET, ref: "r1", expiresAt: now + 60, now }); + assert.equal(await gate.verifyPass(good.token), true); + assert.equal(await gate.verifyPass("cp_nonsense.abc"), false); + assert.equal(await gate.verifyPass(null), false); + + const expired = await mintPass({ secret: SECRET, ref: "r2", expiresAt: now - 1, now: now - 61 }); + assert.equal(await gate.verifyPass(expired.token), false); + }); + + it("reads the token off a request, header or bearer", () => { + const gate = gateway(100); + const header = new Request(`${SITE}/`, { headers: { "x-crawl-pass": " tok " } }); + assert.equal(gate.passFrom(header), "tok"); + const bearer = new Request(`${SITE}/`, { + headers: { authorization: "Bearer cp_abc.def" }, + }); + assert.equal(gate.passFrom(bearer), "cp_abc.def"); + assert.equal(gate.passFrom(new Request(`${SITE}/`)), null); + }); +});