diff --git a/.gitignore b/.gitignore index 8dfb15e8..f2c68ab6 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,6 @@ supabase/.temp/ # Local agent/editor config — never a repo artifact, and it was committed # as a symlink to one machine's home directory, broken for everyone else. .claude + +# Written by the Socket patch postinstall; not source. +.socket/ diff --git a/package.json b/package.json index a0ea883a..722b5263 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,8 @@ "@hookform/resolvers": "^5.2.2", "@profullstack/autoblog": "github:profullstack/autoblog#75e54af", "@profullstack/stack": "^0.1.0", - "@profullstack/x402-gateway": "^0.3.0", + "@profullstack/throttle": "^0.2.2", + "@profullstack/x402-gateway": "^0.6.0", "@stripe/stripe-js": "^8.6.1", "@supabase/ssr": "^0.8.0", "@supabase/supabase-js": "^2.90.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2121b04d..b854f1ea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,9 +17,12 @@ importers: '@profullstack/stack': specifier: ^0.1.0 version: 0.1.3(@supabase/ssr@0.8.0(@supabase/supabase-js@2.90.1))(next@16.1.1(@babel/core@7.28.6(supports-color@7.2.0))(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) + '@profullstack/throttle': + specifier: ^0.2.2 + version: 0.2.2 '@profullstack/x402-gateway': - specifier: ^0.3.0 - version: 0.3.0 + specifier: ^0.6.0 + version: 0.6.0 '@stripe/stripe-js': specifier: ^8.6.1 version: 8.6.1 @@ -909,8 +912,12 @@ packages: react: optional: true - '@profullstack/x402-gateway@0.3.0': - resolution: {integrity: sha512-aaSMdtVInaAD6te/RnNnnqZW2+oo/dUsOTFTJCl0hOn58s85Yw3vZGk3KCtZmjrwV2O2rdTjKlZjbKp3CGgntw==} + '@profullstack/throttle@0.2.2': + resolution: {integrity: sha512-qpb9yKE7frwTP29YILx+QePqyEPWGVZxPYve7ITqSfARzOl0ZKhfjeizxPKZUgYhFQfNk6kWCa6/53RVoj304g==} + engines: {node: '>=20.11'} + + '@profullstack/x402-gateway@0.6.0': + resolution: {integrity: sha512-n9/7NixIM9wVOMGWDc+3RtFsGABG+HAYfkZ11JxoEIlF/FDRGEXbvHFlNL/M97RNy01964FhAbQp34OMJoxqZg==} engines: {node: '>=20.11'} '@rolldown/pluginutils@1.0.0-beta.53': @@ -4644,7 +4651,11 @@ snapshots: next: 16.1.1(@babel/core@7.28.6(supports-color@7.2.0))(react-dom@19.2.3(react@19.2.3))(react@19.2.3) react: 19.2.3 - '@profullstack/x402-gateway@0.3.0': {} + '@profullstack/throttle@0.2.2': + dependencies: + '@profullstack/x402-gateway': 0.6.0 + + '@profullstack/x402-gateway@0.6.0': {} '@rolldown/pluginutils@1.0.0-beta.53': {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 00000000..b667c6c7 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,23 @@ +# Not a monorepo: this file exists only to name the two packages that are newer +# than pnpm's minimum-release-age policy, rather than relaxing the policy. +# +# `packages` has to be here even though it is empty. Without it pnpm still +# treats the directory as a workspace root and `pnpm store path` fails with +# "packages field missing or empty", which is what broke setup-node's pnpm +# cache step in CI. +packages: [] + +allowBuilds: + '@scarf/scarf': set this to true or false + '@tree-sitter-grammars/tree-sitter-yaml': set this to true or false + core-js: set this to true or false + core-js-pure: set this to true or false + esbuild: set this to true or false + sharp: set this to true or false + tree-sitter: set this to true or false + tree-sitter-json: set this to true or false + unrs-resolver: set this to true or false + +minimumReleaseAgeExclude: + - '@profullstack/throttle@0.2.2' + - '@profullstack/x402-gateway@0.6.0' diff --git a/src/lib/throttle.test.ts b/src/lib/throttle.test.ts new file mode 100644 index 00000000..1d46f529 --- /dev/null +++ b/src/lib/throttle.test.ts @@ -0,0 +1,76 @@ +/** + * The site-wide allowance. + * + * Nothing here metered a page route before. The polling cache in proxy.ts is a + * different tool for a different job -- it serves a repeat poller a cached body + * so the database is spared -- and it only knows about four endpoints. A + * caller walking the gig listings had never been counted at all, which is + * exactly the gap a headless browser walked through on coinpayportal. + * + * Tested against the throttle rather than through proxy.ts: importing the + * proxy pulls in @profullstack/stack, whose dist imports a bare `next/server` + * that vitest cannot resolve. That is a pre-existing packaging problem and not + * something this test should be the first to discover. + */ + +import { describe, it, expect } from "vitest"; +import { meter } from "./throttle"; + +const BROWSER_UA = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"; + +function request(path: string, ip: string, headers: Record = {}) { + return new Request(`https://ugig.net${path}`, { + headers: { + "user-agent": BROWSER_UA, + "sec-fetch-mode": "navigate", + "x-real-ip": ip, + ...headers, + }, + }); +} + +/** How many land before the throttle refuses. Each case needs its own address. */ +async function countUntilLimited( + path: string, + ip: string, + attempts: number, + headers: Record = {}, +): Promise { + let allowed = 0; + for (let i = 0; i < attempts; i++) { + if (await meter(request(path, ip, headers))) break; + allowed++; + } + return allowed; +} + +describe("the site-wide allowance", () => { + it("meters a page route, which nothing here did before", async () => { + expect(await countUntilLimited("/gigs/some-listing", "10.7.0.1", 140)).toBe(100); + }); + + it("gives each caller its own allowance", async () => { + expect(await countUntilLimited("/gigs/some-listing", "10.7.0.2", 5)).toBe(5); + expect(await countUntilLimited("/gigs/some-listing", "10.7.0.3", 5)).toBe(5); + }); + + it("keeps sign-in address-bucketed however it is credentialed", async () => { + // Or a brute-force bolts on an Authorization header and buys the member budget. + const allowed = await countUntilLimited("/api/auth/callback", "10.7.0.4", 40, { + authorization: "Bearer anything", + }); + expect(allowed).toBe(10); + }); + + it("lets a polling client keep the page open", async () => { + // proxy.ts already serves these from its cache when they repeat inside 30s, + // so a client with the page open costs the app nothing and is not refused. + expect(await countUntilLimited("/api/notifications", "10.7.0.5", 200)).toBe(200); + }); + + it("gives a signed-in member the larger budget, but still counts them", async () => { + const session = { cookie: "sb-abcdef-auth-token=eyJhbGciOi.session.value" }; + expect(await countUntilLimited("/gigs/some-listing", "10.7.0.6", 200, session)).toBe(200); + }); +}); diff --git a/src/lib/throttle.ts b/src/lib/throttle.ts new file mode 100644 index 00000000..d7bb98be --- /dev/null +++ b/src/lib/throttle.ts @@ -0,0 +1,61 @@ +/** + * The site-wide allowance: a hundred requests a minute, per caller, on every + * route. Going over is answered 402 with the crawl gateway's offer, not 429. + * + * WHY. Nothing here metered a page route. The polling cache below it in + * proxy.ts is a different tool for a different job -- it serves a repeat + * poller a cached body so the database is spared, and it only knows about + * four endpoints. A caller walking the job listings has never been counted at + * all. + * + * That is the shape that failed on coinpayportal on 2026-09-08: a headless + * browser found a route nobody had listed and walked 19,000 of its URLs a day + * for two days, declaring nothing, tripping no list. The gate sells to + * crawlers that say who they are; this sells to the ones that do not. + * + * Imports nothing Node-only: the proxy may run at the edge. + */ + +import { createThrottle } from '@profullstack/throttle'; +import { gateway } from '@/lib/crawl-gateway'; + +/** The Supabase session cookie, as a bucket key rather than a boolean. */ +function sessionKey(request: Request): string | null { + const cookie = request.headers.get('cookie') ?? ''; + return /(?:^|;\s*)(sb-[^=;]*-auth-token(?:\.\d+)?)=([^;]+)/.exec(cookie)?.[2] ?? null; +} + +export const throttle = createThrottle({ + gateway, + /* + * A signed-in member gets the larger budget rather than the anonymous one. + * The gate exempts a session outright -- it is deciding whether to charge a + * crawler, and a session is good evidence of a person. Here they are still + * counted, because an unmetered site for anyone willing to sign up first is + * a worse trade than metering a member generously. + */ + credentialFrom: (request) => + sessionKey(request) ?? + request.headers.get('x-api-key')?.trim() ?? + /^(\S+)\s+(\S+)/.exec(request.headers.get('authorization')?.trim() ?? '')?.[2] ?? + null, + credential: { limit: 600, ceiling: 1200 }, + rules: [ + /* Sign-in stays address-bucketed, or a guess buys the member budget. */ + { path: '/api/auth/', limit: 10, credential: false }, + /* + * The polled endpoints are already served from proxy.ts's cache when they + * repeat inside 30s, so a client with the page open costs the app nothing + * and should not be refused for keeping it open. + */ + { path: '/api/wallet/balance', limit: 600 }, + { path: '/api/wallet/transactions', limit: 600 }, + { path: '/api/notifications', limit: 600 }, + { path: '/api/funding/total', limit: 600 }, + /* Payment processors deliver on their own schedule; signature-verified. */ + { path: '/api/webhooks/', limit: 600, credential: false }, + ], +}); + +/** Resolves to a Response for a caller over the allowance, or undefined. */ +export const meter = (request: Request) => throttle.handle(request); diff --git a/src/proxy.ts b/src/proxy.ts index ff551030..9c51b24e 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -1,6 +1,7 @@ import { type NextRequest, NextResponse } from "next/server"; import { updateSession } from "@/lib/supabase/middleware"; import { gate } from "@/lib/crawl-gateway"; +import { meter } from "@/lib/throttle"; const REDIRECTS: Record = { // Pages now exist at /api-docs, /cli-docs, /openapi, /employers @@ -113,6 +114,14 @@ export async function proxy(request: NextRequest) { const answer = await gate(request); if (answer) return answer; + // Then the site-wide allowance, which meters every route: 100 requests a + // minute per caller, and going over is answered 402 with the same offer the + // gate makes rather than 429. The polling cache further down is a different + // tool -- it spares the database a repeat poll it already answered -- and it + // only ever knew about four endpoints. Nothing counted a page route at all. + const overLimit = await meter(request); + if (overLimit) return overLimit; + const ip = getClientIp(request); const method = request.method; const path = request.nextUrl.pathname;