diff --git a/package.json b/package.json index 6058fff..ec257b9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/x402-gateway", - "version": "0.2.1", + "version": "0.2.2", "type": "module", "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, and a robots.txt that keeps search crawlers welcome.", "keywords": [ diff --git a/src/edge.js b/src/edge.js index 1f10154..a85deeb 100644 --- a/src/edge.js +++ b/src/edge.js @@ -84,6 +84,17 @@ export function clientIp(request) { const CLAIMS_CHROMIUM = /\bChrome\/\d+/; +/** + * A crawler that says so. Googlebot's evergreen string is + * "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; + * Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36" + * -- it claims Chrome, it sends no Sec-Fetch-Mode, and it is the last thing + * on earth to charge. Bingbot is built the same way. Anything that declares + * itself is judged by the lists, never by this check: the whole point of the + * check is the client that declares nothing. + */ +const DECLARES_ITSELF = /compatible;|\bbot\b|bot\/|crawler|spider|slurp/i; + /** * A request that claims a Chromium user agent but carries none of the * fetch-metadata headers Chromium cannot omit. @@ -97,5 +108,6 @@ const CLAIMS_CHROMIUM = /\bChrome\/\d+/; export function isSpoofedBrowser(request) { const ua = request.headers.get('user-agent') ?? ''; if (!CLAIMS_CHROMIUM.test(ua)) return false; + if (DECLARES_ITSELF.test(ua)) return false; return !request.headers.has('sec-fetch-mode'); } diff --git a/test/gateway.test.js b/test/gateway.test.js index a693223..938011d 100644 --- a/test/gateway.test.js +++ b/test/gateway.test.js @@ -389,6 +389,17 @@ describe('crawlers that do not say who they are', () => { assert.equal(isSpoofedBrowser(req('/', { ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0' })), false, 'Firefox is not judged'); assert.equal(isSpoofedBrowser(req('/', { ua: 'curl/8.0' })), false, 'an honest client is not judged either'); assert.equal(isSpoofedBrowser(req('/', { ua: META })), false, 'a declared crawler is charged by name, not by this'); + // The evergreen search crawlers claim Chrome and send no Sec-Fetch-Mode. + const GOOGLEBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/145.0.0.0 Safari/537.36'; + const BINGBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/145.0.0.0 Safari/537.36'; + const APPLEBOT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)'; + for (const ua of [GOOGLEBOT, BINGBOT, APPLEBOT]) assert.equal(isSpoofedBrowser(req('/', { ua })), false, ua.slice(0, 40)); + }); + + it('never charges Googlebot, with or without the spoof check', async () => { + const GOOGLEBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/145.0.0.0 Safari/537.36'; + const { gateway } = gatewayFor({ chargeSpoofedBrowsers: true }); + assert.equal(await gateway.handle(req('/topics/x', { ua: GOOGLEBOT })), null); }); it('charges a spoofed browser only when asked to, and never one that answers the question', async () => {