0.2.2: spoof check skips declared crawlers (Googlebot evergreen UA)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/x402-gateway",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "type": "module",
   "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, and a robots.txt that keeps search crawlers welcome.",
   "keywords": [

src/edge.js

@@ -84,6 +84,17 @@ export function clientIp(request) {
 
 const CLAIMS_CHROMIUM = /\bChrome\/\d+/;
 
+/**
+ * A crawler that says so. Googlebot's evergreen string is
+ * "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible;
+ * Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36"
+ * -- it claims Chrome, it sends no Sec-Fetch-Mode, and it is the last thing
+ * on earth to charge. Bingbot is built the same way. Anything that declares
+ * itself is judged by the lists, never by this check: the whole point of the
+ * check is the client that declares nothing.
+ */
+const DECLARES_ITSELF = /compatible;|\bbot\b|bot\/|crawler|spider|slurp/i;
+
 /**
  * A request that claims a Chromium user agent but carries none of the
  * fetch-metadata headers Chromium cannot omit.
@@ -97,5 +108,6 @@ const CLAIMS_CHROMIUM = /\bChrome\/\d+/;
 export function isSpoofedBrowser(request) {
   const ua = request.headers.get('user-agent') ?? '';
   if (!CLAIMS_CHROMIUM.test(ua)) return false;
+  if (DECLARES_ITSELF.test(ua)) return false;
   return !request.headers.has('sec-fetch-mode');
 }

test/gateway.test.js

@@ -389,6 +389,17 @@ describe('crawlers that do not say who they are', () => {
     assert.equal(isSpoofedBrowser(req('/', { ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0' })), false, 'Firefox is not judged');
     assert.equal(isSpoofedBrowser(req('/', { ua: 'curl/8.0' })), false, 'an honest client is not judged either');
     assert.equal(isSpoofedBrowser(req('/', { ua: META })), false, 'a declared crawler is charged by name, not by this');
+    // The evergreen search crawlers claim Chrome and send no Sec-Fetch-Mode.
+    const GOOGLEBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/145.0.0.0 Safari/537.36';
+    const BINGBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/145.0.0.0 Safari/537.36';
+    const APPLEBOT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)';
+    for (const ua of [GOOGLEBOT, BINGBOT, APPLEBOT]) assert.equal(isSpoofedBrowser(req('/', { ua })), false, ua.slice(0, 40));
+  });
+
+  it('never charges Googlebot, with or without the spoof check', async () => {
+    const GOOGLEBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/145.0.0.0 Safari/537.36';
+    const { gateway } = gatewayFor({ chargeSpoofedBrowsers: true });
+    assert.equal(await gateway.handle(req('/topics/x', { ua: GOOGLEBOT })), null);
   });
 
   it('charges a spoofed browser only when asked to, and never one that answers the question', async () => {