From 8ca583f2bc0067dc2e9ceb3f9af45939474b4ff5 Mon Sep 17 00:00:00 2001
From: Anthony Ettinger <anthony@chovy.com>
Date: Sat, 5 Sep 2026 17:10:50 +0000
Subject: [PATCH] 0.2.2: the spoof check never judges a crawler that declares
 itself

Googlebot's evergreen user agent claims Chrome and sends no Sec-Fetch-Mode,
so 0.2.0's spoof check would have charged Google -- and Bing -- the moment a
site switched it on. Caught on rssamplifier before it shipped, by the site's
own exemption; now the library refuses to judge anything that says
"compatible;", "bot", "crawler", "spider" or "slurp". Those are charged by
the lists or not at all. The check is for the client that declares nothing.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WJaXiqE9BDoNfoJBhfXroC
---
 package.json         |  2 +-
 src/edge.js          | 12 ++++++++++++
 test/gateway.test.js | 11 +++++++++++
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 6058fff..ec257b9 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/x402-gateway",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "type": "module",
   "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, and a robots.txt that keeps search crawlers welcome.",
   "keywords": [
diff --git a/src/edge.js b/src/edge.js
index 1f10154..a85deeb 100644
--- a/src/edge.js
+++ b/src/edge.js
@@ -84,6 +84,17 @@ export function clientIp(request) {
 
 const CLAIMS_CHROMIUM = /\bChrome\/\d+/;
 
+/**
+ * A crawler that says so. Googlebot's evergreen string is
+ * "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible;
+ * Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36"
+ * -- it claims Chrome, it sends no Sec-Fetch-Mode, and it is the last thing
+ * on earth to charge. Bingbot is built the same way. Anything that declares
+ * itself is judged by the lists, never by this check: the whole point of the
+ * check is the client that declares nothing.
+ */
+const DECLARES_ITSELF = /compatible;|\bbot\b|bot\/|crawler|spider|slurp/i;
+
 /**
  * A request that claims a Chromium user agent but carries none of the
  * fetch-metadata headers Chromium cannot omit.
@@ -97,5 +108,6 @@ const CLAIMS_CHROMIUM = /\bChrome\/\d+/;
 export function isSpoofedBrowser(request) {
   const ua = request.headers.get('user-agent') ?? '';
   if (!CLAIMS_CHROMIUM.test(ua)) return false;
+  if (DECLARES_ITSELF.test(ua)) return false;
   return !request.headers.has('sec-fetch-mode');
 }
diff --git a/test/gateway.test.js b/test/gateway.test.js
index a693223..938011d 100644
--- a/test/gateway.test.js
+++ b/test/gateway.test.js
@@ -389,6 +389,17 @@ describe('crawlers that do not say who they are', () => {
     assert.equal(isSpoofedBrowser(req('/', { ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0' })), false, 'Firefox is not judged');
     assert.equal(isSpoofedBrowser(req('/', { ua: 'curl/8.0' })), false, 'an honest client is not judged either');
     assert.equal(isSpoofedBrowser(req('/', { ua: META })), false, 'a declared crawler is charged by name, not by this');
+    // The evergreen search crawlers claim Chrome and send no Sec-Fetch-Mode.
+    const GOOGLEBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/145.0.0.0 Safari/537.36';
+    const BINGBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/145.0.0.0 Safari/537.36';
+    const APPLEBOT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)';
+    for (const ua of [GOOGLEBOT, BINGBOT, APPLEBOT]) assert.equal(isSpoofedBrowser(req('/', { ua })), false, ua.slice(0, 40));
+  });
+
+  it('never charges Googlebot, with or without the spoof check', async () => {
+    const GOOGLEBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/145.0.0.0 Safari/537.36';
+    const { gateway } = gatewayFor({ chargeSpoofedBrowsers: true });
+    assert.equal(await gateway.handle(req('/topics/x', { ua: GOOGLEBOT })), null);
   });
 
   it('charges a spoofed browser only when asked to, and never one that answers the question', async () => {