From ec52251445ce83e18c289fe56311d440bfea01bc Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:34:10 +0000 Subject: [PATCH] Charge AI training crawlers for access (@profullstack/x402-gateway) Training crawlers (GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider, Applebot-Extended) get 402 Payment Required with an x402 offer, or the sales page at /crawl, and a paid pass opens the site for a day. People, search engines and retrieval crawlers pass through untouched. robots.txt is now generated from the same lists. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- .env.example | 7 +++- apps/web/app/robots.txt/route.js | 9 +++++ apps/web/lib/crawl-gateway.js | 27 +++++++++++++++ apps/web/middleware.js | 7 ++++ apps/web/package.json | 1 + apps/web/public/robots.txt | 27 --------------- pnpm-lock.yaml | 57 +++++++++++++++++++++++--------- 7 files changed, 91 insertions(+), 44 deletions(-) create mode 100644 apps/web/app/robots.txt/route.js create mode 100644 apps/web/lib/crawl-gateway.js delete mode 100644 apps/web/public/robots.txt diff --git a/.env.example b/.env.example index 4536e85..1c962da 100644 --- a/.env.example +++ b/.env.example @@ -38,4 +38,9 @@ EDGE_PRODUCT_ID=your-edge-product-id EDGE_CLIENT_ID=your-edge-client-id EDGE_API_KEY=your-edge-api-key -GITHUB_PAT=your-github-pat \ No newline at end of file +GITHUB_PAT=your-github-pat +# Crawl gateway (@profullstack/x402-gateway): AI training crawlers pay $1/day over +# x402. A SCOPED CoinPay key (payments:create) and the EVM address that receives +# the USDC. Unset = crawlers still get 402, nothing sold. +COINPAY_X402_KEY= +CRAWL_PAY_TO= diff --git a/apps/web/app/robots.txt/route.js b/apps/web/app/robots.txt/route.js new file mode 100644 index 0000000..5f40219 --- /dev/null +++ b/apps/web/app/robots.txt/route.js @@ -0,0 +1,9 @@ +import { robotsRoute } from "@profullstack/x402-gateway/next"; +import { gateway } from "@/lib/crawl-gateway"; + +// Generated from the same crawler lists the gateway enforces: training +// crawlers are refused everywhere but /crawl (where they can buy a pass), +// retrieval crawlers are named as welcome, everyone else gets the rules below. +export const GET = robotsRoute(gateway, { + disallow: ["/api/", "/dashboard"], +}); diff --git a/apps/web/lib/crawl-gateway.js b/apps/web/lib/crawl-gateway.js new file mode 100644 index 0000000..31e8983 --- /dev/null +++ b/apps/web/lib/crawl-gateway.js @@ -0,0 +1,27 @@ +import { createGateway } from "@profullstack/x402-gateway"; +import { x402Proxy } from "@profullstack/x402-gateway/next"; + +/** + * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, + * meta-externalagent, Bytespider, Applebot-Extended, ...) by the day over + * x402, settled by CoinPay in USDC. People, Googlebot and the retrieval + * crawlers behind AI search pass through untouched. + * + * Runs inside the middleware, so nothing here may import Node-only modules. + * The env is read through a non-literal key on purpose: Next inlines + * `process.env.NAME` at build time, and these are runtime secrets. Without + * COINPAY_X402_KEY and CRAWL_PAY_TO the gateway still answers training + * crawlers with 402, just with an empty offer. + */ +const env = (name) => process.env[name]; + +export const gateway = createGateway({ + siteUrl: env("SITE_URL") || env("NEXT_PUBLIC_SITE_URL") || "https://marksyncr.com", + siteName: "MarkSyncr", + coinpay: { apiKey: env("COINPAY_X402_KEY") }, + payTo: env("CRAWL_PAY_TO"), + contact: "mailto:support@marksyncr.com", +}); + +/** Resolves to a Response for a refused crawler, or undefined to carry on. */ +export const gate = x402Proxy(gateway); diff --git a/apps/web/middleware.js b/apps/web/middleware.js index 8c31708..3f6abf3 100644 --- a/apps/web/middleware.js +++ b/apps/web/middleware.js @@ -1,3 +1,4 @@ +import { gate } from "@/lib/crawl-gateway"; import { updateSession } from '@profullstack/stack/supabase'; /** @@ -9,6 +10,12 @@ import { updateSession } from '@profullstack/stack/supabase'; * if needed, and writes the updated cookies back to the response. */ export async function middleware(request) { + // Crawl gateway first: AI training crawlers get 402 Payment Required (or the + // sales page at /crawl) unless they present a paid pass. People, Googlebot + // and retrieval crawlers fall through to everything below. + const answer = await gate(request); + if (answer) return answer; + const { response } = await updateSession(request); return response; } diff --git a/apps/web/package.json b/apps/web/package.json index c3d2037..f7f3a9f 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -16,6 +16,7 @@ "@marksyncr/sources": "workspace:*", "@marksyncr/types": "workspace:*", "@profullstack/stack": "^0.1.3", + "@profullstack/x402-gateway": "^0.1.0", "@supabase/ssr": "^0.5.2", "@supabase/supabase-js": "^2.47.10", "next": "^16.1.0", diff --git a/apps/web/public/robots.txt b/apps/web/public/robots.txt deleted file mode 100644 index fdfc20b..0000000 --- a/apps/web/public/robots.txt +++ /dev/null @@ -1,27 +0,0 @@ -User-agent: * -Allow: / -Disallow: /api/ -Disallow: /dashboard - -User-agent: GPTBot -Allow: / - -User-agent: ClaudeBot -Allow: / - -User-agent: PerplexityBot -Allow: / - -User-agent: Google-Extended -Allow: / - -User-agent: OAI-SearchBot -Allow: / - -User-agent: Applebot-Extended -Allow: / - -User-agent: CCBot -Allow: / - -Sitemap: https://marksyncr.com/sitemap.xml diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e363b56..18c5538 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -129,7 +129,10 @@ importers: version: link:../../packages/types '@profullstack/stack': specifier: ^0.1.3 - version: 0.1.3(@supabase/ssr@0.5.2(@supabase/supabase-js@2.89.0))(next@16.1.0(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) + version: 0.1.3(@supabase/ssr@0.5.2(@supabase/supabase-js@2.89.0))(next@16.1.0(@babel/core@7.28.5)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) + '@profullstack/x402-gateway': + specifier: ^0.1.0 + version: 0.1.0 '@supabase/ssr': specifier: ^0.5.2 version: 0.5.2(@supabase/supabase-js@2.89.0) @@ -138,7 +141,7 @@ importers: version: 2.89.0 next: specifier: ^16.1.0 - version: 16.1.0(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + version: 16.1.0(@babel/core@7.28.5)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) posthog-js: specifier: ^1.381.0 version: 1.381.0 @@ -178,7 +181,7 @@ importers: version: 19.2.3(@types/react@19.2.7) '@vitejs/plugin-react': specifier: ^4.3.4 - version: 4.7.0(vite@6.4.1(@types/node@22.19.3)(jiti@1.21.7)(tsx@4.21.0)) + version: 4.7.0(vite@5.4.21(@types/node@22.19.3)) autoprefixer: specifier: ^10.4.20 version: 10.4.23(postcss@8.5.6) @@ -1220,6 +1223,10 @@ packages: react: optional: true + '@profullstack/x402-gateway@0.1.0': + resolution: {integrity: sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ==} + engines: {node: '>=20.11'} + '@rolldown/pluginutils@1.0.0-beta.27': resolution: {integrity: sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==} @@ -2418,6 +2425,7 @@ packages: eslint@9.39.0: resolution: {integrity: sha512-iy2GE3MHrYTL5lrCtMZ0X1KLEKKUjmK0kzwcnefhR66txcEmXZD2YWgR5GNdcEwkNx3a0siYkSvl0vIC+Svjmg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true peerDependencies: jiti: '*' @@ -2428,6 +2436,7 @@ packages: eslint@9.39.2: resolution: {integrity: sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true peerDependencies: jiti: '*' @@ -5074,15 +5083,17 @@ snapshots: optionalDependencies: react: 19.2.3 - '@profullstack/stack@0.1.3(@supabase/ssr@0.5.2(@supabase/supabase-js@2.89.0))(next@16.1.0(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)': + '@profullstack/stack@0.1.3(@supabase/ssr@0.5.2(@supabase/supabase-js@2.89.0))(next@16.1.0(@babel/core@7.28.5)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)': dependencies: '@profullstack/emailer': 1.0.1 '@profullstack/referrals': 0.1.0(react@19.2.3) optionalDependencies: '@supabase/ssr': 0.5.2(@supabase/supabase-js@2.89.0) - next: 16.1.0(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + next: 16.1.0(@babel/core@7.28.5)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) react: 19.2.3 + '@profullstack/x402-gateway@0.1.0': {} + '@rolldown/pluginutils@1.0.0-beta.27': {} '@rollup/rollup-android-arm-eabi@4.54.0': @@ -5448,6 +5459,18 @@ snapshots: '@unrs/resolver-binding-win32-x64-msvc@1.11.1': optional: true + '@vitejs/plugin-react@4.7.0(vite@5.4.21(@types/node@22.19.3))': + dependencies: + '@babel/core': 7.28.5 + '@babel/plugin-transform-react-jsx-self': 7.27.1(@babel/core@7.28.5) + '@babel/plugin-transform-react-jsx-source': 7.27.1(@babel/core@7.28.5) + '@rolldown/pluginutils': 1.0.0-beta.27 + '@types/babel__core': 7.20.5 + react-refresh: 0.17.0 + vite: 5.4.21(@types/node@22.19.3) + transitivePeerDependencies: + - supports-color + '@vitejs/plugin-react@4.7.0(vite@6.4.1(@types/node@22.19.3)(jiti@1.21.7)(tsx@4.21.0))': dependencies: '@babel/core': 7.28.5 @@ -6350,8 +6373,8 @@ snapshots: '@next/eslint-plugin-next': 16.1.0 eslint: 9.39.2(jiti@1.21.7) eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)) - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.2(jiti@1.21.7)) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.2(jiti@1.21.7)) eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.2(jiti@1.21.7)) eslint-plugin-react: 7.37.5(eslint@9.39.2(jiti@1.21.7)) eslint-plugin-react-hooks: 7.0.1(eslint@9.39.2(jiti@1.21.7)) @@ -6373,7 +6396,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)): + eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.2(jiti@1.21.7)): dependencies: '@nolyfill/is-core-module': 1.0.39 debug: 4.4.3 @@ -6384,22 +6407,22 @@ snapshots: tinyglobby: 0.2.15 unrs-resolver: 1.11.1 optionalDependencies: - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.2(jiti@1.21.7)) transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.1(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)): + eslint-module-utils@2.12.1(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.2(jiti@1.21.7)): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3) eslint: 9.39.2(jiti@1.21.7) eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.2(jiti@1.21.7)) transitivePeerDependencies: - supports-color - eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)): + eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.2(jiti@1.21.7)): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.9 @@ -6410,7 +6433,7 @@ snapshots: doctrine: 2.1.0 eslint: 9.39.2(jiti@1.21.7) eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7)) + eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.50.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.2(jiti@1.21.7)) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 @@ -7292,7 +7315,7 @@ snapshots: natural-compare@1.4.0: {} - next@16.1.0(react-dom@19.2.3(react@19.2.3))(react@19.2.3): + next@16.1.0(@babel/core@7.28.5)(react-dom@19.2.3(react@19.2.3))(react@19.2.3): dependencies: '@next/env': 16.1.0 '@swc/helpers': 0.5.15 @@ -7301,7 +7324,7 @@ snapshots: postcss: 8.4.31 react: 19.2.3 react-dom: 19.2.3(react@19.2.3) - styled-jsx: 5.1.6(react@19.2.3) + styled-jsx: 5.1.6(@babel/core@7.28.5)(react@19.2.3) optionalDependencies: '@next/swc-darwin-arm64': 16.1.0 '@next/swc-darwin-x64': 16.1.0 @@ -8053,10 +8076,12 @@ snapshots: stubborn-utils@1.0.2: {} - styled-jsx@5.1.6(react@19.2.3): + styled-jsx@5.1.6(@babel/core@7.28.5)(react@19.2.3): dependencies: client-only: 0.0.1 react: 19.2.3 + optionalDependencies: + '@babel/core': 7.28.5 sucrase@3.35.1: dependencies: