From 071acf6fffeac45da47e42ed5184c4408de7afdc Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:30:50 +0000 Subject: [PATCH 1/3] Charge AI training crawlers for access (@profullstack/x402-gateway) Training crawlers (GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider, Applebot-Extended) get 402 Payment Required with an x402 offer, or the sales page at /crawl, and a paid pass opens the site for a day. People, search engines and retrieval crawlers pass through untouched. robots.txt is now generated from the same lists. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- .env.example | 6 ++++ apps/logicsrc-web/package.json | 1 + apps/logicsrc-web/src/app/robots.ts | 30 ------------------- apps/logicsrc-web/src/app/robots.txt/route.ts | 9 ++++++ apps/logicsrc-web/src/lib/crawl-gateway.ts | 27 +++++++++++++++++ apps/logicsrc-web/src/proxy.ts | 9 +++++- package-lock.json | 10 +++++++ 7 files changed, 61 insertions(+), 31 deletions(-) delete mode 100644 apps/logicsrc-web/src/app/robots.ts create mode 100644 apps/logicsrc-web/src/app/robots.txt/route.ts create mode 100644 apps/logicsrc-web/src/lib/crawl-gateway.ts diff --git a/.env.example b/.env.example index 1db40cb..bc09c99 100644 --- a/.env.example +++ b/.env.example @@ -36,3 +36,9 @@ BLOG_WEBHOOK_SECRET= # Team credential sharing lives in its own app — see apps/pwa/.env.example # (Express + libSQL/Turso; auth + end-to-end-encrypted team vaults). + +# Crawl gateway (@profullstack/x402-gateway): AI training crawlers pay $1/day over +# x402. A SCOPED CoinPay key (payments:create) and the EVM address that receives +# the USDC. Unset = crawlers still get 402, nothing sold. +COINPAY_X402_KEY= +CRAWL_PAY_TO= diff --git a/apps/logicsrc-web/package.json b/apps/logicsrc-web/package.json index 1c9dae6..9dce280 100644 --- a/apps/logicsrc-web/package.json +++ b/apps/logicsrc-web/package.json @@ -16,6 +16,7 @@ "@logicsrc/openprd": "file:../../packages/openprd", "@profullstack/autoblog": "github:profullstack/autoblog#75e54af", "@profullstack/stack": "^0.1.3", + "@profullstack/x402-gateway": "^0.1.0", "@supabase/supabase-js": "^2.105.4", "marked": "^18.0.5", "next": "16.2.6", diff --git a/apps/logicsrc-web/src/app/robots.ts b/apps/logicsrc-web/src/app/robots.ts deleted file mode 100644 index 8597581..0000000 --- a/apps/logicsrc-web/src/app/robots.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { MetadataRoute } from "next"; - -const SITE_URL = (process.env.PUBLIC_URL ?? "https://logicsrc.com").replace(/\/$/, ""); - -// Welcome mainstream + AI crawlers; keep them out of the API surface. -export default function robots(): MetadataRoute.Robots { - return { - rules: [ - { - userAgent: [ - "*", - "GPTBot", - "OAI-SearchBot", - "ChatGPT-User", - "ClaudeBot", - "Claude-Web", - "anthropic-ai", - "PerplexityBot", - "Google-Extended", - "Applebot-Extended", - "CCBot", - ], - allow: "/", - disallow: ["/api/", "/health"], - }, - ], - sitemap: `${SITE_URL}/sitemap.xml`, - host: SITE_URL, - }; -} diff --git a/apps/logicsrc-web/src/app/robots.txt/route.ts b/apps/logicsrc-web/src/app/robots.txt/route.ts new file mode 100644 index 0000000..b6ae202 --- /dev/null +++ b/apps/logicsrc-web/src/app/robots.txt/route.ts @@ -0,0 +1,9 @@ +import { robotsRoute } from "@profullstack/x402-gateway/next"; +import { gateway } from "@/lib/crawl-gateway"; + +// Generated from the same crawler lists the gateway enforces: training +// crawlers are refused everywhere but /crawl (where they can buy a pass), +// retrieval crawlers are named as welcome, everyone else gets the rules below. +export const GET = robotsRoute(gateway, { + disallow: ["/api/", "/health"], +}); diff --git a/apps/logicsrc-web/src/lib/crawl-gateway.ts b/apps/logicsrc-web/src/lib/crawl-gateway.ts new file mode 100644 index 0000000..a26495b --- /dev/null +++ b/apps/logicsrc-web/src/lib/crawl-gateway.ts @@ -0,0 +1,27 @@ +import { createGateway } from "@profullstack/x402-gateway"; +import { x402Proxy } from "@profullstack/x402-gateway/next"; + +/** + * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, + * meta-externalagent, Bytespider, Applebot-Extended, ...) by the day over + * x402, settled by CoinPay in USDC. People, Googlebot and the retrieval + * crawlers behind AI search pass through untouched. + * + * Runs inside the middleware, so nothing here may import Node-only modules. + * The env is read through a non-literal key on purpose: Next inlines + * `process.env.NAME` at build time, and these are runtime secrets. Without + * COINPAY_X402_KEY and CRAWL_PAY_TO the gateway still answers training + * crawlers with 402, just with an empty offer. + */ +const env = (name: string) => process.env[name]; + +export const gateway = createGateway({ + siteUrl: env("SITE_URL") || env("NEXT_PUBLIC_SITE_URL") || "https://logicsrc.com", + siteName: "LogicSRC", + coinpay: { apiKey: env("COINPAY_X402_KEY") }, + payTo: env("CRAWL_PAY_TO"), + contact: "mailto:support@logicsrc.com", +}); + +/** Resolves to a Response for a refused crawler, or undefined to carry on. */ +export const gate = x402Proxy(gateway); diff --git a/apps/logicsrc-web/src/proxy.ts b/apps/logicsrc-web/src/proxy.ts index 3a319c0..bfa8419 100644 --- a/apps/logicsrc-web/src/proxy.ts +++ b/apps/logicsrc-web/src/proxy.ts @@ -1,3 +1,4 @@ +import { gate } from "@/lib/crawl-gateway"; import { NextResponse } from "next/server"; import type { NextRequest } from "next/server"; @@ -6,7 +7,13 @@ import type { NextRequest } from "next/server"; // This is the Next 16 "proxy" (formerly middleware) entrypoint. const ALLOWED_APEX = process.env.PUBLIC_DOMAIN || "logicsrc.com"; -export function proxy(request: NextRequest): NextResponse { +export async function proxy(request: NextRequest): Promise { + // Crawl gateway first: AI training crawlers get 402 Payment Required (or the + // sales page at /crawl) unless they present a paid pass. People, Googlebot + // and retrieval crawlers fall through to everything below. + const answer = await gate(request); + if (answer) return answer; + const host = request.headers.get("host") ?? ""; if (host === `www.${ALLOWED_APEX}`) { const { pathname, search } = request.nextUrl; diff --git a/package-lock.json b/package-lock.json index 8e8a2ad..2daa765 100644 --- a/package-lock.json +++ b/package-lock.json @@ -141,6 +141,7 @@ "@logicsrc/openprd": "file:../../packages/openprd", "@profullstack/autoblog": "github:profullstack/autoblog#75e54af", "@profullstack/stack": "^0.1.3", + "@profullstack/x402-gateway": "^0.1.0", "@supabase/supabase-js": "^2.105.4", "marked": "^18.0.5", "next": "16.2.6", @@ -2618,6 +2619,15 @@ } } }, + "node_modules/@profullstack/x402-gateway": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@profullstack/x402-gateway/-/x402-gateway-0.1.0.tgz", + "integrity": "sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ==", + "license": "MIT", + "engines": { + "node": ">=20.11" + } + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.3.tgz", From c3c5406c70987f1244d1c9d1f615a66e57ec59cd Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:39:18 +0000 Subject: [PATCH 2/3] Type the middleware as returning Response | NextResponse The crawl gateway answers with a plain Fetch Response. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- apps/logicsrc-web/src/proxy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/logicsrc-web/src/proxy.ts b/apps/logicsrc-web/src/proxy.ts index bfa8419..a7eeb01 100644 --- a/apps/logicsrc-web/src/proxy.ts +++ b/apps/logicsrc-web/src/proxy.ts @@ -7,7 +7,7 @@ import type { NextRequest } from "next/server"; // This is the Next 16 "proxy" (formerly middleware) entrypoint. const ALLOWED_APEX = process.env.PUBLIC_DOMAIN || "logicsrc.com"; -export async function proxy(request: NextRequest): Promise { +export async function proxy(request: NextRequest): Promise { // Crawl gateway first: AI training crawlers get 402 Payment Required (or the // sales page at /crawl) unless they present a paid pass. People, Googlebot // and retrieval crawlers fall through to everything below. From b09121ac5238278a0dbf4906db7a2c8e86658289 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:45:34 +0000 Subject: [PATCH 3/3] Contract test awaits the now-async proxy Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- apps/logicsrc-web/contract/logicsrc-web.contract.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/logicsrc-web/contract/logicsrc-web.contract.test.ts b/apps/logicsrc-web/contract/logicsrc-web.contract.test.ts index 849765f..7e20271 100644 --- a/apps/logicsrc-web/contract/logicsrc-web.contract.test.ts +++ b/apps/logicsrc-web/contract/logicsrc-web.contract.test.ts @@ -60,21 +60,21 @@ function jsonResponse(body: unknown, status = 200): Response { } describe("www canonical redirect (proxy.ts)", () => { - it("301s www to the apex host over https, preserving path + query", () => { + it("301s www to the apex host over https, preserving path + query", async () => { const request = new NextRequest("https://www.logicsrc.com/openspec?ref=email", { headers: { host: "www.logicsrc.com" } }); - const response = proxy(request); + const response = await proxy(request); expect(response.status).toBe(301); expect(response.headers.get("location")).toBe("https://logicsrc.com/openspec?ref=email"); }); - it("passes through apex requests untouched", () => { + it("passes through apex requests untouched", async () => { const request = new NextRequest("https://logicsrc.com/hire-us", { headers: { host: "logicsrc.com" } }); - const response = proxy(request); + const response = await proxy(request); // NextResponse.next() yields a non-redirect response. expect(response.status).toBe(200);