diff --git a/.env.example b/.env.example index d7a3a029..34f23c94 100644 --- a/.env.example +++ b/.env.example @@ -54,6 +54,15 @@ COINPAYPORTAL_OAUTH_BASE_URL=https://coinpayportal.com # Optional: Override API URL for testing # COINPAYPORTAL_API_URL=https://coinpayportal.com/api +# Crawl gateway (@profullstack/x402-gateway): AI training crawlers pay $1/day +# over x402 (USDC), settled by CoinPay. Until BOTH are set, training crawlers +# still get 402 but with an empty offer (nothing sold, nothing given away). +# A SCOPED CoinPay key (cp_live_..., API Keys tab) with payments:create; +# the legacy business key is refused by CoinPay's x402 routes. +COINPAY_X402_KEY= +# EVM address that receives the USDC (Base, Polygon and Ethereum alike) +CRAWL_PAY_TO= + # Subscription Plans (in cents) PREMIUM_PLAN_PRICE_CENTS=499 FAMILY_PLAN_PRICE_CENTS=999 diff --git a/middleware.ts b/middleware.ts deleted file mode 100644 index 4ed3344b..00000000 --- a/middleware.ts +++ /dev/null @@ -1,254 +0,0 @@ -/** - * Next.js Middleware - * - * Handles auth token refresh for Supabase session management. - * Runs before every matching route to ensure the auth cookie - * has fresh tokens. Without this, expired access tokens cause - * `getCurrentUser()` to fail because the refreshed tokens from - * `setSession()` are never written back to the cookie. - * - * This is the standard approach for Supabase + Next.js App Router auth. - * - * CIRCUIT BREAKER: Under memory pressure or repeated failures, - * token refresh is skipped to prevent cascading failures. - * The stale token will be handled gracefully by getCurrentUser(). - */ - -import { NextRequest, NextResponse } from 'next/server'; -import { trackReferralCode } from '@profullstack/stack/referrals'; - -const AUTH_COOKIE_NAME = 'sb-auth-token'; -const COOKIE_MAX_AGE = 7 * 24 * 60 * 60; // 7 days - -// Token refresh timeout - short to prevent blocking requests -const REFRESH_TIMEOUT_MS = 3000; // 3 seconds - -/** - * Circuit breaker state for token refresh - * Prevents cascading failures when the system is under pressure - */ -let consecutiveFailures = 0; -let lastFailureTime = 0; -const MAX_CONSECUTIVE_FAILURES = 10; -const CIRCUIT_RESET_MS = 15000; // 15 seconds - -/** - * Check if circuit breaker is open (should skip refresh) - */ -function isCircuitOpen(): boolean { - if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) { - // Check if enough time has passed to reset - if (Date.now() - lastFailureTime > CIRCUIT_RESET_MS) { - consecutiveFailures = 0; - return false; - } - return true; - } - return false; -} - -/** - * Record a failure for circuit breaker - */ -function recordFailure(): void { - consecutiveFailures++; - lastFailureTime = Date.now(); -} - -/** - * Record a success - reset circuit breaker - */ -function recordSuccess(): void { - consecutiveFailures = 0; -} - -/** - * Decode a JWT payload without verifying signature. - * Used only to check expiry — actual validation is done by Supabase. - */ -function decodeJwtPayload(token: string): { exp?: number } | null { - try { - const parts = token.split('.'); - if (parts.length !== 3) return null; - const payload = Buffer.from(parts[1], 'base64url').toString('utf-8'); - return JSON.parse(payload) as { exp?: number }; - } catch { - return null; - } -} - -interface SessionTokens { - access_token: string; - refresh_token: string; -} - -interface SupabaseRefreshResponse { - access_token: string; - refresh_token: string; - expires_in: number; - token_type: string; -} - -function getClientIp(request: NextRequest): string { - return ( - request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() || - request.headers.get("x-real-ip") || - "unknown" - ); -} - -export default async function middleware(request: NextRequest): Promise { - // Log real client IP for API requests - const path = request.nextUrl.pathname; - if (path.startsWith("/api/")) { - console.log(`[${request.method}] ${path} — ${getClientIp(request)}`); - } - - const response = NextResponse.next(); - - const authCookie = request.cookies.get(AUTH_COOKIE_NAME); - if (!authCookie?.value) { - return response; - } - - // Parse the stored session - let session: SessionTokens; - try { - session = JSON.parse(decodeURIComponent(authCookie.value)) as SessionTokens; - } catch { - return response; - } - - if (!session.access_token || !session.refresh_token) { - return response; - } - - // Check if the access token is expired or about to expire (within 60s) - const payload = decodeJwtPayload(session.access_token); - if (!payload?.exp) { - return response; - } - - const now = Math.floor(Date.now() / 1000); - const timeUntilExpiry = payload.exp - now; - - // Token still fresh (more than 60 seconds until expiry) — no refresh needed - if (timeUntilExpiry > 60) { - return response; - } - - // Token expired or about to expire — refresh it - const supabaseUrl = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL; - const supabaseAnonKey = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || process.env.SUPABASE_ANON_KEY; - - if (!supabaseUrl || !supabaseAnonKey) { - // Only log once per circuit reset to avoid spam - if (consecutiveFailures === 0) { - console.error('[Middleware] Missing SUPABASE_URL or SUPABASE_ANON_KEY for token refresh'); - } - return response; - } - - // Circuit breaker: skip refresh if we've had too many recent failures - if (isCircuitOpen()) { - // Silently skip - don't spam logs when circuit is open - return response; - } - - try { - // Use AbortController for timeout to prevent hanging requests - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), REFRESH_TIMEOUT_MS); - - const refreshResponse = await fetch(`${supabaseUrl}/auth/v1/token?grant_type=refresh_token`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'apikey': supabaseAnonKey, - }, - body: JSON.stringify({ refresh_token: session.refresh_token }), - signal: controller.signal, - }); - - clearTimeout(timeoutId); - - if (!refreshResponse.ok) { - recordFailure(); - // Only log if this is the first failure in a series - if (consecutiveFailures === 1) { - console.error('[Middleware] Token refresh failed:', refreshResponse.status); - } - // DON'T clear the cookie on refresh failure — the stale token - // may still work for API routes that do their own refresh via setSession(). - // Only clear on 401 (token truly revoked), not on transient errors. - if (refreshResponse.status === 401) { - response.cookies.set(AUTH_COOKIE_NAME, '', { - path: '/', - httpOnly: true, - sameSite: 'lax', - secure: process.env.NODE_ENV === 'production', - maxAge: 0, - }); - } - return response; - } - - const data = await refreshResponse.json() as SupabaseRefreshResponse; - - if (!data.access_token || !data.refresh_token) { - recordFailure(); - if (consecutiveFailures === 1) { - console.error('[Middleware] Token refresh returned incomplete data'); - } - return response; - } - - // Success! Reset circuit breaker - recordSuccess(); - - // Write the refreshed tokens back to the cookie - const newCookieValue = encodeURIComponent( - JSON.stringify({ - access_token: data.access_token, - refresh_token: data.refresh_token, - }) - ); - - response.cookies.set(AUTH_COOKIE_NAME, newCookieValue, { - path: '/', - httpOnly: true, - sameSite: 'lax', - secure: process.env.NODE_ENV === 'production', - maxAge: COOKIE_MAX_AGE, - }); - } catch (error) { - recordFailure(); - // Only log first failure to avoid spam during outages - if (consecutiveFailures === 1) { - console.error('[Middleware] Token refresh error:', error); - } - // Don't break the request — let getCurrentUser() handle the stale token - } - - const ref = request.nextUrl.searchParams.get('ref'); - // Validate ref before storing: alphanumeric + hyphens/underscores, max 64 chars. - // Without validation, an attacker can inject arbitrary values via a crafted URL, - // enabling referral fraud and overflowing the cookie header. - if (ref && /^[a-zA-Z0-9_-]{1,64}$/.test(ref)) { - trackReferralCode(request, response); - } - return response; -} - -export const config = { - matcher: [ - /* - * Match all routes except: - * - _next/static (static files) - * - _next/image (image optimization) - * - favicon.ico, logo.svg, etc. - * - Public assets - */ - '/((?!_next/static|_next/image|favicon\\.ico|logo\\.svg|.*\\.(?:png|jpg|jpeg|gif|webp|svg|ico|woff2?|ttf|eot|css|js|map)$).*)', - ], -}; diff --git a/package.json b/package.json index d8902ab1..aae9f30e 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "@profullstack/player": "0.3.1", "@profullstack/referrals": "^0.1.0", "@profullstack/stack": "^0.1.3", + "@profullstack/x402-gateway": "0.1.0", "@radix-ui/react-dialog": "^1.1.23", "@radix-ui/react-dropdown-menu": "^2.1.24", "@radix-ui/react-icons": "^1.3.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 38ce42e4..dc96adec 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -37,6 +37,9 @@ importers: '@profullstack/stack': specifier: ^0.1.3 version: 0.1.3(next@16.3.3(@babel/core@7.29.7)(@playwright/test@1.62.1)(@types/node@26.4.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8))(react@19.2.8) + '@profullstack/x402-gateway': + specifier: 0.1.0 + version: 0.1.0 '@radix-ui/react-dialog': specifier: ^1.1.23 version: 1.1.23(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -1314,6 +1317,10 @@ packages: react: optional: true + '@profullstack/x402-gateway@0.1.0': + resolution: {integrity: sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ==} + engines: {node: '>=20.11'} + '@puppeteer/browsers@3.2.1': resolution: {integrity: sha512-KDz+3qDRdBAlRlMjmKyj6dEs33YHTk/xRHEENSXq6TNnhgoU15ruSHtEBeVF6OZ9tBDY55Se4P0nFMNsipzU9A==} engines: {node: '>=22.12.0'} @@ -6204,6 +6211,8 @@ snapshots: next: 16.3.3(@babel/core@7.29.7)(@playwright/test@1.62.1)(@types/node@26.4.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) react: 19.2.8 + '@profullstack/x402-gateway@0.1.0': {} + '@puppeteer/browsers@3.2.1(yauzl@2.10.0)': dependencies: modern-tar: 0.8.4 diff --git a/public/ai.txt b/public/ai.txt index 8caacdc6..9cb52fde 100644 --- a/public/ai.txt +++ b/public/ai.txt @@ -2,12 +2,16 @@ # Base: https://bittorrented.com # Contact: support@bittorrented.com # Models: * -# Capabilities: chat, embed, fine_tune, crawl, train +# Capabilities: chat, embed, crawl +# +# Training crawlers pay for access: $1 buys a day, settled over x402 (USDC). +# Read https://bittorrented.com/crawl for how. Search and retrieval crawlers +# read free, and robots.txt names which is which. User-agent: * Allow: /* -Training: allow -Retention: allow -Commercial-Use: allow +Training: disallow +Retention: disallow +Commercial-Use: disallow Rate-Limit-RPS: 10 diff --git a/public/robots.txt b/public/robots.txt deleted file mode 100644 index a3e6a331..00000000 --- a/public/robots.txt +++ /dev/null @@ -1,5 +0,0 @@ -User-agent: * -Disallow: /api/ -Disallow: /login -Disallow: /settings -Sitemap: https://bittorrented.com/sitemap.xml diff --git a/src/app/robots.ts b/src/app/robots.ts deleted file mode 100644 index 3fa1c358..00000000 --- a/src/app/robots.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { MetadataRoute } from 'next'; - -export default function robots(): MetadataRoute.Robots { - return { - rules: [ - { - userAgent: '*', - allow: '/', - disallow: ['/api/', '/dashboard/'], - }, - ], - sitemap: 'https://bittorrented.com/sitemap.xml', - }; -} diff --git a/src/app/robots.txt/route.test.ts b/src/app/robots.txt/route.test.ts new file mode 100644 index 00000000..bece002e --- /dev/null +++ b/src/app/robots.txt/route.test.ts @@ -0,0 +1,77 @@ +/** + * robots.txt Route Tests + * + * Behavior: + * - Training crawlers (GPTBot, meta-externalagent, ...) are refused with Disallow: / + * and may only read /crawl, where they can pay for access + * - Retrieval crawlers (OAI-SearchBot, ...) are named and allowed + * - Private paths (/api/, /login, /settings, /dashboard/) stay disallowed for everyone + */ + +import { describe, it, expect } from 'vitest'; +import { GET } from './route'; + +/** Split robots.txt into groups, keyed by the user agent that opens each one. */ +function groupsOf(text: string): Map { + const groups = new Map(); + for (const block of text.split(/\n\s*\n/)) { + const lines = block.trim().split('\n'); + const agent = lines[0]?.match(/^User-agent:\s*(.+)$/i)?.[1]; + if (agent) groups.set(agent, lines.slice(1)); + } + return groups; +} + +async function fetchRobots(): Promise<{ status: number; type: string | null; text: string }> { + const res = GET(); + return { status: res.status, type: res.headers.get('content-type'), text: await res.text() }; +} + +describe('robots.txt route', () => { + it('serves plain text', async () => { + const { status, type } = await fetchRobots(); + expect(status).toBe(200); + expect(type).toContain('text/plain'); + }); + + it('refuses GPTBot everywhere but the sales page', async () => { + const { text } = await fetchRobots(); + const rules = groupsOf(text).get('GPTBot'); + expect(rules).toBeDefined(); + expect(rules).toContain('Disallow: /'); + expect(rules).toContain('Allow: /crawl'); + }); + + it('refuses meta-externalagent everywhere but the sales page', async () => { + const { text } = await fetchRobots(); + const rules = groupsOf(text).get('meta-externalagent'); + expect(rules).toBeDefined(); + expect(rules).toContain('Disallow: /'); + expect(rules).toContain('Allow: /crawl'); + }); + + it('names OAI-SearchBot as welcome', async () => { + const { text } = await fetchRobots(); + const rules = groupsOf(text).get('OAI-SearchBot'); + expect(rules).toBeDefined(); + expect(rules).toContain('Allow: /'); + expect(rules).not.toContain('Disallow: /'); + }); + + it('keeps private paths disallowed for everyone', async () => { + const { text } = await fetchRobots(); + const groups = groupsOf(text); + for (const agent of ['*', 'OAI-SearchBot']) { + const rules = groups.get(agent); + expect(rules).toContain('Disallow: /api/'); + expect(rules).toContain('Disallow: /login'); + expect(rules).toContain('Disallow: /settings'); + expect(rules).toContain('Disallow: /dashboard/'); + } + }); + + it('points at the sitemap', async () => { + const { text } = await fetchRobots(); + expect(text).toMatch(/^Sitemap: https?:\/\/.+\/sitemap\.xml$/m); + }); +}); diff --git a/src/app/robots.txt/route.ts b/src/app/robots.txt/route.ts new file mode 100644 index 00000000..76f39f30 --- /dev/null +++ b/src/app/robots.txt/route.ts @@ -0,0 +1,13 @@ +/** + * robots.txt, generated from the crawl gateway's lists so the file and the + * gate agree: training crawlers are refused everywhere but /crawl (where they + * can pay), retrieval crawlers are named as welcome, everyone else gets the + * wildcard rules. + */ + +import { robotsRoute } from '@profullstack/x402-gateway/next'; +import { gateway } from '@/lib/crawl-gateway'; + +export const GET = robotsRoute(gateway, { + disallow: ['/api/', '/login', '/settings', '/dashboard/'], +}); diff --git a/src/lib/crawl-gateway.ts b/src/lib/crawl-gateway.ts new file mode 100644 index 00000000..dec579fa --- /dev/null +++ b/src/lib/crawl-gateway.ts @@ -0,0 +1,22 @@ +/** + * Crawl gateway: sells a day of crawl access to AI training crawlers over x402. + * + * Training crawlers (GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider, + * Applebot-Extended, ...) get `402 Payment Required` with an x402 offer, or the + * HTML sales page at /crawl. A paid pass in the `x-crawl-pass` header lets them + * through. People, Googlebot and retrieval crawlers are untouched. + * + * Used by src/proxy.ts (the gate) and src/app/robots.txt/route.ts (the lists), + * so robots.txt and the gate never disagree about who is who. + * + * Imports nothing Node-only: the proxy may run at the edge. + */ + +import { createGateway } from '@profullstack/x402-gateway'; + +export const gateway = createGateway({ + siteUrl: process.env.NEXT_PUBLIC_APP_URL ?? 'https://bittorrented.com', + siteName: 'bittorrented', + coinpay: { apiKey: process.env.COINPAY_X402_KEY }, + payTo: process.env.CRAWL_PAY_TO, +}); diff --git a/src/proxy.test.ts b/src/proxy.test.ts index 4beb79a4..9da55223 100644 --- a/src/proxy.test.ts +++ b/src/proxy.test.ts @@ -2,15 +2,26 @@ * Middleware Tests — Rate limiting and bot handling on API routes * * Behavior: + * - Training crawlers (GPTBot, meta-externalagent, ...): 402 Payment Required + * with an x402 offer on every route (the crawl gateway runs first) * - Good bots (Googlebot, Bingbot, Applebot): rate-limited (10/min), NOT blocked * - Bad bots on expensive routes (/api/search/*, /api/dht/*): blocked (403) * - Bad bots on other API routes: rate-limited (5/min), allowed through * - Normal browsers: rate-limited on expensive routes (30/min) + * - Supabase session: refreshed (cookie rewritten) when the access token expires within 60s + * - ?ref=CODE: stored in the referral_code cookie when valid */ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { proxy as middleware } from './proxy'; -import { NextRequest } from 'next/server'; +import { NextRequest, NextResponse } from 'next/server'; + +/** A response that lets the request carry on to the app (NextResponse.next()). */ +function expectPassThrough(res: Response | undefined): asserts res is NextResponse { + expect(res).toBeDefined(); + expect(res!.status).toBe(200); + expect(res!.headers.get('x-middleware-next')).toBe('1'); +} describe('Bot Handling Middleware', () => { function callMiddleware(pathname: string, userAgent: string | null) { @@ -24,57 +35,57 @@ describe('Bot Handling Middleware', () => { return middleware(req); } - it('should allow Googlebot on non-expensive API routes (rate-limited, not blocked)', () => { - const res = callMiddleware('/api/torrents/123', 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'); + it('should allow Googlebot on non-expensive API routes (rate-limited, not blocked)', async () => { + const res = await callMiddleware('/api/torrents/123', 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'); // Good bots are allowed through (rate-limited at 10/min but first request passes) - expect(res).toBeUndefined(); + expectPassThrough(res); }); - it('should allow Bingbot on non-expensive API routes', () => { - const res = callMiddleware('/api/stream', 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)'); - expect(res).toBeUndefined(); + it('should allow Bingbot on non-expensive API routes', async () => { + const res = await callMiddleware('/api/stream', 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)'); + expectPassThrough(res); }); - it('should block bad bots from expensive API routes with 403', () => { - const res = callMiddleware('/api/search/torrents', 'SomeBot/1.0'); + it('should block bad bots from expensive API routes with 403', async () => { + const res = await callMiddleware('/api/search/torrents', 'SomeBot/1.0'); expect(res).toBeDefined(); expect(res!.status).toBe(403); }); - it('should block GPTBot from expensive API routes', () => { - const res = callMiddleware('/api/dht/browse', 'GPTBot/1.0'); + it('should charge GPTBot on expensive API routes (402 from the gateway, not 403)', async () => { + const res = await callMiddleware('/api/dht/browse', 'GPTBot/1.0'); expect(res).toBeDefined(); - expect(res!.status).toBe(403); + expect(res!.status).toBe(402); }); - it('should allow bad bots on non-expensive API routes (rate-limited)', () => { + it('should allow bad bots on non-expensive API routes (rate-limited)', async () => { // Bad bots on non-expensive routes are rate-limited but not immediately blocked - const res = callMiddleware('/api/torrents/123', 'SomeBot/1.0'); - expect(res).toBeUndefined(); + const res = await callMiddleware('/api/torrents/123', 'SomeBot/1.0'); + expectPassThrough(res); }); - it('should allow normal browsers to access API routes', () => { - const res = callMiddleware('/api/torrents/123', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'); - expect(res).toBeUndefined(); + it('should allow normal browsers to access API routes', async () => { + const res = await callMiddleware('/api/torrents/123', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'); + expectPassThrough(res); }); - it('should allow requests with no user-agent', () => { - const res = callMiddleware('/api/torrents/123', null); - expect(res).toBeUndefined(); + it('should allow requests with no user-agent', async () => { + const res = await callMiddleware('/api/torrents/123', null); + expectPassThrough(res); }); - it('should not block bots from non-API routes', () => { - const res = callMiddleware('/torrents/123', 'Googlebot/2.1'); - expect(res).toBeUndefined(); + it('should not block bots from non-API routes', async () => { + const res = await callMiddleware('/torrents/123', 'Googlebot/2.1'); + expectPassThrough(res); }); - it('should block AhrefsBot from expensive API routes', () => { - const res = callMiddleware('/api/search/torrents', 'Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)'); + it('should block AhrefsBot from expensive API routes', async () => { + const res = await callMiddleware('/api/search/torrents', 'Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)'); expect(res).toBeDefined(); expect(res!.status).toBe(403); }); - it('should block social media preview bots from expensive API routes', () => { + it('should block social media preview bots from expensive API routes', async () => { const agents = [ 'facebookexternalhit/1.1', 'Twitterbot/1.0', @@ -84,9 +95,211 @@ describe('Bot Handling Middleware', () => { 'Discordbot/2.0', ]; for (const ua of agents) { - const res = callMiddleware('/api/search/torrents', ua); + const res = await callMiddleware('/api/search/torrents', ua); expect(res).toBeDefined(); expect(res!.status).toBe(403); } }); }); + +describe('Crawl Gateway (x402)', () => { + const CHROME_UA = + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'; + const META_UA = 'meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)'; + + function callProxy(pathname: string, userAgent: string | null, headers: Record = {}) { + const url = new URL(`http://localhost${pathname}`); + const req = new NextRequest(url, { + headers: { + ...(userAgent ? { 'user-agent': userAgent } : {}), + 'x-forwarded-for': `${Math.random().toString(36).slice(2)}.1.1.1`, + ...headers, + }, + }); + return middleware(req); + } + + it('answers meta-externalagent on a page route with 402 and an x402 offer', async () => { + const res = await callProxy('/browse', META_UA); + expect(res).toBeDefined(); + expect(res!.status).toBe(402); + expect(res!.headers.get('content-type')).toContain('application/json'); + const body = (await res!.json()) as { x402Version: number; accepts: unknown[]; pass: { buy: string } }; + expect(body.x402Version).toBe(2); + expect(Array.isArray(body.accepts)).toBe(true); + expect(body.pass.buy).toMatch(/\/crawl$/); + }); + + it('answers a training crawler that asks for HTML with the 402 sales page', async () => { + const res = await callProxy('/browse', META_UA, { accept: 'text/html,application/xhtml+xml' }); + expect(res).toBeDefined(); + expect(res!.status).toBe(402); + expect(res!.headers.get('content-type')).toContain('text/html'); + }); + + it('lets a training crawler read robots.txt', async () => { + const res = await callProxy('/robots.txt', META_UA); + expectPassThrough(res); + }); + + it('passes a Chrome browser through to the existing behaviour', async () => { + const res = await callProxy('/browse', CHROME_UA); + expectPassThrough(res); + }); + + it('passes a Chrome browser through on expensive API routes (rate limit, not 402)', async () => { + const res = await callProxy('/api/search/torrents', CHROME_UA); + expectPassThrough(res); + }); + + it('passes Googlebot and a retrieval crawler through', async () => { + expectPassThrough(await callProxy('/browse', 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)')); + expectPassThrough(await callProxy('/browse', 'Mozilla/5.0 (compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)')); + }); + + it('serves the sales page at /crawl to anyone, including a browser', async () => { + const res = await callProxy('/crawl', CHROME_UA, { accept: 'text/html' }); + expect(res).toBeDefined(); + expect(res!.status).toBe(402); + expect(res!.headers.get('content-type')).toContain('text/html'); + }); +}); + +describe('Supabase session refresh and referral cookie', () => { + const CHROME_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/128.0.0.0 Safari/537.36'; + const META_UA = 'meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)'; + const SUPABASE_URL = 'https://sb.test'; + + /** An unsigned JWT whose payload carries only `exp`. */ + function jwt(expiresInSeconds: number): string { + const b64url = (s: string) => Buffer.from(s).toString('base64url'); + const exp = Math.floor(Date.now() / 1000) + expiresInSeconds; + return `${b64url(JSON.stringify({ alg: 'none' }))}.${b64url(JSON.stringify({ exp }))}.sig`; + } + + function authCookie(expiresInSeconds: number, refreshToken = 'old-refresh'): string { + return encodeURIComponent(JSON.stringify({ access_token: jwt(expiresInSeconds), refresh_token: refreshToken })); + } + + function call( + pathname: string, + { ua = CHROME_UA, cookies = {}, headers = {} }: { ua?: string; cookies?: Record; headers?: Record } = {} + ) { + const cookie = Object.entries(cookies).map(([k, v]) => `${k}=${v}`).join('; '); + const req = new NextRequest(new URL(`http://localhost${pathname}`), { + headers: { + 'user-agent': ua, + 'x-forwarded-for': `${Math.random().toString(36).slice(2)}.1.1.1`, + ...(cookie ? { cookie } : {}), + ...headers, + }, + }); + return middleware(req); + } + + const fetchMock = vi.fn(); + + beforeEach(() => { + vi.stubEnv('SUPABASE_URL', SUPABASE_URL); + vi.stubEnv('SUPABASE_ANON_KEY', 'anon-key'); + fetchMock.mockReset(); + fetchMock.mockResolvedValue( + new Response(JSON.stringify({ access_token: jwt(3600), refresh_token: 'new-refresh', expires_in: 3600, token_type: 'bearer' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + ); + vi.stubGlobal('fetch', fetchMock); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + }); + + it('refreshes an expiring session for a browser and writes the new tokens back', async () => { + const res = await call('/browse', { cookies: { 'sb-auth-token': authCookie(30), 'x-profile-id': 'p1' } }); + expectPassThrough(res); + + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(url).toBe(`${SUPABASE_URL}/auth/v1/token?grant_type=refresh_token`); + expect(init.method).toBe('POST'); + expect((init.headers as Record)['apikey']).toBe('anon-key'); + expect(JSON.parse(init.body as string)).toEqual({ refresh_token: 'old-refresh' }); + + const written = res.cookies.get('sb-auth-token'); + expect(written).toBeDefined(); + expect(written!.httpOnly).toBe(true); + expect(written!.maxAge).toBe(7 * 24 * 60 * 60); + expect(JSON.parse(decodeURIComponent(written!.value)).refresh_token).toBe('new-refresh'); + }); + + it('leaves a fresh session alone', async () => { + const res = await call('/browse', { cookies: { 'sb-auth-token': authCookie(3600), 'x-profile-id': 'p1' } }); + expectPassThrough(res); + expect(fetchMock).not.toHaveBeenCalled(); + expect(res.cookies.get('sb-auth-token')).toBeUndefined(); + }); + + it('does nothing for a browser with no session', async () => { + const res = await call('/browse'); + expectPassThrough(res); + expect(fetchMock).not.toHaveBeenCalled(); + expect(res.headers.get('set-cookie')).toBeNull(); + }); + + it('keeps the refreshed session on the select-profile redirect', async () => { + const res = await call('/library', { cookies: { 'sb-auth-token': authCookie(30) } }); + expect(res).toBeDefined(); + expect(res!.status).toBe(307); + expect(new URL(res!.headers.get('location')!).pathname).toBe('/select-profile'); + const written = (res as NextResponse).cookies.get('sb-auth-token'); + expect(JSON.parse(decodeURIComponent(written!.value)).refresh_token).toBe('new-refresh'); + }); + + it('clears the cookie when Supabase says the refresh token is revoked (401)', async () => { + fetchMock.mockResolvedValueOnce(new Response('{"error":"invalid_grant"}', { status: 401 })); + const res = await call('/browse', { cookies: { 'sb-auth-token': authCookie(30), 'x-profile-id': 'p1' } }); + expectPassThrough(res); + const written = res.cookies.get('sb-auth-token'); + expect(written!.value).toBe(''); + expect(written!.maxAge).toBe(0); + }); + + it('keeps the stale cookie on a transient refresh failure', async () => { + fetchMock.mockResolvedValueOnce(new Response('oops', { status: 503 })); + const res = await call('/browse', { cookies: { 'sb-auth-token': authCookie(30), 'x-profile-id': 'p1' } }); + expectPassThrough(res); + expect(res.cookies.get('sb-auth-token')).toBeUndefined(); + }); + + it('answers a training crawler with 402 without touching Supabase', async () => { + const res = await call('/browse', { ua: META_UA, cookies: { 'sb-auth-token': authCookie(30) } }); + expect(res).toBeDefined(); + expect(res!.status).toBe(402); + expect(fetchMock).not.toHaveBeenCalled(); + expect(res!.headers.get('set-cookie')).toBeNull(); + }); + + it('stores a valid ?ref= code in the referral_code cookie', async () => { + const res = await call('/browse?ref=ABC-123_x'); + expectPassThrough(res); + const cookie = res.cookies.get('referral_code'); + expect(cookie?.value).toBe('ABC-123_x'); + expect(cookie?.httpOnly).toBe(false); + }); + + it('ignores a malformed ?ref=', async () => { + const res = await call('/browse?ref=' + encodeURIComponent('