From 4fd79717bcddbd48585ed87ea43e7564474ed711 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 19:42:02 +0000 Subject: [PATCH] Refuse private targets on the audit's page fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeQL flagged the paying fetch in #231 as server-side request forgery, and it was right about the shape if not the novelty: the audit fetches whatever URL a customer submits, and nothing checked where that resolved. A submitted http://169.254.169.254/ would have read the cloud metadata service from our server. The old fetch(url) in smartFetch had the same hole; the wrapper only made the flow visible. lib/net-guard.ts resolves the host and refuses any private, loopback, link-local, CGNAT or multicast answer — the same rules outreach's mailbox discovery applies — before the request, and again on the final URL after redirects. CRAWLPROOF_ALLOW_PRIVATE_TARGETS=1 turns it off for auditing a local server in development. Link-status probes and uptime pings remain on the plain fetch as before. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01NSGS7Sy3QmEgTNZDg4eq7g --- lib/net-guard.ts | 68 ++++++++++++++++++++++++++++++++++++++++ lib/paid-fetch.ts | 55 +++++++++++++++++++++++++------- tests/paid-fetch.test.ts | 38 ++++++++++++++++++++++ 3 files changed, 150 insertions(+), 11 deletions(-) create mode 100644 lib/net-guard.ts create mode 100644 tests/paid-fetch.test.ts diff --git a/lib/net-guard.ts b/lib/net-guard.ts new file mode 100644 index 0000000..6341dca --- /dev/null +++ b/lib/net-guard.ts @@ -0,0 +1,68 @@ +/** + * Refuse to reach into private address space. + * + * An audit fetches whatever URL a customer submits, which is the product, + * and also the textbook shape of server-side request forgery: a submitted + * `http://169.254.169.254/` or `http://10.0.0.5/` would have our server read + * from the cloud metadata service or a neighbour. So the host is resolved + * and every answer checked before the request goes out, and the final URL + * is checked again after redirects, since a public name can bounce inward. + * + * Same rules as outreach's mailbox discovery, kept here so the audit path + * does not import that module's dependencies. + */ + +import dns from "node:dns/promises"; +import net from "node:net"; + +export function isPrivateAddress(addr: string): boolean { + if (net.isIPv4(addr)) { + const [a, b] = addr.split(".").map(Number); + if (a === 10 || a === 127 || a === 0) return true; + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + if (a === 169 && b === 254) return true; + if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT + if (a >= 224) return true; // multicast + reserved + return false; + } + if (net.isIPv6(addr)) { + const v6 = addr.toLowerCase(); + if (v6 === "::1" || v6 === "::") return true; + if (v6.startsWith("fe80") || v6.startsWith("fc") || v6.startsWith("fd")) return true; + const mapped = v6.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); + if (mapped) return isPrivateAddress(mapped[1]); + return false; + } + return true; +} + +export class PrivateTargetError extends Error { + constructor(public readonly host: string) { + super(`refusing to fetch ${host}: it resolves to a private address`); + this.name = "PrivateTargetError"; + } +} + +/** + * Throw unless every address the URL's host resolves to is public. + * + * `localhost` and bare IPs are covered; a name that does not resolve is + * refused too, since there is nothing public to reach. + */ +export async function assertPublicTarget(url: string): Promise { + const host = new URL(url).hostname.replace(/^\[|\]$/g, ""); + let addrs: string[]; + if (net.isIP(host)) { + addrs = [host]; + } else if (host === "localhost" || host.endsWith(".localhost")) { + addrs = ["127.0.0.1"]; + } else { + try { + addrs = (await dns.lookup(host, { all: true })).map((a) => a.address); + } catch { + addrs = []; + } + } + if (!addrs.length || addrs.some(isPrivateAddress)) throw new PrivateTargetError(host); +} diff --git a/lib/paid-fetch.ts b/lib/paid-fetch.ts index e590074..d538188 100644 --- a/lib/paid-fetch.ts +++ b/lib/paid-fetch.ts @@ -1,26 +1,59 @@ /** - * The fetch the audit reaches a customer's site with — paying when asked. + * The fetch the audit reaches a customer's site with — guarded, and paying + * when asked. * - * A site behind an x402 gateway answers a crawler with 402 and an offer. With - * `X402_PRIVATE_KEY` set, the client signs the offer with the shared crawler - * wallet, buys the pass, files it by origin and presents it on every later - * request to that site, so an audit of a gated site reads the site rather - * than the sales page. Without the key this is the global fetch, unchanged. + * Guarded: the URL comes from a customer, so the host is resolved and + * refused if any answer is private, and the final URL is checked again after + * redirects. See lib/net-guard.ts. `CRAWLPROOF_ALLOW_PRIVATE_TARGETS=1` + * switches that off for auditing a local server in development, and for + * nothing else. + * + * Paying: a site behind an x402 gateway answers a crawler with 402 and an + * offer. With `X402_PRIVATE_KEY` set, the client signs the offer with the + * shared crawler wallet, buys the pass, files it by origin and presents it on + * every later request to that site, so an audit of a gated site reads the + * site rather than the sales page. Without the key it is the global fetch. * * Only the page fetch goes through here. Link-status probes and uptime pings * touch many third-party domains per run, and paying a dollar to learn a * link's status would be waste; those stay on the plain fetch. * - * Capped at five dollars a payment. The key is read through a non-literal - * accessor because Next inlines `process.env.NAME` at build time. + * Capped at five dollars a payment. Both env keys are read through + * non-literal accessors because Next inlines `process.env.NAME` at build. */ import { createClient } from "@profullstack/x402-client"; +import { assertPublicTarget, PrivateTargetError } from "./net-guard"; + const key = process.env[["X402", "PRIVATE_KEY"].join("_")]; +const allowPrivate = process.env[["CRAWLPROOF", "ALLOW_PRIVATE_TARGETS"].join("_")] === "1"; export const x402 = key ? createClient({ key, maxUsd: 5 }) : null; -export const paidFetch: typeof fetch = x402 - ? (input, init) => x402.fetch(input, init) - : (input, init) => fetch(input, init); +const underlying: typeof fetch = x402 ? (input, init) => x402.fetch(input, init) : (input, init) => fetch(input, init); + +function urlOf(input: string | URL | Request): string { + if (typeof input === "string") return input; + if (input instanceof URL) return input.href; + return input.url; +} + +export const paidFetch: typeof fetch = async (input, init) => { + const url = urlOf(input); + if (!allowPrivate) await assertPublicTarget(url); + const res = await underlying(input, init); + // A redirect can land inside even when the origin was public. + if (!allowPrivate && res.url && new URL(res.url).hostname !== new URL(url).hostname) { + try { + await assertPublicTarget(res.url); + } catch (err) { + if (err instanceof PrivateTargetError) { + res.body?.cancel().catch(() => {}); + throw err; + } + throw err; + } + } + return res; +}; diff --git a/tests/paid-fetch.test.ts b/tests/paid-fetch.test.ts new file mode 100644 index 0000000..b74df03 --- /dev/null +++ b/tests/paid-fetch.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; + +import { assertPublicTarget, isPrivateAddress, PrivateTargetError } from "@/lib/net-guard"; +import { paidFetch, x402 } from "@/lib/paid-fetch"; + +describe("net-guard", () => { + it("knows private address space", () => { + for (const a of ["127.0.0.1", "10.1.2.3", "172.16.0.1", "172.31.255.255", "192.168.1.1", "169.254.169.254", "100.64.0.1", "0.0.0.0", "224.0.0.1", "::1", "fe80::1", "fd00::1", "::ffff:10.0.0.1"]) { + expect(isPrivateAddress(a), a).toBe(true); + } + for (const a of ["1.1.1.1", "8.8.8.8", "172.32.0.1", "104.18.0.1", "2606:4700::1111"]) { + expect(isPrivateAddress(a), a).toBe(false); + } + expect(isPrivateAddress("not-an-ip")).toBe(true); + }); + + it("refuses loopback, private and metadata targets before any request", async () => { + for (const url of ["http://127.0.0.1:1/", "http://localhost:3000/", "http://[::1]/", "http://169.254.169.254/latest/meta-data/", "http://10.0.0.5/admin"]) { + await expect(assertPublicTarget(url), url).rejects.toBeInstanceOf(PrivateTargetError); + } + }); + + it("accepts a public address", async () => { + await expect(assertPublicTarget("https://1.1.1.1/")).resolves.toBeUndefined(); + }); +}); + +describe("paidFetch", () => { + it("is the plain fetch when no key is configured", () => { + expect(x402).toBeNull(); + }); + + it("never connects to a private target", async () => { + // Port 1 on loopback: if the guard were missing this would be a + // connection error, not a PrivateTargetError. + await expect(paidFetch("http://127.0.0.1:1/")).rejects.toBeInstanceOf(PrivateTargetError); + }); +});