Refuse private targets on the audit's page fetch

lib/net-guard.ts

@@ -0,0 +1,68 @@
+/**
+ * Refuse to reach into private address space.
+ *
+ * An audit fetches whatever URL a customer submits, which is the product,
+ * and also the textbook shape of server-side request forgery: a submitted
+ * `http://169.254.169.254/` or `http://10.0.0.5/` would have our server read
+ * from the cloud metadata service or a neighbour. So the host is resolved
+ * and every answer checked before the request goes out, and the final URL
+ * is checked again after redirects, since a public name can bounce inward.
+ *
+ * Same rules as outreach's mailbox discovery, kept here so the audit path
+ * does not import that module's dependencies.
+ */
+
+import dns from "node:dns/promises";
+import net from "node:net";
+
+export function isPrivateAddress(addr: string): boolean {
+  if (net.isIPv4(addr)) {
+    const [a, b] = addr.split(".").map(Number);
+    if (a === 10 || a === 127 || a === 0) return true;
+    if (a === 172 && b >= 16 && b <= 31) return true;
+    if (a === 192 && b === 168) return true;
+    if (a === 169 && b === 254) return true;
+    if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT
+    if (a >= 224) return true; // multicast + reserved
+    return false;
+  }
+  if (net.isIPv6(addr)) {
+    const v6 = addr.toLowerCase();
+    if (v6 === "::1" || v6 === "::") return true;
+    if (v6.startsWith("fe80") || v6.startsWith("fc") || v6.startsWith("fd")) return true;
+    const mapped = v6.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
+    if (mapped) return isPrivateAddress(mapped[1]);
+    return false;
+  }
+  return true;
+}
+
+export class PrivateTargetError extends Error {
+  constructor(public readonly host: string) {
+    super(`refusing to fetch ${host}: it resolves to a private address`);
+    this.name = "PrivateTargetError";
+  }
+}
+
+/**
+ * Throw unless every address the URL's host resolves to is public.
+ *
+ * `localhost` and bare IPs are covered; a name that does not resolve is
+ * refused too, since there is nothing public to reach.
+ */
+export async function assertPublicTarget(url: string): Promise<void> {
+  const host = new URL(url).hostname.replace(/^\[|\]$/g, "");
+  let addrs: string[];
+  if (net.isIP(host)) {
+    addrs = [host];
+  } else if (host === "localhost" || host.endsWith(".localhost")) {
+    addrs = ["127.0.0.1"];
+  } else {
+    try {
+      addrs = (await dns.lookup(host, { all: true })).map((a) => a.address);
+    } catch {
+      addrs = [];
+    }
+  }
+  if (!addrs.length || addrs.some(isPrivateAddress)) throw new PrivateTargetError(host);
+}

lib/paid-fetch.ts

@@ -1,26 +1,59 @@
 /**
- * The fetch the audit reaches a customer's site with — paying when asked.
+ * The fetch the audit reaches a customer's site with — guarded, and paying
+ * when asked.
  *
- * A site behind an x402 gateway answers a crawler with 402 and an offer. With
- * `X402_PRIVATE_KEY` set, the client signs the offer with the shared crawler
- * wallet, buys the pass, files it by origin and presents it on every later
- * request to that site, so an audit of a gated site reads the site rather
- * than the sales page. Without the key this is the global fetch, unchanged.
+ * Guarded: the URL comes from a customer, so the host is resolved and
+ * refused if any answer is private, and the final URL is checked again after
+ * redirects. See lib/net-guard.ts. `CRAWLPROOF_ALLOW_PRIVATE_TARGETS=1`
+ * switches that off for auditing a local server in development, and for
+ * nothing else.
+ *
+ * Paying: a site behind an x402 gateway answers a crawler with 402 and an
+ * offer. With `X402_PRIVATE_KEY` set, the client signs the offer with the
+ * shared crawler wallet, buys the pass, files it by origin and presents it on
+ * every later request to that site, so an audit of a gated site reads the
+ * site rather than the sales page. Without the key it is the global fetch.
  *
  * Only the page fetch goes through here. Link-status probes and uptime pings
  * touch many third-party domains per run, and paying a dollar to learn a
  * link's status would be waste; those stay on the plain fetch.
  *
- * Capped at five dollars a payment. The key is read through a non-literal
- * accessor because Next inlines `process.env.NAME` at build time.
+ * Capped at five dollars a payment. Both env keys are read through
+ * non-literal accessors because Next inlines `process.env.NAME` at build.
  */
 
 import { createClient } from "@profullstack/x402-client";
 
+import { assertPublicTarget, PrivateTargetError } from "./net-guard";
+
 const key = process.env[["X402", "PRIVATE_KEY"].join("_")];
+const allowPrivate = process.env[["CRAWLPROOF", "ALLOW_PRIVATE_TARGETS"].join("_")] === "1";
 
 export const x402 = key ? createClient({ key, maxUsd: 5 }) : null;
 
-export const paidFetch: typeof fetch = x402
-  ? (input, init) => x402.fetch(input, init)
-  : (input, init) => fetch(input, init);
+const underlying: typeof fetch = x402 ? (input, init) => x402.fetch(input, init) : (input, init) => fetch(input, init);
+
+function urlOf(input: string | URL | Request): string {
+  if (typeof input === "string") return input;
+  if (input instanceof URL) return input.href;
+  return input.url;
+}
+
+export const paidFetch: typeof fetch = async (input, init) => {
+  const url = urlOf(input);
+  if (!allowPrivate) await assertPublicTarget(url);
+  const res = await underlying(input, init);
+  // A redirect can land inside even when the origin was public.
+  if (!allowPrivate && res.url && new URL(res.url).hostname !== new URL(url).hostname) {
+    try {
+      await assertPublicTarget(res.url);
+    } catch (err) {
+      if (err instanceof PrivateTargetError) {
+        res.body?.cancel().catch(() => {});
+        throw err;
+      }
+      throw err;
+    }
+  }
+  return res;
+};

tests/paid-fetch.test.ts

@@ -0,0 +1,38 @@
+import { describe, expect, it } from "vitest";
+
+import { assertPublicTarget, isPrivateAddress, PrivateTargetError } from "@/lib/net-guard";
+import { paidFetch, x402 } from "@/lib/paid-fetch";
+
+describe("net-guard", () => {
+  it("knows private address space", () => {
+    for (const a of ["127.0.0.1", "10.1.2.3", "172.16.0.1", "172.31.255.255", "192.168.1.1", "169.254.169.254", "100.64.0.1", "0.0.0.0", "224.0.0.1", "::1", "fe80::1", "fd00::1", "::ffff:10.0.0.1"]) {
+      expect(isPrivateAddress(a), a).toBe(true);
+    }
+    for (const a of ["1.1.1.1", "8.8.8.8", "172.32.0.1", "104.18.0.1", "2606:4700::1111"]) {
+      expect(isPrivateAddress(a), a).toBe(false);
+    }
+    expect(isPrivateAddress("not-an-ip")).toBe(true);
+  });
+
+  it("refuses loopback, private and metadata targets before any request", async () => {
+    for (const url of ["http://127.0.0.1:1/", "http://localhost:3000/", "http://[::1]/", "http://169.254.169.254/latest/meta-data/", "http://10.0.0.5/admin"]) {
+      await expect(assertPublicTarget(url), url).rejects.toBeInstanceOf(PrivateTargetError);
+    }
+  });
+
+  it("accepts a public address", async () => {
+    await expect(assertPublicTarget("https://1.1.1.1/")).resolves.toBeUndefined();
+  });
+});
+
+describe("paidFetch", () => {
+  it("is the plain fetch when no key is configured", () => {
+    expect(x402).toBeNull();
+  });
+
+  it("never connects to a private target", async () => {
+    // Port 1 on loopback: if the guard were missing this would be a
+    // connection error, not a PrivateTargetError.
+    await expect(paidFetch("http://127.0.0.1:1/")).rejects.toBeInstanceOf(PrivateTargetError);
+  });
+});