Serve the site on every core, not on one of twenty-four

apps/web/Dockerfile

@@ -32,8 +32,17 @@ ENV HOSTNAME=0.0.0.0
 # The web service gets 24 GB on Railway; letting the heap use half of it turns
 # a surge that would have been fatal into one that is merely slow. This is a
 # ceiling, not a reservation — it costs nothing until it is needed.
+#
+# This is the ceiling for a single server, which is what the primary process and
+# a WEB_WORKERS=1 deployment are. When the primary forks workers it computes
+# each one's share of the container's memory and passes it on their command
+# line, where it wins over this — see src/lib/workers.js. Sizing it per worker
+# here instead would mean this number had to be re-derived by hand every time
+# the container changed shape.
 ENV NODE_OPTIONS=--max-old-space-size=12288
 
-# server.mjs is `next start` plus a ceiling on concurrent requests. Reasoning,
-# and the outage behind it, in src/lib/loadShed.js.
+# server.mjs is `next start`, plus a ceiling on concurrent requests, plus one
+# copy of the server per CPU the container is allowed. Reasoning, and the two
+# different outages behind the two parts, in src/lib/loadShed.js (memory) and
+# src/lib/workers.js (CPU).
 CMD ["node", "server.mjs"]

apps/web/server.mjs

@@ -3,54 +3,98 @@ import { createServer } from 'node:http';
 import next from 'next';
 
 import { admit, inflight } from './src/lib/loadShed.js';
+import { forkWorkers, workerCount } from './src/lib/workers.js';
 
 /**
- * The HTTP server, with a ceiling on concurrent work.
+ * The HTTP server, on every core, with a ceiling on concurrent work.
  *
- * This replaces `next start`, and does one thing `next start` cannot: it
- * refuses a request when too many are already in flight. Reasoning, and the
- * outage that motivated it, in src/lib/loadShed.js. Everything else is what
- * `next start` does — the platform's PORT and HOSTNAME, Next's own request
- * handler, no options of our own.
+ * This replaces `next start`, and does two things `next start` cannot. It
+ * refuses a request when too many are already in flight — reasoning, and the
+ * outage that motivated it, in src/lib/loadShed.js. And it runs one copy of the
+ * server per CPU the container is allowed, because JavaScript renders a page on
+ * one thread and a single copy leaves the rest of the machine idle while the
+ * site is down — reasoning, and *that* outage, in src/lib/workers.js.
+ *
+ * Everything else is what `next start` does — the platform's PORT and HOSTNAME,
+ * Next's own request handler, no options of our own.
  *
  * The port and host come from the environment and nothing else. Railway
  * injects PORT, and a hardcoded value here would leave the edge proxy
  * forwarding to a port nothing listens on — see the note in the Dockerfile.
+ *
+ * Every worker listens on the same port; `node:cluster` gives the primary the
+ * socket and hands connections round-robin. Nothing below needs to know whether
+ * it is the only server or one of sixteen, with one exception worth naming: all
+ * of the module state behind these requests — the throttle's counters, the
+ * traffic tally, the verified-key cache — is now per worker rather than per
+ * container. For the counters that bound *memory* that is the correct place for
+ * them, and `loadShed` divides its allowance so the container-wide total is
+ * unchanged. For the counters that meter *a caller* it is a loosening: a client
+ * holding a keep-alive connection stays on one worker, so its own limit is
+ * intact, but a caller opening fresh connections is metered by each worker
+ * separately. That is deliberate. The traffic this was written for arrives one
+ * request per address and defeats a per-caller limit outright, and tightening
+ * those limits by a factor of sixteen during an outage would refuse readers to
+ * no purpose. See src/lib/workers.js on why capacity is not a defence.
  */
 
 const port = Number(process.env.PORT) || 3000;
 const hostname = process.env.HOSTNAME || '0.0.0.0';
 
-const app = next({ dev: false, hostname, port });
-const handle = app.getRequestHandler();
+// The primary forks and then has nothing to do. It must not go on to stand up
+// Next and bind the port itself: that would put a seventeenth server on the
+// socket with none of the workers' heap settings.
+if (
+  forkWorkers({
+    onExit: ({ pid, code, signal }) => {
+      console.warn(`[web] worker ${pid} exited (code ${code}, signal ${signal}), replacing it`);
+    },
+  })
+) {
+  console.log(`[web] primary ${process.pid} running ${workerCount()} workers`);
+} else {
+  await serve();
+}
 
-await app.prepare();
+/**
+ * Stand up Next and answer requests until the process ends.
+ *
+ * @returns {Promise<void>}
+ */
+async function serve() {
+  const app = next({ dev: false, hostname, port });
+  const handle = app.getRequestHandler();
 
-/** Say so when refusing starts, and then once a minute while it goes on. */
-let lastNoted = 0;
+  await app.prepare();
 
-const server = createServer((req, res) => {
-  const release = admit(pathOf(req.url));
+  const server = createServer((req, res) => {
+    const release = admit(pathOf(req.url));
 
-  if (release === null) {
-    refuse(res);
-    return;
-  }
+    if (release === null) {
+      refuse(res);
+      return;
+    }
 
-  // `close` fires whether the response finished or the socket died under it,
-  // which is the one event that means the request is no longer costing us.
-  res.once('close', release);
+    // `close` fires whether the response finished or the socket died under it,
+    // which is the one event that means the request is no longer costing us.
+    res.once('close', release);
 
-  handle(req, res).catch((err) => {
-    console.error('[web] request failed', err);
-    if (!res.headersSent) res.statusCode = 500;
-    res.end();
+    handle(req, res).catch((err) => {
+      console.error('[web] request failed', err);
+      if (!res.headersSent) res.statusCode = 500;
+      res.end();
+    });
   });
-});
 
-server.listen(port, hostname, () => {
-  console.log(`[web] listening on http://${hostname}:${port}, in-flight cap ${inflight().limit}`);
-});
+  server.listen(port, hostname, () => {
+    console.log(
+      `[web] ${process.pid} listening on http://${hostname}:${port}, in-flight cap ${inflight().limit}`,
+    );
+  });
+}
+
+/** Say so when refusing starts, and then once a minute while it goes on. */
+let lastNoted = 0;
 
 /**
  * The refusal: 503, tiny, uncacheable, with a Retry-After a client can obey.
@@ -67,7 +111,9 @@ function refuse(res) {
   if (now - lastNoted > 60_000) {
     lastNoted = now;
     const { active, limit, refused } = inflight();
-    console.warn(`[web] shedding load: ${active}/${limit} in flight, ${refused} refused so far`);
+    console.warn(
+      `[web] ${process.pid} shedding load: ${active}/${limit} in flight, ${refused} refused so far`,
+    );
   }
 
   res.writeHead(503, {

apps/web/src/lib/loadShed.js

@@ -49,8 +49,10 @@
  * standing Next up.
  */
 
+import { share } from './workers.js';
+
 /**
- * How many requests may be in flight.
+ * How many requests may be in flight, across the whole container.
  *
  * Sized from the incident: the process survived an hour at roughly 20 a
  * second with sub-second responses, which is fewer than twenty in flight, and
@@ -60,6 +62,12 @@
  * process's footprint is bounded at the cap times one request's worth of
  * work — tens of megabytes at the top end — rather than at whatever the
  * arrival rate happens to be.
+ *
+ * Since 2026-09-07 the container runs one server per CPU rather than one in
+ * total (`workers.js`), and this number is divided between them. It stayed as
+ * a container-wide figure on purpose: it was sized against a container's heap,
+ * and giving each of sixteen workers the whole of it would raise the real
+ * ceiling to 2,048 and hand back the outage it was written to prevent.
  */
 const DEFAULT_LIMIT = 128;
 
@@ -75,7 +83,12 @@ const DEFAULT_LIMIT = 128;
 const ALWAYS = /^\/(?:_next\/static\/|icons\/|favicon\.ico$|manifest\.webmanifest$|sw\.js$|robots\.txt$)/;
 
 /**
- * The limit, from the environment when it is set to something sensible.
+ * This process's limit, from the environment when it is set to something
+ * sensible.
+ *
+ * `WEB_MAX_INFLIGHT` is read as a container-wide number, like the default it
+ * replaces, and divided the same way — so the dial keeps meaning what it meant
+ * before there were workers, and raising it does not have to be done per CPU.
  *
  * Read through a non-literal property access for the reason `lib/db.js` gives,
  * and junk falls back to the default rather than to unlimited, for the reason
@@ -86,7 +99,8 @@ const ALWAYS = /^\/(?:_next\/static\/|icons\/|favicon\.ico$|manifest\.webmanifes
 export function limit() {
   const env = process.env;
   const raw = Number(env['WEB_MAX_INFLIGHT']);
-  return Number.isInteger(raw) && raw > 0 ? raw : DEFAULT_LIMIT;
+  const total = Number.isInteger(raw) && raw > 0 ? raw : DEFAULT_LIMIT;
+  return share(total);
 }
 
 /** Requests currently being worked on. */