From c36f01de3886f50d2ba2ec93d34a1e6d00c624b2 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Fri, 11 Sep 2026 22:19:20 +0800 Subject: [PATCH 01/26] The ledger's single-writer lock is held for the daemon's lifetime, not until the next garbage collection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit acquireSingleWriterLock's handle was discarded at boot. better-sqlite3 closes a handle whose object is garbage collected, and a closed handle releases the exclusive file lock — so the "one ledger, one daemon" guarantee lasted only until the first collection: measured 2026-09-11 with the fake executor, a second daemon on the same ledger started and listened seconds after the first. The handle is now kept in main.ts and closed on shutdown; the second instance dies at boot naming the conflict, as intended. --- packages/server/src/main.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index 8eee2cc9..ca800461 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -44,10 +44,15 @@ const config = loadConfig(); // One ledger, one daemon — enforced, not assumed. A second instance would // run its own destructive reconcile against sandboxes this one is still -// operating, well before it ever loses the race for the port. +// operating, well before it ever loses the race for the port. The handle +// is kept for the life of the process: better-sqlite3 closes a handle +// whose object is garbage collected, and a closed handle drops the file +// lock — with the return value discarded, a second daemon on the same +// ledger started fine seconds later (measured 2026-09-11, fake executor). +let ledgerLock: ReturnType | undefined; if (config.DORMICE_DB_PATH !== ':memory:') { try { - acquireSingleWriterLock(config.DORMICE_DB_PATH); + ledgerLock = acquireSingleWriterLock(config.DORMICE_DB_PATH); } catch (error) { fatal(error instanceof Error ? error.message : String(error)); } @@ -352,6 +357,7 @@ const close = async (signal: NodeJS.Signals) => { app.log.error(error, `graceful shutdown after ${signal} failed`); process.exitCode = 1; } + ledgerLock?.close(); process.exit(process.exitCode ?? 0); }; const onSigterm = () => void close('SIGTERM'); From 23b9857907898cb2bcab638d9b3ee1b2d219d137 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:19:08 +0800 Subject: [PATCH 02/26] Server exports its auth, keyed queue, ledger lock and shutdown as subpaths the gateway imports without the executor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway in front of several daemons wants four small self-contained pieces of the daemon — the constant-time token compare, the per-name serialization queue, the single-writer file lock and the bounded shutdown — and nothing else. Four subpath entries expose exactly those; deliberately not the package root, whose import graph loads dockerode, execa and the AWS SDK into any process that touches it. The lock's busy sentence becomes the caller's: the gateway takes the same lock over its own database file and must name its own variable in the exit. auth.ts reads the cookie jar structurally so it type-bundles on its own, without @fastify/cookie's request augmentation in its graph. --- packages/server/package.json | 16 ++++++++++++++++ packages/server/src/auth.ts | 9 ++++++++- packages/server/src/db/lock.ts | 16 +++++++++++----- packages/server/tsup.config.ts | 14 +++++++++++++- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 4596500d..f05001ba 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -13,6 +13,22 @@ "./mini-s3": { "types": "./dist/archive/mini-s3.d.ts", "default": "./dist/archive/mini-s3.js" + }, + "./auth": { + "types": "./dist/auth.d.ts", + "default": "./dist/auth.js" + }, + "./keyed-queue": { + "types": "./dist/keyed-queue.d.ts", + "default": "./dist/keyed-queue.js" + }, + "./lock": { + "types": "./dist/db/lock.d.ts", + "default": "./dist/db/lock.js" + }, + "./shutdown": { + "types": "./dist/shutdown.d.ts", + "default": "./dist/shutdown.js" } }, "files": [ diff --git a/packages/server/src/auth.ts b/packages/server/src/auth.ts index 10cd542f..d1b341a5 100644 --- a/packages/server/src/auth.ts +++ b/packages/server/src/auth.ts @@ -161,7 +161,14 @@ function sessionCookieValid( request: FastifyRequest, getSessionSecret: () => string | null, ): boolean { - const cookie = request.cookies?.[SESSION_COOKIE]; + // The jar exists only where the app registered @fastify/cookie: the + // daemon does; the gateway never mounts the console and passes a getter + // that answers null, so no cookie can pass there. Typed structurally so + // this module stands alone as the `@dormice/server/auth` subpath entry + // without that plugin's request augmentation in its build graph. + const jar = (request as { cookies?: Record }) + .cookies; + const cookie = jar?.[SESSION_COOKIE]; const secret = getSessionSecret(); return Boolean( cookie && diff --git a/packages/server/src/db/lock.ts b/packages/server/src/db/lock.ts index afca59e3..2e086d0c 100644 --- a/packages/server/src/db/lock.ts +++ b/packages/server/src/db/lock.ts @@ -13,8 +13,17 @@ import Database from 'better-sqlite3'; * The lock is an OS-level file lock held by a dedicated SQLite handle in * EXCLUSIVE locking mode: released the instant the process dies, crash * included, so there is no stale-pidfile problem to solve. + * + * The returned handle must be kept for the life of the process: better- + * sqlite3 closes a handle whose object is garbage collected, and a closed + * handle drops the lock (main.ts has the 2026-09-11 measurement). The + * busy sentence is the caller's, because the gateway takes the same lock + * over its own file and must name its own variable in the exit. */ -export function acquireSingleWriterLock(dbPath: string): Database.Database { +export function acquireSingleWriterLock( + dbPath: string, + busyMessage = `another daemon is already running against ${dbPath} — one ledger, one daemon. Stop the other instance, or point this one at its own DORMICE_DB_PATH.`, +): Database.Database { mkdirSync(dirname(dbPath), { recursive: true }); // Fail fast: a held lock answers in 100ms instead of the default 5s wait. const lock = new Database(`${dbPath}.lock`, { timeout: 100 }); @@ -29,10 +38,7 @@ export function acquireSingleWriterLock(dbPath: string): Database.Database { } catch (error) { lock.close(); if ((error as { code?: string }).code === 'SQLITE_BUSY') { - throw new Error( - `another daemon is already running against ${dbPath} — one ledger, one daemon. ` + - 'Stop the other instance, or point this one at its own DORMICE_DB_PATH.', - ); + throw new Error(busyMessage); } throw error; } diff --git a/packages/server/tsup.config.ts b/packages/server/tsup.config.ts index e004f519..49d14560 100644 --- a/packages/server/tsup.config.ts +++ b/packages/server/tsup.config.ts @@ -24,7 +24,19 @@ function git(args: string): string { const commitTime = git('log -1 --format=%cI'); export default defineConfig({ - entry: ['src/index.ts', 'src/main.ts', 'src/archive/mini-s3.ts'], + // Subpath entries beyond the root: mini-s3 for the e2e harness; auth, + // keyed-queue, lock and shutdown for the gateway, which reuses the + // daemon's small self-contained pieces without loading its executor + // (the root's import graph drags dockerode, execa and the AWS SDK in). + entry: [ + 'src/index.ts', + 'src/main.ts', + 'src/archive/mini-s3.ts', + 'src/auth.ts', + 'src/keyed-queue.ts', + 'src/db/lock.ts', + 'src/shutdown.ts', + ], format: ['esm'], dts: true, clean: true, From 6bd61ab8aea3e5e80102fbeb4de8cb96aecf1d76 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:22:02 +0800 Subject: [PATCH 03/26] lookupSandbox: a node answers "do you hold this sandbox?" by name or id, inside the name's slot when a create is in flight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway keeps no directory of where sandboxes live — a second copy of a fact the nodes' ledgers already hold, and every copy drifts. When its cache has no answer it asks every node this one read-only question in parallel and routes to the node that says yes. Three steps make the answer truthful about a create in flight. A row that exists answers at once, whatever its state: a restoring sandbox has a row, and waiting for its slot would hold the answer for the whole restore, long past the gateway's two-second patience. No row while the name's slot is busy waits its turn and looks again — the daemon creates first and writes the row second, both under the slot, so a gateway retrying a create whose answer was lost finds the sandbox on the node that built it and never places a second copy. No row and a free slot is a plain no. By id there is no slot to wait on and none is needed: nobody can ask about an id before the create that minted it has answered. The in-flight test parks the fake executor's create to hold the acquire mid-build; removing the wait turns it red. --- packages/server/src/app.test.ts | 97 +++++++++++++++++++++++++ packages/server/src/routes/sandboxes.ts | 45 +++++++++++- packages/shared/src/index.ts | 1 + packages/shared/src/lookup.ts | 40 ++++++++++ 4 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 packages/shared/src/lookup.ts diff --git a/packages/server/src/app.test.ts b/packages/server/src/app.test.ts index 517fbf44..555e2b8f 100644 --- a/packages/server/src/app.test.ts +++ b/packages/server/src/app.test.ts @@ -2358,3 +2358,100 @@ describe('activity attribution', () => { expect((await eventOf(app, 'apikey-created'))?.actor).toBe('console'); }); }); + +describe('POST /lookupSandbox', () => { + it('answers by name and by id with the state, without waking or touching the idle clock', async () => { + const { app, db } = testApp(); + const created = (await acquire(app, { name: 'alice' })).json(); + const row = findById(db, created.sandbox.id); + if (!row) throw new Error('no row'); + // A cold sandbox stays cold: lookup is observation, not use. + transition(db, row.id, 'frozen'); + + const byName = await rpc(app, '/lookupSandbox', { name: 'alice' }); + expect(byName.statusCode).toBe(200); + expect(byName.json()).toEqual({ + found: true, + sandbox: { id: row.id, name: 'alice', state: 'frozen' }, + }); + const byId = await rpc(app, '/lookupSandbox', { id: row.id }); + expect(byId.json()).toEqual(byName.json()); + expect(findById(db, row.id)?.state).toBe('frozen'); + expect(findById(db, row.id)?.lastActiveAt).toBe(row.lastActiveAt); + + expect( + (await rpc(app, '/lookupSandbox', { name: 'nobody' })).json(), + ).toEqual({ found: false }); + expect( + (await rpc(app, '/lookupSandbox', { id: 'no-such-id' })).json(), + ).toEqual({ found: false }); + // Neither a name nor an id is not a question. + expect((await rpc(app, '/lookupSandbox', {})).statusCode).toBe(400); + }); + + it('a name whose slot is busy waits its turn: asked while an acquire is mid-create, it answers found once the row exists', async () => { + // A create that parks inside the executor — the daemon's own shape of + // "in flight": the acquire holds the name's slot, the container is + // being built, the row is not written yet. + let release: () => void = () => {}; + let inCreate: () => void = () => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + const reached = new Promise((resolve) => { + inCreate = resolve; + }); + class ParkedCreate extends FakeExecutor { + override async create( + ...args: Parameters + ): Promise { + inCreate(); + await gate; + return super.create(...args); + } + } + const { app } = testApp(new ParkedCreate()); + const creating = acquire(app, { name: 'alice' }); + await reached; + // No row, slot busy: the question must wait, not answer "no". + const asked = rpc(app, '/lookupSandbox', { name: 'alice' }); + const early = await Promise.race([ + asked.then(() => 'answered'), + new Promise((resolve) => setTimeout(() => resolve('pending'), 50)), + ]); + expect(early).toBe('pending'); + release(); + const created = (await creating).json(); + expect((await asked).json()).toEqual({ + found: true, + sandbox: { id: created.sandbox.id, name: 'alice', state: 'active' }, + }); + }); + + it('a sandbox with a row answers at once even while its slot is held — a restore in progress must not read as silence', async () => { + const { app, db, locks } = testApp(); + const created = (await acquire(app, { name: 'alice' })).json(); + transition(db, created.sandbox.id, 'frozen'); + transition(db, created.sandbox.id, 'stopped'); + transition(db, created.sandbox.id, 'archived'); + transition(db, created.sandbox.id, 'restoring'); + let release: () => void = () => {}; + const held = locks.run( + 'alice', + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const answer = await Promise.race([ + rpc(app, '/lookupSandbox', { name: 'alice' }).then((r) => r.json()), + new Promise((resolve) => setTimeout(() => resolve('pending'), 500)), + ]); + expect(answer).toEqual({ + found: true, + sandbox: { id: created.sandbox.id, name: 'alice', state: 'restoring' }, + }); + release(); + await held; + }); +}); diff --git a/packages/server/src/routes/sandboxes.ts b/packages/server/src/routes/sandboxes.ts index 60b93c94..739f0e9c 100644 --- a/packages/server/src/routes/sandboxes.ts +++ b/packages/server/src/routes/sandboxes.ts @@ -19,6 +19,8 @@ import { listSandboxImagesResponseSchema, listSandboxMetricsRequestSchema, listSandboxMetricsResponseSchema, + lookupSandboxRequestSchema, + lookupSandboxResponseSchema, READ_FILES_TOTAL_LIMIT_BYTES, readFileRequestSchema, readFileResponseSchema, @@ -54,6 +56,7 @@ import type { Db } from '../db/db'; import { countSandboxes, createSandbox, + findById, findByName, listSandboxes, setDiskGb, @@ -81,7 +84,7 @@ import { NotAFileError, } from '../executor/executor'; import { httpError } from '../http-error'; -import type { KeyedQueue } from '../keyed-queue'; +import { type KeyedQueue, SKIPPED } from '../keyed-queue'; import { destroySandbox, rebuildSandbox, wakeSandbox } from '../lifecycle'; import { ArchiveDisabledError, resolvePolicy } from '../policy'; import { resolveSpec } from '../spec'; @@ -1114,6 +1117,46 @@ export const sandboxRoutes: FastifyPluginAsyncZod< }, ); + // The gateway's one question on its own account: does this node hold + // the sandbox? Read-only — never wakes, never touches the idle clock — + // and truthful about a create in flight, in three steps. A row that + // exists answers at once, whatever its state: a restoring sandbox has a + // row, and waiting for its slot would hold the answer for the whole + // restore, long past the gateway's two-second patience — the gateway + // would read a live sandbox as a node that did not answer. No row while + // the name's slot is busy means an acquire may be writing the row right + // now (create first, row second, both under the slot), so the answer + // waits its turn behind it and looks again. No row and a free slot is a + // plain no. By id there is no slot to wait on (slots are keyed by name), + // and none is needed: nobody can ask about an id before the create that + // minted it has answered. + app.post( + '/lookupSandbox', + { + schema: { + body: lookupSandboxRequestSchema, + response: { 200: lookupSandboxResponseSchema }, + }, + }, + async (request) => { + const query = request.body; + const look = () => + 'name' in query ? findByName(db, query.name) : findById(db, query.id); + const answer = (row: SandboxRow | undefined) => + row + ? { + found: true as const, + sandbox: { id: row.id, name: row.name, state: row.state }, + } + : { found: false as const }; + const now = look(); + if (now !== undefined || !('name' in query)) return answer(now); + const unheld = await locks.tryRun(query.name, async () => look()); + if (unheld !== SKIPPED) return answer(unheld); + return answer(await locks.run(query.name, async () => look())); + }, + ); + app.post( '/destroySandbox', { diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 4e392316..e31684ce 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -9,6 +9,7 @@ export * from './host'; export * from './images'; export * from './ingress'; export * from './list'; +export * from './lookup'; export * from './metrics'; export * from './policy'; export * from './rebuild'; diff --git a/packages/shared/src/lookup.ts b/packages/shared/src/lookup.ts new file mode 100644 index 00000000..c449b0af --- /dev/null +++ b/packages/shared/src/lookup.ts @@ -0,0 +1,40 @@ +import { z } from 'zod'; +import { sandboxNameSchema } from './sandbox'; +import { SANDBOX_STATES } from './states'; + +/** + * lookupSandbox({ name }) / lookupSandbox({ id }) — "do you hold this + * sandbox?", the one question a gateway asks a node on its own account. + * The gateway keeps no directory of where sandboxes live (a second copy of + * a fact the nodes' ledgers already hold, and every copy drifts); when its + * cache has no answer it asks every node this, in parallel, and routes to + * the one that says yes. Read-only by construction: it never wakes, never + * touches the idle clock, never creates. + * + * The node answers inside the name's serialization slot when it must: a + * row that exists answers at once, whatever its state; no row while an + * acquire of that name is in flight waits for the acquire (the daemon + * creates first and writes the row second, both under the slot) and looks + * again — so a gateway retrying a create whose answer was lost finds the + * sandbox on the node that built it, and never places a second copy. + */ +export const lookupSandboxRequestSchema = z.union([ + z.object({ name: sandboxNameSchema }), + z.object({ id: z.string().min(1) }), +]); + +export type LookupSandboxRequest = z.infer; + +export const lookupSandboxResponseSchema = z.discriminatedUnion('found', [ + z.object({ + found: z.literal(true), + sandbox: z.object({ + id: z.string(), + name: sandboxNameSchema, + state: z.enum(SANDBOX_STATES), + }), + }), + z.object({ found: z.literal(false) }), +]); + +export type LookupSandboxResponse = z.infer; From bbfb3a810a0313c813dd898816227d42304ece59 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:26:04 +0800 Subject: [PATCH 04/26] A node checks in with its gateway every 15 seconds: readings, build and where it can be reached MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DORMICE_GATEWAY_ENDPOINT makes a daemon a node of a fleet. Set, it POSTs /checkIn to the gateway every DORMICE_CHECK_IN_INTERVAL_SECONDS with its id, the address the gateway may forward to (DORMICE_NODE_ENDPOINT, defaulting to its own loopback — right when gateway and node share a machine), its build and a fresh reading: CPU, memory, swap, the data disk, the ledger's census by state. Unset, the daemon is the whole platform by itself, as before. The token is the one DORMICE_API_TOKEN gateway and nodes share. Who knows the truth speaks: the node knows what it runs and where it lives, so the node reports and the gateway only listens. The gateway learns of a node from its first check-in — no registration verb, no nodes file — and reads two missed check-ins as down; the interval travels in every check-in so both ends measure with the same number. Failures are logged on the change, never every tick, and are never fatal: the gateway is the fleet's front door, not the node's reason to live. The shared host schema is split into its named parts (host reading, data disk, state counts) so getHostMetrics and the check-in describe the machine with one vocabulary; readHostReading is the one function both read through. --- packages/server/src/check-in.test.ts | 179 +++++++++++++++++++++++++++ packages/server/src/check-in.ts | 135 ++++++++++++++++++++ packages/server/src/config.test.ts | 33 +++++ packages/server/src/config.ts | 48 +++++++ packages/server/src/host-metrics.ts | 26 ++++ packages/server/src/main.ts | 32 +++++ packages/server/src/routes/host.ts | 14 +-- packages/shared/src/gateway.ts | 124 +++++++++++++++++++ packages/shared/src/host.ts | 111 ++++++++++------- packages/shared/src/index.ts | 1 + 10 files changed, 643 insertions(+), 60 deletions(-) create mode 100644 packages/server/src/check-in.test.ts create mode 100644 packages/server/src/check-in.ts create mode 100644 packages/shared/src/gateway.ts diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts new file mode 100644 index 00000000..6cbcc69a --- /dev/null +++ b/packages/server/src/check-in.test.ts @@ -0,0 +1,179 @@ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { fileURLToPath } from 'node:url'; +import { checkInRequestSchema } from '@dormice/shared'; +import { afterEach, describe, expect, it } from 'vitest'; +import { CheckIn, type CheckInOptions, readNodeReading } from './check-in'; +import { migrateDb, openDb } from './db/db'; +import { createSandbox } from './db/ledger'; +import { CpuSampler } from './host-metrics'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const TOKEN = 'shared-token-shared-token-shared-token'; + +const servers: http.Server[] = []; +afterEach(async () => { + await Promise.all( + servers + .splice(0) + .map((s) => new Promise((resolve) => s.close(() => resolve()))), + ); +}); + +/** A gateway-shaped listener: records every check-in, answers what the test says. */ +async function gateway(answer: () => { status: number; body: string }) { + const seen: Array<{ headers: http.IncomingHttpHeaders; body: unknown }> = []; + const server = http.createServer((req, res) => { + let text = ''; + req.on('data', (chunk) => { + text += chunk; + }); + req.on('end', () => { + seen.push({ headers: req.headers, body: JSON.parse(text) }); + const a = answer(); + res.writeHead(a.status, { 'content-type': 'application/json' }); + res.end(a.body); + }); + }); + servers.push(server); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return { + endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + seen, + }; +} + +function logSpy() { + const infos: string[] = []; + const warns: string[] = []; + return { + infos, + warns, + log: { + info: (msg: string) => infos.push(msg), + warn: (_obj: unknown, msg: string) => warns.push(msg), + }, + }; +} + +function options( + gatewayEndpoint: string, + log: CheckInOptions['log'], + over: Partial = {}, +): CheckInOptions { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const cpu = new CpuSampler(); + return { + gateway: gatewayEndpoint, + token: TOKEN, + nodeId: 'node-7', + endpoint: 'http://10.0.0.7:80', + intervalSeconds: 1, + build: { + commit: 'abc1234', + title: 'a commit', + committedAt: '2026-09-14T00:00:00.000Z', + }, + readReading: () => readNodeReading(db, cpu, '/nonexistent-data-dir'), + log, + ...over, + }; +} + +describe('CheckIn', () => { + it('posts a check-in the gateway can parse: shared token, id, endpoint, interval, build, reading', async () => { + const gw = await gateway(() => ({ status: 200, body: '{}' })); + const { log, warns } = logSpy(); + await new CheckIn(options(gw.endpoint, log)).once(); + expect(warns).toEqual([]); + expect(gw.seen).toHaveLength(1); + expect(gw.seen[0]?.headers.authorization).toBe(`Bearer ${TOKEN}`); + const body = checkInRequestSchema.parse(gw.seen[0]?.body); + expect(body.nodeId).toBe('node-7'); + expect(body.endpoint).toBe('http://10.0.0.7:80'); + expect(body.intervalSeconds).toBe(1); + expect(body.build?.commit).toBe('abc1234'); + expect(body.reading.host.cpuCount).toBeGreaterThan(0); + // The first sample has no delta, and a data dir that does not exist is + // an honest null, never a made-up disk. + expect(body.reading.host.cpuUsedPct).toBeNull(); + expect(body.reading.dataDisk).toBeNull(); + expect(body.reading.sandboxes).toEqual({ + total: 0, + byState: { active: 0, frozen: 0, stopped: 0, archived: 0, restoring: 0 }, + }); + }); + + it('the reading counts the ledger by state', async () => { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + for (const name of ['a', 'b']) { + createSandbox(db, { + id: `id-${name}`, + name, + nodeId: 'node-7', + policy: { + freezeAfterSeconds: 60, + stopAfterSeconds: null, + archiveAfterSeconds: null, + }, + template: null, + metadata: null, + spec: undefined, + actor: null, + }); + } + const reading = await readNodeReading(db, new CpuSampler(), '/tmp'); + expect(reading.sandboxes.total).toBe(2); + expect(reading.sandboxes.byState.active).toBe(2); + expect(reading.dataDisk?.path).toBe('/tmp'); + }); + + it('logs a failing gateway once, and its recovery once — not every tick', async () => { + let status = 500; + const gw = await gateway(() => ({ status, body: '{"message":"boom"}' })); + const { log, warns, infos } = logSpy(); + const checkIn = new CheckIn(options(gw.endpoint, log)); + await checkIn.once(); + await checkIn.once(); + expect(gw.seen).toHaveLength(2); + expect(warns).toHaveLength(1); + expect(warns[0]).toMatch(/check-in failed/); + status = 200; + await checkIn.once(); + await checkIn.once(); + expect(infos).toEqual([ + `check-in with gateway ${gw.endpoint} answers again`, + ]); + expect(warns).toHaveLength(1); + // A gateway that refuses the token is the same one event. + status = 401; + await checkIn.once(); + await checkIn.once(); + expect(warns).toHaveLength(2); + }); + + it('a gateway that is not there is a logged failure, never a throw', async () => { + const { log, warns } = logSpy(); + const checkIn = new CheckIn(options('http://127.0.0.1:9', log)); + await expect(checkIn.once()).resolves.toBeUndefined(); + expect(warns).toHaveLength(1); + }); + + it('ticks on its interval from start() and stops on stop()', async () => { + const gw = await gateway(() => ({ status: 200, body: '{}' })); + const { log } = logSpy(); + const checkIn = new CheckIn(options(gw.endpoint, log)); + checkIn.start(); + const deadline = Date.now() + 5_000; + while (gw.seen.length < 2 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + expect(gw.seen.length).toBeGreaterThanOrEqual(2); + checkIn.stop(); + const afterStop = gw.seen.length; + await new Promise((resolve) => setTimeout(resolve, 1_200)); + expect(gw.seen.length).toBe(afterStop); + }); +}); diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts new file mode 100644 index 00000000..1fda3f3e --- /dev/null +++ b/packages/server/src/check-in.ts @@ -0,0 +1,135 @@ +import { + type BuildInfo, + type CheckInRequest, + checkInResponseSchema, + type NodeReading, +} from '@dormice/shared'; +import type { Db } from './db/db'; +import { countByState, listSandboxes } from './db/ledger'; +import { type CpuSampler, readHostReading } from './host-metrics'; + +/** + * A node's reading for its check-in: the host half (host-metrics.ts) and + * the ledger's census. The same numbers getHostMetrics answers a caller + * with, minus the daemon-local knobs no gateway places by. + */ +export async function readNodeReading( + db: Db, + cpu: CpuSampler, + dataDir: string, +): Promise { + const { byState, total } = countByState(listSandboxes(db)); + return { + ...(await readHostReading(cpu, dataDir)), + sandboxes: { total, byState }, + }; +} + +export interface CheckInLog { + info(msg: string): void; + warn(obj: unknown, msg: string): void; +} + +export interface CheckInOptions { + /** DORMICE_GATEWAY_ENDPOINT. */ + gateway: string; + /** The token gateway and nodes share (DORMICE_API_TOKEN). */ + token: string; + nodeId: string; + /** Where the gateway reaches this node (DORMICE_NODE_ENDPOINT or the loopback default). */ + endpoint: string; + intervalSeconds: number; + build: BuildInfo | null; + readReading: () => Promise; + log: CheckInLog; + /** Test seam; production uses the platform's fetch. */ + fetchImpl?: typeof fetch; +} + +/** A gateway that has not answered within this is a gateway not answering; the next tick tries again. */ +const CHECK_IN_TIMEOUT_MS = 10_000; + +/** + * The node's check-in ticker: every interval, one POST /checkIn to the + * gateway carrying the node's id, where it can be reached, its build and + * a fresh reading (RULES/协议.md「网关」). The gateway learns of a node from + * its first check-in — no registration verb, no nodes file — and reads + * two missed check-ins as down. + * + * Chained setTimeout, the daemon's discipline: the next tick is scheduled + * when this one is done, so a slow gateway never has ticks pile up. + * Failures are logged on the change — once when the gateway stops + * answering, once when it answers again — never every tick: a gateway + * down for an hour is one event, not two hundred and forty lines. Never + * fatal: the gateway is the fleet's front door and configuration + * authority, not the node's reason to live; the node keeps running its + * sandboxes and keeps trying. + */ +export class CheckIn { + private timer: NodeJS.Timeout | undefined; + private closing = false; + /** The failure the gateway is currently in, or null while it answers. */ + private failing: string | null = null; + + constructor(private readonly opts: CheckInOptions) {} + + start(): void { + this.schedule(0); + } + + stop(): void { + this.closing = true; + clearTimeout(this.timer); + } + + /** One check-in. Never throws: a failure is recorded and the next tick retries. */ + async once(): Promise { + const { opts } = this; + try { + const body: CheckInRequest = { + nodeId: opts.nodeId, + endpoint: opts.endpoint, + intervalSeconds: opts.intervalSeconds, + build: opts.build, + reading: await opts.readReading(), + }; + const res = await (opts.fetchImpl ?? fetch)(`${opts.gateway}/checkIn`, { + method: 'POST', + headers: { + authorization: `Bearer ${opts.token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(CHECK_IN_TIMEOUT_MS), + }); + if (res.status !== 200) { + const text = await res.text(); + throw new Error( + `gateway answered ${res.status}: ${text.slice(0, 200)}`, + ); + } + checkInResponseSchema.parse(await res.json()); + if (this.failing !== null) { + opts.log.info(`check-in with gateway ${opts.gateway} answers again`); + this.failing = null; + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (this.failing === null) { + opts.log.warn( + { gateway: opts.gateway, error: message }, + 'check-in failed; the gateway places nothing here and forwards no new names to this node until it answers again — retrying every interval', + ); + } + this.failing = message; + } + } + + private schedule(delayMs: number): void { + if (this.closing) return; + this.timer = setTimeout(async () => { + await this.once(); + this.schedule(this.opts.intervalSeconds * 1000); + }, delayMs); + } +} diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index 90fea421..083f071c 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -151,3 +151,36 @@ describe('the S3 set', () => { expect(off.DORMICE_S3_FORCE_PATH_STYLE).toBe(false); }); }); + +describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { + it('defaults: no gateway (standalone), no node endpoint, 15s check-in', () => { + const config = loadConfig(TOKEN); + expect(config.DORMICE_GATEWAY_ENDPOINT).toBeUndefined(); + expect(config.DORMICE_NODE_ENDPOINT).toBeUndefined(); + expect(config.DORMICE_CHECK_IN_INTERVAL_SECONDS).toBe(15); + }); + + it('parses both endpoints as full URLs and drops a trailing slash', () => { + const config = loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677/', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80///', + DORMICE_CHECK_IN_INTERVAL_SECONDS: '5', + }); + expect(config.DORMICE_GATEWAY_ENDPOINT).toBe('http://10.0.0.5:3677'); + expect(config.DORMICE_NODE_ENDPOINT).toBe('http://10.0.0.7:80'); + expect(config.DORMICE_CHECK_IN_INTERVAL_SECONDS).toBe(5); + }); + + it('refuses an endpoint without a scheme, naming the variable', () => { + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_ENDPOINT: '10.0.0.5:3677' }), + ).toThrow(/DORMICE_GATEWAY_ENDPOINT must be a full http\(s\) URL/); + expect(() => + loadConfig({ ...TOKEN, DORMICE_NODE_ENDPOINT: 'node-7' }), + ).toThrow(/DORMICE_NODE_ENDPOINT must be a full http\(s\) URL/); + expect(() => + loadConfig({ ...TOKEN, DORMICE_CHECK_IN_INTERVAL_SECONDS: '0' }), + ).toThrow(); + }); +}); diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index 2bc2e96d..acd03501 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -169,6 +169,51 @@ const envSchema = z.object({ DORMICE_S3_REGION: z.string().default('us-east-1'), /** Path-style addressing: MinIO needs true; the clouds route by subdomain. */ DORMICE_S3_FORCE_PATH_STYLE: z.stringbool().default(false), + /** + * The gateway this daemon is a node of — its intranet address, e.g. + * http://10.0.0.5:3677. Set, the daemon checks in with it every + * DORMICE_CHECK_IN_INTERVAL_SECONDS (check-in.ts): its readings, its + * build, and where it can be reached. That check-in is the gateway's + * only source of "which nodes exist and how full are they" — no + * registration, no nodes file. Unset, the daemon is the whole platform + * by itself, as it always was, and checks in with nobody. The token it + * presents is DORMICE_API_TOKEN: gateway and nodes share one, and the + * gateway speaks to every node with the same one. + */ + DORMICE_GATEWAY_ENDPOINT: z + .url({ + protocol: /^https?$/, + error: + 'DORMICE_GATEWAY_ENDPOINT must be a full http(s) URL, e.g. http://10.0.0.5:3677', + }) + .transform((url) => url.replace(/\/+$/, '')) + .optional(), + /** + * Where the gateway reaches this node — the address it forwards to. + * Default: this daemon's own loopback address, right when gateway and + * node share a machine (the single-machine install is a fleet of one). + * On a machine of its own the daemon still binds loopback (the red + * line), so this names the front the gateway may dial — the node's + * Caddy on the intranet interface, e.g. http://10.0.0.7:80. + */ + DORMICE_NODE_ENDPOINT: z + .url({ + protocol: /^https?$/, + error: + 'DORMICE_NODE_ENDPOINT must be a full http(s) URL, e.g. http://10.0.0.7:80', + }) + .transform((url) => url.replace(/\/+$/, '')) + .optional(), + /** + * How often the node checks in with its gateway. The gateway reads two + * missed check-ins as down — the one number both ends of that wire + * share, so the node states it in every check-in. + */ + DORMICE_CHECK_IN_INTERVAL_SECONDS: z.coerce + .number() + .int() + .positive() + .default(15), }); const checkedSchema = envSchema @@ -261,6 +306,9 @@ export const CONFIG_KEYS: Record = { DORMICE_S3_SECRET_ACCESS_KEY: { sensitive: true }, DORMICE_S3_REGION: { sensitive: false }, DORMICE_S3_FORCE_PATH_STYLE: { sensitive: false }, + DORMICE_GATEWAY_ENDPOINT: { sensitive: false }, + DORMICE_NODE_ENDPOINT: { sensitive: false }, + DORMICE_CHECK_IN_INTERVAL_SECONDS: { sensitive: false }, }; export type ConfigSources = Record; diff --git a/packages/server/src/host-metrics.ts b/packages/server/src/host-metrics.ts index d629146c..5a768d7e 100644 --- a/packages/server/src/host-metrics.ts +++ b/packages/server/src/host-metrics.ts @@ -1,5 +1,6 @@ import { readFile, statfs } from 'node:fs/promises'; import os from 'node:os'; +import type { NodeReading } from '@dormice/shared'; /** * Host-side readings for getHostMetrics — the machine's own health, which @@ -158,3 +159,28 @@ export async function readDiskSpace( availableBytes: s.bavail * s.bsize, }; } + +/** + * The host half of a node's reading — what getHostMetrics answers and + * what a check-in reports, from one function so the two never disagree on + * a field: the machine's cores and CPU delta, its memory and swap, and + * the data disk that holds the sandbox disks (null until it exists). The + * CpuSampler is the caller's: a delta spans "since this instance's last + * sample", so each reader owns one (host-metrics route, metrics sampler, + * check-in) and none steals another's window. + */ +export async function readHostReading( + cpu: CpuSampler, + dataDir: string, +): Promise> { + const memory = await readHostMemory(); + const disk = await readDiskSpace(dataDir); + return { + host: { + cpuCount: os.cpus().length, + cpuUsedPct: cpu.sample(), + ...memory, + }, + dataDisk: disk ? { path: dataDir, ...disk } : null, + }; +} diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index ca800461..90b02f3d 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -3,6 +3,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { pino } from 'pino'; import { buildApp } from './app'; +import { CheckIn, readNodeReading } from './check-in'; import { Archiver } from './archive/archiver'; import { LedgerArchiveStore } from './archive/ledger-store'; import { type Config, loadConfig } from './config'; @@ -314,6 +315,36 @@ recordActivity(db, { // the daemon to the outside world is a reverse proxy's job. await app.listen({ host: '127.0.0.1', port: config.DORMICE_PORT }); +// A node of a fleet reports to its gateway; a daemon on its own reports to +// nobody. Started after listen on purpose: the check-in names where the +// gateway may forward to, and that door must be open before the gateway +// hears of it. Its CpuSampler is its own — a delta spans "since this +// instance's last sample", and the route's and the metrics ticker's +// windows must not be stolen (host-metrics.ts). +let checkIn: CheckIn | undefined; +if (config.DORMICE_GATEWAY_ENDPOINT !== undefined) { + const nodeEndpoint = + config.DORMICE_NODE_ENDPOINT ?? `http://127.0.0.1:${config.DORMICE_PORT}`; + const checkInCpu = new CpuSampler(); + checkInCpu.sample(); + checkIn = new CheckIn({ + gateway: config.DORMICE_GATEWAY_ENDPOINT, + token: config.DORMICE_API_TOKEN, + nodeId: config.DORMICE_NODE_ID, + endpoint: nodeEndpoint, + intervalSeconds: config.DORMICE_CHECK_IN_INTERVAL_SECONDS, + build, + readReading: () => readNodeReading(db, checkInCpu, config.DORMICE_DATA_DIR), + log, + }); + checkIn.start(); + log.info( + `node ${config.DORMICE_NODE_ID} checks in with gateway ${config.DORMICE_GATEWAY_ENDPOINT} every ${config.DORMICE_CHECK_IN_INTERVAL_SECONDS}s, reachable at ${nodeEndpoint}`, + ); +} else { + log.info('no gateway: standalone daemon (DORMICE_GATEWAY_ENDPOINT unset)'); +} + // systemd stops the daemon with SIGTERM. Shutdown is bounded on purpose // (shutdown.ts has the measurements): close the app — preClose ends the // long-lived streams with honest end-frames, the listener stops — give @@ -341,6 +372,7 @@ const close = async (signal: NodeJS.Signals) => { process.removeListener('SIGINT', onSigint); clearTimeout(heartbeatTimer); clearTimeout(metricsTimer); + checkIn?.stop(); watchdog.stop(); app.log.info( `${signal} received — shutting down (grace ${SHUTDOWN_GRACE_MS}ms)`, diff --git a/packages/server/src/routes/host.ts b/packages/server/src/routes/host.ts index ca733c19..d7e7d1be 100644 --- a/packages/server/src/routes/host.ts +++ b/packages/server/src/routes/host.ts @@ -1,4 +1,3 @@ -import os from 'node:os'; import { getFleetTimelineRequestSchema, getFleetTimelineResponseSchema, @@ -22,7 +21,7 @@ import { } from '../db/metrics'; import { readRuntimeSettings } from '../db/settings'; import type { Executor } from '../executor/executor'; -import { CpuSampler, readDiskSpace, readHostMemory } from '../host-metrics'; +import { CpuSampler, readHostReading } from '../host-metrics'; export interface HostRoutesOptions { config: Config; @@ -58,17 +57,8 @@ export const hostRoutes: FastifyPluginAsyncZod = async ( const rows = listSandboxes(db); const { byState, total } = countByState(rows); - const memory = await readHostMemory(); - const dataDisk = await readDiskSpace(config.DORMICE_DATA_DIR); return { - host: { - cpuCount: os.cpus().length, - cpuUsedPct: cpu.sample(), - ...memory, - }, - dataDisk: dataDisk - ? { path: config.DORMICE_DATA_DIR, ...dataDisk } - : null, + ...(await readHostReading(cpu, config.DORMICE_DATA_DIR)), sandboxes: { total, // The ledger's live knob, not the env seed — the console edits it. diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts new file mode 100644 index 00000000..11eb6cf8 --- /dev/null +++ b/packages/shared/src/gateway.ts @@ -0,0 +1,124 @@ +import { z } from 'zod'; +import { + dataDiskSchema, + hostReadingSchema, + sandboxStateCountsSchema, +} from './host'; + +/** + * The gateway's wire: the verbs between a node and the gateway that fronts + * it, and the gateway's own observation verbs. A fleet is N daemons that + * know nothing of each other behind one gateway that holds no sandbox + * state — only what the nodes tell it every few seconds and what its own + * configuration tables say. A single machine runs the same two processes + * and is a fleet of one. + */ + +/** The identity a build carries: the commit its dist was built from. Null where the dist was built outside a checkout. */ +export const buildInfoSchema = z.object({ + /** Short hash. */ + commit: z.string(), + /** The commit's subject line. */ + title: z.string(), + /** ISO 8601 UTC — the commit's time, not the build's. */ + committedAt: z.iso.datetime(), +}); + +export type BuildInfo = z.infer; + +/** + * What a node reports about itself at every check-in — everything + * placement decides on: the machine's CPU, memory and data disk, and the + * ledger's census by state. The same host reading getHostMetrics answers + * (host.ts), minus the daemon-local knobs no gateway places by. + */ +export const nodeReadingSchema = z.object({ + host: hostReadingSchema, + dataDisk: dataDiskSchema.nullable(), + sandboxes: z.object({ + total: z.number().int(), + byState: sandboxStateCountsSchema, + }), +}); + +export type NodeReading = z.infer; + +/** + * checkIn — a node reporting for duty, every DORMICE_CHECK_IN_INTERVAL_SECONDS + * (15 by default), authenticated with the token the gateway and every node + * share. The gateway learns of a node from its first check-in: there is no + * registration verb and no nodes file — "which nodes exist" has one home, + * the gateway's nodes table, written by the nodes themselves. Two missed + * check-ins read as down: no new sandbox is placed there and its sandboxes + * answer 502 until it reports again. Who knows the truth speaks: the node + * knows what it runs and where it can be reached; the gateway only listens + * and compares, and never keeps a record of what it told whom. + */ +export const checkInRequestSchema = z.object({ + /** DORMICE_NODE_ID — the node's name in every sandbox's `nodeId`. */ + nodeId: z.string().min(1), + /** Where the gateway forwards to: the node's intranet front (DORMICE_NODE_ENDPOINT). */ + endpoint: z.url({ protocol: /^https?$/ }), + /** How often this node checks in — the gateway's yardstick for "missed two in a row". */ + intervalSeconds: z.number().int().positive(), + build: buildInfoSchema.nullable(), + reading: nodeReadingSchema, +}); + +export type CheckInRequest = z.infer; + +/** Nothing yet: the configuration version the gateway will answer with arrives with the configuration authority. */ +export const checkInResponseSchema = z.object({}); + +export type CheckInResponse = z.infer; + +/** + * listNodes — every node the gateway has ever heard from, with what it + * last said. Everything here is what the gateway already holds; answering + * costs no node anything. + */ +export const listNodesRequestSchema = z.object({}); + +export const nodeViewSchema = z.object({ + id: z.string(), + endpoint: z.string(), + /** ISO 8601 UTC — the first check-in. */ + addedAt: z.iso.datetime(), + /** ISO 8601 UTC — null only right after a gateway start, before the node's next check-in. */ + lastCheckInAt: z.iso.datetime().nullable(), + intervalSeconds: z.number().int().positive().nullable(), + /** Checked in within two of its own intervals. */ + reachable: z.boolean(), + build: buildInfoSchema.nullable(), + reading: nodeReadingSchema.nullable(), + /** Sandboxes the gateway placed here since the last check-in — counted against the node until the next reading shows them. */ + placedSinceCheckIn: z.number().int().nonnegative(), +}); + +export type NodeView = z.infer; + +export const listNodesResponseSchema = z.object({ + nodes: z.array(nodeViewSchema), +}); + +export type ListNodesResponse = z.infer; + +/** + * removeNode — the operator's word that a node is gone for good: its row + * goes, its sandboxes are no longer looked for, and a name that lived only + * there is a new name again. A node that is merely down needs nothing — + * it is back the moment it checks in — and one removed by mistake re-adds + * itself the same way. + */ +export const removeNodeRequestSchema = z.object({ + id: z.string().min(1), +}); + +export type RemoveNodeRequest = z.infer; + +export const removeNodeResponseSchema = z.object({ + /** True when a row existed and was removed; false when there was none. */ + removed: z.boolean(), +}); + +export type RemoveNodeResponse = z.infer; diff --git a/packages/shared/src/host.ts b/packages/shared/src/host.ts index ad695f5a..3eeeefa7 100644 --- a/packages/shared/src/host.ts +++ b/packages/shared/src/host.ts @@ -1,68 +1,83 @@ import { z } from 'zod'; /** - * getHostMetrics() — the observation window into the machine itself: is the - * host healthy, and what do the sandboxes collectively cost it? A single - * point-in-time snapshot; for the machine's past see getHostMetricsHistory - * below. Observation never wakes a sandbox and never touches lifecycle. + * The machine's own readings — the `host` half of getHostMetrics and of a + * node's check-in (gateway.ts nodeReadingSchema), one schema so the two + * never disagree on a field. * * Readings a platform cannot produce are null, honestly — never zero, never - * invented: swap and /proc are Linux facts, and the data directory only - * exists where the docker executor runs. + * invented: swap and /proc are Linux facts. */ -export const hostMetricsResponseSchema = z.object({ - host: z.object({ - cpuCount: z.number().int().positive(), - /** - * Percent of the whole machine, 0-100. Null until the sampler has two - * samples to take a delta between — the first request after daemon - * start reports "don't know yet", not a made-up 0. - */ - cpuUsedPct: z.number().nullable(), - memTotalBytes: z.number(), - /** - * What could still be allocated without swapping — /proc/meminfo's - * MemAvailable (counts reclaimable page cache), not the naive "free". - */ - memAvailableBytes: z.number(), - /** - * Swap is the freeze mechanism's fuel: frozen sandboxes live there, and - * a full swap means "idle is free" stops being true. totalBytes of 0 is - * an honest reading of a machine with no swap configured (doctor warns - * about it); null means the platform offers no reading at all. - */ - swap: z - .object({ - totalBytes: z.number(), - usedBytes: z.number(), - }) - .nullable(), - }), +export const hostReadingSchema = z.object({ + cpuCount: z.number().int().positive(), /** - * The filesystem holding DORMICE_DATA_DIR — where sandbox disks live. A - * full data disk is the real capacity ceiling (past it, even the ledger - * cannot write). Null when the directory does not exist (fake executor, - * fresh install). + * Percent of the whole machine, 0-100. Null until the sampler has two + * samples to take a delta between — the first request after daemon + * start reports "don't know yet", not a made-up 0. */ - dataDisk: z + cpuUsedPct: z.number().nullable(), + memTotalBytes: z.number(), + /** + * What could still be allocated without swapping — /proc/meminfo's + * MemAvailable (counts reclaimable page cache), not the naive "free". + */ + memAvailableBytes: z.number(), + /** + * Swap is the freeze mechanism's fuel: frozen sandboxes live there, and + * a full swap means "idle is free" stops being true. totalBytes of 0 is + * an honest reading of a machine with no swap configured (doctor warns + * about it); null means the platform offers no reading at all. + */ + swap: z .object({ - path: z.string(), totalBytes: z.number(), usedBytes: z.number(), - availableBytes: z.number(), }) .nullable(), +}); + +export type HostReading = z.infer; + +/** + * The filesystem holding DORMICE_DATA_DIR — where sandbox disks live. A + * full data disk is the real capacity ceiling (past it, even the ledger + * cannot write). Readers get null when the directory does not exist (fake + * executor, fresh install). + */ +export const dataDiskSchema = z.object({ + path: z.string(), + totalBytes: z.number(), + usedBytes: z.number(), + availableBytes: z.number(), +}); + +export type DataDisk = z.infer; + +/** The ledger's census by lifecycle state. */ +export const sandboxStateCountsSchema = z.object({ + active: z.number().int(), + frozen: z.number().int(), + stopped: z.number().int(), + archived: z.number().int(), + restoring: z.number().int(), +}); + +export type SandboxStateCounts = z.infer; + +/** + * getHostMetrics() — the observation window into the machine itself: is the + * host healthy, and what do the sandboxes collectively cost it? A single + * point-in-time snapshot; for the machine's past see getHostMetricsHistory + * below. Observation never wakes a sandbox and never touches lifecycle. + */ +export const hostMetricsResponseSchema = z.object({ + host: hostReadingSchema, + dataDisk: dataDiskSchema.nullable(), /** Ledger aggregates: what the daemon believes it is running. */ sandboxes: z.object({ total: z.number().int(), maxSandboxes: z.number().int(), - byState: z.object({ - active: z.number().int(), - frozen: z.number().int(), - stopped: z.number().int(), - archived: z.number().int(), - restoring: z.number().int(), - }), + byState: sandboxStateCountsSchema, }), /** * What the sandbox disks cost, from the executor: nominal is the summed diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index e31684ce..4d8162f9 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -5,6 +5,7 @@ export * from './config'; export * from './destroy'; export * from './exec'; export * from './files'; +export * from './gateway'; export * from './host'; export * from './images'; export * from './ingress'; From bc0e761058ed8db4ccd150e98faa21837e1ab233 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:41:35 +0800 Subject: [PATCH 05/26] =?UTF-8?q?packages/gateway:=20the=20fleet's=20one?= =?UTF-8?q?=20door=20=E2=80=94=20nodes=20check=20in,=20new=20names=20are?= =?UTF-8?q?=20placed,=20existing=20ones=20are=20found=20by=20asking,=20eve?= =?UTF-8?q?rything=20else=20is=20forwarded=20raw?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fleet is N daemons that know nothing of each other behind one gateway. The gateway holds no sandbox state: it keeps the nodes that have checked in (one table, written by the nodes themselves — no registration verb, no nodes file), what each last reported (memory), a cache of where names were last found, and a per-name slot. When the cache has no answer it asks every node "do you hold this sandbox?" in parallel, two seconds, and routes to the one that says yes; two yeses are a 409 it refuses to guess about; no yes while a node is silent is a 503 with Retry-After, because a name that lives only on a silent node must not be built a second time elsewhere. A new name is placed on the emptiest node by active density per core, skipping nodes that are down, above the CPU limit, at the active ceiling or below the data-disk floor, with each pick counted against its node until the next reading. The forwarding plane, error dialects and placement come from the first cut by file (branch router-v1), minus the directory, the claims, the reconciler and the actor header: one token for the whole fleet, and the node trusts the gateway whole. Faces in this cut: the native sandbox verbs, the E2B control plane by name or id, envd by header, the bare signed-URL door as an honest 501, and the gateway's own checkIn / listNodes / removeNode. Daemon-addressed verbs answer 501 until the configuration authority moves here; the sandbox port proxy joins with the sandbox domain. Only the fleet token opens the door in this cut; minted keys arrive with the key table. Reverse-proved: without the name slot twenty simultaneous acquires build several copies; without the placement counter a burst lands on one node; without the 404 re-check a destroy behind the gateway's back leaves a stale entry. --- packages/gateway/drizzle.config.ts | 7 + packages/gateway/drizzle/0000_nodes.sql | 5 + .../gateway/drizzle/meta/0000_snapshot.json | 49 ++ packages/gateway/drizzle/meta/_journal.json | 13 + packages/gateway/package.json | 32 + packages/gateway/src/app.test.ts | 721 +++++++++++++++++ packages/gateway/src/app.ts | 166 ++++ packages/gateway/src/cache.ts | 77 ++ packages/gateway/src/classify.ts | 48 ++ packages/gateway/src/config.test.ts | 52 ++ packages/gateway/src/config.ts | 78 ++ packages/gateway/src/db/db.ts | 26 + packages/gateway/src/db/schema.ts | 32 + packages/gateway/src/errors.ts | 117 +++ packages/gateway/src/find.test.ts | 252 ++++++ packages/gateway/src/find.ts | 122 +++ packages/gateway/src/fleet.test.ts | 94 +++ packages/gateway/src/fleet.ts | 132 ++++ packages/gateway/src/forward.test.ts | 734 ++++++++++++++++++ packages/gateway/src/forward.ts | 367 +++++++++ packages/gateway/src/lookup.ts | 71 ++ packages/gateway/src/main.ts | 135 ++++ packages/gateway/src/placement.test.ts | 181 +++++ packages/gateway/src/placement.ts | 138 ++++ packages/gateway/src/raw.ts | 192 +++++ packages/gateway/src/routes/create.ts | 129 +++ packages/gateway/src/routes/destroy.ts | 44 ++ packages/gateway/src/routes/e2b.ts | 215 +++++ packages/gateway/src/routes/native.ts | 257 ++++++ packages/gateway/src/routes/nodes.ts | 94 +++ packages/gateway/src/routes/verdict.ts | 62 ++ packages/gateway/src/testing.ts | 60 ++ packages/gateway/src/version.ts | 23 + packages/gateway/tsconfig.json | 4 + packages/gateway/tsup.config.ts | 38 + pnpm-lock.yaml | 37 + 36 files changed, 4804 insertions(+) create mode 100644 packages/gateway/drizzle.config.ts create mode 100644 packages/gateway/drizzle/0000_nodes.sql create mode 100644 packages/gateway/drizzle/meta/0000_snapshot.json create mode 100644 packages/gateway/drizzle/meta/_journal.json create mode 100644 packages/gateway/package.json create mode 100644 packages/gateway/src/app.test.ts create mode 100644 packages/gateway/src/app.ts create mode 100644 packages/gateway/src/cache.ts create mode 100644 packages/gateway/src/classify.ts create mode 100644 packages/gateway/src/config.test.ts create mode 100644 packages/gateway/src/config.ts create mode 100644 packages/gateway/src/db/db.ts create mode 100644 packages/gateway/src/db/schema.ts create mode 100644 packages/gateway/src/errors.ts create mode 100644 packages/gateway/src/find.test.ts create mode 100644 packages/gateway/src/find.ts create mode 100644 packages/gateway/src/fleet.test.ts create mode 100644 packages/gateway/src/fleet.ts create mode 100644 packages/gateway/src/forward.test.ts create mode 100644 packages/gateway/src/forward.ts create mode 100644 packages/gateway/src/lookup.ts create mode 100644 packages/gateway/src/main.ts create mode 100644 packages/gateway/src/placement.test.ts create mode 100644 packages/gateway/src/placement.ts create mode 100644 packages/gateway/src/raw.ts create mode 100644 packages/gateway/src/routes/create.ts create mode 100644 packages/gateway/src/routes/destroy.ts create mode 100644 packages/gateway/src/routes/e2b.ts create mode 100644 packages/gateway/src/routes/native.ts create mode 100644 packages/gateway/src/routes/nodes.ts create mode 100644 packages/gateway/src/routes/verdict.ts create mode 100644 packages/gateway/src/testing.ts create mode 100644 packages/gateway/src/version.ts create mode 100644 packages/gateway/tsconfig.json create mode 100644 packages/gateway/tsup.config.ts diff --git a/packages/gateway/drizzle.config.ts b/packages/gateway/drizzle.config.ts new file mode 100644 index 00000000..807d65f7 --- /dev/null +++ b/packages/gateway/drizzle.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'drizzle-kit'; + +export default defineConfig({ + schema: './src/db/schema.ts', + out: './drizzle', + dialect: 'sqlite', +}); diff --git a/packages/gateway/drizzle/0000_nodes.sql b/packages/gateway/drizzle/0000_nodes.sql new file mode 100644 index 00000000..e7f1a6a8 --- /dev/null +++ b/packages/gateway/drizzle/0000_nodes.sql @@ -0,0 +1,5 @@ +CREATE TABLE `nodes` ( + `id` text PRIMARY KEY NOT NULL, + `endpoint` text NOT NULL, + `added_at` text NOT NULL +); diff --git a/packages/gateway/drizzle/meta/0000_snapshot.json b/packages/gateway/drizzle/meta/0000_snapshot.json new file mode 100644 index 00000000..9a496820 --- /dev/null +++ b/packages/gateway/drizzle/meta/0000_snapshot.json @@ -0,0 +1,49 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "b88172b7-a470-4a8c-addb-45c45229585f", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "nodes": { + "name": "nodes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "added_at": { + "name": "added_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/_journal.json b/packages/gateway/drizzle/meta/_journal.json new file mode 100644 index 00000000..d08827ad --- /dev/null +++ b/packages/gateway/drizzle/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "7", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "6", + "when": 1789328056388, + "tag": "0000_nodes", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/packages/gateway/package.json b/packages/gateway/package.json new file mode 100644 index 00000000..d4928d73 --- /dev/null +++ b/packages/gateway/package.json @@ -0,0 +1,32 @@ +{ + "name": "@dormice/gateway", + "version": "0.0.0", + "private": true, + "description": "Dormice gateway: the fleet's one door — places new sandboxes across nodes, finds existing ones by asking, forwards everything else", + "license": "Apache-2.0", + "type": "module", + "files": [ + "dist", + "drizzle" + ], + "scripts": { + "build": "tsup", + "typecheck": "tsc --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@dormice/server": "workspace:*", + "@dormice/shared": "workspace:*", + "better-sqlite3": "^12.11.1", + "drizzle-orm": "^0.45.2", + "fastify": "^5.10.0", + "fastify-type-provider-zod": "^7.0.0", + "pino": "^10.3.1", + "undici": "^8.7.0", + "zod": "^4.4.3" + }, + "devDependencies": { + "@types/better-sqlite3": "^7.6.13", + "drizzle-kit": "^0.31.10" + } +} diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts new file mode 100644 index 00000000..cce3d7a0 --- /dev/null +++ b/packages/gateway/src/app.test.ts @@ -0,0 +1,721 @@ +import { randomUUID } from 'node:crypto'; +import http from 'node:http'; +import net, { type AddressInfo } from 'node:net'; +import { fileURLToPath } from 'node:url'; +import { KeyedQueue } from '@dormice/server/keyed-queue'; +import { afterEach, describe, expect, it } from 'vitest'; +import { buildGatewayApp } from './app'; +import { NameCache } from './cache'; +import { loadConfig } from './config'; +import { migrateDb, openDb } from './db/db'; +import { Finder } from './find'; +import { Fleet } from './fleet'; +import { httpAskNode } from './lookup'; +import { checkInOf, type reading } from './testing'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const TOKEN = 'fleet-token-fleet-token-fleet-token-fleet'; + +/** + * A node as the gateway sees one: the daemon's wire for the handful of + * verbs the gateway touches, over a real socket. Every sandbox it holds is + * a row in `sandboxes`; every request it received is in `hits`. + */ +class FakeNode { + readonly sandboxes = new Map< + string, + { id: string; name: string; files: Map } + >(); + readonly hits: Array<{ path: string; auth: string | undefined }> = []; + creates = 0; + endpoint = ''; + private readonly server: http.Server; + + constructor(readonly id: string) { + this.server = http.createServer((req, res) => { + let text = ''; + req.on('data', (c) => { + text += c; + }); + req.on('end', () => this.answer(req, res, text)); + }); + } + + async start(): Promise { + await new Promise((r) => this.server.listen(0, '127.0.0.1', r)); + this.endpoint = `http://127.0.0.1:${(this.server.address() as AddressInfo).port}`; + return this; + } + + stop(): Promise { + return new Promise((resolve) => { + this.server.closeAllConnections(); + this.server.close(() => resolve()); + }); + } + + lookups(): number { + return this.hits.filter((h) => h.path === '/lookupSandbox').length; + } + + byId(id: string) { + return [...this.sandboxes.values()].find((s) => s.id === id); + } + + private answer( + req: http.IncomingMessage, + res: http.ServerResponse, + text: string, + ): void { + const url = req.url ?? '/'; + const path = url.split('?')[0] ?? url; + const auth = + req.headers.authorization ?? + (req.headers['x-api-key'] as string | undefined); + this.hits.push({ path, auth }); + const json = (status: number, body: unknown) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(body)); + }; + const body = text ? (JSON.parse(text) as Record) : {}; + if (path.startsWith('/e2b/envd/')) { + return json(200, { + envd: this.id, + sandboxId: req.headers['e2b-sandbox-id'], + accessToken: req.headers['x-access-token'], + url, + }); + } + if (path.startsWith('/e2b/api/')) { + if (auth !== `e2b_${TOKEN}`) + return json(401, { code: 401, message: 'invalid API key' }); + const m = path.match(/^\/e2b\/api\/sandboxes(?:\/([^/]+))?(\/.*)?$/); + if (!m) return json(404, { code: 404, message: 'not found' }); + const [, id, rest] = m; + if (id === undefined) { + if (req.method !== 'POST') + return json(404, { code: 404, message: 'not found' }); + const metadata = body.metadata as { name?: string } | undefined; + const name = metadata?.name ?? `e2b-${randomUUID()}`; + // The daemon's create is idempotent on metadata.name. + const sandbox = this.sandboxes.get(name) ?? this.create(name); + return json(201, { sandboxID: sandbox.id, templateID: 'base' }); + } + const sandbox = this.byId(id); + if (!sandbox) return json(404, { code: 404, message: 'not found' }); + if (rest) return json(200, { path: url, sandboxID: sandbox.id }); + if (req.method === 'DELETE') { + this.sandboxes.delete(sandbox.name); + res.writeHead(204); + res.end(); + return; + } + return json(200, { sandboxID: sandbox.id, alias: sandbox.name }); + } + if (auth !== `Bearer ${TOKEN}`) + return json(401, { message: 'missing or invalid API token' }); + const name = body.name as string | undefined; + const found = name === undefined ? undefined : this.sandboxes.get(name); + switch (path) { + case '/lookupSandbox': { + const sandbox = 'id' in body ? this.byId(body.id as string) : found; + return json( + 200, + sandbox + ? { + found: true, + sandbox: { + id: sandbox.id, + name: sandbox.name, + state: 'active', + }, + } + : { found: false }, + ); + } + case '/acquireSandbox': { + if (name === undefined) return json(400, { message: 'name required' }); + const sandbox = found ?? this.create(name); + return json(200, { + status: 'ready', + created: found === undefined, + sandbox: { id: sandbox.id, name, nodeId: this.id }, + }); + } + case '/destroySandbox': { + if (name !== undefined) this.sandboxes.delete(name); + return json(200, { destroyed: found !== undefined }); + } + case '/execCommand': { + if (!found) + return json(404, { + message: `no sandbox named "${name}" — acquire it first`, + }); + return json(200, { stdout: `${this.id}: ${String(body.command)}` }); + } + case '/writeFile': { + if (!found) + return json(404, { + message: `no sandbox named "${name}" — acquire it first`, + }); + found.files.set(String(body.path), String(body.content)); + return json(200, { written: true }); + } + case '/readFile': { + if (!found) + return json(404, { + message: `no sandbox named "${name}" — acquire it first`, + }); + const content = found.files.get(String(body.path)); + if (content === undefined) + return json(404, { message: `no such file: ${String(body.path)}` }); + return json(200, { content }); + } + default: + return json(404, { message: `route ${req.method} ${url} not found` }); + } + } + + private create(name: string) { + this.creates += 1; + const sandbox = { id: randomUUID(), name, files: new Map() }; + this.sandboxes.set(name, sandbox); + return sandbox; + } +} + +interface Harness { + endpoint: string; + nodes: FakeNode[]; + cache: NameCache; + fleet: Fleet; + checkIn( + node: FakeNode, + over?: Parameters[0] & { intervalSeconds?: number }, + ): Promise; + close(): Promise; +} + +const harnesses: Harness[] = []; +afterEach(async () => { + for (const h of harnesses.splice(0)) await h.close(); +}); + +async function gateway( + nodeIds: string[], + env: Record = {}, +): Promise { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const config = loadConfig({ + DORMICE_API_TOKEN: TOKEN, + DORMICE_GATEWAY_DB_PATH: ':memory:', + // A laptop running the suite is not the machine under judgment. + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '100', + ...env, + }); + const fleet = new Fleet(db); + const cache = new NameCache(); + const finder = new Finder(fleet, cache, httpAskNode(TOKEN), { + warn: () => {}, + }); + const app = buildGatewayApp({ + config, + fleet, + finder, + locks: new KeyedQueue(), + logger: false, + build: null, + }); + await app.listen({ host: '127.0.0.1', port: 0 }); + const endpoint = `http://127.0.0.1:${(app.server.address() as AddressInfo).port}`; + const nodes = await Promise.all( + nodeIds.map((id) => new FakeNode(id).start()), + ); + const harness: Harness = { + endpoint, + nodes, + cache, + fleet, + checkIn: async (node, over = {}) => { + const res = await fetch(`${endpoint}/checkIn`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(checkInOf(node.id, node.endpoint, over)), + }); + expect(res.status).toBe(200); + }, + close: async () => { + await app.close(); + for (const node of nodes) await node.stop(); + }, + }; + for (const node of nodes) await harness.checkIn(node); + harnesses.push(harness); + return harness; +} + +async function rpc( + h: Harness, + path: string, + payload: unknown = {}, + token = TOKEN, +): Promise<{ status: number; body: unknown; headers: Headers }> { + const res = await fetch(`${h.endpoint}${path}`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(payload), + }); + const text = await res.text(); + return { + status: res.status, + body: text ? JSON.parse(text) : null, + headers: res.headers, + }; +} + +const message = (r: { body: unknown }) => + (r.body as { message: string }).message; +const sandboxOf = (r: { body: unknown }) => + (r.body as { created: boolean; sandbox: { id: string; nodeId: string } }) + .sandbox; + +/** Polls until the probe answers something — cache verification runs off the request path. */ +async function until( + probe: () => Promise | T | undefined, + timeoutMs = 2_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + const value = await probe(); + if (value !== undefined) return value; + if (Date.now() > deadline) throw new Error('condition never became true'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +describe('the door', () => { + it('/healthz is open; everything else wants the fleet token, on the check-in too', async () => { + const h = await gateway([]); + const health = await fetch(`${h.endpoint}/healthz`); + expect(health.status).toBe(200); + expect(await health.json()).toEqual({ status: 'ok', build: null }); + expect((await rpc(h, '/listNodes', {}, 'w'.repeat(40))).status).toBe(401); + const bare = await fetch(`${h.endpoint}/checkIn`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(checkInOf('x', 'http://127.0.0.1:1')), + }); + expect(bare.status).toBe(401); + expect((await rpc(h, '/acquireSandbox', { name: 'x' }, 'bad')).status).toBe( + 401, + ); + }); + + it('refuses an absolute-form request target before routing', async () => { + const h = await gateway([]); + const url = new URL(h.endpoint); + const answer = await new Promise((resolve, reject) => { + let text = ''; + const socket = net.connect(Number(url.port), url.hostname, () => { + socket.write( + `POST http://evil.example/execCommand HTTP/1.1\r\nhost: gateway\r\nconnection: close\r\n\r\n`, + ); + }); + socket.on('data', (c) => { + text += c; + }); + socket.on('end', () => resolve(text)); + socket.on('error', reject); + }); + expect(answer).toContain('HTTP/1.1 400'); + expect(answer).toContain('origin-form'); + }); +}); + +describe('check-in and the node verbs', () => { + it('a check-in joins the fleet; listNodes shows what it said; a later check-in moves the endpoint; the row outlives the memory', async () => { + const h = await gateway(['b']); + const listed = (await rpc(h, '/listNodes')).body as { + nodes: Array>; + }; + expect(listed.nodes).toHaveLength(1); + expect(listed.nodes[0]).toMatchObject({ + id: 'b', + endpoint: h.nodes[0]?.endpoint, + reachable: true, + intervalSeconds: 15, + placedSinceCheckIn: 0, + build: { commit: 'abc1234' }, + }); + expect( + (listed.nodes[0]?.reading as { host: { cpuCount: number } }).host + .cpuCount, + ).toBe(8); + const moved = new FakeNode('b'); + await moved.start(); + await h.checkIn(moved, { active: 3 }); + expect(h.fleet.get('b')?.endpoint).toBe(moved.endpoint); + expect(h.fleet.get('b')?.reading?.sandboxes.byState.active).toBe(3); + await moved.stop(); + // A malformed check-in is a 400 naming the trouble, not a join. + const bad = await rpc(h, '/checkIn', { nodeId: 'z' }); + expect(bad.status).toBe(400); + expect(h.fleet.get('z')).toBeUndefined(); + }); + + it('removeNode forgets the node and everything cached on it; a removed node that checks in again re-joins', async () => { + const h = await gateway(['b', 'c']); + const created = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'x' })); + expect(h.cache.getByName('x')?.nodeId).toBe(created.nodeId); + expect((await rpc(h, '/removeNode', { id: created.nodeId })).body).toEqual({ + removed: true, + }); + expect(h.cache.getByName('x')).toBeUndefined(); + expect( + ((await rpc(h, '/listNodes')).body as { nodes: unknown[] }).nodes, + ).toHaveLength(1); + expect((await rpc(h, '/removeNode', { id: 'nobody' })).body).toEqual({ + removed: false, + }); + const back = h.nodes.find((n) => n.id === created.nodeId); + if (!back) throw new Error('node lost'); + await h.checkIn(back); + expect( + ((await rpc(h, '/listNodes')).body as { nodes: unknown[] }).nodes, + ).toHaveLength(2); + }); +}); + +describe('acquire: placing and finding', () => { + it('a new name lands on the emptiest node by active density, under the fleet token, and is cached: the second acquire asks nobody', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 10, cores: 8 }); + await h.checkIn(b, { active: 1, cores: 8 }); + const first = await rpc(h, '/acquireSandbox', { name: 'alice' }); + expect(first.status).toBe(200); + expect(sandboxOf(first).nodeId).toBe('b'); + expect(b.creates).toBe(1); + expect(a.creates).toBe(0); + expect(b.hits.find((hit) => hit.path === '/acquireSandbox')?.auth).toBe( + `Bearer ${TOKEN}`, + ); + // Both nodes were asked once — the name was new — and the answer's id + // is cached beside the name. + expect(a.lookups()).toBe(1); + expect(b.lookups()).toBe(1); + expect(h.cache.getById(sandboxOf(first).id)?.nodeId).toBe('b'); + const again = await rpc(h, '/acquireSandbox', { name: 'alice' }); + expect(sandboxOf(again).id).toBe(sandboxOf(first).id); + expect((again.body as { created: boolean }).created).toBe(false); + expect(a.lookups()).toBe(1); + expect(b.lookups()).toBe(1); + expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); + }); + + it('twenty simultaneous acquires of one new name are one create on one node; different names spread', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + const burst = await Promise.all( + Array.from({ length: 20 }, () => + rpc(h, '/acquireSandbox', { name: 'burst' }), + ), + ); + expect(new Set(burst.map((r) => sandboxOf(r).id)).size).toBe(1); + expect(a.creates + b.creates).toBe(1); + expect( + burst.filter((r) => (r.body as { created: boolean }).created), + ).toHaveLength(1); + + const spread = await Promise.all( + Array.from({ length: 6 }, (_, i) => + rpc(h, '/acquireSandbox', { name: `spread-${i}` }), + ), + ); + const landed = spread.map((r) => sandboxOf(r).nodeId); + expect(landed.filter((id) => id === 'a').length).toBeGreaterThan(0); + expect(landed.filter((id) => id === 'b').length).toBeGreaterThan(0); + }); + + it('a sandbox built behind its back is found by asking: routed at once, re-acquired where it is', async () => { + const h = await gateway(['a', 'b']); + const b = h.nodes[1] as FakeNode; + const staged = sandboxOf( + await (async () => { + const res = await fetch(`${b.endpoint}/acquireSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'staged' }), + }); + return { body: await res.json() }; + })(), + ); + const ran = await rpc(h, '/execCommand', { name: 'staged', command: 'id' }); + expect(ran.status).toBe(200); + expect((ran.body as { stdout: string }).stdout).toBe('b: id'); + const found = await rpc(h, '/acquireSandbox', { name: 'staged' }); + expect(sandboxOf(found)).toMatchObject({ id: staged.id, nodeId: 'b' }); + expect((found.body as { created: boolean }).created).toBe(false); + }); + + it('one name on two nodes is a 409 naming both, for every verb; once one copy is gone the name routes again', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + for (const node of [a, b]) { + await fetch(`${node.endpoint}/acquireSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'twin' }), + }); + } + for (const verb of ['/acquireSandbox', '/execCommand', '/destroySandbox']) { + const refused = await rpc(h, verb, { name: 'twin' }); + expect(refused.status).toBe(409); + expect(message(refused)).toContain('exists on nodes a and b'); + } + expect(h.cache.getByName('twin')).toBeUndefined(); + b.sandboxes.delete('twin'); + const healed = await rpc(h, '/acquireSandbox', { name: 'twin' }); + expect(sandboxOf(healed).nodeId).toBe('a'); + }); + + it('a node that does not answer: its cached names 502, and a new name is a 503 with Retry-After naming it — until an operator removes it', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 1 }); + await h.checkIn(b, { active: 50 }); + const onA = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'on-a' })); + expect(onA.nodeId).toBe('a'); + await a.stop(); + const cut = await rpc(h, '/execCommand', { name: 'on-a', command: 'x' }); + expect(cut.status).toBe(502); + expect(message(cut)).toMatch(/did not answer/); + const fresh = await rpc(h, '/acquireSandbox', { name: 'brand-new' }); + expect(fresh.status).toBe(503); + expect(fresh.headers.get('retry-after')).toBe('15'); + expect(message(fresh)).toContain('node a did not answer'); + expect(message(fresh)).toContain('cannot be treated as new'); + expect(b.creates).toBe(0); + expect((await rpc(h, '/removeNode', { id: 'a' })).body).toEqual({ + removed: true, + }); + const placed = await rpc(h, '/acquireSandbox', { name: 'brand-new' }); + expect(placed.status).toBe(200); + expect(sandboxOf(placed).nodeId).toBe('b'); + }); + + it('every node refusing is a 503 that names each one; no node at all says so', async () => { + const h = await gateway(['a', 'b'], { + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', + }); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 2 }); + await h.checkIn(b, { active: 1, diskAvail: 2 ** 30 }); + const refused = await rpc(h, '/acquireSandbox', { name: 'full' }); + expect(refused.status).toBe(503); + expect(message(refused)).toContain('a: 2 active sandboxes + 0 placed'); + expect(message(refused)).toContain('b: data disk has 1.0 GiB available'); + expect(refused.headers.get('retry-after')).toBe('15'); + + const empty = await gateway([]); + const nobody = await rpc(empty, '/acquireSandbox', { name: 'x' }); + expect(nobody.status).toBe(503); + expect(message(nobody)).toMatch(/no node has checked in/); + }); +}); + +describe('using, destroying, and the cache', () => { + it("files round-trip; a node's 404 for a missing file passes through and the sandbox stays cached", async () => { + const h = await gateway(['a']); + const a = h.nodes[0] as FakeNode; + await rpc(h, '/acquireSandbox', { name: 'w' }); + expect( + (await rpc(h, '/writeFile', { name: 'w', path: 'a.txt', content: 'hi' })) + .status, + ).toBe(200); + expect( + (await rpc(h, '/readFile', { name: 'w', path: 'a.txt' })).body, + ).toEqual({ content: 'hi' }); + const missing = await rpc(h, '/readFile', { name: 'w', path: 'nope' }); + expect(missing.status).toBe(404); + expect(message(missing)).toBe('no such file: nope'); + // The 404 sent one question to the node, which said yes: the entry stays. + await until(() => (a.lookups() >= 2 ? true : undefined)); + expect(h.cache.getByName('w')?.nodeId).toBe('a'); + expect( + (await rpc(h, '/execCommand', { name: 'w', command: 'ok' })).status, + ).toBe(200); + }); + + it('a destroy the gateway relays forgets the entry; a destroy behind its back is caught by the 404 re-check', async () => { + const h = await gateway(['a']); + const a = h.nodes[0] as FakeNode; + const first = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'd' })); + expect((await rpc(h, '/destroySandbox', { name: 'd' })).body).toEqual({ + destroyed: true, + }); + expect(h.cache.getByName('d')).toBeUndefined(); + expect(h.cache.getById(first.id)).toBeUndefined(); + expect((await rpc(h, '/destroySandbox', { name: 'd' })).body).toEqual({ + destroyed: false, + }); + const second = await rpc(h, '/acquireSandbox', { name: 'd' }); + expect((second.body as { created: boolean }).created).toBe(true); + expect(sandboxOf(second).id).not.toBe(first.id); + + // Behind the gateway's back: the node's own 404 is relayed as it came, + // the re-check finds the sandbox absent, and the next request asks + // the fleet afresh — a 404 in the gateway's own words. + a.sandboxes.delete('d'); + const relayed = await rpc(h, '/execCommand', { name: 'd', command: 'x' }); + expect(relayed.status).toBe(404); + await until(() => + h.cache.getByName('d') === undefined ? true : undefined, + ); + const before = a.lookups(); + const own = await rpc(h, '/execCommand', { name: 'd', command: 'x' }); + expect(own.status).toBe(404); + expect(message(own)).toBe('no sandbox named "d" — acquire it first'); + expect(a.lookups()).toBe(before + 1); + }); + + it('daemon-addressed verbs are an honest 501, a misspelled verb a 404, a body without a name a 400', async () => { + const h = await gateway(['a']); + const listed = await rpc(h, '/listSandboxes'); + expect(listed.status).toBe(501); + expect(message(listed)).toContain('call the node directly'); + expect((await rpc(h, '/createApiKey', { name: 'k' })).status).toBe(501); + expect((await rpc(h, '/acquireSandbx', { name: 'x' })).status).toBe(404); + expect((await rpc(h, '/execCommand', { command: 'x' })).status).toBe(400); + expect(h.nodes[0]?.hits).toEqual([]); + }); +}); + +describe('the E2B faces', () => { + const e2b = (h: Harness, path: string, init: RequestInit = {}) => + fetch(`${h.endpoint}/e2b/api${path}`, { + ...init, + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + ...(init.headers as Record | undefined), + }, + }); + + it('control plane: a named create is placed and cached; by-id verbs are found and forwarded under e2b_; the kill forgets the entry; an unnamed create is known by id', async () => { + const h = await gateway(['a', 'b']); + const created = await e2b(h, '/sandboxes', { + method: 'POST', + body: JSON.stringify({ metadata: { name: 'e2b-named' } }), + }); + expect(created.status).toBe(201); + const { sandboxID } = (await created.json()) as { sandboxID: string }; + const entry = h.cache.getById(sandboxID); + expect(entry?.name).toBe('e2b-named'); + const node = h.nodes.find((n) => n.id === entry?.nodeId); + if (!node) throw new Error('not placed'); + const info = await e2b(h, `/sandboxes/${sandboxID}`); + expect(info.status).toBe(200); + expect(node.hits.at(-1)?.auth).toBe(`e2b_${TOKEN}`); + const deeper = await e2b(h, `/sandboxes/${sandboxID}/metrics?x=1`, { + method: 'GET', + }); + expect(((await deeper.json()) as { path: string }).path).toBe( + `/e2b/api/sandboxes/${sandboxID}/metrics?x=1`, + ); + // The same name through the E2B face is the same sandbox (the name slot). + const again = await e2b(h, '/sandboxes', { + method: 'POST', + body: JSON.stringify({ metadata: { name: 'e2b-named' } }), + }); + expect(((await again.json()) as { sandboxID: string }).sandboxID).toBe( + sandboxID, + ); + const killed = await e2b(h, `/sandboxes/${sandboxID}`, { + method: 'DELETE', + }); + expect(killed.status).toBe(204); + expect(h.cache.getById(sandboxID)).toBeUndefined(); + const gone = await e2b(h, `/sandboxes/${sandboxID}`); + expect(gone.status).toBe(404); + expect(await gone.json()).toEqual({ + code: 404, + message: `sandbox "${sandboxID}" not found`, + }); + + const anonymous = await e2b(h, '/sandboxes', { + method: 'POST', + body: JSON.stringify({ templateID: 'base' }), + }); + expect(anonymous.status).toBe(201); + const anon = (await anonymous.json()) as { sandboxID: string }; + expect(h.cache.getById(anon.sandboxID)?.name).toBeNull(); + expect((await e2b(h, `/sandboxes/${anon.sandboxID}`)).status).toBe(200); + + expect((await e2b(h, '/v2/sandboxes')).status).toBe(501); + const wrongKey = await fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { 'x-api-key': 'e2b_wrong', 'content-type': 'application/json' }, + body: '{}', + }); + expect(wrongKey.status).toBe(401); + expect(await wrongKey.json()).toEqual({ + code: 401, + message: 'invalid API key', + }); + }); + + it("envd: routed by the E2b-Sandbox-Id header with the caller's own credentials untouched; refusals wear the connect dialect with CORS", async () => { + const h = await gateway(['a']); + const created = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'e' })); + const hit = await fetch(`${h.endpoint}/e2b/envd/files?path=/x`, { + headers: { 'e2b-sandbox-id': created.id, 'x-access-token': 'hmac-1' }, + }); + expect(hit.status).toBe(200); + expect(await hit.json()).toEqual({ + envd: 'a', + sandboxId: created.id, + accessToken: 'hmac-1', + url: '/e2b/envd/files?path=/x', + }); + const headerless = await fetch(`${h.endpoint}/e2b/envd/files`); + expect(headerless.status).toBe(401); + expect(headerless.headers.get('access-control-allow-origin')).toBe('*'); + expect(await headerless.json()).toEqual({ + code: 'unauthenticated', + message: 'missing E2b-Sandbox-Id header', + }); + const stranger = await fetch(`${h.endpoint}/e2b/envd/files`, { + headers: { 'e2b-sandbox-id': randomUUID() }, + }); + expect(stranger.status).toBe(502); + expect(((await stranger.json()) as { code: string }).code).toBe( + 'unavailable', + ); + const preflight = await fetch(`${h.endpoint}/e2b/envd/files`, { + method: 'OPTIONS', + headers: { origin: 'https://app.example' }, + }); + expect(preflight.status).toBe(204); + expect(preflight.headers.get('access-control-allow-origin')).toBe('*'); + const bare = await fetch(`${h.endpoint}/files?signature=x`); + expect(bare.status).toBe(501); + expect(bare.headers.get('access-control-allow-origin')).toBe('*'); + expect(((await bare.json()) as { code: string }).code).toBe( + 'unimplemented', + ); + }); +}); diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts new file mode 100644 index 00000000..906e8e77 --- /dev/null +++ b/packages/gateway/src/app.ts @@ -0,0 +1,166 @@ +import http from 'node:http'; +import { tokensEqual } from '@dormice/server/auth'; +import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import fastify, { type FastifyError, type FastifyServerFactory } from 'fastify'; +import { + serializerCompiler, + validatorCompiler, + type ZodTypeProvider, +} from 'fastify-type-provider-zod'; +import { type Logger, pino } from 'pino'; +import { z } from 'zod'; +import { classify, isOriginForm } from './classify'; +import type { Config } from './config'; +import { renderError } from './errors'; +import type { Finder } from './find'; +import type { Fleet } from './fleet'; +import type { PlacementKnobs } from './placement'; +import { createRawFaces } from './raw'; +import { e2bControlRoutes } from './routes/e2b'; +import { nativeRoutes } from './routes/native'; +import { nodeRoutes } from './routes/nodes'; +import { type BuildInfo, readBuildInfo } from './version'; + +export interface GatewayAppDeps { + config: Config; + fleet: Fleet; + finder: Finder; + /** One queue for the whole gateway: the create and destroy verbs of both faces share per-name slots. */ + locks: KeyedQueue; + /** A pino instance, or a boolean for the default logger (false = silent, for tests). */ + logger?: Logger | boolean; + /** The build identity /healthz reports; null when built outside a checkout. */ + build?: BuildInfo | null; +} + +/** Placement's knobs, read once from the config. */ +export function placementKnobs(config: Config): PlacementKnobs { + return { + cpuLimitPct: config.DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT, + activeLimit: config.DORMICE_GATEWAY_NODE_ACTIVE_LIMIT, + minDiskAvailableBytes: config.DORMICE_GATEWAY_NODE_MIN_DISK_GB * 2 ** 30, + }; +} + +/** + * Builds the gateway's Fastify instance — the daemon's shape (zod as + * validator and serializer, one error dialect, /healthz open), without + * the daemon's body. Building is separate from listening so tests inject + * requests without a port; the raw faces need real sockets. + */ +export function buildGatewayApp({ + config, + fleet, + finder, + locks, + logger = true, + build = readBuildInfo(), +}: GatewayAppDeps) { + const loggerInstance = + typeof logger === 'boolean' ? pino({ enabled: logger }) : logger; + const token = config.DORMICE_API_TOKEN; + + // The face keyed on a header sits in front of Fastify, exactly as the + // daemon's port proxy does (server/app.ts): refuse what is not an + // origin-form target (classify.ts isOriginForm), triage the raw + // request, hand the rest to Fastify. app.inject() bypasses the factory, + // so that face is exercised over real sockets only. + const raw = createRawFaces({ finder, token, log: loggerInstance }); + const serverFactory: FastifyServerFactory = (handler) => { + const server = http.createServer((req, res) => { + if (!isOriginForm(req)) { + renderError(res, 'native', { + status: 400, + message: + 'request target must be origin-form (a path starting with "/")', + }); + return; + } + const kind = classify(req); + if (kind.face === 'fastify') handler(req, res); + else raw.handleRequest(kind, req, res); + }); + server.on('upgrade', (req, socket) => { + raw.handleUpgrade(classify(req), req, socket); + }); + // The gateway only relays; the node's own request timeout is the one + // that should fire on a slow upload, not a second one in front of it. + server.requestTimeout = 0; + return server; + }; + const app = fastify({ + loggerInstance, + serverFactory, + }).withTypeProvider(); + app.setValidatorCompiler(validatorCompiler); + app.setSerializerCompiler(serializerCompiler); + + // The native error dialect, verbatim from the daemon: every non-2xx body + // the gateway itself produces is { message }. Errors a node produced are + // forwarded as they came, in whichever dialect that face speaks. + app.setErrorHandler((error: FastifyError, request, reply) => { + const status = error.statusCode ?? 500; + if (status >= 500) { + request.log.error(error, 'request failed'); + } + reply.code(status).send({ message: error.message }); + }); + app.setNotFoundHandler((request, reply) => { + reply + .code(404) + .send({ message: `route ${request.method} ${request.url} not found` }); + }); + + // Liveness, open by design; the build identity so an operator can tell + // which commit answers without a token. + app.get( + '/healthz', + { + schema: { + response: { + 200: z.object({ + status: z.literal('ok'), + build: z + .object({ + commit: z.string(), + title: z.string(), + committedAt: z.string(), + }) + .nullable(), + }), + }, + }, + }, + async () => ({ status: 'ok' as const, build }), + ); + + // One credential opens the door: the fleet's token, presented as a + // Bearer by callers and by nodes checking in alike. Keys minted by the + // gateway — many, expiring, revocable — arrive with its key table. + const knobs = placementKnobs(config); + app.register(async (api) => { + api.addHook('onRequest', async (request, reply) => { + const header = request.headers.authorization; + const bare = header?.startsWith('Bearer ') ? header.slice(7) : null; + if (bare === null || !tokensEqual(bare, token)) { + await reply.code(401).send({ message: 'missing or invalid API token' }); + } + }); + await api.register(nodeRoutes, { fleet, cache: finder.cache }); + // Its own sub-scope: the byte-preserving body parser it installs must + // not reach the gateway's own verbs, which keep Fastify's JSON parsing. + await api.register(nativeRoutes, { fleet, finder, locks, knobs, token }); + }); + + // The E2B control plane, its own auth and dialect, like the daemon's. + app.register(e2bControlRoutes, { + fleet, + finder, + locks, + knobs, + token, + prefix: '/e2b/api', + }); + + return app; +} diff --git a/packages/gateway/src/cache.ts b/packages/gateway/src/cache.ts new file mode 100644 index 00000000..0803928a --- /dev/null +++ b/packages/gateway/src/cache.ts @@ -0,0 +1,77 @@ +/** + * Where a sandbox was last found: name and id → node. A cache, not a + * directory — nothing here is authoritative, the ledger of the node that + * runs the sandbox is — so it is never persisted, never reconciled and + * never written from anything but a node's own answer: a lookup that said + * yes, a create that answered 2xx. It is dropped on a destroy the gateway + * relayed, on a node's 404 that a fresh lookup confirms (find.ts verify), + * and wholesale for a node an operator removed. An entry that is wrong + * costs one misrouted request, whose answer evicts it; a cache that is + * lost costs one extra round of questions per name. + */ +export interface CacheEntry { + id: string; + /** Null for a sandbox learned by id alone (an unnamed E2B create) until a lookup names it. */ + name: string | null; + nodeId: string; +} + +export class NameCache { + private readonly byName = new Map(); + private readonly byId = new Map(); + + put(entry: CacheEntry): void { + // A name that moves to a new id (destroyed and re-acquired elsewhere + // while this gateway did not see the destroy) drops the old entry + // whole, so no id points at a node that no longer holds it. + if (entry.name !== null) { + const previous = this.byName.get(entry.name); + if (previous !== undefined && previous.id !== entry.id) { + this.byId.delete(previous.id); + } + this.byName.set(entry.name, entry); + } + const known = this.byId.get(entry.id); + if ( + known?.name !== null && + known?.name !== undefined && + known.name !== entry.name + ) { + this.byName.delete(known.name); + } + this.byId.set(entry.id, entry); + } + + getByName(name: string): CacheEntry | undefined { + return this.byName.get(name); + } + + getById(id: string): CacheEntry | undefined { + return this.byId.get(id); + } + + evict(entry: CacheEntry): void { + if (entry.name !== null && this.byName.get(entry.name)?.id === entry.id) { + this.byName.delete(entry.name); + } + if (this.byId.get(entry.id)?.nodeId === entry.nodeId) { + this.byId.delete(entry.id); + } + } + + /** Everything that pointed at a node the operator removed. */ + evictNode(nodeId: string): number { + let evicted = 0; + for (const entry of [...this.byId.values()]) { + if (entry.nodeId === nodeId) { + this.evict(entry); + evicted += 1; + } + } + return evicted; + } + + get size(): number { + return this.byId.size; + } +} diff --git a/packages/gateway/src/classify.ts b/packages/gateway/src/classify.ts new file mode 100644 index 00000000..94f91adc --- /dev/null +++ b/packages/gateway/src/classify.ts @@ -0,0 +1,48 @@ +/** + * Which face a raw request belongs to — the gateway's serverFactory + * triages every request before Fastify's router sees it, because one face + * is keyed on something Fastify cannot route on (a header, on any path). + * Pure, so the order of the tests is a fact here and nowhere else: + * envd /e2b/envd/* — E2B's in-sandbox API, keyed by the + * E2b-Sandbox-Id header. + * signedRoot exactly /files at the root — the bare signed-URL form, + * which carries no sandbox id anywhere the gateway can read + * without the node's signing secret. + * fastify everything else — the native verbs, /e2b/api, /healthz, + * the gateway's own verbs. + * The sandbox port proxy (Host `-.`) joins this list + * when the sandbox domain moves into the gateway's settings. + */ +export type Classified = + | { face: 'envd' } + | { face: 'signedRoot' } + | { face: 'fastify' }; + +/** + * Only the origin-form request target (RFC 9112 §3.2.1, a path starting + * with `/`) is forwarded. The gateway routes on the path Fastify extracts + * — find-my-way strips a scheme and authority off an absolute-form target + * — and sends the target to the node byte for byte, so on an absolute + * form the two would read different requests: `POST http://8000-. + * /execCommand` is `/execCommand` to the gateway (a native verb, + * authenticated, the fleet's token attached) and, to the RFC-7230 hop in + * front of the node (Caddy: Host := the target's authority), a request to + * the node's sandbox proxy — the fleet's credential delivered into the + * tenant's own sandbox (traced 2026-09-12). Absolute form is what a + * client sends a forward proxy, which the gateway is not; every real + * client of an origin server sends origin form, so refusing the rest + * costs nothing and keeps the invariant whole: the gateway and the node + * see the same request. + */ +export function isOriginForm(req: { url?: string }): boolean { + return (req.url ?? '').startsWith('/'); +} + +export function classify(req: { url?: string }): Classified { + const url = req.url ?? ''; + const q = url.indexOf('?'); + const path = q === -1 ? url : url.slice(0, q); + if (url.startsWith('/e2b/envd/')) return { face: 'envd' }; + if (path === '/files') return { face: 'signedRoot' }; + return { face: 'fastify' }; +} diff --git a/packages/gateway/src/config.test.ts b/packages/gateway/src/config.test.ts new file mode 100644 index 00000000..186aa5cb --- /dev/null +++ b/packages/gateway/src/config.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; +import { loadConfig } from './config'; + +const TOKEN = { DORMICE_API_TOKEN: 'x'.repeat(32) }; + +describe('loadConfig', () => { + it('defaults: port 3677, an absolute database path, placement knobs 70% / 400 active / 10 GiB', () => { + const config = loadConfig(TOKEN); + expect(config.DORMICE_GATEWAY_PORT).toBe(3677); + expect(config.DORMICE_GATEWAY_DB_PATH).toBe( + '/var/lib/dormice-gateway/gateway.db', + ); + expect(config.DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT).toBe(70); + expect(config.DORMICE_GATEWAY_NODE_ACTIVE_LIMIT).toBe(400); + expect(config.DORMICE_GATEWAY_NODE_MIN_DISK_GB).toBe(10); + }); + + it('requires the fleet token, at least 32 characters, naming the variable', () => { + expect(() => loadConfig({})).toThrow(/DORMICE_API_TOKEN/); + expect(() => loadConfig({ DORMICE_API_TOKEN: 'short' })).toThrow( + /at least 32 characters/, + ); + }); + + it('refuses a relative database path and accepts :memory:', () => { + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_DB_PATH: 'data/gateway.db' }), + ).toThrow(/DORMICE_GATEWAY_DB_PATH must be an absolute path/); + expect( + loadConfig({ ...TOKEN, DORMICE_GATEWAY_DB_PATH: ':memory:' }) + .DORMICE_GATEWAY_DB_PATH, + ).toBe(':memory:'); + }); + + it('parses the placement knobs and refuses nonsense', () => { + const config = loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '85', + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', + DORMICE_GATEWAY_NODE_MIN_DISK_GB: '0.5', + }); + expect(config.DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT).toBe(85); + expect(config.DORMICE_GATEWAY_NODE_ACTIVE_LIMIT).toBe(2); + expect(config.DORMICE_GATEWAY_NODE_MIN_DISK_GB).toBe(0.5); + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '101' }), + ).toThrow(); + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '0' }), + ).toThrow(); + }); +}); diff --git a/packages/gateway/src/config.ts b/packages/gateway/src/config.ts new file mode 100644 index 00000000..94facb04 --- /dev/null +++ b/packages/gateway/src/config.ts @@ -0,0 +1,78 @@ +import { isAbsolute } from 'node:path'; +import { z } from 'zod'; + +/** + * The gateway's configuration: environment variables, validated once at + * startup so a bad value fails loudly here — the daemon's discipline + * (packages/server/src/config.ts). The gateway's own knobs carry the + * DORMICE_GATEWAY_ prefix so a gateway and a node can share one machine's + * /etc/dormice without colliding; the token deliberately does not — it is + * the very same DORMICE_API_TOKEN every node has, one string for the whole + * fleet, written into both env files by the same hand. + */ +const envSchema = z.object({ + DORMICE_GATEWAY_PORT: z.coerce.number().int().min(1).max(65535).default(3677), + /** + * Absolute, like the daemon's in docker mode and for the same reason: a + * relative path depends on the start directory, and under systemd (no + * WorkingDirectory) that is `/` — a gateway started without this variable + * would keep its tables in /data, silently, until the day someone sets + * the path and every node it ever knew is gone. + */ + DORMICE_GATEWAY_DB_PATH: z + .string() + .default('/var/lib/dormice-gateway/gateway.db') + .refine((path) => path === ':memory:' || isAbsolute(path), { + error: + 'DORMICE_GATEWAY_DB_PATH must be an absolute path, e.g. /var/lib/dormice-gateway/gateway.db', + }), + /** + * The one token of the fleet. Callers present it to the gateway, nodes + * present it when they check in, and the gateway presents it to nodes + * when it forwards. Required, no default: loopback-only is not + * authentication. + */ + DORMICE_API_TOKEN: z.string().min(32, { + error: + 'DORMICE_API_TOKEN must be at least 32 characters — generate one with: openssl rand -hex 32', + }), + /** + * Placement: a node whose last whole-machine CPU reading is above this + * takes no new sandboxes. 70 leaves the headroom a wake burst needs — + * the 2026-09-11 Beijing incident saturated a 128-core host from a + * reading well under 100 in one burst of cold starts. + */ + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: z.coerce + .number() + .min(0) + .max(100) + .default(70), + /** + * Placement: a node already running this many active sandboxes (plus + * those placed on it since its last check-in) takes no more. Active + * only — frozen sandboxes cost swap, not CPU or dockerd attention, and + * a node holds thousands of them; the ceiling this guards is running + * containers per host. + */ + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: z.coerce + .number() + .int() + .positive() + .default(400), + /** + * Placement: a node whose data disk has less than this free takes no new + * sandboxes. Ten GiB is one default sandbox disk: below that, a single + * new sandbox writing its disk full would fill the host, and a full data + * disk is the one failure that stops every sandbox on the node at once + * (the ledger itself cannot write). The node's own create still answers + * its honest 500 past this point; the gate is what keeps the next ten + * names from landing on the same full box. + */ + DORMICE_GATEWAY_NODE_MIN_DISK_GB: z.coerce.number().nonnegative().default(10), +}); + +export type Config = z.infer; + +export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config { + return envSchema.parse(env); +} diff --git a/packages/gateway/src/db/db.ts b/packages/gateway/src/db/db.ts new file mode 100644 index 00000000..f0865403 --- /dev/null +++ b/packages/gateway/src/db/db.ts @@ -0,0 +1,26 @@ +import { mkdirSync } from 'node:fs'; +import { dirname } from 'node:path'; +import Database from 'better-sqlite3'; +import { drizzle } from 'drizzle-orm/better-sqlite3'; +import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; +import * as schema from './schema'; + +/** + * Opens the gateway's database — the daemon's shape (WAL: readers never + * block the single writer, a crash mid-write cannot corrupt the file). + */ +export function openDb(path: string) { + if (path !== ':memory:') { + mkdirSync(dirname(path), { recursive: true }); + } + const sqlite = new Database(path); + sqlite.pragma('journal_mode = WAL'); + return drizzle(sqlite, { schema }); +} + +export type Db = ReturnType; + +/** Applies pending migrations (drizzle-kit output, committed) at every start. */ +export function migrateDb(db: Db, migrationsFolder: string) { + migrate(db, { migrationsFolder }); +} diff --git a/packages/gateway/src/db/schema.ts b/packages/gateway/src/db/schema.ts new file mode 100644 index 00000000..227d9d5b --- /dev/null +++ b/packages/gateway/src/db/schema.ts @@ -0,0 +1,32 @@ +import { sqliteTable, text } from 'drizzle-orm/sqlite-core'; + +/** + * The gateway's tables are "how the fleet is configured and who may + * enter" — never a sandbox's state, which lives in the ledger of the node + * that runs it and is asked for when needed (find.ts). One table today; + * api_keys, settings, templates and console_account arrive here with the + * configuration authority. + */ + +/** + * Every node that has ever checked in (routes/nodes.ts): its id, the + * address the gateway forwards to, and when it first appeared. Written by + * the nodes themselves at their first check-in — there is no registration + * verb and no nodes file, so "which nodes exist" has exactly one home — + * and deleted only by an operator's removeNode. Persistent, not memory, + * for one reason: a node that is down must still be known after a gateway + * restart, or a name that lives only there would be placed anew elsewhere + * and come back as a conflict when the node returns. Everything the node + * last reported (its reading, build, check-in time) is memory: fifteen + * seconds later it is reported again. + */ +export const nodes = sqliteTable('nodes', { + /** DORMICE_NODE_ID as the node states it — the `nodeId` in every sandbox answer. */ + id: text('id').primaryKey(), + /** Where the gateway forwards to; updated when a check-in states a new one. */ + endpoint: text('endpoint').notNull(), + /** ISO 8601 UTC — the first check-in. */ + addedAt: text('added_at').notNull(), +}); + +export type NodeRow = typeof nodes.$inferSelect; diff --git a/packages/gateway/src/errors.ts b/packages/gateway/src/errors.ts new file mode 100644 index 00000000..b310db13 --- /dev/null +++ b/packages/gateway/src/errors.ts @@ -0,0 +1,117 @@ +import type http from 'node:http'; +import { UnreachableError } from './forward'; + +/** What relay needs of a logger — pino's and Fastify's request logger both fit. */ +export interface ErrorLog { + error(obj: unknown, msg?: string): void; +} + +/** + * Every face the gateway fronts has its own error dialect, and an error + * the gateway itself produces must wear the dialect of the face it was + * asked on — a client library parses what it expects: + * native { message } the daemon's API + * control { code: , message } E2B's control plane (openapi-fetch checks error.code === 404) + * connect { code: '', message } E2B's envd (Connect RPC's string codes) + * Errors a node produced are forwarded verbatim and never re-dressed. + */ +export type Dialect = 'native' | 'control' | 'connect'; + +export interface RenderedError { + status: number; + message: string; + /** Connect's string code, used by the connect dialect only. */ + connectCode?: string; + /** Browser-consumable faces carry CORS on errors too, or the refusal is unreadable. */ + cors?: boolean; + /** Seconds — on a 503 whose cause the caller may simply outwait. */ + retryAfterSeconds?: number; +} + +export function errorBody(dialect: Dialect, error: RenderedError): string { + switch (dialect) { + case 'native': + return JSON.stringify({ message: error.message }); + case 'control': + return JSON.stringify({ code: error.status, message: error.message }); + case 'connect': + return JSON.stringify({ + code: error.connectCode ?? 'unknown', + message: error.message, + }); + } +} + +/** Writes an error the gateway produced, on a raw response, in the face's dialect. */ +export function renderError( + res: http.ServerResponse, + dialect: Dialect, + error: RenderedError, +): void { + if (res.headersSent || res.destroyed) { + res.destroy(); + return; + } + const body = errorBody(dialect, error); + const headers: http.OutgoingHttpHeaders = { + 'content-type': 'application/json', + 'content-length': String(Buffer.byteLength(body)), + }; + if (error.cors) headers['access-control-allow-origin'] = '*'; + if (error.retryAfterSeconds !== undefined) { + headers['retry-after'] = String(error.retryAfterSeconds); + } + res.writeHead(error.status, headers); + res.end(body); +} + +/** The preflight answer of the browser-consumable faces — the daemon's cors.ts shape. */ +export function sendPreflight( + req: http.IncomingMessage, + res: http.ServerResponse, +): void { + const requested = req.headers['access-control-request-headers']; + res.writeHead(204, { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET, POST, OPTIONS', + 'access-control-allow-headers': + typeof requested === 'string' && requested !== '' ? requested : '*', + 'access-control-max-age': '7200', + }); + res.end(); +} + +/** + * Runs one forwarding step on a response the gateway owns (a hijacked + * reply, a raw face) and turns its failure into an answer in the face's + * dialect: the node's transport failure is a 502 rendered by `unreachable` + * (what the caller may do next), anything else a 500 that sends the + * operator to the log. Never silence: Fastify writes nothing for a + * hijacked reply whose handler throws, the gateway has no request timeout + * of its own, and a raw face has no framework behind it at all — so a + * step that threw after the node had answered would leave the caller + * waiting forever with the sandbox already built (found by review, + * 2026-09-12). + */ +export async function relay( + res: http.ServerResponse, + dialect: Dialect, + log: ErrorLog, + step: () => Promise, + unreachable: (error: UnreachableError) => RenderedError, +): Promise { + try { + await step(); + } catch (error) { + if (error instanceof UnreachableError) { + renderError(res, dialect, unreachable(error)); + return; + } + log.error(error, 'forwarding failed on the gateway'); + renderError(res, dialect, { + status: 500, + message: 'the gateway failed while forwarding — see its log', + cors: dialect === 'connect', + }); + } +} diff --git a/packages/gateway/src/find.test.ts b/packages/gateway/src/find.test.ts new file mode 100644 index 00000000..f09805af --- /dev/null +++ b/packages/gateway/src/find.test.ts @@ -0,0 +1,252 @@ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { fileURLToPath } from 'node:url'; +import { afterEach, describe, expect, it } from 'vitest'; +import { NameCache } from './cache'; +import { migrateDb, openDb } from './db/db'; +import { Finder } from './find'; +import { Fleet } from './fleet'; +import { type AskNode, httpAskNode, type LookupAnswer } from './lookup'; +import { checkInOf } from './testing'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const TOKEN = 'shared-token-shared-token-shared-token'; +const NOW = new Date('2026-09-14T12:00:00.000Z'); + +function fleetOf(...ids: string[]) { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const fleet = new Fleet(db); + for (const id of ids) fleet.checkIn(checkInOf(id, `http://${id}:80`), NOW); + return fleet; +} + +const silentLog = { warn: () => {} }; + +/** An asker scripted per node: what each node says to any question, and how often it was asked. */ +function scripted(script: Record) { + const asked: string[] = []; + const ask: AskNode = async (node) => { + asked.push(node.id); + return script[node.id] ?? { kind: 'absent' }; + }; + return { ask, asked }; +} + +describe('Finder', () => { + it('asks every node in parallel; exactly one yes wins and is cached by name and id', async () => { + const fleet = fleetOf('a', 'b', 'c'); + const { ask, asked } = scripted({ + b: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + }); + const cache = new NameCache(); + const finder = new Finder(fleet, cache, ask, silentLog); + const found = await finder.byName('alice'); + expect(found).toMatchObject({ kind: 'one', id: 'sb-1', name: 'alice' }); + expect(found.kind === 'one' && found.node.id).toBe('b'); + expect(asked.sort()).toEqual(['a', 'b', 'c']); + // The cache answers the next questions, by either handle, without a + // single node asked. + asked.length = 0; + expect((await finder.byName('alice')).kind).toBe('one'); + expect((await finder.byId('sb-1')).kind).toBe('one'); + expect(asked).toEqual([]); + }); + + it('one yes wins even while another node is silent: the sandbox is where it says it is', async () => { + const fleet = fleetOf('a', 'b'); + const { ask } = scripted({ + a: { kind: 'silent', why: 'ECONNREFUSED' }, + b: { kind: 'found', id: 'sb-1', name: 'alice', state: 'frozen' }, + }); + const found = await new Finder( + fleet, + new NameCache(), + ask, + silentLog, + ).byName('alice'); + expect(found.kind === 'one' && found.node.id).toBe('b'); + }); + + it('two yeses are a conflict naming both nodes, and nothing is cached', async () => { + const fleet = fleetOf('a', 'b', 'c'); + const { ask } = scripted({ + c: { kind: 'found', id: 'sb-2', name: 'alice', state: 'active' }, + a: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + }); + const cache = new NameCache(); + const found = await new Finder(fleet, cache, ask, silentLog).byName( + 'alice', + ); + expect(found).toEqual({ kind: 'conflict', nodeIds: ['a', 'c'] }); + expect(cache.size).toBe(0); + }); + + it('every node says no: the name is new; a silent node among the noes: unsure, naming it', async () => { + const fleet = fleetOf('a', 'b'); + expect( + await new Finder( + fleet, + new NameCache(), + scripted({}).ask, + silentLog, + ).byName('nobody'), + ).toEqual({ kind: 'none' }); + const warned: unknown[] = []; + const unsure = await new Finder( + fleet, + new NameCache(), + scripted({ b: { kind: 'silent', why: 'timeout' } }).ask, + { warn: (obj) => warned.push(obj) }, + ).byName('nobody'); + expect(unsure).toEqual({ + kind: 'unsure', + silent: [{ nodeId: 'b', why: 'timeout' }], + }); + expect(warned).toHaveLength(1); + }); + + it('an empty fleet finds nothing and asks nobody', async () => { + const { ask, asked } = scripted({}); + expect( + await new Finder(fleetOf(), new NameCache(), ask, silentLog).byName('x'), + ).toEqual({ kind: 'none' }); + expect(asked).toEqual([]); + }); + + it('a cached entry whose node was removed is dropped and the fleet asked afresh', async () => { + const fleet = fleetOf('a', 'b'); + const { ask, asked } = scripted({ + a: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + }); + const cache = new NameCache(); + const finder = new Finder(fleet, cache, ask, silentLog); + await finder.byName('alice'); + fleet.remove('a'); + asked.length = 0; + expect(await finder.byName('alice')).toEqual({ kind: 'none' }); + expect(asked).toEqual(['b']); + expect(cache.size).toBe(0); + }); + + it('verify: a node that says absent loses the entry; a silent node keeps it; a removed node loses it', async () => { + const fleet = fleetOf('a', 'b'); + const script: Record = { + a: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + }; + const cache = new NameCache(); + const finder = new Finder( + fleet, + cache, + async (node) => script[node.id] ?? { kind: 'absent' }, + silentLog, + ); + await finder.byName('alice'); + const entry = { id: 'sb-1', name: 'alice', nodeId: 'a' }; + script.a = { kind: 'silent', why: 'timeout' }; + await finder.verify(entry); + expect(cache.getByName('alice')).toEqual(entry); + script.a = { kind: 'absent' }; + await finder.verify(entry); + expect(cache.getByName('alice')).toBeUndefined(); + expect(cache.getById('sb-1')).toBeUndefined(); + + cache.put(entry); + fleet.remove('a'); + await finder.verify(entry); + expect(cache.size).toBe(0); + }); +}); + +describe('NameCache', () => { + it('a name that moved to a new id drops the old id; an id that gained a name is reachable by both', () => { + const cache = new NameCache(); + cache.put({ id: 'old', name: 'alice', nodeId: 'a' }); + cache.put({ id: 'new', name: 'alice', nodeId: 'b' }); + expect(cache.getById('old')).toBeUndefined(); + expect(cache.getByName('alice')?.nodeId).toBe('b'); + cache.put({ id: 'anon', name: null, nodeId: 'a' }); + cache.put({ id: 'anon', name: 'e2b-1', nodeId: 'a' }); + expect(cache.getByName('e2b-1')?.id).toBe('anon'); + expect(cache.size).toBe(2); + expect(cache.evictNode('a')).toBe(1); + expect(cache.getById('anon')).toBeUndefined(); + expect(cache.size).toBe(1); + }); +}); + +describe('httpAskNode', () => { + const servers: http.Server[] = []; + afterEach(async () => { + await Promise.all( + servers + .splice(0) + .map((s) => new Promise((resolve) => s.close(() => resolve()))), + ); + }); + + async function node( + handler: ( + body: unknown, + headers: http.IncomingHttpHeaders, + ) => { status: number; body: string }, + ) { + const server = http.createServer((req, res) => { + let text = ''; + req.on('data', (c) => { + text += c; + }); + req.on('end', () => { + const answer = handler(JSON.parse(text), req.headers); + res.writeHead(answer.status, { 'content-type': 'application/json' }); + res.end(answer.body); + }); + }); + servers.push(server); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + return { + id: 'n', + endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + }; + } + + it('posts the question under the fleet token and reads found / absent; a non-200 or a refused connection is silent with the reason', async () => { + const seen: unknown[] = []; + const yes = await node((body, headers) => { + seen.push({ body, auth: headers.authorization }); + return { + status: 200, + body: JSON.stringify({ + found: true, + sandbox: { id: 'sb-1', name: 'alice', state: 'active' }, + }), + }; + }); + const ask = httpAskNode(TOKEN); + expect(await ask(yes, { name: 'alice' })).toEqual({ + kind: 'found', + id: 'sb-1', + name: 'alice', + state: 'active', + }); + expect(seen).toEqual([ + { body: { name: 'alice' }, auth: `Bearer ${TOKEN}` }, + ]); + const no = await node(() => ({ + status: 200, + body: JSON.stringify({ found: false }), + })); + expect(await ask(no, { id: 'x' })).toEqual({ kind: 'absent' }); + const refusing = await node(() => ({ + status: 401, + body: '{"message":"missing or invalid API token"}', + })); + expect(await ask(refusing, { id: 'x' })).toMatchObject({ + kind: 'silent', + why: expect.stringMatching(/answered 401/), + }); + expect( + await ask({ id: 'gone', endpoint: 'http://127.0.0.1:9' }, { id: 'x' }), + ).toMatchObject({ kind: 'silent', why: 'ECONNREFUSED' }); + }); +}); diff --git a/packages/gateway/src/find.ts b/packages/gateway/src/find.ts new file mode 100644 index 00000000..d6f2ef99 --- /dev/null +++ b/packages/gateway/src/find.ts @@ -0,0 +1,122 @@ +import type { CacheEntry, NameCache } from './cache'; +import type { Fleet, NodeState } from './fleet'; +import type { AskNode, LookupQuery } from './lookup'; + +/** + * Where a sandbox is, adjudicated once for every face: + * one exactly one node holds it — route there + * conflict several nodes hold it — refused, never guessed about (an + * operator destroys one copy directly on its node) + * none every node answered, none holds it — the name is new + * unsure no node holds it as far as anyone answered, but a node did + * not answer, so "new" cannot be proven: a name that lives + * only on a silent node must not be built a second time + * elsewhere (design record #28: a 503 with Retry-After, the + * retry is the application's) + */ +export type Found = + | { kind: 'one'; node: NodeState; id: string; name: string | null } + | { kind: 'conflict'; nodeIds: string[] } + | { kind: 'none' } + | { kind: 'unsure'; silent: Array<{ nodeId: string; why: string }> }; + +export interface FinderLog { + warn(obj: unknown, msg: string): void; +} + +/** + * Finds a sandbox by asking. The cache answers first; on a miss every + * node in the fleet is asked in parallel (lookup.ts), and exactly one + * "yes" wins — the sandbox is wherever it says it is, whether or not some + * other node was slow to say no. Down nodes are asked like any other: + * the reading a node last reported says where NOT to place, never where + * a sandbox is; a node that has missed its check-ins but still answers a + * lookup still holds its sandboxes. + */ +export class Finder { + constructor( + private readonly fleet: Fleet, + readonly cache: NameCache, + private readonly ask: AskNode, + private readonly log: FinderLog, + ) {} + + byName(name: string): Promise { + return this.find(this.cache.getByName(name), { name }); + } + + byId(id: string): Promise { + return this.find(this.cache.getById(id), { id }); + } + + private async find( + cached: CacheEntry | undefined, + query: LookupQuery, + ): Promise { + if (cached !== undefined) { + const node = this.fleet.get(cached.nodeId); + // A node the operator removed while the entry was cached: the + // entry is stale by definition, and the fleet is asked afresh. + if (node !== undefined) { + return { kind: 'one', node, id: cached.id, name: cached.name }; + } + this.cache.evict(cached); + } + const members = this.fleet.all(); + const answers = await Promise.all( + members.map(async (node) => ({ + node, + answer: await this.ask(node, query), + })), + ); + const found = answers.flatMap(({ node, answer }) => + answer.kind === 'found' ? [{ node, answer }] : [], + ); + const first = found[0]; + if (found.length === 1 && first !== undefined) { + const entry = { + id: first.answer.id, + name: first.answer.name, + nodeId: first.node.id, + }; + this.cache.put(entry); + return { kind: 'one', node: first.node, id: entry.id, name: entry.name }; + } + if (found.length > 1) { + return { + kind: 'conflict', + nodeIds: found.map((f) => f.node.id).sort(), + }; + } + const silent = answers.flatMap(({ node, answer }) => + answer.kind === 'silent' ? [{ nodeId: node.id, why: answer.why }] : [], + ); + if (silent.length > 0) { + this.log.warn( + { query, silent }, + 'lookup: a node did not answer; a name it may hold cannot be treated as new', + ); + return { kind: 'unsure', silent }; + } + return { kind: 'none' }; + } + + /** + * After a cached node answered 404 for a name: is the sandbox really + * gone from it? A node's 404 is not "no such sandbox" by itself — + * readFile answers 404 for a missing path too — so the node is asked + * the one question that means exactly that, and only a plain "absent" + * evicts. Silence keeps the entry: the node may be down, and its + * sandboxes are still there. Off the request path (the 404 has already + * been relayed); the next request for the name asks the fleet afresh. + */ + async verify(entry: CacheEntry): Promise { + const node = this.fleet.get(entry.nodeId); + if (node === undefined) { + this.cache.evict(entry); + return; + } + const answer = await this.ask(node, { id: entry.id }); + if (answer.kind === 'absent') this.cache.evict(entry); + } +} diff --git a/packages/gateway/src/fleet.test.ts b/packages/gateway/src/fleet.test.ts new file mode 100644 index 00000000..18e4b523 --- /dev/null +++ b/packages/gateway/src/fleet.test.ts @@ -0,0 +1,94 @@ +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { migrateDb, openDb } from './db/db'; +import { downReason, Fleet } from './fleet'; +import { checkInOf } from './testing'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const NOW = new Date('2026-09-14T12:00:00.000Z'); + +function db() { + const handle = openDb(':memory:'); + migrateDb(handle, MIGRATIONS); + return handle; +} + +describe('Fleet', () => { + it('a first check-in joins the node and persists its row; a gateway restart still knows it, unreached until it checks in again', () => { + const handle = db(); + const fleet = new Fleet(handle); + expect(fleet.all()).toEqual([]); + const { node, joined } = fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), + NOW, + ); + expect(joined).toBe(true); + expect(node.addedAt).toBe(NOW.toISOString()); + expect(node.lastCheckInAt).toBe(NOW); + expect(node.reading?.host.cpuCount).toBe(8); + expect(downReason(node, NOW)).toBeNull(); + + // The same database after a restart: the row is there, the memory is not. + const restarted = new Fleet(handle); + const known = restarted.get('node-b'); + expect(known?.endpoint).toBe('http://10.0.0.7:80'); + expect(known?.addedAt).toBe(NOW.toISOString()); + expect(known?.reading).toBeNull(); + expect(downReason(known as NonNullable, NOW)).toBe( + 'has not checked in since the gateway started', + ); + }); + + it('a later check-in is not a join; a changed endpoint is written through; the placement counter restarts', () => { + const handle = db(); + const fleet = new Fleet(handle); + const first = fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW); + first.node.placedSinceCheckIn = 3; + const second = fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.8:80', { active: 12 }), + new Date(NOW.getTime() + 15_000), + ); + expect(second.joined).toBe(false); + expect(second.node).toBe(first.node); + expect(second.node.endpoint).toBe('http://10.0.0.8:80'); + expect(second.node.reading?.sandboxes.byState.active).toBe(12); + expect(second.node.placedSinceCheckIn).toBe(0); + expect(new Fleet(handle).get('node-b')?.endpoint).toBe( + 'http://10.0.0.8:80', + ); + }); + + it('downReason: fresh within two of its own intervals, down past them', () => { + const fleet = new Fleet(db()); + const { node } = fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), + NOW, + ); + expect(downReason(node, new Date(NOW.getTime() + 29_000))).toBeNull(); + expect(downReason(node, new Date(NOW.getTime() + 31_000))).toBe( + 'has not checked in for 31s', + ); + // A one-second node (the exam's) is judged by its own interval. + const quick = fleet.checkIn( + checkInOf('node-c', 'http://10.0.0.9:80', { intervalSeconds: 1 }), + NOW, + ).node; + expect(downReason(quick, new Date(NOW.getTime() + 1_500))).toBeNull(); + expect(downReason(quick, new Date(NOW.getTime() + 2_500))).toBe( + 'has not checked in for 3s', + ); + }); + + it('remove forgets the node and its row; removing an unknown id is false; a node that checks in again re-joins', () => { + const handle = db(); + const fleet = new Fleet(handle); + fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW); + expect(fleet.remove('node-b')).toBe(true); + expect(fleet.all()).toEqual([]); + expect(new Fleet(handle).all()).toEqual([]); + expect(fleet.remove('node-b')).toBe(false); + expect( + fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW).joined, + ).toBe(true); + }); +}); diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts new file mode 100644 index 00000000..42967ada --- /dev/null +++ b/packages/gateway/src/fleet.ts @@ -0,0 +1,132 @@ +import type { BuildInfo, CheckInRequest, NodeReading } from '@dormice/shared'; +import { eq } from 'drizzle-orm'; +import type { Db } from './db/db'; +import { nodes } from './db/schema'; + +/** + * A node as the gateway knows it: the persistent row (id, endpoint, + * addedAt) and what it last reported (memory — reported again at the next + * check-in, gone with the process and rightly so). placedSinceCheckIn + * counts the sandboxes the gateway sent here since the last reading, so a + * burst inside one interval is counted against the node before its next + * reading shows it (placement.ts). + */ +export interface NodeState { + readonly id: string; + endpoint: string; + readonly addedAt: string; + lastCheckInAt: Date | null; + intervalSeconds: number | null; + build: BuildInfo | null; + reading: NodeReading | null; + placedSinceCheckIn: number; +} + +/** + * Why a node is not to be placed on right now, or null when it is fine: + * never checked in since this gateway started, or silent for two of its + * own intervals — the interval it stated in its last check-in, so the + * exam's one-second nodes and production's fifteen-second nodes are judged + * by the same rule. Two, not one: a check-in delayed by a busy event loop + * or a slow reading is normal; two in a row missing is a node in trouble. + * The same word answers listNodes' `reachable`. + */ +export function downReason(node: NodeState, now: Date): string | null { + if (node.lastCheckInAt === null || node.intervalSeconds === null) { + return 'has not checked in since the gateway started'; + } + const silentMs = now.getTime() - node.lastCheckInAt.getTime(); + if (silentMs > 2 * node.intervalSeconds * 1000) { + return `has not checked in for ${Math.round(silentMs / 1000)}s`; + } + return null; +} + +/** + * The fleet: every node that has ever checked in. Rows come from the + * database at start (a node that is down must still be known — its names + * are not new names); everything else fills in as the nodes report. + */ +export class Fleet { + private readonly members = new Map(); + + constructor(private readonly db: Db) { + for (const row of db.select().from(nodes).all()) { + this.members.set(row.id, { + id: row.id, + endpoint: row.endpoint, + addedAt: row.addedAt, + lastCheckInAt: null, + intervalSeconds: null, + build: null, + reading: null, + placedSinceCheckIn: 0, + }); + } + } + + all(): NodeState[] { + return [...this.members.values()]; + } + + get(id: string): NodeState | undefined { + return this.members.get(id); + } + + /** + * A node reporting for duty. A first check-in adds the node (`joined` + * says so, for the log); a changed endpoint is written through — the + * node states where it lives, the gateway does not remember better. The + * placement counter restarts at zero: what was placed before this + * reading is in it now, and what is still in flight on the node (its + * row is written after the container is up) is in neither figure until + * the next reading — one interval of slack, self-correcting, the same + * as before. + */ + checkIn( + report: CheckInRequest, + now = new Date(), + ): { node: NodeState; joined: boolean } { + let node = this.members.get(report.nodeId); + let joined = false; + if (node === undefined) { + const addedAt = now.toISOString(); + this.db + .insert(nodes) + .values({ id: report.nodeId, endpoint: report.endpoint, addedAt }) + .run(); + node = { + id: report.nodeId, + endpoint: report.endpoint, + addedAt, + lastCheckInAt: null, + intervalSeconds: null, + build: null, + reading: null, + placedSinceCheckIn: 0, + }; + this.members.set(node.id, node); + joined = true; + } else if (node.endpoint !== report.endpoint) { + this.db + .update(nodes) + .set({ endpoint: report.endpoint }) + .where(eq(nodes.id, report.nodeId)) + .run(); + node.endpoint = report.endpoint; + } + node.lastCheckInAt = now; + node.intervalSeconds = report.intervalSeconds; + node.build = report.build; + node.reading = report.reading; + node.placedSinceCheckIn = 0; + return { node, joined }; + } + + /** The operator's word that the node is gone for good; a node still running re-adds itself at its next check-in. */ + remove(id: string): boolean { + const existed = this.members.delete(id); + this.db.delete(nodes).where(eq(nodes.id, id)).run(); + return existed; + } +} diff --git a/packages/gateway/src/forward.test.ts b/packages/gateway/src/forward.test.ts new file mode 100644 index 00000000..48457edd --- /dev/null +++ b/packages/gateway/src/forward.test.ts @@ -0,0 +1,734 @@ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import net from 'node:net'; +import { request } from 'undici'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + forwardCapture, + forwardStream, + forwardUpgrade, + replay, + UnreachableError, +} from './forward'; + +const TOKEN = 'node-token-node-token-node-token-node'; +const servers: http.Server[] = []; +// Every accepted socket, upgraded ones included: http.Server#close waits +// for sockets it no longer tracks after an upgrade (the daemon's +// shutdown.ts has the reference), so teardown cuts them by hand. +const sockets = new Set(); +afterEach(async () => { + for (const socket of sockets) socket.destroy(); + sockets.clear(); + await Promise.all( + servers + .splice(0) + .map((s) => new Promise((resolve) => s.close(() => resolve()))), + ); +}); + +async function listen(server: http.Server): Promise { + servers.push(server); + server.on('connection', (socket) => { + sockets.add(socket); + socket.once('close', () => sockets.delete(socket)); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +/** A gateway-shaped front: every request is forwarded to `endpoint` with the given options. */ +async function front( + endpoint: string, + options: Partial[2]> = {}, +): Promise { + const server = http.createServer((req, res) => { + void forwardStream(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + ...options, + }).catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + res.writeHead(502, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ message })); + }); + }); + return listen(server); +} + +describe('forwardStream', () => { + it('streams the answer frame by frame: the first frame reaches the client before the node writes the second', async () => { + let releaseSecond: () => void = () => {}; + const gate = new Promise((resolve) => { + releaseSecond = resolve; + }); + const node = http.createServer((_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain' }); + res.write('first\n'); + void gate.then(() => res.end('second\n')); + }); + const router = await front(await listen(node)); + + const res = await request(`${router}/execCommand`, { method: 'POST' }); + const chunks: string[] = []; + const reader = res.body[Symbol.asyncIterator](); + const first = await reader.next(); + chunks.push(String(first.value)); + // Proof of streaming: the client holds the first frame while the node + // is still parked before its second. A buffering front could not get here. + expect(chunks).toEqual(['first\n']); + releaseSecond(); + for await (const chunk of { [Symbol.asyncIterator]: () => reader }) { + chunks.push(String(chunk)); + } + expect(chunks.join('')).toBe('first\nsecond\n'); + expect(res.headers['transfer-encoding']).toBe('chunked'); + }); + + it("resolves with the node's status once the answer is relayed — the named verbs read a 404 off it", async () => { + const node = http.createServer((_req, res) => { + res.writeHead(404, { 'content-type': 'application/json' }); + res.end('{"message":"no sandbox named \\"x\\""}'); + }); + const endpoint = await listen(node); + let resolved: number | null | undefined; + const front = http.createServer((req, res) => { + void forwardStream(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + }).then((status) => { + resolved = status; + }); + }); + const url = await listen(front); + const res = await request(`${url}/execCommand`, { method: 'POST' }); + expect(res.statusCode).toBe(404); + await res.body.text(); + const deadline = Date.now() + 1_000; + while (resolved === undefined && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(resolved).toBe(404); + }); + + it('keeps a fixed-length answer fixed-length and passes response headers through — pagination, cookies, CORS', async () => { + const node = http.createServer((_req, res) => { + res.writeHead(200, { + 'content-type': 'application/json', + 'content-length': '5', + 'x-next-token': 'abc', + 'set-cookie': ['a=1', 'b=2'], + 'access-control-allow-origin': '*', + }); + res.end('hello'); + }); + const router = await front(await listen(node)); + const res = await request(`${router}/listSandboxes`, { method: 'POST' }); + expect(res.statusCode).toBe(200); + expect(res.headers['content-length']).toBe('5'); + expect(res.headers['transfer-encoding']).toBeUndefined(); + expect(res.headers['x-next-token']).toBe('abc'); + expect(res.headers['set-cookie']).toEqual(['a=1', 'b=2']); + expect(res.headers['access-control-allow-origin']).toBe('*'); + expect(await res.body.text()).toBe('hello'); + }); + + it('a client that goes away mid-stream takes the node connection down with it', async () => { + let nodeSawClose: () => void = () => {}; + const closed = new Promise((resolve) => { + nodeSawClose = resolve; + }); + const node = http.createServer((req, res) => { + res.writeHead(200); + res.write('tick\n'); + const timer = setInterval(() => res.write('tick\n'), 20); + req.on('close', () => { + clearInterval(timer); + nodeSawClose(); + }); + }); + const router = await front(await listen(node)); + const res = await request(`${router}/stream`, { method: 'POST' }); + const reader = res.body[Symbol.asyncIterator](); + await reader.next(); + res.body.destroy(); + await closed; + }); + + it('forwards the request verbatim — path, query, body — with the credential swapped and the Host renamed to the node unless the face keeps it', async () => { + const seen: Array<{ + url?: string; + headers: http.IncomingHttpHeaders; + body: string; + }> = []; + const node = http.createServer((req, res) => { + let body = ''; + req.on('data', (c) => { + body += c; + }); + req.on('end', () => { + seen.push({ url: req.url, headers: req.headers, body }); + res.writeHead(200); + res.end(); + }); + }); + const endpoint = await listen(node); + const router = await front(endpoint, { credential: 'bearer' }); + await request(`${router}/readFile?x=1`, { + method: 'POST', + headers: { + host: '8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test', + authorization: 'Bearer caller-secret', + 'content-type': 'application/json', + connection: 'keep-alive', + }, + body: '{"name":"alice"}', + }); + expect(seen).toHaveLength(1); + expect(seen[0]?.url).toBe('/readFile?x=1'); + expect(seen[0]?.body).toBe('{"name":"alice"}'); + // The Host named the gateway; the node is asked as itself, so a Caddy + // in front of it that binds the gateway's domain never answers a 308. + expect(seen[0]?.headers.host).toBe(new URL(endpoint).host); + expect(seen[0]?.headers.authorization).toBe(`Bearer ${TOKEN}`); + expect(seen[0]?.headers['content-type']).toBe('application/json'); + + // The E2B face swaps the other header; the credential-less faces touch none. + const asE2b = await front(endpoint, { credential: 'x-api-key' }); + await request(`${asE2b}/sandboxes`, { + method: 'POST', + headers: { 'x-api-key': 'e2b_caller' }, + }); + expect(seen[1]?.headers['x-api-key']).toBe(`e2b_${TOKEN}`); + const asEnvd = await front(endpoint, { credential: 'none' }); + await request(`${asEnvd}/e2b/envd/files`, { + method: 'GET', + headers: { authorization: 'Basic dXNlcjo=', 'x-access-token': 'hmac' }, + }); + expect(seen[2]?.headers.authorization).toBe('Basic dXNlcjo='); + expect(seen[2]?.headers['x-access-token']).toBe('hmac'); + + // The proxy face is the one that keeps the Host: it is the routing + // key on the node as well. + const asProxy = await front(endpoint, { + credential: 'none', + preserveHost: true, + }); + await request(`${asProxy}/`, { + method: 'GET', + headers: { host: '8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test' }, + }); + expect(seen[3]?.headers.host).toBe( + '8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test', + ); + }); + + it('sends the request target byte for byte — dot segments included — instead of a URL it resolved itself', async () => { + const seen: string[] = []; + const node = http.createServer((req, res) => { + seen.push(req.url ?? ''); + res.writeHead(200); + res.end(); + }); + const endpoint = await listen(node); + const router = await front(endpoint); + const url = new URL(router); + // A client library would collapse the segments before they left; the + // socket carries them as an attacker would. + const path = '/e2b/api/sandboxes/some-id/../../sandboxes?x=1'; + await new Promise((resolve, reject) => { + const socket = net.connect(Number(url.port), url.hostname, () => { + socket.write( + `GET ${path} HTTP/1.1\r\nhost: router\r\nconnection: close\r\n\r\n`, + ); + }); + socket.on('data', () => {}); + socket.on('end', resolve); + socket.on('error', reject); + }); + expect(seen).toEqual([path]); + }); + + it('drops the Expect header: the 100-continue handshake ended at the gateway, and undici refuses to carry it', async () => { + const seen: http.IncomingHttpHeaders[] = []; + const node = http.createServer((req, res) => { + req.resume(); + req.on('end', () => { + seen.push(req.headers); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end('{"ok":true}'); + }); + }); + const endpoint = await listen(node); + const router = await front(endpoint); + // curl adds this to every body over 1024 bytes (undici's own client + // refuses to send it, so the exam speaks node:http): Node's server + // answered the 100 before forwardStream saw the request. + const body = JSON.stringify({ name: 'big', content: 'x'.repeat(2048) }); + const res = await new Promise<{ status: number; text: string }>( + (resolve, reject) => { + const req = http.request( + `${router}/writeFile`, + { + method: 'POST', + headers: { + expect: '100-continue', + 'content-type': 'application/json', + 'content-length': String(Buffer.byteLength(body)), + }, + }, + (answer) => { + let text = ''; + answer.on('data', (c) => { + text += c; + }); + answer.on('end', () => + resolve({ status: answer.statusCode ?? 0, text }), + ); + }, + ); + req.on('continue', () => req.end(body)); + req.on('error', reject); + }, + ); + expect(res.status).toBe(200); + expect(JSON.parse(res.text)).toEqual({ ok: true }); + expect(seen[0]?.expect).toBeUndefined(); + }); + + it('a node that does not answer is an UnreachableError, tried exactly once', async () => { + let hits = 0; + const node = http.createServer((req) => { + hits += 1; + req.socket.destroy(); + }); + const endpoint = await listen(node); + const router = await front(endpoint); + const cut = await request(`${router}/acquireSandbox`, { method: 'POST' }); + expect(cut.statusCode).toBe(502); + expect((await cut.body.json()) as { message: string }).toMatchObject({ + message: expect.stringMatching( + /^node http:\/\/127\.0\.0\.1:\d+ did not answer: /, + ), + }); + expect(hits).toBe(1); + + const refused = await front('http://127.0.0.1:9'); + const res = await request(`${refused}/acquireSandbox`, { method: 'POST' }); + expect(res.statusCode).toBe(502); + expect(((await res.body.json()) as { message: string }).message).toMatch( + /did not answer: ECONNREFUSED/, + ); + }); +}); + +describe('forwardCapture + replay', () => { + it('collects the answer whole and replays it verbatim, asks for it uncompressed, and reports a refused connection as unreachable', async () => { + const encodings: Array = []; + const node = http.createServer((req, res) => { + encodings.push(req.headers['accept-encoding']); + res.writeHead(201, { + 'content-type': 'application/json', + 'x-extra': 'y', + }); + res.end('{"sandboxID":"s-1"}'); + }); + const endpoint = await listen(node); + const server = http.createServer((req, res) => { + void forwardCapture(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'x-api-key', + }).then( + (answer) => { + if (answer === null) throw new Error('the client did not leave'); + expect(answer.status).toBe(201); + expect(JSON.parse(answer.body.toString())).toEqual({ + sandboxID: 's-1', + }); + replay(res, answer); + }, + (error: unknown) => { + res.writeHead(502); + res.end(error instanceof Error ? error.name : 'other'); + }, + ); + }); + const router = await listen(server); + const res = await request(`${router}/sandboxes`, { + method: 'POST', + headers: { 'accept-encoding': 'gzip, br' }, + }); + expect(res.statusCode).toBe(201); + expect(res.headers['x-extra']).toBe('y'); + expect(res.headers['content-length']).toBe('19'); + expect(await res.body.json()).toEqual({ sandboxID: 's-1' }); + // A compressing hop between router and node would hand back bytes + // the gateway cannot read the id out of. + expect(encodings).toEqual(['identity']); + + const orphan = new http.IncomingMessage(new net.Socket()); + await expect( + forwardCapture( + Object.assign(orphan, { url: '/x', method: 'POST', headers: {} }), + new http.ServerResponse(orphan), + { + target: { endpoint: 'http://127.0.0.1:9', token: TOKEN }, + credential: 'bearer', + body: Buffer.from('{}'), + }, + ), + ).rejects.toBeInstanceOf(UnreachableError); + }); +}); + +describe('forwardCapture + replay — bodiless answers', () => { + it('replays a 204 without inventing a content-length', async () => { + const node = http.createServer((_req, res) => { + res.writeHead(204); + res.end(); + }); + const endpoint = await listen(node); + const front = http.createServer((req, res) => { + void forwardCapture(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'x-api-key', + }).then((answer) => answer && replay(res, answer)); + }); + const url = await listen(front); + const res = await request(`${url}/e2b/api/sandboxes/x`, { + method: 'DELETE', + }); + expect(res.statusCode).toBe(204); + expect(res.headers['content-length']).toBeUndefined(); + await res.body.text(); + }); +}); + +describe('forwardStream — a client that leaves', () => { + it('withdraws the request from the node when the client leaves before the node answered', async () => { + let nodeSawClose: number | null = null; + const node = http.createServer((req, res) => { + const at = Date.now(); + req.on('close', () => { + nodeSawClose = Date.now() - at; + }); + // Answers late — the client will be gone by then. + setTimeout(() => { + if (!res.destroyed) { + res.writeHead(200); + res.end('late'); + } + }, 1500); + }); + const endpoint = await listen(node); + const routerUrl = await front(endpoint); + const url = new URL(routerUrl); + const client = http.request({ + host: url.hostname, + port: url.port, + path: '/execCommand', + method: 'POST', + headers: { 'content-type': 'application/json' }, + }); + client.on('error', () => {}); + client.end('{"name":"x"}'); + await new Promise((resolve) => setTimeout(resolve, 100)); + client.destroy(); + const deadline = Date.now() + 1_000; + while (nodeSawClose === null && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + // Well before the node's own answer at 1.5s: the gateway aborted it. + expect(nodeSawClose).not.toBeNull(); + expect(nodeSawClose as unknown as number).toBeLessThan(1_000); + }); +}); + +describe('forwardCapture — a client that leaves', () => { + it('withdraws the request from the node when the client leaves before the node answered, and answers null; a client already gone is not sent at all', async () => { + let nodeSawClose: number | null = null; + let requests = 0; + const node = http.createServer((req, res) => { + requests += 1; + const at = Date.now(); + req.on('close', () => { + nodeSawClose = Date.now() - at; + }); + setTimeout(() => { + if (!res.destroyed) { + res.writeHead(200); + res.end('{}'); + } + }, 1500); + }); + const endpoint = await listen(node); + const outcomes: Array<'null' | 'answer' | 'error'> = []; + const front = http.createServer((req, res) => { + void forwardCapture(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + }).then( + (answer) => { + outcomes.push(answer === null ? 'null' : 'answer'); + if (answer !== null) replay(res, answer); + }, + () => outcomes.push('error'), + ); + }); + const url = new URL(await listen(front)); + const client = http.request({ + host: url.hostname, + port: url.port, + path: '/acquireSandbox', + method: 'POST', + headers: { 'content-type': 'application/json' }, + }); + client.on('error', () => {}); + client.end('{"name":"x"}'); + await new Promise((resolve) => setTimeout(resolve, 100)); + client.destroy(); + const deadline = Date.now() + 1_000; + while (outcomes.length === 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + // Well before the node's own answer at 1.5s: the gateway aborted it + // and told the caller there is nothing to send. + expect(outcomes).toEqual(['null']); + expect(nodeSawClose).not.toBeNull(); + expect(nodeSawClose as unknown as number).toBeLessThan(1_000); + + // A verb whose client left while it waited for its slot: the response + // is already gone when the capture begins, and the node hears nothing. + const gone = new http.IncomingMessage(new net.Socket()); + const res = new http.ServerResponse(gone); + res.destroy(); + expect( + await forwardCapture( + Object.assign(gone, { url: '/x', method: 'POST', headers: {} }), + res, + { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + body: Buffer.from('{}'), + }, + ), + ).toBeNull(); + expect(requests).toBe(1); + }); +}); + +describe('forwardCapture — the head arrived, then the client left', () => { + it('reads the small body to the end and answers it, so a 2xx that did land is still learned; replay writes nothing to nobody', async () => { + let finishBody: () => void = () => {}; + let headOut: () => void = () => {}; + const headWritten = new Promise((resolve) => { + headOut = resolve; + }); + const node = http.createServer((_req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.write('{"sandbox":{"id":"sb-1"', headOut); + finishBody = () => res.end('}}'); + }); + const endpoint = await listen(node); + let answer: Awaited> | undefined; + let clientGone: () => void = () => {}; + const gone = new Promise((resolve) => { + clientGone = resolve; + }); + const front = http.createServer((req, res) => { + res.once('close', () => clientGone()); + void forwardCapture(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + }).then((a) => { + answer = a; + if (a !== null) replay(res, a); + }); + }); + const url = new URL(await listen(front)); + const client = http.request({ + host: url.hostname, + port: url.port, + path: '/acquireSandbox', + method: 'POST', + }); + client.on('error', () => {}); + client.end('{"name":"x"}'); + // The node's head is out; the body is not. The client gives up. + await headWritten; + client.destroy(); + await gone; + finishBody(); + const deadline = Date.now() + 2_000; + while (answer === undefined && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(answer).toBeDefined(); + expect(answer).not.toBeNull(); + expect(answer?.status).toBe(200); + expect(JSON.parse(answer?.body.toString('utf8') ?? '')).toEqual({ + sandbox: { id: 'sb-1' }, + }); + }); +}); + +describe('forwardUpgrade', () => { + it('a node behind TLS is refused with a 502 before any dial — upgrades are plain TCP here', async () => { + const router = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + router.on('upgrade', (req, socket, head) => + forwardUpgrade(req, socket, head, { + endpoint: 'https://node.example:443', + token: TOKEN, + }), + ); + const routerUrl = await listen(router); + const client = net.connect(Number(new URL(routerUrl).port), '127.0.0.1'); + const received: string[] = []; + client.on('data', (chunk) => received.push(String(chunk))); + const closed = new Promise((resolve) => client.on('close', resolve)); + client.write( + 'GET /ws HTTP/1.1\r\nHost: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + await closed; + expect(received.join('')).toContain('HTTP/1.1 502 Bad Gateway'); + expect(received.join('')).toContain('not forwarded over TLS'); + expect(received.join('')).not.toContain('node.example'); + }); + + it('a node that dies after the handshake cuts the client without writing an HTTP status into the upgraded stream', async () => { + const node = http.createServer((_req, res) => { + res.writeHead(426); + res.end(); + }); + node.on('upgrade', (req, socket) => { + socket.write( + `HTTP/1.1 101 Switching Protocols\r\nUpgrade: ${req.headers.upgrade}\r\nConnection: Upgrade\r\n\r\n`, + ); + // RST from the node side: the router's upstream sees an error, not a FIN. + socket.on('data', () => (socket as net.Socket).resetAndDestroy()); + }); + const endpoint = await listen(node); + const router = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + router.on('upgrade', (req, socket, head) => + forwardUpgrade(req, socket, head, { endpoint, token: TOKEN }), + ); + const routerUrl = await listen(router); + const client = net.connect(Number(new URL(routerUrl).port), '127.0.0.1'); + const received: string[] = []; + client.on('data', (chunk) => received.push(String(chunk))); + const closed = new Promise((resolve) => client.on('close', resolve)); + client.write( + 'GET /ws HTTP/1.1\r\nHost: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + await new Promise((resolve) => { + const check = () => { + if (received.join('').includes('101')) resolve(); + else client.once('data', check); + }; + check(); + }); + client.write('ping'); + await closed; + expect(received.join('')).not.toContain('502'); + }); + + it("a node that black-holes the dial is a 502 within the connect deadline, not the kernel's SYN budget", async () => { + const router = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + // TEST-NET-1 (RFC 5737) is never routed: the SYN is dropped, or the + // network answers unreachable at once — a 502 either way, and without + // the deadline the dropped case held this test for 75s on macOS. + router.on('upgrade', (req, socket, head) => + forwardUpgrade( + req, + socket, + head, + { endpoint: 'http://192.0.2.1:80', token: TOKEN }, + 500, + ), + ); + const routerUrl = await listen(router); + const client = net.connect(Number(new URL(routerUrl).port), '127.0.0.1'); + const received: string[] = []; + client.on('data', (chunk) => received.push(String(chunk))); + const closed = new Promise((resolve) => client.on('close', resolve)); + const started = Date.now(); + client.write( + 'GET /ws HTTP/1.1\r\nHost: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + await closed; + expect(received.join('')).toContain('HTTP/1.1 502 Bad Gateway'); + expect(received.join('')).toContain('did not answer the upgrade'); + expect(Date.now() - started).toBeLessThan(4000); + }); + + it('replays the upgrade handshake to the node and pipes both ways', async () => { + const node = http.createServer((_req, res) => { + res.writeHead(426); + res.end(); + }); + let nodeSideClosed: () => void = () => {}; + const nodeClosed = new Promise((resolve) => { + nodeSideClosed = resolve; + }); + node.on('upgrade', (req, socket) => { + socket.write( + `HTTP/1.1 101 Switching Protocols\r\nUpgrade: ${req.headers.upgrade}\r\nConnection: Upgrade\r\nX-Seen-Host: ${req.headers.host}\r\n\r\n`, + ); + socket.on('data', (chunk) => socket.write(`echo:${chunk}`)); + // A real upgraded peer (a WebSocket server in the sandbox) ends its + // side when the other side does; http sockets allow half-open, so + // without this the chain would hang on a peer that never answers FIN. + socket.on('end', () => socket.end()); + socket.on('close', nodeSideClosed); + }); + const endpoint = await listen(node); + const router = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + router.on('upgrade', (req, socket, head) => + forwardUpgrade(req, socket, head, { endpoint, token: TOKEN }), + ); + const routerUrl = await listen(router); + const port = Number(new URL(routerUrl).port); + + const client = net.connect(port, '127.0.0.1'); + const received: string[] = []; + const done = new Promise((resolve) => { + client.on('data', (chunk) => { + received.push(String(chunk)); + if (received.join('').includes('echo:ping')) resolve(); + }); + }); + client.write( + 'GET /ws HTTP/1.1\r\nHost: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + await new Promise((resolve) => { + const check = () => { + if (received.join('').includes('101')) resolve(); + else client.once('data', check); + }; + check(); + }); + client.write('ping'); + await done; + const all = received.join(''); + expect(all).toContain('HTTP/1.1 101 Switching Protocols'); + expect(all).toContain( + 'X-Seen-Host: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test', + ); + expect(all).toContain('echo:ping'); + // The client leaving takes the node-side socket down with it. + client.destroy(); + await nodeClosed; + }); +}); diff --git a/packages/gateway/src/forward.ts b/packages/gateway/src/forward.ts new file mode 100644 index 00000000..c025cfda --- /dev/null +++ b/packages/gateway/src/forward.ts @@ -0,0 +1,367 @@ +import type http from 'node:http'; +import net from 'node:net'; +import type { Duplex } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { Agent } from 'undici'; + +/** + * The one place the gateway talks to a node on a caller's behalf. Bytes + * in, bytes out: the request body is streamed (or handed over as the + * buffer Fastify already read), the response is written head-first and + * piped — never buffered, never re-parsed, never re-framed. Every + * long-lived shape a node produces (an exec that answers after an hour, + * an envd process stream, a proxied SSE) rides through untouched. + * + * The dispatcher switches undici's hidden clocks off, for the same reason + * the SDK's does (packages/sdk/src/client.ts): a node legitimately takes + * hours to answer an exec, and a default dispatcher gives up on headers + * after 300s. Connecting keeps a short deadline — a node that will not + * even accept the connection in ten seconds is unreachable now. + * + * Never retried, never re-routed. A create that did not answer may have + * created; sending it again elsewhere is how one name ends up on two + * nodes. The caller gets the honest 502, and its retry finds the sandbox + * by asking (find.ts) wherever it landed. + */ +const CONNECT_TIMEOUT_MS = 10_000; +const agent = new Agent({ + headersTimeout: 0, + bodyTimeout: 0, + connect: { timeout: CONNECT_TIMEOUT_MS }, +}); + +export interface ForwardTarget { + endpoint: string; + /** The fleet's one token: what the gateway presents to every node. */ + token: string; +} + +/** + * How the node is authenticated to: `bearer` replaces the Authorization + * header with the fleet token (native verbs), `x-api-key` replaces + * X-API-KEY with `e2b_` (E2B control plane), `none` touches no + * credential header at all — envd traffic carries its own (an access + * token minted by the node itself) and the gateway has nothing to add. + * No header names the real caller: the gateway is the fleet's one door + * and the node trusts it whole (design record #3). + */ +export type Credential = 'bearer' | 'x-api-key' | 'none'; + +export interface ForwardOptions { + target: ForwardTarget; + credential: Credential; + /** The request body when Fastify already consumed the stream; omit to stream req itself. */ + body?: Buffer | undefined; + /** + * Keep the caller's Host header. Only a face keyed on the Host wants + * this (the sandbox port proxy, once it routes through the gateway). + * Everywhere else the Host names the gateway, and carrying it to a node + * whose Caddy binds that very domain gets a 308 to https instead of the + * daemon — so the default lets undici name the node's own endpoint. + */ + preserveHost?: boolean; +} + +/** The node did not answer at all — no headers came back. */ +export class UnreachableError extends Error { + constructor( + readonly node: string, + /** The transport's word for why (ECONNREFUSED, a timeout, a cut body). */ + readonly why: string, + ) { + super(`node ${node} did not answer: ${why}`); + this.name = 'UnreachableError'; + } +} + +// Hop-by-hop headers describe one connection, not the message; a proxy +// that forwards them lies to the next hop about its framing. `expect` is +// one too: the 100-continue handshake is between the client and the +// first server, and Node's http answered it before this code ran (a +// server with no checkContinue listener sends 100 and reads the body). +// undici refuses to forward it outright (NotSupportedError), and curl +// adds it to every body over 1024 bytes — so without this line a 1.5MB +// writeFile through the front was a 502 (measured 2026-09-12). +const HOP_BY_HOP = new Set([ + 'connection', + 'expect', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade', +]); + +function outboundHeaders( + req: http.IncomingMessage, + options: ForwardOptions, +): Record { + const headers: Record = {}; + for (const [name, value] of Object.entries(req.headers)) { + if (value === undefined || HOP_BY_HOP.has(name)) continue; + // A buffered body is re-framed by undici; the client's length may + // describe a chunked encoding that no longer exists. + if (name === 'content-length' && options.body !== undefined) continue; + if (name === 'host' && options.preserveHost !== true) continue; + headers[name] = value; + } + switch (options.credential) { + case 'bearer': + headers.authorization = `Bearer ${options.target.token}`; + break; + case 'x-api-key': + headers['x-api-key'] = `e2b_${options.target.token}`; + break; + case 'none': + break; + } + return headers; +} + +function inboundHeaders( + headers: Record, +): http.OutgoingHttpHeaders { + const out: http.OutgoingHttpHeaders = {}; + for (const [name, value] of Object.entries(headers)) { + if (value === undefined || HOP_BY_HOP.has(name)) continue; + out[name] = value; + } + return out; +} + +function causeOf(error: unknown): string { + const e = error as { code?: string; message?: string; cause?: unknown }; + const cause = e.cause as { code?: string; message?: string } | undefined; + return cause?.code ?? cause?.message ?? e.code ?? e.message ?? String(error); +} + +/** + * The request target is sent to the node exactly as the caller wrote it — + * origin and path handed to undici separately, never joined into a URL. + * A URL parser resolves dot segments: `/e2b/api/sandboxes//../../ + * sandboxes` is a sub-route of a known id to Fastify, which routed it to + * the id's node, and `/e2b/api/sandboxes` to the parser — so a caller + * holding any live id could reach the node's create, under the fleet's + * credential, past placement and the name slot (reproduced 2026-09-12). + * The node judges the path it is sent; the gateway's routing and the + * node's must see the same bytes. + */ +async function dispatch( + req: http.IncomingMessage, + options: ForwardOptions, + override: Record = {}, + signal?: AbortSignal, +) { + try { + return await agent.request({ + origin: options.target.endpoint, + path: req.url ?? '/', + method: req.method as 'GET', + headers: { ...outboundHeaders(req, options), ...override }, + body: options.body ?? req, + signal, + }); + } catch (error) { + throw new UnreachableError(options.target.endpoint, causeOf(error)); + } +} + +/** + * Forwards the request and streams the node's answer back as it arrives; + * resolves with the node's status once the answer has been relayed (the + * named verbs read a 404 off it to re-check the cache — find.ts verify). + * Throws UnreachableError only before any byte of the answer was written; + * once the head is out, a failure mid-stream can only be a cut connection + * — there is no honest status left to send. Resolves null when the client + * left before the node answered: the node-side request is aborted and + * nothing is rendered (the response is gone). Otherwise an abandoned exec + * against a slow node would hold a gateway→node socket until the node + * answered — on a hung node, until its TCP died. forwardCapture follows + * the same rule. + */ +export async function forwardStream( + req: http.IncomingMessage, + res: http.ServerResponse, + options: ForwardOptions, +): Promise { + const gone = new AbortController(); + const onClose = () => gone.abort(); + res.once('close', onClose); + let upstream: Awaited>; + try { + upstream = await dispatch(req, options, {}, gone.signal); + } catch (error) { + if (gone.signal.aborted) return null; + throw error; + } finally { + res.off('close', onClose); + } + res.writeHead(upstream.statusCode, inboundHeaders(upstream.headers)); + try { + // pipeline destroys both ends on failure: a client that went away + // aborts the node's response, a node that died cuts the client. + await pipeline(upstream.body, res); + } catch { + res.destroy(); + } + return upstream.statusCode; +} + +export interface CapturedResponse { + status: number; + headers: http.OutgoingHttpHeaders; + body: Buffer; +} + +/** + * Forwards and collects the whole answer, for the few verbs whose answer + * the gateway must read before passing it on: a create (to learn the id + * the node minted, for the cache) and a destroy (to forget the entry). + * Their answers are small JSON; everything else streams. + * + * The same abort rule as forwardStream, and for the same reason: a client + * that leaves before the node has answered takes its request with it, and + * `null` says so (the caller writes nothing — the response is gone). These + * verbs hold the name's queue slot, and a request nobody is waiting for + * must not hold it: a node that accepts the connection and never answers + * (a daemon with a hung event loop — 2026-08-13 and 2026-09-11 shapes) + * would otherwise pin the name until its TCP died, every retry queueing + * behind the abandoned attempt and firing at the node together when it + * came back. A queued verb whose client left while it waited for the slot + * is not sent at all. Once the node's head has arrived the small body is + * read to the end regardless, so a 2xx that did land is still learned. + */ +export async function forwardCapture( + req: http.IncomingMessage, + res: http.ServerResponse, + options: ForwardOptions, +): Promise { + if (res.destroyed) return null; + const gone = new AbortController(); + const onClose = () => gone.abort(); + res.once('close', onClose); + let upstream: Awaited>; + try { + // The gateway is the reader of this answer, so it states its own + // preference: identity. A caller's accept-encoding would let a hop + // that compresses (a Caddy with `encode` in front of the node) hand + // back bytes the gateway cannot read the id out of. + upstream = await dispatch( + req, + options, + { 'accept-encoding': 'identity' }, + gone.signal, + ); + } catch (error) { + if (gone.signal.aborted) return null; + throw error; + } finally { + res.off('close', onClose); + } + let body: Buffer; + try { + body = Buffer.from(await upstream.body.arrayBuffer()); + } catch (error) { + throw new UnreachableError( + options.target.endpoint, + `answer cut mid-body: ${causeOf(error)}`, + ); + } + return { + status: upstream.statusCode, + headers: inboundHeaders(upstream.headers), + body, + }; +} + +/** Writes a captured answer through, verbatim. */ +export function replay( + res: http.ServerResponse, + answer: CapturedResponse, +): void { + // The client left while the small body was being read: nothing to say + // to nobody (what the answer taught the cache is kept regardless). + if (res.destroyed) return; + // A 204, a 304 or a 1xx carries no body and must not claim a length + // (RFC 9110 §8.6): the node sent none, and a `content-length: 0` added + // here would change the framing this file promises to keep. + const bodiless = + answer.status === 204 || answer.status === 304 || answer.status < 200; + res.writeHead( + answer.status, + bodiless + ? answer.headers + : { ...answer.headers, 'content-length': String(answer.body.length) }, + ); + res.end(bodiless ? undefined : answer.body); +} + +/** + * The upgrade path (sandbox WebSockets, once the port proxy routes + * through the gateway): the daemon's own replay (sandbox-proxy.ts + * handleUpgrade) — dial the node, write the request line and rawHeaders + * verbatim, then pipe both ways. Plain TCP: node endpoints are + * private-network http, and a TLS node endpoint is refused here rather + * than half-supported. + */ +export function forwardUpgrade( + req: http.IncomingMessage, + socket: Duplex, + head: Buffer, + target: ForwardTarget, + connectTimeoutMs = CONNECT_TIMEOUT_MS, +): void { + socket.on('error', () => socket.destroy()); + const url = new URL(target.endpoint); + if (url.protocol !== 'http:') { + // An operator's configuration, refused without naming the address: + // this face is unauthenticated (raw.ts). + socket.end( + `HTTP/1.1 502 Bad Gateway\r\ncontent-type: application/json\r\nconnection: close\r\n\r\n${JSON.stringify({ message: "the sandbox's node is not a plain http endpoint — upgrades are not forwarded over TLS" })}`, + ); + return; + } + // A URL keeps the brackets of an IPv6 literal in hostname; a dialer + // wants them off. + const host = url.hostname.replace(/^\[(.*)\]$/, '$1'); + let replayed = false; + // The HTTP faces' connect deadline (the dispatcher above), here by hand: + // a node whose host is down without an RST black-holes the SYN, and a + // bare net.connect would hold the client for the OS's own retry budget + // (75s on macOS, about two minutes on Linux — measured 2026-09-12 with + // nc against 192.0.2.1) before the 502 the HTTP faces give in ten. + // Idle after the handshake is legitimate (a quiet WebSocket), so the + // timer is switched off the moment the connection is up. + const upstream = net.connect( + { port: Number(url.port || 80), host, timeout: connectTimeoutMs }, + () => { + upstream.setTimeout(0); + replayed = true; + let raw = `${req.method} ${req.url} HTTP/1.1\r\n`; + for (let i = 0; i < req.rawHeaders.length; i += 2) { + raw += `${req.rawHeaders[i]}: ${req.rawHeaders[i + 1]}\r\n`; + } + upstream.write(`${raw}\r\n`); + if (head.length > 0) upstream.write(head); + socket.pipe(upstream); + upstream.pipe(socket); + }, + ); + upstream.on('timeout', () => upstream.destroy(new Error('connect timeout'))); + upstream.on('error', () => { + // An HTTP status is honest only before the handshake was replayed; + // after it the client is inside an upgraded stream (WebSocket frames), + // where a 502 line would be a corrupt frame, not an answer. + if (!replayed && !socket.writableEnded && socket.writable) { + socket.end( + `HTTP/1.1 502 Bad Gateway\r\ncontent-type: application/json\r\nconnection: close\r\n\r\n${JSON.stringify({ message: "the sandbox's node did not answer the upgrade — retry" })}`, + ); + } else { + socket.destroy(); + } + }); + socket.on('close', () => upstream.destroy()); + upstream.on('close', () => socket.destroy()); +} diff --git a/packages/gateway/src/lookup.ts b/packages/gateway/src/lookup.ts new file mode 100644 index 00000000..11476283 --- /dev/null +++ b/packages/gateway/src/lookup.ts @@ -0,0 +1,71 @@ +import { + type LookupSandboxRequest, + lookupSandboxResponseSchema, + type SandboxState, +} from '@dormice/shared'; +import { request } from 'undici'; + +/** + * Asking one node "do you hold this sandbox?" — the daemon's lookupSandbox + * verb, the one question the gateway puts to a node on its own account + * (everything else it sends is a caller's request, forwarded raw). Two + * seconds, not more: a node that cannot answer a ledger read in two + * seconds is a node in trouble, and the caller is waiting on the whole + * round. There is no second, slower deadline — slow is down (design + * record #35). + */ +export const LOOKUP_TIMEOUT_MS = 2_000; + +export type LookupQuery = LookupSandboxRequest; + +export type LookupAnswer = + | { kind: 'found'; id: string; name: string; state: SandboxState } + | { kind: 'absent' } + /** The node did not answer: no connection, a timeout, a non-200, an unreadable body. `why` is the transport's word. */ + | { kind: 'silent'; why: string }; + +export interface AskedNode { + id: string; + endpoint: string; +} + +export type AskNode = ( + node: AskedNode, + query: LookupQuery, +) => Promise; + +export function causeOf(error: unknown): string { + const e = error as { code?: string; message?: string; cause?: unknown }; + const cause = e.cause as { code?: string; message?: string } | undefined; + return cause?.code ?? cause?.message ?? e.code ?? e.message ?? String(error); +} + +/** The production asker: HTTP to the node's endpoint under the fleet's token. */ +export function httpAskNode(token: string): AskNode { + return async (node, query) => { + try { + const res = await request(`${node.endpoint}/lookupSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(query), + signal: AbortSignal.timeout(LOOKUP_TIMEOUT_MS), + }); + if (res.statusCode !== 200) { + const text = await res.body.text(); + return { + kind: 'silent', + why: `lookupSandbox answered ${res.statusCode}: ${text.slice(0, 200)}`, + }; + } + const answer = lookupSandboxResponseSchema.parse(await res.body.json()); + return answer.found + ? { kind: 'found', ...answer.sandbox } + : { kind: 'absent' }; + } catch (error) { + return { kind: 'silent', why: causeOf(error) }; + } + }; +} diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts new file mode 100644 index 00000000..be302d6c --- /dev/null +++ b/packages/gateway/src/main.ts @@ -0,0 +1,135 @@ +import { fileURLToPath } from 'node:url'; +import { KeyedQueue } from '@dormice/server/keyed-queue'; +import { acquireSingleWriterLock } from '@dormice/server/lock'; +import { closeWithGrace, trackConnections } from '@dormice/server/shutdown'; +import { pino } from 'pino'; +import { z } from 'zod'; +import { buildGatewayApp } from './app'; +import { NameCache } from './cache'; +import { loadConfig } from './config'; +import { migrateDb, openDb } from './db/db'; +import { Finder } from './find'; +import { Fleet } from './fleet'; +import { httpAskNode } from './lookup'; +import { readBuildInfo } from './version'; + +const log = pino(); + +/** An operator mistake, not a bug: one honest line, no stack trace. */ +function fatal(error: unknown): never { + log.fatal( + error instanceof z.ZodError + ? z.prettifyError(error) + : error instanceof Error + ? error.message + : String(error), + ); + process.exit(1); +} + +const config = (() => { + try { + return loadConfig(); + } catch (error) { + return fatal(error); + } +})(); + +// One database file, one gateway — enforced, not assumed (the daemon's own +// lock over the gateway's file). The handle is kept for the life of the +// process on purpose: better-sqlite3 closes a handle when its object is +// garbage collected, and a closed handle drops the file lock (measured +// 2026-09-11 on the daemon with a discarded return value). +let lock: ReturnType | undefined; +if (config.DORMICE_GATEWAY_DB_PATH !== ':memory:') { + try { + lock = acquireSingleWriterLock( + config.DORMICE_GATEWAY_DB_PATH, + `another gateway is already running against ${config.DORMICE_GATEWAY_DB_PATH} — one database, one gateway. Stop the other instance, or point this one at its own DORMICE_GATEWAY_DB_PATH.`, + ); + } catch (error) { + fatal(error); + } +} + +// Migrate on every boot; a fresh install needs no separate setup step. +const db = openDb(config.DORMICE_GATEWAY_DB_PATH); +migrateDb(db, fileURLToPath(new URL('../drizzle', import.meta.url))); + +// The fleet from the nodes table (a node that is down is still a node); +// the cache and the readings fill in as nodes report and callers ask. +const fleet = new Fleet(db); +const finder = new Finder( + fleet, + new NameCache(), + httpAskNode(config.DORMICE_API_TOKEN), + log, +); + +// One queue for the whole gateway: the native acquire/destroy and the E2B +// create/kill must share per-name slots or the serialization means nothing. +const locks = new KeyedQueue(); + +const build = readBuildInfo(); +log.info( + build + ? `dormice-gateway build ${build.commit} (${build.title})` + : 'dormice-gateway build: no version identity (built outside a git checkout)', +); + +const app = buildGatewayApp({ + config, + fleet, + finder, + locks, + logger: log, + build, +}); + +// Same red line as the daemon: loopback only, host not configurable — the +// public face is a reverse proxy's job. Nothing is awaited before the door +// opens: the nodes report themselves within one interval, and until then +// a new name is refused with placement's honest 503 while every existing +// sandbox is found by asking. +await app.listen({ host: '127.0.0.1', port: config.DORMICE_GATEWAY_PORT }); +const known = fleet.all(); +log.info( + known.length === 0 + ? 'no node has checked in yet; nodes join the fleet at their first check-in' + : `fronting ${known.length} node(s) from the last run: ${known.map((n) => `${n.id} at ${n.endpoint}`).join(', ')} — each is placed on again after its first check-in`, +); + +// Bounded shutdown, the daemon's (packages/server/src/shutdown.ts has the +// measurements): close the listener, give in-flight short work the grace, +// cut what is still connected — a forwarded exec may legitimately live for +// hours — and exit explicitly. +const SHUTDOWN_GRACE_MS = 10_000; +const connections = trackConnections(app.server); +let closing = false; +const close = async (signal: NodeJS.Signals) => { + if (closing) return; + closing = true; + process.removeListener('SIGTERM', onSigterm); + process.removeListener('SIGINT', onSigint); + app.log.info( + `${signal} received — shutting down (grace ${SHUTDOWN_GRACE_MS}ms)`, + ); + try { + const cut = await closeWithGrace(app, connections, SHUTDOWN_GRACE_MS); + if (cut > 0) { + app.log.warn( + { cut }, + 'connections still open at the end of the grace period were cut', + ); + } + } catch (error) { + app.log.error(error, `graceful shutdown after ${signal} failed`); + process.exitCode = 1; + } + lock?.close(); + process.exit(process.exitCode ?? 0); +}; +const onSigterm = () => void close('SIGTERM'); +const onSigint = () => void close('SIGINT'); +process.once('SIGTERM', onSigterm); +process.once('SIGINT', onSigint); diff --git a/packages/gateway/src/placement.test.ts b/packages/gateway/src/placement.test.ts new file mode 100644 index 00000000..c9833ee2 --- /dev/null +++ b/packages/gateway/src/placement.test.ts @@ -0,0 +1,181 @@ +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { type Db, migrateDb, openDb } from './db/db'; +import { Fleet, type NodeState } from './fleet'; +import { type PlacementKnobs, pick, refusalMessage } from './placement'; +import { checkInOf, type reading } from './testing'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const NOW = new Date('2026-09-14T12:00:00.000Z'); +const KNOBS: PlacementKnobs = { + cpuLimitPct: 70, + activeLimit: 400, + minDiskAvailableBytes: 10 * 2 ** 30, +}; + +function fleet(): { db: Db; fleet: Fleet } { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + return { db, fleet: new Fleet(db) }; +} + +/** A node that checked in at NOW (or when told) with the given reading. */ +function node( + f: Fleet, + id: string, + over: Parameters[0] & { + intervalSeconds?: number; + placed?: number; + checkedInAt?: Date; + } = {}, +): NodeState { + const { node } = f.checkIn( + checkInOf(id, `http://${id}:80`, over), + over.checkedInAt ?? NOW, + ); + node.placedSinceCheckIn = over.placed ?? 0; + return node; +} + +describe('pick', () => { + it('cpu above the limit refuses, at the limit passes, unknown passes', () => { + const f = fleet().fleet; + expect(pick([node(f, 'a', { cpu: 71 })], KNOBS, NOW).node).toBeNull(); + expect(pick([node(f, 'b', { cpu: 70 })], KNOBS, NOW).node?.id).toBe('b'); + // A freshly restarted node's first reading has no delta: not saturated. + expect(pick([node(f, 'c', { cpu: null })], KNOBS, NOW).node?.id).toBe('c'); + }); + + it('active at the limit refuses; frozen sandboxes are not counted', () => { + const f = fleet().fleet; + expect(pick([node(f, 'a', { active: 400 })], KNOBS, NOW).node).toBeNull(); + expect(pick([node(f, 'b', { active: 399 })], KNOBS, NOW).node?.id).toBe( + 'b', + ); + // Thousands of frozen sandboxes are the normal shape of a node. + expect( + pick([node(f, 'c', { active: 1, frozen: 14_000 })], KNOBS, NOW).node?.id, + ).toBe('c'); + }); + + it('counts what was placed since the reading, so a burst inside one interval cannot overfill a node', () => { + const f = fleet().fleet; + const a = node(f, 'a', { active: 390 }); + expect(pick([a], KNOBS, NOW).node?.id).toBe('a'); + a.placedSinceCheckIn = 10; + expect(pick([a], KNOBS, NOW).node).toBeNull(); + expect(pick([a], KNOBS, NOW).refused[0]?.reason).toMatch( + /390 active sandboxes \+ 10 placed since the reading reach the 400 limit/, + ); + }); + + it('a data disk below the floor refuses, naming the GiB; a missing disk reading passes', () => { + const f = fleet().fleet; + const full = node(f, 'full', { diskAvail: 9 * 2 ** 30, memAvail: 30e9 }); + const room = node(f, 'room', { diskAvail: 50 * 2 ** 30, memAvail: 20e9 }); + expect(pick([full, room], KNOBS, NOW).node?.id).toBe('room'); + expect(pick([full], KNOBS, NOW).refused).toEqual([ + { + nodeId: 'full', + reason: 'data disk has 9.0 GiB available, below the 10.0 GiB floor', + }, + ]); + expect( + pick([node(f, 'unknown', { diskAvail: null })], KNOBS, NOW).node?.id, + ).toBe('unknown'); + }); + + it('scores by active density — (active + placed) per core — then by available memory, then id', () => { + const f = fleet().fleet; + // Fewer sandboxes per core beats more free memory: memory is a + // fifteen-second-old reading, the count moves with every pick. + const a = node(f, 'a', { memAvail: 8e9, active: 5 }); + const b = node(f, 'b', { memAvail: 16e9, active: 50 }); + expect(pick([a, b], KNOBS, NOW).node?.id).toBe('a'); + // Equal density: the most available memory, then the id. + const c = node(f, 'c', { memAvail: 16e9, active: 5 }); + expect(pick([a, b, c], KNOBS, NOW).node?.id).toBe('c'); + const d = node(f, 'd', { memAvail: 16e9, active: 5 }); + expect(pick([d, c], KNOBS, NOW).node?.id).toBe('c'); + // Per core: 40 sandboxes on 64 cores is emptier than 6 on 8. + const big = node(f, 'big', { cores: 64, active: 40, memAvail: 1e9 }); + const small = node(f, 'small', { cores: 8, active: 6, memAvail: 64e9 }); + expect(pick([small, big], KNOBS, NOW).node?.id).toBe('big'); + }); + + it('a burst inside one reading spreads: each pick counts against the node it chose, so the next pick sees it fuller', () => { + const f = fleet().fleet; + const a = node(f, 'a', { memAvail: 17e9, active: 350 }); + const b = node(f, 'b', { memAvail: 16e9, active: 10 }); + const landed = { a: 0, b: 0 }; + for (let i = 0; i < 100; i++) { + const chosen = pick([a, b], KNOBS, NOW).node; + if (chosen === null) throw new Error('every node refused'); + chosen.placedSinceCheckIn += 1; + landed[chosen.id as 'a' | 'b'] += 1; + } + expect(landed).toEqual({ a: 0, b: 100 }); + // And with equal readings the two alternate. + const c = node(f, 'c', { active: 10 }); + const d = node(f, 'd', { active: 10 }); + const alternating: string[] = []; + for (let i = 0; i < 6; i++) { + const chosen = pick([c, d], KNOBS, NOW).node; + if (chosen === null) throw new Error('every node refused'); + chosen.placedSinceCheckIn += 1; + alternating.push(chosen.id); + } + expect(alternating).toEqual(['c', 'd', 'c', 'd', 'c', 'd']); + }); + + it('refuses a node that never checked in since the start, or is silent for two of its intervals', () => { + const { db, fleet: f } = fleet(); + const stale = node(f, 'stale', { + intervalSeconds: 15, + checkedInAt: new Date(NOW.getTime() - 31_000), + }); + const fresh = node(f, 'fresh', { + intervalSeconds: 15, + checkedInAt: new Date(NOW.getTime() - 29_000), + }); + // The same row after a gateway restart: known, but nothing reported yet. + const restarted = new Fleet(db).get('fresh'); + if (!restarted) throw new Error('row lost'); + const result = pick([stale, restarted], KNOBS, NOW); + expect(result.node).toBeNull(); + expect( + Object.fromEntries(result.refused.map((r) => [r.nodeId, r.reason])), + ).toEqual({ + stale: 'has not checked in for 31s', + fresh: 'has not checked in since the gateway started', + }); + expect(pick([fresh], KNOBS, NOW).node?.id).toBe('fresh'); + }); + + it('when every node refuses, each refusal names the node and its reason for the 503; no node at all says so', () => { + const f = fleet().fleet; + const result = pick( + [ + node(f, 'a', { cpu: 90, active: 100, memAvail: 1e9 }), + node(f, 'b', { cpu: 20, active: 400, memAvail: 2e9 }), + ], + KNOBS, + NOW, + ); + expect(result.node).toBeNull(); + expect(result.refused).toEqual([ + { nodeId: 'a', reason: 'cpu 90% is above the 70% limit' }, + { + nodeId: 'b', + reason: + '400 active sandboxes + 0 placed since the reading reach the 400 limit', + }, + ]); + expect(refusalMessage(result)).toBe( + 'no node can take a new sandbox right now — a: cpu 90% is above the 70% limit; b: 400 active sandboxes + 0 placed since the reading reach the 400 limit', + ); + expect(refusalMessage(pick([], KNOBS, NOW))).toMatch( + /no node has checked in/, + ); + }); +}); diff --git a/packages/gateway/src/placement.ts b/packages/gateway/src/placement.ts new file mode 100644 index 00000000..456e26ec --- /dev/null +++ b/packages/gateway/src/placement.ts @@ -0,0 +1,138 @@ +import { downReason, type NodeState } from './fleet'; + +export interface PlacementKnobs { + /** A reading above this refuses the node. */ + cpuLimitPct: number; + /** Active sandboxes plus placements since the reading at or above this refuse the node. */ + activeLimit: number; + /** A data disk with less than this available refuses the node. */ + minDiskAvailableBytes: number; +} + +/** One node's refusal, in the words the 503 repeats: `: `. */ +export interface NodeRefusal { + nodeId: string; + reason: string; +} + +export interface Placement { + /** The node to create on, or null when every node refused. */ + node: NodeState | null; + /** Every node that refused and why — the 503's body when node is null. */ + refused: NodeRefusal[]; +} + +/** + * Where a new sandbox goes: four gates, then one score. Pure — it reads + * what the nodes last reported and never asks a node. + * + * A node that is down (fleet.ts downReason: no check-in since the gateway + * started, or two of its own intervals silent) is refused: placing blind + * is how a gateway puts the twenty-first sandbox on the box that just + * fell over. A node whose whole-machine CPU is above the limit is + * refused; a null reading passes — the first check-in after a node + * restart has no delta to report, and "don't know yet" is not + * "saturated". A node at its active ceiling is refused, counting what + * this gateway placed there since the reading: frozen sandboxes are not + * counted, on purpose — they cost swap, not CPU or dockerd attention, a + * node holds thousands of them, and counting them would shut the gate + * from the first minute of a cut-over and never open it again. A node + * whose data disk is below the floor is refused (design record #36): a + * full data disk stops every sandbox on the node at once, and the node's + * own create answering 500 would otherwise be re-chosen — scored emptiest, + * because nothing it fails to build ever counts — for every new name + * until someone noticed. A missing disk reading passes, like the CPU: + * unknown is not full. + * + * Among the nodes that pass, the emptiest by active density wins: active + * sandboxes plus what this gateway placed there since the reading, per + * core. Per core because the one density ever measured is per core (535 + * sandboxes healthy and 882 dead on 128 cores, 2026-09-11) and a 64-core + * node with 40 sandboxes has more room than an 8-core node with 6; + * counting the in-flight placements is what spreads a burst — a reading + * is fifteen seconds old, a hundred creates arrive inside that window, + * and a score that does not move with each pick sends all hundred to the + * node the reading favoured (reproduced twice, 2026-09-12). Ties go to the + * most available memory — the resource the freeze/swap design overcommits + * and an overloaded host runs out of first — then to the id, so the + * choice is stable and explainable. Memory does not gate placement: a + * node with room per core and little memory left is still chosen, and + * memory pressure is judged where it is felt, by the node's own admission + * (design record #27, after the cluster), not guessed from a + * fifteen-second-old figure the picks do not move. + */ +export function pick( + nodes: readonly NodeState[], + knobs: PlacementKnobs, + now: Date, +): Placement { + const refused: NodeRefusal[] = []; + const candidates: Array<{ + node: NodeState; + /** (active + placed since the reading) per core. */ + load: number; + memAvailableBytes: number; + }> = []; + for (const node of nodes) { + const refuse = (reason: string) => + refused.push({ nodeId: node.id, reason }); + const down = downReason(node, now); + if (down !== null) { + refuse(down); + continue; + } + const reading = node.reading; + if (reading === null) { + refuse('has not reported a reading'); + continue; + } + const cpuUsedPct = reading.host.cpuUsedPct; + if (cpuUsedPct !== null && cpuUsedPct > knobs.cpuLimitPct) { + refuse( + `cpu ${Math.round(cpuUsedPct)}% is above the ${knobs.cpuLimitPct}% limit`, + ); + continue; + } + const active = reading.sandboxes.byState.active; + const placed = node.placedSinceCheckIn; + if (active + placed >= knobs.activeLimit) { + refuse( + `${active} active sandboxes + ${placed} placed since the reading reach the ${knobs.activeLimit} limit`, + ); + continue; + } + const disk = reading.dataDisk; + if (disk !== null && disk.availableBytes < knobs.minDiskAvailableBytes) { + refuse( + `data disk has ${gib(disk.availableBytes)} GiB available, below the ${gib(knobs.minDiskAvailableBytes)} GiB floor`, + ); + continue; + } + candidates.push({ + node, + load: (active + placed) / Math.max(1, reading.host.cpuCount), + memAvailableBytes: reading.host.memAvailableBytes, + }); + } + candidates.sort( + (a, b) => + a.load - b.load || + b.memAvailableBytes - a.memAvailableBytes || + (a.node.id < b.node.id ? -1 : a.node.id > b.node.id ? 1 : 0), + ); + return { node: candidates[0]?.node ?? null, refused }; +} + +function gib(bytes: number): string { + return (bytes / 2 ** 30).toFixed(1); +} + +/** The 503's sentence when every node refused: each node and its reason — or that there is no node at all. */ +export function refusalMessage(placement: Placement): string { + if (placement.refused.length === 0) { + return 'no node can take a new sandbox: no node has checked in with this gateway yet'; + } + return `no node can take a new sandbox right now — ${placement.refused + .map((r) => `${r.nodeId}: ${r.reason}`) + .join('; ')}`; +} diff --git a/packages/gateway/src/raw.ts b/packages/gateway/src/raw.ts new file mode 100644 index 00000000..a5611263 --- /dev/null +++ b/packages/gateway/src/raw.ts @@ -0,0 +1,192 @@ +import type http from 'node:http'; +import type { Duplex } from 'node:stream'; +import type { Logger } from 'pino'; +import type { Classified } from './classify'; +import { relay, renderError, sendPreflight } from './errors'; +import type { Finder, Found } from './find'; +import { forwardStream } from './forward'; + +export interface RawFacesDeps { + finder: Finder; + token: string; + log: Logger; +} + +/** How long a caller refused with "a node did not answer" may wait before asking again — one check-in interval. */ +export const RETRY_AFTER_SECONDS = 15; + +/** + * The faces Fastify never sees — keyed on a header on any path, judged on + * the raw request the serverFactory hands over: + * envd E2B's in-sandbox API, keyed by E2b-Sandbox-Id; forwarded + * with no credential change (the access token is the node's + * own HMAC). Preflights are answered here: the node answers + * them without auth, and a preflight that came back 401 + * without CORS would fail every browser-direct upload. + * signedRoot the bare signed-URL form: no sandbox id anywhere the + * gateway can read without the node's signing secret, so an + * honest 501. + * + * Nobody has authenticated to the gateway on these faces — the node + * judges the credential, after the gateway has picked it — so what the + * gateway says here describes the sandbox's state and never the fleet: no + * node id, no endpoint. A sandbox id is handed to browsers in every + * getHost URL, and the daemon's own proxy says only "not found" for one + * it lacks. The detail an operator needs goes to the log; the + * authenticated faces and listNodes name nodes freely. + */ +export function createRawFaces({ finder, token, log }: RawFacesDeps) { + /** The one sentence per finding for a sandbox id on these faces — generic on purpose (above). */ + function refusal( + res: http.ServerResponse, + id: string, + found: Exclude, + ): void { + switch (found.kind) { + case 'conflict': + renderError(res, 'connect', { + status: 502, + connectCode: 'unavailable', + message: `sandbox "${id}" is held by more than one node — routing resumes once an operator destroys one copy (listNodes and the gateway log name them)`, + cors: true, + }); + return; + case 'none': + renderError(res, 'connect', { + status: 502, + connectCode: 'unavailable', + message: `sandbox "${id}" is on no node — it may have been destroyed`, + cors: true, + }); + return; + case 'unsure': + renderError(res, 'connect', { + status: 503, + connectCode: 'unavailable', + message: `sandbox "${id}": a node did not answer, so its whereabouts cannot be settled — retry`, + cors: true, + retryAfterSeconds: RETRY_AFTER_SECONDS, + }); + return; + } + } + + /** + * Finds the id and forwards, or answers the refusal — the one path + * every keyed face takes. Nothing here may throw past this point: no + * framework stands behind a raw face, so an escaped rejection would be + * the process's, not the request's (errors.ts relay answers instead). + */ + async function route( + req: http.IncomingMessage, + res: http.ServerResponse, + id: string, + ): Promise { + let found: Found; + try { + found = await finder.byId(id); + } catch (error) { + log.error(error, 'envd face: the lookup itself failed'); + renderError(res, 'connect', { + status: 500, + connectCode: 'internal', + message: 'the gateway failed while locating the sandbox — see its log', + cors: true, + }); + return; + } + if (found.kind !== 'one') { + refusal(res, id, found); + return; + } + const node = found.node; + await relay( + res, + 'connect', + log, + async () => { + await forwardStream(req, res, { + target: { endpoint: node.endpoint, token }, + credential: 'none', + }); + }, + (error) => { + log.warn( + { + sandboxId: id, + nodeId: node.id, + endpoint: node.endpoint, + why: error.why, + }, + "envd face: the sandbox's node did not answer", + ); + return { + status: 502, + connectCode: 'unavailable', + message: `sandbox "${id}": its node did not answer (${error.why}) — retry`, + cors: true, + }; + }, + ); + } + + return { + handleRequest( + kind: Exclude, + req: http.IncomingMessage, + res: http.ServerResponse, + ): void { + switch (kind.face) { + case 'envd': { + if (req.method === 'OPTIONS') { + sendPreflight(req, res); + return; + } + const header = req.headers['e2b-sandbox-id']; + const id = Array.isArray(header) ? header[0] : header; + if (!id) { + renderError(res, 'connect', { + status: 401, + connectCode: 'unauthenticated', + message: 'missing E2b-Sandbox-Id header', + cors: true, + }); + return; + } + void route(req, res, id); + return; + } + case 'signedRoot': { + if (req.method === 'OPTIONS') { + sendPreflight(req, res); + return; + } + renderError(res, 'connect', { + status: 501, + connectCode: 'unimplemented', + message: + 'the bare signed-URL form is not routed by the gateway yet — use the sandbox host form, or the node directly', + cors: true, + }); + return; + } + } + }, + + handleUpgrade( + _kind: Classified, + _req: http.IncomingMessage, + socket: Duplex, + ): void { + // First, before anything else: http.Server drops its own error + // listener from the socket before emitting 'upgrade', so a client + // that resets here would raise an uncaught ECONNRESET and take the + // whole gateway down — from an unauthenticated face (the daemon's + // sandbox-proxy.ts handleUpgrade has the same first line). + socket.on('error', () => socket.destroy()); + // No face of this build takes upgrades: sandbox WebSockets ride the + // port proxy, which joins the gateway with the sandbox domain. + socket.destroy(); + }, + }; +} diff --git a/packages/gateway/src/routes/create.ts b/packages/gateway/src/routes/create.ts new file mode 100644 index 00000000..f6e3d174 --- /dev/null +++ b/packages/gateway/src/routes/create.ts @@ -0,0 +1,129 @@ +import type http from 'node:http'; +import type { FastifyReply, FastifyRequest } from 'fastify'; +import { z } from 'zod'; +import type { NameCache } from '../cache'; +import type { Fleet, NodeState } from '../fleet'; +import { + type CapturedResponse, + type Credential, + forwardCapture, +} from '../forward'; +import { type Placement, type PlacementKnobs, pick } from '../placement'; + +/** + * What a create looks like on each face the gateway creates through: the + * credential it forwards under, and where the node's answer keeps the id + * it minted — all the gateway ever reads of that answer. + */ +export interface CreateFace { + credential: Credential; + idOf(answer: unknown): string | null; +} + +const acquireAnswerSchema = z.object({ sandbox: z.object({ id: z.string() }) }); +const e2bAnswerSchema = z.object({ sandboxID: z.string() }); + +export const NATIVE_CREATE: CreateFace = { + credential: 'bearer', + idOf: (answer) => { + const parsed = acquireAnswerSchema.safeParse(answer); + return parsed.success ? parsed.data.sandbox.id : null; + }, +}; + +export const E2B_CREATE: CreateFace = { + credential: 'x-api-key', + idOf: (answer) => { + const parsed = e2bAnswerSchema.safeParse(answer); + return parsed.success ? parsed.data.sandboxID : null; + }, +}; + +/** + * Where a new sandbox goes: placement's pick, counted against the node at + * once — the next acquire in this same interval must see this one as + * already there. Nothing is recorded anywhere else: if the answer is + * lost, the caller's retry asks the fleet (find.ts) and the node that + * built it answers "yes" from inside the name's slot. + */ +export function place( + fleet: Fleet, + knobs: PlacementKnobs, + now: Date, +): Placement { + const placement = pick(fleet.all(), knobs, now); + if (placement.node !== null) placement.node.placedSinceCheckIn += 1; + return placement; +} + +/** + * A create that waited for its name's slot behind a slow destroy and whose + * client left meanwhile: nothing is placed or counted for it — the + * response is gone, forwardCapture would send nothing. The reply is + * hijacked so Fastify writes nothing to the dead socket either. + */ +export function clientGone(reply: FastifyReply): boolean { + if (!reply.raw.destroyed) return false; + reply.hijack(); + return true; +} + +export interface CreateOptions { + target: NodeState; + token: string; + name: string | null; + body: Buffer | undefined; + face: CreateFace; +} + +/** + * Forwards a create and learns from the node's answer: a 2xx with a + * readable id goes into the cache (name and id → this node), so the next + * request for the sandbox skips the round of questions. A 2xx without a + * readable id is logged — the sandbox exists on the node and the next + * lookup finds it there, never silently. Any other answer teaches + * nothing: a 4xx or the node's own 500 means the node holds nothing under + * the name and the next attempt may be placed elsewhere; a 502/503/504 + * from a hop in front of the node, or no answer at all, leaves the + * question to the next lookup — the node that may have built it answers + * from inside the name's slot. + */ +export async function forwardCreate( + cache: NameCache, + request: FastifyRequest, + res: http.ServerResponse, + { target, token, name, body, face }: CreateOptions, +): Promise { + const answer = await forwardCapture(request.raw, res, { + target: { endpoint: target.endpoint, token }, + credential: face.credential, + body, + }); + if (answer === null) return null; + if (answer.status >= 200 && answer.status < 300) { + const id = face.idOf(parseJson(answer.body)); + if (id !== null) { + cache.put({ id, name, nodeId: target.id }); + } else { + request.log.warn( + { node: target.id, name, status: answer.status }, + 'create answered 2xx without a readable sandbox id; the next lookup finds it on the node', + ); + } + } else if (answer.status >= 500) { + request.log.warn( + { node: target.id, name, status: answer.status }, + 'create was refused by the node or a hop in front of it; nothing cached', + ); + } + return answer; +} + +export function parseJson(body: Buffer | undefined): unknown { + if (body === undefined || body.length === 0) return undefined; + try { + return JSON.parse(body.toString('utf8')); + } catch { + return undefined; + } +} diff --git a/packages/gateway/src/routes/destroy.ts b/packages/gateway/src/routes/destroy.ts new file mode 100644 index 00000000..189105c3 --- /dev/null +++ b/packages/gateway/src/routes/destroy.ts @@ -0,0 +1,44 @@ +import type http from 'node:http'; +import type { FastifyRequest } from 'fastify'; +import type { CacheEntry, NameCache } from '../cache'; +import type { NodeState } from '../fleet'; +import { + type CapturedResponse, + type Credential, + forwardCapture, +} from '../forward'; + +export interface DestroyOptions { + target: NodeState; + token: string; + /** The cache entry the verb addresses — forgotten on the node's yes. */ + entry: CacheEntry; + body: Buffer | undefined; + credential: Credential; +} + +/** + * The one destroy path, the mirror of create.ts: the native destroySandbox + * and the E2B kill both forward, wait for the node's whole answer and + * learn from a 2xx — the cache entry goes. Only the node's yes teaches + * anything: its 404 is not "gone" (the daemon answers 404 for a sandbox + * past its kill deadline that its scanner has not torn down yet, a row a + * lookup would still answer yes for), and a 5xx or no answer leaves the + * cache as it was. The cache is a cache: an entry kept one destroy too + * long costs the next request one 404 and a re-check (find.ts verify). + */ +export async function forwardDestroy( + cache: NameCache, + request: FastifyRequest, + res: http.ServerResponse, + { target, token, entry, body, credential }: DestroyOptions, +): Promise { + const answer = await forwardCapture(request.raw, res, { + target: { endpoint: target.endpoint, token }, + credential, + body, + }); + if (answer === null) return null; + if (answer.status >= 200 && answer.status < 300) cache.evict(entry); + return answer; +} diff --git a/packages/gateway/src/routes/e2b.ts b/packages/gateway/src/routes/e2b.ts new file mode 100644 index 00000000..ee5d0e51 --- /dev/null +++ b/packages/gateway/src/routes/e2b.ts @@ -0,0 +1,215 @@ +import { tokensEqual } from '@dormice/server/auth'; +import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import type { FastifyError, FastifyReply, FastifyRequest } from 'fastify'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { relay } from '../errors'; +import type { Finder } from '../find'; +import type { Fleet, NodeState } from '../fleet'; +import { forwardStream, replay } from '../forward'; +import { type PlacementKnobs, refusalMessage } from '../placement'; +import { RETRY_AFTER_SECONDS } from '../raw'; +import { + clientGone, + E2B_CREATE, + forwardCreate, + parseJson, + place, +} from './create'; +import { forwardDestroy } from './destroy'; +import { refuse, verdict } from './verdict'; + +export interface E2bRoutesOptions { + fleet: Fleet; + finder: Finder; + locks: KeyedQueue; + knobs: PlacementKnobs; + token: string; +} + +/** + * The E2B control plane in front of several nodes: what the official SDK + * calls api.e2b.app for, mounted at /e2b/api like the daemon's. Creates + * are placed (by metadata.name when the SDK gave one — the same name slot + * the native acquire takes — else unnamed, known by id alone until a + * lookup names it); everything addressed by id is found and forwarded. + * The dialect is E2B's: { code: , message }. + */ +export const e2bControlRoutes: FastifyPluginAsyncZod = async ( + app, + { fleet, finder, locks, knobs, token }, +) => { + app.addContentTypeParser( + 'application/json', + { parseAs: 'buffer' }, + (_request, body, done) => done(null, body), + ); + app.setErrorHandler((error: FastifyError, request, reply) => { + const status = error.statusCode ?? 500; + if (status >= 500) request.log.error(error, 'request failed'); + reply.code(status).send({ code: status, message: error.message }); + }); + app.setNotFoundHandler((request, reply) => { + reply.code(404).send({ + code: 404, + message: `route ${request.method} ${request.url} not found`, + }); + }); + + // The daemon's own X-API-KEY convention (`e2b_`), over the + // fleet's one token. + app.addHook('onRequest', async (request, reply) => { + const presented = request.headers['x-api-key']; + const key = Array.isArray(presented) ? presented[0] : presented; + const bare = key?.startsWith('e2b_') ? key.slice(4) : key; + if (bare === undefined || !tokensEqual(bare, token)) { + await reply.code(401).send({ code: 401, message: 'invalid API key' }); + } + }); + + const dialect = (message: string) => ({ code: 0, message }); + const send = (reply: FastifyReply, code: number, message: string) => + reply.code(code).send({ code, message }); + + /** From the hijack on the raw response is the gateway's to finish, in E2B's dialect (errors.ts relay). */ + const forwarded = ( + request: FastifyRequest, + reply: FastifyReply, + tail: string, + step: () => Promise, + ) => { + reply.hijack(); + return relay(reply.raw, 'control', request.log, step, (error) => ({ + status: 502, + message: `${error.message}${tail}`, + })); + }; + + const RETRY_FINDS_IT = + ' — retry: if the sandbox was built, the node answers for it'; + + function create( + request: FastifyRequest, + reply: FastifyReply, + target: NodeState, + name: string | null, + body: Buffer | undefined, + ) { + return forwarded(request, reply, RETRY_FINDS_IT, async () => { + const answer = await forwardCreate(finder.cache, request, reply.raw, { + target, + token, + name, + body, + face: E2B_CREATE, + }); + if (answer !== null) replay(reply.raw, answer); + }); + } + + function placed(reply: FastifyReply): NodeState | null { + if (clientGone(reply)) return null; + const placement = place(fleet, knobs, new Date()); + if (placement.node === null) { + reply.header('retry-after', String(RETRY_AFTER_SECONDS)); + send(reply, 503, refusalMessage(placement)); + return null; + } + return placement.node; + } + + app.post('/sandboxes', async (request, reply) => { + const body = request.body as Buffer | undefined; + const parsed = parseJson(body) as + | { metadata?: { name?: unknown } } + | undefined; + const name = parsed?.metadata?.name; + if (typeof name === 'string' && name.length > 0) { + return locks.run(name, async () => { + const judged = verdict(await finder.byName(name), `sandbox "${name}"`); + if (judged.kind === 'refuse') { + return refuse(reply, judged, (message) => ({ + ...dialect(message), + code: judged.status, + })); + } + const target = judged.kind === 'node' ? judged.node : placed(reply); + if (target === null) return reply; + return create(request, reply, target, name, body); + }); + } + const target = placed(reply); + if (target === null) return reply; + return create(request, reply, target, null, body); + }); + + app.get('/v2/sandboxes', async (_request, reply) => + send( + reply, + 501, + 'listing sandboxes is not routed by the gateway yet — call the node directly', + ), + ); + + const byId = async (request: FastifyRequest, reply: FastifyReply) => { + const { id } = request.params as { id: string }; + const judged = verdict(await finder.byId(id), `sandbox "${id}"`); + if (judged.kind === 'refuse') { + return refuse(reply, judged, (message) => ({ + ...dialect(message), + code: judged.status, + })); + } + if (judged.kind === 'none') { + return send(reply, 404, `sandbox "${id}" not found`); + } + const body = request.body as Buffer | undefined; + const { node, name } = judged; + // Only the kill itself — DELETE /sandboxes/, nothing deeper — is + // captured and learned from (routes/destroy.ts has what is learned). + // A DELETE on a sub-route the node lacks answers the same 404 shape as + // a kill of an unknown id (measured 2026-09-12 with + // DELETE /sandboxes//bogus). The kill takes the name's slot like + // the native destroy (the daemon's own kill does too). + if (request.method === 'DELETE' && isKill(request)) { + return locks.run(name ?? id, () => + forwarded(request, reply, ' — retry', async () => { + const answer = await forwardDestroy( + finder.cache, + request, + reply.raw, + { + target: node, + token, + entry: { id, name, nodeId: node.id }, + body, + credential: 'x-api-key', + }, + ); + if (answer !== null) replay(reply.raw, answer); + }), + ); + } + return forwarded(request, reply, ' — retry', async () => { + const status = await forwardStream(request.raw, reply.raw, { + target: { endpoint: node.endpoint, token }, + credential: 'x-api-key', + body, + }); + if (status === 404) void finder.verify({ id, name, nodeId: node.id }); + }); + }; + // Every method, not the ones this build of the daemon happens to serve: + // the node answers for its own routes (e2b 2.31 already PUTs + // /sandboxes/:id/network, which the daemon 404s — its 404, not the + // gateway's, is the honest one). The path reaches the node exactly as + // written (forward.ts dispatch), so a sub-route is a sub-route there + // too, never a create resolved out of dot segments. + const methods = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE'] as const; + app.route({ method: [...methods], url: '/sandboxes/:id', handler: byId }); + app.route({ method: [...methods], url: '/sandboxes/:id/*', handler: byId }); +}; + +/** The kill route exactly: `/sandboxes/` with nothing after the id. */ +function isKill(request: FastifyRequest): boolean { + return (request.params as { '*'?: string })['*'] === undefined; +} diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts new file mode 100644 index 00000000..fd8d3fed --- /dev/null +++ b/packages/gateway/src/routes/native.ts @@ -0,0 +1,257 @@ +import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import { WRITE_FILES_BODY_LIMIT_BYTES } from '@dormice/shared'; +import type { FastifyReply, FastifyRequest } from 'fastify'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { relay } from '../errors'; +import type { Finder } from '../find'; +import type { Fleet } from '../fleet'; +import { forwardStream, replay } from '../forward'; +import { type PlacementKnobs, refusalMessage } from '../placement'; +import { RETRY_AFTER_SECONDS } from '../raw'; +import { + clientGone, + forwardCreate, + NATIVE_CREATE, + parseJson, + place, +} from './create'; +import { forwardDestroy } from './destroy'; +import { refuse, verdict } from './verdict'; + +/** + * The native verbs the gateway routes: every verb addressed to one sandbox + * by `name`. Registered one by one — never as `/:verb` — so a misspelled + * verb is a 404 like on the daemon, and nothing else under the root is + * swallowed. + */ +export const NAMED_VERBS = [ + 'acquireSandbox', + 'execCommand', + 'writeFile', + 'writeFiles', + 'readFile', + 'readFiles', + 'rebuildSandbox', + 'updatePolicy', + 'updateSpec', + 'updateTemplate', + 'expandDisk', + 'updateMetadata', + 'destroySandbox', + 'getSandboxMetrics', + 'getSandboxMetricsHistory', +] as const; + +/** + * The verbs that address the daemon, not a sandbox: fleet lists, host + * readings, templates, settings, ingress, upgrade, keys. Asking every + * node and merging, or answering from the gateway's own tables, comes + * with the configuration authority; until then each answers an honest + * 501 naming the alternative, instead of a misleading answer from + * whichever node the gateway happened to pick. + */ +export const UNNAMED_VERBS = [ + 'listSandboxes', + 'listSandboxMetrics', + 'listSandboxImages', + 'listActivity', + 'getFleetTimeline', + 'getHostMetrics', + 'getHostMetricsHistory', + 'getConfig', + 'checkUpgrade', + 'applyUpgrade', + 'getUpgradeStatus', + 'getIngress', + 'setIngress', + 'registerTemplate', + 'listTemplates', + 'removeTemplate', + 'updateSettings', + 'createApiKey', + 'listApiKeys', + 'updateApiKey', + 'revokeApiKey', +] as const; + +export interface NativeRoutesOptions { + fleet: Fleet; + finder: Finder; + locks: KeyedQueue; + knobs: PlacementKnobs; + token: string; +} + +export const nativeRoutes: FastifyPluginAsyncZod = async ( + app, + { fleet, finder, locks, knobs, token }, +) => { + // Bodies stay bytes: the gateway reads one field (`name`) and forwards + // the bytes it received, never a re-serialization. Scoped to this + // plugin — the gateway's own verbs keep Fastify's JSON parsing. + app.addContentTypeParser( + 'application/json', + { parseAs: 'buffer' }, + (_request, body, done) => done(null, body), + ); + + for (const verb of UNNAMED_VERBS) { + app.post(`/${verb}`, async (_request, reply) => + reply.code(501).send({ + message: `${verb} is not routed by the gateway yet — call the node directly (this version routes only sandbox-addressed verbs)`, + }), + ); + } + + for (const verb of NAMED_VERBS) { + const bodyLimit = + verb === 'writeFile' || verb === 'writeFiles' + ? WRITE_FILES_BODY_LIMIT_BYTES + : undefined; + app.post( + `/${verb}`, + bodyLimit === undefined ? {} : { bodyLimit }, + async (request, reply) => { + const body = request.body as Buffer | undefined; + const name = nameOf(body); + if (name === null) { + return reply + .code(400) + .send({ message: 'name is required and must be a string' }); + } + // Only the two verbs that create or remove take the name's slot — + // the daemon's own discipline (its other verbs run unserialized + // too). The slot is what keeps twenty simultaneous acquires of a + // new name from each placing their own copy: the first finds + // nothing and places, the rest find its cache entry. + if (verb === 'acquireSandbox') { + return locks.run(name, () => acquire(request, reply, name, body)); + } + if (verb === 'destroySandbox') { + return locks.run(name, () => destroy(request, reply, name, body)); + } + return forwardNamed(request, reply, name, body); + }, + ); + } + + /** + * From the hijack on, the raw response is the gateway's to finish: the + * node's answer is replayed, or the gateway's own 502/500 rendered, in + * the native dialect (errors.ts relay). `tail` is what the caller may + * do after a node that did not answer. + */ + const forwarded = ( + request: FastifyRequest, + reply: FastifyReply, + tail: string, + step: () => Promise, + ) => { + reply.hijack(); + return relay(reply.raw, 'native', request.log, step, (error) => ({ + status: 502, + message: `${error.message}${tail}`, + })); + }; + + /** A create that did not answer: the sandbox may exist; the retry asks the fleet and lands on the node that built it. */ + const RETRY_FINDS_IT = + ' — retry: if the sandbox was built, the node answers for it'; + + async function acquire( + request: FastifyRequest, + reply: FastifyReply, + name: string, + body: Buffer | undefined, + ) { + const judged = verdict(await finder.byName(name), `sandbox "${name}"`); + if (judged.kind === 'refuse') return refuse(reply, judged); + let target = judged.kind === 'node' ? judged.node : null; + if (target === null) { + if (clientGone(reply)) return; + const placement = place(fleet, knobs, new Date()); + if (placement.node === null) { + reply.header('retry-after', String(RETRY_AFTER_SECONDS)); + return reply.code(503).send({ message: refusalMessage(placement) }); + } + target = placement.node; + } + const chosen = target; + return forwarded(request, reply, RETRY_FINDS_IT, async () => { + const answer = await forwardCreate(finder.cache, request, reply.raw, { + target: chosen, + token, + name, + body, + face: NATIVE_CREATE, + }); + if (answer !== null) replay(reply.raw, answer); + }); + } + + async function destroy( + request: FastifyRequest, + reply: FastifyReply, + name: string, + body: Buffer | undefined, + ) { + const judged = verdict(await finder.byName(name), `sandbox "${name}"`); + if (judged.kind === 'refuse') return refuse(reply, judged); + // Nothing to forward: every node answered and none holds the name, + // which is exactly the daemon's own idempotent answer. + if (judged.kind === 'none') + return reply.code(200).send({ destroyed: false }); + const { node, id } = judged; + return forwarded(request, reply, ' — retry', async () => { + const answer = await forwardDestroy(finder.cache, request, reply.raw, { + target: node, + token, + entry: { id, name, nodeId: node.id }, + body, + credential: 'bearer', + }); + if (answer !== null) replay(reply.raw, answer); + }); + } + + /** + * Every other named verb: find the node, stream the answer through as + * it came — status, headers, body. A node's 404 is not "no such + * sandbox" by itself (readFile answers 404 for a missing path too), so + * it is relayed as it came and the cache entry is re-checked off the + * request path (find.ts verify): only a node that says "absent" to the + * one question that means it loses the entry. + */ + async function forwardNamed( + request: FastifyRequest, + reply: FastifyReply, + name: string, + body: Buffer | undefined, + ) { + const judged = verdict(await finder.byName(name), `sandbox "${name}"`); + if (judged.kind === 'refuse') return refuse(reply, judged); + if (judged.kind === 'none') { + return reply + .code(404) + .send({ message: `no sandbox named "${name}" — acquire it first` }); + } + const { node, id } = judged; + return forwarded(request, reply, ' — retry', async () => { + const status = await forwardStream(request.raw, reply.raw, { + target: { endpoint: node.endpoint, token }, + credential: 'bearer', + body, + }); + if (status === 404) { + void finder.verify({ id, name, nodeId: node.id }); + } + }); + } +}; + +/** The one field the gateway reads from a native body. */ +export function nameOf(body: Buffer | undefined): string | null { + const parsed = parseJson(body) as { name?: unknown } | undefined; + const name = parsed?.name; + return typeof name === 'string' && name.length > 0 ? name : null; +} diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts new file mode 100644 index 00000000..83ddd779 --- /dev/null +++ b/packages/gateway/src/routes/nodes.ts @@ -0,0 +1,94 @@ +import { + checkInRequestSchema, + checkInResponseSchema, + listNodesRequestSchema, + listNodesResponseSchema, + removeNodeRequestSchema, + removeNodeResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import type { NameCache } from '../cache'; +import { downReason, type Fleet } from '../fleet'; + +export interface NodeRoutesOptions { + fleet: Fleet; + cache: NameCache; +} + +/** + * The gateway's own verbs about its nodes: the check-in the nodes send + * (RULES/协议.md「网关」), and what an operator reads and does about them. + * Everything listNodes answers is what the gateway already holds, so + * answering costs no node anything. + */ +export const nodeRoutes: FastifyPluginAsyncZod = async ( + app, + { fleet, cache }, +) => { + app.post( + '/checkIn', + { + schema: { + body: checkInRequestSchema, + response: { 200: checkInResponseSchema }, + }, + }, + async (request) => { + const { node, joined } = fleet.checkIn(request.body); + if (joined) { + request.log.info( + { nodeId: node.id, endpoint: node.endpoint }, + 'a node checked in for the first time and joined the fleet', + ); + } + return {}; + }, + ); + + app.post( + '/listNodes', + { + schema: { + body: listNodesRequestSchema, + response: { 200: listNodesResponseSchema }, + }, + }, + async () => { + const now = new Date(); + return { + nodes: fleet.all().map((node) => ({ + id: node.id, + endpoint: node.endpoint, + addedAt: node.addedAt, + lastCheckInAt: node.lastCheckInAt?.toISOString() ?? null, + intervalSeconds: node.intervalSeconds, + reachable: downReason(node, now) === null, + build: node.build, + reading: node.reading, + placedSinceCheckIn: node.placedSinceCheckIn, + })), + }; + }, + ); + + app.post( + '/removeNode', + { + schema: { + body: removeNodeRequestSchema, + response: { 200: removeNodeResponseSchema }, + }, + }, + async (request) => { + const removed = fleet.remove(request.body.id); + const evicted = cache.evictNode(request.body.id); + if (removed) { + request.log.warn( + { nodeId: request.body.id, evicted }, + 'a node was removed by an operator: its sandboxes are no longer looked for, and names that lived only there are new names again', + ); + } + return { removed }; + }, + ); +}; diff --git a/packages/gateway/src/routes/verdict.ts b/packages/gateway/src/routes/verdict.ts new file mode 100644 index 00000000..e4636633 --- /dev/null +++ b/packages/gateway/src/routes/verdict.ts @@ -0,0 +1,62 @@ +import type { FastifyReply } from 'fastify'; +import type { Found } from '../find'; +import type { NodeState } from '../fleet'; +import { RETRY_AFTER_SECONDS } from '../raw'; + +/** + * A finding turned into a routing decision for an authenticated face: the + * node to forward to (with what the cache or the lookup knows of the + * sandbox), nothing (every node answered and none holds it — a create + * places, a destroy gives its idempotent answer, everything else a 404), + * or a refusal with its status and sentence, which the face sends in its + * own dialect. A value, so every verb reads the decision the same way. + */ +export type Verdict = + | { kind: 'node'; node: NodeState; id: string; name: string | null } + | { kind: 'none' } + | { + kind: 'refuse'; + status: number; + message: string; + retryAfterSeconds?: number; + }; + +export function verdict(found: Found, what: string): Verdict { + switch (found.kind) { + case 'one': + return { kind: 'node', node: found.node, id: found.id, name: found.name }; + case 'none': + return { kind: 'none' }; + case 'conflict': + // The gateway refuses every verb for this name with this very 409, + // destroy included — it will not guess which copy the caller means. + return { + kind: 'refuse', + status: 409, + message: `${what} exists on nodes ${found.nodeIds.join(' and ')} — destroy one copy directly on its node before routing can resume`, + }; + case 'unsure': + return { + kind: 'refuse', + status: 503, + message: `${what}: ${found.silent + .map((s) => `node ${s.nodeId} did not answer (${s.why})`) + .join( + ', ', + )} — it cannot be treated as new while a node that may hold it is silent; retry, or remove the node if it is gone for good`, + retryAfterSeconds: RETRY_AFTER_SECONDS, + }; + } +} + +/** Sends a refusal in the native dialect, with Retry-After where the verdict carries one. */ +export function refuse( + reply: FastifyReply, + refusal: Extract, + body: (message: string) => unknown = (message) => ({ message }), +) { + if (refusal.retryAfterSeconds !== undefined) { + reply.header('retry-after', String(refusal.retryAfterSeconds)); + } + return reply.code(refusal.status).send(body(refusal.message)); +} diff --git a/packages/gateway/src/testing.ts b/packages/gateway/src/testing.ts new file mode 100644 index 00000000..083a1be6 --- /dev/null +++ b/packages/gateway/src/testing.ts @@ -0,0 +1,60 @@ +import type { CheckInRequest, NodeReading } from '@dormice/shared'; + +/** + * Test scaffolding shared by the gateway's suites: a node's reading and + * check-in with a few knobs turned. Not shipped — nothing under src/ but + * main.ts is bundled (tsup.config.ts). + */ +export function reading( + over: { + cpu?: number | null; + cores?: number; + active?: number; + frozen?: number; + memAvail?: number; + diskAvail?: number | null; + } = {}, +): NodeReading { + const frozen = over.frozen ?? 0; + const active = over.active ?? 10; + return { + host: { + cpuCount: over.cores ?? 8, + cpuUsedPct: over.cpu === undefined ? 10 : over.cpu, + memTotalBytes: 32e9, + memAvailableBytes: over.memAvail ?? 16e9, + swap: null, + }, + dataDisk: + over.diskAvail === null + ? null + : { + path: '/var/lib/dormice', + totalBytes: 1e12, + usedBytes: 1e12 - (over.diskAvail ?? 5e11), + availableBytes: over.diskAvail ?? 5e11, + }, + sandboxes: { + total: active + frozen, + byState: { active, frozen, stopped: 0, archived: 0, restoring: 0 }, + }, + }; +} + +export function checkInOf( + nodeId: string, + endpoint: string, + over: Parameters[0] & { intervalSeconds?: number } = {}, +): CheckInRequest { + return { + nodeId, + endpoint, + intervalSeconds: over.intervalSeconds ?? 15, + build: { + commit: 'abc1234', + title: 'a commit', + committedAt: '2026-09-14T00:00:00.000Z', + }, + reading: reading(over), + }; +} diff --git a/packages/gateway/src/version.ts b/packages/gateway/src/version.ts new file mode 100644 index 00000000..62ed4335 --- /dev/null +++ b/packages/gateway/src/version.ts @@ -0,0 +1,23 @@ +/** + * The identity tsup baked into this build (tsup.config.ts): the commit the + * dist was built from. In the built daemon these `process.env` reads are + * compile-time literals; running from source (tests, tsx) they fall + * through to the real environment and come back empty — an unbuilt tree + * has no build identity, and null says so honestly. + */ +export interface BuildInfo { + /** Short hash. */ + commit: string; + /** The commit's subject line. */ + title: string; + /** ISO 8601 UTC — the commit's time, not the build's. */ + committedAt: string; +} + +export function readBuildInfo(): BuildInfo | null { + const commit = process.env.DORMICE_BUILD_COMMIT; + const title = process.env.DORMICE_BUILD_COMMIT_TITLE; + const committedAt = process.env.DORMICE_BUILD_COMMIT_AT; + if (!commit || !title || !committedAt) return null; + return { commit, title, committedAt }; +} diff --git a/packages/gateway/tsconfig.json b/packages/gateway/tsconfig.json new file mode 100644 index 00000000..564a5990 --- /dev/null +++ b/packages/gateway/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src"] +} diff --git a/packages/gateway/tsup.config.ts b/packages/gateway/tsup.config.ts new file mode 100644 index 00000000..d607d033 --- /dev/null +++ b/packages/gateway/tsup.config.ts @@ -0,0 +1,38 @@ +import { execSync } from 'node:child_process'; +import { defineConfig } from 'tsup'; + +/** + * Same build identity as the daemon (packages/server/tsup.config.ts has + * the reasoning): the commit the dist was built from, baked in as + * literals that src/version.ts reads back — what the process IS, not what + * the checkout says after a pull. + */ +function git(args: string): string { + try { + return execSync(`git ${args}`, { + stdio: ['ignore', 'pipe', 'ignore'], + }) + .toString() + .trim(); + } catch { + return ''; + } +} + +const commitTime = git('log -1 --format=%cI'); + +export default defineConfig({ + // A service, not a library: main.ts is the one entry and nothing imports + // the package (e2e boots dist/main.js as a process), so no index and no + // declarations. + entry: ['src/main.ts'], + format: ['esm'], + clean: true, + env: { + DORMICE_BUILD_COMMIT: git('rev-parse --short HEAD'), + DORMICE_BUILD_COMMIT_TITLE: git('log -1 --format=%s'), + DORMICE_BUILD_COMMIT_AT: commitTime + ? new Date(commitTime).toISOString() + : '', + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3eeb1852..44238dfe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -177,6 +177,43 @@ importers: specifier: ^8.1.3 version: 8.1.3(@types/node@26.1.0)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.0)(yaml@2.9.0) + packages/gateway: + dependencies: + '@dormice/server': + specifier: workspace:* + version: link:../server + '@dormice/shared': + specifier: workspace:* + version: link:../shared + better-sqlite3: + specifier: ^12.11.1 + version: 12.11.1 + drizzle-orm: + specifier: ^0.45.2 + version: 0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.11.1)(kysely@0.28.17) + fastify: + specifier: ^5.10.0 + version: 5.10.0 + fastify-type-provider-zod: + specifier: ^7.0.0 + version: 7.0.0(@fastify/swagger@9.7.0)(fastify@5.10.0)(openapi-types@12.1.3)(zod@4.4.3) + pino: + specifier: ^10.3.1 + version: 10.3.1 + undici: + specifier: ^8.7.0 + version: 8.7.0 + zod: + specifier: ^4.4.3 + version: 4.4.3 + devDependencies: + '@types/better-sqlite3': + specifier: ^7.6.13 + version: 7.6.13 + drizzle-kit: + specifier: ^0.31.10 + version: 0.31.10 + packages/sdk: dependencies: '@dormice/shared': From 7b560eff7024c461d2cda9a3c8a1a32c41fe2df8 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:48:51 +0800 Subject: [PATCH 06/26] A sandbox placed and destroyed inside one check-in interval no longer holds a placement slot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The placement counter exists so a burst inside one interval is counted against a node before its next reading shows it. It was only ever reset by the reading, so a sandbox placed and destroyed within the same interval — a short job, or the exam's churn — kept holding a slot the reading would never show; with the exam's two-per-node gate that shut placement for a whole interval after a handful of fast tests. Each node now remembers the ids its creates answered with since the last reading; a destroy of one of them takes the placement off the count, and a create the node itself refused is uncounted at once. A hop's 502/503/504, or no answer, still counts until the reading: the sandbox may exist. --- packages/gateway/src/app.test.ts | 7 +++++- packages/gateway/src/fleet.ts | 9 ++++++- packages/gateway/src/routes/create.ts | 34 ++++++++++++++++++-------- packages/gateway/src/routes/destroy.ts | 11 +++++++-- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index cce3d7a0..63e03031 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -62,11 +62,13 @@ class FakeNode { return [...this.sandboxes.values()].find((s) => s.id === id); } + // Inferred return type on purpose: the early `return json(...)` exits + // read as statements, and an explicit void would flag each one. private answer( req: http.IncomingMessage, res: http.ServerResponse, text: string, - ): void { + ) { const url = req.url ?? '/'; const path = url.split('?')[0] ?? url; const auth = @@ -569,6 +571,9 @@ describe('using, destroying, and the cache', () => { }); expect(h.cache.getByName('d')).toBeUndefined(); expect(h.cache.getById(first.id)).toBeUndefined(); + // Placed and destroyed inside one interval: the placement no longer + // counts against the node — the reading will never show it. + expect(h.fleet.get('a')?.placedSinceCheckIn).toBe(0); expect((await rpc(h, '/destroySandbox', { name: 'd' })).body).toEqual({ destroyed: false, }); diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 42967ada..1e3611d1 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -9,7 +9,10 @@ import { nodes } from './db/schema'; * check-in, gone with the process and rightly so). placedSinceCheckIn * counts the sandboxes the gateway sent here since the last reading, so a * burst inside one interval is counted against the node before its next - * reading shows it (placement.ts). + * reading shows it (placement.ts); placedIds are the ones among them whose + * create answered with an id, so a sandbox placed and destroyed inside one + * interval — a short job, the exam's churn — is taken off the count again + * instead of holding a slot the reading will never show (routes/destroy.ts). */ export interface NodeState { readonly id: string; @@ -20,6 +23,7 @@ export interface NodeState { build: BuildInfo | null; reading: NodeReading | null; placedSinceCheckIn: number; + placedIds: Set; } /** @@ -61,6 +65,7 @@ export class Fleet { build: null, reading: null, placedSinceCheckIn: 0, + placedIds: new Set(), }); } } @@ -104,6 +109,7 @@ export class Fleet { build: null, reading: null, placedSinceCheckIn: 0, + placedIds: new Set(), }; this.members.set(node.id, node); joined = true; @@ -120,6 +126,7 @@ export class Fleet { node.build = report.build; node.reading = report.reading; node.placedSinceCheckIn = 0; + node.placedIds.clear(); return { node, joined }; } diff --git a/packages/gateway/src/routes/create.ts b/packages/gateway/src/routes/create.ts index f6e3d174..8448458c 100644 --- a/packages/gateway/src/routes/create.ts +++ b/packages/gateway/src/routes/create.ts @@ -79,15 +79,19 @@ export interface CreateOptions { /** * Forwards a create and learns from the node's answer: a 2xx with a * readable id goes into the cache (name and id → this node), so the next - * request for the sandbox skips the round of questions. A 2xx without a - * readable id is logged — the sandbox exists on the node and the next - * lookup finds it there, never silently. Any other answer teaches - * nothing: a 4xx or the node's own 500 means the node holds nothing under - * the name and the next attempt may be placed elsewhere; a 502/503/504 - * from a hop in front of the node, or no answer at all, leaves the - * question to the next lookup — the node that may have built it answers - * from inside the name's slot. + * request for the sandbox skips the round of questions, and into the + * node's placedIds, so a destroy inside the same interval can take the + * placement off the count again. A 2xx without a readable id is logged — + * the sandbox exists on the node and the next lookup finds it there, + * never silently. A 4xx or the node's own 500 means the node built + * nothing: the placement is taken off the count at once (it would have + * held a slot for a whole interval otherwise) and the next attempt may be + * placed elsewhere. A 502/503/504 from a hop in front of the node, or no + * answer at all, leaves the count as it is and the question to the next + * lookup — the node that may have built it answers from inside the + * name's slot. */ +const HOP_STATUSES = new Set([502, 503, 504]); export async function forwardCreate( cache: NameCache, request: FastifyRequest, @@ -104,16 +108,26 @@ export async function forwardCreate( const id = face.idOf(parseJson(answer.body)); if (id !== null) { cache.put({ id, name, nodeId: target.id }); + target.placedIds.add(id); } else { request.log.warn( { node: target.id, name, status: answer.status }, 'create answered 2xx without a readable sandbox id; the next lookup finds it on the node', ); } - } else if (answer.status >= 500) { + } else if (!HOP_STATUSES.has(answer.status)) { + // The node itself answered no: nothing was built there. + target.placedSinceCheckIn = Math.max(0, target.placedSinceCheckIn - 1); + if (answer.status >= 500) { + request.log.warn( + { node: target.id, name, status: answer.status }, + 'the node itself failed the create; nothing cached, the placement is uncounted', + ); + } + } else { request.log.warn( { node: target.id, name, status: answer.status }, - 'create was refused by the node or a hop in front of it; nothing cached', + 'a hop in front of the node answered the create; nothing cached, the placement stays counted until the next reading', ); } return answer; diff --git a/packages/gateway/src/routes/destroy.ts b/packages/gateway/src/routes/destroy.ts index 189105c3..e501a489 100644 --- a/packages/gateway/src/routes/destroy.ts +++ b/packages/gateway/src/routes/destroy.ts @@ -20,7 +20,9 @@ export interface DestroyOptions { /** * The one destroy path, the mirror of create.ts: the native destroySandbox * and the E2B kill both forward, wait for the node's whole answer and - * learn from a 2xx — the cache entry goes. Only the node's yes teaches + * learn from a 2xx — the cache entry goes, and a sandbox placed since the + * node's last reading comes off its placement count (fleet.ts placedIds). + * Only the node's yes teaches * anything: its 404 is not "gone" (the daemon answers 404 for a sandbox * past its kill deadline that its scanner has not torn down yet, a row a * lookup would still answer yes for), and a 5xx or no answer leaves the @@ -39,6 +41,11 @@ export async function forwardDestroy( body, }); if (answer === null) return null; - if (answer.status >= 200 && answer.status < 300) cache.evict(entry); + if (answer.status >= 200 && answer.status < 300) { + cache.evict(entry); + if (target.placedIds.delete(entry.id)) { + target.placedSinceCheckIn = Math.max(0, target.placedSinceCheckIn - 1); + } + } return answer; } From 2fc3ea9f4909d7a252959013bb25418284d82a32 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:48:51 +0800 Subject: [PATCH 07/26] =?UTF-8?q?e2e:=20the=20gateway=20exam=20=E2=80=94?= =?UTF-8?q?=20two=20daemons=20behind=20a=20real=20gateway,=20a=20third=20t?= =?UTF-8?q?hat=20joins=20and=20dies,=20all=20driven=20over=20the=20wire?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The setup boots node A exactly as before (every existing suite is untouched) and, outside docker mode, a gateway plus nodes B and C that share its token and check in every second. The exam speaks only the SDK, the official e2b package and plain fetch; direct calls to a node stage what the gateway must then find: a sandbox built behind its back is routed at once (no reconcile to wait for), a name on two nodes is a 409 naming both, a destroy behind its back is caught by the 404 re-check. Placement, the active gate and its Retry-After, five simultaneous acquires, streaming through two hops, envd and the bare signed door are graded; a node booted by the test itself joins at its first check-in, and when it dies its sandboxes 502 and a new name is a 503 naming it until removeNode. A second gateway on the same database file dies at boot naming the conflict, as does a second daemon on node A's ledger. --- e2e/package.json | 1 + e2e/src/gateway.test.ts | 559 ++++++++++++++++++++++++++++++++++++++++ e2e/src/native.test.ts | 37 +++ e2e/src/setup/daemon.ts | 281 +++++++++++++++----- pnpm-lock.yaml | 3 + 5 files changed, 813 insertions(+), 68 deletions(-) create mode 100644 e2e/src/gateway.test.ts diff --git a/e2e/package.json b/e2e/package.json index 9d104a0d..7c574ba9 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -14,6 +14,7 @@ "e2b": "^2.31.0" }, "devDependencies": { + "@dormice/gateway": "workspace:*", "@dormice/server": "workspace:*" } } diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts new file mode 100644 index 00000000..31c2f481 --- /dev/null +++ b/e2e/src/gateway.test.ts @@ -0,0 +1,559 @@ +import { spawn } from 'node:child_process'; +import { randomBytes, randomUUID } from 'node:crypto'; +import { mkdtemp, rm } from 'node:fs/promises'; +import net, { type AddressInfo } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Dormice } from '@dormice/sdk'; +import { Sandbox } from 'e2b'; +import { describe, expect, inject, it } from 'vitest'; + +// The gateway exam: two real daemons behind a real gateway, all three +// booted the production way and driven only over the wire — the SDK, the +// official e2b package and plain fetch. Direct calls to a node exist only +// to stage what the gateway must then find (a sandbox built behind its +// back, a name on two nodes, a destroy it did not see). Skipped in docker +// mode, where the setup boots node A alone. +const skip = inject('dormiceGatewayEndpoint') === null; + +const gateway = () => inject('dormiceGatewayEndpoint') as string; +const token = () => inject('dormiceGatewayToken') as string; +const nodes = () => + inject('dormiceGatewayNodes') as Array<{ id: string; endpoint: string }>; +const viaGateway = () => new Dormice({ endpoint: gateway(), token: token() }); +function direct(id: string) { + const node = nodes().find((n) => n.id === id); + if (!node) throw new Error(`no node ${id} in the exam fleet`); + return new Dormice({ endpoint: node.endpoint, token: token() }); +} +const other = (id: string) => (id === 'node-b' ? 'node-c' : 'node-b'); + +/** Polls until the probe answers something — nodes check in on their own clock, not ours. */ +async function until( + probe: () => Promise, + timeoutMs = 10_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + const value = await probe(); + if (value !== undefined) return value; + if (Date.now() > deadline) throw new Error('condition never became true'); + await new Promise((resolve) => setTimeout(resolve, 100)); + } +} + +async function rpc( + path: string, + payload: unknown = {}, + bearer = token(), + endpoint = gateway(), +): Promise<{ status: number; body: unknown; headers: Headers }> { + const res = await fetch(`${endpoint}${path}`, { + method: 'POST', + headers: { + authorization: `Bearer ${bearer}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(payload), + }); + const text = await res.text(); + return { + status: res.status, + body: text ? JSON.parse(text) : null, + headers: res.headers, + }; +} + +interface ListedNode { + id: string; + endpoint: string; + reachable: boolean; + lastCheckInAt: string | null; + build: { commit: string } | null; + reading: { sandboxes: { byState: { active: number } } } | null; + placedSinceCheckIn: number; +} +async function listNodes(): Promise { + const { status, body } = await rpc('/listNodes'); + expect(status).toBe(200); + return (body as { nodes: ListedNode[] }).nodes; +} + +const status = (error: unknown) => (error as { status?: number }).status; +const message = (r: { body: unknown }) => + (r.body as { message: string }).message; + +describe.skipIf(skip)('the gateway in front of two daemons', () => { + it('/healthz is open and names the build; a wrong token is refused', async () => { + const health = (await (await fetch(`${gateway()}/healthz`)).json()) as { + status: string; + build: { commit: string } | null; + }; + expect(health.status).toBe('ok'); + expect(health.build?.commit).toMatch(/^[0-9a-f]{7,}$/); + expect((await rpc('/listNodes', {}, 'x'.repeat(64))).status).toBe(401); + }); + + it('both nodes checked in: reachable, with a reading and the build they run', async () => { + const listed = await until(async () => { + const seen = await listNodes(); + return seen.length === 2 && seen.every((n) => n.reachable) + ? seen + : undefined; + }); + expect(listed.map((n) => n.id).sort()).toEqual(['node-b', 'node-c']); + for (const node of listed) { + expect(node.reading?.sandboxes.byState.active).toBeGreaterThanOrEqual(0); + expect(node.build?.commit).toMatch(/^[0-9a-f]{7,}$/); + expect(node.endpoint).toBe( + nodes().find((n) => n.id === node.id)?.endpoint, + ); + } + }); + + it('acquire is placed on one node and lands in exactly one ledger; re-acquire finds it there', async () => { + const created = await viaGateway().acquireSandbox('gw-place'); + try { + expect(created.status).toBe('ready'); + expect(created.created).toBe(true); + expect(['node-b', 'node-c']).toContain(created.sandbox.nodeId); + const here = await direct(created.sandbox.nodeId).listSandboxes(); + expect(here.some((s) => s.id === created.sandbox.id)).toBe(true); + const there = await direct(other(created.sandbox.nodeId)).listSandboxes(); + expect(there.some((s) => s.name === 'gw-place')).toBe(false); + + const again = await viaGateway().acquireSandbox('gw-place'); + expect(again.created).toBe(false); + expect(again.sandbox.id).toBe(created.sandbox.id); + expect(again.sandbox.nodeId).toBe(created.sandbox.nodeId); + } finally { + await viaGateway().destroySandbox('gw-place'); + } + }); + + it("files and commands round-trip; a node's 404 for a missing file passes through and the sandbox stays routable", async () => { + await viaGateway().acquireSandbox('gw-work'); + try { + await viaGateway().writeFile('gw-work', 'hello.txt', 'via gateway'); + const read = await viaGateway().readFile('gw-work', 'hello.txt'); + expect(new TextDecoder().decode(read.content)).toBe('via gateway'); + const ran = await viaGateway().execCommand('gw-work', 'echo through'); + expect(ran.stdout).toBe('through\n'); + + await expect( + viaGateway().readFile('gw-work', 'missing.txt'), + ).rejects.toMatchObject({ name: 'DormiceApiError', status: 404 }); + expect( + (await viaGateway().execCommand('gw-work', 'echo still-here')).stdout, + ).toBe('still-here\n'); + } finally { + await viaGateway().destroySandbox('gw-work'); + } + }); + + it('five simultaneous acquires of a new name are one sandbox on one node', async () => { + const results = await Promise.all( + Array.from({ length: 5 }, () => viaGateway().acquireSandbox('gw-burst')), + ); + try { + expect(new Set(results.map((r) => r.sandbox.id)).size).toBe(1); + expect(new Set(results.map((r) => r.sandbox.nodeId)).size).toBe(1); + expect(results.filter((r) => r.created)).toHaveLength(1); + } finally { + await viaGateway().destroySandbox('gw-burst'); + } + }); + + it('a sandbox built behind its back is found by asking — at once, no reconcile to wait for', async () => { + const staged = await direct('node-b').acquireSandbox('gw-staged'); + try { + const ran = await viaGateway().execCommand('gw-staged', 'echo found'); + expect(ran.stdout).toBe('found\n'); + const found = await viaGateway().acquireSandbox('gw-staged'); + expect(found.created).toBe(false); + expect(found.sandbox.id).toBe(staged.sandbox.id); + expect(found.sandbox.nodeId).toBe('node-b'); + } finally { + await viaGateway().destroySandbox('gw-staged'); + } + }); + + it('one name on two nodes is refused with a 409 naming both, and routable again once one copy is destroyed', async () => { + await direct('node-b').acquireSandbox('gw-twin'); + await direct('node-c').acquireSandbox('gw-twin'); + try { + await expect( + viaGateway().acquireSandbox('gw-twin'), + ).rejects.toMatchObject({ + status: 409, + message: expect.stringMatching(/node-b and node-c/), + }); + await expect( + viaGateway().execCommand('gw-twin', 'true'), + ).rejects.toMatchObject({ status: 409 }); + await direct('node-c').destroySandbox('gw-twin'); + const healed = await viaGateway().acquireSandbox('gw-twin'); + expect(healed.created).toBe(false); + expect(healed.sandbox.nodeId).toBe('node-b'); + } finally { + await direct('node-b').destroySandbox('gw-twin'); + await direct('node-c').destroySandbox('gw-twin'); + } + }); + + it('a destroy through the gateway is final; a destroy behind its back is caught on the next use and the name is asked for afresh', async () => { + const created = await viaGateway().acquireSandbox('gw-gone'); + expect(await viaGateway().destroySandbox('gw-gone')).toEqual({ + destroyed: true, + }); + await expect( + viaGateway().execCommand('gw-gone', 'true'), + ).rejects.toMatchObject({ + status: 404, + message: expect.stringMatching(/no sandbox named "gw-gone"/), + }); + expect(await viaGateway().destroySandbox('gw-gone')).toEqual({ + destroyed: false, + }); + const reborn = await viaGateway().acquireSandbox('gw-gone'); + expect(reborn.created).toBe(true); + expect(reborn.sandbox.id).not.toBe(created.sandbox.id); + try { + // Destroyed directly on its node, then rebuilt directly on the other: + // the gateway's first use relays the node's own 404, re-checks the + // cache off the request path, and the next use finds the new home. + // Without the re-check every use would keep going to the old node. + await direct(reborn.sandbox.nodeId).destroySandbox('gw-gone'); + await expect( + viaGateway().execCommand('gw-gone', 'true'), + ).rejects.toMatchObject({ status: 404 }); + const moved = await direct(other(reborn.sandbox.nodeId)).acquireSandbox( + 'gw-gone', + ); + await until(async () => { + try { + return await viaGateway().execCommand('gw-gone', 'echo moved'); + } catch (error) { + if (status(error) === 404) return undefined; + throw error; + } + }); + const found = await viaGateway().acquireSandbox('gw-gone'); + expect(found.created).toBe(false); + expect(found.sandbox.id).toBe(moved.sandbox.id); + } finally { + await viaGateway().destroySandbox('gw-gone'); + } + }); + + it('the active gate (2 per node here) refuses with a 503 naming every node, and reopens when a sandbox is destroyed', async () => { + const baseline = new Map( + (await listNodes()).map((n) => [ + n.id, + n.reading?.sandboxes.byState.active ?? 0, + ]), + ); + const names: string[] = []; + try { + const refusal = await until(async () => { + if (names.length > 8) throw new Error('gate never closed'); + const name = `gw-fill-${names.length}`; + const answer = await rpc('/acquireSandbox', { name }); + if (answer.status === 200) { + names.push(name); + return undefined; + } + return answer.status === 503 ? answer : undefined; + }, 20_000); + expect(message(refusal)).toContain('node-b'); + expect(message(refusal)).toContain('node-c'); + expect(message(refusal)).toContain('reach the 2 limit'); + expect(refusal.headers.get('retry-after')).toBe('15'); + + const freed = names.pop(); + if (freed) await viaGateway().destroySandbox(freed); + const reopened = await until(async () => { + try { + return await viaGateway().acquireSandbox('gw-fill-late'); + } catch (error) { + if (status(error) === 503) return undefined; + throw error; + } + }); + names.push('gw-fill-late'); + expect(reopened.status).toBe('ready'); + } finally { + for (const name of names) await viaGateway().destroySandbox(name); + // Leave the fleet as found: the gate reads the nodes' readings, which + // catch up with the destroys at their next check-in — the next test + // must not inherit a closed gate. + await until(async () => + (await listNodes()).every( + (n) => + n.reading?.sandboxes.byState.active === baseline.get(n.id) && + n.placedSinceCheckIn === 0, + ) + ? true + : undefined, + ); + } + }); + + it('daemon-addressed verbs are an honest 501; a misspelled verb is a 404', async () => { + await expect(viaGateway().listSandboxes()).rejects.toMatchObject({ + status: 501, + message: expect.stringMatching(/call the node directly/), + }); + const e2bList = await fetch(`${gateway()}/e2b/api/v2/sandboxes`, { + headers: { 'x-api-key': `e2b_${token()}` }, + }); + expect(e2bList.status).toBe(501); + expect(((await e2bList.json()) as { code: number }).code).toBe(501); + expect((await rpc('/acquireSandbx', { name: 'x' })).status).toBe(404); + }); + + it('the official e2b package works through the gateway: create, live streaming, files, kill; an unnamed create routes by id', async () => { + const connection = { + apiKey: `e2b_${token()}`, + apiUrl: `${gateway()}/e2b/api`, + sandboxUrl: `${gateway()}/e2b/envd`, + }; + const sbx = await Sandbox.create({ + ...connection, + metadata: { name: 'gw-e2b' }, + }); + try { + const chunks: Array<{ text: string; at: number }> = []; + const result = await sbx.commands.run( + 'echo first; sleep 1; echo second', + { + onStdout: (text) => { + chunks.push({ text, at: Date.now() }); + }, + }, + ); + expect(result.stdout).toBe('first\nsecond\n'); + // Streaming through two hops: a real gap between the frames, as in + // e2b.test.ts against the daemon alone — a buffering hop would + // deliver both at once. + const at = chunks.map((c) => c.at); + expect(at.length).toBeGreaterThanOrEqual(2); + expect((at.at(-1) ?? 0) - (at[0] ?? 0)).toBeGreaterThanOrEqual(500); + + await sbx.files.write('/home/user/gateway.txt', 'through the gateway'); + expect(await sbx.files.read('/home/user/gateway.txt')).toBe( + 'through the gateway', + ); + // The same name through the native face is the same sandbox. + const found = await viaGateway().acquireSandbox('gw-e2b'); + expect(found.created).toBe(false); + expect(found.sandbox.id).toBe(sbx.sandboxId); + } finally { + await sbx.kill(); + } + expect(await viaGateway().destroySandbox('gw-e2b')).toEqual({ + destroyed: false, + }); + + const anonymous = await Sandbox.create(connection); + try { + const info = await fetch( + `${connection.apiUrl}/sandboxes/${anonymous.sandboxId}`, + { headers: { 'x-api-key': connection.apiKey } }, + ); + expect(info.status).toBe(200); + } finally { + await anonymous.kill(); + } + }); + + it('envd preflights are answered without a header; the bare signed-URL door is a 501 with CORS', async () => { + const preflight = await fetch(`${gateway()}/e2b/envd/files`, { + method: 'OPTIONS', + headers: { origin: 'https://app.example' }, + }); + expect(preflight.status).toBe(204); + expect(preflight.headers.get('access-control-allow-origin')).toBe('*'); + const bare = await fetch(`${gateway()}/files?signature=x&path=/y`); + expect(bare.status).toBe(501); + expect(bare.headers.get('access-control-allow-origin')).toBe('*'); + expect(((await bare.json()) as { code: string }).code).toBe( + 'unimplemented', + ); + const stranger = await fetch(`${gateway()}/e2b/envd/files`, { + headers: { 'e2b-sandbox-id': randomUUID() }, + }); + expect(stranger.status).toBe(502); + expect(((await stranger.json()) as { code: string }).code).toBe( + 'unavailable', + ); + }); + + it('a third node joins at its first check-in; when it dies its sandboxes 502 and new names are a 503 naming it, until an operator removes it', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dormice-e2e-node-d-')); + const port = await freePort(); + const child = spawn('node', [inject('dormiceDaemonMain')], { + env: { + PATH: process.env.PATH ?? '', + DORMICE_NODE_ID: 'node-d', + DORMICE_PORT: String(port), + DORMICE_DATA_DIR: dir, + DORMICE_DB_PATH: join(dir, 'dormice.db'), + DORMICE_API_TOKEN: token(), + DORMICE_GATEWAY_ENDPOINT: gateway(), + DORMICE_CHECK_IN_INTERVAL_SECONDS: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stderr.on('data', (chunk) => { + output += chunk; + }); + const exited = new Promise((resolve) => + child.on('exit', () => resolve()), + ); + const d = new Dormice({ + endpoint: `http://127.0.0.1:${port}`, + token: token(), + }); + try { + await until(async () => + (await listNodes()).some((n) => n.id === 'node-d' && n.reachable) + ? true + : undefined, + ).catch((error) => { + throw new Error(`${String(error)}\n${output}`); + }); + const staged = await d.acquireSandbox('gw-on-d'); + expect( + (await viaGateway().execCommand('gw-on-d', 'echo on-d')).stdout, + ).toBe('on-d\n'); + + child.kill(); + await exited; + await expect( + viaGateway().execCommand('gw-on-d', 'true'), + ).rejects.toMatchObject({ + status: 502, + message: expect.stringMatching(/did not answer/), + }); + const refused = await rpc('/acquireSandbox', { name: 'gw-while-d-down' }); + expect(refused.status).toBe(503); + expect(message(refused)).toContain('node node-d did not answer'); + expect(refused.headers.get('retry-after')).toBe('15'); + // Down is also what listNodes says, on the node's own interval. + await until(async () => + (await listNodes()).some((n) => n.id === 'node-d' && !n.reachable) + ? true + : undefined, + ); + + expect((await rpc('/removeNode', { id: 'node-d' })).body).toEqual({ + removed: true, + }); + expect((await listNodes()).some((n) => n.id === 'node-d')).toBe(false); + const placed = await viaGateway().acquireSandbox('gw-while-d-down'); + try { + expect(['node-b', 'node-c']).toContain(placed.sandbox.nodeId); + } finally { + await viaGateway().destroySandbox('gw-while-d-down'); + } + expect(staged.sandbox.nodeId).toBe('node-d'); + } finally { + child.kill(); + await exited; + await rm(dir, { recursive: true, force: true }); + } + }); + + it("a gateway's boot, black-box: a second gateway on the same database file dies naming the conflict", async () => { + const dir = await mkdtemp(join(tmpdir(), 'dormice-e2e-gateway-boot-')); + try { + const env = async () => ({ + PATH: process.env.PATH ?? '', + DORMICE_GATEWAY_PORT: String(await freePort()), + DORMICE_GATEWAY_DB_PATH: join(dir, 'gateway.db'), + DORMICE_API_TOKEN: randomBytes(32).toString('hex'), + }); + const first = await bootGateway(await env()); + expect(first.outcome, first.output).toBe('healthy'); + try { + const second = await bootGateway(await env()); + expect(second.outcome, second.output).toEqual({ exitCode: 1 }); + expect(second.output).toMatch( + /another gateway is already running against .*gateway\.db/, + ); + expect(second.output).toContain('DORMICE_GATEWAY_DB_PATH'); + } finally { + await first.kill(); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); + +/** A port the OS just handed out and released — for a process this test boots itself. */ +function freePort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + const { port } = server.address() as AddressInfo; + server.close(() => resolve(port)); + }); + }); +} + +/** + * Boots a gateway the production way (`node dist/main.js` + environment) + * and reports how the boot ended: healthy at its port, or the exit code + * it died with — its whole output kept for the assertion either way. + */ +async function bootGateway(env: Record): Promise<{ + outcome: 'healthy' | { exitCode: number | null }; + output: string; + kill: () => Promise; +}> { + const child = spawn('node', [inject('dormiceGatewayMain')], { + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stderr.on('data', (chunk) => { + output += chunk; + }); + let exited: number | null | undefined; + const exit = new Promise((resolve) => { + child.on('exit', (code) => { + exited = code; + resolve(); + }); + }); + const endpoint = `http://127.0.0.1:${env.DORMICE_GATEWAY_PORT}`; + const deadline = Date.now() + 15_000; + while (Date.now() < deadline) { + if (exited !== undefined) { + return { outcome: { exitCode: exited }, output, kill: async () => {} }; + } + const healthy = await fetch(`${endpoint}/healthz`) + .then((r) => r.ok) + .catch(() => false); + if (healthy) break; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + return { + outcome: exited === undefined ? 'healthy' : { exitCode: exited }, + output, + kill: async () => { + child.kill(); + await exit; + }, + }; +} diff --git a/e2e/src/native.test.ts b/e2e/src/native.test.ts index 65635d55..cd0c2b48 100644 --- a/e2e/src/native.test.ts +++ b/e2e/src/native.test.ts @@ -1,3 +1,4 @@ +import { spawn } from 'node:child_process'; import { DEFAULT_LIFECYCLE_POLICY, Dormice } from '@dormice/sdk'; import { describe, expect, inject, it } from 'vitest'; @@ -111,6 +112,42 @@ describe('native API over a real daemon', () => { ).resolves.toBe(400); }); + it('a second daemon on the same ledger dies at boot naming the conflict — one ledger, one daemon', async () => { + // The same environment as the running daemon, another port: the port + // is not what must refuse it, the ledger lock is (measured 2026-09-11: + // with the lock handle garbage-collected, this second daemon started). + const child = spawn('node', [inject('dormiceDaemonMain')], { + env: { + ...inject('dormiceNodeAEnv'), + DORMICE_PORT: String( + Number(new URL(inject('dormiceEndpoint')).port) + 1000, + ), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stderr.on('data', (chunk) => { + output += chunk; + }); + const code = await new Promise((resolve) => { + const timer = setTimeout(() => { + child.kill(); + resolve(null); + }, 10_000); + child.on('exit', (exitCode) => { + clearTimeout(timer); + resolve(exitCode); + }); + }); + expect(code, output).toBe(1); + expect(output).toMatch( + /another daemon is already running against .*dormice\.db/, + ); + }); + it('destroys a sandbox: gone, forgotten, idempotent', async () => { const created = await client().acquireSandbox('destroy-key'); diff --git a/e2e/src/setup/daemon.ts b/e2e/src/setup/daemon.ts index 075aa6c9..41b11a4c 100644 --- a/e2e/src/setup/daemon.ts +++ b/e2e/src/setup/daemon.ts @@ -1,4 +1,4 @@ -import { spawn } from 'node:child_process'; +import { type ChildProcess, spawn } from 'node:child_process'; import { randomBytes } from 'node:crypto'; import { existsSync } from 'node:fs'; import { mkdtemp, rm } from 'node:fs/promises'; @@ -8,12 +8,31 @@ import { fileURLToPath } from 'node:url'; import { startMiniS3 } from '@dormice/server/mini-s3'; import type { TestProject } from 'vitest/node'; +export interface FleetNodeHandle { + id: string; + endpoint: string; +} + declare module 'vitest' { export interface ProvidedContext { dormiceEndpoint: string; dormiceToken: string; /** The Caddy config file the exam daemon owns — an operator-visible artifact. */ dormiceIngressFile: string; + /** The built daemon entry, for tests that boot a daemon of their own. */ + dormiceDaemonMain: string; + /** Node A's exact environment — a second daemon on the same ledger must refuse to start. */ + dormiceNodeAEnv: Record; + /** The gateway fronting nodes B and C; null in docker mode, where only node A runs. */ + dormiceGatewayEndpoint: string | null; + /** The fleet's one token: the gateway's, and every fronted node's. */ + dormiceGatewayToken: string | null; + /** The fronted nodes, reachable directly — to stage what the gateway must then find. */ + dormiceGatewayNodes: FleetNodeHandle[] | null; + /** The exam's S3, for a node a test boots itself. */ + dormiceMiniS3Url: string; + /** The built gateway entry, for tests that boot a gateway of their own (its boot refusals). */ + dormiceGatewayMain: string; } } @@ -25,26 +44,23 @@ declare module 'vitest' { const MAIN = fileURLToPath( new URL('../../../packages/server/dist/main.js', import.meta.url), ); +const GATEWAY_MAIN = fileURLToPath( + new URL('../../../packages/gateway/dist/main.js', import.meta.url), +); -export default async function setup(project: TestProject) { - if (!existsSync(MAIN)) { - throw new Error( - `daemon build not found at ${MAIN} — run \`pnpm build\` first`, - ); - } - - const token = randomBytes(32).toString('hex'); - const dataDir = await mkdtemp(join(tmpdir(), 'dormice-e2e-')); - // Random high port: never collides with a locally running daemon on 3676. - const port = 20000 + Math.floor(Math.random() * 20000); - const endpoint = `http://127.0.0.1:${port}`; - - // The exam's own S3 (in-process, test-only): the archive lifecycle runs - // black-box in every mode — the fake executor exports real files, the - // store speaks real HTTP. It also mimics OSS's checksum strictness, so a - // daemon that regressed pit #8 fails here too. - const miniS3 = await startMiniS3(); +interface DaemonSpec { + /** DORMICE_NODE_ID; omitted keeps the daemon's default, as node A always has. */ + nodeId?: string; + port: number; + token: string; + dataDir: string; + miniS3Url: string; + extraEnv?: Record; +} +/** Boots one daemon the production way and waits for /healthz. */ +async function bootDaemon(spec: DaemonSpec) { + const endpoint = `http://127.0.0.1:${spec.port}`; // An explicit allowlist instead of inheriting the whole environment: // whatever DORMICE_* knobs happen to be exported in the developer's shell // must not silently reconfigure the daemon under test. The three docker @@ -63,82 +79,211 @@ export default async function setup(project: TestProject) { inherited[name] = value; } } + const env: Record = { + // Exam disks evaporate with the exam: without this default, a docker + // run without an exported DORMICE_DATA_DIR drops its sandbox disks + // into /var/lib/dormice — the resident daemon's data dir, whose + // startup guard then refuses to start (measured 2026-07-10). An + // exported value still wins through `inherited` below. + DORMICE_DATA_DIR: spec.dataDir, + ...inherited, + ...(spec.nodeId === undefined ? {} : { DORMICE_NODE_ID: spec.nodeId }), + DORMICE_PORT: String(spec.port), + DORMICE_DB_PATH: join(spec.dataDir, 'dormice.db'), + DORMICE_API_TOKEN: spec.token, + // Sweep every second so lifecycle tests run on second-scale policies + // instead of the production default of days. + DORMICE_SCAN_INTERVAL_SECONDS: '1', + // Sample every second so history verbs have rows to answer with + // inside a test's lifetime. + DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS: '1', + // A wildcard sandbox domain so getHost() and the port proxy are + // exercised — no DNS needed, tests spoof the Host header locally. + // A first-boot seed since the setting moved into the ledger: every + // exam starts on a fresh DB, so the seed lands every run, and + // settings-hot.test.ts exercises the live edit on top of it. + DORMICE_SANDBOX_DOMAIN: 'sbx.dormice.test', + // The archiver, pointed at the exam's mini S3 — likewise a + // first-boot seed for the ledger's S3 settings. Deliberately NOT in + // the inherited allowlist: a developer's real DORMICE_S3_* exports + // must never leak an exam's archives into a production bucket. + DORMICE_S3_ENDPOINT: spec.miniS3Url, + DORMICE_S3_BUCKET: 'e2e-archive', + DORMICE_S3_ACCESS_KEY_ID: 'e2e-key', + DORMICE_S3_SECRET_ACCESS_KEY: 'e2e-secret', + DORMICE_S3_FORCE_PATH_STYLE: 'true', + // A managed ingress so the domain-binding verbs run black-box. The + // reload command is a no-op: the exam grades what the daemon writes + // and answers, not Caddy — Caddy's side is real-machine acceptance. + DORMICE_INGRESS_FILE: join(spec.dataDir, 'Caddyfile'), + DORMICE_INGRESS_RELOAD_CMD: 'true', + ...spec.extraEnv, + }; const child = spawn('node', [MAIN], { - env: { - // Exam disks evaporate with the exam: without this default, a docker - // run without an exported DORMICE_DATA_DIR drops its sandbox disks - // into /var/lib/dormice — the resident daemon's data dir, whose - // startup guard then refuses to start (measured 2026-07-10). An - // exported value still wins through `inherited` below. - DORMICE_DATA_DIR: dataDir, - ...inherited, - DORMICE_PORT: String(port), - DORMICE_DB_PATH: join(dataDir, 'dormice.db'), - DORMICE_API_TOKEN: token, - // Sweep every second so lifecycle tests run on second-scale policies - // instead of the production default of days. - DORMICE_SCAN_INTERVAL_SECONDS: '1', - // Sample every second so history verbs have rows to answer with - // inside a test's lifetime. - DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS: '1', - // A wildcard sandbox domain so getHost() and the port proxy are - // exercised — no DNS needed, tests spoof the Host header locally. - // A first-boot seed since the setting moved into the ledger: every - // exam starts on a fresh DB, so the seed lands every run, and - // settings-hot.test.ts exercises the live edit on top of it. - DORMICE_SANDBOX_DOMAIN: 'sbx.dormice.test', - // The archiver, pointed at the exam's mini S3 — likewise a - // first-boot seed for the ledger's S3 settings. Deliberately NOT in - // the inherited allowlist: a developer's real DORMICE_S3_* exports - // must never leak an exam's archives into a production bucket. - DORMICE_S3_ENDPOINT: miniS3.url, - DORMICE_S3_BUCKET: 'e2e-archive', - DORMICE_S3_ACCESS_KEY_ID: 'e2e-key', - DORMICE_S3_SECRET_ACCESS_KEY: 'e2e-secret', - DORMICE_S3_FORCE_PATH_STYLE: 'true', - // A managed ingress so the domain-binding verbs run black-box. The - // reload command is a no-op: the exam grades what the daemon writes - // and answers, not Caddy — Caddy's side is real-machine acceptance. - DORMICE_INGRESS_FILE: join(dataDir, 'Caddyfile'), - DORMICE_INGRESS_RELOAD_CMD: 'true', - }, + env, stdio: ['ignore', 'pipe', 'pipe'], }); + await waitHealthy(child, endpoint, `daemon ${spec.nodeId ?? 'A'}`); + return { + endpoint, + token: spec.token, + env, + ingressFile: join(spec.dataDir, 'Caddyfile'), + kill: () => child.kill(), + }; +} + +async function waitHealthy( + child: ChildProcess, + endpoint: string, + what: string, +): Promise { let output = ''; - child.stdout.on('data', (chunk) => { + child.stdout?.on('data', (chunk) => { output += chunk; }); - child.stderr.on('data', (chunk) => { + child.stderr?.on('data', (chunk) => { output += chunk; }); - const deadline = Date.now() + 10_000; for (;;) { if (child.exitCode !== null) { - throw new Error(`daemon exited during startup:\n${output}`); + throw new Error(`${what} exited during startup:\n${output}`); } try { const res = await fetch(`${endpoint}/healthz`); if (res.ok) { - break; + return; } } catch { // Not listening yet; keep probing until the deadline. } if (Date.now() > deadline) { child.kill(); - throw new Error(`daemon did not come up within 10s:\n${output}`); + throw new Error(`${what} did not come up within 10s:\n${output}`); } await new Promise((resolve) => setTimeout(resolve, 100)); } +} + +export default async function setup(project: TestProject) { + if (!existsSync(MAIN)) { + throw new Error( + `daemon build not found at ${MAIN} — run \`pnpm build\` first`, + ); + } + + const dataDir = await mkdtemp(join(tmpdir(), 'dormice-e2e-')); + // Random high base port: never collides with a locally running daemon on + // 3676 or a gateway on 3677; the fleet takes the next three. Below + // 32768, where Linux hands out ephemeral ports — a fleet port already + // taken by some outbound connection would be a boot failure for nothing. + const base = 20000 + Math.floor(Math.random() * 12000); - project.provide('dormiceEndpoint', endpoint); - project.provide('dormiceToken', token); - project.provide('dormiceIngressFile', join(dataDir, 'Caddyfile')); + // The exam's own S3 (in-process, test-only): the archive lifecycle runs + // black-box in every mode — the fake executor exports real files, the + // store speaks real HTTP. It also mimics OSS's checksum strictness, so a + // daemon that regressed pit #8 fails here too. + const miniS3 = await startMiniS3(); + + // Node A: the daemon every existing suite talks to, exactly as before — + // a standalone daemon that checks in with nobody. + const a = await bootDaemon({ + port: base, + token: randomBytes(32).toString('hex'), + dataDir, + miniS3Url: miniS3.url, + }); + project.provide('dormiceEndpoint', a.endpoint); + project.provide('dormiceToken', a.token); + project.provide('dormiceIngressFile', a.ingressFile); + project.provide('dormiceDaemonMain', MAIN); + project.provide('dormiceNodeAEnv', a.env); + project.provide('dormiceMiniS3Url', miniS3.url); + project.provide('dormiceGatewayMain', GATEWAY_MAIN); + + // The gateway exam: two more daemons behind a gateway, sharing A's mini + // S3 bucket the way a real fleet shares one, and sharing one token with + // the gateway the way a real fleet does. Fake mode only — in docker mode + // the startup guard judges containers by label across the whole machine, + // so two daemons on one docker would refuse each other. + const fleet: Array<{ kill: () => void }> = []; + const dirs: string[] = [dataDir]; + // Whatever came up before a boot failed is killed here: vitest never + // calls the teardown of a setup that threw, and a spawned daemon does + // not die with its parent. + const abandon = async (error: unknown) => { + for (const process of fleet.reverse()) process.kill(); + a.kill(); + await miniS3.close(); + throw error; + }; + if (process.env.DORMICE_EXECUTOR !== 'docker') { + if (!existsSync(GATEWAY_MAIN)) { + await abandon( + new Error( + `gateway build not found at ${GATEWAY_MAIN} — run \`pnpm build\` first`, + ), + ); + } + const gatewayPort = base + 3; + const gatewayEndpoint = `http://127.0.0.1:${gatewayPort}`; + const fleetToken = randomBytes(32).toString('hex'); + // The gateway first, so the nodes' first check-in lands; a gateway + // that comes up after its nodes learns them at their next check-in + // anyway, but the exam should not start on that slack. + const gateway = spawn('node', [GATEWAY_MAIN], { + env: { + PATH: process.env.PATH ?? '', + DORMICE_GATEWAY_PORT: String(gatewayPort), + DORMICE_GATEWAY_DB_PATH: join(dataDir, 'gateway.db'), + DORMICE_API_TOKEN: fleetToken, + // A tiny active limit so the gate can be reached with a handful of + // sandboxes; the CPU gate is opened wide — a laptop running the + // suite is not the machine under judgment; the disk floor is off + // for the same reason. + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '100', + DORMICE_GATEWAY_NODE_MIN_DISK_GB: '0', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + fleet.push({ kill: () => gateway.kill() }); + await waitHealthy(gateway, gatewayEndpoint, 'gateway').catch(abandon); + const nodes: FleetNodeHandle[] = []; + for (const [index, id] of (['node-b', 'node-c'] as const).entries()) { + const nodeDir = await mkdtemp(join(tmpdir(), `dormice-e2e-${id}-`)); + dirs.push(nodeDir); + const node = await bootDaemon({ + nodeId: id, + port: base + 1 + index, + token: fleetToken, + dataDir: nodeDir, + miniS3Url: miniS3.url, + extraEnv: { + DORMICE_GATEWAY_ENDPOINT: gatewayEndpoint, + // Second-scale check-ins so a node's readings and its absence + // show inside a test. + DORMICE_CHECK_IN_INTERVAL_SECONDS: '1', + }, + }).catch(abandon); + fleet.push(node); + nodes.push({ id, endpoint: node.endpoint }); + } + project.provide('dormiceGatewayEndpoint', gatewayEndpoint); + project.provide('dormiceGatewayToken', fleetToken); + project.provide('dormiceGatewayNodes', nodes); + } else { + project.provide('dormiceGatewayEndpoint', null); + project.provide('dormiceGatewayToken', null); + project.provide('dormiceGatewayNodes', null); + } return async () => { - child.kill(); + // The nodes first (they stop checking in), then the gateway. + for (const process of fleet.reverse()) process.kill(); + a.kill(); await miniS3.close(); - await rm(dataDir, { recursive: true, force: true }); + for (const dir of dirs) await rm(dir, { recursive: true, force: true }); }; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 44238dfe..c520a471 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -36,6 +36,9 @@ importers: specifier: ^2.31.0 version: 2.31.0 devDependencies: + '@dormice/gateway': + specifier: workspace:* + version: link:../packages/gateway '@dormice/server': specifier: workspace:* version: link:../packages/server From 4c697dd44760e2b3d47bda0ae4cfe89aaa1b29f5 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:48:51 +0800 Subject: [PATCH 08/26] The gateway's systemd unit, its row in the README, and the shared changeset for the fleet wire The unit is copied into place by hand until install.sh learns the two roles; it carries no docker dependency, because the gateway never touches a container. --- .changeset/gateway-check-in-and-lookup.md | 5 +++++ README.md | 1 + deploy/dormice-gateway.service | 24 +++++++++++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 .changeset/gateway-check-in-and-lookup.md create mode 100644 deploy/dormice-gateway.service diff --git a/.changeset/gateway-check-in-and-lookup.md b/.changeset/gateway-check-in-and-lookup.md new file mode 100644 index 00000000..35d961a0 --- /dev/null +++ b/.changeset/gateway-check-in-and-lookup.md @@ -0,0 +1,5 @@ +--- +"@dormice/shared": minor +--- + +Wire schemas for a fleet behind a gateway: `lookupSandbox` (a node answers "do you hold this sandbox?" by name or id), the node check-in (`checkInRequestSchema`, readings, build), and the gateway's `listNodes` / `removeNode`. The host-metrics schema is split into named parts (`hostReadingSchema`, `dataDiskSchema`, `sandboxStateCountsSchema`) that `hostMetricsResponseSchema` still composes unchanged. diff --git a/README.md b/README.md index 4ea0e84a..8ca14744 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,7 @@ pnpm monorepo: | `packages/sdk` | `@dormice/sdk` — TypeScript client for the native API | | `packages/cli` | `dormice` command-line tool (`dor` for short) | | `packages/console` | Web console: React SPA, served by the daemon at `/console` | +| `packages/gateway` | The fleet's one door in front of one or more daemons: places new sandboxes, finds existing ones by asking the nodes, forwards everything else | | `e2e` | Black-box suite: boots the built daemon, drives it over the wire | | `examples` | Runnable demos: the native SDK, the official `e2b` package, a resident agent | diff --git a/deploy/dormice-gateway.service b/deploy/dormice-gateway.service new file mode 100644 index 00000000..e8369bbc --- /dev/null +++ b/deploy/dormice-gateway.service @@ -0,0 +1,24 @@ +# Dormice gateway: the fleet's one door in front of one or more daemons. +# Configuration lives in /etc/dormice/gateway.env (full-line comments only +# there — systemd's EnvironmentFile treats an inline comment as part of the +# value). install.sh does not install this unit yet: the two-role install +# (gateway + node on one machine, `--role node` elsewhere) is a later step; +# until then it is copied into place by hand (docs/测试机搭建手册.md). +[Unit] +Description=Dormice gateway (fleet front door) +Wants=network-online.target +After=network-online.target + +[Service] +ExecStart=/usr/local/bin/node /opt/dormice/packages/gateway/dist/main.js +EnvironmentFile=/etc/dormice/gateway.env +# Crash-only: the gateway holds no sandbox state — nodes check in again +# within seconds and the cache refills by asking — so restarting it is +# always safe. +Restart=always +RestartSec=3 +# One process per database file. The SQLite lock enforces this; never wrap +# the gateway in a process manager that forks workers. + +[Install] +WantedBy=multi-user.target From f2df652d685a6de2296199032f56c680005ec347 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:49:10 +0800 Subject: [PATCH 09/26] main.ts: import order, as biome sorts it --- packages/server/src/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index 90b02f3d..bd9fbe9f 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -3,9 +3,9 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { pino } from 'pino'; import { buildApp } from './app'; -import { CheckIn, readNodeReading } from './check-in'; import { Archiver } from './archive/archiver'; import { LedgerArchiveStore } from './archive/ledger-store'; +import { CheckIn, readNodeReading } from './check-in'; import { type Config, loadConfig } from './config'; import { recordActivity } from './db/activity'; import { migrateDb, openDb } from './db/db'; From fceecefaf58808b8051e8d9547c234a45f772056 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 04:21:13 +0800 Subject: [PATCH 10/26] Gateway review: a wake is not a placement, an invalid name is refused at the door, and a check-in that moves or shares an endpoint is said in the log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit forwardCreate took every 2xx id into placedIds and paid every node-side 4xx back to placedSinceCheckIn, wakes included. A wake was never counted in the first place, so destroying one inside the same interval uncounted a real placement and let the active gate admit one sandbox more than the node's reading allows. The callers now say whether pick() chose the node (a placement) or the name was found on it (a wake); only a placement moves the count either way. Reverse-proved: without the flag the new test reads b's count as 0 with a placement still in flight. nameOf and the E2B metadata.name are judged by the shared sandboxNameSchema before any node is asked. A node's 400 to lookupSandbox read as silence, so a 129-character name came back as a 503 with Retry-After — retry forever — instead of the 400 it deserves. Fleet.checkIn reports the endpoint a node moved from; the check-in route warns on a move and when two nodes report one endpoint. Both are misconfigurations (two machines sharing DORMICE_NODE_ID, whose default is node-1; a DORMICE_NODE_ENDPOINT naming the wrong machine) whose symptom downstream is a 409 on every name, and the check-in is the only place that sees them. e2b.ts: the `dialect` helper that existed only to be overridden is gone. --- packages/gateway/src/app.test.ts | 77 +++++++++++++++++++++++++++ packages/gateway/src/fleet.test.ts | 2 + packages/gateway/src/fleet.ts | 10 ++-- packages/gateway/src/routes/create.ts | 41 +++++++++----- packages/gateway/src/routes/e2b.ts | 46 +++++++++++----- packages/gateway/src/routes/native.ts | 42 +++++++++++---- packages/gateway/src/routes/nodes.ts | 29 +++++++++- 7 files changed, 206 insertions(+), 41 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 63e03031..f4490e9c 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -288,6 +288,19 @@ const sandboxOf = (r: { body: unknown }) => (r.body as { created: boolean; sandbox: { id: string; nodeId: string } }) .sandbox; +/** Builds a sandbox directly on a node, behind the gateway's back. */ +async function stage(node: FakeNode, name: string) { + const res = await fetch(`${node.endpoint}/acquireSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name }), + }); + return sandboxOf({ body: await res.json() }); +} + /** Polls until the probe answers something — cache verification runs off the request path. */ async function until( probe: () => Promise | T | undefined, @@ -470,6 +483,45 @@ describe('acquire: placing and finding', () => { expect((found.body as { created: boolean }).created).toBe(false); }); + it('a wake is not a placement: it neither counts against the node nor pays a placement back when destroyed — on either face', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 10 }); + await h.checkIn(b, { active: 1 }); + const placedOnB = () => h.fleet.get('b')?.placedSinceCheckIn; + // Sandboxes that already live on b: the gateway finds and wakes them + // there, and b's reading already counts them. + await stage(b, 'sleepy'); + await stage(b, 'dozy'); + expect( + sandboxOf(await rpc(h, '/acquireSandbox', { name: 'sleepy' })).nodeId, + ).toBe('b'); + expect(placedOnB()).toBe(0); + const e2bWake = await fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ metadata: { name: 'dozy' } }), + }); + expect(e2bWake.status).toBe(201); + expect(placedOnB()).toBe(0); + // A new name placed on b (the emptiest) is counted once. + expect( + sandboxOf(await rpc(h, '/acquireSandbox', { name: 'fresh' })).nodeId, + ).toBe('b'); + expect(placedOnB()).toBe(1); + // Destroying the woken sandboxes must not pay back a placement that + // never was: b would be judged one sandbox emptier than it is. + await rpc(h, '/destroySandbox', { name: 'sleepy' }); + expect(placedOnB()).toBe(1); + await rpc(h, '/destroySandbox', { name: 'dozy' }); + expect(placedOnB()).toBe(1); + await rpc(h, '/destroySandbox', { name: 'fresh' }); + expect(placedOnB()).toBe(0); + }); + it('one name on two nodes is a 409 naming both, for every verb; once one copy is gone the name routes again', async () => { const h = await gateway(['a', 'b']); const [a, b] = h.nodes as [FakeNode, FakeNode]; @@ -607,6 +659,31 @@ describe('using, destroying, and the cache', () => { expect((await rpc(h, '/execCommand', { command: 'x' })).status).toBe(400); expect(h.nodes[0]?.hits).toEqual([]); }); + + it('a name the wire refuses is a 400 at the door on both faces, and no node is asked', async () => { + const h = await gateway(['a']); + const long = 'x'.repeat(129); + const native = await rpc(h, '/acquireSandbox', { name: long }); + expect(native.status).toBe(400); + expect(message(native)).toMatch(/^invalid name: /); + expect( + (await rpc(h, '/execCommand', { name: 7, command: 'x' })).status, + ).toBe(400); + const e2b = await fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ metadata: { name: long } }), + }); + expect(e2b.status).toBe(400); + expect(await e2b.json()).toMatchObject({ + code: 400, + message: expect.stringMatching(/^invalid metadata\.name: /), + }); + expect(h.nodes[0]?.hits).toEqual([]); + }); }); describe('the E2B faces', () => { diff --git a/packages/gateway/src/fleet.test.ts b/packages/gateway/src/fleet.test.ts index 18e4b523..97e3ab8b 100644 --- a/packages/gateway/src/fleet.test.ts +++ b/packages/gateway/src/fleet.test.ts @@ -49,6 +49,8 @@ describe('Fleet', () => { new Date(NOW.getTime() + 15_000), ); expect(second.joined).toBe(false); + expect(first.movedFrom).toBeNull(); + expect(second.movedFrom).toBe('http://10.0.0.7:80'); expect(second.node).toBe(first.node); expect(second.node.endpoint).toBe('http://10.0.0.8:80'); expect(second.node.reading?.sandboxes.byState.active).toBe(12); diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 1e3611d1..69e02140 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -81,7 +81,9 @@ export class Fleet { /** * A node reporting for duty. A first check-in adds the node (`joined` * says so, for the log); a changed endpoint is written through — the - * node states where it lives, the gateway does not remember better. The + * node states where it lives, the gateway does not remember better — + * and `movedFrom` names the old one, for the log: a node that moves at + * every check-in is two machines sharing one DORMICE_NODE_ID. The * placement counter restarts at zero: what was placed before this * reading is in it now, and what is still in flight on the node (its * row is written after the container is up) is in neither figure until @@ -91,9 +93,10 @@ export class Fleet { checkIn( report: CheckInRequest, now = new Date(), - ): { node: NodeState; joined: boolean } { + ): { node: NodeState; joined: boolean; movedFrom: string | null } { let node = this.members.get(report.nodeId); let joined = false; + let movedFrom: string | null = null; if (node === undefined) { const addedAt = now.toISOString(); this.db @@ -119,6 +122,7 @@ export class Fleet { .set({ endpoint: report.endpoint }) .where(eq(nodes.id, report.nodeId)) .run(); + movedFrom = node.endpoint; node.endpoint = report.endpoint; } node.lastCheckInAt = now; @@ -127,7 +131,7 @@ export class Fleet { node.reading = report.reading; node.placedSinceCheckIn = 0; node.placedIds.clear(); - return { node, joined }; + return { node, joined, movedFrom }; } /** The operator's word that the node is gone for good; a node still running re-adds itself at its next check-in. */ diff --git a/packages/gateway/src/routes/create.ts b/packages/gateway/src/routes/create.ts index 8448458c..d9f700b1 100644 --- a/packages/gateway/src/routes/create.ts +++ b/packages/gateway/src/routes/create.ts @@ -74,21 +74,34 @@ export interface CreateOptions { name: string | null; body: Buffer | undefined; face: CreateFace; + /** + * True when place() chose the target for a name no node held — the + * placement was counted against it in placedSinceCheckIn. False for a + * wake: the name was found on the target, its sandbox is in the reading + * already, and nothing was counted that could be uncounted. + */ + placed: boolean; } /** * Forwards a create and learns from the node's answer: a 2xx with a * readable id goes into the cache (name and id → this node), so the next - * request for the sandbox skips the round of questions, and into the - * node's placedIds, so a destroy inside the same interval can take the - * placement off the count again. A 2xx without a readable id is logged — - * the sandbox exists on the node and the next lookup finds it there, - * never silently. A 4xx or the node's own 500 means the node built - * nothing: the placement is taken off the count at once (it would have - * held a slot for a whole interval otherwise) and the next attempt may be - * placed elsewhere. A 502/503/504 from a hop in front of the node, or no - * answer at all, leaves the count as it is and the question to the next - * lookup — the node that may have built it answers from inside the + * request for the sandbox skips the round of questions. A 2xx without a + * readable id is logged — the sandbox exists on the node and the next + * lookup finds it there, never silently. + * + * The placement count moves only for a placement (`placed`): a 2xx id + * also goes into the node's placedIds, so a destroy inside the same + * interval takes the placement off the count again; a 4xx or the node's + * own 500 means the node built nothing, and the placement comes off the + * count at once (it would have held a slot for a whole interval + * otherwise), so the next attempt may be placed elsewhere. A wake — a + * name found on the node — moves nothing either way: its sandbox is in + * the reading already, and paying back a placement that never was would + * open the gate one sandbox wider than the reading allows (found by + * review, 2026-09-14). A 502/503/504 from a hop in front of the node, or + * no answer at all, leaves the count as it is and the question to the + * next lookup — the node that may have built it answers from inside the * name's slot. */ const HOP_STATUSES = new Set([502, 503, 504]); @@ -96,7 +109,7 @@ export async function forwardCreate( cache: NameCache, request: FastifyRequest, res: http.ServerResponse, - { target, token, name, body, face }: CreateOptions, + { target, token, name, body, face, placed }: CreateOptions, ): Promise { const answer = await forwardCapture(request.raw, res, { target: { endpoint: target.endpoint, token }, @@ -108,7 +121,7 @@ export async function forwardCreate( const id = face.idOf(parseJson(answer.body)); if (id !== null) { cache.put({ id, name, nodeId: target.id }); - target.placedIds.add(id); + if (placed) target.placedIds.add(id); } else { request.log.warn( { node: target.id, name, status: answer.status }, @@ -117,7 +130,9 @@ export async function forwardCreate( } } else if (!HOP_STATUSES.has(answer.status)) { // The node itself answered no: nothing was built there. - target.placedSinceCheckIn = Math.max(0, target.placedSinceCheckIn - 1); + if (placed) { + target.placedSinceCheckIn = Math.max(0, target.placedSinceCheckIn - 1); + } if (answer.status >= 500) { request.log.warn( { node: target.id, name, status: answer.status }, diff --git a/packages/gateway/src/routes/e2b.ts b/packages/gateway/src/routes/e2b.ts index ee5d0e51..a80443e9 100644 --- a/packages/gateway/src/routes/e2b.ts +++ b/packages/gateway/src/routes/e2b.ts @@ -1,5 +1,6 @@ import { tokensEqual } from '@dormice/server/auth'; import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import { sandboxNameSchema } from '@dormice/shared'; import type { FastifyError, FastifyReply, FastifyRequest } from 'fastify'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { relay } from '../errors'; @@ -66,7 +67,6 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( } }); - const dialect = (message: string) => ({ code: 0, message }); const send = (reply: FastifyReply, code: number, message: string) => reply.code(code).send({ code, message }); @@ -87,12 +87,14 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( const RETRY_FINDS_IT = ' — retry: if the sandbox was built, the node answers for it'; + /** `placed`: pick() chose the node for a new name (counted there) — false for a name found on it, a wake. */ function create( request: FastifyRequest, reply: FastifyReply, target: NodeState, name: string | null, body: Buffer | undefined, + placed: boolean, ) { return forwarded(request, reply, RETRY_FINDS_IT, async () => { const answer = await forwardCreate(finder.cache, request, reply.raw, { @@ -101,12 +103,14 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( name, body, face: E2B_CREATE, + placed, }); if (answer !== null) replay(reply.raw, answer); }); } - function placed(reply: FastifyReply): NodeState | null { + /** Places a new sandbox, or sends the 503 and answers null (null too for a client that already left). */ + function placeOrRefuse(reply: FastifyReply): NodeState | null { if (clientGone(reply)) return null; const placement = place(fleet, knobs, new Date()); if (placement.node === null) { @@ -122,24 +126,38 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( const parsed = parseJson(body) as | { metadata?: { name?: unknown } } | undefined; - const name = parsed?.metadata?.name; - if (typeof name === 'string' && name.length > 0) { + if (parsed?.metadata?.name !== undefined) { + // Judged by the wire's own rule before any node is asked (native.ts + // nameOf has why); the node's stricter E2B pattern still answers its + // own 400, relayed as it came. + const judged = sandboxNameSchema.safeParse(parsed.metadata.name); + if (!judged.success) { + return send( + reply, + 400, + `invalid metadata.name: ${judged.error.issues[0]?.message ?? 'refused by the wire'}`, + ); + } + const name = judged.data; return locks.run(name, async () => { - const judged = verdict(await finder.byName(name), `sandbox "${name}"`); - if (judged.kind === 'refuse') { - return refuse(reply, judged, (message) => ({ - ...dialect(message), - code: judged.status, + const found = verdict(await finder.byName(name), `sandbox "${name}"`); + if (found.kind === 'refuse') { + return refuse(reply, found, (message) => ({ + code: found.status, + message, })); } - const target = judged.kind === 'node' ? judged.node : placed(reply); + if (found.kind === 'node') { + return create(request, reply, found.node, name, body, false); + } + const target = placeOrRefuse(reply); if (target === null) return reply; - return create(request, reply, target, name, body); + return create(request, reply, target, name, body, true); }); } - const target = placed(reply); + const target = placeOrRefuse(reply); if (target === null) return reply; - return create(request, reply, target, null, body); + return create(request, reply, target, null, body, true); }); app.get('/v2/sandboxes', async (_request, reply) => @@ -155,8 +173,8 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( const judged = verdict(await finder.byId(id), `sandbox "${id}"`); if (judged.kind === 'refuse') { return refuse(reply, judged, (message) => ({ - ...dialect(message), code: judged.status, + message, })); } if (judged.kind === 'none') { diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index fd8d3fed..17cfafa3 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -1,5 +1,8 @@ import type { KeyedQueue } from '@dormice/server/keyed-queue'; -import { WRITE_FILES_BODY_LIMIT_BYTES } from '@dormice/shared'; +import { + sandboxNameSchema, + WRITE_FILES_BODY_LIMIT_BYTES, +} from '@dormice/shared'; import type { FastifyReply, FastifyRequest } from 'fastify'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { relay } from '../errors'; @@ -113,12 +116,11 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( bodyLimit === undefined ? {} : { bodyLimit }, async (request, reply) => { const body = request.body as Buffer | undefined; - const name = nameOf(body); - if (name === null) { - return reply - .code(400) - .send({ message: 'name is required and must be a string' }); + const named = nameOf(body); + if ('refusal' in named) { + return reply.code(400).send({ message: named.refusal }); } + const { name } = named; // Only the two verbs that create or remove take the name's slot — // the daemon's own discipline (its other verbs run unserialized // too). The slot is what keeps twenty simultaneous acquires of a @@ -167,6 +169,9 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( const judged = verdict(await finder.byName(name), `sandbox "${name}"`); if (judged.kind === 'refuse') return refuse(reply, judged); let target = judged.kind === 'node' ? judged.node : null; + // Found nowhere: a placement, counted against the node it lands on. + // Found somewhere: a wake, already in that node's reading. + const placed = target === null; if (target === null) { if (clientGone(reply)) return; const placement = place(fleet, knobs, new Date()); @@ -184,6 +189,7 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( name, body, face: NATIVE_CREATE, + placed, }); if (answer !== null) replay(reply.raw, answer); }); @@ -249,9 +255,25 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( } }; -/** The one field the gateway reads from a native body. */ -export function nameOf(body: Buffer | undefined): string | null { +/** + * The one field the gateway reads from a native body, judged by the wire's + * own rule (shared sandboxNameSchema — what lookupSandbox validates + * against) so no node is ever asked a question it would refuse: a node's + * 400 to the lookup reads as silence, and the caller would get a 503 with + * Retry-After for a name that can never be valid (found by review, + * 2026-09-14). + */ +export function nameOf( + body: Buffer | undefined, +): { name: string } | { refusal: string } { const parsed = parseJson(body) as { name?: unknown } | undefined; - const name = parsed?.name; - return typeof name === 'string' && name.length > 0 ? name : null; + if (parsed?.name === undefined) { + return { refusal: 'name is required and must be a string' }; + } + const judged = sandboxNameSchema.safeParse(parsed.name); + return judged.success + ? { name: judged.data } + : { + refusal: `invalid name: ${judged.error.issues[0]?.message ?? 'refused by the wire'}`, + }; } diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index 83ddd779..1240e56a 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -34,13 +34,40 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( }, }, async (request) => { - const { node, joined } = fleet.checkIn(request.body); + const { node, joined, movedFrom } = fleet.checkIn(request.body); if (joined) { request.log.info( { nodeId: node.id, endpoint: node.endpoint }, 'a node checked in for the first time and joined the fleet', ); } + // Two misconfigurations show only here, so they are said here. A + // node whose endpoint moves at every check-in is two machines + // sharing one DORMICE_NODE_ID (the daemon's default is `node-1`). + // Two nodes reporting one endpoint is a DORMICE_NODE_ENDPOINT that + // names the wrong machine — the daemon refuses the loopback default + // when its gateway is remote, but a hand-written value still can. + // Either way the symptom downstream is a 409 on every name (both + // "nodes" answer the lookup) or sandboxes on the wrong machine. + if (movedFrom !== null) { + request.log.warn( + { nodeId: node.id, from: movedFrom, to: node.endpoint }, + 'a node checked in from a new endpoint; the gateway forwards there from now on', + ); + } + const twins = fleet + .all() + .filter((n) => n.id !== node.id && n.endpoint === node.endpoint); + if (twins.length > 0) { + request.log.warn( + { + nodeId: node.id, + endpoint: node.endpoint, + alsoReportedBy: twins.map((n) => n.id), + }, + 'two nodes report the same endpoint — check DORMICE_NODE_ID and DORMICE_NODE_ENDPOINT on both; their sandboxes will be found twice (409) or land on the wrong machine', + ); + } return {}; }, ); From 6701d5983fe51e2b821f68656409407b773f4df0 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 04:21:13 +0800 Subject: [PATCH 11/26] A node whose gateway is on another machine must set DORMICE_NODE_ENDPOINT: the loopback default names the gateway's own machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Left unset, the node reports http://127.0.0.1: as where it can be reached, and a remote gateway dials its own daemon (or nothing) for this node: every sandbox placed "here" lands there, and every lookup then finds it twice — a 409 on every name, from one forgotten variable. Refused at boot, naming the variable and the address to write, whenever DORMICE_GATEWAY_ENDPOINT is not loopback. An explicit value is the operator's word and is taken as written. The loopback test tolerates an unparsable URL: zod runs the object-level refinements even when a field failed, and the field's own error ("must be a full http(s) URL") must be the one that shows, not an Invalid URL thrown from inside the rule. --- packages/server/src/config.test.ts | 28 +++++++++++++++++++ packages/server/src/config.ts | 44 +++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index 083f071c..963e89cb 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -183,4 +183,32 @@ describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { loadConfig({ ...TOKEN, DORMICE_CHECK_IN_INTERVAL_SECONDS: '0' }), ).toThrow(); }); + + it('a gateway on another machine requires the node endpoint, naming why; a loopback gateway does not', () => { + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + }), + ).toThrow( + /DORMICE_NODE_ENDPOINT is required when DORMICE_GATEWAY_ENDPOINT is not loopback/, + ); + expect( + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80', + }).DORMICE_NODE_ENDPOINT, + ).toBe('http://10.0.0.7:80'); + for (const local of [ + 'http://127.0.0.1:3677', + 'http://localhost:3677', + 'http://[::1]:3677', + ]) { + expect( + loadConfig({ ...TOKEN, DORMICE_GATEWAY_ENDPOINT: local }) + .DORMICE_NODE_ENDPOINT, + ).toBeUndefined(); + } + }); }); diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index acd03501..c31d7289 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -194,7 +194,11 @@ const envSchema = z.object({ * node share a machine (the single-machine install is a fleet of one). * On a machine of its own the daemon still binds loopback (the red * line), so this names the front the gateway may dial — the node's - * Caddy on the intranet interface, e.g. http://10.0.0.7:80. + * Caddy on the intranet interface, e.g. http://10.0.0.7:80 — and is + * required there (checkedSchema below): left at the default, the node + * would tell a remote gateway to dial 127.0.0.1, an address on the + * gateway's own machine, and every sandbox placed "here" would land on + * whatever daemon lives there and be found twice. */ DORMICE_NODE_ENDPOINT: z .url({ @@ -216,6 +220,27 @@ const envSchema = z.object({ .default(15), }); +/** + * Loopback as an operator writes it: 127.0.0.0/8, ::1, localhost. Null + * when the URL does not parse — its own field has already said so, and an + * object-level rule must not throw over it (zod runs the refinements even + * when a field failed). + */ +function isLoopbackUrl(url: string): boolean | null { + let host: string; + try { + host = new URL(url).hostname; + } catch { + return null; + } + return ( + host === 'localhost' || + host === '::1' || + host === '[::1]' || + host.startsWith('127.') + ); +} + const checkedSchema = envSchema .refine( (cfg) => cfg.DORMICE_EXECUTOR !== 'docker' || !!cfg.DORMICE_BASE_IMAGE, @@ -248,6 +273,23 @@ const checkedSchema = envSchema path: ['DORMICE_DATA_DIR'], }, ) + // A node whose gateway is on another machine must say where it is. The + // check-in's default endpoint is this daemon's loopback address, which + // on the gateway's machine names the gateway's own daemon (or nothing): + // a remote gateway dialing 127.0.0.1 for this node would place sandboxes + // on the wrong machine and then find every one of them twice (409). An + // explicit value is the operator's word and is taken as written. + .refine( + (cfg) => + cfg.DORMICE_GATEWAY_ENDPOINT === undefined || + isLoopbackUrl(cfg.DORMICE_GATEWAY_ENDPOINT) !== false || + cfg.DORMICE_NODE_ENDPOINT !== undefined, + { + message: + "DORMICE_NODE_ENDPOINT is required when DORMICE_GATEWAY_ENDPOINT is not loopback: the gateway is on another machine, and without it this node would report http://127.0.0.1: — an address on the gateway's machine, not this one. Name this node's address on the network, e.g. http://10.0.0.7:80", + path: ['DORMICE_NODE_ENDPOINT'], + }, + ) // All-or-none: a half-configured store would make the archiver's // existence ambiguous, and ambiguity here decides real policy defaults. .superRefine((cfg, ctx) => { From 042b26541008ad280046665e67ec77db5b8e38bc Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 04:21:13 +0800 Subject: [PATCH 12/26] lookupSandbox takes the name's slot in one step: locks.run already answers at once when the slot is free MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tryRun-then-run did the same thing in three lines. run() executes the task immediately when nothing holds the key and queues it behind the holder otherwise — exactly "a plain no when the slot is free, wait for the acquire when it is not". SKIPPED is no longer imported here. --- packages/server/src/routes/sandboxes.ts | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/packages/server/src/routes/sandboxes.ts b/packages/server/src/routes/sandboxes.ts index 739f0e9c..7488481d 100644 --- a/packages/server/src/routes/sandboxes.ts +++ b/packages/server/src/routes/sandboxes.ts @@ -84,7 +84,7 @@ import { NotAFileError, } from '../executor/executor'; import { httpError } from '../http-error'; -import { type KeyedQueue, SKIPPED } from '../keyed-queue'; +import type { KeyedQueue } from '../keyed-queue'; import { destroySandbox, rebuildSandbox, wakeSandbox } from '../lifecycle'; import { ArchiveDisabledError, resolvePolicy } from '../policy'; import { resolveSpec } from '../spec'; @@ -1119,17 +1119,17 @@ export const sandboxRoutes: FastifyPluginAsyncZod< // The gateway's one question on its own account: does this node hold // the sandbox? Read-only — never wakes, never touches the idle clock — - // and truthful about a create in flight, in three steps. A row that + // and truthful about a create in flight, in two steps. A row that // exists answers at once, whatever its state: a restoring sandbox has a // row, and waiting for its slot would hold the answer for the whole // restore, long past the gateway's two-second patience — the gateway - // would read a live sandbox as a node that did not answer. No row while - // the name's slot is busy means an acquire may be writing the row right - // now (create first, row second, both under the slot), so the answer - // waits its turn behind it and looks again. No row and a free slot is a - // plain no. By id there is no slot to wait on (slots are keyed by name), - // and none is needed: nobody can ask about an id before the create that - // minted it has answered. + // would read a live sandbox as a node that did not answer. No row means + // an acquire may be writing it right now (create first, row second, + // both under the name's slot), so the answer takes the slot itself and + // looks again: at once when the slot is free (a plain no), behind the + // acquire when it is not. By id there is no slot to wait on (slots are + // keyed by name), and none is needed: nobody can ask about an id before + // the create that minted it has answered. app.post( '/lookupSandbox', { @@ -1151,8 +1151,6 @@ export const sandboxRoutes: FastifyPluginAsyncZod< : { found: false as const }; const now = look(); if (now !== undefined || !('name' in query)) return answer(now); - const unheld = await locks.tryRun(query.name, async () => look()); - if (unheld !== SKIPPED) return answer(unheld); return answer(await locks.run(query.name, async () => look())); }, ); From 1c4f1a250f27f18f0f95f525186c49042b7b3522 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 04:49:41 +0800 Subject: [PATCH 13/26] A node endpoint is an origin, and a redirected check-in is reported instead of followed The gateway hands a node's endpoint to undici as the request's origin when it forwards, and undici refuses an origin that carries a path (UND_ERR_INVALID_ARG, measured). Its own lookup joins `/lookupSandbox` as a string and works with a path. So a node reporting http://10.0.0.7:80/dormice was found by every lookup and reached by no forward. The shared wire now has `endpointSchema`: a trailing slash is dropped (the two ends must agree byte for byte, and `//lookupSandbox` is a 404 the gateway reads as silence), a path or query is a 400 at /checkIn, and DORMICE_NODE_ENDPOINT refuses the same at the daemon's boot, where the operator is looking. The node's check-in used fetch's default redirect handling. A front that answers plain http with a 308 to https (a Caddy binding the gateway's domain) would be followed across origins, and the Fetch standard drops Authorization on the way, so the gateway answered 401 and the log said "wrong token" where the address was wrong. The check-in no longer follows: it reports the 3xx and its Location, and names the variable to fix. Found by the second review pass of 2026-09-14; each fix reverse-proved (the new test goes red with the fix removed). --- .changeset/gateway-check-in-and-lookup.md | 2 +- packages/gateway/src/app.test.ts | 15 +++++++++ packages/server/src/check-in.test.ts | 35 +++++++++++++++++++-- packages/server/src/check-in.ts | 13 +++++++- packages/server/src/config.test.ts | 22 +++++++++++++ packages/server/src/config.ts | 13 +++++++- packages/shared/src/gateway.ts | 38 +++++++++++++++++++++-- 7 files changed, 130 insertions(+), 8 deletions(-) diff --git a/.changeset/gateway-check-in-and-lookup.md b/.changeset/gateway-check-in-and-lookup.md index 35d961a0..7561e255 100644 --- a/.changeset/gateway-check-in-and-lookup.md +++ b/.changeset/gateway-check-in-and-lookup.md @@ -2,4 +2,4 @@ "@dormice/shared": minor --- -Wire schemas for a fleet behind a gateway: `lookupSandbox` (a node answers "do you hold this sandbox?" by name or id), the node check-in (`checkInRequestSchema`, readings, build), and the gateway's `listNodes` / `removeNode`. The host-metrics schema is split into named parts (`hostReadingSchema`, `dataDiskSchema`, `sandboxStateCountsSchema`) that `hostMetricsResponseSchema` still composes unchanged. +Wire schemas for a fleet behind a gateway: `lookupSandbox` (a node answers "do you hold this sandbox?" by name or id), the node check-in (`checkInRequestSchema`, readings, build; its `endpoint` is an origin — `endpointSchema` drops a trailing slash and refuses a path), and the gateway's `listNodes` / `removeNode`. The host-metrics schema is split into named parts (`hostReadingSchema`, `dataDiskSchema`, `sandboxStateCountsSchema`) that `hostMetricsResponseSchema` still composes unchanged. diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index f4490e9c..573eda1f 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -383,6 +383,21 @@ describe('check-in and the node verbs', () => { const bad = await rpc(h, '/checkIn', { nodeId: 'z' }); expect(bad.status).toBe(400); expect(h.fleet.get('z')).toBeUndefined(); + // An endpoint the gateway could not use is refused at the wire (shared + // endpointSchema has the measurement): undici takes the endpoint as + // the request's origin and throws on a path — a node every lookup + // would find and no forward could reach. A trailing slash is merely + // dropped; the two ends must agree byte for byte. + const pathy = await rpc( + h, + '/checkIn', + checkInOf('p', 'http://10.0.0.9:80/dormice'), + ); + expect(pathy.status).toBe(400); + expect(message(pathy)).toMatch(/an endpoint is an origin/); + expect(h.fleet.get('p')).toBeUndefined(); + await rpc(h, '/checkIn', checkInOf('s', 'http://10.0.0.9:80/')); + expect(h.fleet.get('s')?.endpoint).toBe('http://10.0.0.9:80'); }); it('removeNode forgets the node and everything cached on it; a removed node that checks in again re-joins', async () => { diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index 6cbcc69a..f4636b82 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -21,7 +21,13 @@ afterEach(async () => { }); /** A gateway-shaped listener: records every check-in, answers what the test says. */ -async function gateway(answer: () => { status: number; body: string }) { +async function gateway( + answer: () => { + status: number; + body: string; + headers?: Record; + }, +) { const seen: Array<{ headers: http.IncomingHttpHeaders; body: unknown }> = []; const server = http.createServer((req, res) => { let text = ''; @@ -31,7 +37,10 @@ async function gateway(answer: () => { status: number; body: string }) { req.on('end', () => { seen.push({ headers: req.headers, body: JSON.parse(text) }); const a = answer(); - res.writeHead(a.status, { 'content-type': 'application/json' }); + res.writeHead(a.status, { + 'content-type': 'application/json', + ...a.headers, + }); res.end(a.body); }); }); @@ -46,12 +55,17 @@ async function gateway(answer: () => { status: number; body: string }) { function logSpy() { const infos: string[] = []; const warns: string[] = []; + const details: unknown[] = []; return { infos, warns, + details, log: { info: (msg: string) => infos.push(msg), - warn: (_obj: unknown, msg: string) => warns.push(msg), + warn: (obj: unknown, msg: string) => { + warns.push(msg); + details.push(obj); + }, }, }; } @@ -154,6 +168,21 @@ describe('CheckIn', () => { expect(warns).toHaveLength(2); }); + it('a front that redirects is reported as a wrong address, not followed with the token stripped', async () => { + const gw = await gateway(() => ({ + status: 308, + body: '', + headers: { location: 'https://gateway.example/checkIn' }, + })); + const { log, warns, details } = logSpy(); + await new CheckIn(options(gw.endpoint, log)).once(); + expect(gw.seen).toHaveLength(1); + expect(warns).toEqual([expect.stringMatching(/check-in failed/)]); + expect((details[0] as { error: string }).error).toMatch( + /gateway answered 308 redirecting to https:\/\/gateway\.example\/checkIn — DORMICE_GATEWAY_ENDPOINT must be the gateway's own address/, + ); + }); + it('a gateway that is not there is a logged failure, never a throw', async () => { const { log, warns } = logSpy(); const checkIn = new CheckIn(options('http://127.0.0.1:9', log)); diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index 1fda3f3e..d5acc4da 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -101,11 +101,22 @@ export class CheckIn { }, body: JSON.stringify(body), signal: AbortSignal.timeout(CHECK_IN_TIMEOUT_MS), + // A front that redirects (a Caddy binding the gateway's domain + // answers plain http with a 308 to https) is reported as what it + // is. Followed, the redirect would cross origins and fetch would + // drop the Authorization header on the way (the Fetch standard's + // rule), so the gateway would answer 401 — and the operator would + // read a wrong token where there is a wrong address (found by + // review, 2026-09-14). + redirect: 'manual', }); if (res.status !== 200) { const text = await res.text(); + const location = res.headers.get('location'); throw new Error( - `gateway answered ${res.status}: ${text.slice(0, 200)}`, + location === null + ? `gateway answered ${res.status}: ${text.slice(0, 200)}` + : `gateway answered ${res.status} redirecting to ${location} — DORMICE_GATEWAY_ENDPOINT must be the gateway's own address, not a front that redirects`, ); } checkInResponseSchema.parse(await res.json()); diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index 963e89cb..d07cebfa 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -184,6 +184,28 @@ describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { ).toThrow(); }); + it('a node endpoint with a path or a query is refused at boot, naming why; a trailing slash is still just dropped', () => { + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80/dormice', + }), + ).toThrow( + /DORMICE_NODE_ENDPOINT must name the node's front without a path/, + ); + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80/?x=1', + }), + ).toThrow(/without a path/); + expect( + loadConfig({ ...TOKEN, DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80/' }) + .DORMICE_NODE_ENDPOINT, + ).toBe('http://10.0.0.7:80'); + }); + it('a gateway on another machine requires the node endpoint, naming why; a loopback gateway does not', () => { expect(() => loadConfig({ diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index c31d7289..ddfac547 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -1,5 +1,9 @@ import { isAbsolute } from 'node:path'; -import { bareHostnameRegex, PIDS_LIMIT_MIN } from '@dormice/shared'; +import { + bareHostnameRegex, + isOriginUrl, + PIDS_LIMIT_MIN, +} from '@dormice/shared'; import { z } from 'zod'; import type { S3Settings } from './archive/s3-store'; @@ -207,6 +211,13 @@ const envSchema = z.object({ 'DORMICE_NODE_ENDPOINT must be a full http(s) URL, e.g. http://10.0.0.7:80', }) .transform((url) => url.replace(/\/+$/, '')) + // An origin, nothing more (shared endpointSchema has the measurement): + // the gateway would answer this node's every check-in with a 400 + // otherwise — refused here, at boot, where the operator is looking. + .refine(isOriginUrl, { + error: + "DORMICE_NODE_ENDPOINT must name the node's front without a path — scheme, host and port only, e.g. http://10.0.0.7:80 (the gateway dials /)", + }) .optional(), /** * How often the node checks in with its gateway. The gateway reads two diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts index 11eb6cf8..12d2c6b7 100644 --- a/packages/shared/src/gateway.ts +++ b/packages/shared/src/gateway.ts @@ -26,6 +26,40 @@ export const buildInfoSchema = z.object({ export type BuildInfo = z.infer; +/** + * Whether a URL is an origin and nothing more: scheme, host and port — no + * path, query or fragment. False for a string that is not a URL at all. + */ +export function isOriginUrl(url: string): boolean { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return false; + } + return parsed.pathname === '/' && parsed.search === '' && parsed.hash === ''; +} + +/** + * Where a process can be dialled — an origin. The gateway joins + * `/` for its own lookup and hands the endpoint to undici + * as the request's origin when it forwards, and undici refuses an origin + * that carries a path (UND_ERR_INVALID_ARG, measured 2026-09-14): a node + * reporting `http://10.0.0.7:80/dormice` would be found by every lookup + * and reached by no forward. So a path is refused at the wire. A trailing + * slash is dropped rather than refused — the same address to everyone but + * a string compare, and the two ends must agree byte for byte (the join + * above would otherwise ask for `//lookupSandbox`, a 404 the gateway reads + * as silence). + */ +export const endpointSchema = z + .url({ protocol: /^https?$/ }) + .transform((url) => url.replace(/\/+$/, '')) + .refine(isOriginUrl, { + error: + 'an endpoint is an origin — scheme, host and port only, no path or query, e.g. http://10.0.0.7:80', + }); + /** * What a node reports about itself at every check-in — everything * placement decides on: the machine's CPU, memory and data disk, and the @@ -57,8 +91,8 @@ export type NodeReading = z.infer; export const checkInRequestSchema = z.object({ /** DORMICE_NODE_ID — the node's name in every sandbox's `nodeId`. */ nodeId: z.string().min(1), - /** Where the gateway forwards to: the node's intranet front (DORMICE_NODE_ENDPOINT). */ - endpoint: z.url({ protocol: /^https?$/ }), + /** Where the gateway forwards to: the node's intranet front (DORMICE_NODE_ENDPOINT) — an origin, endpointSchema has why. */ + endpoint: endpointSchema, /** How often this node checks in — the gateway's yardstick for "missed two in a row". */ intervalSeconds: z.number().int().positive(), build: buildInfoSchema.nullable(), From 918073765ace14888562b5ada4a91ad183041bc0 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 04:49:41 +0800 Subject: [PATCH 14/26] forwardStream sends nothing for a client that already left MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit forwardCapture opened with `if (res.destroyed) return null`; forwardStream did not. A caller of a streamed verb (execCommand) that hangs up during the lookup round — up to two seconds when a node is slow — has a destroyed response whose 'close' has already fired, so the abort wired to 'close' would never come, and the node would run the command to its end for nobody. Same first line now, and the doc comment that already claimed the two followed one rule is true. Found by the second review pass of 2026-09-14; reverse-proved. --- packages/gateway/src/forward.test.ts | 32 ++++++++++++++++++++++++++++ packages/gateway/src/forward.ts | 10 +++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/packages/gateway/src/forward.test.ts b/packages/gateway/src/forward.test.ts index 48457edd..58ef946d 100644 --- a/packages/gateway/src/forward.test.ts +++ b/packages/gateway/src/forward.test.ts @@ -155,6 +155,38 @@ describe('forwardStream', () => { await closed; }); + it('a client already gone when the forward begins is not sent to the node at all, and answers null', async () => { + let requests = 0; + const node = http.createServer((_req, res) => { + requests += 1; + res.end('{}'); + }); + const endpoint = await listen(node); + // The shape forwardNamed produces: the lookup round took a while and + // the caller hung up meanwhile — the response is destroyed before the + // forward starts, so its 'close' has already fired and no abort would + // ever follow. + const gone = new http.IncomingMessage(new net.Socket()); + const res = new http.ServerResponse(gone); + res.destroy(); + expect( + await forwardStream( + Object.assign(gone, { + url: '/execCommand', + method: 'POST', + headers: {}, + }), + res, + { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + body: Buffer.from('{"name":"x","command":"sleep 3600"}'), + }, + ), + ).toBeNull(); + expect(requests).toBe(0); + }); + it('forwards the request verbatim — path, query, body — with the credential swapped and the Host renamed to the node unless the face keeps it', async () => { const seen: Array<{ url?: string; diff --git a/packages/gateway/src/forward.ts b/packages/gateway/src/forward.ts index c025cfda..5fdc09a4 100644 --- a/packages/gateway/src/forward.ts +++ b/packages/gateway/src/forward.ts @@ -175,8 +175,9 @@ async function dispatch( * Throws UnreachableError only before any byte of the answer was written; * once the head is out, a failure mid-stream can only be a cut connection * — there is no honest status left to send. Resolves null when the client - * left before the node answered: the node-side request is aborted and - * nothing is rendered (the response is gone). Otherwise an abandoned exec + * left before the node answered: the node-side request is aborted — or, + * for a client already gone, never sent — and nothing is rendered (the + * response is gone). Otherwise an abandoned exec * against a slow node would hold a gateway→node socket until the node * answered — on a hung node, until its TCP died. forwardCapture follows * the same rule. @@ -186,6 +187,11 @@ export async function forwardStream( res: http.ServerResponse, options: ForwardOptions, ): Promise { + // The client left while its sandbox was being found (a lookup round is + // up to two seconds): 'close' has fired already, the abort below would + // never come, and the node would run an exec to its end for nobody — + // forwardCapture's first line, missing here (found by review, 2026-09-14). + if (res.destroyed) return null; const gone = new AbortController(); const onClose = () => gone.abort(); res.once('close', onClose); From a9b735bad3fe0f0627c34294346b623b44796106 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 05:14:43 +0800 Subject: [PATCH 15/26] A creator confirms its cached node before waking there; removeNode and a second reporter under one id are refused while the node is live MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three ways the fleet could end up with one name on two nodes, or a placement past every gate, found by the second review pass of 2026-09-14. The cache is a hint for every verb but the two creators: a stale entry costs a reader one 404 that evicts it, but acquireSandbox and the E2B named create are create-or-wake on the node, so a cache hit forwarded as a wake to a node that has since deleted the row (an E2B deadline kill is the scanner's routine, five minutes by default) rebuilt the sandbox there — past placement's four gates, uncounted in placedSinceCheckIn, every re-create of an expired name pinned to its first node. The creators now confirm a cache hit with that one node by id (Finder.byName confirm): found is a wake, absent evicts and asks the fleet, silent is the same 503 a silent stranger earns. One RTT per warm acquire. removeNode on a node still checking in deleted its row; a name of its acquired before its next check-in was placed elsewhere, then the node re-added itself and the name was on two nodes — a 409 an operator clears by hand. It is now a 409 up front, naming what to do: stop the daemon, wait two of its intervals (what "down" means), then remove. A check-in that changed a node's endpoint inside the previous reporter's own interval was written through. Two daemons sharing one DORMICE_NODE_ID (the default is node-1) flipped the endpoint at every check-in; a lookup asked whichever was current, a name on the other read as new and was built again, on two nodes, with no 409 ever. Such a check-in is now refused (409) naming both addresses; the first reporter keeps the id; a node that really moved is taken an interval later. The 503 for a silent node now says the node may be busy building that very name (its lookup waits for the name slot, and a cold create outlasts the gateway's two seconds), and the shared lookup comment no longer promises that a retry finds a sandbox still being built. --- packages/gateway/src/app.test.ts | 86 +++++++++++++++++++++++++- packages/gateway/src/find.ts | 50 ++++++++++++--- packages/gateway/src/fleet.test.ts | 58 ++++++++++++----- packages/gateway/src/fleet.ts | 36 ++++++++--- packages/gateway/src/placement.test.ts | 4 +- packages/gateway/src/routes/e2b.ts | 7 ++- packages/gateway/src/routes/native.ts | 10 ++- packages/gateway/src/routes/nodes.ts | 36 ++++++++++- packages/gateway/src/routes/verdict.ts | 2 +- packages/shared/src/gateway.ts | 5 +- packages/shared/src/lookup.ts | 7 ++- 11 files changed, 260 insertions(+), 41 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 573eda1f..080bee39 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -375,6 +375,20 @@ describe('check-in and the node verbs', () => { ).toBe(8); const moved = new FakeNode('b'); await moved.start(); + // Inside the first reporter's interval a different address is a second + // daemon under one DORMICE_NODE_ID, refused with why; the first keeps + // the id. An interval later the same report is a move. + const twin = await rpc( + h, + '/checkIn', + checkInOf('b', moved.endpoint, { active: 3 }), + ); + expect(twin.status).toBe(409); + expect(message(twin)).toMatch(/two daemons share one DORMICE_NODE_ID/); + expect(h.fleet.get('b')?.endpoint).toBe(h.nodes[0]?.endpoint); + const first = h.fleet.get('b'); + if (!first) throw new Error('node lost'); + first.lastCheckInAt = new Date(Date.now() - 16_000); await h.checkIn(moved, { active: 3 }); expect(h.fleet.get('b')?.endpoint).toBe(moved.endpoint); expect(h.fleet.get('b')?.reading?.sandboxes.byState.active).toBe(3); @@ -404,6 +418,17 @@ describe('check-in and the node verbs', () => { const h = await gateway(['b', 'c']); const created = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'x' })); expect(h.cache.getByName('x')?.nodeId).toBe(created.nodeId); + // Still checking in: refused, with what to do instead — its names + // would be placed elsewhere before its next check-in and come back on + // two nodes. + const live = await rpc(h, '/removeNode', { id: created.nodeId }); + expect(live.status).toBe(409); + expect(message(live)).toMatch(/checked in \ds ago — it is running/); + expect(h.fleet.get(created.nodeId)).toBeDefined(); + // Silent for two of its intervals: down, and removable. + const silent = h.fleet.get(created.nodeId); + if (!silent) throw new Error('node lost'); + silent.lastCheckInAt = new Date(Date.now() - 31_000); expect((await rpc(h, '/removeNode', { id: created.nodeId })).body).toEqual({ removed: true, }); @@ -424,7 +449,7 @@ describe('check-in and the node verbs', () => { }); describe('acquire: placing and finding', () => { - it('a new name lands on the emptiest node by active density, under the fleet token, and is cached: the second acquire asks nobody', async () => { + it('a new name lands on the emptiest node by active density, under the fleet token, and is cached: the second acquire asks only its node, once', async () => { const h = await gateway(['a', 'b']); const [a, b] = h.nodes as [FakeNode, FakeNode]; await h.checkIn(a, { active: 10, cores: 8 }); @@ -445,8 +470,58 @@ describe('acquire: placing and finding', () => { const again = await rpc(h, '/acquireSandbox', { name: 'alice' }); expect(sandboxOf(again).id).toBe(sandboxOf(first).id); expect((again.body as { created: boolean }).created).toBe(false); + // A creator confirms its cache hit with that one node (by id) before + // trusting it; the other node hears nothing. expect(a.lookups()).toBe(1); - expect(b.lookups()).toBe(1); + expect(b.lookups()).toBe(2); + expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); + }); + + it('a name whose sandbox its node has since removed on its own is a placement again: the cached node is asked first, the gate judges afresh, the count moves — on either face', async () => { + const h = await gateway(['a', 'b'], { + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', + }); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 0 }); + await h.checkIn(b, { active: 1 }); + const born = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'ttl' })); + expect(born.nodeId).toBe('a'); + expect(h.cache.getByName('ttl')?.nodeId).toBe('a'); + // The node reaps it on its own (an E2B deadline kill is the scanner's + // routine): the gateway hears nothing, the entry is stale. Meanwhile a + // fills up and b empties — the gate must judge afresh, not wake. + a.sandboxes.delete('ttl'); + await h.checkIn(a, { active: 2 }); + await h.checkIn(b, { active: 0 }); + const again = await rpc(h, '/acquireSandbox', { name: 'ttl' }); + expect(again.status).toBe(200); + expect(sandboxOf(again).nodeId).toBe('b'); + expect((again.body as { created: boolean }).created).toBe(true); + // Nothing was rebuilt on the full node; the placement is counted where + // it landed and the cache follows. + expect(a.creates).toBe(1); + expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); + expect(h.cache.getByName('ttl')?.nodeId).toBe('b'); + + // The E2B face takes the same slot and the same confirmation. + const create = (name: string) => + fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ metadata: { name } }), + }); + await h.checkIn(a, { active: 0 }); + await h.checkIn(b, { active: 2 }); + expect((await create('ttl-e2b')).status).toBe(201); + expect(h.cache.getByName('ttl-e2b')?.nodeId).toBe('a'); + a.sandboxes.delete('ttl-e2b'); + await h.checkIn(a, { active: 2 }); + await h.checkIn(b, { active: 0 }); + expect((await create('ttl-e2b')).status).toBe(201); + expect(h.cache.getByName('ttl-e2b')?.nodeId).toBe('b'); expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); }); @@ -578,6 +653,13 @@ describe('acquire: placing and finding', () => { expect(message(fresh)).toContain('node a did not answer'); expect(message(fresh)).toContain('cannot be treated as new'); expect(b.creates).toBe(0); + // Its socket is dead but its check-in was seconds ago: not yet "down", + // and removeNode says so. Two of its intervals of silence later it is. + const early = await rpc(h, '/removeNode', { id: 'a' }); + expect(early.status).toBe(409); + const silent = h.fleet.get('a'); + if (!silent) throw new Error('node lost'); + silent.lastCheckInAt = new Date(Date.now() - 31_000); expect((await rpc(h, '/removeNode', { id: 'a' })).body).toEqual({ removed: true, }); diff --git a/packages/gateway/src/find.ts b/packages/gateway/src/find.ts index d6f2ef99..8cd34a1f 100644 --- a/packages/gateway/src/find.ts +++ b/packages/gateway/src/find.ts @@ -41,26 +41,62 @@ export class Finder { private readonly log: FinderLog, ) {} - byName(name: string): Promise { - return this.find(this.cache.getByName(name), { name }); + /** + * `confirm`: a creator's cache hit is checked with the cached node before + * it is trusted. For every other verb a stale entry costs one misrouted + * request whose 404 evicts it (verify below); a create forwarded as a + * wake to a node that no longer holds the name *builds* — the daemon's + * acquire and E2B create are create-or-wake — on a node placement never + * judged and never counted, and the daemon deletes rows on its own (an + * E2B deadline kill is the scanner's routine, five minutes by default), + * so every re-create of an expired name would pin to its first node + * past every gate. One question to one node, by id so no name slot is + * waited on; "absent" evicts and the fleet is asked afresh (found by + * review, 2026-09-14). + */ + byName(name: string, options: { confirm?: boolean } = {}): Promise { + return this.find( + this.cache.getByName(name), + { name }, + options.confirm === true, + ); } byId(id: string): Promise { - return this.find(this.cache.getById(id), { id }); + return this.find(this.cache.getById(id), { id }, false); } private async find( cached: CacheEntry | undefined, query: LookupQuery, + confirm: boolean, ): Promise { if (cached !== undefined) { const node = this.fleet.get(cached.nodeId); - // A node the operator removed while the entry was cached: the - // entry is stale by definition, and the fleet is asked afresh. - if (node !== undefined) { + if (node === undefined) { + // A node the operator removed while the entry was cached: the + // entry is stale by definition, and the fleet is asked afresh. + this.cache.evict(cached); + } else if (!confirm) { return { kind: 'one', node, id: cached.id, name: cached.name }; + } else { + const answer = await this.ask(node, { id: cached.id }); + if (answer.kind === 'found') { + return { kind: 'one', node, id: answer.id, name: answer.name }; + } + if (answer.kind === 'silent') { + // The one node that may hold it did not answer: not new, not + // known to be there — the same refusal a silent stranger earns. + const silent = [{ nodeId: node.id, why: answer.why }]; + this.log.warn( + { query, silent }, + 'lookup: the cached node did not confirm; the name cannot be treated as new', + ); + return { kind: 'unsure', silent }; + } + // The node itself says the sandbox is gone: the entry was stale. + this.cache.evict(cached); } - this.cache.evict(cached); } const members = this.fleet.all(); const answers = await Promise.all( diff --git a/packages/gateway/src/fleet.test.ts b/packages/gateway/src/fleet.test.ts index 97e3ab8b..287e184e 100644 --- a/packages/gateway/src/fleet.test.ts +++ b/packages/gateway/src/fleet.test.ts @@ -1,12 +1,18 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; import { migrateDb, openDb } from './db/db'; -import { downReason, Fleet } from './fleet'; +import { type CheckInOutcome, downReason, Fleet } from './fleet'; import { checkInOf } from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); const NOW = new Date('2026-09-14T12:00:00.000Z'); +/** A check-in the test expects to be taken. */ +function taken(outcome: CheckInOutcome) { + if ('refused' in outcome) throw new Error(outcome.refused); + return outcome; +} + function db() { const handle = openDb(':memory:'); migrateDb(handle, MIGRATIONS); @@ -18,9 +24,11 @@ describe('Fleet', () => { const handle = db(); const fleet = new Fleet(handle); expect(fleet.all()).toEqual([]); - const { node, joined } = fleet.checkIn( - checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), - NOW, + const { node, joined } = taken( + fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), + NOW, + ), ); expect(joined).toBe(true); expect(node.addedAt).toBe(NOW.toISOString()); @@ -39,14 +47,27 @@ describe('Fleet', () => { ); }); - it('a later check-in is not a join; a changed endpoint is written through; the placement counter restarts', () => { + it('a later check-in is not a join; a changed endpoint inside the interval is refused as a second daemon, an interval later written through as a move; the placement counter restarts', () => { const handle = db(); const fleet = new Fleet(handle); - const first = fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW); + const first = taken( + fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW), + ); first.node.placedSinceCheckIn = 3; - const second = fleet.checkIn( - checkInOf('node-b', 'http://10.0.0.8:80', { active: 12 }), - new Date(NOW.getTime() + 15_000), + const twin = fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.8:80'), + new Date(NOW.getTime() + 7_000), + ); + expect('refused' in twin && twin.refused).toMatch( + /^node node-b checked in from http:\/\/10\.0\.0\.7:80 7s ago and now from http:\/\/10\.0\.0\.8:80 — two daemons share one DORMICE_NODE_ID/, + ); + expect(first.node.endpoint).toBe('http://10.0.0.7:80'); + expect(first.node.placedSinceCheckIn).toBe(3); + const second = taken( + fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.8:80', { active: 12 }), + new Date(NOW.getTime() + 15_000), + ), ); expect(second.joined).toBe(false); expect(first.movedFrom).toBeNull(); @@ -62,18 +83,22 @@ describe('Fleet', () => { it('downReason: fresh within two of its own intervals, down past them', () => { const fleet = new Fleet(db()); - const { node } = fleet.checkIn( - checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), - NOW, + const { node } = taken( + fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), + NOW, + ), ); expect(downReason(node, new Date(NOW.getTime() + 29_000))).toBeNull(); expect(downReason(node, new Date(NOW.getTime() + 31_000))).toBe( 'has not checked in for 31s', ); // A one-second node (the exam's) is judged by its own interval. - const quick = fleet.checkIn( - checkInOf('node-c', 'http://10.0.0.9:80', { intervalSeconds: 1 }), - NOW, + const quick = taken( + fleet.checkIn( + checkInOf('node-c', 'http://10.0.0.9:80', { intervalSeconds: 1 }), + NOW, + ), ).node; expect(downReason(quick, new Date(NOW.getTime() + 1_500))).toBeNull(); expect(downReason(quick, new Date(NOW.getTime() + 2_500))).toBe( @@ -90,7 +115,8 @@ describe('Fleet', () => { expect(new Fleet(handle).all()).toEqual([]); expect(fleet.remove('node-b')).toBe(false); expect( - fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW).joined, + taken(fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW)) + .joined, ).toBe(true); }); }); diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 69e02140..079700bc 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -46,6 +46,11 @@ export function downReason(node: NodeState, now: Date): string | null { return null; } +/** What a check-in came to: taken (and whether it joined or moved), or refused with the sentence the node is told (routes/nodes.ts answers 409). */ +export type CheckInOutcome = + | { node: NodeState; joined: boolean; movedFrom: string | null } + | { refused: string }; + /** * The fleet: every node that has ever checked in. Rows come from the * database at start (a node that is down must still be known — its names @@ -82,18 +87,22 @@ export class Fleet { * A node reporting for duty. A first check-in adds the node (`joined` * says so, for the log); a changed endpoint is written through — the * node states where it lives, the gateway does not remember better — - * and `movedFrom` names the old one, for the log: a node that moves at - * every check-in is two machines sharing one DORMICE_NODE_ID. The + * and `movedFrom` names the old one, for the log. Except inside the + * previous reporter's own interval: a different address that soon is a + * second machine with the same DORMICE_NODE_ID (the daemon's default is + * node-1), not a move, and is refused — written through, the two would + * flip the endpoint at every check-in, a lookup would ask whichever is + * current, a name on the other would read as new and be built again, on + * two nodes, with no 409 ever (traced by review, 2026-09-14). The first + * reporter keeps the id; the second is told why. A node that really + * moved is taken at its next check-in, one interval on. The * placement counter restarts at zero: what was placed before this * reading is in it now, and what is still in flight on the node (its * row is written after the container is up) is in neither figure until * the next reading — one interval of slack, self-correcting, the same * as before. */ - checkIn( - report: CheckInRequest, - now = new Date(), - ): { node: NodeState; joined: boolean; movedFrom: string | null } { + checkIn(report: CheckInRequest, now = new Date()): CheckInOutcome { let node = this.members.get(report.nodeId); let joined = false; let movedFrom: string | null = null; @@ -117,6 +126,19 @@ export class Fleet { this.members.set(node.id, node); joined = true; } else if (node.endpoint !== report.endpoint) { + if ( + node.lastCheckInAt !== null && + node.intervalSeconds !== null && + now.getTime() - node.lastCheckInAt.getTime() < + node.intervalSeconds * 1000 + ) { + const ago = Math.round( + (now.getTime() - node.lastCheckInAt.getTime()) / 1000, + ); + return { + refused: `node ${report.nodeId} checked in from ${node.endpoint} ${ago}s ago and now from ${report.endpoint} — two daemons share one DORMICE_NODE_ID (give this one its own), or the node just moved (then its next check-in, an interval later, is taken)`, + }; + } this.db .update(nodes) .set({ endpoint: report.endpoint }) @@ -134,7 +156,7 @@ export class Fleet { return { node, joined, movedFrom }; } - /** The operator's word that the node is gone for good; a node still running re-adds itself at its next check-in. */ + /** The operator's word that the node is gone for good (routes/nodes.ts refuses it for a node still checking in); one removed while briefly silent re-adds itself at its next check-in. */ remove(id: string): boolean { const existed = this.members.delete(id); this.db.delete(nodes).where(eq(nodes.id, id)).run(); diff --git a/packages/gateway/src/placement.test.ts b/packages/gateway/src/placement.test.ts index c9833ee2..165cf6de 100644 --- a/packages/gateway/src/placement.test.ts +++ b/packages/gateway/src/placement.test.ts @@ -29,10 +29,12 @@ function node( checkedInAt?: Date; } = {}, ): NodeState { - const { node } = f.checkIn( + const outcome = f.checkIn( checkInOf(id, `http://${id}:80`, over), over.checkedInAt ?? NOW, ); + if ('refused' in outcome) throw new Error(outcome.refused); + const { node } = outcome; node.placedSinceCheckIn = over.placed ?? 0; return node; } diff --git a/packages/gateway/src/routes/e2b.ts b/packages/gateway/src/routes/e2b.ts index a80443e9..3af2dcf8 100644 --- a/packages/gateway/src/routes/e2b.ts +++ b/packages/gateway/src/routes/e2b.ts @@ -140,7 +140,12 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( } const name = judged.data; return locks.run(name, async () => { - const found = verdict(await finder.byName(name), `sandbox "${name}"`); + // Confirmed with the cached node first (find.ts byName has why): + // the daemon's create builds what it does not find. + const found = verdict( + await finder.byName(name, { confirm: true }), + `sandbox "${name}"`, + ); if (found.kind === 'refuse') { return refuse(reply, found, (message) => ({ code: found.status, diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index 17cfafa3..1aec5452 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -166,11 +166,17 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( name: string, body: Buffer | undefined, ) { - const judged = verdict(await finder.byName(name), `sandbox "${name}"`); + // A creator confirms a cache hit with its node first (find.ts byName + // has why): the daemon's acquire builds what it does not find. + const judged = verdict( + await finder.byName(name, { confirm: true }), + `sandbox "${name}"`, + ); if (judged.kind === 'refuse') return refuse(reply, judged); let target = judged.kind === 'node' ? judged.node : null; // Found nowhere: a placement, counted against the node it lands on. - // Found somewhere: a wake, already in that node's reading. + // Found somewhere — and confirmed there: a wake, already in that + // node's reading. const placed = target === null; if (target === null) { if (clientGone(reply)) return; diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index 1240e56a..bf6987dc 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -15,6 +15,11 @@ export interface NodeRoutesOptions { cache: NameCache; } +/** A refusal in the native dialect, rendered by the app's error handler as `{ message }` under its status. */ +function refusal(statusCode: number, message: string): Error { + return Object.assign(new Error(message), { statusCode }); +} + /** * The gateway's own verbs about its nodes: the check-in the nodes send * (RULES/协议.md「网关」), and what an operator reads and does about them. @@ -34,7 +39,15 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( }, }, async (request) => { - const { node, joined, movedFrom } = fleet.checkIn(request.body); + const outcome = fleet.checkIn(request.body); + if ('refused' in outcome) { + request.log.warn( + { nodeId: request.body.nodeId, endpoint: request.body.endpoint }, + outcome.refused, + ); + throw refusal(409, outcome.refused); + } + const { node, joined, movedFrom } = outcome; if (joined) { request.log.info( { nodeId: node.id, endpoint: node.endpoint }, @@ -107,6 +120,27 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( }, }, async (request) => { + // "Gone for good" is refused for a node that is still checking in: + // its row would go, its names would be new names, and any of them + // acquired in the seconds before its next check-in would be built + // elsewhere — then the node re-adds itself and every such name is on + // two nodes, a 409 an operator clears by hand. Stop the daemon + // first; two of its intervals of silence is what "down" means + // (fleet.ts downReason), and a down node is removable (found by + // review, 2026-09-14). + const node = fleet.get(request.body.id); + if (node !== undefined) { + const now = new Date(); + if (downReason(node, now) === null && node.lastCheckInAt !== null) { + const ago = Math.round( + (now.getTime() - node.lastCheckInAt.getTime()) / 1000, + ); + throw refusal( + 409, + `node ${node.id} checked in ${ago}s ago — it is running, and its names would be placed elsewhere before it checked in again and come back as a 409 on two nodes; stop its daemon, wait two of its intervals (${node.intervalSeconds}s each), then remove it`, + ); + } + } const removed = fleet.remove(request.body.id); const evicted = cache.evictNode(request.body.id); if (removed) { diff --git a/packages/gateway/src/routes/verdict.ts b/packages/gateway/src/routes/verdict.ts index e4636633..f7eabd36 100644 --- a/packages/gateway/src/routes/verdict.ts +++ b/packages/gateway/src/routes/verdict.ts @@ -43,7 +43,7 @@ export function verdict(found: Found, what: string): Verdict { .map((s) => `node ${s.nodeId} did not answer (${s.why})`) .join( ', ', - )} — it cannot be treated as new while a node that may hold it is silent; retry, or remove the node if it is gone for good`, + )} — it cannot be treated as new while a node that may hold it is silent: that node may be down, or busy building this very name (its lookup waits for that); retry after Retry-After, and remove the node only if it is gone for good`, retryAfterSeconds: RETRY_AFTER_SECONDS, }; } diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts index 12d2c6b7..7022acfc 100644 --- a/packages/shared/src/gateway.ts +++ b/packages/shared/src/gateway.ts @@ -142,7 +142,10 @@ export type ListNodesResponse = z.infer; * goes, its sandboxes are no longer looked for, and a name that lived only * there is a new name again. A node that is merely down needs nothing — * it is back the moment it checks in — and one removed by mistake re-adds - * itself the same way. + * itself the same way. Refused (409) while the node is still checking in: + * a name of its acquired in the seconds before its next check-in would be + * built elsewhere and come back on two nodes. Stop the daemon, wait two of + * its intervals (that is "down"), then remove. */ export const removeNodeRequestSchema = z.object({ id: z.string().min(1), diff --git a/packages/shared/src/lookup.ts b/packages/shared/src/lookup.ts index c449b0af..8eb0a281 100644 --- a/packages/shared/src/lookup.ts +++ b/packages/shared/src/lookup.ts @@ -15,8 +15,11 @@ import { SANDBOX_STATES } from './states'; * row that exists answers at once, whatever its state; no row while an * acquire of that name is in flight waits for the acquire (the daemon * creates first and writes the row second, both under the slot) and looks - * again — so a gateway retrying a create whose answer was lost finds the - * sandbox on the node that built it, and never places a second copy. + * again — so a gateway retrying a create whose answer was lost never + * places a second copy: while the build is still running the gateway's + * two-second patience runs out first and the caller is told to retry + * (503 with Retry-After); once the row is written, the retry finds the + * sandbox on the node that built it. */ export const lookupSandboxRequestSchema = z.union([ z.object({ name: sandboxNameSchema }), From d3cc1309c3e182bcaf3890b83644dc013eb6f9af Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 05:14:43 +0800 Subject: [PATCH 16/26] forwardStream flushes the node's head as soon as it arrives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node holds a written head until the first body byte. A node stream that opens and then waits for its first event looked, through the gateway, like a node that had not answered — measured: a 300ms pause before the first byte held the status line for 300ms. The head now goes out when the node's did. --- packages/gateway/src/forward.test.ts | 23 +++++++++++++++++++++++ packages/gateway/src/forward.ts | 6 ++++++ 2 files changed, 29 insertions(+) diff --git a/packages/gateway/src/forward.test.ts b/packages/gateway/src/forward.test.ts index 58ef946d..60e5e24a 100644 --- a/packages/gateway/src/forward.test.ts +++ b/packages/gateway/src/forward.test.ts @@ -85,6 +85,29 @@ describe('forwardStream', () => { expect(res.headers['transfer-encoding']).toBe('chunked'); }); + it("the node's head reaches the client when it arrives, not with the first body byte", async () => { + let nodeEnded = false; + const node = http.createServer((_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain' }); + res.flushHeaders(); + setTimeout(() => { + nodeEnded = true; + res.end('late'); + }, 800); + }); + const endpoint = await listen(node); + const res = await request(`${await front(endpoint)}/execCommand`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: '{"name":"x"}', + }); + // undici resolves on the head; the node has not written a body byte. + expect(res.statusCode).toBe(200); + expect(nodeEnded).toBe(false); + expect(await res.body.text()).toBe('late'); + expect(nodeEnded).toBe(true); + }); + it("resolves with the node's status once the answer is relayed — the named verbs read a 404 off it", async () => { const node = http.createServer((_req, res) => { res.writeHead(404, { 'content-type': 'application/json' }); diff --git a/packages/gateway/src/forward.ts b/packages/gateway/src/forward.ts index 5fdc09a4..6b9917bf 100644 --- a/packages/gateway/src/forward.ts +++ b/packages/gateway/src/forward.ts @@ -205,6 +205,12 @@ export async function forwardStream( res.off('close', onClose); } res.writeHead(upstream.statusCode, inboundHeaders(upstream.headers)); + // Node holds a written head until the first body byte; the node's head + // has arrived, so it goes out now — a stream that opens and then waits + // (a process stream before its first event) must look open to the + // caller, not like a node that has not answered (found by review, + // 2026-09-14). + res.flushHeaders(); try { // pipeline destroys both ends on failure: a client that went away // aborts the node's response, a node that died cuts the client. From 4bb011a0bed301f65f3babb4b4c86a8b4c4ee24f Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 05:14:43 +0800 Subject: [PATCH 17/26] The check-in's first CPU reading is an honest null, not a percentage over a few milliseconds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sampler was primed right before the first check-in, so its first delta spanned the sliver between the two calls and read near 0 or near 100 by luck; a freshly restarted node could sit out its first interval on a number that meant nothing. Unprimed, the first reading is null — "no interval yet" — which placement lets through as unknown, as it was written to. --- packages/server/src/main.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index bd9fbe9f..6bc55694 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -325,8 +325,13 @@ let checkIn: CheckIn | undefined; if (config.DORMICE_GATEWAY_ENDPOINT !== undefined) { const nodeEndpoint = config.DORMICE_NODE_ENDPOINT ?? `http://127.0.0.1:${config.DORMICE_PORT}`; + // Not primed: the first check-in then reports cpuUsedPct null — "no + // interval yet" — which placement lets through as unknown. A sample a + // few milliseconds before it would make that first reading a percentage + // over the sliver in between, near 0 or near 100 by luck, and a freshly + // restarted node could sit out its first interval on a number that + // meant nothing (found by review, 2026-09-14). const checkInCpu = new CpuSampler(); - checkInCpu.sample(); checkIn = new CheckIn({ gateway: config.DORMICE_GATEWAY_ENDPOINT, token: config.DORMICE_API_TOKEN, From 00f6e458c845fbe2efacc38482e22aa676aa96e8 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 12:46:03 +0800 Subject: [PATCH 18/26] The lookup gives up in two seconds on a black-holed node too, and says why in words MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit undici's request() ignores an abort signal while the socket is still connecting, so a node whose host drops the SYN (a deleted VM, a closed security group — the case removeNode exists for) held every question for the dispatcher's 10s connect timeout, not the promised two seconds; and the creator's confirmation runs inside the name's slot, so queued acquires of a name cached there paid that in series. fetch honours the signal mid-connect (measured: request 10 500ms, fetch 2 001ms against 192.0.2.1); it follows no redirect here either. causeOf now prefers string codes: a DOMException's numeric legacy code rendered a timed-out node as "did not answer (23)". One causeOf, in lookup.ts, shared with forward.ts. The refused-connection test moved off port 9, which fetch refuses as a "bad port" without dialling. --- packages/gateway/src/find.test.ts | 34 ++++++++++++++++++++++-- packages/gateway/src/forward.ts | 7 +---- packages/gateway/src/lookup.ts | 44 ++++++++++++++++++++++++------- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/packages/gateway/src/find.test.ts b/packages/gateway/src/find.test.ts index f09805af..dcb4b260 100644 --- a/packages/gateway/src/find.test.ts +++ b/packages/gateway/src/find.test.ts @@ -6,7 +6,12 @@ import { NameCache } from './cache'; import { migrateDb, openDb } from './db/db'; import { Finder } from './find'; import { Fleet } from './fleet'; -import { type AskNode, httpAskNode, type LookupAnswer } from './lookup'; +import { + type AskNode, + httpAskNode, + LOOKUP_TIMEOUT_MS, + type LookupAnswer, +} from './lookup'; import { checkInOf } from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); @@ -245,8 +250,33 @@ describe('httpAskNode', () => { kind: 'silent', why: expect.stringMatching(/answered 401/), }); + // A port nobody listens on any more: the OS hands one out and it is + // released before the question is asked. (Not port 9 — fetch refuses + // the Fetch standard's "bad ports" without dialling.) + const released = http.createServer(); + await new Promise((r) => released.listen(0, '127.0.0.1', r)); + const port = (released.address() as AddressInfo).port; + await new Promise((r) => released.close(() => r())); expect( - await ask({ id: 'gone', endpoint: 'http://127.0.0.1:9' }, { id: 'x' }), + await ask( + { id: 'gone', endpoint: `http://127.0.0.1:${port}` }, + { id: 'x' }, + ), ).toMatchObject({ kind: 'silent', why: 'ECONNREFUSED' }); }); + + it('a node whose host swallows the connection is silent after the two seconds, not after a connect timeout — and the why is a word, not a number', async () => { + // 192.0.2.1 (TEST-NET-1) is routed nowhere: the SYN is dropped, the + // connect hangs. Where a network refuses it outright instead + // (ENETUNREACH), the answer is immediate and the assertion still holds. + const started = Date.now(); + const answer = await httpAskNode(TOKEN)( + { id: 'hole', endpoint: 'http://192.0.2.1:80' }, + { id: 'x' }, + ); + expect(Date.now() - started).toBeLessThan(LOOKUP_TIMEOUT_MS + 1_500); + expect(answer.kind).toBe('silent'); + expect(answer.kind === 'silent' && typeof answer.why).toBe('string'); + expect(answer.kind === 'silent' && answer.why).not.toMatch(/^\d+$/); + }, 15_000); }); diff --git a/packages/gateway/src/forward.ts b/packages/gateway/src/forward.ts index 6b9917bf..a867546d 100644 --- a/packages/gateway/src/forward.ts +++ b/packages/gateway/src/forward.ts @@ -3,6 +3,7 @@ import net from 'node:net'; import type { Duplex } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { Agent } from 'undici'; +import { causeOf } from './lookup'; /** * The one place the gateway talks to a node on a caller's behalf. Bytes @@ -131,12 +132,6 @@ function inboundHeaders( return out; } -function causeOf(error: unknown): string { - const e = error as { code?: string; message?: string; cause?: unknown }; - const cause = e.cause as { code?: string; message?: string } | undefined; - return cause?.code ?? cause?.message ?? e.code ?? e.message ?? String(error); -} - /** * The request target is sent to the node exactly as the caller wrote it — * origin and path handed to undici separately, never joined into a URL. diff --git a/packages/gateway/src/lookup.ts b/packages/gateway/src/lookup.ts index 11476283..ad471898 100644 --- a/packages/gateway/src/lookup.ts +++ b/packages/gateway/src/lookup.ts @@ -3,7 +3,6 @@ import { lookupSandboxResponseSchema, type SandboxState, } from '@dormice/shared'; -import { request } from 'undici'; /** * Asking one node "do you hold this sandbox?" — the daemon's lookupSandbox @@ -34,17 +33,41 @@ export type AskNode = ( query: LookupQuery, ) => Promise; +/** + * The transport's word for a failure, for a log line or a 503's sentence: + * a system code where there is one (ECONNREFUSED, UND_ERR_CONNECT_TIMEOUT), + * else the message. Only string codes count — a DOMException carries a + * numeric legacy `code` (TimeoutError is 23), and "node b did not answer + * (23)" tells an operator nothing (found by review, 2026-09-14). + */ export function causeOf(error: unknown): string { - const e = error as { code?: string; message?: string; cause?: unknown }; - const cause = e.cause as { code?: string; message?: string } | undefined; - return cause?.code ?? cause?.message ?? e.code ?? e.message ?? String(error); + const e = error as { code?: unknown; message?: string; cause?: unknown }; + const cause = e.cause as { code?: unknown; message?: string } | undefined; + if (typeof cause?.code === 'string') return cause.code; + if (cause?.message !== undefined) return cause.message; + if (typeof e.code === 'string') return e.code; + return e.message ?? String(error); } -/** The production asker: HTTP to the node's endpoint under the fleet's token. */ +/** + * The production asker: HTTP to the node's endpoint under the fleet's + * token. `fetch`, not undici's `request`, for the deadline: `request` + * ignores an abort signal while the socket is still connecting, so a node + * whose host drops the SYN (a VM deleted, a security group closed — the + * machine-gone case removeNode exists for) held every question for + * undici's 10s connect timeout, not two seconds — and the creator's + * confirmation runs inside the name's slot, so twenty queued acquires of a + * name cached there would have waited 3.5 minutes for the last (measured + * 2026-09-14: request 10 500ms, fetch 2 001ms, against 192.0.2.1). No + * redirect is followed: a front that redirects is not a node, and fetch + * would drop the Authorization header across origins on the way. One + * quirk comes with fetch: it refuses the Fetch standard's "bad ports" + * (9, 22, 25, 6000 …) without dialling — no node's front lives on one. + */ export function httpAskNode(token: string): AskNode { return async (node, query) => { try { - const res = await request(`${node.endpoint}/lookupSandbox`, { + const res = await fetch(`${node.endpoint}/lookupSandbox`, { method: 'POST', headers: { authorization: `Bearer ${token}`, @@ -52,15 +75,16 @@ export function httpAskNode(token: string): AskNode { }, body: JSON.stringify(query), signal: AbortSignal.timeout(LOOKUP_TIMEOUT_MS), + redirect: 'manual', }); - if (res.statusCode !== 200) { - const text = await res.body.text(); + if (res.status !== 200) { + const text = await res.text(); return { kind: 'silent', - why: `lookupSandbox answered ${res.statusCode}: ${text.slice(0, 200)}`, + why: `lookupSandbox answered ${res.status}: ${text.slice(0, 200)}`, }; } - const answer = lookupSandboxResponseSchema.parse(await res.body.json()); + const answer = lookupSandboxResponseSchema.parse(await res.json()); return answer.found ? { kind: 'found', ...answer.sandbox } : { kind: 'absent' }; From bafb6021122d486b377ca3f53b5fde8772f19667 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 12:46:52 +0800 Subject: [PATCH 19/26] A creator whose client left while it waited for the slot asks no node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clientGone was checked only at placement, after the lookup round: a creator queued behind a slow create or destroy whose client had given up still confirmed with the cached node — a question inside the slot, paid in series by every abandoned request behind a stuck name. It is now the first thing inside the slot on both faces, and still checked after the round of questions. The fake node learned a slow destroy for the test. --- packages/gateway/src/app.test.ts | 49 +++++++++++++++++++++++++-- packages/gateway/src/routes/create.ts | 10 +++--- packages/gateway/src/routes/e2b.ts | 3 ++ packages/gateway/src/routes/native.ts | 7 ++++ 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 080bee39..5820b2ea 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -29,6 +29,8 @@ class FakeNode { readonly hits: Array<{ path: string; auth: string | undefined }> = []; creates = 0; endpoint = ''; + /** How long a destroy takes to answer — a slow node holding the name's slot. */ + destroyTakesMs = 0; private readonly server: http.Server; constructor(readonly id: string) { @@ -145,8 +147,15 @@ class FakeNode { }); } case '/destroySandbox': { - if (name !== undefined) this.sandboxes.delete(name); - return json(200, { destroyed: found !== undefined }); + const destroy = () => { + if (name !== undefined) this.sandboxes.delete(name); + json(200, { destroyed: found !== undefined }); + }; + if (this.destroyTakesMs > 0) { + setTimeout(destroy, this.destroyTakesMs); + return; + } + return destroy(); } case '/execCommand': { if (!found) @@ -525,6 +534,42 @@ describe('acquire: placing and finding', () => { expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); }); + it('an acquire whose client left while it waited for the name\'s slot asks no node and builds nothing', async () => { + const h = await gateway(['a']); + const a = h.nodes[0] as FakeNode; + await rpc(h, '/acquireSandbox', { name: 'q' }); + // A destroy holds the slot; an acquire queues behind it, and its + // client gives up while queued. + a.destroyTakesMs = 400; + const destroying = rpc(h, '/destroySandbox', { name: 'q' }); + await until(() => + a.hits.some((hit) => hit.path === '/destroySandbox') ? true : undefined, + ); + const asked = a.lookups(); + const left = new AbortController(); + const abandoned = fetch(`${h.endpoint}/acquireSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'q' }), + signal: left.signal, + }).then( + () => 'answered', + () => 'left', + ); + await new Promise((r) => setTimeout(r, 50)); + left.abort(); + expect(await abandoned).toBe('left'); + expect((await destroying).body).toEqual({ destroyed: true }); + // The slot is free; the abandoned acquire ran and did nothing. + await new Promise((r) => setTimeout(r, 100)); + expect(a.lookups()).toBe(asked); + expect(a.creates).toBe(1); + expect(a.sandboxes.has('q')).toBe(false); + }); + it('twenty simultaneous acquires of one new name are one create on one node; different names spread', async () => { const h = await gateway(['a', 'b']); const [a, b] = h.nodes as [FakeNode, FakeNode]; diff --git a/packages/gateway/src/routes/create.ts b/packages/gateway/src/routes/create.ts index d9f700b1..9dd3ca31 100644 --- a/packages/gateway/src/routes/create.ts +++ b/packages/gateway/src/routes/create.ts @@ -57,10 +57,12 @@ export function place( } /** - * A create that waited for its name's slot behind a slow destroy and whose - * client left meanwhile: nothing is placed or counted for it — the - * response is gone, forwardCapture would send nothing. The reply is - * hijacked so Fastify writes nothing to the dead socket either. + * A create whose client has already left: nothing is asked, placed or + * counted for it — the response is gone, forwardCapture would send + * nothing. Checked first thing inside the slot (a creator that waited + * behind a slow create or destroy) and again after the lookup round (up + * to two seconds). The reply is hijacked so Fastify writes nothing to the + * dead socket either. */ export function clientGone(reply: FastifyReply): boolean { if (!reply.raw.destroyed) return false; diff --git a/packages/gateway/src/routes/e2b.ts b/packages/gateway/src/routes/e2b.ts index 3af2dcf8..4b5ad57e 100644 --- a/packages/gateway/src/routes/e2b.ts +++ b/packages/gateway/src/routes/e2b.ts @@ -140,6 +140,9 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( } const name = judged.data; return locks.run(name, async () => { + // A client that left while waiting for the slot asks nobody + // (native.ts acquire has why). + if (clientGone(reply)) return reply; // Confirmed with the cached node first (find.ts byName has why): // the daemon's create builds what it does not find. const found = verdict( diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index 1aec5452..9c61107a 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -166,6 +166,12 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( name: string, body: Buffer | undefined, ) { + // First, before any node is asked: a creator that waited for the slot + // behind a slow create or destroy and whose client left meanwhile + // asks nobody — the confirmation below is a question to a node inside + // the slot, and twenty abandoned acquires of one name would otherwise + // hold the slot for twenty answers nobody reads (create.ts clientGone). + if (clientGone(reply)) return; // A creator confirms a cache hit with its node first (find.ts byName // has why): the daemon's acquire builds what it does not find. const judged = verdict( @@ -179,6 +185,7 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( // node's reading. const placed = target === null; if (target === null) { + // Asked again: the round of questions took up to two seconds. if (clientGone(reply)) return; const placement = place(fleet, knobs, new Date()); if (placement.node === null) { From 42f9bd635abcbfb7dd464a19126e8b64072e098d Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 12:52:31 +0800 Subject: [PATCH 20/26] install.sh builds the gateway with the daemon and restarts a hand-installed gateway unit first A machine that runs both (the single-machine install is a fleet of one) upgraded with install.sh rebuilt only the server, CLI and console: the gateway's dist stayed on the older commit and ran it at its next restart, against a daemon whose check-in it may no longer parse. The gateway is now in the build filter, and a dormice-gateway unit that is running or enabled is restarted before the daemon so its first check-in lands on the new one. --- deploy/dormice-gateway.service | 4 +++- deploy/install.sh | 24 +++++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/deploy/dormice-gateway.service b/deploy/dormice-gateway.service index e8369bbc..45f3eded 100644 --- a/deploy/dormice-gateway.service +++ b/deploy/dormice-gateway.service @@ -3,7 +3,9 @@ # there — systemd's EnvironmentFile treats an inline comment as part of the # value). install.sh does not install this unit yet: the two-role install # (gateway + node on one machine, `--role node` elsewhere) is a later step; -# until then it is copied into place by hand (docs/测试机搭建手册.md). +# until then it is copied into place by hand (docs/测试机搭建手册.md). Once it +# is enabled, install.sh re-runs rebuild the gateway with the daemon and +# restart this unit before the daemon's, so both run one commit. [Unit] Description=Dormice gateway (fleet front door) Wants=network-online.target diff --git a/deploy/install.sh b/deploy/install.sh index 858f8b52..8b49fa30 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -520,11 +520,15 @@ build_repo() { else pnpm install --frozen-lockfile fi - # Build only what a daemon host runs: the server (plus its workspace deps), - # the CLI, and the console SPA. The website package is the project's Next.js - # marketing site — building it here would cost minutes and import a frontend - # toolchain's failure modes into an installer whose job is the daemon. - pnpm --filter "@dormice/server..." --filter "@dormice/cli..." --filter "@dormice/console" build + # Build only what a Dormice host runs: the server (plus its workspace deps), + # the CLI, the console SPA — and the gateway, whose dist must move with the + # daemon's on a machine that runs both (the single-machine install is a + # fleet of one): a gateway left on an older dist runs it at its next + # restart, against a daemon whose check-in it may no longer parse. Seconds + # to build. The website package is the project's Next.js marketing site — + # building it here would cost minutes and import a frontend toolchain's + # failure modes into an installer whose job is the daemon. + pnpm --filter "@dormice/server..." --filter "@dormice/gateway..." --filter "@dormice/cli..." --filter "@dormice/console" build } log "Dormice code ($INSTALL_DIR)" @@ -728,6 +732,16 @@ log 'systemd service' cp "$INSTALL_DIR/deploy/dormice.service" /etc/systemd/system/dormice.service systemctl daemon-reload systemctl enable dormice >/dev/null 2>&1 +# A gateway installed by hand on this machine (deploy/dormice-gateway.service; +# install.sh does not install it yet) was just rebuilt with the daemon and is +# restarted first, so the daemon's first check-in lands on the new one: the two +# processes of a fleet of one run one commit, never two. Running or enabled — +# a unit started by hand and never enabled is running the old dist just the +# same. +if systemctl is-active -q dormice-gateway 2>/dev/null || systemctl is-enabled -q dormice-gateway 2>/dev/null; then + systemctl restart dormice-gateway + note 'restarted dormice-gateway (hand-installed unit, rebuilt with the daemon)' +fi # Restart, not start: a re-run just built fresh code, and the daemon is # crash-only by design — restarting it is always safe. systemctl restart dormice From fb058e8d2fb1ccbaf27bf27f175de803d5bce3d1 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 12:52:45 +0800 Subject: [PATCH 21/26] A check-in refusal reaches the log whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The non-200 branch cut the gateway's answer at 200 characters. The gateway's longest refusal, the 409 naming both endpoints of a shared node id, runs to about 260 — the operator saw the diagnosis and lost the remedy. Cut at 400 now: whole enough for every sentence the gateway writes, short enough that a front's HTML error page does not flood the log. --- packages/server/src/check-in.test.ts | 14 ++++++++++++++ packages/server/src/check-in.ts | 7 ++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index f4636b82..5b162e7b 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -183,6 +183,20 @@ describe('CheckIn', () => { ); }); + it("a refusal's whole sentence reaches the log — the 409 for a shared node id names two endpoints and a remedy past the 200th character", async () => { + const refusal = + 'node node-7 checked in from http://10.0.0.7:80 3s ago and now from http://10.0.0.8:80 — two daemons share one DORMICE_NODE_ID (give this one its own), or the node just moved (then its next check-in, an interval later, is taken)'; + const gw = await gateway(() => ({ + status: 409, + body: JSON.stringify({ message: refusal }), + })); + const { log, details } = logSpy(); + await new CheckIn(options(gw.endpoint, log)).once(); + expect((details[0] as { error: string }).error).toBe( + `gateway answered 409: ${JSON.stringify({ message: refusal })}`, + ); + }); + it('a gateway that is not there is a logged failure, never a throw', async () => { const { log, warns } = logSpy(); const checkIn = new CheckIn(options('http://127.0.0.1:9', log)); diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index d5acc4da..c81c741e 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -113,9 +113,14 @@ export class CheckIn { if (res.status !== 200) { const text = await res.text(); const location = res.headers.get('location'); + // Whole enough for the gateway's own refusals (its longest, the + // 409 naming both endpoints of a shared node id, runs to about 260 + // characters — cut at 200 it lost its remedy), short enough that + // a front's HTML error page does not flood the log. + const body = text.slice(0, 400); throw new Error( location === null - ? `gateway answered ${res.status}: ${text.slice(0, 200)}` + ? `gateway answered ${res.status}: ${body}` : `gateway answered ${res.status} redirecting to ${location} — DORMICE_GATEWAY_ENDPOINT must be the gateway's own address, not a front that redirects`, ); } From 4996ab25f5a0c5d335793c8fdf9d3d879e3f3977 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 13:00:39 +0800 Subject: [PATCH 22/26] The slot comment in the native face tells the truth about the daemon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It claimed the daemon runs its other verbs unserialized; the daemon holds a slot per name for thirteen of the fifteen. The gateway's own reason for taking the slot only for acquire and destroy — the two verbs whose outcome it acts on — stands on its own and is now the one stated. --- packages/gateway/src/routes/native.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index 9c61107a..161d2f85 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -121,11 +121,14 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( return reply.code(400).send({ message: named.refusal }); } const { name } = named; - // Only the two verbs that create or remove take the name's slot — - // the daemon's own discipline (its other verbs run unserialized - // too). The slot is what keeps twenty simultaneous acquires of a - // new name from each placing their own copy: the first finds - // nothing and places, the rest find its cache entry. + // The gateway takes the name's slot for the two verbs whose + // outcome it acts on — a create it may have to place, a destroy it + // must forget — so twenty simultaneous acquires of a new name + // place once: the first finds nothing and places, the rest find + // its cache entry. Every other verb is relayed unserialized: the + // daemon holds its own slot per name for the ones that touch the + // sandbox (routes/sandboxes.ts locks.run), and a relay in front of + // it has nothing to add to their order. if (verb === 'acquireSandbox') { return locks.run(name, () => acquire(request, reply, name, body)); } From 7b914769553093b5ba4f32fa461d278d703a7068 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 13:00:39 +0800 Subject: [PATCH 23/26] The name cache is bounded, least recently used first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node deletes rows on its own (an E2B deadline kill is the scanner's routine) and tells no gateway, so the cache kept an entry for every sandbox ever created through the process for as long as it lived — an unnamed E2B create a minute is half a million dead entries a year. A hundred thousand entries, generous next to a fleet's live population; past it the entry nobody asked about for longest goes, and a hit moves an entry to the young end. A wrongly evicted entry costs the one round of questions any miss costs. --- packages/gateway/src/cache.test.ts | 60 ++++++++++++++++++++++++++++++ packages/gateway/src/cache.ts | 34 ++++++++++++++++- 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 packages/gateway/src/cache.test.ts diff --git a/packages/gateway/src/cache.test.ts b/packages/gateway/src/cache.test.ts new file mode 100644 index 00000000..bd1e464a --- /dev/null +++ b/packages/gateway/src/cache.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; +import { NameCache } from './cache'; + +const on = (id: string, name: string | null, nodeId = 'a') => ({ + id, + name, + nodeId, +}); + +describe('NameCache', () => { + it('answers by name and by id; a name that moves to a new id drops the old id whole; an id that changes name frees the old name', () => { + const cache = new NameCache(); + cache.put(on('sb-1', 'alice')); + expect(cache.getByName('alice')?.id).toBe('sb-1'); + expect(cache.getById('sb-1')?.name).toBe('alice'); + cache.put(on('sb-2', 'alice', 'b')); + expect(cache.getById('sb-1')).toBeUndefined(); + expect(cache.getByName('alice')?.nodeId).toBe('b'); + cache.put(on('sb-2', 'alicia', 'b')); + expect(cache.getByName('alice')).toBeUndefined(); + expect(cache.getByName('alicia')?.id).toBe('sb-2'); + expect(cache.size).toBe(1); + }); + + it('evict drops one entry, both handles; evictNode drops every entry of a node and counts them', () => { + const cache = new NameCache(); + cache.put(on('sb-1', 'alice')); + cache.put(on('sb-2', null)); + cache.put(on('sb-3', 'carol', 'b')); + cache.evict(on('sb-1', 'alice')); + expect(cache.getByName('alice')).toBeUndefined(); + expect(cache.getById('sb-1')).toBeUndefined(); + expect(cache.evictNode('a')).toBe(1); + expect(cache.getById('sb-2')).toBeUndefined(); + expect(cache.getByName('carol')?.id).toBe('sb-3'); + expect(cache.size).toBe(1); + }); + + it('is bounded: past the limit the entry least recently asked about goes, both handles; one just read stays', () => { + const cache = new NameCache(3); + cache.put(on('sb-1', 'alice')); + cache.put(on('sb-2', 'bob')); + cache.put(on('sb-3', null)); + // alice is the oldest write, but she was just asked about. + expect(cache.getByName('alice')).toBeDefined(); + cache.put(on('sb-4', 'dora')); + expect(cache.size).toBe(3); + expect(cache.getByName('bob')).toBeUndefined(); + expect(cache.getById('sb-2')).toBeUndefined(); + expect(cache.getByName('alice')?.id).toBe('sb-1'); + expect(cache.getById('sb-3')).toBeDefined(); + expect(cache.getByName('dora')).toBeDefined(); + // A re-put of a known id is a use too, not a second entry. + cache.put(on('sb-3', 'cathy')); + expect(cache.size).toBe(3); + cache.put(on('sb-5', 'eve')); + expect(cache.getByName('alice')).toBeUndefined(); + expect(cache.getByName('cathy')?.id).toBe('sb-3'); + }); +}); diff --git a/packages/gateway/src/cache.ts b/packages/gateway/src/cache.ts index 0803928a..a9736694 100644 --- a/packages/gateway/src/cache.ts +++ b/packages/gateway/src/cache.ts @@ -8,6 +8,16 @@ * and wholesale for a node an operator removed. An entry that is wrong * costs one misrouted request, whose answer evicts it; a cache that is * lost costs one extra round of questions per name. + * + * Bounded, least recently used first. A node deletes rows on its own — an + * E2B deadline kill is the scanner's routine — and tells no gateway, so + * an unbounded map would keep an entry for every sandbox ever created + * through this process, for as long as it lived: an unnamed E2B create + * every minute is half a million dead entries a year. The bound is + * generous next to a fleet's live population (Beijing holds some fifteen + * thousand rows, 2026-09), and past it the entry nobody has asked about + * for longest goes; asked about again, it costs the one round of + * questions any miss costs. */ export interface CacheEntry { id: string; @@ -16,10 +26,15 @@ export interface CacheEntry { nodeId: string; } +export const CACHE_LIMIT = 100_000; + export class NameCache { private readonly byName = new Map(); + /** Insertion order is recency: a hit re-inserts, and the first key is the least recently used. */ private readonly byId = new Map(); + constructor(private readonly limit = CACHE_LIMIT) {} + put(entry: CacheEntry): void { // A name that moves to a new id (destroyed and re-acquired elsewhere // while this gateway did not see the destroy) drops the old entry @@ -39,15 +54,30 @@ export class NameCache { ) { this.byName.delete(known.name); } + this.byId.delete(entry.id); this.byId.set(entry.id, entry); + if (this.byId.size > this.limit) { + const oldest = this.byId.values().next().value; + if (oldest !== undefined) this.evict(oldest); + } } getByName(name: string): CacheEntry | undefined { - return this.byName.get(name); + const entry = this.byName.get(name); + if (entry !== undefined) this.touch(entry); + return entry; } getById(id: string): CacheEntry | undefined { - return this.byId.get(id); + const entry = this.byId.get(id); + if (entry !== undefined) this.touch(entry); + return entry; + } + + /** Marks the entry as just used: back to the end of the recency order. */ + private touch(entry: CacheEntry): void { + this.byId.delete(entry.id); + this.byId.set(entry.id, entry); } evict(entry: CacheEntry): void { From e1790906979f988ec6006c05e70cc140459219c8 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 13:00:39 +0800 Subject: [PATCH 24/26] A 409 for two node ids at one endpoint says they are one daemon, and names the way out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node whose DORMICE_NODE_ID changed checks in as a new node while its old id keeps its row; both are asked, both answer from the same address, and every name there was a 409 telling the operator to destroy one copy — of a sandbox that exists once. The finding now carries the endpoints, and when they are one the sentence says so: remove the id that no longer checks in (or correct a DORMICE_NODE_ENDPOINT that names the wrong machine). Nothing is collapsed or healed; the fleet's list is wrong and the operator is told exactly how. --- packages/gateway/src/app.test.ts | 37 +++++++++++++++++++++++++- packages/gateway/src/find.test.ts | 36 ++++++++++++++++++++++++- packages/gateway/src/find.ts | 8 ++++-- packages/gateway/src/routes/verdict.ts | 20 ++++++++++++-- 4 files changed, 95 insertions(+), 6 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 5820b2ea..3d13bea4 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -534,7 +534,7 @@ describe('acquire: placing and finding', () => { expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); }); - it('an acquire whose client left while it waited for the name\'s slot asks no node and builds nothing', async () => { + it("an acquire whose client left while it waited for the name's slot asks no node and builds nothing", async () => { const h = await gateway(['a']); const a = h.nodes[0] as FakeNode; await rpc(h, '/acquireSandbox', { name: 'q' }); @@ -681,6 +681,41 @@ describe('acquire: placing and finding', () => { expect(sandboxOf(healed).nodeId).toBe('a'); }); + it("a node that checked in again under a new id answers every lookup twice: the 409 says the two ids are one endpoint and names the way out, not 'destroy one copy'", async () => { + const h = await gateway(['a', 'b']); + const [a] = h.nodes as [FakeNode, FakeNode]; + expect( + sandboxOf(await rpc(h, '/acquireSandbox', { name: 'kept' })).nodeId, + ).toBe('a'); + // The operator renamed DORMICE_NODE_ID on a's machine; the old row stays. + const renamed = await fetch(`${h.endpoint}/checkIn`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(checkInOf('a-renamed', a.endpoint)), + }); + expect(renamed.status).toBe(200); + h.cache.evict({ + id: sandboxOf(await rpc(h, '/acquireSandbox', { name: 'kept' })).id, + name: 'kept', + nodeId: 'a', + }); + const refused = await rpc(h, '/execCommand', { + name: 'kept', + command: 'true', + }); + expect(refused.status).toBe(409); + expect(message(refused)).toContain( + `nodes a and a-renamed, which are one endpoint (${a.endpoint})`, + ); + expect(message(refused)).toContain( + 'removeNode the id that no longer checks in', + ); + expect(message(refused)).not.toContain('destroy one copy'); + }); + it('a node that does not answer: its cached names 502, and a new name is a 503 with Retry-After naming it — until an operator removes it', async () => { const h = await gateway(['a', 'b']); const [a, b] = h.nodes as [FakeNode, FakeNode]; diff --git a/packages/gateway/src/find.test.ts b/packages/gateway/src/find.test.ts index dcb4b260..1b8a6023 100644 --- a/packages/gateway/src/find.test.ts +++ b/packages/gateway/src/find.test.ts @@ -83,10 +83,44 @@ describe('Finder', () => { const found = await new Finder(fleet, cache, ask, silentLog).byName( 'alice', ); - expect(found).toEqual({ kind: 'conflict', nodeIds: ['a', 'c'] }); + expect(found).toEqual({ + kind: 'conflict', + nodes: [ + { id: 'a', endpoint: 'http://a:80' }, + { id: 'c', endpoint: 'http://c:80' }, + ], + }); expect(cache.size).toBe(0); }); + it('two ids checked in from one endpoint answer twice for every name there: a conflict whose nodes share the endpoint', async () => { + const fleet = fleetOf('a', 'b'); + // The node at a's address checks in again under a new id: a rename. + fleet.checkIn(checkInOf('a-renamed', 'http://a:80'), NOW); + const { ask } = scripted({ + a: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + 'a-renamed': { + kind: 'found', + id: 'sb-1', + name: 'alice', + state: 'active', + }, + }); + const found = await new Finder( + fleet, + new NameCache(), + ask, + silentLog, + ).byName('alice'); + expect(found).toEqual({ + kind: 'conflict', + nodes: [ + { id: 'a', endpoint: 'http://a:80' }, + { id: 'a-renamed', endpoint: 'http://a:80' }, + ], + }); + }); + it('every node says no: the name is new; a silent node among the noes: unsure, naming it', async () => { const fleet = fleetOf('a', 'b'); expect( diff --git a/packages/gateway/src/find.ts b/packages/gateway/src/find.ts index 8cd34a1f..bc6191ea 100644 --- a/packages/gateway/src/find.ts +++ b/packages/gateway/src/find.ts @@ -16,7 +16,7 @@ import type { AskNode, LookupQuery } from './lookup'; */ export type Found = | { kind: 'one'; node: NodeState; id: string; name: string | null } - | { kind: 'conflict'; nodeIds: string[] } + | { kind: 'conflict'; nodes: Array<{ id: string; endpoint: string }> } | { kind: 'none' } | { kind: 'unsure'; silent: Array<{ nodeId: string; why: string }> }; @@ -119,9 +119,13 @@ export class Finder { return { kind: 'one', node: first.node, id: entry.id, name: entry.name }; } if (found.length > 1) { + // The endpoints ride along: two ids answering from one endpoint is + // one daemon under two names, which verdict.ts diagnoses as such. return { kind: 'conflict', - nodeIds: found.map((f) => f.node.id).sort(), + nodes: found + .map((f) => ({ id: f.node.id, endpoint: f.node.endpoint })) + .sort((x, y) => x.id.localeCompare(y.id)), }; } const silent = answers.flatMap(({ node, answer }) => diff --git a/packages/gateway/src/routes/verdict.ts b/packages/gateway/src/routes/verdict.ts index f7eabd36..038c09f6 100644 --- a/packages/gateway/src/routes/verdict.ts +++ b/packages/gateway/src/routes/verdict.ts @@ -27,14 +27,30 @@ export function verdict(found: Found, what: string): Verdict { return { kind: 'node', node: found.node, id: found.id, name: found.name }; case 'none': return { kind: 'none' }; - case 'conflict': + case 'conflict': { // The gateway refuses every verb for this name with this very 409, // destroy included — it will not guess which copy the caller means. + const ids = found.nodes.map((n) => n.id).join(' and '); + const endpoints = new Set(found.nodes.map((n) => n.endpoint)); + if (endpoints.size === 1) { + // Two ids, one endpoint: not two copies but one daemon answering + // twice — a node whose DORMICE_NODE_ID changed (its old id keeps + // its row until removed), or two nodes whose DORMICE_NODE_ENDPOINT + // name the same machine. Nothing to destroy; the fleet's list of + // nodes is wrong, and the check-in log said so when it happened + // (routes/nodes.ts). + return { + kind: 'refuse', + status: 409, + message: `${what} was answered for by nodes ${ids}, which are one endpoint (${[...endpoints][0]}) — one daemon under two node ids: a node whose DORMICE_NODE_ID changed and whose old id still has its row (removeNode the id that no longer checks in; listNodes shows which), or two nodes whose DORMICE_NODE_ENDPOINT name the same machine (correct the wrong one)`, + }; + } return { kind: 'refuse', status: 409, - message: `${what} exists on nodes ${found.nodeIds.join(' and ')} — destroy one copy directly on its node before routing can resume`, + message: `${what} exists on nodes ${ids} — destroy one copy directly on its node before routing can resume`, }; + } case 'unsure': return { kind: 'refuse', From 18024fc44d68e70f02ad7ce7b78be1a509f33985 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 13:03:29 +0800 Subject: [PATCH 25/26] The check-in log says why in the transport's word, and speaks again when the failure changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetch says "fetch failed" and keeps ECONNREFUSED, ENOTFOUND or the TLS error in cause; the log carried only the first half. And a failure was logged once per streak: a gateway that was unreachable and then, up again, refused this node as a twin of another (409) never made the log — the streak had already been announced. What is wrong is now compared with the numbers blanked, so a change of failure is one more line and a 409 that says 3s ago, then 4s ago, is still one. --- packages/server/src/check-in.test.ts | 37 ++++++++++++++++++++++++--- packages/server/src/check-in.ts | 38 ++++++++++++++++++++-------- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index 5b162e7b..666cfa82 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -197,11 +197,42 @@ describe('CheckIn', () => { ); }); - it('a gateway that is not there is a logged failure, never a throw', async () => { - const { log, warns } = logSpy(); - const checkIn = new CheckIn(options('http://127.0.0.1:9', log)); + it('a failure that changes is logged again — unreachable, then refused, are two events — while the same refusal with another number in it is not', async () => { + let answer = { status: 500, body: '{"message":"boom"}' }; + const gw = await gateway(() => answer); + const { log, warns, details } = logSpy(); + const checkIn = new CheckIn(options(gw.endpoint, log)); + await checkIn.once(); + await checkIn.once(); + expect(warns).toHaveLength(1); + const refusal = (ago: number) => + JSON.stringify({ + message: `node node-7 checked in from http://10.0.0.7:80 ${ago}s ago and now from http://10.0.0.8:80 — two daemons share one DORMICE_NODE_ID`, + }); + answer = { status: 409, body: refusal(3) }; + await checkIn.once(); + answer = { status: 409, body: refusal(4) }; + await checkIn.once(); + expect(warns).toHaveLength(2); + expect(warns[1]).toMatch(/still failing, differently/); + expect((details[1] as { error: string }).error).toMatch( + /gateway answered 409: .*3s ago/, + ); + }); + + it("a gateway that is not there is a logged failure, never a throw, and the log says why in the transport's word", async () => { + // A port the OS just released: dialling it is refused, not black-holed. + const probe = http.createServer(); + await new Promise((resolve) => probe.listen(0, '127.0.0.1', resolve)); + const port = (probe.address() as AddressInfo).port; + await new Promise((resolve) => probe.close(() => resolve())); + const { log, warns, details } = logSpy(); + const checkIn = new CheckIn(options(`http://127.0.0.1:${port}`, log)); await expect(checkIn.once()).resolves.toBeUndefined(); expect(warns).toHaveLength(1); + expect((details[0] as { error: string }).error).toMatch( + /fetch failed \(ECONNREFUSED\)/, + ); }); it('ticks on its interval from start() and stops on stop()', async () => { diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index c81c741e..a208d1a0 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -59,16 +59,31 @@ const CHECK_IN_TIMEOUT_MS = 10_000; * Chained setTimeout, the daemon's discipline: the next tick is scheduled * when this one is done, so a slow gateway never has ticks pile up. * Failures are logged on the change — once when the gateway stops - * answering, once when it answers again — never every tick: a gateway - * down for an hour is one event, not two hundred and forty lines. Never - * fatal: the gateway is the fleet's front door and configuration - * authority, not the node's reason to live; the node keeps running its - * sandboxes and keeps trying. + * answering, once more when what is wrong changes (a gateway that was + * unreachable and now refuses this node is news), once when it answers + * again — never every tick: a gateway down for an hour is one event, not + * two hundred and forty lines. Never fatal: the gateway is the fleet's + * front door and configuration authority, not the node's reason to live; + * the node keeps running its sandboxes and keeps trying. */ +/** + * A failure in the operator's words. fetch says "fetch failed" and keeps + * the reason (ECONNREFUSED, ENOTFOUND, a TLS error) in `cause`; a + * timeout is a DOMException whose only code is a legacy number. The + * transport's word is the one the operator acts on, so it is appended. + */ +function describe(error: unknown): string { + const e = error as { message?: string; cause?: unknown }; + const cause = e.cause as { code?: unknown; message?: string } | undefined; + const message = e.message ?? String(error); + const why = typeof cause?.code === 'string' ? cause.code : cause?.message; + return why === undefined ? message : `${message} (${why})`; +} + export class CheckIn { private timer: NodeJS.Timeout | undefined; private closing = false; - /** The failure the gateway is currently in, or null while it answers. */ + /** The failure the gateway is currently in (its sentence with the numbers blanked, so a 409 that says "3s ago" and then "4s ago" is one failure), or null while it answers. */ private failing: string | null = null; constructor(private readonly opts: CheckInOptions) {} @@ -130,14 +145,17 @@ export class CheckIn { this.failing = null; } } catch (error) { - const message = error instanceof Error ? error.message : String(error); - if (this.failing === null) { + const message = describe(error); + const failure = message.replace(/\d+/g, '#'); + if (failure !== this.failing) { opts.log.warn( { gateway: opts.gateway, error: message }, - 'check-in failed; the gateway places nothing here and forwards no new names to this node until it answers again — retrying every interval', + this.failing === null + ? 'check-in failed; the gateway places nothing here and forwards no new names to this node until it answers again — retrying every interval' + : 'check-in still failing, differently — retrying every interval', ); } - this.failing = message; + this.failing = failure; } } From 63dc74f3f58eb37e850cf114dbf4ec133478e9ce Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 13:03:29 +0800 Subject: [PATCH 26/26] A node whose gateway is on another machine must state its own DORMICE_NODE_ID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway tells nodes apart by it, and node-1 is what every other unconfigured node says too: the second to check in was refused as a twin at every check-in, or — when the first had been silent for an interval — taken for it having moved, and the first's names placed again elsewhere. The same rule as DORMICE_NODE_ENDPOINT, for the same reason: refused at boot, where the operator is looking. Beside its gateway, or alone, the default still serves. --- packages/server/src/config.test.ts | 29 +++++++++++++++++++++++++++++ packages/server/src/config.ts | 26 ++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index d07cebfa..c9a76efd 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -165,6 +165,7 @@ describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { ...TOKEN, DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677/', DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80///', + DORMICE_NODE_ID: 'node-7', DORMICE_CHECK_IN_INTERVAL_SECONDS: '5', }); expect(config.DORMICE_GATEWAY_ENDPOINT).toBe('http://10.0.0.5:3677'); @@ -220,6 +221,7 @@ describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { ...TOKEN, DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80', + DORMICE_NODE_ID: 'node-7', }).DORMICE_NODE_ENDPOINT, ).toBe('http://10.0.0.7:80'); for (const local of [ @@ -233,4 +235,31 @@ describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { ).toBeUndefined(); } }); + + it('a gateway on another machine requires a node id of its own, naming why; beside its gateway the default serves', () => { + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80', + }), + ).toThrow( + /DORMICE_NODE_ID is required when DORMICE_GATEWAY_ENDPOINT is not loopback/, + ); + expect( + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80', + DORMICE_NODE_ID: 'bj-7', + }).DORMICE_NODE_ID, + ).toBe('bj-7'); + expect( + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://127.0.0.1:3677', + }).DORMICE_NODE_ID, + ).toBe('node-1'); + expect(loadConfig(TOKEN).DORMICE_NODE_ID).toBe('node-1'); + }); }); diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index ddfac547..59a0e184 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -20,8 +20,13 @@ import type { S3Settings } from './archive/s3-store'; const envSchema = z.object({ DORMICE_PORT: z.coerce.number().int().min(1).max(65535).default(3676), DORMICE_DB_PATH: z.string().default('data/dormice.db'), - /** Identifies this machine in the ledger. Single-machine today; the field keeps the ledger shardable. */ - DORMICE_NODE_ID: z.string().default('node-1'), + /** + * This node's name: in its ledger's rows, and to its gateway, which + * tells nodes apart by it. The default serves a daemon that is the + * whole platform, or shares its machine with its gateway; a node whose + * gateway is elsewhere must state its own (checkedSchema below). + */ + DORMICE_NODE_ID: z.string().min(1).default('node-1'), /** How often the idle scanner sweeps the ledger. */ DORMICE_SCAN_INTERVAL_SECONDS: z.coerce.number().int().positive().default(60), /** @@ -301,6 +306,23 @@ const checkedSchema = envSchema path: ['DORMICE_NODE_ENDPOINT'], }, ) + // ...and who it is. The gateway tells nodes apart by DORMICE_NODE_ID, + // and node-1 is what every other unconfigured node says too: the + // second node-1 to check in is refused as a twin (409) at every + // check-in — or, when the first has been silent for an interval, taken + // for it having moved, and the first's names are placed again + // elsewhere. Refused here, at boot, where the operator is looking. + .refine( + (cfg) => + cfg.DORMICE_GATEWAY_ENDPOINT === undefined || + isLoopbackUrl(cfg.DORMICE_GATEWAY_ENDPOINT) !== false || + cfg.DORMICE_NODE_ID !== 'node-1', + { + message: + 'DORMICE_NODE_ID is required when DORMICE_GATEWAY_ENDPOINT is not loopback: the gateway tells nodes apart by it, and node-1 (the default) is what every other unconfigured node says — the second to check in is refused as a twin. Give this node a name of its own, e.g. its hostname', + path: ['DORMICE_NODE_ID'], + }, + ) // All-or-none: a half-configured store would make the archiver's // existence ambiguous, and ambiguity here decides real policy defaults. .superRefine((cfg, ctx) => {