diff --git a/.changeset/gateway-check-in-and-lookup.md b/.changeset/gateway-check-in-and-lookup.md new file mode 100644 index 00000000..7561e255 --- /dev/null +++ b/.changeset/gateway-check-in-and-lookup.md @@ -0,0 +1,5 @@ +--- +"@dormice/shared": minor +--- + +Wire schemas for a fleet behind a gateway: `lookupSandbox` (a node answers "do you hold this sandbox?" by name or id), the node check-in (`checkInRequestSchema`, readings, build; its `endpoint` is an origin — `endpointSchema` drops a trailing slash and refuses a path), and the gateway's `listNodes` / `removeNode`. The host-metrics schema is split into named parts (`hostReadingSchema`, `dataDiskSchema`, `sandboxStateCountsSchema`) that `hostMetricsResponseSchema` still composes unchanged. diff --git a/README.md b/README.md index 4ea0e84a..8ca14744 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,7 @@ pnpm monorepo: | `packages/sdk` | `@dormice/sdk` — TypeScript client for the native API | | `packages/cli` | `dormice` command-line tool (`dor` for short) | | `packages/console` | Web console: React SPA, served by the daemon at `/console` | +| `packages/gateway` | The fleet's one door in front of one or more daemons: places new sandboxes, finds existing ones by asking the nodes, forwards everything else | | `e2e` | Black-box suite: boots the built daemon, drives it over the wire | | `examples` | Runnable demos: the native SDK, the official `e2b` package, a resident agent | diff --git a/deploy/dormice-gateway.service b/deploy/dormice-gateway.service new file mode 100644 index 00000000..45f3eded --- /dev/null +++ b/deploy/dormice-gateway.service @@ -0,0 +1,26 @@ +# Dormice gateway: the fleet's one door in front of one or more daemons. +# Configuration lives in /etc/dormice/gateway.env (full-line comments only +# there — systemd's EnvironmentFile treats an inline comment as part of the +# value). install.sh does not install this unit yet: the two-role install +# (gateway + node on one machine, `--role node` elsewhere) is a later step; +# until then it is copied into place by hand (docs/测试机搭建手册.md). Once it +# is enabled, install.sh re-runs rebuild the gateway with the daemon and +# restart this unit before the daemon's, so both run one commit. +[Unit] +Description=Dormice gateway (fleet front door) +Wants=network-online.target +After=network-online.target + +[Service] +ExecStart=/usr/local/bin/node /opt/dormice/packages/gateway/dist/main.js +EnvironmentFile=/etc/dormice/gateway.env +# Crash-only: the gateway holds no sandbox state — nodes check in again +# within seconds and the cache refills by asking — so restarting it is +# always safe. +Restart=always +RestartSec=3 +# One process per database file. The SQLite lock enforces this; never wrap +# the gateway in a process manager that forks workers. + +[Install] +WantedBy=multi-user.target diff --git a/deploy/install.sh b/deploy/install.sh index 858f8b52..8b49fa30 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -520,11 +520,15 @@ build_repo() { else pnpm install --frozen-lockfile fi - # Build only what a daemon host runs: the server (plus its workspace deps), - # the CLI, and the console SPA. The website package is the project's Next.js - # marketing site — building it here would cost minutes and import a frontend - # toolchain's failure modes into an installer whose job is the daemon. - pnpm --filter "@dormice/server..." --filter "@dormice/cli..." --filter "@dormice/console" build + # Build only what a Dormice host runs: the server (plus its workspace deps), + # the CLI, the console SPA — and the gateway, whose dist must move with the + # daemon's on a machine that runs both (the single-machine install is a + # fleet of one): a gateway left on an older dist runs it at its next + # restart, against a daemon whose check-in it may no longer parse. Seconds + # to build. The website package is the project's Next.js marketing site — + # building it here would cost minutes and import a frontend toolchain's + # failure modes into an installer whose job is the daemon. + pnpm --filter "@dormice/server..." --filter "@dormice/gateway..." --filter "@dormice/cli..." --filter "@dormice/console" build } log "Dormice code ($INSTALL_DIR)" @@ -728,6 +732,16 @@ log 'systemd service' cp "$INSTALL_DIR/deploy/dormice.service" /etc/systemd/system/dormice.service systemctl daemon-reload systemctl enable dormice >/dev/null 2>&1 +# A gateway installed by hand on this machine (deploy/dormice-gateway.service; +# install.sh does not install it yet) was just rebuilt with the daemon and is +# restarted first, so the daemon's first check-in lands on the new one: the two +# processes of a fleet of one run one commit, never two. Running or enabled — +# a unit started by hand and never enabled is running the old dist just the +# same. +if systemctl is-active -q dormice-gateway 2>/dev/null || systemctl is-enabled -q dormice-gateway 2>/dev/null; then + systemctl restart dormice-gateway + note 'restarted dormice-gateway (hand-installed unit, rebuilt with the daemon)' +fi # Restart, not start: a re-run just built fresh code, and the daemon is # crash-only by design — restarting it is always safe. systemctl restart dormice diff --git a/e2e/package.json b/e2e/package.json index 9d104a0d..7c574ba9 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -14,6 +14,7 @@ "e2b": "^2.31.0" }, "devDependencies": { + "@dormice/gateway": "workspace:*", "@dormice/server": "workspace:*" } } diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts new file mode 100644 index 00000000..31c2f481 --- /dev/null +++ b/e2e/src/gateway.test.ts @@ -0,0 +1,559 @@ +import { spawn } from 'node:child_process'; +import { randomBytes, randomUUID } from 'node:crypto'; +import { mkdtemp, rm } from 'node:fs/promises'; +import net, { type AddressInfo } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Dormice } from '@dormice/sdk'; +import { Sandbox } from 'e2b'; +import { describe, expect, inject, it } from 'vitest'; + +// The gateway exam: two real daemons behind a real gateway, all three +// booted the production way and driven only over the wire — the SDK, the +// official e2b package and plain fetch. Direct calls to a node exist only +// to stage what the gateway must then find (a sandbox built behind its +// back, a name on two nodes, a destroy it did not see). Skipped in docker +// mode, where the setup boots node A alone. +const skip = inject('dormiceGatewayEndpoint') === null; + +const gateway = () => inject('dormiceGatewayEndpoint') as string; +const token = () => inject('dormiceGatewayToken') as string; +const nodes = () => + inject('dormiceGatewayNodes') as Array<{ id: string; endpoint: string }>; +const viaGateway = () => new Dormice({ endpoint: gateway(), token: token() }); +function direct(id: string) { + const node = nodes().find((n) => n.id === id); + if (!node) throw new Error(`no node ${id} in the exam fleet`); + return new Dormice({ endpoint: node.endpoint, token: token() }); +} +const other = (id: string) => (id === 'node-b' ? 'node-c' : 'node-b'); + +/** Polls until the probe answers something — nodes check in on their own clock, not ours. */ +async function until( + probe: () => Promise, + timeoutMs = 10_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + const value = await probe(); + if (value !== undefined) return value; + if (Date.now() > deadline) throw new Error('condition never became true'); + await new Promise((resolve) => setTimeout(resolve, 100)); + } +} + +async function rpc( + path: string, + payload: unknown = {}, + bearer = token(), + endpoint = gateway(), +): Promise<{ status: number; body: unknown; headers: Headers }> { + const res = await fetch(`${endpoint}${path}`, { + method: 'POST', + headers: { + authorization: `Bearer ${bearer}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(payload), + }); + const text = await res.text(); + return { + status: res.status, + body: text ? JSON.parse(text) : null, + headers: res.headers, + }; +} + +interface ListedNode { + id: string; + endpoint: string; + reachable: boolean; + lastCheckInAt: string | null; + build: { commit: string } | null; + reading: { sandboxes: { byState: { active: number } } } | null; + placedSinceCheckIn: number; +} +async function listNodes(): Promise { + const { status, body } = await rpc('/listNodes'); + expect(status).toBe(200); + return (body as { nodes: ListedNode[] }).nodes; +} + +const status = (error: unknown) => (error as { status?: number }).status; +const message = (r: { body: unknown }) => + (r.body as { message: string }).message; + +describe.skipIf(skip)('the gateway in front of two daemons', () => { + it('/healthz is open and names the build; a wrong token is refused', async () => { + const health = (await (await fetch(`${gateway()}/healthz`)).json()) as { + status: string; + build: { commit: string } | null; + }; + expect(health.status).toBe('ok'); + expect(health.build?.commit).toMatch(/^[0-9a-f]{7,}$/); + expect((await rpc('/listNodes', {}, 'x'.repeat(64))).status).toBe(401); + }); + + it('both nodes checked in: reachable, with a reading and the build they run', async () => { + const listed = await until(async () => { + const seen = await listNodes(); + return seen.length === 2 && seen.every((n) => n.reachable) + ? seen + : undefined; + }); + expect(listed.map((n) => n.id).sort()).toEqual(['node-b', 'node-c']); + for (const node of listed) { + expect(node.reading?.sandboxes.byState.active).toBeGreaterThanOrEqual(0); + expect(node.build?.commit).toMatch(/^[0-9a-f]{7,}$/); + expect(node.endpoint).toBe( + nodes().find((n) => n.id === node.id)?.endpoint, + ); + } + }); + + it('acquire is placed on one node and lands in exactly one ledger; re-acquire finds it there', async () => { + const created = await viaGateway().acquireSandbox('gw-place'); + try { + expect(created.status).toBe('ready'); + expect(created.created).toBe(true); + expect(['node-b', 'node-c']).toContain(created.sandbox.nodeId); + const here = await direct(created.sandbox.nodeId).listSandboxes(); + expect(here.some((s) => s.id === created.sandbox.id)).toBe(true); + const there = await direct(other(created.sandbox.nodeId)).listSandboxes(); + expect(there.some((s) => s.name === 'gw-place')).toBe(false); + + const again = await viaGateway().acquireSandbox('gw-place'); + expect(again.created).toBe(false); + expect(again.sandbox.id).toBe(created.sandbox.id); + expect(again.sandbox.nodeId).toBe(created.sandbox.nodeId); + } finally { + await viaGateway().destroySandbox('gw-place'); + } + }); + + it("files and commands round-trip; a node's 404 for a missing file passes through and the sandbox stays routable", async () => { + await viaGateway().acquireSandbox('gw-work'); + try { + await viaGateway().writeFile('gw-work', 'hello.txt', 'via gateway'); + const read = await viaGateway().readFile('gw-work', 'hello.txt'); + expect(new TextDecoder().decode(read.content)).toBe('via gateway'); + const ran = await viaGateway().execCommand('gw-work', 'echo through'); + expect(ran.stdout).toBe('through\n'); + + await expect( + viaGateway().readFile('gw-work', 'missing.txt'), + ).rejects.toMatchObject({ name: 'DormiceApiError', status: 404 }); + expect( + (await viaGateway().execCommand('gw-work', 'echo still-here')).stdout, + ).toBe('still-here\n'); + } finally { + await viaGateway().destroySandbox('gw-work'); + } + }); + + it('five simultaneous acquires of a new name are one sandbox on one node', async () => { + const results = await Promise.all( + Array.from({ length: 5 }, () => viaGateway().acquireSandbox('gw-burst')), + ); + try { + expect(new Set(results.map((r) => r.sandbox.id)).size).toBe(1); + expect(new Set(results.map((r) => r.sandbox.nodeId)).size).toBe(1); + expect(results.filter((r) => r.created)).toHaveLength(1); + } finally { + await viaGateway().destroySandbox('gw-burst'); + } + }); + + it('a sandbox built behind its back is found by asking — at once, no reconcile to wait for', async () => { + const staged = await direct('node-b').acquireSandbox('gw-staged'); + try { + const ran = await viaGateway().execCommand('gw-staged', 'echo found'); + expect(ran.stdout).toBe('found\n'); + const found = await viaGateway().acquireSandbox('gw-staged'); + expect(found.created).toBe(false); + expect(found.sandbox.id).toBe(staged.sandbox.id); + expect(found.sandbox.nodeId).toBe('node-b'); + } finally { + await viaGateway().destroySandbox('gw-staged'); + } + }); + + it('one name on two nodes is refused with a 409 naming both, and routable again once one copy is destroyed', async () => { + await direct('node-b').acquireSandbox('gw-twin'); + await direct('node-c').acquireSandbox('gw-twin'); + try { + await expect( + viaGateway().acquireSandbox('gw-twin'), + ).rejects.toMatchObject({ + status: 409, + message: expect.stringMatching(/node-b and node-c/), + }); + await expect( + viaGateway().execCommand('gw-twin', 'true'), + ).rejects.toMatchObject({ status: 409 }); + await direct('node-c').destroySandbox('gw-twin'); + const healed = await viaGateway().acquireSandbox('gw-twin'); + expect(healed.created).toBe(false); + expect(healed.sandbox.nodeId).toBe('node-b'); + } finally { + await direct('node-b').destroySandbox('gw-twin'); + await direct('node-c').destroySandbox('gw-twin'); + } + }); + + it('a destroy through the gateway is final; a destroy behind its back is caught on the next use and the name is asked for afresh', async () => { + const created = await viaGateway().acquireSandbox('gw-gone'); + expect(await viaGateway().destroySandbox('gw-gone')).toEqual({ + destroyed: true, + }); + await expect( + viaGateway().execCommand('gw-gone', 'true'), + ).rejects.toMatchObject({ + status: 404, + message: expect.stringMatching(/no sandbox named "gw-gone"/), + }); + expect(await viaGateway().destroySandbox('gw-gone')).toEqual({ + destroyed: false, + }); + const reborn = await viaGateway().acquireSandbox('gw-gone'); + expect(reborn.created).toBe(true); + expect(reborn.sandbox.id).not.toBe(created.sandbox.id); + try { + // Destroyed directly on its node, then rebuilt directly on the other: + // the gateway's first use relays the node's own 404, re-checks the + // cache off the request path, and the next use finds the new home. + // Without the re-check every use would keep going to the old node. + await direct(reborn.sandbox.nodeId).destroySandbox('gw-gone'); + await expect( + viaGateway().execCommand('gw-gone', 'true'), + ).rejects.toMatchObject({ status: 404 }); + const moved = await direct(other(reborn.sandbox.nodeId)).acquireSandbox( + 'gw-gone', + ); + await until(async () => { + try { + return await viaGateway().execCommand('gw-gone', 'echo moved'); + } catch (error) { + if (status(error) === 404) return undefined; + throw error; + } + }); + const found = await viaGateway().acquireSandbox('gw-gone'); + expect(found.created).toBe(false); + expect(found.sandbox.id).toBe(moved.sandbox.id); + } finally { + await viaGateway().destroySandbox('gw-gone'); + } + }); + + it('the active gate (2 per node here) refuses with a 503 naming every node, and reopens when a sandbox is destroyed', async () => { + const baseline = new Map( + (await listNodes()).map((n) => [ + n.id, + n.reading?.sandboxes.byState.active ?? 0, + ]), + ); + const names: string[] = []; + try { + const refusal = await until(async () => { + if (names.length > 8) throw new Error('gate never closed'); + const name = `gw-fill-${names.length}`; + const answer = await rpc('/acquireSandbox', { name }); + if (answer.status === 200) { + names.push(name); + return undefined; + } + return answer.status === 503 ? answer : undefined; + }, 20_000); + expect(message(refusal)).toContain('node-b'); + expect(message(refusal)).toContain('node-c'); + expect(message(refusal)).toContain('reach the 2 limit'); + expect(refusal.headers.get('retry-after')).toBe('15'); + + const freed = names.pop(); + if (freed) await viaGateway().destroySandbox(freed); + const reopened = await until(async () => { + try { + return await viaGateway().acquireSandbox('gw-fill-late'); + } catch (error) { + if (status(error) === 503) return undefined; + throw error; + } + }); + names.push('gw-fill-late'); + expect(reopened.status).toBe('ready'); + } finally { + for (const name of names) await viaGateway().destroySandbox(name); + // Leave the fleet as found: the gate reads the nodes' readings, which + // catch up with the destroys at their next check-in — the next test + // must not inherit a closed gate. + await until(async () => + (await listNodes()).every( + (n) => + n.reading?.sandboxes.byState.active === baseline.get(n.id) && + n.placedSinceCheckIn === 0, + ) + ? true + : undefined, + ); + } + }); + + it('daemon-addressed verbs are an honest 501; a misspelled verb is a 404', async () => { + await expect(viaGateway().listSandboxes()).rejects.toMatchObject({ + status: 501, + message: expect.stringMatching(/call the node directly/), + }); + const e2bList = await fetch(`${gateway()}/e2b/api/v2/sandboxes`, { + headers: { 'x-api-key': `e2b_${token()}` }, + }); + expect(e2bList.status).toBe(501); + expect(((await e2bList.json()) as { code: number }).code).toBe(501); + expect((await rpc('/acquireSandbx', { name: 'x' })).status).toBe(404); + }); + + it('the official e2b package works through the gateway: create, live streaming, files, kill; an unnamed create routes by id', async () => { + const connection = { + apiKey: `e2b_${token()}`, + apiUrl: `${gateway()}/e2b/api`, + sandboxUrl: `${gateway()}/e2b/envd`, + }; + const sbx = await Sandbox.create({ + ...connection, + metadata: { name: 'gw-e2b' }, + }); + try { + const chunks: Array<{ text: string; at: number }> = []; + const result = await sbx.commands.run( + 'echo first; sleep 1; echo second', + { + onStdout: (text) => { + chunks.push({ text, at: Date.now() }); + }, + }, + ); + expect(result.stdout).toBe('first\nsecond\n'); + // Streaming through two hops: a real gap between the frames, as in + // e2b.test.ts against the daemon alone — a buffering hop would + // deliver both at once. + const at = chunks.map((c) => c.at); + expect(at.length).toBeGreaterThanOrEqual(2); + expect((at.at(-1) ?? 0) - (at[0] ?? 0)).toBeGreaterThanOrEqual(500); + + await sbx.files.write('/home/user/gateway.txt', 'through the gateway'); + expect(await sbx.files.read('/home/user/gateway.txt')).toBe( + 'through the gateway', + ); + // The same name through the native face is the same sandbox. + const found = await viaGateway().acquireSandbox('gw-e2b'); + expect(found.created).toBe(false); + expect(found.sandbox.id).toBe(sbx.sandboxId); + } finally { + await sbx.kill(); + } + expect(await viaGateway().destroySandbox('gw-e2b')).toEqual({ + destroyed: false, + }); + + const anonymous = await Sandbox.create(connection); + try { + const info = await fetch( + `${connection.apiUrl}/sandboxes/${anonymous.sandboxId}`, + { headers: { 'x-api-key': connection.apiKey } }, + ); + expect(info.status).toBe(200); + } finally { + await anonymous.kill(); + } + }); + + it('envd preflights are answered without a header; the bare signed-URL door is a 501 with CORS', async () => { + const preflight = await fetch(`${gateway()}/e2b/envd/files`, { + method: 'OPTIONS', + headers: { origin: 'https://app.example' }, + }); + expect(preflight.status).toBe(204); + expect(preflight.headers.get('access-control-allow-origin')).toBe('*'); + const bare = await fetch(`${gateway()}/files?signature=x&path=/y`); + expect(bare.status).toBe(501); + expect(bare.headers.get('access-control-allow-origin')).toBe('*'); + expect(((await bare.json()) as { code: string }).code).toBe( + 'unimplemented', + ); + const stranger = await fetch(`${gateway()}/e2b/envd/files`, { + headers: { 'e2b-sandbox-id': randomUUID() }, + }); + expect(stranger.status).toBe(502); + expect(((await stranger.json()) as { code: string }).code).toBe( + 'unavailable', + ); + }); + + it('a third node joins at its first check-in; when it dies its sandboxes 502 and new names are a 503 naming it, until an operator removes it', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dormice-e2e-node-d-')); + const port = await freePort(); + const child = spawn('node', [inject('dormiceDaemonMain')], { + env: { + PATH: process.env.PATH ?? '', + DORMICE_NODE_ID: 'node-d', + DORMICE_PORT: String(port), + DORMICE_DATA_DIR: dir, + DORMICE_DB_PATH: join(dir, 'dormice.db'), + DORMICE_API_TOKEN: token(), + DORMICE_GATEWAY_ENDPOINT: gateway(), + DORMICE_CHECK_IN_INTERVAL_SECONDS: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stderr.on('data', (chunk) => { + output += chunk; + }); + const exited = new Promise((resolve) => + child.on('exit', () => resolve()), + ); + const d = new Dormice({ + endpoint: `http://127.0.0.1:${port}`, + token: token(), + }); + try { + await until(async () => + (await listNodes()).some((n) => n.id === 'node-d' && n.reachable) + ? true + : undefined, + ).catch((error) => { + throw new Error(`${String(error)}\n${output}`); + }); + const staged = await d.acquireSandbox('gw-on-d'); + expect( + (await viaGateway().execCommand('gw-on-d', 'echo on-d')).stdout, + ).toBe('on-d\n'); + + child.kill(); + await exited; + await expect( + viaGateway().execCommand('gw-on-d', 'true'), + ).rejects.toMatchObject({ + status: 502, + message: expect.stringMatching(/did not answer/), + }); + const refused = await rpc('/acquireSandbox', { name: 'gw-while-d-down' }); + expect(refused.status).toBe(503); + expect(message(refused)).toContain('node node-d did not answer'); + expect(refused.headers.get('retry-after')).toBe('15'); + // Down is also what listNodes says, on the node's own interval. + await until(async () => + (await listNodes()).some((n) => n.id === 'node-d' && !n.reachable) + ? true + : undefined, + ); + + expect((await rpc('/removeNode', { id: 'node-d' })).body).toEqual({ + removed: true, + }); + expect((await listNodes()).some((n) => n.id === 'node-d')).toBe(false); + const placed = await viaGateway().acquireSandbox('gw-while-d-down'); + try { + expect(['node-b', 'node-c']).toContain(placed.sandbox.nodeId); + } finally { + await viaGateway().destroySandbox('gw-while-d-down'); + } + expect(staged.sandbox.nodeId).toBe('node-d'); + } finally { + child.kill(); + await exited; + await rm(dir, { recursive: true, force: true }); + } + }); + + it("a gateway's boot, black-box: a second gateway on the same database file dies naming the conflict", async () => { + const dir = await mkdtemp(join(tmpdir(), 'dormice-e2e-gateway-boot-')); + try { + const env = async () => ({ + PATH: process.env.PATH ?? '', + DORMICE_GATEWAY_PORT: String(await freePort()), + DORMICE_GATEWAY_DB_PATH: join(dir, 'gateway.db'), + DORMICE_API_TOKEN: randomBytes(32).toString('hex'), + }); + const first = await bootGateway(await env()); + expect(first.outcome, first.output).toBe('healthy'); + try { + const second = await bootGateway(await env()); + expect(second.outcome, second.output).toEqual({ exitCode: 1 }); + expect(second.output).toMatch( + /another gateway is already running against .*gateway\.db/, + ); + expect(second.output).toContain('DORMICE_GATEWAY_DB_PATH'); + } finally { + await first.kill(); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); + +/** A port the OS just handed out and released — for a process this test boots itself. */ +function freePort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + const { port } = server.address() as AddressInfo; + server.close(() => resolve(port)); + }); + }); +} + +/** + * Boots a gateway the production way (`node dist/main.js` + environment) + * and reports how the boot ended: healthy at its port, or the exit code + * it died with — its whole output kept for the assertion either way. + */ +async function bootGateway(env: Record): Promise<{ + outcome: 'healthy' | { exitCode: number | null }; + output: string; + kill: () => Promise; +}> { + const child = spawn('node', [inject('dormiceGatewayMain')], { + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stderr.on('data', (chunk) => { + output += chunk; + }); + let exited: number | null | undefined; + const exit = new Promise((resolve) => { + child.on('exit', (code) => { + exited = code; + resolve(); + }); + }); + const endpoint = `http://127.0.0.1:${env.DORMICE_GATEWAY_PORT}`; + const deadline = Date.now() + 15_000; + while (Date.now() < deadline) { + if (exited !== undefined) { + return { outcome: { exitCode: exited }, output, kill: async () => {} }; + } + const healthy = await fetch(`${endpoint}/healthz`) + .then((r) => r.ok) + .catch(() => false); + if (healthy) break; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + return { + outcome: exited === undefined ? 'healthy' : { exitCode: exited }, + output, + kill: async () => { + child.kill(); + await exit; + }, + }; +} diff --git a/e2e/src/native.test.ts b/e2e/src/native.test.ts index 65635d55..cd0c2b48 100644 --- a/e2e/src/native.test.ts +++ b/e2e/src/native.test.ts @@ -1,3 +1,4 @@ +import { spawn } from 'node:child_process'; import { DEFAULT_LIFECYCLE_POLICY, Dormice } from '@dormice/sdk'; import { describe, expect, inject, it } from 'vitest'; @@ -111,6 +112,42 @@ describe('native API over a real daemon', () => { ).resolves.toBe(400); }); + it('a second daemon on the same ledger dies at boot naming the conflict — one ledger, one daemon', async () => { + // The same environment as the running daemon, another port: the port + // is not what must refuse it, the ledger lock is (measured 2026-09-11: + // with the lock handle garbage-collected, this second daemon started). + const child = spawn('node', [inject('dormiceDaemonMain')], { + env: { + ...inject('dormiceNodeAEnv'), + DORMICE_PORT: String( + Number(new URL(inject('dormiceEndpoint')).port) + 1000, + ), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stderr.on('data', (chunk) => { + output += chunk; + }); + const code = await new Promise((resolve) => { + const timer = setTimeout(() => { + child.kill(); + resolve(null); + }, 10_000); + child.on('exit', (exitCode) => { + clearTimeout(timer); + resolve(exitCode); + }); + }); + expect(code, output).toBe(1); + expect(output).toMatch( + /another daemon is already running against .*dormice\.db/, + ); + }); + it('destroys a sandbox: gone, forgotten, idempotent', async () => { const created = await client().acquireSandbox('destroy-key'); diff --git a/e2e/src/setup/daemon.ts b/e2e/src/setup/daemon.ts index 075aa6c9..41b11a4c 100644 --- a/e2e/src/setup/daemon.ts +++ b/e2e/src/setup/daemon.ts @@ -1,4 +1,4 @@ -import { spawn } from 'node:child_process'; +import { type ChildProcess, spawn } from 'node:child_process'; import { randomBytes } from 'node:crypto'; import { existsSync } from 'node:fs'; import { mkdtemp, rm } from 'node:fs/promises'; @@ -8,12 +8,31 @@ import { fileURLToPath } from 'node:url'; import { startMiniS3 } from '@dormice/server/mini-s3'; import type { TestProject } from 'vitest/node'; +export interface FleetNodeHandle { + id: string; + endpoint: string; +} + declare module 'vitest' { export interface ProvidedContext { dormiceEndpoint: string; dormiceToken: string; /** The Caddy config file the exam daemon owns — an operator-visible artifact. */ dormiceIngressFile: string; + /** The built daemon entry, for tests that boot a daemon of their own. */ + dormiceDaemonMain: string; + /** Node A's exact environment — a second daemon on the same ledger must refuse to start. */ + dormiceNodeAEnv: Record; + /** The gateway fronting nodes B and C; null in docker mode, where only node A runs. */ + dormiceGatewayEndpoint: string | null; + /** The fleet's one token: the gateway's, and every fronted node's. */ + dormiceGatewayToken: string | null; + /** The fronted nodes, reachable directly — to stage what the gateway must then find. */ + dormiceGatewayNodes: FleetNodeHandle[] | null; + /** The exam's S3, for a node a test boots itself. */ + dormiceMiniS3Url: string; + /** The built gateway entry, for tests that boot a gateway of their own (its boot refusals). */ + dormiceGatewayMain: string; } } @@ -25,26 +44,23 @@ declare module 'vitest' { const MAIN = fileURLToPath( new URL('../../../packages/server/dist/main.js', import.meta.url), ); +const GATEWAY_MAIN = fileURLToPath( + new URL('../../../packages/gateway/dist/main.js', import.meta.url), +); -export default async function setup(project: TestProject) { - if (!existsSync(MAIN)) { - throw new Error( - `daemon build not found at ${MAIN} — run \`pnpm build\` first`, - ); - } - - const token = randomBytes(32).toString('hex'); - const dataDir = await mkdtemp(join(tmpdir(), 'dormice-e2e-')); - // Random high port: never collides with a locally running daemon on 3676. - const port = 20000 + Math.floor(Math.random() * 20000); - const endpoint = `http://127.0.0.1:${port}`; - - // The exam's own S3 (in-process, test-only): the archive lifecycle runs - // black-box in every mode — the fake executor exports real files, the - // store speaks real HTTP. It also mimics OSS's checksum strictness, so a - // daemon that regressed pit #8 fails here too. - const miniS3 = await startMiniS3(); +interface DaemonSpec { + /** DORMICE_NODE_ID; omitted keeps the daemon's default, as node A always has. */ + nodeId?: string; + port: number; + token: string; + dataDir: string; + miniS3Url: string; + extraEnv?: Record; +} +/** Boots one daemon the production way and waits for /healthz. */ +async function bootDaemon(spec: DaemonSpec) { + const endpoint = `http://127.0.0.1:${spec.port}`; // An explicit allowlist instead of inheriting the whole environment: // whatever DORMICE_* knobs happen to be exported in the developer's shell // must not silently reconfigure the daemon under test. The three docker @@ -63,82 +79,211 @@ export default async function setup(project: TestProject) { inherited[name] = value; } } + const env: Record = { + // Exam disks evaporate with the exam: without this default, a docker + // run without an exported DORMICE_DATA_DIR drops its sandbox disks + // into /var/lib/dormice — the resident daemon's data dir, whose + // startup guard then refuses to start (measured 2026-07-10). An + // exported value still wins through `inherited` below. + DORMICE_DATA_DIR: spec.dataDir, + ...inherited, + ...(spec.nodeId === undefined ? {} : { DORMICE_NODE_ID: spec.nodeId }), + DORMICE_PORT: String(spec.port), + DORMICE_DB_PATH: join(spec.dataDir, 'dormice.db'), + DORMICE_API_TOKEN: spec.token, + // Sweep every second so lifecycle tests run on second-scale policies + // instead of the production default of days. + DORMICE_SCAN_INTERVAL_SECONDS: '1', + // Sample every second so history verbs have rows to answer with + // inside a test's lifetime. + DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS: '1', + // A wildcard sandbox domain so getHost() and the port proxy are + // exercised — no DNS needed, tests spoof the Host header locally. + // A first-boot seed since the setting moved into the ledger: every + // exam starts on a fresh DB, so the seed lands every run, and + // settings-hot.test.ts exercises the live edit on top of it. + DORMICE_SANDBOX_DOMAIN: 'sbx.dormice.test', + // The archiver, pointed at the exam's mini S3 — likewise a + // first-boot seed for the ledger's S3 settings. Deliberately NOT in + // the inherited allowlist: a developer's real DORMICE_S3_* exports + // must never leak an exam's archives into a production bucket. + DORMICE_S3_ENDPOINT: spec.miniS3Url, + DORMICE_S3_BUCKET: 'e2e-archive', + DORMICE_S3_ACCESS_KEY_ID: 'e2e-key', + DORMICE_S3_SECRET_ACCESS_KEY: 'e2e-secret', + DORMICE_S3_FORCE_PATH_STYLE: 'true', + // A managed ingress so the domain-binding verbs run black-box. The + // reload command is a no-op: the exam grades what the daemon writes + // and answers, not Caddy — Caddy's side is real-machine acceptance. + DORMICE_INGRESS_FILE: join(spec.dataDir, 'Caddyfile'), + DORMICE_INGRESS_RELOAD_CMD: 'true', + ...spec.extraEnv, + }; const child = spawn('node', [MAIN], { - env: { - // Exam disks evaporate with the exam: without this default, a docker - // run without an exported DORMICE_DATA_DIR drops its sandbox disks - // into /var/lib/dormice — the resident daemon's data dir, whose - // startup guard then refuses to start (measured 2026-07-10). An - // exported value still wins through `inherited` below. - DORMICE_DATA_DIR: dataDir, - ...inherited, - DORMICE_PORT: String(port), - DORMICE_DB_PATH: join(dataDir, 'dormice.db'), - DORMICE_API_TOKEN: token, - // Sweep every second so lifecycle tests run on second-scale policies - // instead of the production default of days. - DORMICE_SCAN_INTERVAL_SECONDS: '1', - // Sample every second so history verbs have rows to answer with - // inside a test's lifetime. - DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS: '1', - // A wildcard sandbox domain so getHost() and the port proxy are - // exercised — no DNS needed, tests spoof the Host header locally. - // A first-boot seed since the setting moved into the ledger: every - // exam starts on a fresh DB, so the seed lands every run, and - // settings-hot.test.ts exercises the live edit on top of it. - DORMICE_SANDBOX_DOMAIN: 'sbx.dormice.test', - // The archiver, pointed at the exam's mini S3 — likewise a - // first-boot seed for the ledger's S3 settings. Deliberately NOT in - // the inherited allowlist: a developer's real DORMICE_S3_* exports - // must never leak an exam's archives into a production bucket. - DORMICE_S3_ENDPOINT: miniS3.url, - DORMICE_S3_BUCKET: 'e2e-archive', - DORMICE_S3_ACCESS_KEY_ID: 'e2e-key', - DORMICE_S3_SECRET_ACCESS_KEY: 'e2e-secret', - DORMICE_S3_FORCE_PATH_STYLE: 'true', - // A managed ingress so the domain-binding verbs run black-box. The - // reload command is a no-op: the exam grades what the daemon writes - // and answers, not Caddy — Caddy's side is real-machine acceptance. - DORMICE_INGRESS_FILE: join(dataDir, 'Caddyfile'), - DORMICE_INGRESS_RELOAD_CMD: 'true', - }, + env, stdio: ['ignore', 'pipe', 'pipe'], }); + await waitHealthy(child, endpoint, `daemon ${spec.nodeId ?? 'A'}`); + return { + endpoint, + token: spec.token, + env, + ingressFile: join(spec.dataDir, 'Caddyfile'), + kill: () => child.kill(), + }; +} + +async function waitHealthy( + child: ChildProcess, + endpoint: string, + what: string, +): Promise { let output = ''; - child.stdout.on('data', (chunk) => { + child.stdout?.on('data', (chunk) => { output += chunk; }); - child.stderr.on('data', (chunk) => { + child.stderr?.on('data', (chunk) => { output += chunk; }); - const deadline = Date.now() + 10_000; for (;;) { if (child.exitCode !== null) { - throw new Error(`daemon exited during startup:\n${output}`); + throw new Error(`${what} exited during startup:\n${output}`); } try { const res = await fetch(`${endpoint}/healthz`); if (res.ok) { - break; + return; } } catch { // Not listening yet; keep probing until the deadline. } if (Date.now() > deadline) { child.kill(); - throw new Error(`daemon did not come up within 10s:\n${output}`); + throw new Error(`${what} did not come up within 10s:\n${output}`); } await new Promise((resolve) => setTimeout(resolve, 100)); } +} + +export default async function setup(project: TestProject) { + if (!existsSync(MAIN)) { + throw new Error( + `daemon build not found at ${MAIN} — run \`pnpm build\` first`, + ); + } + + const dataDir = await mkdtemp(join(tmpdir(), 'dormice-e2e-')); + // Random high base port: never collides with a locally running daemon on + // 3676 or a gateway on 3677; the fleet takes the next three. Below + // 32768, where Linux hands out ephemeral ports — a fleet port already + // taken by some outbound connection would be a boot failure for nothing. + const base = 20000 + Math.floor(Math.random() * 12000); - project.provide('dormiceEndpoint', endpoint); - project.provide('dormiceToken', token); - project.provide('dormiceIngressFile', join(dataDir, 'Caddyfile')); + // The exam's own S3 (in-process, test-only): the archive lifecycle runs + // black-box in every mode — the fake executor exports real files, the + // store speaks real HTTP. It also mimics OSS's checksum strictness, so a + // daemon that regressed pit #8 fails here too. + const miniS3 = await startMiniS3(); + + // Node A: the daemon every existing suite talks to, exactly as before — + // a standalone daemon that checks in with nobody. + const a = await bootDaemon({ + port: base, + token: randomBytes(32).toString('hex'), + dataDir, + miniS3Url: miniS3.url, + }); + project.provide('dormiceEndpoint', a.endpoint); + project.provide('dormiceToken', a.token); + project.provide('dormiceIngressFile', a.ingressFile); + project.provide('dormiceDaemonMain', MAIN); + project.provide('dormiceNodeAEnv', a.env); + project.provide('dormiceMiniS3Url', miniS3.url); + project.provide('dormiceGatewayMain', GATEWAY_MAIN); + + // The gateway exam: two more daemons behind a gateway, sharing A's mini + // S3 bucket the way a real fleet shares one, and sharing one token with + // the gateway the way a real fleet does. Fake mode only — in docker mode + // the startup guard judges containers by label across the whole machine, + // so two daemons on one docker would refuse each other. + const fleet: Array<{ kill: () => void }> = []; + const dirs: string[] = [dataDir]; + // Whatever came up before a boot failed is killed here: vitest never + // calls the teardown of a setup that threw, and a spawned daemon does + // not die with its parent. + const abandon = async (error: unknown) => { + for (const process of fleet.reverse()) process.kill(); + a.kill(); + await miniS3.close(); + throw error; + }; + if (process.env.DORMICE_EXECUTOR !== 'docker') { + if (!existsSync(GATEWAY_MAIN)) { + await abandon( + new Error( + `gateway build not found at ${GATEWAY_MAIN} — run \`pnpm build\` first`, + ), + ); + } + const gatewayPort = base + 3; + const gatewayEndpoint = `http://127.0.0.1:${gatewayPort}`; + const fleetToken = randomBytes(32).toString('hex'); + // The gateway first, so the nodes' first check-in lands; a gateway + // that comes up after its nodes learns them at their next check-in + // anyway, but the exam should not start on that slack. + const gateway = spawn('node', [GATEWAY_MAIN], { + env: { + PATH: process.env.PATH ?? '', + DORMICE_GATEWAY_PORT: String(gatewayPort), + DORMICE_GATEWAY_DB_PATH: join(dataDir, 'gateway.db'), + DORMICE_API_TOKEN: fleetToken, + // A tiny active limit so the gate can be reached with a handful of + // sandboxes; the CPU gate is opened wide — a laptop running the + // suite is not the machine under judgment; the disk floor is off + // for the same reason. + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '100', + DORMICE_GATEWAY_NODE_MIN_DISK_GB: '0', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + fleet.push({ kill: () => gateway.kill() }); + await waitHealthy(gateway, gatewayEndpoint, 'gateway').catch(abandon); + const nodes: FleetNodeHandle[] = []; + for (const [index, id] of (['node-b', 'node-c'] as const).entries()) { + const nodeDir = await mkdtemp(join(tmpdir(), `dormice-e2e-${id}-`)); + dirs.push(nodeDir); + const node = await bootDaemon({ + nodeId: id, + port: base + 1 + index, + token: fleetToken, + dataDir: nodeDir, + miniS3Url: miniS3.url, + extraEnv: { + DORMICE_GATEWAY_ENDPOINT: gatewayEndpoint, + // Second-scale check-ins so a node's readings and its absence + // show inside a test. + DORMICE_CHECK_IN_INTERVAL_SECONDS: '1', + }, + }).catch(abandon); + fleet.push(node); + nodes.push({ id, endpoint: node.endpoint }); + } + project.provide('dormiceGatewayEndpoint', gatewayEndpoint); + project.provide('dormiceGatewayToken', fleetToken); + project.provide('dormiceGatewayNodes', nodes); + } else { + project.provide('dormiceGatewayEndpoint', null); + project.provide('dormiceGatewayToken', null); + project.provide('dormiceGatewayNodes', null); + } return async () => { - child.kill(); + // The nodes first (they stop checking in), then the gateway. + for (const process of fleet.reverse()) process.kill(); + a.kill(); await miniS3.close(); - await rm(dataDir, { recursive: true, force: true }); + for (const dir of dirs) await rm(dir, { recursive: true, force: true }); }; } diff --git a/packages/gateway/drizzle.config.ts b/packages/gateway/drizzle.config.ts new file mode 100644 index 00000000..807d65f7 --- /dev/null +++ b/packages/gateway/drizzle.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'drizzle-kit'; + +export default defineConfig({ + schema: './src/db/schema.ts', + out: './drizzle', + dialect: 'sqlite', +}); diff --git a/packages/gateway/drizzle/0000_nodes.sql b/packages/gateway/drizzle/0000_nodes.sql new file mode 100644 index 00000000..e7f1a6a8 --- /dev/null +++ b/packages/gateway/drizzle/0000_nodes.sql @@ -0,0 +1,5 @@ +CREATE TABLE `nodes` ( + `id` text PRIMARY KEY NOT NULL, + `endpoint` text NOT NULL, + `added_at` text NOT NULL +); diff --git a/packages/gateway/drizzle/meta/0000_snapshot.json b/packages/gateway/drizzle/meta/0000_snapshot.json new file mode 100644 index 00000000..9a496820 --- /dev/null +++ b/packages/gateway/drizzle/meta/0000_snapshot.json @@ -0,0 +1,49 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "b88172b7-a470-4a8c-addb-45c45229585f", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "nodes": { + "name": "nodes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "added_at": { + "name": "added_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/_journal.json b/packages/gateway/drizzle/meta/_journal.json new file mode 100644 index 00000000..d08827ad --- /dev/null +++ b/packages/gateway/drizzle/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "7", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "6", + "when": 1789328056388, + "tag": "0000_nodes", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/packages/gateway/package.json b/packages/gateway/package.json new file mode 100644 index 00000000..d4928d73 --- /dev/null +++ b/packages/gateway/package.json @@ -0,0 +1,32 @@ +{ + "name": "@dormice/gateway", + "version": "0.0.0", + "private": true, + "description": "Dormice gateway: the fleet's one door — places new sandboxes across nodes, finds existing ones by asking, forwards everything else", + "license": "Apache-2.0", + "type": "module", + "files": [ + "dist", + "drizzle" + ], + "scripts": { + "build": "tsup", + "typecheck": "tsc --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@dormice/server": "workspace:*", + "@dormice/shared": "workspace:*", + "better-sqlite3": "^12.11.1", + "drizzle-orm": "^0.45.2", + "fastify": "^5.10.0", + "fastify-type-provider-zod": "^7.0.0", + "pino": "^10.3.1", + "undici": "^8.7.0", + "zod": "^4.4.3" + }, + "devDependencies": { + "@types/better-sqlite3": "^7.6.13", + "drizzle-kit": "^0.31.10" + } +} diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts new file mode 100644 index 00000000..3d13bea4 --- /dev/null +++ b/packages/gateway/src/app.test.ts @@ -0,0 +1,980 @@ +import { randomUUID } from 'node:crypto'; +import http from 'node:http'; +import net, { type AddressInfo } from 'node:net'; +import { fileURLToPath } from 'node:url'; +import { KeyedQueue } from '@dormice/server/keyed-queue'; +import { afterEach, describe, expect, it } from 'vitest'; +import { buildGatewayApp } from './app'; +import { NameCache } from './cache'; +import { loadConfig } from './config'; +import { migrateDb, openDb } from './db/db'; +import { Finder } from './find'; +import { Fleet } from './fleet'; +import { httpAskNode } from './lookup'; +import { checkInOf, type reading } from './testing'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const TOKEN = 'fleet-token-fleet-token-fleet-token-fleet'; + +/** + * A node as the gateway sees one: the daemon's wire for the handful of + * verbs the gateway touches, over a real socket. Every sandbox it holds is + * a row in `sandboxes`; every request it received is in `hits`. + */ +class FakeNode { + readonly sandboxes = new Map< + string, + { id: string; name: string; files: Map } + >(); + readonly hits: Array<{ path: string; auth: string | undefined }> = []; + creates = 0; + endpoint = ''; + /** How long a destroy takes to answer — a slow node holding the name's slot. */ + destroyTakesMs = 0; + private readonly server: http.Server; + + constructor(readonly id: string) { + this.server = http.createServer((req, res) => { + let text = ''; + req.on('data', (c) => { + text += c; + }); + req.on('end', () => this.answer(req, res, text)); + }); + } + + async start(): Promise { + await new Promise((r) => this.server.listen(0, '127.0.0.1', r)); + this.endpoint = `http://127.0.0.1:${(this.server.address() as AddressInfo).port}`; + return this; + } + + stop(): Promise { + return new Promise((resolve) => { + this.server.closeAllConnections(); + this.server.close(() => resolve()); + }); + } + + lookups(): number { + return this.hits.filter((h) => h.path === '/lookupSandbox').length; + } + + byId(id: string) { + return [...this.sandboxes.values()].find((s) => s.id === id); + } + + // Inferred return type on purpose: the early `return json(...)` exits + // read as statements, and an explicit void would flag each one. + private answer( + req: http.IncomingMessage, + res: http.ServerResponse, + text: string, + ) { + const url = req.url ?? '/'; + const path = url.split('?')[0] ?? url; + const auth = + req.headers.authorization ?? + (req.headers['x-api-key'] as string | undefined); + this.hits.push({ path, auth }); + const json = (status: number, body: unknown) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(body)); + }; + const body = text ? (JSON.parse(text) as Record) : {}; + if (path.startsWith('/e2b/envd/')) { + return json(200, { + envd: this.id, + sandboxId: req.headers['e2b-sandbox-id'], + accessToken: req.headers['x-access-token'], + url, + }); + } + if (path.startsWith('/e2b/api/')) { + if (auth !== `e2b_${TOKEN}`) + return json(401, { code: 401, message: 'invalid API key' }); + const m = path.match(/^\/e2b\/api\/sandboxes(?:\/([^/]+))?(\/.*)?$/); + if (!m) return json(404, { code: 404, message: 'not found' }); + const [, id, rest] = m; + if (id === undefined) { + if (req.method !== 'POST') + return json(404, { code: 404, message: 'not found' }); + const metadata = body.metadata as { name?: string } | undefined; + const name = metadata?.name ?? `e2b-${randomUUID()}`; + // The daemon's create is idempotent on metadata.name. + const sandbox = this.sandboxes.get(name) ?? this.create(name); + return json(201, { sandboxID: sandbox.id, templateID: 'base' }); + } + const sandbox = this.byId(id); + if (!sandbox) return json(404, { code: 404, message: 'not found' }); + if (rest) return json(200, { path: url, sandboxID: sandbox.id }); + if (req.method === 'DELETE') { + this.sandboxes.delete(sandbox.name); + res.writeHead(204); + res.end(); + return; + } + return json(200, { sandboxID: sandbox.id, alias: sandbox.name }); + } + if (auth !== `Bearer ${TOKEN}`) + return json(401, { message: 'missing or invalid API token' }); + const name = body.name as string | undefined; + const found = name === undefined ? undefined : this.sandboxes.get(name); + switch (path) { + case '/lookupSandbox': { + const sandbox = 'id' in body ? this.byId(body.id as string) : found; + return json( + 200, + sandbox + ? { + found: true, + sandbox: { + id: sandbox.id, + name: sandbox.name, + state: 'active', + }, + } + : { found: false }, + ); + } + case '/acquireSandbox': { + if (name === undefined) return json(400, { message: 'name required' }); + const sandbox = found ?? this.create(name); + return json(200, { + status: 'ready', + created: found === undefined, + sandbox: { id: sandbox.id, name, nodeId: this.id }, + }); + } + case '/destroySandbox': { + const destroy = () => { + if (name !== undefined) this.sandboxes.delete(name); + json(200, { destroyed: found !== undefined }); + }; + if (this.destroyTakesMs > 0) { + setTimeout(destroy, this.destroyTakesMs); + return; + } + return destroy(); + } + case '/execCommand': { + if (!found) + return json(404, { + message: `no sandbox named "${name}" — acquire it first`, + }); + return json(200, { stdout: `${this.id}: ${String(body.command)}` }); + } + case '/writeFile': { + if (!found) + return json(404, { + message: `no sandbox named "${name}" — acquire it first`, + }); + found.files.set(String(body.path), String(body.content)); + return json(200, { written: true }); + } + case '/readFile': { + if (!found) + return json(404, { + message: `no sandbox named "${name}" — acquire it first`, + }); + const content = found.files.get(String(body.path)); + if (content === undefined) + return json(404, { message: `no such file: ${String(body.path)}` }); + return json(200, { content }); + } + default: + return json(404, { message: `route ${req.method} ${url} not found` }); + } + } + + private create(name: string) { + this.creates += 1; + const sandbox = { id: randomUUID(), name, files: new Map() }; + this.sandboxes.set(name, sandbox); + return sandbox; + } +} + +interface Harness { + endpoint: string; + nodes: FakeNode[]; + cache: NameCache; + fleet: Fleet; + checkIn( + node: FakeNode, + over?: Parameters[0] & { intervalSeconds?: number }, + ): Promise; + close(): Promise; +} + +const harnesses: Harness[] = []; +afterEach(async () => { + for (const h of harnesses.splice(0)) await h.close(); +}); + +async function gateway( + nodeIds: string[], + env: Record = {}, +): Promise { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const config = loadConfig({ + DORMICE_API_TOKEN: TOKEN, + DORMICE_GATEWAY_DB_PATH: ':memory:', + // A laptop running the suite is not the machine under judgment. + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '100', + ...env, + }); + const fleet = new Fleet(db); + const cache = new NameCache(); + const finder = new Finder(fleet, cache, httpAskNode(TOKEN), { + warn: () => {}, + }); + const app = buildGatewayApp({ + config, + fleet, + finder, + locks: new KeyedQueue(), + logger: false, + build: null, + }); + await app.listen({ host: '127.0.0.1', port: 0 }); + const endpoint = `http://127.0.0.1:${(app.server.address() as AddressInfo).port}`; + const nodes = await Promise.all( + nodeIds.map((id) => new FakeNode(id).start()), + ); + const harness: Harness = { + endpoint, + nodes, + cache, + fleet, + checkIn: async (node, over = {}) => { + const res = await fetch(`${endpoint}/checkIn`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(checkInOf(node.id, node.endpoint, over)), + }); + expect(res.status).toBe(200); + }, + close: async () => { + await app.close(); + for (const node of nodes) await node.stop(); + }, + }; + for (const node of nodes) await harness.checkIn(node); + harnesses.push(harness); + return harness; +} + +async function rpc( + h: Harness, + path: string, + payload: unknown = {}, + token = TOKEN, +): Promise<{ status: number; body: unknown; headers: Headers }> { + const res = await fetch(`${h.endpoint}${path}`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(payload), + }); + const text = await res.text(); + return { + status: res.status, + body: text ? JSON.parse(text) : null, + headers: res.headers, + }; +} + +const message = (r: { body: unknown }) => + (r.body as { message: string }).message; +const sandboxOf = (r: { body: unknown }) => + (r.body as { created: boolean; sandbox: { id: string; nodeId: string } }) + .sandbox; + +/** Builds a sandbox directly on a node, behind the gateway's back. */ +async function stage(node: FakeNode, name: string) { + const res = await fetch(`${node.endpoint}/acquireSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name }), + }); + return sandboxOf({ body: await res.json() }); +} + +/** Polls until the probe answers something — cache verification runs off the request path. */ +async function until( + probe: () => Promise | T | undefined, + timeoutMs = 2_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + const value = await probe(); + if (value !== undefined) return value; + if (Date.now() > deadline) throw new Error('condition never became true'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +describe('the door', () => { + it('/healthz is open; everything else wants the fleet token, on the check-in too', async () => { + const h = await gateway([]); + const health = await fetch(`${h.endpoint}/healthz`); + expect(health.status).toBe(200); + expect(await health.json()).toEqual({ status: 'ok', build: null }); + expect((await rpc(h, '/listNodes', {}, 'w'.repeat(40))).status).toBe(401); + const bare = await fetch(`${h.endpoint}/checkIn`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(checkInOf('x', 'http://127.0.0.1:1')), + }); + expect(bare.status).toBe(401); + expect((await rpc(h, '/acquireSandbox', { name: 'x' }, 'bad')).status).toBe( + 401, + ); + }); + + it('refuses an absolute-form request target before routing', async () => { + const h = await gateway([]); + const url = new URL(h.endpoint); + const answer = await new Promise((resolve, reject) => { + let text = ''; + const socket = net.connect(Number(url.port), url.hostname, () => { + socket.write( + `POST http://evil.example/execCommand HTTP/1.1\r\nhost: gateway\r\nconnection: close\r\n\r\n`, + ); + }); + socket.on('data', (c) => { + text += c; + }); + socket.on('end', () => resolve(text)); + socket.on('error', reject); + }); + expect(answer).toContain('HTTP/1.1 400'); + expect(answer).toContain('origin-form'); + }); +}); + +describe('check-in and the node verbs', () => { + it('a check-in joins the fleet; listNodes shows what it said; a later check-in moves the endpoint; the row outlives the memory', async () => { + const h = await gateway(['b']); + const listed = (await rpc(h, '/listNodes')).body as { + nodes: Array>; + }; + expect(listed.nodes).toHaveLength(1); + expect(listed.nodes[0]).toMatchObject({ + id: 'b', + endpoint: h.nodes[0]?.endpoint, + reachable: true, + intervalSeconds: 15, + placedSinceCheckIn: 0, + build: { commit: 'abc1234' }, + }); + expect( + (listed.nodes[0]?.reading as { host: { cpuCount: number } }).host + .cpuCount, + ).toBe(8); + const moved = new FakeNode('b'); + await moved.start(); + // Inside the first reporter's interval a different address is a second + // daemon under one DORMICE_NODE_ID, refused with why; the first keeps + // the id. An interval later the same report is a move. + const twin = await rpc( + h, + '/checkIn', + checkInOf('b', moved.endpoint, { active: 3 }), + ); + expect(twin.status).toBe(409); + expect(message(twin)).toMatch(/two daemons share one DORMICE_NODE_ID/); + expect(h.fleet.get('b')?.endpoint).toBe(h.nodes[0]?.endpoint); + const first = h.fleet.get('b'); + if (!first) throw new Error('node lost'); + first.lastCheckInAt = new Date(Date.now() - 16_000); + await h.checkIn(moved, { active: 3 }); + expect(h.fleet.get('b')?.endpoint).toBe(moved.endpoint); + expect(h.fleet.get('b')?.reading?.sandboxes.byState.active).toBe(3); + await moved.stop(); + // A malformed check-in is a 400 naming the trouble, not a join. + const bad = await rpc(h, '/checkIn', { nodeId: 'z' }); + expect(bad.status).toBe(400); + expect(h.fleet.get('z')).toBeUndefined(); + // An endpoint the gateway could not use is refused at the wire (shared + // endpointSchema has the measurement): undici takes the endpoint as + // the request's origin and throws on a path — a node every lookup + // would find and no forward could reach. A trailing slash is merely + // dropped; the two ends must agree byte for byte. + const pathy = await rpc( + h, + '/checkIn', + checkInOf('p', 'http://10.0.0.9:80/dormice'), + ); + expect(pathy.status).toBe(400); + expect(message(pathy)).toMatch(/an endpoint is an origin/); + expect(h.fleet.get('p')).toBeUndefined(); + await rpc(h, '/checkIn', checkInOf('s', 'http://10.0.0.9:80/')); + expect(h.fleet.get('s')?.endpoint).toBe('http://10.0.0.9:80'); + }); + + it('removeNode forgets the node and everything cached on it; a removed node that checks in again re-joins', async () => { + const h = await gateway(['b', 'c']); + const created = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'x' })); + expect(h.cache.getByName('x')?.nodeId).toBe(created.nodeId); + // Still checking in: refused, with what to do instead — its names + // would be placed elsewhere before its next check-in and come back on + // two nodes. + const live = await rpc(h, '/removeNode', { id: created.nodeId }); + expect(live.status).toBe(409); + expect(message(live)).toMatch(/checked in \ds ago — it is running/); + expect(h.fleet.get(created.nodeId)).toBeDefined(); + // Silent for two of its intervals: down, and removable. + const silent = h.fleet.get(created.nodeId); + if (!silent) throw new Error('node lost'); + silent.lastCheckInAt = new Date(Date.now() - 31_000); + expect((await rpc(h, '/removeNode', { id: created.nodeId })).body).toEqual({ + removed: true, + }); + expect(h.cache.getByName('x')).toBeUndefined(); + expect( + ((await rpc(h, '/listNodes')).body as { nodes: unknown[] }).nodes, + ).toHaveLength(1); + expect((await rpc(h, '/removeNode', { id: 'nobody' })).body).toEqual({ + removed: false, + }); + const back = h.nodes.find((n) => n.id === created.nodeId); + if (!back) throw new Error('node lost'); + await h.checkIn(back); + expect( + ((await rpc(h, '/listNodes')).body as { nodes: unknown[] }).nodes, + ).toHaveLength(2); + }); +}); + +describe('acquire: placing and finding', () => { + it('a new name lands on the emptiest node by active density, under the fleet token, and is cached: the second acquire asks only its node, once', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 10, cores: 8 }); + await h.checkIn(b, { active: 1, cores: 8 }); + const first = await rpc(h, '/acquireSandbox', { name: 'alice' }); + expect(first.status).toBe(200); + expect(sandboxOf(first).nodeId).toBe('b'); + expect(b.creates).toBe(1); + expect(a.creates).toBe(0); + expect(b.hits.find((hit) => hit.path === '/acquireSandbox')?.auth).toBe( + `Bearer ${TOKEN}`, + ); + // Both nodes were asked once — the name was new — and the answer's id + // is cached beside the name. + expect(a.lookups()).toBe(1); + expect(b.lookups()).toBe(1); + expect(h.cache.getById(sandboxOf(first).id)?.nodeId).toBe('b'); + const again = await rpc(h, '/acquireSandbox', { name: 'alice' }); + expect(sandboxOf(again).id).toBe(sandboxOf(first).id); + expect((again.body as { created: boolean }).created).toBe(false); + // A creator confirms its cache hit with that one node (by id) before + // trusting it; the other node hears nothing. + expect(a.lookups()).toBe(1); + expect(b.lookups()).toBe(2); + expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); + }); + + it('a name whose sandbox its node has since removed on its own is a placement again: the cached node is asked first, the gate judges afresh, the count moves — on either face', async () => { + const h = await gateway(['a', 'b'], { + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', + }); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 0 }); + await h.checkIn(b, { active: 1 }); + const born = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'ttl' })); + expect(born.nodeId).toBe('a'); + expect(h.cache.getByName('ttl')?.nodeId).toBe('a'); + // The node reaps it on its own (an E2B deadline kill is the scanner's + // routine): the gateway hears nothing, the entry is stale. Meanwhile a + // fills up and b empties — the gate must judge afresh, not wake. + a.sandboxes.delete('ttl'); + await h.checkIn(a, { active: 2 }); + await h.checkIn(b, { active: 0 }); + const again = await rpc(h, '/acquireSandbox', { name: 'ttl' }); + expect(again.status).toBe(200); + expect(sandboxOf(again).nodeId).toBe('b'); + expect((again.body as { created: boolean }).created).toBe(true); + // Nothing was rebuilt on the full node; the placement is counted where + // it landed and the cache follows. + expect(a.creates).toBe(1); + expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); + expect(h.cache.getByName('ttl')?.nodeId).toBe('b'); + + // The E2B face takes the same slot and the same confirmation. + const create = (name: string) => + fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ metadata: { name } }), + }); + await h.checkIn(a, { active: 0 }); + await h.checkIn(b, { active: 2 }); + expect((await create('ttl-e2b')).status).toBe(201); + expect(h.cache.getByName('ttl-e2b')?.nodeId).toBe('a'); + a.sandboxes.delete('ttl-e2b'); + await h.checkIn(a, { active: 2 }); + await h.checkIn(b, { active: 0 }); + expect((await create('ttl-e2b')).status).toBe(201); + expect(h.cache.getByName('ttl-e2b')?.nodeId).toBe('b'); + expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); + }); + + it("an acquire whose client left while it waited for the name's slot asks no node and builds nothing", async () => { + const h = await gateway(['a']); + const a = h.nodes[0] as FakeNode; + await rpc(h, '/acquireSandbox', { name: 'q' }); + // A destroy holds the slot; an acquire queues behind it, and its + // client gives up while queued. + a.destroyTakesMs = 400; + const destroying = rpc(h, '/destroySandbox', { name: 'q' }); + await until(() => + a.hits.some((hit) => hit.path === '/destroySandbox') ? true : undefined, + ); + const asked = a.lookups(); + const left = new AbortController(); + const abandoned = fetch(`${h.endpoint}/acquireSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'q' }), + signal: left.signal, + }).then( + () => 'answered', + () => 'left', + ); + await new Promise((r) => setTimeout(r, 50)); + left.abort(); + expect(await abandoned).toBe('left'); + expect((await destroying).body).toEqual({ destroyed: true }); + // The slot is free; the abandoned acquire ran and did nothing. + await new Promise((r) => setTimeout(r, 100)); + expect(a.lookups()).toBe(asked); + expect(a.creates).toBe(1); + expect(a.sandboxes.has('q')).toBe(false); + }); + + it('twenty simultaneous acquires of one new name are one create on one node; different names spread', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + const burst = await Promise.all( + Array.from({ length: 20 }, () => + rpc(h, '/acquireSandbox', { name: 'burst' }), + ), + ); + expect(new Set(burst.map((r) => sandboxOf(r).id)).size).toBe(1); + expect(a.creates + b.creates).toBe(1); + expect( + burst.filter((r) => (r.body as { created: boolean }).created), + ).toHaveLength(1); + + const spread = await Promise.all( + Array.from({ length: 6 }, (_, i) => + rpc(h, '/acquireSandbox', { name: `spread-${i}` }), + ), + ); + const landed = spread.map((r) => sandboxOf(r).nodeId); + expect(landed.filter((id) => id === 'a').length).toBeGreaterThan(0); + expect(landed.filter((id) => id === 'b').length).toBeGreaterThan(0); + }); + + it('a sandbox built behind its back is found by asking: routed at once, re-acquired where it is', async () => { + const h = await gateway(['a', 'b']); + const b = h.nodes[1] as FakeNode; + const staged = sandboxOf( + await (async () => { + const res = await fetch(`${b.endpoint}/acquireSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'staged' }), + }); + return { body: await res.json() }; + })(), + ); + const ran = await rpc(h, '/execCommand', { name: 'staged', command: 'id' }); + expect(ran.status).toBe(200); + expect((ran.body as { stdout: string }).stdout).toBe('b: id'); + const found = await rpc(h, '/acquireSandbox', { name: 'staged' }); + expect(sandboxOf(found)).toMatchObject({ id: staged.id, nodeId: 'b' }); + expect((found.body as { created: boolean }).created).toBe(false); + }); + + it('a wake is not a placement: it neither counts against the node nor pays a placement back when destroyed — on either face', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 10 }); + await h.checkIn(b, { active: 1 }); + const placedOnB = () => h.fleet.get('b')?.placedSinceCheckIn; + // Sandboxes that already live on b: the gateway finds and wakes them + // there, and b's reading already counts them. + await stage(b, 'sleepy'); + await stage(b, 'dozy'); + expect( + sandboxOf(await rpc(h, '/acquireSandbox', { name: 'sleepy' })).nodeId, + ).toBe('b'); + expect(placedOnB()).toBe(0); + const e2bWake = await fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ metadata: { name: 'dozy' } }), + }); + expect(e2bWake.status).toBe(201); + expect(placedOnB()).toBe(0); + // A new name placed on b (the emptiest) is counted once. + expect( + sandboxOf(await rpc(h, '/acquireSandbox', { name: 'fresh' })).nodeId, + ).toBe('b'); + expect(placedOnB()).toBe(1); + // Destroying the woken sandboxes must not pay back a placement that + // never was: b would be judged one sandbox emptier than it is. + await rpc(h, '/destroySandbox', { name: 'sleepy' }); + expect(placedOnB()).toBe(1); + await rpc(h, '/destroySandbox', { name: 'dozy' }); + expect(placedOnB()).toBe(1); + await rpc(h, '/destroySandbox', { name: 'fresh' }); + expect(placedOnB()).toBe(0); + }); + + it('one name on two nodes is a 409 naming both, for every verb; once one copy is gone the name routes again', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + for (const node of [a, b]) { + await fetch(`${node.endpoint}/acquireSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'twin' }), + }); + } + for (const verb of ['/acquireSandbox', '/execCommand', '/destroySandbox']) { + const refused = await rpc(h, verb, { name: 'twin' }); + expect(refused.status).toBe(409); + expect(message(refused)).toContain('exists on nodes a and b'); + } + expect(h.cache.getByName('twin')).toBeUndefined(); + b.sandboxes.delete('twin'); + const healed = await rpc(h, '/acquireSandbox', { name: 'twin' }); + expect(sandboxOf(healed).nodeId).toBe('a'); + }); + + it("a node that checked in again under a new id answers every lookup twice: the 409 says the two ids are one endpoint and names the way out, not 'destroy one copy'", async () => { + const h = await gateway(['a', 'b']); + const [a] = h.nodes as [FakeNode, FakeNode]; + expect( + sandboxOf(await rpc(h, '/acquireSandbox', { name: 'kept' })).nodeId, + ).toBe('a'); + // The operator renamed DORMICE_NODE_ID on a's machine; the old row stays. + const renamed = await fetch(`${h.endpoint}/checkIn`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(checkInOf('a-renamed', a.endpoint)), + }); + expect(renamed.status).toBe(200); + h.cache.evict({ + id: sandboxOf(await rpc(h, '/acquireSandbox', { name: 'kept' })).id, + name: 'kept', + nodeId: 'a', + }); + const refused = await rpc(h, '/execCommand', { + name: 'kept', + command: 'true', + }); + expect(refused.status).toBe(409); + expect(message(refused)).toContain( + `nodes a and a-renamed, which are one endpoint (${a.endpoint})`, + ); + expect(message(refused)).toContain( + 'removeNode the id that no longer checks in', + ); + expect(message(refused)).not.toContain('destroy one copy'); + }); + + it('a node that does not answer: its cached names 502, and a new name is a 503 with Retry-After naming it — until an operator removes it', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 1 }); + await h.checkIn(b, { active: 50 }); + const onA = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'on-a' })); + expect(onA.nodeId).toBe('a'); + await a.stop(); + const cut = await rpc(h, '/execCommand', { name: 'on-a', command: 'x' }); + expect(cut.status).toBe(502); + expect(message(cut)).toMatch(/did not answer/); + const fresh = await rpc(h, '/acquireSandbox', { name: 'brand-new' }); + expect(fresh.status).toBe(503); + expect(fresh.headers.get('retry-after')).toBe('15'); + expect(message(fresh)).toContain('node a did not answer'); + expect(message(fresh)).toContain('cannot be treated as new'); + expect(b.creates).toBe(0); + // Its socket is dead but its check-in was seconds ago: not yet "down", + // and removeNode says so. Two of its intervals of silence later it is. + const early = await rpc(h, '/removeNode', { id: 'a' }); + expect(early.status).toBe(409); + const silent = h.fleet.get('a'); + if (!silent) throw new Error('node lost'); + silent.lastCheckInAt = new Date(Date.now() - 31_000); + expect((await rpc(h, '/removeNode', { id: 'a' })).body).toEqual({ + removed: true, + }); + const placed = await rpc(h, '/acquireSandbox', { name: 'brand-new' }); + expect(placed.status).toBe(200); + expect(sandboxOf(placed).nodeId).toBe('b'); + }); + + it('every node refusing is a 503 that names each one; no node at all says so', async () => { + const h = await gateway(['a', 'b'], { + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', + }); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 2 }); + await h.checkIn(b, { active: 1, diskAvail: 2 ** 30 }); + const refused = await rpc(h, '/acquireSandbox', { name: 'full' }); + expect(refused.status).toBe(503); + expect(message(refused)).toContain('a: 2 active sandboxes + 0 placed'); + expect(message(refused)).toContain('b: data disk has 1.0 GiB available'); + expect(refused.headers.get('retry-after')).toBe('15'); + + const empty = await gateway([]); + const nobody = await rpc(empty, '/acquireSandbox', { name: 'x' }); + expect(nobody.status).toBe(503); + expect(message(nobody)).toMatch(/no node has checked in/); + }); +}); + +describe('using, destroying, and the cache', () => { + it("files round-trip; a node's 404 for a missing file passes through and the sandbox stays cached", async () => { + const h = await gateway(['a']); + const a = h.nodes[0] as FakeNode; + await rpc(h, '/acquireSandbox', { name: 'w' }); + expect( + (await rpc(h, '/writeFile', { name: 'w', path: 'a.txt', content: 'hi' })) + .status, + ).toBe(200); + expect( + (await rpc(h, '/readFile', { name: 'w', path: 'a.txt' })).body, + ).toEqual({ content: 'hi' }); + const missing = await rpc(h, '/readFile', { name: 'w', path: 'nope' }); + expect(missing.status).toBe(404); + expect(message(missing)).toBe('no such file: nope'); + // The 404 sent one question to the node, which said yes: the entry stays. + await until(() => (a.lookups() >= 2 ? true : undefined)); + expect(h.cache.getByName('w')?.nodeId).toBe('a'); + expect( + (await rpc(h, '/execCommand', { name: 'w', command: 'ok' })).status, + ).toBe(200); + }); + + it('a destroy the gateway relays forgets the entry; a destroy behind its back is caught by the 404 re-check', async () => { + const h = await gateway(['a']); + const a = h.nodes[0] as FakeNode; + const first = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'd' })); + expect((await rpc(h, '/destroySandbox', { name: 'd' })).body).toEqual({ + destroyed: true, + }); + expect(h.cache.getByName('d')).toBeUndefined(); + expect(h.cache.getById(first.id)).toBeUndefined(); + // Placed and destroyed inside one interval: the placement no longer + // counts against the node — the reading will never show it. + expect(h.fleet.get('a')?.placedSinceCheckIn).toBe(0); + expect((await rpc(h, '/destroySandbox', { name: 'd' })).body).toEqual({ + destroyed: false, + }); + const second = await rpc(h, '/acquireSandbox', { name: 'd' }); + expect((second.body as { created: boolean }).created).toBe(true); + expect(sandboxOf(second).id).not.toBe(first.id); + + // Behind the gateway's back: the node's own 404 is relayed as it came, + // the re-check finds the sandbox absent, and the next request asks + // the fleet afresh — a 404 in the gateway's own words. + a.sandboxes.delete('d'); + const relayed = await rpc(h, '/execCommand', { name: 'd', command: 'x' }); + expect(relayed.status).toBe(404); + await until(() => + h.cache.getByName('d') === undefined ? true : undefined, + ); + const before = a.lookups(); + const own = await rpc(h, '/execCommand', { name: 'd', command: 'x' }); + expect(own.status).toBe(404); + expect(message(own)).toBe('no sandbox named "d" — acquire it first'); + expect(a.lookups()).toBe(before + 1); + }); + + it('daemon-addressed verbs are an honest 501, a misspelled verb a 404, a body without a name a 400', async () => { + const h = await gateway(['a']); + const listed = await rpc(h, '/listSandboxes'); + expect(listed.status).toBe(501); + expect(message(listed)).toContain('call the node directly'); + expect((await rpc(h, '/createApiKey', { name: 'k' })).status).toBe(501); + expect((await rpc(h, '/acquireSandbx', { name: 'x' })).status).toBe(404); + expect((await rpc(h, '/execCommand', { command: 'x' })).status).toBe(400); + expect(h.nodes[0]?.hits).toEqual([]); + }); + + it('a name the wire refuses is a 400 at the door on both faces, and no node is asked', async () => { + const h = await gateway(['a']); + const long = 'x'.repeat(129); + const native = await rpc(h, '/acquireSandbox', { name: long }); + expect(native.status).toBe(400); + expect(message(native)).toMatch(/^invalid name: /); + expect( + (await rpc(h, '/execCommand', { name: 7, command: 'x' })).status, + ).toBe(400); + const e2b = await fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ metadata: { name: long } }), + }); + expect(e2b.status).toBe(400); + expect(await e2b.json()).toMatchObject({ + code: 400, + message: expect.stringMatching(/^invalid metadata\.name: /), + }); + expect(h.nodes[0]?.hits).toEqual([]); + }); +}); + +describe('the E2B faces', () => { + const e2b = (h: Harness, path: string, init: RequestInit = {}) => + fetch(`${h.endpoint}/e2b/api${path}`, { + ...init, + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + ...(init.headers as Record | undefined), + }, + }); + + it('control plane: a named create is placed and cached; by-id verbs are found and forwarded under e2b_; the kill forgets the entry; an unnamed create is known by id', async () => { + const h = await gateway(['a', 'b']); + const created = await e2b(h, '/sandboxes', { + method: 'POST', + body: JSON.stringify({ metadata: { name: 'e2b-named' } }), + }); + expect(created.status).toBe(201); + const { sandboxID } = (await created.json()) as { sandboxID: string }; + const entry = h.cache.getById(sandboxID); + expect(entry?.name).toBe('e2b-named'); + const node = h.nodes.find((n) => n.id === entry?.nodeId); + if (!node) throw new Error('not placed'); + const info = await e2b(h, `/sandboxes/${sandboxID}`); + expect(info.status).toBe(200); + expect(node.hits.at(-1)?.auth).toBe(`e2b_${TOKEN}`); + const deeper = await e2b(h, `/sandboxes/${sandboxID}/metrics?x=1`, { + method: 'GET', + }); + expect(((await deeper.json()) as { path: string }).path).toBe( + `/e2b/api/sandboxes/${sandboxID}/metrics?x=1`, + ); + // The same name through the E2B face is the same sandbox (the name slot). + const again = await e2b(h, '/sandboxes', { + method: 'POST', + body: JSON.stringify({ metadata: { name: 'e2b-named' } }), + }); + expect(((await again.json()) as { sandboxID: string }).sandboxID).toBe( + sandboxID, + ); + const killed = await e2b(h, `/sandboxes/${sandboxID}`, { + method: 'DELETE', + }); + expect(killed.status).toBe(204); + expect(h.cache.getById(sandboxID)).toBeUndefined(); + const gone = await e2b(h, `/sandboxes/${sandboxID}`); + expect(gone.status).toBe(404); + expect(await gone.json()).toEqual({ + code: 404, + message: `sandbox "${sandboxID}" not found`, + }); + + const anonymous = await e2b(h, '/sandboxes', { + method: 'POST', + body: JSON.stringify({ templateID: 'base' }), + }); + expect(anonymous.status).toBe(201); + const anon = (await anonymous.json()) as { sandboxID: string }; + expect(h.cache.getById(anon.sandboxID)?.name).toBeNull(); + expect((await e2b(h, `/sandboxes/${anon.sandboxID}`)).status).toBe(200); + + expect((await e2b(h, '/v2/sandboxes')).status).toBe(501); + const wrongKey = await fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { 'x-api-key': 'e2b_wrong', 'content-type': 'application/json' }, + body: '{}', + }); + expect(wrongKey.status).toBe(401); + expect(await wrongKey.json()).toEqual({ + code: 401, + message: 'invalid API key', + }); + }); + + it("envd: routed by the E2b-Sandbox-Id header with the caller's own credentials untouched; refusals wear the connect dialect with CORS", async () => { + const h = await gateway(['a']); + const created = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'e' })); + const hit = await fetch(`${h.endpoint}/e2b/envd/files?path=/x`, { + headers: { 'e2b-sandbox-id': created.id, 'x-access-token': 'hmac-1' }, + }); + expect(hit.status).toBe(200); + expect(await hit.json()).toEqual({ + envd: 'a', + sandboxId: created.id, + accessToken: 'hmac-1', + url: '/e2b/envd/files?path=/x', + }); + const headerless = await fetch(`${h.endpoint}/e2b/envd/files`); + expect(headerless.status).toBe(401); + expect(headerless.headers.get('access-control-allow-origin')).toBe('*'); + expect(await headerless.json()).toEqual({ + code: 'unauthenticated', + message: 'missing E2b-Sandbox-Id header', + }); + const stranger = await fetch(`${h.endpoint}/e2b/envd/files`, { + headers: { 'e2b-sandbox-id': randomUUID() }, + }); + expect(stranger.status).toBe(502); + expect(((await stranger.json()) as { code: string }).code).toBe( + 'unavailable', + ); + const preflight = await fetch(`${h.endpoint}/e2b/envd/files`, { + method: 'OPTIONS', + headers: { origin: 'https://app.example' }, + }); + expect(preflight.status).toBe(204); + expect(preflight.headers.get('access-control-allow-origin')).toBe('*'); + const bare = await fetch(`${h.endpoint}/files?signature=x`); + expect(bare.status).toBe(501); + expect(bare.headers.get('access-control-allow-origin')).toBe('*'); + expect(((await bare.json()) as { code: string }).code).toBe( + 'unimplemented', + ); + }); +}); diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts new file mode 100644 index 00000000..906e8e77 --- /dev/null +++ b/packages/gateway/src/app.ts @@ -0,0 +1,166 @@ +import http from 'node:http'; +import { tokensEqual } from '@dormice/server/auth'; +import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import fastify, { type FastifyError, type FastifyServerFactory } from 'fastify'; +import { + serializerCompiler, + validatorCompiler, + type ZodTypeProvider, +} from 'fastify-type-provider-zod'; +import { type Logger, pino } from 'pino'; +import { z } from 'zod'; +import { classify, isOriginForm } from './classify'; +import type { Config } from './config'; +import { renderError } from './errors'; +import type { Finder } from './find'; +import type { Fleet } from './fleet'; +import type { PlacementKnobs } from './placement'; +import { createRawFaces } from './raw'; +import { e2bControlRoutes } from './routes/e2b'; +import { nativeRoutes } from './routes/native'; +import { nodeRoutes } from './routes/nodes'; +import { type BuildInfo, readBuildInfo } from './version'; + +export interface GatewayAppDeps { + config: Config; + fleet: Fleet; + finder: Finder; + /** One queue for the whole gateway: the create and destroy verbs of both faces share per-name slots. */ + locks: KeyedQueue; + /** A pino instance, or a boolean for the default logger (false = silent, for tests). */ + logger?: Logger | boolean; + /** The build identity /healthz reports; null when built outside a checkout. */ + build?: BuildInfo | null; +} + +/** Placement's knobs, read once from the config. */ +export function placementKnobs(config: Config): PlacementKnobs { + return { + cpuLimitPct: config.DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT, + activeLimit: config.DORMICE_GATEWAY_NODE_ACTIVE_LIMIT, + minDiskAvailableBytes: config.DORMICE_GATEWAY_NODE_MIN_DISK_GB * 2 ** 30, + }; +} + +/** + * Builds the gateway's Fastify instance — the daemon's shape (zod as + * validator and serializer, one error dialect, /healthz open), without + * the daemon's body. Building is separate from listening so tests inject + * requests without a port; the raw faces need real sockets. + */ +export function buildGatewayApp({ + config, + fleet, + finder, + locks, + logger = true, + build = readBuildInfo(), +}: GatewayAppDeps) { + const loggerInstance = + typeof logger === 'boolean' ? pino({ enabled: logger }) : logger; + const token = config.DORMICE_API_TOKEN; + + // The face keyed on a header sits in front of Fastify, exactly as the + // daemon's port proxy does (server/app.ts): refuse what is not an + // origin-form target (classify.ts isOriginForm), triage the raw + // request, hand the rest to Fastify. app.inject() bypasses the factory, + // so that face is exercised over real sockets only. + const raw = createRawFaces({ finder, token, log: loggerInstance }); + const serverFactory: FastifyServerFactory = (handler) => { + const server = http.createServer((req, res) => { + if (!isOriginForm(req)) { + renderError(res, 'native', { + status: 400, + message: + 'request target must be origin-form (a path starting with "/")', + }); + return; + } + const kind = classify(req); + if (kind.face === 'fastify') handler(req, res); + else raw.handleRequest(kind, req, res); + }); + server.on('upgrade', (req, socket) => { + raw.handleUpgrade(classify(req), req, socket); + }); + // The gateway only relays; the node's own request timeout is the one + // that should fire on a slow upload, not a second one in front of it. + server.requestTimeout = 0; + return server; + }; + const app = fastify({ + loggerInstance, + serverFactory, + }).withTypeProvider(); + app.setValidatorCompiler(validatorCompiler); + app.setSerializerCompiler(serializerCompiler); + + // The native error dialect, verbatim from the daemon: every non-2xx body + // the gateway itself produces is { message }. Errors a node produced are + // forwarded as they came, in whichever dialect that face speaks. + app.setErrorHandler((error: FastifyError, request, reply) => { + const status = error.statusCode ?? 500; + if (status >= 500) { + request.log.error(error, 'request failed'); + } + reply.code(status).send({ message: error.message }); + }); + app.setNotFoundHandler((request, reply) => { + reply + .code(404) + .send({ message: `route ${request.method} ${request.url} not found` }); + }); + + // Liveness, open by design; the build identity so an operator can tell + // which commit answers without a token. + app.get( + '/healthz', + { + schema: { + response: { + 200: z.object({ + status: z.literal('ok'), + build: z + .object({ + commit: z.string(), + title: z.string(), + committedAt: z.string(), + }) + .nullable(), + }), + }, + }, + }, + async () => ({ status: 'ok' as const, build }), + ); + + // One credential opens the door: the fleet's token, presented as a + // Bearer by callers and by nodes checking in alike. Keys minted by the + // gateway — many, expiring, revocable — arrive with its key table. + const knobs = placementKnobs(config); + app.register(async (api) => { + api.addHook('onRequest', async (request, reply) => { + const header = request.headers.authorization; + const bare = header?.startsWith('Bearer ') ? header.slice(7) : null; + if (bare === null || !tokensEqual(bare, token)) { + await reply.code(401).send({ message: 'missing or invalid API token' }); + } + }); + await api.register(nodeRoutes, { fleet, cache: finder.cache }); + // Its own sub-scope: the byte-preserving body parser it installs must + // not reach the gateway's own verbs, which keep Fastify's JSON parsing. + await api.register(nativeRoutes, { fleet, finder, locks, knobs, token }); + }); + + // The E2B control plane, its own auth and dialect, like the daemon's. + app.register(e2bControlRoutes, { + fleet, + finder, + locks, + knobs, + token, + prefix: '/e2b/api', + }); + + return app; +} diff --git a/packages/gateway/src/cache.test.ts b/packages/gateway/src/cache.test.ts new file mode 100644 index 00000000..bd1e464a --- /dev/null +++ b/packages/gateway/src/cache.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; +import { NameCache } from './cache'; + +const on = (id: string, name: string | null, nodeId = 'a') => ({ + id, + name, + nodeId, +}); + +describe('NameCache', () => { + it('answers by name and by id; a name that moves to a new id drops the old id whole; an id that changes name frees the old name', () => { + const cache = new NameCache(); + cache.put(on('sb-1', 'alice')); + expect(cache.getByName('alice')?.id).toBe('sb-1'); + expect(cache.getById('sb-1')?.name).toBe('alice'); + cache.put(on('sb-2', 'alice', 'b')); + expect(cache.getById('sb-1')).toBeUndefined(); + expect(cache.getByName('alice')?.nodeId).toBe('b'); + cache.put(on('sb-2', 'alicia', 'b')); + expect(cache.getByName('alice')).toBeUndefined(); + expect(cache.getByName('alicia')?.id).toBe('sb-2'); + expect(cache.size).toBe(1); + }); + + it('evict drops one entry, both handles; evictNode drops every entry of a node and counts them', () => { + const cache = new NameCache(); + cache.put(on('sb-1', 'alice')); + cache.put(on('sb-2', null)); + cache.put(on('sb-3', 'carol', 'b')); + cache.evict(on('sb-1', 'alice')); + expect(cache.getByName('alice')).toBeUndefined(); + expect(cache.getById('sb-1')).toBeUndefined(); + expect(cache.evictNode('a')).toBe(1); + expect(cache.getById('sb-2')).toBeUndefined(); + expect(cache.getByName('carol')?.id).toBe('sb-3'); + expect(cache.size).toBe(1); + }); + + it('is bounded: past the limit the entry least recently asked about goes, both handles; one just read stays', () => { + const cache = new NameCache(3); + cache.put(on('sb-1', 'alice')); + cache.put(on('sb-2', 'bob')); + cache.put(on('sb-3', null)); + // alice is the oldest write, but she was just asked about. + expect(cache.getByName('alice')).toBeDefined(); + cache.put(on('sb-4', 'dora')); + expect(cache.size).toBe(3); + expect(cache.getByName('bob')).toBeUndefined(); + expect(cache.getById('sb-2')).toBeUndefined(); + expect(cache.getByName('alice')?.id).toBe('sb-1'); + expect(cache.getById('sb-3')).toBeDefined(); + expect(cache.getByName('dora')).toBeDefined(); + // A re-put of a known id is a use too, not a second entry. + cache.put(on('sb-3', 'cathy')); + expect(cache.size).toBe(3); + cache.put(on('sb-5', 'eve')); + expect(cache.getByName('alice')).toBeUndefined(); + expect(cache.getByName('cathy')?.id).toBe('sb-3'); + }); +}); diff --git a/packages/gateway/src/cache.ts b/packages/gateway/src/cache.ts new file mode 100644 index 00000000..a9736694 --- /dev/null +++ b/packages/gateway/src/cache.ts @@ -0,0 +1,107 @@ +/** + * Where a sandbox was last found: name and id → node. A cache, not a + * directory — nothing here is authoritative, the ledger of the node that + * runs the sandbox is — so it is never persisted, never reconciled and + * never written from anything but a node's own answer: a lookup that said + * yes, a create that answered 2xx. It is dropped on a destroy the gateway + * relayed, on a node's 404 that a fresh lookup confirms (find.ts verify), + * and wholesale for a node an operator removed. An entry that is wrong + * costs one misrouted request, whose answer evicts it; a cache that is + * lost costs one extra round of questions per name. + * + * Bounded, least recently used first. A node deletes rows on its own — an + * E2B deadline kill is the scanner's routine — and tells no gateway, so + * an unbounded map would keep an entry for every sandbox ever created + * through this process, for as long as it lived: an unnamed E2B create + * every minute is half a million dead entries a year. The bound is + * generous next to a fleet's live population (Beijing holds some fifteen + * thousand rows, 2026-09), and past it the entry nobody has asked about + * for longest goes; asked about again, it costs the one round of + * questions any miss costs. + */ +export interface CacheEntry { + id: string; + /** Null for a sandbox learned by id alone (an unnamed E2B create) until a lookup names it. */ + name: string | null; + nodeId: string; +} + +export const CACHE_LIMIT = 100_000; + +export class NameCache { + private readonly byName = new Map(); + /** Insertion order is recency: a hit re-inserts, and the first key is the least recently used. */ + private readonly byId = new Map(); + + constructor(private readonly limit = CACHE_LIMIT) {} + + put(entry: CacheEntry): void { + // A name that moves to a new id (destroyed and re-acquired elsewhere + // while this gateway did not see the destroy) drops the old entry + // whole, so no id points at a node that no longer holds it. + if (entry.name !== null) { + const previous = this.byName.get(entry.name); + if (previous !== undefined && previous.id !== entry.id) { + this.byId.delete(previous.id); + } + this.byName.set(entry.name, entry); + } + const known = this.byId.get(entry.id); + if ( + known?.name !== null && + known?.name !== undefined && + known.name !== entry.name + ) { + this.byName.delete(known.name); + } + this.byId.delete(entry.id); + this.byId.set(entry.id, entry); + if (this.byId.size > this.limit) { + const oldest = this.byId.values().next().value; + if (oldest !== undefined) this.evict(oldest); + } + } + + getByName(name: string): CacheEntry | undefined { + const entry = this.byName.get(name); + if (entry !== undefined) this.touch(entry); + return entry; + } + + getById(id: string): CacheEntry | undefined { + const entry = this.byId.get(id); + if (entry !== undefined) this.touch(entry); + return entry; + } + + /** Marks the entry as just used: back to the end of the recency order. */ + private touch(entry: CacheEntry): void { + this.byId.delete(entry.id); + this.byId.set(entry.id, entry); + } + + evict(entry: CacheEntry): void { + if (entry.name !== null && this.byName.get(entry.name)?.id === entry.id) { + this.byName.delete(entry.name); + } + if (this.byId.get(entry.id)?.nodeId === entry.nodeId) { + this.byId.delete(entry.id); + } + } + + /** Everything that pointed at a node the operator removed. */ + evictNode(nodeId: string): number { + let evicted = 0; + for (const entry of [...this.byId.values()]) { + if (entry.nodeId === nodeId) { + this.evict(entry); + evicted += 1; + } + } + return evicted; + } + + get size(): number { + return this.byId.size; + } +} diff --git a/packages/gateway/src/classify.ts b/packages/gateway/src/classify.ts new file mode 100644 index 00000000..94f91adc --- /dev/null +++ b/packages/gateway/src/classify.ts @@ -0,0 +1,48 @@ +/** + * Which face a raw request belongs to — the gateway's serverFactory + * triages every request before Fastify's router sees it, because one face + * is keyed on something Fastify cannot route on (a header, on any path). + * Pure, so the order of the tests is a fact here and nowhere else: + * envd /e2b/envd/* — E2B's in-sandbox API, keyed by the + * E2b-Sandbox-Id header. + * signedRoot exactly /files at the root — the bare signed-URL form, + * which carries no sandbox id anywhere the gateway can read + * without the node's signing secret. + * fastify everything else — the native verbs, /e2b/api, /healthz, + * the gateway's own verbs. + * The sandbox port proxy (Host `-.`) joins this list + * when the sandbox domain moves into the gateway's settings. + */ +export type Classified = + | { face: 'envd' } + | { face: 'signedRoot' } + | { face: 'fastify' }; + +/** + * Only the origin-form request target (RFC 9112 §3.2.1, a path starting + * with `/`) is forwarded. The gateway routes on the path Fastify extracts + * — find-my-way strips a scheme and authority off an absolute-form target + * — and sends the target to the node byte for byte, so on an absolute + * form the two would read different requests: `POST http://8000-. + * /execCommand` is `/execCommand` to the gateway (a native verb, + * authenticated, the fleet's token attached) and, to the RFC-7230 hop in + * front of the node (Caddy: Host := the target's authority), a request to + * the node's sandbox proxy — the fleet's credential delivered into the + * tenant's own sandbox (traced 2026-09-12). Absolute form is what a + * client sends a forward proxy, which the gateway is not; every real + * client of an origin server sends origin form, so refusing the rest + * costs nothing and keeps the invariant whole: the gateway and the node + * see the same request. + */ +export function isOriginForm(req: { url?: string }): boolean { + return (req.url ?? '').startsWith('/'); +} + +export function classify(req: { url?: string }): Classified { + const url = req.url ?? ''; + const q = url.indexOf('?'); + const path = q === -1 ? url : url.slice(0, q); + if (url.startsWith('/e2b/envd/')) return { face: 'envd' }; + if (path === '/files') return { face: 'signedRoot' }; + return { face: 'fastify' }; +} diff --git a/packages/gateway/src/config.test.ts b/packages/gateway/src/config.test.ts new file mode 100644 index 00000000..186aa5cb --- /dev/null +++ b/packages/gateway/src/config.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; +import { loadConfig } from './config'; + +const TOKEN = { DORMICE_API_TOKEN: 'x'.repeat(32) }; + +describe('loadConfig', () => { + it('defaults: port 3677, an absolute database path, placement knobs 70% / 400 active / 10 GiB', () => { + const config = loadConfig(TOKEN); + expect(config.DORMICE_GATEWAY_PORT).toBe(3677); + expect(config.DORMICE_GATEWAY_DB_PATH).toBe( + '/var/lib/dormice-gateway/gateway.db', + ); + expect(config.DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT).toBe(70); + expect(config.DORMICE_GATEWAY_NODE_ACTIVE_LIMIT).toBe(400); + expect(config.DORMICE_GATEWAY_NODE_MIN_DISK_GB).toBe(10); + }); + + it('requires the fleet token, at least 32 characters, naming the variable', () => { + expect(() => loadConfig({})).toThrow(/DORMICE_API_TOKEN/); + expect(() => loadConfig({ DORMICE_API_TOKEN: 'short' })).toThrow( + /at least 32 characters/, + ); + }); + + it('refuses a relative database path and accepts :memory:', () => { + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_DB_PATH: 'data/gateway.db' }), + ).toThrow(/DORMICE_GATEWAY_DB_PATH must be an absolute path/); + expect( + loadConfig({ ...TOKEN, DORMICE_GATEWAY_DB_PATH: ':memory:' }) + .DORMICE_GATEWAY_DB_PATH, + ).toBe(':memory:'); + }); + + it('parses the placement knobs and refuses nonsense', () => { + const config = loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '85', + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', + DORMICE_GATEWAY_NODE_MIN_DISK_GB: '0.5', + }); + expect(config.DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT).toBe(85); + expect(config.DORMICE_GATEWAY_NODE_ACTIVE_LIMIT).toBe(2); + expect(config.DORMICE_GATEWAY_NODE_MIN_DISK_GB).toBe(0.5); + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '101' }), + ).toThrow(); + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '0' }), + ).toThrow(); + }); +}); diff --git a/packages/gateway/src/config.ts b/packages/gateway/src/config.ts new file mode 100644 index 00000000..94facb04 --- /dev/null +++ b/packages/gateway/src/config.ts @@ -0,0 +1,78 @@ +import { isAbsolute } from 'node:path'; +import { z } from 'zod'; + +/** + * The gateway's configuration: environment variables, validated once at + * startup so a bad value fails loudly here — the daemon's discipline + * (packages/server/src/config.ts). The gateway's own knobs carry the + * DORMICE_GATEWAY_ prefix so a gateway and a node can share one machine's + * /etc/dormice without colliding; the token deliberately does not — it is + * the very same DORMICE_API_TOKEN every node has, one string for the whole + * fleet, written into both env files by the same hand. + */ +const envSchema = z.object({ + DORMICE_GATEWAY_PORT: z.coerce.number().int().min(1).max(65535).default(3677), + /** + * Absolute, like the daemon's in docker mode and for the same reason: a + * relative path depends on the start directory, and under systemd (no + * WorkingDirectory) that is `/` — a gateway started without this variable + * would keep its tables in /data, silently, until the day someone sets + * the path and every node it ever knew is gone. + */ + DORMICE_GATEWAY_DB_PATH: z + .string() + .default('/var/lib/dormice-gateway/gateway.db') + .refine((path) => path === ':memory:' || isAbsolute(path), { + error: + 'DORMICE_GATEWAY_DB_PATH must be an absolute path, e.g. /var/lib/dormice-gateway/gateway.db', + }), + /** + * The one token of the fleet. Callers present it to the gateway, nodes + * present it when they check in, and the gateway presents it to nodes + * when it forwards. Required, no default: loopback-only is not + * authentication. + */ + DORMICE_API_TOKEN: z.string().min(32, { + error: + 'DORMICE_API_TOKEN must be at least 32 characters — generate one with: openssl rand -hex 32', + }), + /** + * Placement: a node whose last whole-machine CPU reading is above this + * takes no new sandboxes. 70 leaves the headroom a wake burst needs — + * the 2026-09-11 Beijing incident saturated a 128-core host from a + * reading well under 100 in one burst of cold starts. + */ + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: z.coerce + .number() + .min(0) + .max(100) + .default(70), + /** + * Placement: a node already running this many active sandboxes (plus + * those placed on it since its last check-in) takes no more. Active + * only — frozen sandboxes cost swap, not CPU or dockerd attention, and + * a node holds thousands of them; the ceiling this guards is running + * containers per host. + */ + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: z.coerce + .number() + .int() + .positive() + .default(400), + /** + * Placement: a node whose data disk has less than this free takes no new + * sandboxes. Ten GiB is one default sandbox disk: below that, a single + * new sandbox writing its disk full would fill the host, and a full data + * disk is the one failure that stops every sandbox on the node at once + * (the ledger itself cannot write). The node's own create still answers + * its honest 500 past this point; the gate is what keeps the next ten + * names from landing on the same full box. + */ + DORMICE_GATEWAY_NODE_MIN_DISK_GB: z.coerce.number().nonnegative().default(10), +}); + +export type Config = z.infer; + +export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config { + return envSchema.parse(env); +} diff --git a/packages/gateway/src/db/db.ts b/packages/gateway/src/db/db.ts new file mode 100644 index 00000000..f0865403 --- /dev/null +++ b/packages/gateway/src/db/db.ts @@ -0,0 +1,26 @@ +import { mkdirSync } from 'node:fs'; +import { dirname } from 'node:path'; +import Database from 'better-sqlite3'; +import { drizzle } from 'drizzle-orm/better-sqlite3'; +import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; +import * as schema from './schema'; + +/** + * Opens the gateway's database — the daemon's shape (WAL: readers never + * block the single writer, a crash mid-write cannot corrupt the file). + */ +export function openDb(path: string) { + if (path !== ':memory:') { + mkdirSync(dirname(path), { recursive: true }); + } + const sqlite = new Database(path); + sqlite.pragma('journal_mode = WAL'); + return drizzle(sqlite, { schema }); +} + +export type Db = ReturnType; + +/** Applies pending migrations (drizzle-kit output, committed) at every start. */ +export function migrateDb(db: Db, migrationsFolder: string) { + migrate(db, { migrationsFolder }); +} diff --git a/packages/gateway/src/db/schema.ts b/packages/gateway/src/db/schema.ts new file mode 100644 index 00000000..227d9d5b --- /dev/null +++ b/packages/gateway/src/db/schema.ts @@ -0,0 +1,32 @@ +import { sqliteTable, text } from 'drizzle-orm/sqlite-core'; + +/** + * The gateway's tables are "how the fleet is configured and who may + * enter" — never a sandbox's state, which lives in the ledger of the node + * that runs it and is asked for when needed (find.ts). One table today; + * api_keys, settings, templates and console_account arrive here with the + * configuration authority. + */ + +/** + * Every node that has ever checked in (routes/nodes.ts): its id, the + * address the gateway forwards to, and when it first appeared. Written by + * the nodes themselves at their first check-in — there is no registration + * verb and no nodes file, so "which nodes exist" has exactly one home — + * and deleted only by an operator's removeNode. Persistent, not memory, + * for one reason: a node that is down must still be known after a gateway + * restart, or a name that lives only there would be placed anew elsewhere + * and come back as a conflict when the node returns. Everything the node + * last reported (its reading, build, check-in time) is memory: fifteen + * seconds later it is reported again. + */ +export const nodes = sqliteTable('nodes', { + /** DORMICE_NODE_ID as the node states it — the `nodeId` in every sandbox answer. */ + id: text('id').primaryKey(), + /** Where the gateway forwards to; updated when a check-in states a new one. */ + endpoint: text('endpoint').notNull(), + /** ISO 8601 UTC — the first check-in. */ + addedAt: text('added_at').notNull(), +}); + +export type NodeRow = typeof nodes.$inferSelect; diff --git a/packages/gateway/src/errors.ts b/packages/gateway/src/errors.ts new file mode 100644 index 00000000..b310db13 --- /dev/null +++ b/packages/gateway/src/errors.ts @@ -0,0 +1,117 @@ +import type http from 'node:http'; +import { UnreachableError } from './forward'; + +/** What relay needs of a logger — pino's and Fastify's request logger both fit. */ +export interface ErrorLog { + error(obj: unknown, msg?: string): void; +} + +/** + * Every face the gateway fronts has its own error dialect, and an error + * the gateway itself produces must wear the dialect of the face it was + * asked on — a client library parses what it expects: + * native { message } the daemon's API + * control { code: , message } E2B's control plane (openapi-fetch checks error.code === 404) + * connect { code: '', message } E2B's envd (Connect RPC's string codes) + * Errors a node produced are forwarded verbatim and never re-dressed. + */ +export type Dialect = 'native' | 'control' | 'connect'; + +export interface RenderedError { + status: number; + message: string; + /** Connect's string code, used by the connect dialect only. */ + connectCode?: string; + /** Browser-consumable faces carry CORS on errors too, or the refusal is unreadable. */ + cors?: boolean; + /** Seconds — on a 503 whose cause the caller may simply outwait. */ + retryAfterSeconds?: number; +} + +export function errorBody(dialect: Dialect, error: RenderedError): string { + switch (dialect) { + case 'native': + return JSON.stringify({ message: error.message }); + case 'control': + return JSON.stringify({ code: error.status, message: error.message }); + case 'connect': + return JSON.stringify({ + code: error.connectCode ?? 'unknown', + message: error.message, + }); + } +} + +/** Writes an error the gateway produced, on a raw response, in the face's dialect. */ +export function renderError( + res: http.ServerResponse, + dialect: Dialect, + error: RenderedError, +): void { + if (res.headersSent || res.destroyed) { + res.destroy(); + return; + } + const body = errorBody(dialect, error); + const headers: http.OutgoingHttpHeaders = { + 'content-type': 'application/json', + 'content-length': String(Buffer.byteLength(body)), + }; + if (error.cors) headers['access-control-allow-origin'] = '*'; + if (error.retryAfterSeconds !== undefined) { + headers['retry-after'] = String(error.retryAfterSeconds); + } + res.writeHead(error.status, headers); + res.end(body); +} + +/** The preflight answer of the browser-consumable faces — the daemon's cors.ts shape. */ +export function sendPreflight( + req: http.IncomingMessage, + res: http.ServerResponse, +): void { + const requested = req.headers['access-control-request-headers']; + res.writeHead(204, { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET, POST, OPTIONS', + 'access-control-allow-headers': + typeof requested === 'string' && requested !== '' ? requested : '*', + 'access-control-max-age': '7200', + }); + res.end(); +} + +/** + * Runs one forwarding step on a response the gateway owns (a hijacked + * reply, a raw face) and turns its failure into an answer in the face's + * dialect: the node's transport failure is a 502 rendered by `unreachable` + * (what the caller may do next), anything else a 500 that sends the + * operator to the log. Never silence: Fastify writes nothing for a + * hijacked reply whose handler throws, the gateway has no request timeout + * of its own, and a raw face has no framework behind it at all — so a + * step that threw after the node had answered would leave the caller + * waiting forever with the sandbox already built (found by review, + * 2026-09-12). + */ +export async function relay( + res: http.ServerResponse, + dialect: Dialect, + log: ErrorLog, + step: () => Promise, + unreachable: (error: UnreachableError) => RenderedError, +): Promise { + try { + await step(); + } catch (error) { + if (error instanceof UnreachableError) { + renderError(res, dialect, unreachable(error)); + return; + } + log.error(error, 'forwarding failed on the gateway'); + renderError(res, dialect, { + status: 500, + message: 'the gateway failed while forwarding — see its log', + cors: dialect === 'connect', + }); + } +} diff --git a/packages/gateway/src/find.test.ts b/packages/gateway/src/find.test.ts new file mode 100644 index 00000000..1b8a6023 --- /dev/null +++ b/packages/gateway/src/find.test.ts @@ -0,0 +1,316 @@ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { fileURLToPath } from 'node:url'; +import { afterEach, describe, expect, it } from 'vitest'; +import { NameCache } from './cache'; +import { migrateDb, openDb } from './db/db'; +import { Finder } from './find'; +import { Fleet } from './fleet'; +import { + type AskNode, + httpAskNode, + LOOKUP_TIMEOUT_MS, + type LookupAnswer, +} from './lookup'; +import { checkInOf } from './testing'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const TOKEN = 'shared-token-shared-token-shared-token'; +const NOW = new Date('2026-09-14T12:00:00.000Z'); + +function fleetOf(...ids: string[]) { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const fleet = new Fleet(db); + for (const id of ids) fleet.checkIn(checkInOf(id, `http://${id}:80`), NOW); + return fleet; +} + +const silentLog = { warn: () => {} }; + +/** An asker scripted per node: what each node says to any question, and how often it was asked. */ +function scripted(script: Record) { + const asked: string[] = []; + const ask: AskNode = async (node) => { + asked.push(node.id); + return script[node.id] ?? { kind: 'absent' }; + }; + return { ask, asked }; +} + +describe('Finder', () => { + it('asks every node in parallel; exactly one yes wins and is cached by name and id', async () => { + const fleet = fleetOf('a', 'b', 'c'); + const { ask, asked } = scripted({ + b: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + }); + const cache = new NameCache(); + const finder = new Finder(fleet, cache, ask, silentLog); + const found = await finder.byName('alice'); + expect(found).toMatchObject({ kind: 'one', id: 'sb-1', name: 'alice' }); + expect(found.kind === 'one' && found.node.id).toBe('b'); + expect(asked.sort()).toEqual(['a', 'b', 'c']); + // The cache answers the next questions, by either handle, without a + // single node asked. + asked.length = 0; + expect((await finder.byName('alice')).kind).toBe('one'); + expect((await finder.byId('sb-1')).kind).toBe('one'); + expect(asked).toEqual([]); + }); + + it('one yes wins even while another node is silent: the sandbox is where it says it is', async () => { + const fleet = fleetOf('a', 'b'); + const { ask } = scripted({ + a: { kind: 'silent', why: 'ECONNREFUSED' }, + b: { kind: 'found', id: 'sb-1', name: 'alice', state: 'frozen' }, + }); + const found = await new Finder( + fleet, + new NameCache(), + ask, + silentLog, + ).byName('alice'); + expect(found.kind === 'one' && found.node.id).toBe('b'); + }); + + it('two yeses are a conflict naming both nodes, and nothing is cached', async () => { + const fleet = fleetOf('a', 'b', 'c'); + const { ask } = scripted({ + c: { kind: 'found', id: 'sb-2', name: 'alice', state: 'active' }, + a: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + }); + const cache = new NameCache(); + const found = await new Finder(fleet, cache, ask, silentLog).byName( + 'alice', + ); + expect(found).toEqual({ + kind: 'conflict', + nodes: [ + { id: 'a', endpoint: 'http://a:80' }, + { id: 'c', endpoint: 'http://c:80' }, + ], + }); + expect(cache.size).toBe(0); + }); + + it('two ids checked in from one endpoint answer twice for every name there: a conflict whose nodes share the endpoint', async () => { + const fleet = fleetOf('a', 'b'); + // The node at a's address checks in again under a new id: a rename. + fleet.checkIn(checkInOf('a-renamed', 'http://a:80'), NOW); + const { ask } = scripted({ + a: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + 'a-renamed': { + kind: 'found', + id: 'sb-1', + name: 'alice', + state: 'active', + }, + }); + const found = await new Finder( + fleet, + new NameCache(), + ask, + silentLog, + ).byName('alice'); + expect(found).toEqual({ + kind: 'conflict', + nodes: [ + { id: 'a', endpoint: 'http://a:80' }, + { id: 'a-renamed', endpoint: 'http://a:80' }, + ], + }); + }); + + it('every node says no: the name is new; a silent node among the noes: unsure, naming it', async () => { + const fleet = fleetOf('a', 'b'); + expect( + await new Finder( + fleet, + new NameCache(), + scripted({}).ask, + silentLog, + ).byName('nobody'), + ).toEqual({ kind: 'none' }); + const warned: unknown[] = []; + const unsure = await new Finder( + fleet, + new NameCache(), + scripted({ b: { kind: 'silent', why: 'timeout' } }).ask, + { warn: (obj) => warned.push(obj) }, + ).byName('nobody'); + expect(unsure).toEqual({ + kind: 'unsure', + silent: [{ nodeId: 'b', why: 'timeout' }], + }); + expect(warned).toHaveLength(1); + }); + + it('an empty fleet finds nothing and asks nobody', async () => { + const { ask, asked } = scripted({}); + expect( + await new Finder(fleetOf(), new NameCache(), ask, silentLog).byName('x'), + ).toEqual({ kind: 'none' }); + expect(asked).toEqual([]); + }); + + it('a cached entry whose node was removed is dropped and the fleet asked afresh', async () => { + const fleet = fleetOf('a', 'b'); + const { ask, asked } = scripted({ + a: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + }); + const cache = new NameCache(); + const finder = new Finder(fleet, cache, ask, silentLog); + await finder.byName('alice'); + fleet.remove('a'); + asked.length = 0; + expect(await finder.byName('alice')).toEqual({ kind: 'none' }); + expect(asked).toEqual(['b']); + expect(cache.size).toBe(0); + }); + + it('verify: a node that says absent loses the entry; a silent node keeps it; a removed node loses it', async () => { + const fleet = fleetOf('a', 'b'); + const script: Record = { + a: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + }; + const cache = new NameCache(); + const finder = new Finder( + fleet, + cache, + async (node) => script[node.id] ?? { kind: 'absent' }, + silentLog, + ); + await finder.byName('alice'); + const entry = { id: 'sb-1', name: 'alice', nodeId: 'a' }; + script.a = { kind: 'silent', why: 'timeout' }; + await finder.verify(entry); + expect(cache.getByName('alice')).toEqual(entry); + script.a = { kind: 'absent' }; + await finder.verify(entry); + expect(cache.getByName('alice')).toBeUndefined(); + expect(cache.getById('sb-1')).toBeUndefined(); + + cache.put(entry); + fleet.remove('a'); + await finder.verify(entry); + expect(cache.size).toBe(0); + }); +}); + +describe('NameCache', () => { + it('a name that moved to a new id drops the old id; an id that gained a name is reachable by both', () => { + const cache = new NameCache(); + cache.put({ id: 'old', name: 'alice', nodeId: 'a' }); + cache.put({ id: 'new', name: 'alice', nodeId: 'b' }); + expect(cache.getById('old')).toBeUndefined(); + expect(cache.getByName('alice')?.nodeId).toBe('b'); + cache.put({ id: 'anon', name: null, nodeId: 'a' }); + cache.put({ id: 'anon', name: 'e2b-1', nodeId: 'a' }); + expect(cache.getByName('e2b-1')?.id).toBe('anon'); + expect(cache.size).toBe(2); + expect(cache.evictNode('a')).toBe(1); + expect(cache.getById('anon')).toBeUndefined(); + expect(cache.size).toBe(1); + }); +}); + +describe('httpAskNode', () => { + const servers: http.Server[] = []; + afterEach(async () => { + await Promise.all( + servers + .splice(0) + .map((s) => new Promise((resolve) => s.close(() => resolve()))), + ); + }); + + async function node( + handler: ( + body: unknown, + headers: http.IncomingHttpHeaders, + ) => { status: number; body: string }, + ) { + const server = http.createServer((req, res) => { + let text = ''; + req.on('data', (c) => { + text += c; + }); + req.on('end', () => { + const answer = handler(JSON.parse(text), req.headers); + res.writeHead(answer.status, { 'content-type': 'application/json' }); + res.end(answer.body); + }); + }); + servers.push(server); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + return { + id: 'n', + endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + }; + } + + it('posts the question under the fleet token and reads found / absent; a non-200 or a refused connection is silent with the reason', async () => { + const seen: unknown[] = []; + const yes = await node((body, headers) => { + seen.push({ body, auth: headers.authorization }); + return { + status: 200, + body: JSON.stringify({ + found: true, + sandbox: { id: 'sb-1', name: 'alice', state: 'active' }, + }), + }; + }); + const ask = httpAskNode(TOKEN); + expect(await ask(yes, { name: 'alice' })).toEqual({ + kind: 'found', + id: 'sb-1', + name: 'alice', + state: 'active', + }); + expect(seen).toEqual([ + { body: { name: 'alice' }, auth: `Bearer ${TOKEN}` }, + ]); + const no = await node(() => ({ + status: 200, + body: JSON.stringify({ found: false }), + })); + expect(await ask(no, { id: 'x' })).toEqual({ kind: 'absent' }); + const refusing = await node(() => ({ + status: 401, + body: '{"message":"missing or invalid API token"}', + })); + expect(await ask(refusing, { id: 'x' })).toMatchObject({ + kind: 'silent', + why: expect.stringMatching(/answered 401/), + }); + // A port nobody listens on any more: the OS hands one out and it is + // released before the question is asked. (Not port 9 — fetch refuses + // the Fetch standard's "bad ports" without dialling.) + const released = http.createServer(); + await new Promise((r) => released.listen(0, '127.0.0.1', r)); + const port = (released.address() as AddressInfo).port; + await new Promise((r) => released.close(() => r())); + expect( + await ask( + { id: 'gone', endpoint: `http://127.0.0.1:${port}` }, + { id: 'x' }, + ), + ).toMatchObject({ kind: 'silent', why: 'ECONNREFUSED' }); + }); + + it('a node whose host swallows the connection is silent after the two seconds, not after a connect timeout — and the why is a word, not a number', async () => { + // 192.0.2.1 (TEST-NET-1) is routed nowhere: the SYN is dropped, the + // connect hangs. Where a network refuses it outright instead + // (ENETUNREACH), the answer is immediate and the assertion still holds. + const started = Date.now(); + const answer = await httpAskNode(TOKEN)( + { id: 'hole', endpoint: 'http://192.0.2.1:80' }, + { id: 'x' }, + ); + expect(Date.now() - started).toBeLessThan(LOOKUP_TIMEOUT_MS + 1_500); + expect(answer.kind).toBe('silent'); + expect(answer.kind === 'silent' && typeof answer.why).toBe('string'); + expect(answer.kind === 'silent' && answer.why).not.toMatch(/^\d+$/); + }, 15_000); +}); diff --git a/packages/gateway/src/find.ts b/packages/gateway/src/find.ts new file mode 100644 index 00000000..bc6191ea --- /dev/null +++ b/packages/gateway/src/find.ts @@ -0,0 +1,162 @@ +import type { CacheEntry, NameCache } from './cache'; +import type { Fleet, NodeState } from './fleet'; +import type { AskNode, LookupQuery } from './lookup'; + +/** + * Where a sandbox is, adjudicated once for every face: + * one exactly one node holds it — route there + * conflict several nodes hold it — refused, never guessed about (an + * operator destroys one copy directly on its node) + * none every node answered, none holds it — the name is new + * unsure no node holds it as far as anyone answered, but a node did + * not answer, so "new" cannot be proven: a name that lives + * only on a silent node must not be built a second time + * elsewhere (design record #28: a 503 with Retry-After, the + * retry is the application's) + */ +export type Found = + | { kind: 'one'; node: NodeState; id: string; name: string | null } + | { kind: 'conflict'; nodes: Array<{ id: string; endpoint: string }> } + | { kind: 'none' } + | { kind: 'unsure'; silent: Array<{ nodeId: string; why: string }> }; + +export interface FinderLog { + warn(obj: unknown, msg: string): void; +} + +/** + * Finds a sandbox by asking. The cache answers first; on a miss every + * node in the fleet is asked in parallel (lookup.ts), and exactly one + * "yes" wins — the sandbox is wherever it says it is, whether or not some + * other node was slow to say no. Down nodes are asked like any other: + * the reading a node last reported says where NOT to place, never where + * a sandbox is; a node that has missed its check-ins but still answers a + * lookup still holds its sandboxes. + */ +export class Finder { + constructor( + private readonly fleet: Fleet, + readonly cache: NameCache, + private readonly ask: AskNode, + private readonly log: FinderLog, + ) {} + + /** + * `confirm`: a creator's cache hit is checked with the cached node before + * it is trusted. For every other verb a stale entry costs one misrouted + * request whose 404 evicts it (verify below); a create forwarded as a + * wake to a node that no longer holds the name *builds* — the daemon's + * acquire and E2B create are create-or-wake — on a node placement never + * judged and never counted, and the daemon deletes rows on its own (an + * E2B deadline kill is the scanner's routine, five minutes by default), + * so every re-create of an expired name would pin to its first node + * past every gate. One question to one node, by id so no name slot is + * waited on; "absent" evicts and the fleet is asked afresh (found by + * review, 2026-09-14). + */ + byName(name: string, options: { confirm?: boolean } = {}): Promise { + return this.find( + this.cache.getByName(name), + { name }, + options.confirm === true, + ); + } + + byId(id: string): Promise { + return this.find(this.cache.getById(id), { id }, false); + } + + private async find( + cached: CacheEntry | undefined, + query: LookupQuery, + confirm: boolean, + ): Promise { + if (cached !== undefined) { + const node = this.fleet.get(cached.nodeId); + if (node === undefined) { + // A node the operator removed while the entry was cached: the + // entry is stale by definition, and the fleet is asked afresh. + this.cache.evict(cached); + } else if (!confirm) { + return { kind: 'one', node, id: cached.id, name: cached.name }; + } else { + const answer = await this.ask(node, { id: cached.id }); + if (answer.kind === 'found') { + return { kind: 'one', node, id: answer.id, name: answer.name }; + } + if (answer.kind === 'silent') { + // The one node that may hold it did not answer: not new, not + // known to be there — the same refusal a silent stranger earns. + const silent = [{ nodeId: node.id, why: answer.why }]; + this.log.warn( + { query, silent }, + 'lookup: the cached node did not confirm; the name cannot be treated as new', + ); + return { kind: 'unsure', silent }; + } + // The node itself says the sandbox is gone: the entry was stale. + this.cache.evict(cached); + } + } + const members = this.fleet.all(); + const answers = await Promise.all( + members.map(async (node) => ({ + node, + answer: await this.ask(node, query), + })), + ); + const found = answers.flatMap(({ node, answer }) => + answer.kind === 'found' ? [{ node, answer }] : [], + ); + const first = found[0]; + if (found.length === 1 && first !== undefined) { + const entry = { + id: first.answer.id, + name: first.answer.name, + nodeId: first.node.id, + }; + this.cache.put(entry); + return { kind: 'one', node: first.node, id: entry.id, name: entry.name }; + } + if (found.length > 1) { + // The endpoints ride along: two ids answering from one endpoint is + // one daemon under two names, which verdict.ts diagnoses as such. + return { + kind: 'conflict', + nodes: found + .map((f) => ({ id: f.node.id, endpoint: f.node.endpoint })) + .sort((x, y) => x.id.localeCompare(y.id)), + }; + } + const silent = answers.flatMap(({ node, answer }) => + answer.kind === 'silent' ? [{ nodeId: node.id, why: answer.why }] : [], + ); + if (silent.length > 0) { + this.log.warn( + { query, silent }, + 'lookup: a node did not answer; a name it may hold cannot be treated as new', + ); + return { kind: 'unsure', silent }; + } + return { kind: 'none' }; + } + + /** + * After a cached node answered 404 for a name: is the sandbox really + * gone from it? A node's 404 is not "no such sandbox" by itself — + * readFile answers 404 for a missing path too — so the node is asked + * the one question that means exactly that, and only a plain "absent" + * evicts. Silence keeps the entry: the node may be down, and its + * sandboxes are still there. Off the request path (the 404 has already + * been relayed); the next request for the name asks the fleet afresh. + */ + async verify(entry: CacheEntry): Promise { + const node = this.fleet.get(entry.nodeId); + if (node === undefined) { + this.cache.evict(entry); + return; + } + const answer = await this.ask(node, { id: entry.id }); + if (answer.kind === 'absent') this.cache.evict(entry); + } +} diff --git a/packages/gateway/src/fleet.test.ts b/packages/gateway/src/fleet.test.ts new file mode 100644 index 00000000..287e184e --- /dev/null +++ b/packages/gateway/src/fleet.test.ts @@ -0,0 +1,122 @@ +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { migrateDb, openDb } from './db/db'; +import { type CheckInOutcome, downReason, Fleet } from './fleet'; +import { checkInOf } from './testing'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const NOW = new Date('2026-09-14T12:00:00.000Z'); + +/** A check-in the test expects to be taken. */ +function taken(outcome: CheckInOutcome) { + if ('refused' in outcome) throw new Error(outcome.refused); + return outcome; +} + +function db() { + const handle = openDb(':memory:'); + migrateDb(handle, MIGRATIONS); + return handle; +} + +describe('Fleet', () => { + it('a first check-in joins the node and persists its row; a gateway restart still knows it, unreached until it checks in again', () => { + const handle = db(); + const fleet = new Fleet(handle); + expect(fleet.all()).toEqual([]); + const { node, joined } = taken( + fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), + NOW, + ), + ); + expect(joined).toBe(true); + expect(node.addedAt).toBe(NOW.toISOString()); + expect(node.lastCheckInAt).toBe(NOW); + expect(node.reading?.host.cpuCount).toBe(8); + expect(downReason(node, NOW)).toBeNull(); + + // The same database after a restart: the row is there, the memory is not. + const restarted = new Fleet(handle); + const known = restarted.get('node-b'); + expect(known?.endpoint).toBe('http://10.0.0.7:80'); + expect(known?.addedAt).toBe(NOW.toISOString()); + expect(known?.reading).toBeNull(); + expect(downReason(known as NonNullable, NOW)).toBe( + 'has not checked in since the gateway started', + ); + }); + + it('a later check-in is not a join; a changed endpoint inside the interval is refused as a second daemon, an interval later written through as a move; the placement counter restarts', () => { + const handle = db(); + const fleet = new Fleet(handle); + const first = taken( + fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW), + ); + first.node.placedSinceCheckIn = 3; + const twin = fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.8:80'), + new Date(NOW.getTime() + 7_000), + ); + expect('refused' in twin && twin.refused).toMatch( + /^node node-b checked in from http:\/\/10\.0\.0\.7:80 7s ago and now from http:\/\/10\.0\.0\.8:80 — two daemons share one DORMICE_NODE_ID/, + ); + expect(first.node.endpoint).toBe('http://10.0.0.7:80'); + expect(first.node.placedSinceCheckIn).toBe(3); + const second = taken( + fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.8:80', { active: 12 }), + new Date(NOW.getTime() + 15_000), + ), + ); + expect(second.joined).toBe(false); + expect(first.movedFrom).toBeNull(); + expect(second.movedFrom).toBe('http://10.0.0.7:80'); + expect(second.node).toBe(first.node); + expect(second.node.endpoint).toBe('http://10.0.0.8:80'); + expect(second.node.reading?.sandboxes.byState.active).toBe(12); + expect(second.node.placedSinceCheckIn).toBe(0); + expect(new Fleet(handle).get('node-b')?.endpoint).toBe( + 'http://10.0.0.8:80', + ); + }); + + it('downReason: fresh within two of its own intervals, down past them', () => { + const fleet = new Fleet(db()); + const { node } = taken( + fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), + NOW, + ), + ); + expect(downReason(node, new Date(NOW.getTime() + 29_000))).toBeNull(); + expect(downReason(node, new Date(NOW.getTime() + 31_000))).toBe( + 'has not checked in for 31s', + ); + // A one-second node (the exam's) is judged by its own interval. + const quick = taken( + fleet.checkIn( + checkInOf('node-c', 'http://10.0.0.9:80', { intervalSeconds: 1 }), + NOW, + ), + ).node; + expect(downReason(quick, new Date(NOW.getTime() + 1_500))).toBeNull(); + expect(downReason(quick, new Date(NOW.getTime() + 2_500))).toBe( + 'has not checked in for 3s', + ); + }); + + it('remove forgets the node and its row; removing an unknown id is false; a node that checks in again re-joins', () => { + const handle = db(); + const fleet = new Fleet(handle); + fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW); + expect(fleet.remove('node-b')).toBe(true); + expect(fleet.all()).toEqual([]); + expect(new Fleet(handle).all()).toEqual([]); + expect(fleet.remove('node-b')).toBe(false); + expect( + taken(fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW)) + .joined, + ).toBe(true); + }); +}); diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts new file mode 100644 index 00000000..079700bc --- /dev/null +++ b/packages/gateway/src/fleet.ts @@ -0,0 +1,165 @@ +import type { BuildInfo, CheckInRequest, NodeReading } from '@dormice/shared'; +import { eq } from 'drizzle-orm'; +import type { Db } from './db/db'; +import { nodes } from './db/schema'; + +/** + * A node as the gateway knows it: the persistent row (id, endpoint, + * addedAt) and what it last reported (memory — reported again at the next + * check-in, gone with the process and rightly so). placedSinceCheckIn + * counts the sandboxes the gateway sent here since the last reading, so a + * burst inside one interval is counted against the node before its next + * reading shows it (placement.ts); placedIds are the ones among them whose + * create answered with an id, so a sandbox placed and destroyed inside one + * interval — a short job, the exam's churn — is taken off the count again + * instead of holding a slot the reading will never show (routes/destroy.ts). + */ +export interface NodeState { + readonly id: string; + endpoint: string; + readonly addedAt: string; + lastCheckInAt: Date | null; + intervalSeconds: number | null; + build: BuildInfo | null; + reading: NodeReading | null; + placedSinceCheckIn: number; + placedIds: Set; +} + +/** + * Why a node is not to be placed on right now, or null when it is fine: + * never checked in since this gateway started, or silent for two of its + * own intervals — the interval it stated in its last check-in, so the + * exam's one-second nodes and production's fifteen-second nodes are judged + * by the same rule. Two, not one: a check-in delayed by a busy event loop + * or a slow reading is normal; two in a row missing is a node in trouble. + * The same word answers listNodes' `reachable`. + */ +export function downReason(node: NodeState, now: Date): string | null { + if (node.lastCheckInAt === null || node.intervalSeconds === null) { + return 'has not checked in since the gateway started'; + } + const silentMs = now.getTime() - node.lastCheckInAt.getTime(); + if (silentMs > 2 * node.intervalSeconds * 1000) { + return `has not checked in for ${Math.round(silentMs / 1000)}s`; + } + return null; +} + +/** What a check-in came to: taken (and whether it joined or moved), or refused with the sentence the node is told (routes/nodes.ts answers 409). */ +export type CheckInOutcome = + | { node: NodeState; joined: boolean; movedFrom: string | null } + | { refused: string }; + +/** + * The fleet: every node that has ever checked in. Rows come from the + * database at start (a node that is down must still be known — its names + * are not new names); everything else fills in as the nodes report. + */ +export class Fleet { + private readonly members = new Map(); + + constructor(private readonly db: Db) { + for (const row of db.select().from(nodes).all()) { + this.members.set(row.id, { + id: row.id, + endpoint: row.endpoint, + addedAt: row.addedAt, + lastCheckInAt: null, + intervalSeconds: null, + build: null, + reading: null, + placedSinceCheckIn: 0, + placedIds: new Set(), + }); + } + } + + all(): NodeState[] { + return [...this.members.values()]; + } + + get(id: string): NodeState | undefined { + return this.members.get(id); + } + + /** + * A node reporting for duty. A first check-in adds the node (`joined` + * says so, for the log); a changed endpoint is written through — the + * node states where it lives, the gateway does not remember better — + * and `movedFrom` names the old one, for the log. Except inside the + * previous reporter's own interval: a different address that soon is a + * second machine with the same DORMICE_NODE_ID (the daemon's default is + * node-1), not a move, and is refused — written through, the two would + * flip the endpoint at every check-in, a lookup would ask whichever is + * current, a name on the other would read as new and be built again, on + * two nodes, with no 409 ever (traced by review, 2026-09-14). The first + * reporter keeps the id; the second is told why. A node that really + * moved is taken at its next check-in, one interval on. The + * placement counter restarts at zero: what was placed before this + * reading is in it now, and what is still in flight on the node (its + * row is written after the container is up) is in neither figure until + * the next reading — one interval of slack, self-correcting, the same + * as before. + */ + checkIn(report: CheckInRequest, now = new Date()): CheckInOutcome { + let node = this.members.get(report.nodeId); + let joined = false; + let movedFrom: string | null = null; + if (node === undefined) { + const addedAt = now.toISOString(); + this.db + .insert(nodes) + .values({ id: report.nodeId, endpoint: report.endpoint, addedAt }) + .run(); + node = { + id: report.nodeId, + endpoint: report.endpoint, + addedAt, + lastCheckInAt: null, + intervalSeconds: null, + build: null, + reading: null, + placedSinceCheckIn: 0, + placedIds: new Set(), + }; + this.members.set(node.id, node); + joined = true; + } else if (node.endpoint !== report.endpoint) { + if ( + node.lastCheckInAt !== null && + node.intervalSeconds !== null && + now.getTime() - node.lastCheckInAt.getTime() < + node.intervalSeconds * 1000 + ) { + const ago = Math.round( + (now.getTime() - node.lastCheckInAt.getTime()) / 1000, + ); + return { + refused: `node ${report.nodeId} checked in from ${node.endpoint} ${ago}s ago and now from ${report.endpoint} — two daemons share one DORMICE_NODE_ID (give this one its own), or the node just moved (then its next check-in, an interval later, is taken)`, + }; + } + this.db + .update(nodes) + .set({ endpoint: report.endpoint }) + .where(eq(nodes.id, report.nodeId)) + .run(); + movedFrom = node.endpoint; + node.endpoint = report.endpoint; + } + node.lastCheckInAt = now; + node.intervalSeconds = report.intervalSeconds; + node.build = report.build; + node.reading = report.reading; + node.placedSinceCheckIn = 0; + node.placedIds.clear(); + return { node, joined, movedFrom }; + } + + /** The operator's word that the node is gone for good (routes/nodes.ts refuses it for a node still checking in); one removed while briefly silent re-adds itself at its next check-in. */ + remove(id: string): boolean { + const existed = this.members.delete(id); + this.db.delete(nodes).where(eq(nodes.id, id)).run(); + return existed; + } +} diff --git a/packages/gateway/src/forward.test.ts b/packages/gateway/src/forward.test.ts new file mode 100644 index 00000000..60e5e24a --- /dev/null +++ b/packages/gateway/src/forward.test.ts @@ -0,0 +1,789 @@ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import net from 'node:net'; +import { request } from 'undici'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + forwardCapture, + forwardStream, + forwardUpgrade, + replay, + UnreachableError, +} from './forward'; + +const TOKEN = 'node-token-node-token-node-token-node'; +const servers: http.Server[] = []; +// Every accepted socket, upgraded ones included: http.Server#close waits +// for sockets it no longer tracks after an upgrade (the daemon's +// shutdown.ts has the reference), so teardown cuts them by hand. +const sockets = new Set(); +afterEach(async () => { + for (const socket of sockets) socket.destroy(); + sockets.clear(); + await Promise.all( + servers + .splice(0) + .map((s) => new Promise((resolve) => s.close(() => resolve()))), + ); +}); + +async function listen(server: http.Server): Promise { + servers.push(server); + server.on('connection', (socket) => { + sockets.add(socket); + socket.once('close', () => sockets.delete(socket)); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +/** A gateway-shaped front: every request is forwarded to `endpoint` with the given options. */ +async function front( + endpoint: string, + options: Partial[2]> = {}, +): Promise { + const server = http.createServer((req, res) => { + void forwardStream(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + ...options, + }).catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + res.writeHead(502, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ message })); + }); + }); + return listen(server); +} + +describe('forwardStream', () => { + it('streams the answer frame by frame: the first frame reaches the client before the node writes the second', async () => { + let releaseSecond: () => void = () => {}; + const gate = new Promise((resolve) => { + releaseSecond = resolve; + }); + const node = http.createServer((_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain' }); + res.write('first\n'); + void gate.then(() => res.end('second\n')); + }); + const router = await front(await listen(node)); + + const res = await request(`${router}/execCommand`, { method: 'POST' }); + const chunks: string[] = []; + const reader = res.body[Symbol.asyncIterator](); + const first = await reader.next(); + chunks.push(String(first.value)); + // Proof of streaming: the client holds the first frame while the node + // is still parked before its second. A buffering front could not get here. + expect(chunks).toEqual(['first\n']); + releaseSecond(); + for await (const chunk of { [Symbol.asyncIterator]: () => reader }) { + chunks.push(String(chunk)); + } + expect(chunks.join('')).toBe('first\nsecond\n'); + expect(res.headers['transfer-encoding']).toBe('chunked'); + }); + + it("the node's head reaches the client when it arrives, not with the first body byte", async () => { + let nodeEnded = false; + const node = http.createServer((_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain' }); + res.flushHeaders(); + setTimeout(() => { + nodeEnded = true; + res.end('late'); + }, 800); + }); + const endpoint = await listen(node); + const res = await request(`${await front(endpoint)}/execCommand`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: '{"name":"x"}', + }); + // undici resolves on the head; the node has not written a body byte. + expect(res.statusCode).toBe(200); + expect(nodeEnded).toBe(false); + expect(await res.body.text()).toBe('late'); + expect(nodeEnded).toBe(true); + }); + + it("resolves with the node's status once the answer is relayed — the named verbs read a 404 off it", async () => { + const node = http.createServer((_req, res) => { + res.writeHead(404, { 'content-type': 'application/json' }); + res.end('{"message":"no sandbox named \\"x\\""}'); + }); + const endpoint = await listen(node); + let resolved: number | null | undefined; + const front = http.createServer((req, res) => { + void forwardStream(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + }).then((status) => { + resolved = status; + }); + }); + const url = await listen(front); + const res = await request(`${url}/execCommand`, { method: 'POST' }); + expect(res.statusCode).toBe(404); + await res.body.text(); + const deadline = Date.now() + 1_000; + while (resolved === undefined && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(resolved).toBe(404); + }); + + it('keeps a fixed-length answer fixed-length and passes response headers through — pagination, cookies, CORS', async () => { + const node = http.createServer((_req, res) => { + res.writeHead(200, { + 'content-type': 'application/json', + 'content-length': '5', + 'x-next-token': 'abc', + 'set-cookie': ['a=1', 'b=2'], + 'access-control-allow-origin': '*', + }); + res.end('hello'); + }); + const router = await front(await listen(node)); + const res = await request(`${router}/listSandboxes`, { method: 'POST' }); + expect(res.statusCode).toBe(200); + expect(res.headers['content-length']).toBe('5'); + expect(res.headers['transfer-encoding']).toBeUndefined(); + expect(res.headers['x-next-token']).toBe('abc'); + expect(res.headers['set-cookie']).toEqual(['a=1', 'b=2']); + expect(res.headers['access-control-allow-origin']).toBe('*'); + expect(await res.body.text()).toBe('hello'); + }); + + it('a client that goes away mid-stream takes the node connection down with it', async () => { + let nodeSawClose: () => void = () => {}; + const closed = new Promise((resolve) => { + nodeSawClose = resolve; + }); + const node = http.createServer((req, res) => { + res.writeHead(200); + res.write('tick\n'); + const timer = setInterval(() => res.write('tick\n'), 20); + req.on('close', () => { + clearInterval(timer); + nodeSawClose(); + }); + }); + const router = await front(await listen(node)); + const res = await request(`${router}/stream`, { method: 'POST' }); + const reader = res.body[Symbol.asyncIterator](); + await reader.next(); + res.body.destroy(); + await closed; + }); + + it('a client already gone when the forward begins is not sent to the node at all, and answers null', async () => { + let requests = 0; + const node = http.createServer((_req, res) => { + requests += 1; + res.end('{}'); + }); + const endpoint = await listen(node); + // The shape forwardNamed produces: the lookup round took a while and + // the caller hung up meanwhile — the response is destroyed before the + // forward starts, so its 'close' has already fired and no abort would + // ever follow. + const gone = new http.IncomingMessage(new net.Socket()); + const res = new http.ServerResponse(gone); + res.destroy(); + expect( + await forwardStream( + Object.assign(gone, { + url: '/execCommand', + method: 'POST', + headers: {}, + }), + res, + { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + body: Buffer.from('{"name":"x","command":"sleep 3600"}'), + }, + ), + ).toBeNull(); + expect(requests).toBe(0); + }); + + it('forwards the request verbatim — path, query, body — with the credential swapped and the Host renamed to the node unless the face keeps it', async () => { + const seen: Array<{ + url?: string; + headers: http.IncomingHttpHeaders; + body: string; + }> = []; + const node = http.createServer((req, res) => { + let body = ''; + req.on('data', (c) => { + body += c; + }); + req.on('end', () => { + seen.push({ url: req.url, headers: req.headers, body }); + res.writeHead(200); + res.end(); + }); + }); + const endpoint = await listen(node); + const router = await front(endpoint, { credential: 'bearer' }); + await request(`${router}/readFile?x=1`, { + method: 'POST', + headers: { + host: '8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test', + authorization: 'Bearer caller-secret', + 'content-type': 'application/json', + connection: 'keep-alive', + }, + body: '{"name":"alice"}', + }); + expect(seen).toHaveLength(1); + expect(seen[0]?.url).toBe('/readFile?x=1'); + expect(seen[0]?.body).toBe('{"name":"alice"}'); + // The Host named the gateway; the node is asked as itself, so a Caddy + // in front of it that binds the gateway's domain never answers a 308. + expect(seen[0]?.headers.host).toBe(new URL(endpoint).host); + expect(seen[0]?.headers.authorization).toBe(`Bearer ${TOKEN}`); + expect(seen[0]?.headers['content-type']).toBe('application/json'); + + // The E2B face swaps the other header; the credential-less faces touch none. + const asE2b = await front(endpoint, { credential: 'x-api-key' }); + await request(`${asE2b}/sandboxes`, { + method: 'POST', + headers: { 'x-api-key': 'e2b_caller' }, + }); + expect(seen[1]?.headers['x-api-key']).toBe(`e2b_${TOKEN}`); + const asEnvd = await front(endpoint, { credential: 'none' }); + await request(`${asEnvd}/e2b/envd/files`, { + method: 'GET', + headers: { authorization: 'Basic dXNlcjo=', 'x-access-token': 'hmac' }, + }); + expect(seen[2]?.headers.authorization).toBe('Basic dXNlcjo='); + expect(seen[2]?.headers['x-access-token']).toBe('hmac'); + + // The proxy face is the one that keeps the Host: it is the routing + // key on the node as well. + const asProxy = await front(endpoint, { + credential: 'none', + preserveHost: true, + }); + await request(`${asProxy}/`, { + method: 'GET', + headers: { host: '8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test' }, + }); + expect(seen[3]?.headers.host).toBe( + '8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test', + ); + }); + + it('sends the request target byte for byte — dot segments included — instead of a URL it resolved itself', async () => { + const seen: string[] = []; + const node = http.createServer((req, res) => { + seen.push(req.url ?? ''); + res.writeHead(200); + res.end(); + }); + const endpoint = await listen(node); + const router = await front(endpoint); + const url = new URL(router); + // A client library would collapse the segments before they left; the + // socket carries them as an attacker would. + const path = '/e2b/api/sandboxes/some-id/../../sandboxes?x=1'; + await new Promise((resolve, reject) => { + const socket = net.connect(Number(url.port), url.hostname, () => { + socket.write( + `GET ${path} HTTP/1.1\r\nhost: router\r\nconnection: close\r\n\r\n`, + ); + }); + socket.on('data', () => {}); + socket.on('end', resolve); + socket.on('error', reject); + }); + expect(seen).toEqual([path]); + }); + + it('drops the Expect header: the 100-continue handshake ended at the gateway, and undici refuses to carry it', async () => { + const seen: http.IncomingHttpHeaders[] = []; + const node = http.createServer((req, res) => { + req.resume(); + req.on('end', () => { + seen.push(req.headers); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end('{"ok":true}'); + }); + }); + const endpoint = await listen(node); + const router = await front(endpoint); + // curl adds this to every body over 1024 bytes (undici's own client + // refuses to send it, so the exam speaks node:http): Node's server + // answered the 100 before forwardStream saw the request. + const body = JSON.stringify({ name: 'big', content: 'x'.repeat(2048) }); + const res = await new Promise<{ status: number; text: string }>( + (resolve, reject) => { + const req = http.request( + `${router}/writeFile`, + { + method: 'POST', + headers: { + expect: '100-continue', + 'content-type': 'application/json', + 'content-length': String(Buffer.byteLength(body)), + }, + }, + (answer) => { + let text = ''; + answer.on('data', (c) => { + text += c; + }); + answer.on('end', () => + resolve({ status: answer.statusCode ?? 0, text }), + ); + }, + ); + req.on('continue', () => req.end(body)); + req.on('error', reject); + }, + ); + expect(res.status).toBe(200); + expect(JSON.parse(res.text)).toEqual({ ok: true }); + expect(seen[0]?.expect).toBeUndefined(); + }); + + it('a node that does not answer is an UnreachableError, tried exactly once', async () => { + let hits = 0; + const node = http.createServer((req) => { + hits += 1; + req.socket.destroy(); + }); + const endpoint = await listen(node); + const router = await front(endpoint); + const cut = await request(`${router}/acquireSandbox`, { method: 'POST' }); + expect(cut.statusCode).toBe(502); + expect((await cut.body.json()) as { message: string }).toMatchObject({ + message: expect.stringMatching( + /^node http:\/\/127\.0\.0\.1:\d+ did not answer: /, + ), + }); + expect(hits).toBe(1); + + const refused = await front('http://127.0.0.1:9'); + const res = await request(`${refused}/acquireSandbox`, { method: 'POST' }); + expect(res.statusCode).toBe(502); + expect(((await res.body.json()) as { message: string }).message).toMatch( + /did not answer: ECONNREFUSED/, + ); + }); +}); + +describe('forwardCapture + replay', () => { + it('collects the answer whole and replays it verbatim, asks for it uncompressed, and reports a refused connection as unreachable', async () => { + const encodings: Array = []; + const node = http.createServer((req, res) => { + encodings.push(req.headers['accept-encoding']); + res.writeHead(201, { + 'content-type': 'application/json', + 'x-extra': 'y', + }); + res.end('{"sandboxID":"s-1"}'); + }); + const endpoint = await listen(node); + const server = http.createServer((req, res) => { + void forwardCapture(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'x-api-key', + }).then( + (answer) => { + if (answer === null) throw new Error('the client did not leave'); + expect(answer.status).toBe(201); + expect(JSON.parse(answer.body.toString())).toEqual({ + sandboxID: 's-1', + }); + replay(res, answer); + }, + (error: unknown) => { + res.writeHead(502); + res.end(error instanceof Error ? error.name : 'other'); + }, + ); + }); + const router = await listen(server); + const res = await request(`${router}/sandboxes`, { + method: 'POST', + headers: { 'accept-encoding': 'gzip, br' }, + }); + expect(res.statusCode).toBe(201); + expect(res.headers['x-extra']).toBe('y'); + expect(res.headers['content-length']).toBe('19'); + expect(await res.body.json()).toEqual({ sandboxID: 's-1' }); + // A compressing hop between router and node would hand back bytes + // the gateway cannot read the id out of. + expect(encodings).toEqual(['identity']); + + const orphan = new http.IncomingMessage(new net.Socket()); + await expect( + forwardCapture( + Object.assign(orphan, { url: '/x', method: 'POST', headers: {} }), + new http.ServerResponse(orphan), + { + target: { endpoint: 'http://127.0.0.1:9', token: TOKEN }, + credential: 'bearer', + body: Buffer.from('{}'), + }, + ), + ).rejects.toBeInstanceOf(UnreachableError); + }); +}); + +describe('forwardCapture + replay — bodiless answers', () => { + it('replays a 204 without inventing a content-length', async () => { + const node = http.createServer((_req, res) => { + res.writeHead(204); + res.end(); + }); + const endpoint = await listen(node); + const front = http.createServer((req, res) => { + void forwardCapture(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'x-api-key', + }).then((answer) => answer && replay(res, answer)); + }); + const url = await listen(front); + const res = await request(`${url}/e2b/api/sandboxes/x`, { + method: 'DELETE', + }); + expect(res.statusCode).toBe(204); + expect(res.headers['content-length']).toBeUndefined(); + await res.body.text(); + }); +}); + +describe('forwardStream — a client that leaves', () => { + it('withdraws the request from the node when the client leaves before the node answered', async () => { + let nodeSawClose: number | null = null; + const node = http.createServer((req, res) => { + const at = Date.now(); + req.on('close', () => { + nodeSawClose = Date.now() - at; + }); + // Answers late — the client will be gone by then. + setTimeout(() => { + if (!res.destroyed) { + res.writeHead(200); + res.end('late'); + } + }, 1500); + }); + const endpoint = await listen(node); + const routerUrl = await front(endpoint); + const url = new URL(routerUrl); + const client = http.request({ + host: url.hostname, + port: url.port, + path: '/execCommand', + method: 'POST', + headers: { 'content-type': 'application/json' }, + }); + client.on('error', () => {}); + client.end('{"name":"x"}'); + await new Promise((resolve) => setTimeout(resolve, 100)); + client.destroy(); + const deadline = Date.now() + 1_000; + while (nodeSawClose === null && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + // Well before the node's own answer at 1.5s: the gateway aborted it. + expect(nodeSawClose).not.toBeNull(); + expect(nodeSawClose as unknown as number).toBeLessThan(1_000); + }); +}); + +describe('forwardCapture — a client that leaves', () => { + it('withdraws the request from the node when the client leaves before the node answered, and answers null; a client already gone is not sent at all', async () => { + let nodeSawClose: number | null = null; + let requests = 0; + const node = http.createServer((req, res) => { + requests += 1; + const at = Date.now(); + req.on('close', () => { + nodeSawClose = Date.now() - at; + }); + setTimeout(() => { + if (!res.destroyed) { + res.writeHead(200); + res.end('{}'); + } + }, 1500); + }); + const endpoint = await listen(node); + const outcomes: Array<'null' | 'answer' | 'error'> = []; + const front = http.createServer((req, res) => { + void forwardCapture(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + }).then( + (answer) => { + outcomes.push(answer === null ? 'null' : 'answer'); + if (answer !== null) replay(res, answer); + }, + () => outcomes.push('error'), + ); + }); + const url = new URL(await listen(front)); + const client = http.request({ + host: url.hostname, + port: url.port, + path: '/acquireSandbox', + method: 'POST', + headers: { 'content-type': 'application/json' }, + }); + client.on('error', () => {}); + client.end('{"name":"x"}'); + await new Promise((resolve) => setTimeout(resolve, 100)); + client.destroy(); + const deadline = Date.now() + 1_000; + while (outcomes.length === 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + // Well before the node's own answer at 1.5s: the gateway aborted it + // and told the caller there is nothing to send. + expect(outcomes).toEqual(['null']); + expect(nodeSawClose).not.toBeNull(); + expect(nodeSawClose as unknown as number).toBeLessThan(1_000); + + // A verb whose client left while it waited for its slot: the response + // is already gone when the capture begins, and the node hears nothing. + const gone = new http.IncomingMessage(new net.Socket()); + const res = new http.ServerResponse(gone); + res.destroy(); + expect( + await forwardCapture( + Object.assign(gone, { url: '/x', method: 'POST', headers: {} }), + res, + { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + body: Buffer.from('{}'), + }, + ), + ).toBeNull(); + expect(requests).toBe(1); + }); +}); + +describe('forwardCapture — the head arrived, then the client left', () => { + it('reads the small body to the end and answers it, so a 2xx that did land is still learned; replay writes nothing to nobody', async () => { + let finishBody: () => void = () => {}; + let headOut: () => void = () => {}; + const headWritten = new Promise((resolve) => { + headOut = resolve; + }); + const node = http.createServer((_req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.write('{"sandbox":{"id":"sb-1"', headOut); + finishBody = () => res.end('}}'); + }); + const endpoint = await listen(node); + let answer: Awaited> | undefined; + let clientGone: () => void = () => {}; + const gone = new Promise((resolve) => { + clientGone = resolve; + }); + const front = http.createServer((req, res) => { + res.once('close', () => clientGone()); + void forwardCapture(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + }).then((a) => { + answer = a; + if (a !== null) replay(res, a); + }); + }); + const url = new URL(await listen(front)); + const client = http.request({ + host: url.hostname, + port: url.port, + path: '/acquireSandbox', + method: 'POST', + }); + client.on('error', () => {}); + client.end('{"name":"x"}'); + // The node's head is out; the body is not. The client gives up. + await headWritten; + client.destroy(); + await gone; + finishBody(); + const deadline = Date.now() + 2_000; + while (answer === undefined && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(answer).toBeDefined(); + expect(answer).not.toBeNull(); + expect(answer?.status).toBe(200); + expect(JSON.parse(answer?.body.toString('utf8') ?? '')).toEqual({ + sandbox: { id: 'sb-1' }, + }); + }); +}); + +describe('forwardUpgrade', () => { + it('a node behind TLS is refused with a 502 before any dial — upgrades are plain TCP here', async () => { + const router = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + router.on('upgrade', (req, socket, head) => + forwardUpgrade(req, socket, head, { + endpoint: 'https://node.example:443', + token: TOKEN, + }), + ); + const routerUrl = await listen(router); + const client = net.connect(Number(new URL(routerUrl).port), '127.0.0.1'); + const received: string[] = []; + client.on('data', (chunk) => received.push(String(chunk))); + const closed = new Promise((resolve) => client.on('close', resolve)); + client.write( + 'GET /ws HTTP/1.1\r\nHost: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + await closed; + expect(received.join('')).toContain('HTTP/1.1 502 Bad Gateway'); + expect(received.join('')).toContain('not forwarded over TLS'); + expect(received.join('')).not.toContain('node.example'); + }); + + it('a node that dies after the handshake cuts the client without writing an HTTP status into the upgraded stream', async () => { + const node = http.createServer((_req, res) => { + res.writeHead(426); + res.end(); + }); + node.on('upgrade', (req, socket) => { + socket.write( + `HTTP/1.1 101 Switching Protocols\r\nUpgrade: ${req.headers.upgrade}\r\nConnection: Upgrade\r\n\r\n`, + ); + // RST from the node side: the router's upstream sees an error, not a FIN. + socket.on('data', () => (socket as net.Socket).resetAndDestroy()); + }); + const endpoint = await listen(node); + const router = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + router.on('upgrade', (req, socket, head) => + forwardUpgrade(req, socket, head, { endpoint, token: TOKEN }), + ); + const routerUrl = await listen(router); + const client = net.connect(Number(new URL(routerUrl).port), '127.0.0.1'); + const received: string[] = []; + client.on('data', (chunk) => received.push(String(chunk))); + const closed = new Promise((resolve) => client.on('close', resolve)); + client.write( + 'GET /ws HTTP/1.1\r\nHost: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + await new Promise((resolve) => { + const check = () => { + if (received.join('').includes('101')) resolve(); + else client.once('data', check); + }; + check(); + }); + client.write('ping'); + await closed; + expect(received.join('')).not.toContain('502'); + }); + + it("a node that black-holes the dial is a 502 within the connect deadline, not the kernel's SYN budget", async () => { + const router = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + // TEST-NET-1 (RFC 5737) is never routed: the SYN is dropped, or the + // network answers unreachable at once — a 502 either way, and without + // the deadline the dropped case held this test for 75s on macOS. + router.on('upgrade', (req, socket, head) => + forwardUpgrade( + req, + socket, + head, + { endpoint: 'http://192.0.2.1:80', token: TOKEN }, + 500, + ), + ); + const routerUrl = await listen(router); + const client = net.connect(Number(new URL(routerUrl).port), '127.0.0.1'); + const received: string[] = []; + client.on('data', (chunk) => received.push(String(chunk))); + const closed = new Promise((resolve) => client.on('close', resolve)); + const started = Date.now(); + client.write( + 'GET /ws HTTP/1.1\r\nHost: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + await closed; + expect(received.join('')).toContain('HTTP/1.1 502 Bad Gateway'); + expect(received.join('')).toContain('did not answer the upgrade'); + expect(Date.now() - started).toBeLessThan(4000); + }); + + it('replays the upgrade handshake to the node and pipes both ways', async () => { + const node = http.createServer((_req, res) => { + res.writeHead(426); + res.end(); + }); + let nodeSideClosed: () => void = () => {}; + const nodeClosed = new Promise((resolve) => { + nodeSideClosed = resolve; + }); + node.on('upgrade', (req, socket) => { + socket.write( + `HTTP/1.1 101 Switching Protocols\r\nUpgrade: ${req.headers.upgrade}\r\nConnection: Upgrade\r\nX-Seen-Host: ${req.headers.host}\r\n\r\n`, + ); + socket.on('data', (chunk) => socket.write(`echo:${chunk}`)); + // A real upgraded peer (a WebSocket server in the sandbox) ends its + // side when the other side does; http sockets allow half-open, so + // without this the chain would hang on a peer that never answers FIN. + socket.on('end', () => socket.end()); + socket.on('close', nodeSideClosed); + }); + const endpoint = await listen(node); + const router = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + router.on('upgrade', (req, socket, head) => + forwardUpgrade(req, socket, head, { endpoint, token: TOKEN }), + ); + const routerUrl = await listen(router); + const port = Number(new URL(routerUrl).port); + + const client = net.connect(port, '127.0.0.1'); + const received: string[] = []; + const done = new Promise((resolve) => { + client.on('data', (chunk) => { + received.push(String(chunk)); + if (received.join('').includes('echo:ping')) resolve(); + }); + }); + client.write( + 'GET /ws HTTP/1.1\r\nHost: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + await new Promise((resolve) => { + const check = () => { + if (received.join('').includes('101')) resolve(); + else client.once('data', check); + }; + check(); + }); + client.write('ping'); + await done; + const all = received.join(''); + expect(all).toContain('HTTP/1.1 101 Switching Protocols'); + expect(all).toContain( + 'X-Seen-Host: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test', + ); + expect(all).toContain('echo:ping'); + // The client leaving takes the node-side socket down with it. + client.destroy(); + await nodeClosed; + }); +}); diff --git a/packages/gateway/src/forward.ts b/packages/gateway/src/forward.ts new file mode 100644 index 00000000..a867546d --- /dev/null +++ b/packages/gateway/src/forward.ts @@ -0,0 +1,374 @@ +import type http from 'node:http'; +import net from 'node:net'; +import type { Duplex } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { Agent } from 'undici'; +import { causeOf } from './lookup'; + +/** + * The one place the gateway talks to a node on a caller's behalf. Bytes + * in, bytes out: the request body is streamed (or handed over as the + * buffer Fastify already read), the response is written head-first and + * piped — never buffered, never re-parsed, never re-framed. Every + * long-lived shape a node produces (an exec that answers after an hour, + * an envd process stream, a proxied SSE) rides through untouched. + * + * The dispatcher switches undici's hidden clocks off, for the same reason + * the SDK's does (packages/sdk/src/client.ts): a node legitimately takes + * hours to answer an exec, and a default dispatcher gives up on headers + * after 300s. Connecting keeps a short deadline — a node that will not + * even accept the connection in ten seconds is unreachable now. + * + * Never retried, never re-routed. A create that did not answer may have + * created; sending it again elsewhere is how one name ends up on two + * nodes. The caller gets the honest 502, and its retry finds the sandbox + * by asking (find.ts) wherever it landed. + */ +const CONNECT_TIMEOUT_MS = 10_000; +const agent = new Agent({ + headersTimeout: 0, + bodyTimeout: 0, + connect: { timeout: CONNECT_TIMEOUT_MS }, +}); + +export interface ForwardTarget { + endpoint: string; + /** The fleet's one token: what the gateway presents to every node. */ + token: string; +} + +/** + * How the node is authenticated to: `bearer` replaces the Authorization + * header with the fleet token (native verbs), `x-api-key` replaces + * X-API-KEY with `e2b_` (E2B control plane), `none` touches no + * credential header at all — envd traffic carries its own (an access + * token minted by the node itself) and the gateway has nothing to add. + * No header names the real caller: the gateway is the fleet's one door + * and the node trusts it whole (design record #3). + */ +export type Credential = 'bearer' | 'x-api-key' | 'none'; + +export interface ForwardOptions { + target: ForwardTarget; + credential: Credential; + /** The request body when Fastify already consumed the stream; omit to stream req itself. */ + body?: Buffer | undefined; + /** + * Keep the caller's Host header. Only a face keyed on the Host wants + * this (the sandbox port proxy, once it routes through the gateway). + * Everywhere else the Host names the gateway, and carrying it to a node + * whose Caddy binds that very domain gets a 308 to https instead of the + * daemon — so the default lets undici name the node's own endpoint. + */ + preserveHost?: boolean; +} + +/** The node did not answer at all — no headers came back. */ +export class UnreachableError extends Error { + constructor( + readonly node: string, + /** The transport's word for why (ECONNREFUSED, a timeout, a cut body). */ + readonly why: string, + ) { + super(`node ${node} did not answer: ${why}`); + this.name = 'UnreachableError'; + } +} + +// Hop-by-hop headers describe one connection, not the message; a proxy +// that forwards them lies to the next hop about its framing. `expect` is +// one too: the 100-continue handshake is between the client and the +// first server, and Node's http answered it before this code ran (a +// server with no checkContinue listener sends 100 and reads the body). +// undici refuses to forward it outright (NotSupportedError), and curl +// adds it to every body over 1024 bytes — so without this line a 1.5MB +// writeFile through the front was a 502 (measured 2026-09-12). +const HOP_BY_HOP = new Set([ + 'connection', + 'expect', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade', +]); + +function outboundHeaders( + req: http.IncomingMessage, + options: ForwardOptions, +): Record { + const headers: Record = {}; + for (const [name, value] of Object.entries(req.headers)) { + if (value === undefined || HOP_BY_HOP.has(name)) continue; + // A buffered body is re-framed by undici; the client's length may + // describe a chunked encoding that no longer exists. + if (name === 'content-length' && options.body !== undefined) continue; + if (name === 'host' && options.preserveHost !== true) continue; + headers[name] = value; + } + switch (options.credential) { + case 'bearer': + headers.authorization = `Bearer ${options.target.token}`; + break; + case 'x-api-key': + headers['x-api-key'] = `e2b_${options.target.token}`; + break; + case 'none': + break; + } + return headers; +} + +function inboundHeaders( + headers: Record, +): http.OutgoingHttpHeaders { + const out: http.OutgoingHttpHeaders = {}; + for (const [name, value] of Object.entries(headers)) { + if (value === undefined || HOP_BY_HOP.has(name)) continue; + out[name] = value; + } + return out; +} + +/** + * The request target is sent to the node exactly as the caller wrote it — + * origin and path handed to undici separately, never joined into a URL. + * A URL parser resolves dot segments: `/e2b/api/sandboxes//../../ + * sandboxes` is a sub-route of a known id to Fastify, which routed it to + * the id's node, and `/e2b/api/sandboxes` to the parser — so a caller + * holding any live id could reach the node's create, under the fleet's + * credential, past placement and the name slot (reproduced 2026-09-12). + * The node judges the path it is sent; the gateway's routing and the + * node's must see the same bytes. + */ +async function dispatch( + req: http.IncomingMessage, + options: ForwardOptions, + override: Record = {}, + signal?: AbortSignal, +) { + try { + return await agent.request({ + origin: options.target.endpoint, + path: req.url ?? '/', + method: req.method as 'GET', + headers: { ...outboundHeaders(req, options), ...override }, + body: options.body ?? req, + signal, + }); + } catch (error) { + throw new UnreachableError(options.target.endpoint, causeOf(error)); + } +} + +/** + * Forwards the request and streams the node's answer back as it arrives; + * resolves with the node's status once the answer has been relayed (the + * named verbs read a 404 off it to re-check the cache — find.ts verify). + * Throws UnreachableError only before any byte of the answer was written; + * once the head is out, a failure mid-stream can only be a cut connection + * — there is no honest status left to send. Resolves null when the client + * left before the node answered: the node-side request is aborted — or, + * for a client already gone, never sent — and nothing is rendered (the + * response is gone). Otherwise an abandoned exec + * against a slow node would hold a gateway→node socket until the node + * answered — on a hung node, until its TCP died. forwardCapture follows + * the same rule. + */ +export async function forwardStream( + req: http.IncomingMessage, + res: http.ServerResponse, + options: ForwardOptions, +): Promise { + // The client left while its sandbox was being found (a lookup round is + // up to two seconds): 'close' has fired already, the abort below would + // never come, and the node would run an exec to its end for nobody — + // forwardCapture's first line, missing here (found by review, 2026-09-14). + if (res.destroyed) return null; + const gone = new AbortController(); + const onClose = () => gone.abort(); + res.once('close', onClose); + let upstream: Awaited>; + try { + upstream = await dispatch(req, options, {}, gone.signal); + } catch (error) { + if (gone.signal.aborted) return null; + throw error; + } finally { + res.off('close', onClose); + } + res.writeHead(upstream.statusCode, inboundHeaders(upstream.headers)); + // Node holds a written head until the first body byte; the node's head + // has arrived, so it goes out now — a stream that opens and then waits + // (a process stream before its first event) must look open to the + // caller, not like a node that has not answered (found by review, + // 2026-09-14). + res.flushHeaders(); + try { + // pipeline destroys both ends on failure: a client that went away + // aborts the node's response, a node that died cuts the client. + await pipeline(upstream.body, res); + } catch { + res.destroy(); + } + return upstream.statusCode; +} + +export interface CapturedResponse { + status: number; + headers: http.OutgoingHttpHeaders; + body: Buffer; +} + +/** + * Forwards and collects the whole answer, for the few verbs whose answer + * the gateway must read before passing it on: a create (to learn the id + * the node minted, for the cache) and a destroy (to forget the entry). + * Their answers are small JSON; everything else streams. + * + * The same abort rule as forwardStream, and for the same reason: a client + * that leaves before the node has answered takes its request with it, and + * `null` says so (the caller writes nothing — the response is gone). These + * verbs hold the name's queue slot, and a request nobody is waiting for + * must not hold it: a node that accepts the connection and never answers + * (a daemon with a hung event loop — 2026-08-13 and 2026-09-11 shapes) + * would otherwise pin the name until its TCP died, every retry queueing + * behind the abandoned attempt and firing at the node together when it + * came back. A queued verb whose client left while it waited for the slot + * is not sent at all. Once the node's head has arrived the small body is + * read to the end regardless, so a 2xx that did land is still learned. + */ +export async function forwardCapture( + req: http.IncomingMessage, + res: http.ServerResponse, + options: ForwardOptions, +): Promise { + if (res.destroyed) return null; + const gone = new AbortController(); + const onClose = () => gone.abort(); + res.once('close', onClose); + let upstream: Awaited>; + try { + // The gateway is the reader of this answer, so it states its own + // preference: identity. A caller's accept-encoding would let a hop + // that compresses (a Caddy with `encode` in front of the node) hand + // back bytes the gateway cannot read the id out of. + upstream = await dispatch( + req, + options, + { 'accept-encoding': 'identity' }, + gone.signal, + ); + } catch (error) { + if (gone.signal.aborted) return null; + throw error; + } finally { + res.off('close', onClose); + } + let body: Buffer; + try { + body = Buffer.from(await upstream.body.arrayBuffer()); + } catch (error) { + throw new UnreachableError( + options.target.endpoint, + `answer cut mid-body: ${causeOf(error)}`, + ); + } + return { + status: upstream.statusCode, + headers: inboundHeaders(upstream.headers), + body, + }; +} + +/** Writes a captured answer through, verbatim. */ +export function replay( + res: http.ServerResponse, + answer: CapturedResponse, +): void { + // The client left while the small body was being read: nothing to say + // to nobody (what the answer taught the cache is kept regardless). + if (res.destroyed) return; + // A 204, a 304 or a 1xx carries no body and must not claim a length + // (RFC 9110 §8.6): the node sent none, and a `content-length: 0` added + // here would change the framing this file promises to keep. + const bodiless = + answer.status === 204 || answer.status === 304 || answer.status < 200; + res.writeHead( + answer.status, + bodiless + ? answer.headers + : { ...answer.headers, 'content-length': String(answer.body.length) }, + ); + res.end(bodiless ? undefined : answer.body); +} + +/** + * The upgrade path (sandbox WebSockets, once the port proxy routes + * through the gateway): the daemon's own replay (sandbox-proxy.ts + * handleUpgrade) — dial the node, write the request line and rawHeaders + * verbatim, then pipe both ways. Plain TCP: node endpoints are + * private-network http, and a TLS node endpoint is refused here rather + * than half-supported. + */ +export function forwardUpgrade( + req: http.IncomingMessage, + socket: Duplex, + head: Buffer, + target: ForwardTarget, + connectTimeoutMs = CONNECT_TIMEOUT_MS, +): void { + socket.on('error', () => socket.destroy()); + const url = new URL(target.endpoint); + if (url.protocol !== 'http:') { + // An operator's configuration, refused without naming the address: + // this face is unauthenticated (raw.ts). + socket.end( + `HTTP/1.1 502 Bad Gateway\r\ncontent-type: application/json\r\nconnection: close\r\n\r\n${JSON.stringify({ message: "the sandbox's node is not a plain http endpoint — upgrades are not forwarded over TLS" })}`, + ); + return; + } + // A URL keeps the brackets of an IPv6 literal in hostname; a dialer + // wants them off. + const host = url.hostname.replace(/^\[(.*)\]$/, '$1'); + let replayed = false; + // The HTTP faces' connect deadline (the dispatcher above), here by hand: + // a node whose host is down without an RST black-holes the SYN, and a + // bare net.connect would hold the client for the OS's own retry budget + // (75s on macOS, about two minutes on Linux — measured 2026-09-12 with + // nc against 192.0.2.1) before the 502 the HTTP faces give in ten. + // Idle after the handshake is legitimate (a quiet WebSocket), so the + // timer is switched off the moment the connection is up. + const upstream = net.connect( + { port: Number(url.port || 80), host, timeout: connectTimeoutMs }, + () => { + upstream.setTimeout(0); + replayed = true; + let raw = `${req.method} ${req.url} HTTP/1.1\r\n`; + for (let i = 0; i < req.rawHeaders.length; i += 2) { + raw += `${req.rawHeaders[i]}: ${req.rawHeaders[i + 1]}\r\n`; + } + upstream.write(`${raw}\r\n`); + if (head.length > 0) upstream.write(head); + socket.pipe(upstream); + upstream.pipe(socket); + }, + ); + upstream.on('timeout', () => upstream.destroy(new Error('connect timeout'))); + upstream.on('error', () => { + // An HTTP status is honest only before the handshake was replayed; + // after it the client is inside an upgraded stream (WebSocket frames), + // where a 502 line would be a corrupt frame, not an answer. + if (!replayed && !socket.writableEnded && socket.writable) { + socket.end( + `HTTP/1.1 502 Bad Gateway\r\ncontent-type: application/json\r\nconnection: close\r\n\r\n${JSON.stringify({ message: "the sandbox's node did not answer the upgrade — retry" })}`, + ); + } else { + socket.destroy(); + } + }); + socket.on('close', () => upstream.destroy()); + upstream.on('close', () => socket.destroy()); +} diff --git a/packages/gateway/src/lookup.ts b/packages/gateway/src/lookup.ts new file mode 100644 index 00000000..ad471898 --- /dev/null +++ b/packages/gateway/src/lookup.ts @@ -0,0 +1,95 @@ +import { + type LookupSandboxRequest, + lookupSandboxResponseSchema, + type SandboxState, +} from '@dormice/shared'; + +/** + * Asking one node "do you hold this sandbox?" — the daemon's lookupSandbox + * verb, the one question the gateway puts to a node on its own account + * (everything else it sends is a caller's request, forwarded raw). Two + * seconds, not more: a node that cannot answer a ledger read in two + * seconds is a node in trouble, and the caller is waiting on the whole + * round. There is no second, slower deadline — slow is down (design + * record #35). + */ +export const LOOKUP_TIMEOUT_MS = 2_000; + +export type LookupQuery = LookupSandboxRequest; + +export type LookupAnswer = + | { kind: 'found'; id: string; name: string; state: SandboxState } + | { kind: 'absent' } + /** The node did not answer: no connection, a timeout, a non-200, an unreadable body. `why` is the transport's word. */ + | { kind: 'silent'; why: string }; + +export interface AskedNode { + id: string; + endpoint: string; +} + +export type AskNode = ( + node: AskedNode, + query: LookupQuery, +) => Promise; + +/** + * The transport's word for a failure, for a log line or a 503's sentence: + * a system code where there is one (ECONNREFUSED, UND_ERR_CONNECT_TIMEOUT), + * else the message. Only string codes count — a DOMException carries a + * numeric legacy `code` (TimeoutError is 23), and "node b did not answer + * (23)" tells an operator nothing (found by review, 2026-09-14). + */ +export function causeOf(error: unknown): string { + const e = error as { code?: unknown; message?: string; cause?: unknown }; + const cause = e.cause as { code?: unknown; message?: string } | undefined; + if (typeof cause?.code === 'string') return cause.code; + if (cause?.message !== undefined) return cause.message; + if (typeof e.code === 'string') return e.code; + return e.message ?? String(error); +} + +/** + * The production asker: HTTP to the node's endpoint under the fleet's + * token. `fetch`, not undici's `request`, for the deadline: `request` + * ignores an abort signal while the socket is still connecting, so a node + * whose host drops the SYN (a VM deleted, a security group closed — the + * machine-gone case removeNode exists for) held every question for + * undici's 10s connect timeout, not two seconds — and the creator's + * confirmation runs inside the name's slot, so twenty queued acquires of a + * name cached there would have waited 3.5 minutes for the last (measured + * 2026-09-14: request 10 500ms, fetch 2 001ms, against 192.0.2.1). No + * redirect is followed: a front that redirects is not a node, and fetch + * would drop the Authorization header across origins on the way. One + * quirk comes with fetch: it refuses the Fetch standard's "bad ports" + * (9, 22, 25, 6000 …) without dialling — no node's front lives on one. + */ +export function httpAskNode(token: string): AskNode { + return async (node, query) => { + try { + const res = await fetch(`${node.endpoint}/lookupSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(query), + signal: AbortSignal.timeout(LOOKUP_TIMEOUT_MS), + redirect: 'manual', + }); + if (res.status !== 200) { + const text = await res.text(); + return { + kind: 'silent', + why: `lookupSandbox answered ${res.status}: ${text.slice(0, 200)}`, + }; + } + const answer = lookupSandboxResponseSchema.parse(await res.json()); + return answer.found + ? { kind: 'found', ...answer.sandbox } + : { kind: 'absent' }; + } catch (error) { + return { kind: 'silent', why: causeOf(error) }; + } + }; +} diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts new file mode 100644 index 00000000..be302d6c --- /dev/null +++ b/packages/gateway/src/main.ts @@ -0,0 +1,135 @@ +import { fileURLToPath } from 'node:url'; +import { KeyedQueue } from '@dormice/server/keyed-queue'; +import { acquireSingleWriterLock } from '@dormice/server/lock'; +import { closeWithGrace, trackConnections } from '@dormice/server/shutdown'; +import { pino } from 'pino'; +import { z } from 'zod'; +import { buildGatewayApp } from './app'; +import { NameCache } from './cache'; +import { loadConfig } from './config'; +import { migrateDb, openDb } from './db/db'; +import { Finder } from './find'; +import { Fleet } from './fleet'; +import { httpAskNode } from './lookup'; +import { readBuildInfo } from './version'; + +const log = pino(); + +/** An operator mistake, not a bug: one honest line, no stack trace. */ +function fatal(error: unknown): never { + log.fatal( + error instanceof z.ZodError + ? z.prettifyError(error) + : error instanceof Error + ? error.message + : String(error), + ); + process.exit(1); +} + +const config = (() => { + try { + return loadConfig(); + } catch (error) { + return fatal(error); + } +})(); + +// One database file, one gateway — enforced, not assumed (the daemon's own +// lock over the gateway's file). The handle is kept for the life of the +// process on purpose: better-sqlite3 closes a handle when its object is +// garbage collected, and a closed handle drops the file lock (measured +// 2026-09-11 on the daemon with a discarded return value). +let lock: ReturnType | undefined; +if (config.DORMICE_GATEWAY_DB_PATH !== ':memory:') { + try { + lock = acquireSingleWriterLock( + config.DORMICE_GATEWAY_DB_PATH, + `another gateway is already running against ${config.DORMICE_GATEWAY_DB_PATH} — one database, one gateway. Stop the other instance, or point this one at its own DORMICE_GATEWAY_DB_PATH.`, + ); + } catch (error) { + fatal(error); + } +} + +// Migrate on every boot; a fresh install needs no separate setup step. +const db = openDb(config.DORMICE_GATEWAY_DB_PATH); +migrateDb(db, fileURLToPath(new URL('../drizzle', import.meta.url))); + +// The fleet from the nodes table (a node that is down is still a node); +// the cache and the readings fill in as nodes report and callers ask. +const fleet = new Fleet(db); +const finder = new Finder( + fleet, + new NameCache(), + httpAskNode(config.DORMICE_API_TOKEN), + log, +); + +// One queue for the whole gateway: the native acquire/destroy and the E2B +// create/kill must share per-name slots or the serialization means nothing. +const locks = new KeyedQueue(); + +const build = readBuildInfo(); +log.info( + build + ? `dormice-gateway build ${build.commit} (${build.title})` + : 'dormice-gateway build: no version identity (built outside a git checkout)', +); + +const app = buildGatewayApp({ + config, + fleet, + finder, + locks, + logger: log, + build, +}); + +// Same red line as the daemon: loopback only, host not configurable — the +// public face is a reverse proxy's job. Nothing is awaited before the door +// opens: the nodes report themselves within one interval, and until then +// a new name is refused with placement's honest 503 while every existing +// sandbox is found by asking. +await app.listen({ host: '127.0.0.1', port: config.DORMICE_GATEWAY_PORT }); +const known = fleet.all(); +log.info( + known.length === 0 + ? 'no node has checked in yet; nodes join the fleet at their first check-in' + : `fronting ${known.length} node(s) from the last run: ${known.map((n) => `${n.id} at ${n.endpoint}`).join(', ')} — each is placed on again after its first check-in`, +); + +// Bounded shutdown, the daemon's (packages/server/src/shutdown.ts has the +// measurements): close the listener, give in-flight short work the grace, +// cut what is still connected — a forwarded exec may legitimately live for +// hours — and exit explicitly. +const SHUTDOWN_GRACE_MS = 10_000; +const connections = trackConnections(app.server); +let closing = false; +const close = async (signal: NodeJS.Signals) => { + if (closing) return; + closing = true; + process.removeListener('SIGTERM', onSigterm); + process.removeListener('SIGINT', onSigint); + app.log.info( + `${signal} received — shutting down (grace ${SHUTDOWN_GRACE_MS}ms)`, + ); + try { + const cut = await closeWithGrace(app, connections, SHUTDOWN_GRACE_MS); + if (cut > 0) { + app.log.warn( + { cut }, + 'connections still open at the end of the grace period were cut', + ); + } + } catch (error) { + app.log.error(error, `graceful shutdown after ${signal} failed`); + process.exitCode = 1; + } + lock?.close(); + process.exit(process.exitCode ?? 0); +}; +const onSigterm = () => void close('SIGTERM'); +const onSigint = () => void close('SIGINT'); +process.once('SIGTERM', onSigterm); +process.once('SIGINT', onSigint); diff --git a/packages/gateway/src/placement.test.ts b/packages/gateway/src/placement.test.ts new file mode 100644 index 00000000..165cf6de --- /dev/null +++ b/packages/gateway/src/placement.test.ts @@ -0,0 +1,183 @@ +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { type Db, migrateDb, openDb } from './db/db'; +import { Fleet, type NodeState } from './fleet'; +import { type PlacementKnobs, pick, refusalMessage } from './placement'; +import { checkInOf, type reading } from './testing'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const NOW = new Date('2026-09-14T12:00:00.000Z'); +const KNOBS: PlacementKnobs = { + cpuLimitPct: 70, + activeLimit: 400, + minDiskAvailableBytes: 10 * 2 ** 30, +}; + +function fleet(): { db: Db; fleet: Fleet } { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + return { db, fleet: new Fleet(db) }; +} + +/** A node that checked in at NOW (or when told) with the given reading. */ +function node( + f: Fleet, + id: string, + over: Parameters[0] & { + intervalSeconds?: number; + placed?: number; + checkedInAt?: Date; + } = {}, +): NodeState { + const outcome = f.checkIn( + checkInOf(id, `http://${id}:80`, over), + over.checkedInAt ?? NOW, + ); + if ('refused' in outcome) throw new Error(outcome.refused); + const { node } = outcome; + node.placedSinceCheckIn = over.placed ?? 0; + return node; +} + +describe('pick', () => { + it('cpu above the limit refuses, at the limit passes, unknown passes', () => { + const f = fleet().fleet; + expect(pick([node(f, 'a', { cpu: 71 })], KNOBS, NOW).node).toBeNull(); + expect(pick([node(f, 'b', { cpu: 70 })], KNOBS, NOW).node?.id).toBe('b'); + // A freshly restarted node's first reading has no delta: not saturated. + expect(pick([node(f, 'c', { cpu: null })], KNOBS, NOW).node?.id).toBe('c'); + }); + + it('active at the limit refuses; frozen sandboxes are not counted', () => { + const f = fleet().fleet; + expect(pick([node(f, 'a', { active: 400 })], KNOBS, NOW).node).toBeNull(); + expect(pick([node(f, 'b', { active: 399 })], KNOBS, NOW).node?.id).toBe( + 'b', + ); + // Thousands of frozen sandboxes are the normal shape of a node. + expect( + pick([node(f, 'c', { active: 1, frozen: 14_000 })], KNOBS, NOW).node?.id, + ).toBe('c'); + }); + + it('counts what was placed since the reading, so a burst inside one interval cannot overfill a node', () => { + const f = fleet().fleet; + const a = node(f, 'a', { active: 390 }); + expect(pick([a], KNOBS, NOW).node?.id).toBe('a'); + a.placedSinceCheckIn = 10; + expect(pick([a], KNOBS, NOW).node).toBeNull(); + expect(pick([a], KNOBS, NOW).refused[0]?.reason).toMatch( + /390 active sandboxes \+ 10 placed since the reading reach the 400 limit/, + ); + }); + + it('a data disk below the floor refuses, naming the GiB; a missing disk reading passes', () => { + const f = fleet().fleet; + const full = node(f, 'full', { diskAvail: 9 * 2 ** 30, memAvail: 30e9 }); + const room = node(f, 'room', { diskAvail: 50 * 2 ** 30, memAvail: 20e9 }); + expect(pick([full, room], KNOBS, NOW).node?.id).toBe('room'); + expect(pick([full], KNOBS, NOW).refused).toEqual([ + { + nodeId: 'full', + reason: 'data disk has 9.0 GiB available, below the 10.0 GiB floor', + }, + ]); + expect( + pick([node(f, 'unknown', { diskAvail: null })], KNOBS, NOW).node?.id, + ).toBe('unknown'); + }); + + it('scores by active density — (active + placed) per core — then by available memory, then id', () => { + const f = fleet().fleet; + // Fewer sandboxes per core beats more free memory: memory is a + // fifteen-second-old reading, the count moves with every pick. + const a = node(f, 'a', { memAvail: 8e9, active: 5 }); + const b = node(f, 'b', { memAvail: 16e9, active: 50 }); + expect(pick([a, b], KNOBS, NOW).node?.id).toBe('a'); + // Equal density: the most available memory, then the id. + const c = node(f, 'c', { memAvail: 16e9, active: 5 }); + expect(pick([a, b, c], KNOBS, NOW).node?.id).toBe('c'); + const d = node(f, 'd', { memAvail: 16e9, active: 5 }); + expect(pick([d, c], KNOBS, NOW).node?.id).toBe('c'); + // Per core: 40 sandboxes on 64 cores is emptier than 6 on 8. + const big = node(f, 'big', { cores: 64, active: 40, memAvail: 1e9 }); + const small = node(f, 'small', { cores: 8, active: 6, memAvail: 64e9 }); + expect(pick([small, big], KNOBS, NOW).node?.id).toBe('big'); + }); + + it('a burst inside one reading spreads: each pick counts against the node it chose, so the next pick sees it fuller', () => { + const f = fleet().fleet; + const a = node(f, 'a', { memAvail: 17e9, active: 350 }); + const b = node(f, 'b', { memAvail: 16e9, active: 10 }); + const landed = { a: 0, b: 0 }; + for (let i = 0; i < 100; i++) { + const chosen = pick([a, b], KNOBS, NOW).node; + if (chosen === null) throw new Error('every node refused'); + chosen.placedSinceCheckIn += 1; + landed[chosen.id as 'a' | 'b'] += 1; + } + expect(landed).toEqual({ a: 0, b: 100 }); + // And with equal readings the two alternate. + const c = node(f, 'c', { active: 10 }); + const d = node(f, 'd', { active: 10 }); + const alternating: string[] = []; + for (let i = 0; i < 6; i++) { + const chosen = pick([c, d], KNOBS, NOW).node; + if (chosen === null) throw new Error('every node refused'); + chosen.placedSinceCheckIn += 1; + alternating.push(chosen.id); + } + expect(alternating).toEqual(['c', 'd', 'c', 'd', 'c', 'd']); + }); + + it('refuses a node that never checked in since the start, or is silent for two of its intervals', () => { + const { db, fleet: f } = fleet(); + const stale = node(f, 'stale', { + intervalSeconds: 15, + checkedInAt: new Date(NOW.getTime() - 31_000), + }); + const fresh = node(f, 'fresh', { + intervalSeconds: 15, + checkedInAt: new Date(NOW.getTime() - 29_000), + }); + // The same row after a gateway restart: known, but nothing reported yet. + const restarted = new Fleet(db).get('fresh'); + if (!restarted) throw new Error('row lost'); + const result = pick([stale, restarted], KNOBS, NOW); + expect(result.node).toBeNull(); + expect( + Object.fromEntries(result.refused.map((r) => [r.nodeId, r.reason])), + ).toEqual({ + stale: 'has not checked in for 31s', + fresh: 'has not checked in since the gateway started', + }); + expect(pick([fresh], KNOBS, NOW).node?.id).toBe('fresh'); + }); + + it('when every node refuses, each refusal names the node and its reason for the 503; no node at all says so', () => { + const f = fleet().fleet; + const result = pick( + [ + node(f, 'a', { cpu: 90, active: 100, memAvail: 1e9 }), + node(f, 'b', { cpu: 20, active: 400, memAvail: 2e9 }), + ], + KNOBS, + NOW, + ); + expect(result.node).toBeNull(); + expect(result.refused).toEqual([ + { nodeId: 'a', reason: 'cpu 90% is above the 70% limit' }, + { + nodeId: 'b', + reason: + '400 active sandboxes + 0 placed since the reading reach the 400 limit', + }, + ]); + expect(refusalMessage(result)).toBe( + 'no node can take a new sandbox right now — a: cpu 90% is above the 70% limit; b: 400 active sandboxes + 0 placed since the reading reach the 400 limit', + ); + expect(refusalMessage(pick([], KNOBS, NOW))).toMatch( + /no node has checked in/, + ); + }); +}); diff --git a/packages/gateway/src/placement.ts b/packages/gateway/src/placement.ts new file mode 100644 index 00000000..456e26ec --- /dev/null +++ b/packages/gateway/src/placement.ts @@ -0,0 +1,138 @@ +import { downReason, type NodeState } from './fleet'; + +export interface PlacementKnobs { + /** A reading above this refuses the node. */ + cpuLimitPct: number; + /** Active sandboxes plus placements since the reading at or above this refuse the node. */ + activeLimit: number; + /** A data disk with less than this available refuses the node. */ + minDiskAvailableBytes: number; +} + +/** One node's refusal, in the words the 503 repeats: `: `. */ +export interface NodeRefusal { + nodeId: string; + reason: string; +} + +export interface Placement { + /** The node to create on, or null when every node refused. */ + node: NodeState | null; + /** Every node that refused and why — the 503's body when node is null. */ + refused: NodeRefusal[]; +} + +/** + * Where a new sandbox goes: four gates, then one score. Pure — it reads + * what the nodes last reported and never asks a node. + * + * A node that is down (fleet.ts downReason: no check-in since the gateway + * started, or two of its own intervals silent) is refused: placing blind + * is how a gateway puts the twenty-first sandbox on the box that just + * fell over. A node whose whole-machine CPU is above the limit is + * refused; a null reading passes — the first check-in after a node + * restart has no delta to report, and "don't know yet" is not + * "saturated". A node at its active ceiling is refused, counting what + * this gateway placed there since the reading: frozen sandboxes are not + * counted, on purpose — they cost swap, not CPU or dockerd attention, a + * node holds thousands of them, and counting them would shut the gate + * from the first minute of a cut-over and never open it again. A node + * whose data disk is below the floor is refused (design record #36): a + * full data disk stops every sandbox on the node at once, and the node's + * own create answering 500 would otherwise be re-chosen — scored emptiest, + * because nothing it fails to build ever counts — for every new name + * until someone noticed. A missing disk reading passes, like the CPU: + * unknown is not full. + * + * Among the nodes that pass, the emptiest by active density wins: active + * sandboxes plus what this gateway placed there since the reading, per + * core. Per core because the one density ever measured is per core (535 + * sandboxes healthy and 882 dead on 128 cores, 2026-09-11) and a 64-core + * node with 40 sandboxes has more room than an 8-core node with 6; + * counting the in-flight placements is what spreads a burst — a reading + * is fifteen seconds old, a hundred creates arrive inside that window, + * and a score that does not move with each pick sends all hundred to the + * node the reading favoured (reproduced twice, 2026-09-12). Ties go to the + * most available memory — the resource the freeze/swap design overcommits + * and an overloaded host runs out of first — then to the id, so the + * choice is stable and explainable. Memory does not gate placement: a + * node with room per core and little memory left is still chosen, and + * memory pressure is judged where it is felt, by the node's own admission + * (design record #27, after the cluster), not guessed from a + * fifteen-second-old figure the picks do not move. + */ +export function pick( + nodes: readonly NodeState[], + knobs: PlacementKnobs, + now: Date, +): Placement { + const refused: NodeRefusal[] = []; + const candidates: Array<{ + node: NodeState; + /** (active + placed since the reading) per core. */ + load: number; + memAvailableBytes: number; + }> = []; + for (const node of nodes) { + const refuse = (reason: string) => + refused.push({ nodeId: node.id, reason }); + const down = downReason(node, now); + if (down !== null) { + refuse(down); + continue; + } + const reading = node.reading; + if (reading === null) { + refuse('has not reported a reading'); + continue; + } + const cpuUsedPct = reading.host.cpuUsedPct; + if (cpuUsedPct !== null && cpuUsedPct > knobs.cpuLimitPct) { + refuse( + `cpu ${Math.round(cpuUsedPct)}% is above the ${knobs.cpuLimitPct}% limit`, + ); + continue; + } + const active = reading.sandboxes.byState.active; + const placed = node.placedSinceCheckIn; + if (active + placed >= knobs.activeLimit) { + refuse( + `${active} active sandboxes + ${placed} placed since the reading reach the ${knobs.activeLimit} limit`, + ); + continue; + } + const disk = reading.dataDisk; + if (disk !== null && disk.availableBytes < knobs.minDiskAvailableBytes) { + refuse( + `data disk has ${gib(disk.availableBytes)} GiB available, below the ${gib(knobs.minDiskAvailableBytes)} GiB floor`, + ); + continue; + } + candidates.push({ + node, + load: (active + placed) / Math.max(1, reading.host.cpuCount), + memAvailableBytes: reading.host.memAvailableBytes, + }); + } + candidates.sort( + (a, b) => + a.load - b.load || + b.memAvailableBytes - a.memAvailableBytes || + (a.node.id < b.node.id ? -1 : a.node.id > b.node.id ? 1 : 0), + ); + return { node: candidates[0]?.node ?? null, refused }; +} + +function gib(bytes: number): string { + return (bytes / 2 ** 30).toFixed(1); +} + +/** The 503's sentence when every node refused: each node and its reason — or that there is no node at all. */ +export function refusalMessage(placement: Placement): string { + if (placement.refused.length === 0) { + return 'no node can take a new sandbox: no node has checked in with this gateway yet'; + } + return `no node can take a new sandbox right now — ${placement.refused + .map((r) => `${r.nodeId}: ${r.reason}`) + .join('; ')}`; +} diff --git a/packages/gateway/src/raw.ts b/packages/gateway/src/raw.ts new file mode 100644 index 00000000..a5611263 --- /dev/null +++ b/packages/gateway/src/raw.ts @@ -0,0 +1,192 @@ +import type http from 'node:http'; +import type { Duplex } from 'node:stream'; +import type { Logger } from 'pino'; +import type { Classified } from './classify'; +import { relay, renderError, sendPreflight } from './errors'; +import type { Finder, Found } from './find'; +import { forwardStream } from './forward'; + +export interface RawFacesDeps { + finder: Finder; + token: string; + log: Logger; +} + +/** How long a caller refused with "a node did not answer" may wait before asking again — one check-in interval. */ +export const RETRY_AFTER_SECONDS = 15; + +/** + * The faces Fastify never sees — keyed on a header on any path, judged on + * the raw request the serverFactory hands over: + * envd E2B's in-sandbox API, keyed by E2b-Sandbox-Id; forwarded + * with no credential change (the access token is the node's + * own HMAC). Preflights are answered here: the node answers + * them without auth, and a preflight that came back 401 + * without CORS would fail every browser-direct upload. + * signedRoot the bare signed-URL form: no sandbox id anywhere the + * gateway can read without the node's signing secret, so an + * honest 501. + * + * Nobody has authenticated to the gateway on these faces — the node + * judges the credential, after the gateway has picked it — so what the + * gateway says here describes the sandbox's state and never the fleet: no + * node id, no endpoint. A sandbox id is handed to browsers in every + * getHost URL, and the daemon's own proxy says only "not found" for one + * it lacks. The detail an operator needs goes to the log; the + * authenticated faces and listNodes name nodes freely. + */ +export function createRawFaces({ finder, token, log }: RawFacesDeps) { + /** The one sentence per finding for a sandbox id on these faces — generic on purpose (above). */ + function refusal( + res: http.ServerResponse, + id: string, + found: Exclude, + ): void { + switch (found.kind) { + case 'conflict': + renderError(res, 'connect', { + status: 502, + connectCode: 'unavailable', + message: `sandbox "${id}" is held by more than one node — routing resumes once an operator destroys one copy (listNodes and the gateway log name them)`, + cors: true, + }); + return; + case 'none': + renderError(res, 'connect', { + status: 502, + connectCode: 'unavailable', + message: `sandbox "${id}" is on no node — it may have been destroyed`, + cors: true, + }); + return; + case 'unsure': + renderError(res, 'connect', { + status: 503, + connectCode: 'unavailable', + message: `sandbox "${id}": a node did not answer, so its whereabouts cannot be settled — retry`, + cors: true, + retryAfterSeconds: RETRY_AFTER_SECONDS, + }); + return; + } + } + + /** + * Finds the id and forwards, or answers the refusal — the one path + * every keyed face takes. Nothing here may throw past this point: no + * framework stands behind a raw face, so an escaped rejection would be + * the process's, not the request's (errors.ts relay answers instead). + */ + async function route( + req: http.IncomingMessage, + res: http.ServerResponse, + id: string, + ): Promise { + let found: Found; + try { + found = await finder.byId(id); + } catch (error) { + log.error(error, 'envd face: the lookup itself failed'); + renderError(res, 'connect', { + status: 500, + connectCode: 'internal', + message: 'the gateway failed while locating the sandbox — see its log', + cors: true, + }); + return; + } + if (found.kind !== 'one') { + refusal(res, id, found); + return; + } + const node = found.node; + await relay( + res, + 'connect', + log, + async () => { + await forwardStream(req, res, { + target: { endpoint: node.endpoint, token }, + credential: 'none', + }); + }, + (error) => { + log.warn( + { + sandboxId: id, + nodeId: node.id, + endpoint: node.endpoint, + why: error.why, + }, + "envd face: the sandbox's node did not answer", + ); + return { + status: 502, + connectCode: 'unavailable', + message: `sandbox "${id}": its node did not answer (${error.why}) — retry`, + cors: true, + }; + }, + ); + } + + return { + handleRequest( + kind: Exclude, + req: http.IncomingMessage, + res: http.ServerResponse, + ): void { + switch (kind.face) { + case 'envd': { + if (req.method === 'OPTIONS') { + sendPreflight(req, res); + return; + } + const header = req.headers['e2b-sandbox-id']; + const id = Array.isArray(header) ? header[0] : header; + if (!id) { + renderError(res, 'connect', { + status: 401, + connectCode: 'unauthenticated', + message: 'missing E2b-Sandbox-Id header', + cors: true, + }); + return; + } + void route(req, res, id); + return; + } + case 'signedRoot': { + if (req.method === 'OPTIONS') { + sendPreflight(req, res); + return; + } + renderError(res, 'connect', { + status: 501, + connectCode: 'unimplemented', + message: + 'the bare signed-URL form is not routed by the gateway yet — use the sandbox host form, or the node directly', + cors: true, + }); + return; + } + } + }, + + handleUpgrade( + _kind: Classified, + _req: http.IncomingMessage, + socket: Duplex, + ): void { + // First, before anything else: http.Server drops its own error + // listener from the socket before emitting 'upgrade', so a client + // that resets here would raise an uncaught ECONNRESET and take the + // whole gateway down — from an unauthenticated face (the daemon's + // sandbox-proxy.ts handleUpgrade has the same first line). + socket.on('error', () => socket.destroy()); + // No face of this build takes upgrades: sandbox WebSockets ride the + // port proxy, which joins the gateway with the sandbox domain. + socket.destroy(); + }, + }; +} diff --git a/packages/gateway/src/routes/create.ts b/packages/gateway/src/routes/create.ts new file mode 100644 index 00000000..9dd3ca31 --- /dev/null +++ b/packages/gateway/src/routes/create.ts @@ -0,0 +1,160 @@ +import type http from 'node:http'; +import type { FastifyReply, FastifyRequest } from 'fastify'; +import { z } from 'zod'; +import type { NameCache } from '../cache'; +import type { Fleet, NodeState } from '../fleet'; +import { + type CapturedResponse, + type Credential, + forwardCapture, +} from '../forward'; +import { type Placement, type PlacementKnobs, pick } from '../placement'; + +/** + * What a create looks like on each face the gateway creates through: the + * credential it forwards under, and where the node's answer keeps the id + * it minted — all the gateway ever reads of that answer. + */ +export interface CreateFace { + credential: Credential; + idOf(answer: unknown): string | null; +} + +const acquireAnswerSchema = z.object({ sandbox: z.object({ id: z.string() }) }); +const e2bAnswerSchema = z.object({ sandboxID: z.string() }); + +export const NATIVE_CREATE: CreateFace = { + credential: 'bearer', + idOf: (answer) => { + const parsed = acquireAnswerSchema.safeParse(answer); + return parsed.success ? parsed.data.sandbox.id : null; + }, +}; + +export const E2B_CREATE: CreateFace = { + credential: 'x-api-key', + idOf: (answer) => { + const parsed = e2bAnswerSchema.safeParse(answer); + return parsed.success ? parsed.data.sandboxID : null; + }, +}; + +/** + * Where a new sandbox goes: placement's pick, counted against the node at + * once — the next acquire in this same interval must see this one as + * already there. Nothing is recorded anywhere else: if the answer is + * lost, the caller's retry asks the fleet (find.ts) and the node that + * built it answers "yes" from inside the name's slot. + */ +export function place( + fleet: Fleet, + knobs: PlacementKnobs, + now: Date, +): Placement { + const placement = pick(fleet.all(), knobs, now); + if (placement.node !== null) placement.node.placedSinceCheckIn += 1; + return placement; +} + +/** + * A create whose client has already left: nothing is asked, placed or + * counted for it — the response is gone, forwardCapture would send + * nothing. Checked first thing inside the slot (a creator that waited + * behind a slow create or destroy) and again after the lookup round (up + * to two seconds). The reply is hijacked so Fastify writes nothing to the + * dead socket either. + */ +export function clientGone(reply: FastifyReply): boolean { + if (!reply.raw.destroyed) return false; + reply.hijack(); + return true; +} + +export interface CreateOptions { + target: NodeState; + token: string; + name: string | null; + body: Buffer | undefined; + face: CreateFace; + /** + * True when place() chose the target for a name no node held — the + * placement was counted against it in placedSinceCheckIn. False for a + * wake: the name was found on the target, its sandbox is in the reading + * already, and nothing was counted that could be uncounted. + */ + placed: boolean; +} + +/** + * Forwards a create and learns from the node's answer: a 2xx with a + * readable id goes into the cache (name and id → this node), so the next + * request for the sandbox skips the round of questions. A 2xx without a + * readable id is logged — the sandbox exists on the node and the next + * lookup finds it there, never silently. + * + * The placement count moves only for a placement (`placed`): a 2xx id + * also goes into the node's placedIds, so a destroy inside the same + * interval takes the placement off the count again; a 4xx or the node's + * own 500 means the node built nothing, and the placement comes off the + * count at once (it would have held a slot for a whole interval + * otherwise), so the next attempt may be placed elsewhere. A wake — a + * name found on the node — moves nothing either way: its sandbox is in + * the reading already, and paying back a placement that never was would + * open the gate one sandbox wider than the reading allows (found by + * review, 2026-09-14). A 502/503/504 from a hop in front of the node, or + * no answer at all, leaves the count as it is and the question to the + * next lookup — the node that may have built it answers from inside the + * name's slot. + */ +const HOP_STATUSES = new Set([502, 503, 504]); +export async function forwardCreate( + cache: NameCache, + request: FastifyRequest, + res: http.ServerResponse, + { target, token, name, body, face, placed }: CreateOptions, +): Promise { + const answer = await forwardCapture(request.raw, res, { + target: { endpoint: target.endpoint, token }, + credential: face.credential, + body, + }); + if (answer === null) return null; + if (answer.status >= 200 && answer.status < 300) { + const id = face.idOf(parseJson(answer.body)); + if (id !== null) { + cache.put({ id, name, nodeId: target.id }); + if (placed) target.placedIds.add(id); + } else { + request.log.warn( + { node: target.id, name, status: answer.status }, + 'create answered 2xx without a readable sandbox id; the next lookup finds it on the node', + ); + } + } else if (!HOP_STATUSES.has(answer.status)) { + // The node itself answered no: nothing was built there. + if (placed) { + target.placedSinceCheckIn = Math.max(0, target.placedSinceCheckIn - 1); + } + if (answer.status >= 500) { + request.log.warn( + { node: target.id, name, status: answer.status }, + 'the node itself failed the create; nothing cached, the placement is uncounted', + ); + } + } else { + request.log.warn( + { node: target.id, name, status: answer.status }, + 'a hop in front of the node answered the create; nothing cached, the placement stays counted until the next reading', + ); + } + return answer; +} + +export function parseJson(body: Buffer | undefined): unknown { + if (body === undefined || body.length === 0) return undefined; + try { + return JSON.parse(body.toString('utf8')); + } catch { + return undefined; + } +} diff --git a/packages/gateway/src/routes/destroy.ts b/packages/gateway/src/routes/destroy.ts new file mode 100644 index 00000000..e501a489 --- /dev/null +++ b/packages/gateway/src/routes/destroy.ts @@ -0,0 +1,51 @@ +import type http from 'node:http'; +import type { FastifyRequest } from 'fastify'; +import type { CacheEntry, NameCache } from '../cache'; +import type { NodeState } from '../fleet'; +import { + type CapturedResponse, + type Credential, + forwardCapture, +} from '../forward'; + +export interface DestroyOptions { + target: NodeState; + token: string; + /** The cache entry the verb addresses — forgotten on the node's yes. */ + entry: CacheEntry; + body: Buffer | undefined; + credential: Credential; +} + +/** + * The one destroy path, the mirror of create.ts: the native destroySandbox + * and the E2B kill both forward, wait for the node's whole answer and + * learn from a 2xx — the cache entry goes, and a sandbox placed since the + * node's last reading comes off its placement count (fleet.ts placedIds). + * Only the node's yes teaches + * anything: its 404 is not "gone" (the daemon answers 404 for a sandbox + * past its kill deadline that its scanner has not torn down yet, a row a + * lookup would still answer yes for), and a 5xx or no answer leaves the + * cache as it was. The cache is a cache: an entry kept one destroy too + * long costs the next request one 404 and a re-check (find.ts verify). + */ +export async function forwardDestroy( + cache: NameCache, + request: FastifyRequest, + res: http.ServerResponse, + { target, token, entry, body, credential }: DestroyOptions, +): Promise { + const answer = await forwardCapture(request.raw, res, { + target: { endpoint: target.endpoint, token }, + credential, + body, + }); + if (answer === null) return null; + if (answer.status >= 200 && answer.status < 300) { + cache.evict(entry); + if (target.placedIds.delete(entry.id)) { + target.placedSinceCheckIn = Math.max(0, target.placedSinceCheckIn - 1); + } + } + return answer; +} diff --git a/packages/gateway/src/routes/e2b.ts b/packages/gateway/src/routes/e2b.ts new file mode 100644 index 00000000..4b5ad57e --- /dev/null +++ b/packages/gateway/src/routes/e2b.ts @@ -0,0 +1,241 @@ +import { tokensEqual } from '@dormice/server/auth'; +import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import { sandboxNameSchema } from '@dormice/shared'; +import type { FastifyError, FastifyReply, FastifyRequest } from 'fastify'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { relay } from '../errors'; +import type { Finder } from '../find'; +import type { Fleet, NodeState } from '../fleet'; +import { forwardStream, replay } from '../forward'; +import { type PlacementKnobs, refusalMessage } from '../placement'; +import { RETRY_AFTER_SECONDS } from '../raw'; +import { + clientGone, + E2B_CREATE, + forwardCreate, + parseJson, + place, +} from './create'; +import { forwardDestroy } from './destroy'; +import { refuse, verdict } from './verdict'; + +export interface E2bRoutesOptions { + fleet: Fleet; + finder: Finder; + locks: KeyedQueue; + knobs: PlacementKnobs; + token: string; +} + +/** + * The E2B control plane in front of several nodes: what the official SDK + * calls api.e2b.app for, mounted at /e2b/api like the daemon's. Creates + * are placed (by metadata.name when the SDK gave one — the same name slot + * the native acquire takes — else unnamed, known by id alone until a + * lookup names it); everything addressed by id is found and forwarded. + * The dialect is E2B's: { code: , message }. + */ +export const e2bControlRoutes: FastifyPluginAsyncZod = async ( + app, + { fleet, finder, locks, knobs, token }, +) => { + app.addContentTypeParser( + 'application/json', + { parseAs: 'buffer' }, + (_request, body, done) => done(null, body), + ); + app.setErrorHandler((error: FastifyError, request, reply) => { + const status = error.statusCode ?? 500; + if (status >= 500) request.log.error(error, 'request failed'); + reply.code(status).send({ code: status, message: error.message }); + }); + app.setNotFoundHandler((request, reply) => { + reply.code(404).send({ + code: 404, + message: `route ${request.method} ${request.url} not found`, + }); + }); + + // The daemon's own X-API-KEY convention (`e2b_`), over the + // fleet's one token. + app.addHook('onRequest', async (request, reply) => { + const presented = request.headers['x-api-key']; + const key = Array.isArray(presented) ? presented[0] : presented; + const bare = key?.startsWith('e2b_') ? key.slice(4) : key; + if (bare === undefined || !tokensEqual(bare, token)) { + await reply.code(401).send({ code: 401, message: 'invalid API key' }); + } + }); + + const send = (reply: FastifyReply, code: number, message: string) => + reply.code(code).send({ code, message }); + + /** From the hijack on the raw response is the gateway's to finish, in E2B's dialect (errors.ts relay). */ + const forwarded = ( + request: FastifyRequest, + reply: FastifyReply, + tail: string, + step: () => Promise, + ) => { + reply.hijack(); + return relay(reply.raw, 'control', request.log, step, (error) => ({ + status: 502, + message: `${error.message}${tail}`, + })); + }; + + const RETRY_FINDS_IT = + ' — retry: if the sandbox was built, the node answers for it'; + + /** `placed`: pick() chose the node for a new name (counted there) — false for a name found on it, a wake. */ + function create( + request: FastifyRequest, + reply: FastifyReply, + target: NodeState, + name: string | null, + body: Buffer | undefined, + placed: boolean, + ) { + return forwarded(request, reply, RETRY_FINDS_IT, async () => { + const answer = await forwardCreate(finder.cache, request, reply.raw, { + target, + token, + name, + body, + face: E2B_CREATE, + placed, + }); + if (answer !== null) replay(reply.raw, answer); + }); + } + + /** Places a new sandbox, or sends the 503 and answers null (null too for a client that already left). */ + function placeOrRefuse(reply: FastifyReply): NodeState | null { + if (clientGone(reply)) return null; + const placement = place(fleet, knobs, new Date()); + if (placement.node === null) { + reply.header('retry-after', String(RETRY_AFTER_SECONDS)); + send(reply, 503, refusalMessage(placement)); + return null; + } + return placement.node; + } + + app.post('/sandboxes', async (request, reply) => { + const body = request.body as Buffer | undefined; + const parsed = parseJson(body) as + | { metadata?: { name?: unknown } } + | undefined; + if (parsed?.metadata?.name !== undefined) { + // Judged by the wire's own rule before any node is asked (native.ts + // nameOf has why); the node's stricter E2B pattern still answers its + // own 400, relayed as it came. + const judged = sandboxNameSchema.safeParse(parsed.metadata.name); + if (!judged.success) { + return send( + reply, + 400, + `invalid metadata.name: ${judged.error.issues[0]?.message ?? 'refused by the wire'}`, + ); + } + const name = judged.data; + return locks.run(name, async () => { + // A client that left while waiting for the slot asks nobody + // (native.ts acquire has why). + if (clientGone(reply)) return reply; + // Confirmed with the cached node first (find.ts byName has why): + // the daemon's create builds what it does not find. + const found = verdict( + await finder.byName(name, { confirm: true }), + `sandbox "${name}"`, + ); + if (found.kind === 'refuse') { + return refuse(reply, found, (message) => ({ + code: found.status, + message, + })); + } + if (found.kind === 'node') { + return create(request, reply, found.node, name, body, false); + } + const target = placeOrRefuse(reply); + if (target === null) return reply; + return create(request, reply, target, name, body, true); + }); + } + const target = placeOrRefuse(reply); + if (target === null) return reply; + return create(request, reply, target, null, body, true); + }); + + app.get('/v2/sandboxes', async (_request, reply) => + send( + reply, + 501, + 'listing sandboxes is not routed by the gateway yet — call the node directly', + ), + ); + + const byId = async (request: FastifyRequest, reply: FastifyReply) => { + const { id } = request.params as { id: string }; + const judged = verdict(await finder.byId(id), `sandbox "${id}"`); + if (judged.kind === 'refuse') { + return refuse(reply, judged, (message) => ({ + code: judged.status, + message, + })); + } + if (judged.kind === 'none') { + return send(reply, 404, `sandbox "${id}" not found`); + } + const body = request.body as Buffer | undefined; + const { node, name } = judged; + // Only the kill itself — DELETE /sandboxes/, nothing deeper — is + // captured and learned from (routes/destroy.ts has what is learned). + // A DELETE on a sub-route the node lacks answers the same 404 shape as + // a kill of an unknown id (measured 2026-09-12 with + // DELETE /sandboxes//bogus). The kill takes the name's slot like + // the native destroy (the daemon's own kill does too). + if (request.method === 'DELETE' && isKill(request)) { + return locks.run(name ?? id, () => + forwarded(request, reply, ' — retry', async () => { + const answer = await forwardDestroy( + finder.cache, + request, + reply.raw, + { + target: node, + token, + entry: { id, name, nodeId: node.id }, + body, + credential: 'x-api-key', + }, + ); + if (answer !== null) replay(reply.raw, answer); + }), + ); + } + return forwarded(request, reply, ' — retry', async () => { + const status = await forwardStream(request.raw, reply.raw, { + target: { endpoint: node.endpoint, token }, + credential: 'x-api-key', + body, + }); + if (status === 404) void finder.verify({ id, name, nodeId: node.id }); + }); + }; + // Every method, not the ones this build of the daemon happens to serve: + // the node answers for its own routes (e2b 2.31 already PUTs + // /sandboxes/:id/network, which the daemon 404s — its 404, not the + // gateway's, is the honest one). The path reaches the node exactly as + // written (forward.ts dispatch), so a sub-route is a sub-route there + // too, never a create resolved out of dot segments. + const methods = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE'] as const; + app.route({ method: [...methods], url: '/sandboxes/:id', handler: byId }); + app.route({ method: [...methods], url: '/sandboxes/:id/*', handler: byId }); +}; + +/** The kill route exactly: `/sandboxes/` with nothing after the id. */ +function isKill(request: FastifyRequest): boolean { + return (request.params as { '*'?: string })['*'] === undefined; +} diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts new file mode 100644 index 00000000..161d2f85 --- /dev/null +++ b/packages/gateway/src/routes/native.ts @@ -0,0 +1,295 @@ +import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import { + sandboxNameSchema, + WRITE_FILES_BODY_LIMIT_BYTES, +} from '@dormice/shared'; +import type { FastifyReply, FastifyRequest } from 'fastify'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { relay } from '../errors'; +import type { Finder } from '../find'; +import type { Fleet } from '../fleet'; +import { forwardStream, replay } from '../forward'; +import { type PlacementKnobs, refusalMessage } from '../placement'; +import { RETRY_AFTER_SECONDS } from '../raw'; +import { + clientGone, + forwardCreate, + NATIVE_CREATE, + parseJson, + place, +} from './create'; +import { forwardDestroy } from './destroy'; +import { refuse, verdict } from './verdict'; + +/** + * The native verbs the gateway routes: every verb addressed to one sandbox + * by `name`. Registered one by one — never as `/:verb` — so a misspelled + * verb is a 404 like on the daemon, and nothing else under the root is + * swallowed. + */ +export const NAMED_VERBS = [ + 'acquireSandbox', + 'execCommand', + 'writeFile', + 'writeFiles', + 'readFile', + 'readFiles', + 'rebuildSandbox', + 'updatePolicy', + 'updateSpec', + 'updateTemplate', + 'expandDisk', + 'updateMetadata', + 'destroySandbox', + 'getSandboxMetrics', + 'getSandboxMetricsHistory', +] as const; + +/** + * The verbs that address the daemon, not a sandbox: fleet lists, host + * readings, templates, settings, ingress, upgrade, keys. Asking every + * node and merging, or answering from the gateway's own tables, comes + * with the configuration authority; until then each answers an honest + * 501 naming the alternative, instead of a misleading answer from + * whichever node the gateway happened to pick. + */ +export const UNNAMED_VERBS = [ + 'listSandboxes', + 'listSandboxMetrics', + 'listSandboxImages', + 'listActivity', + 'getFleetTimeline', + 'getHostMetrics', + 'getHostMetricsHistory', + 'getConfig', + 'checkUpgrade', + 'applyUpgrade', + 'getUpgradeStatus', + 'getIngress', + 'setIngress', + 'registerTemplate', + 'listTemplates', + 'removeTemplate', + 'updateSettings', + 'createApiKey', + 'listApiKeys', + 'updateApiKey', + 'revokeApiKey', +] as const; + +export interface NativeRoutesOptions { + fleet: Fleet; + finder: Finder; + locks: KeyedQueue; + knobs: PlacementKnobs; + token: string; +} + +export const nativeRoutes: FastifyPluginAsyncZod = async ( + app, + { fleet, finder, locks, knobs, token }, +) => { + // Bodies stay bytes: the gateway reads one field (`name`) and forwards + // the bytes it received, never a re-serialization. Scoped to this + // plugin — the gateway's own verbs keep Fastify's JSON parsing. + app.addContentTypeParser( + 'application/json', + { parseAs: 'buffer' }, + (_request, body, done) => done(null, body), + ); + + for (const verb of UNNAMED_VERBS) { + app.post(`/${verb}`, async (_request, reply) => + reply.code(501).send({ + message: `${verb} is not routed by the gateway yet — call the node directly (this version routes only sandbox-addressed verbs)`, + }), + ); + } + + for (const verb of NAMED_VERBS) { + const bodyLimit = + verb === 'writeFile' || verb === 'writeFiles' + ? WRITE_FILES_BODY_LIMIT_BYTES + : undefined; + app.post( + `/${verb}`, + bodyLimit === undefined ? {} : { bodyLimit }, + async (request, reply) => { + const body = request.body as Buffer | undefined; + const named = nameOf(body); + if ('refusal' in named) { + return reply.code(400).send({ message: named.refusal }); + } + const { name } = named; + // The gateway takes the name's slot for the two verbs whose + // outcome it acts on — a create it may have to place, a destroy it + // must forget — so twenty simultaneous acquires of a new name + // place once: the first finds nothing and places, the rest find + // its cache entry. Every other verb is relayed unserialized: the + // daemon holds its own slot per name for the ones that touch the + // sandbox (routes/sandboxes.ts locks.run), and a relay in front of + // it has nothing to add to their order. + if (verb === 'acquireSandbox') { + return locks.run(name, () => acquire(request, reply, name, body)); + } + if (verb === 'destroySandbox') { + return locks.run(name, () => destroy(request, reply, name, body)); + } + return forwardNamed(request, reply, name, body); + }, + ); + } + + /** + * From the hijack on, the raw response is the gateway's to finish: the + * node's answer is replayed, or the gateway's own 502/500 rendered, in + * the native dialect (errors.ts relay). `tail` is what the caller may + * do after a node that did not answer. + */ + const forwarded = ( + request: FastifyRequest, + reply: FastifyReply, + tail: string, + step: () => Promise, + ) => { + reply.hijack(); + return relay(reply.raw, 'native', request.log, step, (error) => ({ + status: 502, + message: `${error.message}${tail}`, + })); + }; + + /** A create that did not answer: the sandbox may exist; the retry asks the fleet and lands on the node that built it. */ + const RETRY_FINDS_IT = + ' — retry: if the sandbox was built, the node answers for it'; + + async function acquire( + request: FastifyRequest, + reply: FastifyReply, + name: string, + body: Buffer | undefined, + ) { + // First, before any node is asked: a creator that waited for the slot + // behind a slow create or destroy and whose client left meanwhile + // asks nobody — the confirmation below is a question to a node inside + // the slot, and twenty abandoned acquires of one name would otherwise + // hold the slot for twenty answers nobody reads (create.ts clientGone). + if (clientGone(reply)) return; + // A creator confirms a cache hit with its node first (find.ts byName + // has why): the daemon's acquire builds what it does not find. + const judged = verdict( + await finder.byName(name, { confirm: true }), + `sandbox "${name}"`, + ); + if (judged.kind === 'refuse') return refuse(reply, judged); + let target = judged.kind === 'node' ? judged.node : null; + // Found nowhere: a placement, counted against the node it lands on. + // Found somewhere — and confirmed there: a wake, already in that + // node's reading. + const placed = target === null; + if (target === null) { + // Asked again: the round of questions took up to two seconds. + if (clientGone(reply)) return; + const placement = place(fleet, knobs, new Date()); + if (placement.node === null) { + reply.header('retry-after', String(RETRY_AFTER_SECONDS)); + return reply.code(503).send({ message: refusalMessage(placement) }); + } + target = placement.node; + } + const chosen = target; + return forwarded(request, reply, RETRY_FINDS_IT, async () => { + const answer = await forwardCreate(finder.cache, request, reply.raw, { + target: chosen, + token, + name, + body, + face: NATIVE_CREATE, + placed, + }); + if (answer !== null) replay(reply.raw, answer); + }); + } + + async function destroy( + request: FastifyRequest, + reply: FastifyReply, + name: string, + body: Buffer | undefined, + ) { + const judged = verdict(await finder.byName(name), `sandbox "${name}"`); + if (judged.kind === 'refuse') return refuse(reply, judged); + // Nothing to forward: every node answered and none holds the name, + // which is exactly the daemon's own idempotent answer. + if (judged.kind === 'none') + return reply.code(200).send({ destroyed: false }); + const { node, id } = judged; + return forwarded(request, reply, ' — retry', async () => { + const answer = await forwardDestroy(finder.cache, request, reply.raw, { + target: node, + token, + entry: { id, name, nodeId: node.id }, + body, + credential: 'bearer', + }); + if (answer !== null) replay(reply.raw, answer); + }); + } + + /** + * Every other named verb: find the node, stream the answer through as + * it came — status, headers, body. A node's 404 is not "no such + * sandbox" by itself (readFile answers 404 for a missing path too), so + * it is relayed as it came and the cache entry is re-checked off the + * request path (find.ts verify): only a node that says "absent" to the + * one question that means it loses the entry. + */ + async function forwardNamed( + request: FastifyRequest, + reply: FastifyReply, + name: string, + body: Buffer | undefined, + ) { + const judged = verdict(await finder.byName(name), `sandbox "${name}"`); + if (judged.kind === 'refuse') return refuse(reply, judged); + if (judged.kind === 'none') { + return reply + .code(404) + .send({ message: `no sandbox named "${name}" — acquire it first` }); + } + const { node, id } = judged; + return forwarded(request, reply, ' — retry', async () => { + const status = await forwardStream(request.raw, reply.raw, { + target: { endpoint: node.endpoint, token }, + credential: 'bearer', + body, + }); + if (status === 404) { + void finder.verify({ id, name, nodeId: node.id }); + } + }); + } +}; + +/** + * The one field the gateway reads from a native body, judged by the wire's + * own rule (shared sandboxNameSchema — what lookupSandbox validates + * against) so no node is ever asked a question it would refuse: a node's + * 400 to the lookup reads as silence, and the caller would get a 503 with + * Retry-After for a name that can never be valid (found by review, + * 2026-09-14). + */ +export function nameOf( + body: Buffer | undefined, +): { name: string } | { refusal: string } { + const parsed = parseJson(body) as { name?: unknown } | undefined; + if (parsed?.name === undefined) { + return { refusal: 'name is required and must be a string' }; + } + const judged = sandboxNameSchema.safeParse(parsed.name); + return judged.success + ? { name: judged.data } + : { + refusal: `invalid name: ${judged.error.issues[0]?.message ?? 'refused by the wire'}`, + }; +} diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts new file mode 100644 index 00000000..bf6987dc --- /dev/null +++ b/packages/gateway/src/routes/nodes.ts @@ -0,0 +1,155 @@ +import { + checkInRequestSchema, + checkInResponseSchema, + listNodesRequestSchema, + listNodesResponseSchema, + removeNodeRequestSchema, + removeNodeResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import type { NameCache } from '../cache'; +import { downReason, type Fleet } from '../fleet'; + +export interface NodeRoutesOptions { + fleet: Fleet; + cache: NameCache; +} + +/** A refusal in the native dialect, rendered by the app's error handler as `{ message }` under its status. */ +function refusal(statusCode: number, message: string): Error { + return Object.assign(new Error(message), { statusCode }); +} + +/** + * The gateway's own verbs about its nodes: the check-in the nodes send + * (RULES/协议.md「网关」), and what an operator reads and does about them. + * Everything listNodes answers is what the gateway already holds, so + * answering costs no node anything. + */ +export const nodeRoutes: FastifyPluginAsyncZod = async ( + app, + { fleet, cache }, +) => { + app.post( + '/checkIn', + { + schema: { + body: checkInRequestSchema, + response: { 200: checkInResponseSchema }, + }, + }, + async (request) => { + const outcome = fleet.checkIn(request.body); + if ('refused' in outcome) { + request.log.warn( + { nodeId: request.body.nodeId, endpoint: request.body.endpoint }, + outcome.refused, + ); + throw refusal(409, outcome.refused); + } + const { node, joined, movedFrom } = outcome; + if (joined) { + request.log.info( + { nodeId: node.id, endpoint: node.endpoint }, + 'a node checked in for the first time and joined the fleet', + ); + } + // Two misconfigurations show only here, so they are said here. A + // node whose endpoint moves at every check-in is two machines + // sharing one DORMICE_NODE_ID (the daemon's default is `node-1`). + // Two nodes reporting one endpoint is a DORMICE_NODE_ENDPOINT that + // names the wrong machine — the daemon refuses the loopback default + // when its gateway is remote, but a hand-written value still can. + // Either way the symptom downstream is a 409 on every name (both + // "nodes" answer the lookup) or sandboxes on the wrong machine. + if (movedFrom !== null) { + request.log.warn( + { nodeId: node.id, from: movedFrom, to: node.endpoint }, + 'a node checked in from a new endpoint; the gateway forwards there from now on', + ); + } + const twins = fleet + .all() + .filter((n) => n.id !== node.id && n.endpoint === node.endpoint); + if (twins.length > 0) { + request.log.warn( + { + nodeId: node.id, + endpoint: node.endpoint, + alsoReportedBy: twins.map((n) => n.id), + }, + 'two nodes report the same endpoint — check DORMICE_NODE_ID and DORMICE_NODE_ENDPOINT on both; their sandboxes will be found twice (409) or land on the wrong machine', + ); + } + return {}; + }, + ); + + app.post( + '/listNodes', + { + schema: { + body: listNodesRequestSchema, + response: { 200: listNodesResponseSchema }, + }, + }, + async () => { + const now = new Date(); + return { + nodes: fleet.all().map((node) => ({ + id: node.id, + endpoint: node.endpoint, + addedAt: node.addedAt, + lastCheckInAt: node.lastCheckInAt?.toISOString() ?? null, + intervalSeconds: node.intervalSeconds, + reachable: downReason(node, now) === null, + build: node.build, + reading: node.reading, + placedSinceCheckIn: node.placedSinceCheckIn, + })), + }; + }, + ); + + app.post( + '/removeNode', + { + schema: { + body: removeNodeRequestSchema, + response: { 200: removeNodeResponseSchema }, + }, + }, + async (request) => { + // "Gone for good" is refused for a node that is still checking in: + // its row would go, its names would be new names, and any of them + // acquired in the seconds before its next check-in would be built + // elsewhere — then the node re-adds itself and every such name is on + // two nodes, a 409 an operator clears by hand. Stop the daemon + // first; two of its intervals of silence is what "down" means + // (fleet.ts downReason), and a down node is removable (found by + // review, 2026-09-14). + const node = fleet.get(request.body.id); + if (node !== undefined) { + const now = new Date(); + if (downReason(node, now) === null && node.lastCheckInAt !== null) { + const ago = Math.round( + (now.getTime() - node.lastCheckInAt.getTime()) / 1000, + ); + throw refusal( + 409, + `node ${node.id} checked in ${ago}s ago — it is running, and its names would be placed elsewhere before it checked in again and come back as a 409 on two nodes; stop its daemon, wait two of its intervals (${node.intervalSeconds}s each), then remove it`, + ); + } + } + const removed = fleet.remove(request.body.id); + const evicted = cache.evictNode(request.body.id); + if (removed) { + request.log.warn( + { nodeId: request.body.id, evicted }, + 'a node was removed by an operator: its sandboxes are no longer looked for, and names that lived only there are new names again', + ); + } + return { removed }; + }, + ); +}; diff --git a/packages/gateway/src/routes/verdict.ts b/packages/gateway/src/routes/verdict.ts new file mode 100644 index 00000000..038c09f6 --- /dev/null +++ b/packages/gateway/src/routes/verdict.ts @@ -0,0 +1,78 @@ +import type { FastifyReply } from 'fastify'; +import type { Found } from '../find'; +import type { NodeState } from '../fleet'; +import { RETRY_AFTER_SECONDS } from '../raw'; + +/** + * A finding turned into a routing decision for an authenticated face: the + * node to forward to (with what the cache or the lookup knows of the + * sandbox), nothing (every node answered and none holds it — a create + * places, a destroy gives its idempotent answer, everything else a 404), + * or a refusal with its status and sentence, which the face sends in its + * own dialect. A value, so every verb reads the decision the same way. + */ +export type Verdict = + | { kind: 'node'; node: NodeState; id: string; name: string | null } + | { kind: 'none' } + | { + kind: 'refuse'; + status: number; + message: string; + retryAfterSeconds?: number; + }; + +export function verdict(found: Found, what: string): Verdict { + switch (found.kind) { + case 'one': + return { kind: 'node', node: found.node, id: found.id, name: found.name }; + case 'none': + return { kind: 'none' }; + case 'conflict': { + // The gateway refuses every verb for this name with this very 409, + // destroy included — it will not guess which copy the caller means. + const ids = found.nodes.map((n) => n.id).join(' and '); + const endpoints = new Set(found.nodes.map((n) => n.endpoint)); + if (endpoints.size === 1) { + // Two ids, one endpoint: not two copies but one daemon answering + // twice — a node whose DORMICE_NODE_ID changed (its old id keeps + // its row until removed), or two nodes whose DORMICE_NODE_ENDPOINT + // name the same machine. Nothing to destroy; the fleet's list of + // nodes is wrong, and the check-in log said so when it happened + // (routes/nodes.ts). + return { + kind: 'refuse', + status: 409, + message: `${what} was answered for by nodes ${ids}, which are one endpoint (${[...endpoints][0]}) — one daemon under two node ids: a node whose DORMICE_NODE_ID changed and whose old id still has its row (removeNode the id that no longer checks in; listNodes shows which), or two nodes whose DORMICE_NODE_ENDPOINT name the same machine (correct the wrong one)`, + }; + } + return { + kind: 'refuse', + status: 409, + message: `${what} exists on nodes ${ids} — destroy one copy directly on its node before routing can resume`, + }; + } + case 'unsure': + return { + kind: 'refuse', + status: 503, + message: `${what}: ${found.silent + .map((s) => `node ${s.nodeId} did not answer (${s.why})`) + .join( + ', ', + )} — it cannot be treated as new while a node that may hold it is silent: that node may be down, or busy building this very name (its lookup waits for that); retry after Retry-After, and remove the node only if it is gone for good`, + retryAfterSeconds: RETRY_AFTER_SECONDS, + }; + } +} + +/** Sends a refusal in the native dialect, with Retry-After where the verdict carries one. */ +export function refuse( + reply: FastifyReply, + refusal: Extract, + body: (message: string) => unknown = (message) => ({ message }), +) { + if (refusal.retryAfterSeconds !== undefined) { + reply.header('retry-after', String(refusal.retryAfterSeconds)); + } + return reply.code(refusal.status).send(body(refusal.message)); +} diff --git a/packages/gateway/src/testing.ts b/packages/gateway/src/testing.ts new file mode 100644 index 00000000..083a1be6 --- /dev/null +++ b/packages/gateway/src/testing.ts @@ -0,0 +1,60 @@ +import type { CheckInRequest, NodeReading } from '@dormice/shared'; + +/** + * Test scaffolding shared by the gateway's suites: a node's reading and + * check-in with a few knobs turned. Not shipped — nothing under src/ but + * main.ts is bundled (tsup.config.ts). + */ +export function reading( + over: { + cpu?: number | null; + cores?: number; + active?: number; + frozen?: number; + memAvail?: number; + diskAvail?: number | null; + } = {}, +): NodeReading { + const frozen = over.frozen ?? 0; + const active = over.active ?? 10; + return { + host: { + cpuCount: over.cores ?? 8, + cpuUsedPct: over.cpu === undefined ? 10 : over.cpu, + memTotalBytes: 32e9, + memAvailableBytes: over.memAvail ?? 16e9, + swap: null, + }, + dataDisk: + over.diskAvail === null + ? null + : { + path: '/var/lib/dormice', + totalBytes: 1e12, + usedBytes: 1e12 - (over.diskAvail ?? 5e11), + availableBytes: over.diskAvail ?? 5e11, + }, + sandboxes: { + total: active + frozen, + byState: { active, frozen, stopped: 0, archived: 0, restoring: 0 }, + }, + }; +} + +export function checkInOf( + nodeId: string, + endpoint: string, + over: Parameters[0] & { intervalSeconds?: number } = {}, +): CheckInRequest { + return { + nodeId, + endpoint, + intervalSeconds: over.intervalSeconds ?? 15, + build: { + commit: 'abc1234', + title: 'a commit', + committedAt: '2026-09-14T00:00:00.000Z', + }, + reading: reading(over), + }; +} diff --git a/packages/gateway/src/version.ts b/packages/gateway/src/version.ts new file mode 100644 index 00000000..62ed4335 --- /dev/null +++ b/packages/gateway/src/version.ts @@ -0,0 +1,23 @@ +/** + * The identity tsup baked into this build (tsup.config.ts): the commit the + * dist was built from. In the built daemon these `process.env` reads are + * compile-time literals; running from source (tests, tsx) they fall + * through to the real environment and come back empty — an unbuilt tree + * has no build identity, and null says so honestly. + */ +export interface BuildInfo { + /** Short hash. */ + commit: string; + /** The commit's subject line. */ + title: string; + /** ISO 8601 UTC — the commit's time, not the build's. */ + committedAt: string; +} + +export function readBuildInfo(): BuildInfo | null { + const commit = process.env.DORMICE_BUILD_COMMIT; + const title = process.env.DORMICE_BUILD_COMMIT_TITLE; + const committedAt = process.env.DORMICE_BUILD_COMMIT_AT; + if (!commit || !title || !committedAt) return null; + return { commit, title, committedAt }; +} diff --git a/packages/gateway/tsconfig.json b/packages/gateway/tsconfig.json new file mode 100644 index 00000000..564a5990 --- /dev/null +++ b/packages/gateway/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src"] +} diff --git a/packages/gateway/tsup.config.ts b/packages/gateway/tsup.config.ts new file mode 100644 index 00000000..d607d033 --- /dev/null +++ b/packages/gateway/tsup.config.ts @@ -0,0 +1,38 @@ +import { execSync } from 'node:child_process'; +import { defineConfig } from 'tsup'; + +/** + * Same build identity as the daemon (packages/server/tsup.config.ts has + * the reasoning): the commit the dist was built from, baked in as + * literals that src/version.ts reads back — what the process IS, not what + * the checkout says after a pull. + */ +function git(args: string): string { + try { + return execSync(`git ${args}`, { + stdio: ['ignore', 'pipe', 'ignore'], + }) + .toString() + .trim(); + } catch { + return ''; + } +} + +const commitTime = git('log -1 --format=%cI'); + +export default defineConfig({ + // A service, not a library: main.ts is the one entry and nothing imports + // the package (e2e boots dist/main.js as a process), so no index and no + // declarations. + entry: ['src/main.ts'], + format: ['esm'], + clean: true, + env: { + DORMICE_BUILD_COMMIT: git('rev-parse --short HEAD'), + DORMICE_BUILD_COMMIT_TITLE: git('log -1 --format=%s'), + DORMICE_BUILD_COMMIT_AT: commitTime + ? new Date(commitTime).toISOString() + : '', + }, +}); diff --git a/packages/server/package.json b/packages/server/package.json index 4596500d..f05001ba 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -13,6 +13,22 @@ "./mini-s3": { "types": "./dist/archive/mini-s3.d.ts", "default": "./dist/archive/mini-s3.js" + }, + "./auth": { + "types": "./dist/auth.d.ts", + "default": "./dist/auth.js" + }, + "./keyed-queue": { + "types": "./dist/keyed-queue.d.ts", + "default": "./dist/keyed-queue.js" + }, + "./lock": { + "types": "./dist/db/lock.d.ts", + "default": "./dist/db/lock.js" + }, + "./shutdown": { + "types": "./dist/shutdown.d.ts", + "default": "./dist/shutdown.js" } }, "files": [ diff --git a/packages/server/src/app.test.ts b/packages/server/src/app.test.ts index 517fbf44..555e2b8f 100644 --- a/packages/server/src/app.test.ts +++ b/packages/server/src/app.test.ts @@ -2358,3 +2358,100 @@ describe('activity attribution', () => { expect((await eventOf(app, 'apikey-created'))?.actor).toBe('console'); }); }); + +describe('POST /lookupSandbox', () => { + it('answers by name and by id with the state, without waking or touching the idle clock', async () => { + const { app, db } = testApp(); + const created = (await acquire(app, { name: 'alice' })).json(); + const row = findById(db, created.sandbox.id); + if (!row) throw new Error('no row'); + // A cold sandbox stays cold: lookup is observation, not use. + transition(db, row.id, 'frozen'); + + const byName = await rpc(app, '/lookupSandbox', { name: 'alice' }); + expect(byName.statusCode).toBe(200); + expect(byName.json()).toEqual({ + found: true, + sandbox: { id: row.id, name: 'alice', state: 'frozen' }, + }); + const byId = await rpc(app, '/lookupSandbox', { id: row.id }); + expect(byId.json()).toEqual(byName.json()); + expect(findById(db, row.id)?.state).toBe('frozen'); + expect(findById(db, row.id)?.lastActiveAt).toBe(row.lastActiveAt); + + expect( + (await rpc(app, '/lookupSandbox', { name: 'nobody' })).json(), + ).toEqual({ found: false }); + expect( + (await rpc(app, '/lookupSandbox', { id: 'no-such-id' })).json(), + ).toEqual({ found: false }); + // Neither a name nor an id is not a question. + expect((await rpc(app, '/lookupSandbox', {})).statusCode).toBe(400); + }); + + it('a name whose slot is busy waits its turn: asked while an acquire is mid-create, it answers found once the row exists', async () => { + // A create that parks inside the executor — the daemon's own shape of + // "in flight": the acquire holds the name's slot, the container is + // being built, the row is not written yet. + let release: () => void = () => {}; + let inCreate: () => void = () => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + const reached = new Promise((resolve) => { + inCreate = resolve; + }); + class ParkedCreate extends FakeExecutor { + override async create( + ...args: Parameters + ): Promise { + inCreate(); + await gate; + return super.create(...args); + } + } + const { app } = testApp(new ParkedCreate()); + const creating = acquire(app, { name: 'alice' }); + await reached; + // No row, slot busy: the question must wait, not answer "no". + const asked = rpc(app, '/lookupSandbox', { name: 'alice' }); + const early = await Promise.race([ + asked.then(() => 'answered'), + new Promise((resolve) => setTimeout(() => resolve('pending'), 50)), + ]); + expect(early).toBe('pending'); + release(); + const created = (await creating).json(); + expect((await asked).json()).toEqual({ + found: true, + sandbox: { id: created.sandbox.id, name: 'alice', state: 'active' }, + }); + }); + + it('a sandbox with a row answers at once even while its slot is held — a restore in progress must not read as silence', async () => { + const { app, db, locks } = testApp(); + const created = (await acquire(app, { name: 'alice' })).json(); + transition(db, created.sandbox.id, 'frozen'); + transition(db, created.sandbox.id, 'stopped'); + transition(db, created.sandbox.id, 'archived'); + transition(db, created.sandbox.id, 'restoring'); + let release: () => void = () => {}; + const held = locks.run( + 'alice', + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const answer = await Promise.race([ + rpc(app, '/lookupSandbox', { name: 'alice' }).then((r) => r.json()), + new Promise((resolve) => setTimeout(() => resolve('pending'), 500)), + ]); + expect(answer).toEqual({ + found: true, + sandbox: { id: created.sandbox.id, name: 'alice', state: 'restoring' }, + }); + release(); + await held; + }); +}); diff --git a/packages/server/src/auth.ts b/packages/server/src/auth.ts index 10cd542f..d1b341a5 100644 --- a/packages/server/src/auth.ts +++ b/packages/server/src/auth.ts @@ -161,7 +161,14 @@ function sessionCookieValid( request: FastifyRequest, getSessionSecret: () => string | null, ): boolean { - const cookie = request.cookies?.[SESSION_COOKIE]; + // The jar exists only where the app registered @fastify/cookie: the + // daemon does; the gateway never mounts the console and passes a getter + // that answers null, so no cookie can pass there. Typed structurally so + // this module stands alone as the `@dormice/server/auth` subpath entry + // without that plugin's request augmentation in its build graph. + const jar = (request as { cookies?: Record }) + .cookies; + const cookie = jar?.[SESSION_COOKIE]; const secret = getSessionSecret(); return Boolean( cookie && diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts new file mode 100644 index 00000000..666cfa82 --- /dev/null +++ b/packages/server/src/check-in.test.ts @@ -0,0 +1,253 @@ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { fileURLToPath } from 'node:url'; +import { checkInRequestSchema } from '@dormice/shared'; +import { afterEach, describe, expect, it } from 'vitest'; +import { CheckIn, type CheckInOptions, readNodeReading } from './check-in'; +import { migrateDb, openDb } from './db/db'; +import { createSandbox } from './db/ledger'; +import { CpuSampler } from './host-metrics'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const TOKEN = 'shared-token-shared-token-shared-token'; + +const servers: http.Server[] = []; +afterEach(async () => { + await Promise.all( + servers + .splice(0) + .map((s) => new Promise((resolve) => s.close(() => resolve()))), + ); +}); + +/** A gateway-shaped listener: records every check-in, answers what the test says. */ +async function gateway( + answer: () => { + status: number; + body: string; + headers?: Record; + }, +) { + const seen: Array<{ headers: http.IncomingHttpHeaders; body: unknown }> = []; + const server = http.createServer((req, res) => { + let text = ''; + req.on('data', (chunk) => { + text += chunk; + }); + req.on('end', () => { + seen.push({ headers: req.headers, body: JSON.parse(text) }); + const a = answer(); + res.writeHead(a.status, { + 'content-type': 'application/json', + ...a.headers, + }); + res.end(a.body); + }); + }); + servers.push(server); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return { + endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + seen, + }; +} + +function logSpy() { + const infos: string[] = []; + const warns: string[] = []; + const details: unknown[] = []; + return { + infos, + warns, + details, + log: { + info: (msg: string) => infos.push(msg), + warn: (obj: unknown, msg: string) => { + warns.push(msg); + details.push(obj); + }, + }, + }; +} + +function options( + gatewayEndpoint: string, + log: CheckInOptions['log'], + over: Partial = {}, +): CheckInOptions { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const cpu = new CpuSampler(); + return { + gateway: gatewayEndpoint, + token: TOKEN, + nodeId: 'node-7', + endpoint: 'http://10.0.0.7:80', + intervalSeconds: 1, + build: { + commit: 'abc1234', + title: 'a commit', + committedAt: '2026-09-14T00:00:00.000Z', + }, + readReading: () => readNodeReading(db, cpu, '/nonexistent-data-dir'), + log, + ...over, + }; +} + +describe('CheckIn', () => { + it('posts a check-in the gateway can parse: shared token, id, endpoint, interval, build, reading', async () => { + const gw = await gateway(() => ({ status: 200, body: '{}' })); + const { log, warns } = logSpy(); + await new CheckIn(options(gw.endpoint, log)).once(); + expect(warns).toEqual([]); + expect(gw.seen).toHaveLength(1); + expect(gw.seen[0]?.headers.authorization).toBe(`Bearer ${TOKEN}`); + const body = checkInRequestSchema.parse(gw.seen[0]?.body); + expect(body.nodeId).toBe('node-7'); + expect(body.endpoint).toBe('http://10.0.0.7:80'); + expect(body.intervalSeconds).toBe(1); + expect(body.build?.commit).toBe('abc1234'); + expect(body.reading.host.cpuCount).toBeGreaterThan(0); + // The first sample has no delta, and a data dir that does not exist is + // an honest null, never a made-up disk. + expect(body.reading.host.cpuUsedPct).toBeNull(); + expect(body.reading.dataDisk).toBeNull(); + expect(body.reading.sandboxes).toEqual({ + total: 0, + byState: { active: 0, frozen: 0, stopped: 0, archived: 0, restoring: 0 }, + }); + }); + + it('the reading counts the ledger by state', async () => { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + for (const name of ['a', 'b']) { + createSandbox(db, { + id: `id-${name}`, + name, + nodeId: 'node-7', + policy: { + freezeAfterSeconds: 60, + stopAfterSeconds: null, + archiveAfterSeconds: null, + }, + template: null, + metadata: null, + spec: undefined, + actor: null, + }); + } + const reading = await readNodeReading(db, new CpuSampler(), '/tmp'); + expect(reading.sandboxes.total).toBe(2); + expect(reading.sandboxes.byState.active).toBe(2); + expect(reading.dataDisk?.path).toBe('/tmp'); + }); + + it('logs a failing gateway once, and its recovery once — not every tick', async () => { + let status = 500; + const gw = await gateway(() => ({ status, body: '{"message":"boom"}' })); + const { log, warns, infos } = logSpy(); + const checkIn = new CheckIn(options(gw.endpoint, log)); + await checkIn.once(); + await checkIn.once(); + expect(gw.seen).toHaveLength(2); + expect(warns).toHaveLength(1); + expect(warns[0]).toMatch(/check-in failed/); + status = 200; + await checkIn.once(); + await checkIn.once(); + expect(infos).toEqual([ + `check-in with gateway ${gw.endpoint} answers again`, + ]); + expect(warns).toHaveLength(1); + // A gateway that refuses the token is the same one event. + status = 401; + await checkIn.once(); + await checkIn.once(); + expect(warns).toHaveLength(2); + }); + + it('a front that redirects is reported as a wrong address, not followed with the token stripped', async () => { + const gw = await gateway(() => ({ + status: 308, + body: '', + headers: { location: 'https://gateway.example/checkIn' }, + })); + const { log, warns, details } = logSpy(); + await new CheckIn(options(gw.endpoint, log)).once(); + expect(gw.seen).toHaveLength(1); + expect(warns).toEqual([expect.stringMatching(/check-in failed/)]); + expect((details[0] as { error: string }).error).toMatch( + /gateway answered 308 redirecting to https:\/\/gateway\.example\/checkIn — DORMICE_GATEWAY_ENDPOINT must be the gateway's own address/, + ); + }); + + it("a refusal's whole sentence reaches the log — the 409 for a shared node id names two endpoints and a remedy past the 200th character", async () => { + const refusal = + 'node node-7 checked in from http://10.0.0.7:80 3s ago and now from http://10.0.0.8:80 — two daemons share one DORMICE_NODE_ID (give this one its own), or the node just moved (then its next check-in, an interval later, is taken)'; + const gw = await gateway(() => ({ + status: 409, + body: JSON.stringify({ message: refusal }), + })); + const { log, details } = logSpy(); + await new CheckIn(options(gw.endpoint, log)).once(); + expect((details[0] as { error: string }).error).toBe( + `gateway answered 409: ${JSON.stringify({ message: refusal })}`, + ); + }); + + it('a failure that changes is logged again — unreachable, then refused, are two events — while the same refusal with another number in it is not', async () => { + let answer = { status: 500, body: '{"message":"boom"}' }; + const gw = await gateway(() => answer); + const { log, warns, details } = logSpy(); + const checkIn = new CheckIn(options(gw.endpoint, log)); + await checkIn.once(); + await checkIn.once(); + expect(warns).toHaveLength(1); + const refusal = (ago: number) => + JSON.stringify({ + message: `node node-7 checked in from http://10.0.0.7:80 ${ago}s ago and now from http://10.0.0.8:80 — two daemons share one DORMICE_NODE_ID`, + }); + answer = { status: 409, body: refusal(3) }; + await checkIn.once(); + answer = { status: 409, body: refusal(4) }; + await checkIn.once(); + expect(warns).toHaveLength(2); + expect(warns[1]).toMatch(/still failing, differently/); + expect((details[1] as { error: string }).error).toMatch( + /gateway answered 409: .*3s ago/, + ); + }); + + it("a gateway that is not there is a logged failure, never a throw, and the log says why in the transport's word", async () => { + // A port the OS just released: dialling it is refused, not black-holed. + const probe = http.createServer(); + await new Promise((resolve) => probe.listen(0, '127.0.0.1', resolve)); + const port = (probe.address() as AddressInfo).port; + await new Promise((resolve) => probe.close(() => resolve())); + const { log, warns, details } = logSpy(); + const checkIn = new CheckIn(options(`http://127.0.0.1:${port}`, log)); + await expect(checkIn.once()).resolves.toBeUndefined(); + expect(warns).toHaveLength(1); + expect((details[0] as { error: string }).error).toMatch( + /fetch failed \(ECONNREFUSED\)/, + ); + }); + + it('ticks on its interval from start() and stops on stop()', async () => { + const gw = await gateway(() => ({ status: 200, body: '{}' })); + const { log } = logSpy(); + const checkIn = new CheckIn(options(gw.endpoint, log)); + checkIn.start(); + const deadline = Date.now() + 5_000; + while (gw.seen.length < 2 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + expect(gw.seen.length).toBeGreaterThanOrEqual(2); + checkIn.stop(); + const afterStop = gw.seen.length; + await new Promise((resolve) => setTimeout(resolve, 1_200)); + expect(gw.seen.length).toBe(afterStop); + }); +}); diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts new file mode 100644 index 00000000..a208d1a0 --- /dev/null +++ b/packages/server/src/check-in.ts @@ -0,0 +1,169 @@ +import { + type BuildInfo, + type CheckInRequest, + checkInResponseSchema, + type NodeReading, +} from '@dormice/shared'; +import type { Db } from './db/db'; +import { countByState, listSandboxes } from './db/ledger'; +import { type CpuSampler, readHostReading } from './host-metrics'; + +/** + * A node's reading for its check-in: the host half (host-metrics.ts) and + * the ledger's census. The same numbers getHostMetrics answers a caller + * with, minus the daemon-local knobs no gateway places by. + */ +export async function readNodeReading( + db: Db, + cpu: CpuSampler, + dataDir: string, +): Promise { + const { byState, total } = countByState(listSandboxes(db)); + return { + ...(await readHostReading(cpu, dataDir)), + sandboxes: { total, byState }, + }; +} + +export interface CheckInLog { + info(msg: string): void; + warn(obj: unknown, msg: string): void; +} + +export interface CheckInOptions { + /** DORMICE_GATEWAY_ENDPOINT. */ + gateway: string; + /** The token gateway and nodes share (DORMICE_API_TOKEN). */ + token: string; + nodeId: string; + /** Where the gateway reaches this node (DORMICE_NODE_ENDPOINT or the loopback default). */ + endpoint: string; + intervalSeconds: number; + build: BuildInfo | null; + readReading: () => Promise; + log: CheckInLog; + /** Test seam; production uses the platform's fetch. */ + fetchImpl?: typeof fetch; +} + +/** A gateway that has not answered within this is a gateway not answering; the next tick tries again. */ +const CHECK_IN_TIMEOUT_MS = 10_000; + +/** + * The node's check-in ticker: every interval, one POST /checkIn to the + * gateway carrying the node's id, where it can be reached, its build and + * a fresh reading (RULES/协议.md「网关」). The gateway learns of a node from + * its first check-in — no registration verb, no nodes file — and reads + * two missed check-ins as down. + * + * Chained setTimeout, the daemon's discipline: the next tick is scheduled + * when this one is done, so a slow gateway never has ticks pile up. + * Failures are logged on the change — once when the gateway stops + * answering, once more when what is wrong changes (a gateway that was + * unreachable and now refuses this node is news), once when it answers + * again — never every tick: a gateway down for an hour is one event, not + * two hundred and forty lines. Never fatal: the gateway is the fleet's + * front door and configuration authority, not the node's reason to live; + * the node keeps running its sandboxes and keeps trying. + */ +/** + * A failure in the operator's words. fetch says "fetch failed" and keeps + * the reason (ECONNREFUSED, ENOTFOUND, a TLS error) in `cause`; a + * timeout is a DOMException whose only code is a legacy number. The + * transport's word is the one the operator acts on, so it is appended. + */ +function describe(error: unknown): string { + const e = error as { message?: string; cause?: unknown }; + const cause = e.cause as { code?: unknown; message?: string } | undefined; + const message = e.message ?? String(error); + const why = typeof cause?.code === 'string' ? cause.code : cause?.message; + return why === undefined ? message : `${message} (${why})`; +} + +export class CheckIn { + private timer: NodeJS.Timeout | undefined; + private closing = false; + /** The failure the gateway is currently in (its sentence with the numbers blanked, so a 409 that says "3s ago" and then "4s ago" is one failure), or null while it answers. */ + private failing: string | null = null; + + constructor(private readonly opts: CheckInOptions) {} + + start(): void { + this.schedule(0); + } + + stop(): void { + this.closing = true; + clearTimeout(this.timer); + } + + /** One check-in. Never throws: a failure is recorded and the next tick retries. */ + async once(): Promise { + const { opts } = this; + try { + const body: CheckInRequest = { + nodeId: opts.nodeId, + endpoint: opts.endpoint, + intervalSeconds: opts.intervalSeconds, + build: opts.build, + reading: await opts.readReading(), + }; + const res = await (opts.fetchImpl ?? fetch)(`${opts.gateway}/checkIn`, { + method: 'POST', + headers: { + authorization: `Bearer ${opts.token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(CHECK_IN_TIMEOUT_MS), + // A front that redirects (a Caddy binding the gateway's domain + // answers plain http with a 308 to https) is reported as what it + // is. Followed, the redirect would cross origins and fetch would + // drop the Authorization header on the way (the Fetch standard's + // rule), so the gateway would answer 401 — and the operator would + // read a wrong token where there is a wrong address (found by + // review, 2026-09-14). + redirect: 'manual', + }); + if (res.status !== 200) { + const text = await res.text(); + const location = res.headers.get('location'); + // Whole enough for the gateway's own refusals (its longest, the + // 409 naming both endpoints of a shared node id, runs to about 260 + // characters — cut at 200 it lost its remedy), short enough that + // a front's HTML error page does not flood the log. + const body = text.slice(0, 400); + throw new Error( + location === null + ? `gateway answered ${res.status}: ${body}` + : `gateway answered ${res.status} redirecting to ${location} — DORMICE_GATEWAY_ENDPOINT must be the gateway's own address, not a front that redirects`, + ); + } + checkInResponseSchema.parse(await res.json()); + if (this.failing !== null) { + opts.log.info(`check-in with gateway ${opts.gateway} answers again`); + this.failing = null; + } + } catch (error) { + const message = describe(error); + const failure = message.replace(/\d+/g, '#'); + if (failure !== this.failing) { + opts.log.warn( + { gateway: opts.gateway, error: message }, + this.failing === null + ? 'check-in failed; the gateway places nothing here and forwards no new names to this node until it answers again — retrying every interval' + : 'check-in still failing, differently — retrying every interval', + ); + } + this.failing = failure; + } + } + + private schedule(delayMs: number): void { + if (this.closing) return; + this.timer = setTimeout(async () => { + await this.once(); + this.schedule(this.opts.intervalSeconds * 1000); + }, delayMs); + } +} diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index 90fea421..c9a76efd 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -151,3 +151,115 @@ describe('the S3 set', () => { expect(off.DORMICE_S3_FORCE_PATH_STYLE).toBe(false); }); }); + +describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { + it('defaults: no gateway (standalone), no node endpoint, 15s check-in', () => { + const config = loadConfig(TOKEN); + expect(config.DORMICE_GATEWAY_ENDPOINT).toBeUndefined(); + expect(config.DORMICE_NODE_ENDPOINT).toBeUndefined(); + expect(config.DORMICE_CHECK_IN_INTERVAL_SECONDS).toBe(15); + }); + + it('parses both endpoints as full URLs and drops a trailing slash', () => { + const config = loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677/', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80///', + DORMICE_NODE_ID: 'node-7', + DORMICE_CHECK_IN_INTERVAL_SECONDS: '5', + }); + expect(config.DORMICE_GATEWAY_ENDPOINT).toBe('http://10.0.0.5:3677'); + expect(config.DORMICE_NODE_ENDPOINT).toBe('http://10.0.0.7:80'); + expect(config.DORMICE_CHECK_IN_INTERVAL_SECONDS).toBe(5); + }); + + it('refuses an endpoint without a scheme, naming the variable', () => { + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_ENDPOINT: '10.0.0.5:3677' }), + ).toThrow(/DORMICE_GATEWAY_ENDPOINT must be a full http\(s\) URL/); + expect(() => + loadConfig({ ...TOKEN, DORMICE_NODE_ENDPOINT: 'node-7' }), + ).toThrow(/DORMICE_NODE_ENDPOINT must be a full http\(s\) URL/); + expect(() => + loadConfig({ ...TOKEN, DORMICE_CHECK_IN_INTERVAL_SECONDS: '0' }), + ).toThrow(); + }); + + it('a node endpoint with a path or a query is refused at boot, naming why; a trailing slash is still just dropped', () => { + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80/dormice', + }), + ).toThrow( + /DORMICE_NODE_ENDPOINT must name the node's front without a path/, + ); + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80/?x=1', + }), + ).toThrow(/without a path/); + expect( + loadConfig({ ...TOKEN, DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80/' }) + .DORMICE_NODE_ENDPOINT, + ).toBe('http://10.0.0.7:80'); + }); + + it('a gateway on another machine requires the node endpoint, naming why; a loopback gateway does not', () => { + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + }), + ).toThrow( + /DORMICE_NODE_ENDPOINT is required when DORMICE_GATEWAY_ENDPOINT is not loopback/, + ); + expect( + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80', + DORMICE_NODE_ID: 'node-7', + }).DORMICE_NODE_ENDPOINT, + ).toBe('http://10.0.0.7:80'); + for (const local of [ + 'http://127.0.0.1:3677', + 'http://localhost:3677', + 'http://[::1]:3677', + ]) { + expect( + loadConfig({ ...TOKEN, DORMICE_GATEWAY_ENDPOINT: local }) + .DORMICE_NODE_ENDPOINT, + ).toBeUndefined(); + } + }); + + it('a gateway on another machine requires a node id of its own, naming why; beside its gateway the default serves', () => { + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80', + }), + ).toThrow( + /DORMICE_NODE_ID is required when DORMICE_GATEWAY_ENDPOINT is not loopback/, + ); + expect( + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80', + DORMICE_NODE_ID: 'bj-7', + }).DORMICE_NODE_ID, + ).toBe('bj-7'); + expect( + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://127.0.0.1:3677', + }).DORMICE_NODE_ID, + ).toBe('node-1'); + expect(loadConfig(TOKEN).DORMICE_NODE_ID).toBe('node-1'); + }); +}); diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index 2bc2e96d..59a0e184 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -1,5 +1,9 @@ import { isAbsolute } from 'node:path'; -import { bareHostnameRegex, PIDS_LIMIT_MIN } from '@dormice/shared'; +import { + bareHostnameRegex, + isOriginUrl, + PIDS_LIMIT_MIN, +} from '@dormice/shared'; import { z } from 'zod'; import type { S3Settings } from './archive/s3-store'; @@ -16,8 +20,13 @@ import type { S3Settings } from './archive/s3-store'; const envSchema = z.object({ DORMICE_PORT: z.coerce.number().int().min(1).max(65535).default(3676), DORMICE_DB_PATH: z.string().default('data/dormice.db'), - /** Identifies this machine in the ledger. Single-machine today; the field keeps the ledger shardable. */ - DORMICE_NODE_ID: z.string().default('node-1'), + /** + * This node's name: in its ledger's rows, and to its gateway, which + * tells nodes apart by it. The default serves a daemon that is the + * whole platform, or shares its machine with its gateway; a node whose + * gateway is elsewhere must state its own (checkedSchema below). + */ + DORMICE_NODE_ID: z.string().min(1).default('node-1'), /** How often the idle scanner sweeps the ledger. */ DORMICE_SCAN_INTERVAL_SECONDS: z.coerce.number().int().positive().default(60), /** @@ -169,8 +178,85 @@ const envSchema = z.object({ DORMICE_S3_REGION: z.string().default('us-east-1'), /** Path-style addressing: MinIO needs true; the clouds route by subdomain. */ DORMICE_S3_FORCE_PATH_STYLE: z.stringbool().default(false), + /** + * The gateway this daemon is a node of — its intranet address, e.g. + * http://10.0.0.5:3677. Set, the daemon checks in with it every + * DORMICE_CHECK_IN_INTERVAL_SECONDS (check-in.ts): its readings, its + * build, and where it can be reached. That check-in is the gateway's + * only source of "which nodes exist and how full are they" — no + * registration, no nodes file. Unset, the daemon is the whole platform + * by itself, as it always was, and checks in with nobody. The token it + * presents is DORMICE_API_TOKEN: gateway and nodes share one, and the + * gateway speaks to every node with the same one. + */ + DORMICE_GATEWAY_ENDPOINT: z + .url({ + protocol: /^https?$/, + error: + 'DORMICE_GATEWAY_ENDPOINT must be a full http(s) URL, e.g. http://10.0.0.5:3677', + }) + .transform((url) => url.replace(/\/+$/, '')) + .optional(), + /** + * Where the gateway reaches this node — the address it forwards to. + * Default: this daemon's own loopback address, right when gateway and + * node share a machine (the single-machine install is a fleet of one). + * On a machine of its own the daemon still binds loopback (the red + * line), so this names the front the gateway may dial — the node's + * Caddy on the intranet interface, e.g. http://10.0.0.7:80 — and is + * required there (checkedSchema below): left at the default, the node + * would tell a remote gateway to dial 127.0.0.1, an address on the + * gateway's own machine, and every sandbox placed "here" would land on + * whatever daemon lives there and be found twice. + */ + DORMICE_NODE_ENDPOINT: z + .url({ + protocol: /^https?$/, + error: + 'DORMICE_NODE_ENDPOINT must be a full http(s) URL, e.g. http://10.0.0.7:80', + }) + .transform((url) => url.replace(/\/+$/, '')) + // An origin, nothing more (shared endpointSchema has the measurement): + // the gateway would answer this node's every check-in with a 400 + // otherwise — refused here, at boot, where the operator is looking. + .refine(isOriginUrl, { + error: + "DORMICE_NODE_ENDPOINT must name the node's front without a path — scheme, host and port only, e.g. http://10.0.0.7:80 (the gateway dials /)", + }) + .optional(), + /** + * How often the node checks in with its gateway. The gateway reads two + * missed check-ins as down — the one number both ends of that wire + * share, so the node states it in every check-in. + */ + DORMICE_CHECK_IN_INTERVAL_SECONDS: z.coerce + .number() + .int() + .positive() + .default(15), }); +/** + * Loopback as an operator writes it: 127.0.0.0/8, ::1, localhost. Null + * when the URL does not parse — its own field has already said so, and an + * object-level rule must not throw over it (zod runs the refinements even + * when a field failed). + */ +function isLoopbackUrl(url: string): boolean | null { + let host: string; + try { + host = new URL(url).hostname; + } catch { + return null; + } + return ( + host === 'localhost' || + host === '::1' || + host === '[::1]' || + host.startsWith('127.') + ); +} + const checkedSchema = envSchema .refine( (cfg) => cfg.DORMICE_EXECUTOR !== 'docker' || !!cfg.DORMICE_BASE_IMAGE, @@ -203,6 +289,40 @@ const checkedSchema = envSchema path: ['DORMICE_DATA_DIR'], }, ) + // A node whose gateway is on another machine must say where it is. The + // check-in's default endpoint is this daemon's loopback address, which + // on the gateway's machine names the gateway's own daemon (or nothing): + // a remote gateway dialing 127.0.0.1 for this node would place sandboxes + // on the wrong machine and then find every one of them twice (409). An + // explicit value is the operator's word and is taken as written. + .refine( + (cfg) => + cfg.DORMICE_GATEWAY_ENDPOINT === undefined || + isLoopbackUrl(cfg.DORMICE_GATEWAY_ENDPOINT) !== false || + cfg.DORMICE_NODE_ENDPOINT !== undefined, + { + message: + "DORMICE_NODE_ENDPOINT is required when DORMICE_GATEWAY_ENDPOINT is not loopback: the gateway is on another machine, and without it this node would report http://127.0.0.1: — an address on the gateway's machine, not this one. Name this node's address on the network, e.g. http://10.0.0.7:80", + path: ['DORMICE_NODE_ENDPOINT'], + }, + ) + // ...and who it is. The gateway tells nodes apart by DORMICE_NODE_ID, + // and node-1 is what every other unconfigured node says too: the + // second node-1 to check in is refused as a twin (409) at every + // check-in — or, when the first has been silent for an interval, taken + // for it having moved, and the first's names are placed again + // elsewhere. Refused here, at boot, where the operator is looking. + .refine( + (cfg) => + cfg.DORMICE_GATEWAY_ENDPOINT === undefined || + isLoopbackUrl(cfg.DORMICE_GATEWAY_ENDPOINT) !== false || + cfg.DORMICE_NODE_ID !== 'node-1', + { + message: + 'DORMICE_NODE_ID is required when DORMICE_GATEWAY_ENDPOINT is not loopback: the gateway tells nodes apart by it, and node-1 (the default) is what every other unconfigured node says — the second to check in is refused as a twin. Give this node a name of its own, e.g. its hostname', + path: ['DORMICE_NODE_ID'], + }, + ) // All-or-none: a half-configured store would make the archiver's // existence ambiguous, and ambiguity here decides real policy defaults. .superRefine((cfg, ctx) => { @@ -261,6 +381,9 @@ export const CONFIG_KEYS: Record = { DORMICE_S3_SECRET_ACCESS_KEY: { sensitive: true }, DORMICE_S3_REGION: { sensitive: false }, DORMICE_S3_FORCE_PATH_STYLE: { sensitive: false }, + DORMICE_GATEWAY_ENDPOINT: { sensitive: false }, + DORMICE_NODE_ENDPOINT: { sensitive: false }, + DORMICE_CHECK_IN_INTERVAL_SECONDS: { sensitive: false }, }; export type ConfigSources = Record; diff --git a/packages/server/src/db/lock.ts b/packages/server/src/db/lock.ts index afca59e3..2e086d0c 100644 --- a/packages/server/src/db/lock.ts +++ b/packages/server/src/db/lock.ts @@ -13,8 +13,17 @@ import Database from 'better-sqlite3'; * The lock is an OS-level file lock held by a dedicated SQLite handle in * EXCLUSIVE locking mode: released the instant the process dies, crash * included, so there is no stale-pidfile problem to solve. + * + * The returned handle must be kept for the life of the process: better- + * sqlite3 closes a handle whose object is garbage collected, and a closed + * handle drops the lock (main.ts has the 2026-09-11 measurement). The + * busy sentence is the caller's, because the gateway takes the same lock + * over its own file and must name its own variable in the exit. */ -export function acquireSingleWriterLock(dbPath: string): Database.Database { +export function acquireSingleWriterLock( + dbPath: string, + busyMessage = `another daemon is already running against ${dbPath} — one ledger, one daemon. Stop the other instance, or point this one at its own DORMICE_DB_PATH.`, +): Database.Database { mkdirSync(dirname(dbPath), { recursive: true }); // Fail fast: a held lock answers in 100ms instead of the default 5s wait. const lock = new Database(`${dbPath}.lock`, { timeout: 100 }); @@ -29,10 +38,7 @@ export function acquireSingleWriterLock(dbPath: string): Database.Database { } catch (error) { lock.close(); if ((error as { code?: string }).code === 'SQLITE_BUSY') { - throw new Error( - `another daemon is already running against ${dbPath} — one ledger, one daemon. ` + - 'Stop the other instance, or point this one at its own DORMICE_DB_PATH.', - ); + throw new Error(busyMessage); } throw error; } diff --git a/packages/server/src/host-metrics.ts b/packages/server/src/host-metrics.ts index d629146c..5a768d7e 100644 --- a/packages/server/src/host-metrics.ts +++ b/packages/server/src/host-metrics.ts @@ -1,5 +1,6 @@ import { readFile, statfs } from 'node:fs/promises'; import os from 'node:os'; +import type { NodeReading } from '@dormice/shared'; /** * Host-side readings for getHostMetrics — the machine's own health, which @@ -158,3 +159,28 @@ export async function readDiskSpace( availableBytes: s.bavail * s.bsize, }; } + +/** + * The host half of a node's reading — what getHostMetrics answers and + * what a check-in reports, from one function so the two never disagree on + * a field: the machine's cores and CPU delta, its memory and swap, and + * the data disk that holds the sandbox disks (null until it exists). The + * CpuSampler is the caller's: a delta spans "since this instance's last + * sample", so each reader owns one (host-metrics route, metrics sampler, + * check-in) and none steals another's window. + */ +export async function readHostReading( + cpu: CpuSampler, + dataDir: string, +): Promise> { + const memory = await readHostMemory(); + const disk = await readDiskSpace(dataDir); + return { + host: { + cpuCount: os.cpus().length, + cpuUsedPct: cpu.sample(), + ...memory, + }, + dataDisk: disk ? { path: dataDir, ...disk } : null, + }; +} diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index 8eee2cc9..6bc55694 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -5,6 +5,7 @@ import { pino } from 'pino'; import { buildApp } from './app'; import { Archiver } from './archive/archiver'; import { LedgerArchiveStore } from './archive/ledger-store'; +import { CheckIn, readNodeReading } from './check-in'; import { type Config, loadConfig } from './config'; import { recordActivity } from './db/activity'; import { migrateDb, openDb } from './db/db'; @@ -44,10 +45,15 @@ const config = loadConfig(); // One ledger, one daemon — enforced, not assumed. A second instance would // run its own destructive reconcile against sandboxes this one is still -// operating, well before it ever loses the race for the port. +// operating, well before it ever loses the race for the port. The handle +// is kept for the life of the process: better-sqlite3 closes a handle +// whose object is garbage collected, and a closed handle drops the file +// lock — with the return value discarded, a second daemon on the same +// ledger started fine seconds later (measured 2026-09-11, fake executor). +let ledgerLock: ReturnType | undefined; if (config.DORMICE_DB_PATH !== ':memory:') { try { - acquireSingleWriterLock(config.DORMICE_DB_PATH); + ledgerLock = acquireSingleWriterLock(config.DORMICE_DB_PATH); } catch (error) { fatal(error instanceof Error ? error.message : String(error)); } @@ -309,6 +315,41 @@ recordActivity(db, { // the daemon to the outside world is a reverse proxy's job. await app.listen({ host: '127.0.0.1', port: config.DORMICE_PORT }); +// A node of a fleet reports to its gateway; a daemon on its own reports to +// nobody. Started after listen on purpose: the check-in names where the +// gateway may forward to, and that door must be open before the gateway +// hears of it. Its CpuSampler is its own — a delta spans "since this +// instance's last sample", and the route's and the metrics ticker's +// windows must not be stolen (host-metrics.ts). +let checkIn: CheckIn | undefined; +if (config.DORMICE_GATEWAY_ENDPOINT !== undefined) { + const nodeEndpoint = + config.DORMICE_NODE_ENDPOINT ?? `http://127.0.0.1:${config.DORMICE_PORT}`; + // Not primed: the first check-in then reports cpuUsedPct null — "no + // interval yet" — which placement lets through as unknown. A sample a + // few milliseconds before it would make that first reading a percentage + // over the sliver in between, near 0 or near 100 by luck, and a freshly + // restarted node could sit out its first interval on a number that + // meant nothing (found by review, 2026-09-14). + const checkInCpu = new CpuSampler(); + checkIn = new CheckIn({ + gateway: config.DORMICE_GATEWAY_ENDPOINT, + token: config.DORMICE_API_TOKEN, + nodeId: config.DORMICE_NODE_ID, + endpoint: nodeEndpoint, + intervalSeconds: config.DORMICE_CHECK_IN_INTERVAL_SECONDS, + build, + readReading: () => readNodeReading(db, checkInCpu, config.DORMICE_DATA_DIR), + log, + }); + checkIn.start(); + log.info( + `node ${config.DORMICE_NODE_ID} checks in with gateway ${config.DORMICE_GATEWAY_ENDPOINT} every ${config.DORMICE_CHECK_IN_INTERVAL_SECONDS}s, reachable at ${nodeEndpoint}`, + ); +} else { + log.info('no gateway: standalone daemon (DORMICE_GATEWAY_ENDPOINT unset)'); +} + // systemd stops the daemon with SIGTERM. Shutdown is bounded on purpose // (shutdown.ts has the measurements): close the app — preClose ends the // long-lived streams with honest end-frames, the listener stops — give @@ -336,6 +377,7 @@ const close = async (signal: NodeJS.Signals) => { process.removeListener('SIGINT', onSigint); clearTimeout(heartbeatTimer); clearTimeout(metricsTimer); + checkIn?.stop(); watchdog.stop(); app.log.info( `${signal} received — shutting down (grace ${SHUTDOWN_GRACE_MS}ms)`, @@ -352,6 +394,7 @@ const close = async (signal: NodeJS.Signals) => { app.log.error(error, `graceful shutdown after ${signal} failed`); process.exitCode = 1; } + ledgerLock?.close(); process.exit(process.exitCode ?? 0); }; const onSigterm = () => void close('SIGTERM'); diff --git a/packages/server/src/routes/host.ts b/packages/server/src/routes/host.ts index ca733c19..d7e7d1be 100644 --- a/packages/server/src/routes/host.ts +++ b/packages/server/src/routes/host.ts @@ -1,4 +1,3 @@ -import os from 'node:os'; import { getFleetTimelineRequestSchema, getFleetTimelineResponseSchema, @@ -22,7 +21,7 @@ import { } from '../db/metrics'; import { readRuntimeSettings } from '../db/settings'; import type { Executor } from '../executor/executor'; -import { CpuSampler, readDiskSpace, readHostMemory } from '../host-metrics'; +import { CpuSampler, readHostReading } from '../host-metrics'; export interface HostRoutesOptions { config: Config; @@ -58,17 +57,8 @@ export const hostRoutes: FastifyPluginAsyncZod = async ( const rows = listSandboxes(db); const { byState, total } = countByState(rows); - const memory = await readHostMemory(); - const dataDisk = await readDiskSpace(config.DORMICE_DATA_DIR); return { - host: { - cpuCount: os.cpus().length, - cpuUsedPct: cpu.sample(), - ...memory, - }, - dataDisk: dataDisk - ? { path: config.DORMICE_DATA_DIR, ...dataDisk } - : null, + ...(await readHostReading(cpu, config.DORMICE_DATA_DIR)), sandboxes: { total, // The ledger's live knob, not the env seed — the console edits it. diff --git a/packages/server/src/routes/sandboxes.ts b/packages/server/src/routes/sandboxes.ts index 60b93c94..7488481d 100644 --- a/packages/server/src/routes/sandboxes.ts +++ b/packages/server/src/routes/sandboxes.ts @@ -19,6 +19,8 @@ import { listSandboxImagesResponseSchema, listSandboxMetricsRequestSchema, listSandboxMetricsResponseSchema, + lookupSandboxRequestSchema, + lookupSandboxResponseSchema, READ_FILES_TOTAL_LIMIT_BYTES, readFileRequestSchema, readFileResponseSchema, @@ -54,6 +56,7 @@ import type { Db } from '../db/db'; import { countSandboxes, createSandbox, + findById, findByName, listSandboxes, setDiskGb, @@ -1114,6 +1117,44 @@ export const sandboxRoutes: FastifyPluginAsyncZod< }, ); + // The gateway's one question on its own account: does this node hold + // the sandbox? Read-only — never wakes, never touches the idle clock — + // and truthful about a create in flight, in two steps. A row that + // exists answers at once, whatever its state: a restoring sandbox has a + // row, and waiting for its slot would hold the answer for the whole + // restore, long past the gateway's two-second patience — the gateway + // would read a live sandbox as a node that did not answer. No row means + // an acquire may be writing it right now (create first, row second, + // both under the name's slot), so the answer takes the slot itself and + // looks again: at once when the slot is free (a plain no), behind the + // acquire when it is not. By id there is no slot to wait on (slots are + // keyed by name), and none is needed: nobody can ask about an id before + // the create that minted it has answered. + app.post( + '/lookupSandbox', + { + schema: { + body: lookupSandboxRequestSchema, + response: { 200: lookupSandboxResponseSchema }, + }, + }, + async (request) => { + const query = request.body; + const look = () => + 'name' in query ? findByName(db, query.name) : findById(db, query.id); + const answer = (row: SandboxRow | undefined) => + row + ? { + found: true as const, + sandbox: { id: row.id, name: row.name, state: row.state }, + } + : { found: false as const }; + const now = look(); + if (now !== undefined || !('name' in query)) return answer(now); + return answer(await locks.run(query.name, async () => look())); + }, + ); + app.post( '/destroySandbox', { diff --git a/packages/server/tsup.config.ts b/packages/server/tsup.config.ts index e004f519..49d14560 100644 --- a/packages/server/tsup.config.ts +++ b/packages/server/tsup.config.ts @@ -24,7 +24,19 @@ function git(args: string): string { const commitTime = git('log -1 --format=%cI'); export default defineConfig({ - entry: ['src/index.ts', 'src/main.ts', 'src/archive/mini-s3.ts'], + // Subpath entries beyond the root: mini-s3 for the e2e harness; auth, + // keyed-queue, lock and shutdown for the gateway, which reuses the + // daemon's small self-contained pieces without loading its executor + // (the root's import graph drags dockerode, execa and the AWS SDK in). + entry: [ + 'src/index.ts', + 'src/main.ts', + 'src/archive/mini-s3.ts', + 'src/auth.ts', + 'src/keyed-queue.ts', + 'src/db/lock.ts', + 'src/shutdown.ts', + ], format: ['esm'], dts: true, clean: true, diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts new file mode 100644 index 00000000..7022acfc --- /dev/null +++ b/packages/shared/src/gateway.ts @@ -0,0 +1,161 @@ +import { z } from 'zod'; +import { + dataDiskSchema, + hostReadingSchema, + sandboxStateCountsSchema, +} from './host'; + +/** + * The gateway's wire: the verbs between a node and the gateway that fronts + * it, and the gateway's own observation verbs. A fleet is N daemons that + * know nothing of each other behind one gateway that holds no sandbox + * state — only what the nodes tell it every few seconds and what its own + * configuration tables say. A single machine runs the same two processes + * and is a fleet of one. + */ + +/** The identity a build carries: the commit its dist was built from. Null where the dist was built outside a checkout. */ +export const buildInfoSchema = z.object({ + /** Short hash. */ + commit: z.string(), + /** The commit's subject line. */ + title: z.string(), + /** ISO 8601 UTC — the commit's time, not the build's. */ + committedAt: z.iso.datetime(), +}); + +export type BuildInfo = z.infer; + +/** + * Whether a URL is an origin and nothing more: scheme, host and port — no + * path, query or fragment. False for a string that is not a URL at all. + */ +export function isOriginUrl(url: string): boolean { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return false; + } + return parsed.pathname === '/' && parsed.search === '' && parsed.hash === ''; +} + +/** + * Where a process can be dialled — an origin. The gateway joins + * `/` for its own lookup and hands the endpoint to undici + * as the request's origin when it forwards, and undici refuses an origin + * that carries a path (UND_ERR_INVALID_ARG, measured 2026-09-14): a node + * reporting `http://10.0.0.7:80/dormice` would be found by every lookup + * and reached by no forward. So a path is refused at the wire. A trailing + * slash is dropped rather than refused — the same address to everyone but + * a string compare, and the two ends must agree byte for byte (the join + * above would otherwise ask for `//lookupSandbox`, a 404 the gateway reads + * as silence). + */ +export const endpointSchema = z + .url({ protocol: /^https?$/ }) + .transform((url) => url.replace(/\/+$/, '')) + .refine(isOriginUrl, { + error: + 'an endpoint is an origin — scheme, host and port only, no path or query, e.g. http://10.0.0.7:80', + }); + +/** + * What a node reports about itself at every check-in — everything + * placement decides on: the machine's CPU, memory and data disk, and the + * ledger's census by state. The same host reading getHostMetrics answers + * (host.ts), minus the daemon-local knobs no gateway places by. + */ +export const nodeReadingSchema = z.object({ + host: hostReadingSchema, + dataDisk: dataDiskSchema.nullable(), + sandboxes: z.object({ + total: z.number().int(), + byState: sandboxStateCountsSchema, + }), +}); + +export type NodeReading = z.infer; + +/** + * checkIn — a node reporting for duty, every DORMICE_CHECK_IN_INTERVAL_SECONDS + * (15 by default), authenticated with the token the gateway and every node + * share. The gateway learns of a node from its first check-in: there is no + * registration verb and no nodes file — "which nodes exist" has one home, + * the gateway's nodes table, written by the nodes themselves. Two missed + * check-ins read as down: no new sandbox is placed there and its sandboxes + * answer 502 until it reports again. Who knows the truth speaks: the node + * knows what it runs and where it can be reached; the gateway only listens + * and compares, and never keeps a record of what it told whom. + */ +export const checkInRequestSchema = z.object({ + /** DORMICE_NODE_ID — the node's name in every sandbox's `nodeId`. */ + nodeId: z.string().min(1), + /** Where the gateway forwards to: the node's intranet front (DORMICE_NODE_ENDPOINT) — an origin, endpointSchema has why. */ + endpoint: endpointSchema, + /** How often this node checks in — the gateway's yardstick for "missed two in a row". */ + intervalSeconds: z.number().int().positive(), + build: buildInfoSchema.nullable(), + reading: nodeReadingSchema, +}); + +export type CheckInRequest = z.infer; + +/** Nothing yet: the configuration version the gateway will answer with arrives with the configuration authority. */ +export const checkInResponseSchema = z.object({}); + +export type CheckInResponse = z.infer; + +/** + * listNodes — every node the gateway has ever heard from, with what it + * last said. Everything here is what the gateway already holds; answering + * costs no node anything. + */ +export const listNodesRequestSchema = z.object({}); + +export const nodeViewSchema = z.object({ + id: z.string(), + endpoint: z.string(), + /** ISO 8601 UTC — the first check-in. */ + addedAt: z.iso.datetime(), + /** ISO 8601 UTC — null only right after a gateway start, before the node's next check-in. */ + lastCheckInAt: z.iso.datetime().nullable(), + intervalSeconds: z.number().int().positive().nullable(), + /** Checked in within two of its own intervals. */ + reachable: z.boolean(), + build: buildInfoSchema.nullable(), + reading: nodeReadingSchema.nullable(), + /** Sandboxes the gateway placed here since the last check-in — counted against the node until the next reading shows them. */ + placedSinceCheckIn: z.number().int().nonnegative(), +}); + +export type NodeView = z.infer; + +export const listNodesResponseSchema = z.object({ + nodes: z.array(nodeViewSchema), +}); + +export type ListNodesResponse = z.infer; + +/** + * removeNode — the operator's word that a node is gone for good: its row + * goes, its sandboxes are no longer looked for, and a name that lived only + * there is a new name again. A node that is merely down needs nothing — + * it is back the moment it checks in — and one removed by mistake re-adds + * itself the same way. Refused (409) while the node is still checking in: + * a name of its acquired in the seconds before its next check-in would be + * built elsewhere and come back on two nodes. Stop the daemon, wait two of + * its intervals (that is "down"), then remove. + */ +export const removeNodeRequestSchema = z.object({ + id: z.string().min(1), +}); + +export type RemoveNodeRequest = z.infer; + +export const removeNodeResponseSchema = z.object({ + /** True when a row existed and was removed; false when there was none. */ + removed: z.boolean(), +}); + +export type RemoveNodeResponse = z.infer; diff --git a/packages/shared/src/host.ts b/packages/shared/src/host.ts index ad695f5a..3eeeefa7 100644 --- a/packages/shared/src/host.ts +++ b/packages/shared/src/host.ts @@ -1,68 +1,83 @@ import { z } from 'zod'; /** - * getHostMetrics() — the observation window into the machine itself: is the - * host healthy, and what do the sandboxes collectively cost it? A single - * point-in-time snapshot; for the machine's past see getHostMetricsHistory - * below. Observation never wakes a sandbox and never touches lifecycle. + * The machine's own readings — the `host` half of getHostMetrics and of a + * node's check-in (gateway.ts nodeReadingSchema), one schema so the two + * never disagree on a field. * * Readings a platform cannot produce are null, honestly — never zero, never - * invented: swap and /proc are Linux facts, and the data directory only - * exists where the docker executor runs. + * invented: swap and /proc are Linux facts. */ -export const hostMetricsResponseSchema = z.object({ - host: z.object({ - cpuCount: z.number().int().positive(), - /** - * Percent of the whole machine, 0-100. Null until the sampler has two - * samples to take a delta between — the first request after daemon - * start reports "don't know yet", not a made-up 0. - */ - cpuUsedPct: z.number().nullable(), - memTotalBytes: z.number(), - /** - * What could still be allocated without swapping — /proc/meminfo's - * MemAvailable (counts reclaimable page cache), not the naive "free". - */ - memAvailableBytes: z.number(), - /** - * Swap is the freeze mechanism's fuel: frozen sandboxes live there, and - * a full swap means "idle is free" stops being true. totalBytes of 0 is - * an honest reading of a machine with no swap configured (doctor warns - * about it); null means the platform offers no reading at all. - */ - swap: z - .object({ - totalBytes: z.number(), - usedBytes: z.number(), - }) - .nullable(), - }), +export const hostReadingSchema = z.object({ + cpuCount: z.number().int().positive(), /** - * The filesystem holding DORMICE_DATA_DIR — where sandbox disks live. A - * full data disk is the real capacity ceiling (past it, even the ledger - * cannot write). Null when the directory does not exist (fake executor, - * fresh install). + * Percent of the whole machine, 0-100. Null until the sampler has two + * samples to take a delta between — the first request after daemon + * start reports "don't know yet", not a made-up 0. */ - dataDisk: z + cpuUsedPct: z.number().nullable(), + memTotalBytes: z.number(), + /** + * What could still be allocated without swapping — /proc/meminfo's + * MemAvailable (counts reclaimable page cache), not the naive "free". + */ + memAvailableBytes: z.number(), + /** + * Swap is the freeze mechanism's fuel: frozen sandboxes live there, and + * a full swap means "idle is free" stops being true. totalBytes of 0 is + * an honest reading of a machine with no swap configured (doctor warns + * about it); null means the platform offers no reading at all. + */ + swap: z .object({ - path: z.string(), totalBytes: z.number(), usedBytes: z.number(), - availableBytes: z.number(), }) .nullable(), +}); + +export type HostReading = z.infer; + +/** + * The filesystem holding DORMICE_DATA_DIR — where sandbox disks live. A + * full data disk is the real capacity ceiling (past it, even the ledger + * cannot write). Readers get null when the directory does not exist (fake + * executor, fresh install). + */ +export const dataDiskSchema = z.object({ + path: z.string(), + totalBytes: z.number(), + usedBytes: z.number(), + availableBytes: z.number(), +}); + +export type DataDisk = z.infer; + +/** The ledger's census by lifecycle state. */ +export const sandboxStateCountsSchema = z.object({ + active: z.number().int(), + frozen: z.number().int(), + stopped: z.number().int(), + archived: z.number().int(), + restoring: z.number().int(), +}); + +export type SandboxStateCounts = z.infer; + +/** + * getHostMetrics() — the observation window into the machine itself: is the + * host healthy, and what do the sandboxes collectively cost it? A single + * point-in-time snapshot; for the machine's past see getHostMetricsHistory + * below. Observation never wakes a sandbox and never touches lifecycle. + */ +export const hostMetricsResponseSchema = z.object({ + host: hostReadingSchema, + dataDisk: dataDiskSchema.nullable(), /** Ledger aggregates: what the daemon believes it is running. */ sandboxes: z.object({ total: z.number().int(), maxSandboxes: z.number().int(), - byState: z.object({ - active: z.number().int(), - frozen: z.number().int(), - stopped: z.number().int(), - archived: z.number().int(), - restoring: z.number().int(), - }), + byState: sandboxStateCountsSchema, }), /** * What the sandbox disks cost, from the executor: nominal is the summed diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 4e392316..4d8162f9 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -5,10 +5,12 @@ export * from './config'; export * from './destroy'; export * from './exec'; export * from './files'; +export * from './gateway'; export * from './host'; export * from './images'; export * from './ingress'; export * from './list'; +export * from './lookup'; export * from './metrics'; export * from './policy'; export * from './rebuild'; diff --git a/packages/shared/src/lookup.ts b/packages/shared/src/lookup.ts new file mode 100644 index 00000000..8eb0a281 --- /dev/null +++ b/packages/shared/src/lookup.ts @@ -0,0 +1,43 @@ +import { z } from 'zod'; +import { sandboxNameSchema } from './sandbox'; +import { SANDBOX_STATES } from './states'; + +/** + * lookupSandbox({ name }) / lookupSandbox({ id }) — "do you hold this + * sandbox?", the one question a gateway asks a node on its own account. + * The gateway keeps no directory of where sandboxes live (a second copy of + * a fact the nodes' ledgers already hold, and every copy drifts); when its + * cache has no answer it asks every node this, in parallel, and routes to + * the one that says yes. Read-only by construction: it never wakes, never + * touches the idle clock, never creates. + * + * The node answers inside the name's serialization slot when it must: a + * row that exists answers at once, whatever its state; no row while an + * acquire of that name is in flight waits for the acquire (the daemon + * creates first and writes the row second, both under the slot) and looks + * again — so a gateway retrying a create whose answer was lost never + * places a second copy: while the build is still running the gateway's + * two-second patience runs out first and the caller is told to retry + * (503 with Retry-After); once the row is written, the retry finds the + * sandbox on the node that built it. + */ +export const lookupSandboxRequestSchema = z.union([ + z.object({ name: sandboxNameSchema }), + z.object({ id: z.string().min(1) }), +]); + +export type LookupSandboxRequest = z.infer; + +export const lookupSandboxResponseSchema = z.discriminatedUnion('found', [ + z.object({ + found: z.literal(true), + sandbox: z.object({ + id: z.string(), + name: sandboxNameSchema, + state: z.enum(SANDBOX_STATES), + }), + }), + z.object({ found: z.literal(false) }), +]); + +export type LookupSandboxResponse = z.infer; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3eeb1852..c520a471 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -36,6 +36,9 @@ importers: specifier: ^2.31.0 version: 2.31.0 devDependencies: + '@dormice/gateway': + specifier: workspace:* + version: link:../packages/gateway '@dormice/server': specifier: workspace:* version: link:../packages/server @@ -177,6 +180,43 @@ importers: specifier: ^8.1.3 version: 8.1.3(@types/node@26.1.0)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.0)(yaml@2.9.0) + packages/gateway: + dependencies: + '@dormice/server': + specifier: workspace:* + version: link:../server + '@dormice/shared': + specifier: workspace:* + version: link:../shared + better-sqlite3: + specifier: ^12.11.1 + version: 12.11.1 + drizzle-orm: + specifier: ^0.45.2 + version: 0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.11.1)(kysely@0.28.17) + fastify: + specifier: ^5.10.0 + version: 5.10.0 + fastify-type-provider-zod: + specifier: ^7.0.0 + version: 7.0.0(@fastify/swagger@9.7.0)(fastify@5.10.0)(openapi-types@12.1.3)(zod@4.4.3) + pino: + specifier: ^10.3.1 + version: 10.3.1 + undici: + specifier: ^8.7.0 + version: 8.7.0 + zod: + specifier: ^4.4.3 + version: 4.4.3 + devDependencies: + '@types/better-sqlite3': + specifier: ^7.6.13 + version: 7.6.13 + drizzle-kit: + specifier: ^0.31.10 + version: 0.31.10 + packages/sdk: dependencies: '@dormice/shared':