From c36f01de3886f50d2ba2ec93d34a1e6d00c624b2 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Fri, 11 Sep 2026 22:19:20 +0800 Subject: [PATCH 01/89] The ledger's single-writer lock is held for the daemon's lifetime, not until the next garbage collection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit acquireSingleWriterLock's handle was discarded at boot. better-sqlite3 closes a handle whose object is garbage collected, and a closed handle releases the exclusive file lock — so the "one ledger, one daemon" guarantee lasted only until the first collection: measured 2026-09-11 with the fake executor, a second daemon on the same ledger started and listened seconds after the first. The handle is now kept in main.ts and closed on shutdown; the second instance dies at boot naming the conflict, as intended. --- packages/server/src/main.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index 8eee2cc9..ca800461 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -44,10 +44,15 @@ const config = loadConfig(); // One ledger, one daemon — enforced, not assumed. A second instance would // run its own destructive reconcile against sandboxes this one is still -// operating, well before it ever loses the race for the port. +// operating, well before it ever loses the race for the port. The handle +// is kept for the life of the process: better-sqlite3 closes a handle +// whose object is garbage collected, and a closed handle drops the file +// lock — with the return value discarded, a second daemon on the same +// ledger started fine seconds later (measured 2026-09-11, fake executor). +let ledgerLock: ReturnType | undefined; if (config.DORMICE_DB_PATH !== ':memory:') { try { - acquireSingleWriterLock(config.DORMICE_DB_PATH); + ledgerLock = acquireSingleWriterLock(config.DORMICE_DB_PATH); } catch (error) { fatal(error instanceof Error ? error.message : String(error)); } @@ -352,6 +357,7 @@ const close = async (signal: NodeJS.Signals) => { app.log.error(error, `graceful shutdown after ${signal} failed`); process.exitCode = 1; } + ledgerLock?.close(); process.exit(process.exitCode ?? 0); }; const onSigterm = () => void close('SIGTERM'); From 23b9857907898cb2bcab638d9b3ee1b2d219d137 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:19:08 +0800 Subject: [PATCH 02/89] Server exports its auth, keyed queue, ledger lock and shutdown as subpaths the gateway imports without the executor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway in front of several daemons wants four small self-contained pieces of the daemon — the constant-time token compare, the per-name serialization queue, the single-writer file lock and the bounded shutdown — and nothing else. Four subpath entries expose exactly those; deliberately not the package root, whose import graph loads dockerode, execa and the AWS SDK into any process that touches it. The lock's busy sentence becomes the caller's: the gateway takes the same lock over its own database file and must name its own variable in the exit. auth.ts reads the cookie jar structurally so it type-bundles on its own, without @fastify/cookie's request augmentation in its graph. --- packages/server/package.json | 16 ++++++++++++++++ packages/server/src/auth.ts | 9 ++++++++- packages/server/src/db/lock.ts | 16 +++++++++++----- packages/server/tsup.config.ts | 14 +++++++++++++- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 4596500d..f05001ba 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -13,6 +13,22 @@ "./mini-s3": { "types": "./dist/archive/mini-s3.d.ts", "default": "./dist/archive/mini-s3.js" + }, + "./auth": { + "types": "./dist/auth.d.ts", + "default": "./dist/auth.js" + }, + "./keyed-queue": { + "types": "./dist/keyed-queue.d.ts", + "default": "./dist/keyed-queue.js" + }, + "./lock": { + "types": "./dist/db/lock.d.ts", + "default": "./dist/db/lock.js" + }, + "./shutdown": { + "types": "./dist/shutdown.d.ts", + "default": "./dist/shutdown.js" } }, "files": [ diff --git a/packages/server/src/auth.ts b/packages/server/src/auth.ts index 10cd542f..d1b341a5 100644 --- a/packages/server/src/auth.ts +++ b/packages/server/src/auth.ts @@ -161,7 +161,14 @@ function sessionCookieValid( request: FastifyRequest, getSessionSecret: () => string | null, ): boolean { - const cookie = request.cookies?.[SESSION_COOKIE]; + // The jar exists only where the app registered @fastify/cookie: the + // daemon does; the gateway never mounts the console and passes a getter + // that answers null, so no cookie can pass there. Typed structurally so + // this module stands alone as the `@dormice/server/auth` subpath entry + // without that plugin's request augmentation in its build graph. + const jar = (request as { cookies?: Record }) + .cookies; + const cookie = jar?.[SESSION_COOKIE]; const secret = getSessionSecret(); return Boolean( cookie && diff --git a/packages/server/src/db/lock.ts b/packages/server/src/db/lock.ts index afca59e3..2e086d0c 100644 --- a/packages/server/src/db/lock.ts +++ b/packages/server/src/db/lock.ts @@ -13,8 +13,17 @@ import Database from 'better-sqlite3'; * The lock is an OS-level file lock held by a dedicated SQLite handle in * EXCLUSIVE locking mode: released the instant the process dies, crash * included, so there is no stale-pidfile problem to solve. + * + * The returned handle must be kept for the life of the process: better- + * sqlite3 closes a handle whose object is garbage collected, and a closed + * handle drops the lock (main.ts has the 2026-09-11 measurement). The + * busy sentence is the caller's, because the gateway takes the same lock + * over its own file and must name its own variable in the exit. */ -export function acquireSingleWriterLock(dbPath: string): Database.Database { +export function acquireSingleWriterLock( + dbPath: string, + busyMessage = `another daemon is already running against ${dbPath} — one ledger, one daemon. Stop the other instance, or point this one at its own DORMICE_DB_PATH.`, +): Database.Database { mkdirSync(dirname(dbPath), { recursive: true }); // Fail fast: a held lock answers in 100ms instead of the default 5s wait. const lock = new Database(`${dbPath}.lock`, { timeout: 100 }); @@ -29,10 +38,7 @@ export function acquireSingleWriterLock(dbPath: string): Database.Database { } catch (error) { lock.close(); if ((error as { code?: string }).code === 'SQLITE_BUSY') { - throw new Error( - `another daemon is already running against ${dbPath} — one ledger, one daemon. ` + - 'Stop the other instance, or point this one at its own DORMICE_DB_PATH.', - ); + throw new Error(busyMessage); } throw error; } diff --git a/packages/server/tsup.config.ts b/packages/server/tsup.config.ts index e004f519..49d14560 100644 --- a/packages/server/tsup.config.ts +++ b/packages/server/tsup.config.ts @@ -24,7 +24,19 @@ function git(args: string): string { const commitTime = git('log -1 --format=%cI'); export default defineConfig({ - entry: ['src/index.ts', 'src/main.ts', 'src/archive/mini-s3.ts'], + // Subpath entries beyond the root: mini-s3 for the e2e harness; auth, + // keyed-queue, lock and shutdown for the gateway, which reuses the + // daemon's small self-contained pieces without loading its executor + // (the root's import graph drags dockerode, execa and the AWS SDK in). + entry: [ + 'src/index.ts', + 'src/main.ts', + 'src/archive/mini-s3.ts', + 'src/auth.ts', + 'src/keyed-queue.ts', + 'src/db/lock.ts', + 'src/shutdown.ts', + ], format: ['esm'], dts: true, clean: true, From 6bd61ab8aea3e5e80102fbeb4de8cb96aecf1d76 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:22:02 +0800 Subject: [PATCH 03/89] lookupSandbox: a node answers "do you hold this sandbox?" by name or id, inside the name's slot when a create is in flight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway keeps no directory of where sandboxes live — a second copy of a fact the nodes' ledgers already hold, and every copy drifts. When its cache has no answer it asks every node this one read-only question in parallel and routes to the node that says yes. Three steps make the answer truthful about a create in flight. A row that exists answers at once, whatever its state: a restoring sandbox has a row, and waiting for its slot would hold the answer for the whole restore, long past the gateway's two-second patience. No row while the name's slot is busy waits its turn and looks again — the daemon creates first and writes the row second, both under the slot, so a gateway retrying a create whose answer was lost finds the sandbox on the node that built it and never places a second copy. No row and a free slot is a plain no. By id there is no slot to wait on and none is needed: nobody can ask about an id before the create that minted it has answered. The in-flight test parks the fake executor's create to hold the acquire mid-build; removing the wait turns it red. --- packages/server/src/app.test.ts | 97 +++++++++++++++++++++++++ packages/server/src/routes/sandboxes.ts | 45 +++++++++++- packages/shared/src/index.ts | 1 + packages/shared/src/lookup.ts | 40 ++++++++++ 4 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 packages/shared/src/lookup.ts diff --git a/packages/server/src/app.test.ts b/packages/server/src/app.test.ts index 517fbf44..555e2b8f 100644 --- a/packages/server/src/app.test.ts +++ b/packages/server/src/app.test.ts @@ -2358,3 +2358,100 @@ describe('activity attribution', () => { expect((await eventOf(app, 'apikey-created'))?.actor).toBe('console'); }); }); + +describe('POST /lookupSandbox', () => { + it('answers by name and by id with the state, without waking or touching the idle clock', async () => { + const { app, db } = testApp(); + const created = (await acquire(app, { name: 'alice' })).json(); + const row = findById(db, created.sandbox.id); + if (!row) throw new Error('no row'); + // A cold sandbox stays cold: lookup is observation, not use. + transition(db, row.id, 'frozen'); + + const byName = await rpc(app, '/lookupSandbox', { name: 'alice' }); + expect(byName.statusCode).toBe(200); + expect(byName.json()).toEqual({ + found: true, + sandbox: { id: row.id, name: 'alice', state: 'frozen' }, + }); + const byId = await rpc(app, '/lookupSandbox', { id: row.id }); + expect(byId.json()).toEqual(byName.json()); + expect(findById(db, row.id)?.state).toBe('frozen'); + expect(findById(db, row.id)?.lastActiveAt).toBe(row.lastActiveAt); + + expect( + (await rpc(app, '/lookupSandbox', { name: 'nobody' })).json(), + ).toEqual({ found: false }); + expect( + (await rpc(app, '/lookupSandbox', { id: 'no-such-id' })).json(), + ).toEqual({ found: false }); + // Neither a name nor an id is not a question. + expect((await rpc(app, '/lookupSandbox', {})).statusCode).toBe(400); + }); + + it('a name whose slot is busy waits its turn: asked while an acquire is mid-create, it answers found once the row exists', async () => { + // A create that parks inside the executor — the daemon's own shape of + // "in flight": the acquire holds the name's slot, the container is + // being built, the row is not written yet. + let release: () => void = () => {}; + let inCreate: () => void = () => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + const reached = new Promise((resolve) => { + inCreate = resolve; + }); + class ParkedCreate extends FakeExecutor { + override async create( + ...args: Parameters + ): Promise { + inCreate(); + await gate; + return super.create(...args); + } + } + const { app } = testApp(new ParkedCreate()); + const creating = acquire(app, { name: 'alice' }); + await reached; + // No row, slot busy: the question must wait, not answer "no". + const asked = rpc(app, '/lookupSandbox', { name: 'alice' }); + const early = await Promise.race([ + asked.then(() => 'answered'), + new Promise((resolve) => setTimeout(() => resolve('pending'), 50)), + ]); + expect(early).toBe('pending'); + release(); + const created = (await creating).json(); + expect((await asked).json()).toEqual({ + found: true, + sandbox: { id: created.sandbox.id, name: 'alice', state: 'active' }, + }); + }); + + it('a sandbox with a row answers at once even while its slot is held — a restore in progress must not read as silence', async () => { + const { app, db, locks } = testApp(); + const created = (await acquire(app, { name: 'alice' })).json(); + transition(db, created.sandbox.id, 'frozen'); + transition(db, created.sandbox.id, 'stopped'); + transition(db, created.sandbox.id, 'archived'); + transition(db, created.sandbox.id, 'restoring'); + let release: () => void = () => {}; + const held = locks.run( + 'alice', + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const answer = await Promise.race([ + rpc(app, '/lookupSandbox', { name: 'alice' }).then((r) => r.json()), + new Promise((resolve) => setTimeout(() => resolve('pending'), 500)), + ]); + expect(answer).toEqual({ + found: true, + sandbox: { id: created.sandbox.id, name: 'alice', state: 'restoring' }, + }); + release(); + await held; + }); +}); diff --git a/packages/server/src/routes/sandboxes.ts b/packages/server/src/routes/sandboxes.ts index 60b93c94..739f0e9c 100644 --- a/packages/server/src/routes/sandboxes.ts +++ b/packages/server/src/routes/sandboxes.ts @@ -19,6 +19,8 @@ import { listSandboxImagesResponseSchema, listSandboxMetricsRequestSchema, listSandboxMetricsResponseSchema, + lookupSandboxRequestSchema, + lookupSandboxResponseSchema, READ_FILES_TOTAL_LIMIT_BYTES, readFileRequestSchema, readFileResponseSchema, @@ -54,6 +56,7 @@ import type { Db } from '../db/db'; import { countSandboxes, createSandbox, + findById, findByName, listSandboxes, setDiskGb, @@ -81,7 +84,7 @@ import { NotAFileError, } from '../executor/executor'; import { httpError } from '../http-error'; -import type { KeyedQueue } from '../keyed-queue'; +import { type KeyedQueue, SKIPPED } from '../keyed-queue'; import { destroySandbox, rebuildSandbox, wakeSandbox } from '../lifecycle'; import { ArchiveDisabledError, resolvePolicy } from '../policy'; import { resolveSpec } from '../spec'; @@ -1114,6 +1117,46 @@ export const sandboxRoutes: FastifyPluginAsyncZod< }, ); + // The gateway's one question on its own account: does this node hold + // the sandbox? Read-only — never wakes, never touches the idle clock — + // and truthful about a create in flight, in three steps. A row that + // exists answers at once, whatever its state: a restoring sandbox has a + // row, and waiting for its slot would hold the answer for the whole + // restore, long past the gateway's two-second patience — the gateway + // would read a live sandbox as a node that did not answer. No row while + // the name's slot is busy means an acquire may be writing the row right + // now (create first, row second, both under the slot), so the answer + // waits its turn behind it and looks again. No row and a free slot is a + // plain no. By id there is no slot to wait on (slots are keyed by name), + // and none is needed: nobody can ask about an id before the create that + // minted it has answered. + app.post( + '/lookupSandbox', + { + schema: { + body: lookupSandboxRequestSchema, + response: { 200: lookupSandboxResponseSchema }, + }, + }, + async (request) => { + const query = request.body; + const look = () => + 'name' in query ? findByName(db, query.name) : findById(db, query.id); + const answer = (row: SandboxRow | undefined) => + row + ? { + found: true as const, + sandbox: { id: row.id, name: row.name, state: row.state }, + } + : { found: false as const }; + const now = look(); + if (now !== undefined || !('name' in query)) return answer(now); + const unheld = await locks.tryRun(query.name, async () => look()); + if (unheld !== SKIPPED) return answer(unheld); + return answer(await locks.run(query.name, async () => look())); + }, + ); + app.post( '/destroySandbox', { diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 4e392316..e31684ce 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -9,6 +9,7 @@ export * from './host'; export * from './images'; export * from './ingress'; export * from './list'; +export * from './lookup'; export * from './metrics'; export * from './policy'; export * from './rebuild'; diff --git a/packages/shared/src/lookup.ts b/packages/shared/src/lookup.ts new file mode 100644 index 00000000..c449b0af --- /dev/null +++ b/packages/shared/src/lookup.ts @@ -0,0 +1,40 @@ +import { z } from 'zod'; +import { sandboxNameSchema } from './sandbox'; +import { SANDBOX_STATES } from './states'; + +/** + * lookupSandbox({ name }) / lookupSandbox({ id }) — "do you hold this + * sandbox?", the one question a gateway asks a node on its own account. + * The gateway keeps no directory of where sandboxes live (a second copy of + * a fact the nodes' ledgers already hold, and every copy drifts); when its + * cache has no answer it asks every node this, in parallel, and routes to + * the one that says yes. Read-only by construction: it never wakes, never + * touches the idle clock, never creates. + * + * The node answers inside the name's serialization slot when it must: a + * row that exists answers at once, whatever its state; no row while an + * acquire of that name is in flight waits for the acquire (the daemon + * creates first and writes the row second, both under the slot) and looks + * again — so a gateway retrying a create whose answer was lost finds the + * sandbox on the node that built it, and never places a second copy. + */ +export const lookupSandboxRequestSchema = z.union([ + z.object({ name: sandboxNameSchema }), + z.object({ id: z.string().min(1) }), +]); + +export type LookupSandboxRequest = z.infer; + +export const lookupSandboxResponseSchema = z.discriminatedUnion('found', [ + z.object({ + found: z.literal(true), + sandbox: z.object({ + id: z.string(), + name: sandboxNameSchema, + state: z.enum(SANDBOX_STATES), + }), + }), + z.object({ found: z.literal(false) }), +]); + +export type LookupSandboxResponse = z.infer; From bbfb3a810a0313c813dd898816227d42304ece59 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:26:04 +0800 Subject: [PATCH 04/89] A node checks in with its gateway every 15 seconds: readings, build and where it can be reached MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DORMICE_GATEWAY_ENDPOINT makes a daemon a node of a fleet. Set, it POSTs /checkIn to the gateway every DORMICE_CHECK_IN_INTERVAL_SECONDS with its id, the address the gateway may forward to (DORMICE_NODE_ENDPOINT, defaulting to its own loopback — right when gateway and node share a machine), its build and a fresh reading: CPU, memory, swap, the data disk, the ledger's census by state. Unset, the daemon is the whole platform by itself, as before. The token is the one DORMICE_API_TOKEN gateway and nodes share. Who knows the truth speaks: the node knows what it runs and where it lives, so the node reports and the gateway only listens. The gateway learns of a node from its first check-in — no registration verb, no nodes file — and reads two missed check-ins as down; the interval travels in every check-in so both ends measure with the same number. Failures are logged on the change, never every tick, and are never fatal: the gateway is the fleet's front door, not the node's reason to live. The shared host schema is split into its named parts (host reading, data disk, state counts) so getHostMetrics and the check-in describe the machine with one vocabulary; readHostReading is the one function both read through. --- packages/server/src/check-in.test.ts | 179 +++++++++++++++++++++++++++ packages/server/src/check-in.ts | 135 ++++++++++++++++++++ packages/server/src/config.test.ts | 33 +++++ packages/server/src/config.ts | 48 +++++++ packages/server/src/host-metrics.ts | 26 ++++ packages/server/src/main.ts | 32 +++++ packages/server/src/routes/host.ts | 14 +-- packages/shared/src/gateway.ts | 124 +++++++++++++++++++ packages/shared/src/host.ts | 111 ++++++++++------- packages/shared/src/index.ts | 1 + 10 files changed, 643 insertions(+), 60 deletions(-) create mode 100644 packages/server/src/check-in.test.ts create mode 100644 packages/server/src/check-in.ts create mode 100644 packages/shared/src/gateway.ts diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts new file mode 100644 index 00000000..6cbcc69a --- /dev/null +++ b/packages/server/src/check-in.test.ts @@ -0,0 +1,179 @@ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { fileURLToPath } from 'node:url'; +import { checkInRequestSchema } from '@dormice/shared'; +import { afterEach, describe, expect, it } from 'vitest'; +import { CheckIn, type CheckInOptions, readNodeReading } from './check-in'; +import { migrateDb, openDb } from './db/db'; +import { createSandbox } from './db/ledger'; +import { CpuSampler } from './host-metrics'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const TOKEN = 'shared-token-shared-token-shared-token'; + +const servers: http.Server[] = []; +afterEach(async () => { + await Promise.all( + servers + .splice(0) + .map((s) => new Promise((resolve) => s.close(() => resolve()))), + ); +}); + +/** A gateway-shaped listener: records every check-in, answers what the test says. */ +async function gateway(answer: () => { status: number; body: string }) { + const seen: Array<{ headers: http.IncomingHttpHeaders; body: unknown }> = []; + const server = http.createServer((req, res) => { + let text = ''; + req.on('data', (chunk) => { + text += chunk; + }); + req.on('end', () => { + seen.push({ headers: req.headers, body: JSON.parse(text) }); + const a = answer(); + res.writeHead(a.status, { 'content-type': 'application/json' }); + res.end(a.body); + }); + }); + servers.push(server); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return { + endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + seen, + }; +} + +function logSpy() { + const infos: string[] = []; + const warns: string[] = []; + return { + infos, + warns, + log: { + info: (msg: string) => infos.push(msg), + warn: (_obj: unknown, msg: string) => warns.push(msg), + }, + }; +} + +function options( + gatewayEndpoint: string, + log: CheckInOptions['log'], + over: Partial = {}, +): CheckInOptions { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const cpu = new CpuSampler(); + return { + gateway: gatewayEndpoint, + token: TOKEN, + nodeId: 'node-7', + endpoint: 'http://10.0.0.7:80', + intervalSeconds: 1, + build: { + commit: 'abc1234', + title: 'a commit', + committedAt: '2026-09-14T00:00:00.000Z', + }, + readReading: () => readNodeReading(db, cpu, '/nonexistent-data-dir'), + log, + ...over, + }; +} + +describe('CheckIn', () => { + it('posts a check-in the gateway can parse: shared token, id, endpoint, interval, build, reading', async () => { + const gw = await gateway(() => ({ status: 200, body: '{}' })); + const { log, warns } = logSpy(); + await new CheckIn(options(gw.endpoint, log)).once(); + expect(warns).toEqual([]); + expect(gw.seen).toHaveLength(1); + expect(gw.seen[0]?.headers.authorization).toBe(`Bearer ${TOKEN}`); + const body = checkInRequestSchema.parse(gw.seen[0]?.body); + expect(body.nodeId).toBe('node-7'); + expect(body.endpoint).toBe('http://10.0.0.7:80'); + expect(body.intervalSeconds).toBe(1); + expect(body.build?.commit).toBe('abc1234'); + expect(body.reading.host.cpuCount).toBeGreaterThan(0); + // The first sample has no delta, and a data dir that does not exist is + // an honest null, never a made-up disk. + expect(body.reading.host.cpuUsedPct).toBeNull(); + expect(body.reading.dataDisk).toBeNull(); + expect(body.reading.sandboxes).toEqual({ + total: 0, + byState: { active: 0, frozen: 0, stopped: 0, archived: 0, restoring: 0 }, + }); + }); + + it('the reading counts the ledger by state', async () => { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + for (const name of ['a', 'b']) { + createSandbox(db, { + id: `id-${name}`, + name, + nodeId: 'node-7', + policy: { + freezeAfterSeconds: 60, + stopAfterSeconds: null, + archiveAfterSeconds: null, + }, + template: null, + metadata: null, + spec: undefined, + actor: null, + }); + } + const reading = await readNodeReading(db, new CpuSampler(), '/tmp'); + expect(reading.sandboxes.total).toBe(2); + expect(reading.sandboxes.byState.active).toBe(2); + expect(reading.dataDisk?.path).toBe('/tmp'); + }); + + it('logs a failing gateway once, and its recovery once — not every tick', async () => { + let status = 500; + const gw = await gateway(() => ({ status, body: '{"message":"boom"}' })); + const { log, warns, infos } = logSpy(); + const checkIn = new CheckIn(options(gw.endpoint, log)); + await checkIn.once(); + await checkIn.once(); + expect(gw.seen).toHaveLength(2); + expect(warns).toHaveLength(1); + expect(warns[0]).toMatch(/check-in failed/); + status = 200; + await checkIn.once(); + await checkIn.once(); + expect(infos).toEqual([ + `check-in with gateway ${gw.endpoint} answers again`, + ]); + expect(warns).toHaveLength(1); + // A gateway that refuses the token is the same one event. + status = 401; + await checkIn.once(); + await checkIn.once(); + expect(warns).toHaveLength(2); + }); + + it('a gateway that is not there is a logged failure, never a throw', async () => { + const { log, warns } = logSpy(); + const checkIn = new CheckIn(options('http://127.0.0.1:9', log)); + await expect(checkIn.once()).resolves.toBeUndefined(); + expect(warns).toHaveLength(1); + }); + + it('ticks on its interval from start() and stops on stop()', async () => { + const gw = await gateway(() => ({ status: 200, body: '{}' })); + const { log } = logSpy(); + const checkIn = new CheckIn(options(gw.endpoint, log)); + checkIn.start(); + const deadline = Date.now() + 5_000; + while (gw.seen.length < 2 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + expect(gw.seen.length).toBeGreaterThanOrEqual(2); + checkIn.stop(); + const afterStop = gw.seen.length; + await new Promise((resolve) => setTimeout(resolve, 1_200)); + expect(gw.seen.length).toBe(afterStop); + }); +}); diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts new file mode 100644 index 00000000..1fda3f3e --- /dev/null +++ b/packages/server/src/check-in.ts @@ -0,0 +1,135 @@ +import { + type BuildInfo, + type CheckInRequest, + checkInResponseSchema, + type NodeReading, +} from '@dormice/shared'; +import type { Db } from './db/db'; +import { countByState, listSandboxes } from './db/ledger'; +import { type CpuSampler, readHostReading } from './host-metrics'; + +/** + * A node's reading for its check-in: the host half (host-metrics.ts) and + * the ledger's census. The same numbers getHostMetrics answers a caller + * with, minus the daemon-local knobs no gateway places by. + */ +export async function readNodeReading( + db: Db, + cpu: CpuSampler, + dataDir: string, +): Promise { + const { byState, total } = countByState(listSandboxes(db)); + return { + ...(await readHostReading(cpu, dataDir)), + sandboxes: { total, byState }, + }; +} + +export interface CheckInLog { + info(msg: string): void; + warn(obj: unknown, msg: string): void; +} + +export interface CheckInOptions { + /** DORMICE_GATEWAY_ENDPOINT. */ + gateway: string; + /** The token gateway and nodes share (DORMICE_API_TOKEN). */ + token: string; + nodeId: string; + /** Where the gateway reaches this node (DORMICE_NODE_ENDPOINT or the loopback default). */ + endpoint: string; + intervalSeconds: number; + build: BuildInfo | null; + readReading: () => Promise; + log: CheckInLog; + /** Test seam; production uses the platform's fetch. */ + fetchImpl?: typeof fetch; +} + +/** A gateway that has not answered within this is a gateway not answering; the next tick tries again. */ +const CHECK_IN_TIMEOUT_MS = 10_000; + +/** + * The node's check-in ticker: every interval, one POST /checkIn to the + * gateway carrying the node's id, where it can be reached, its build and + * a fresh reading (RULES/协议.md「网关」). The gateway learns of a node from + * its first check-in — no registration verb, no nodes file — and reads + * two missed check-ins as down. + * + * Chained setTimeout, the daemon's discipline: the next tick is scheduled + * when this one is done, so a slow gateway never has ticks pile up. + * Failures are logged on the change — once when the gateway stops + * answering, once when it answers again — never every tick: a gateway + * down for an hour is one event, not two hundred and forty lines. Never + * fatal: the gateway is the fleet's front door and configuration + * authority, not the node's reason to live; the node keeps running its + * sandboxes and keeps trying. + */ +export class CheckIn { + private timer: NodeJS.Timeout | undefined; + private closing = false; + /** The failure the gateway is currently in, or null while it answers. */ + private failing: string | null = null; + + constructor(private readonly opts: CheckInOptions) {} + + start(): void { + this.schedule(0); + } + + stop(): void { + this.closing = true; + clearTimeout(this.timer); + } + + /** One check-in. Never throws: a failure is recorded and the next tick retries. */ + async once(): Promise { + const { opts } = this; + try { + const body: CheckInRequest = { + nodeId: opts.nodeId, + endpoint: opts.endpoint, + intervalSeconds: opts.intervalSeconds, + build: opts.build, + reading: await opts.readReading(), + }; + const res = await (opts.fetchImpl ?? fetch)(`${opts.gateway}/checkIn`, { + method: 'POST', + headers: { + authorization: `Bearer ${opts.token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(CHECK_IN_TIMEOUT_MS), + }); + if (res.status !== 200) { + const text = await res.text(); + throw new Error( + `gateway answered ${res.status}: ${text.slice(0, 200)}`, + ); + } + checkInResponseSchema.parse(await res.json()); + if (this.failing !== null) { + opts.log.info(`check-in with gateway ${opts.gateway} answers again`); + this.failing = null; + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (this.failing === null) { + opts.log.warn( + { gateway: opts.gateway, error: message }, + 'check-in failed; the gateway places nothing here and forwards no new names to this node until it answers again — retrying every interval', + ); + } + this.failing = message; + } + } + + private schedule(delayMs: number): void { + if (this.closing) return; + this.timer = setTimeout(async () => { + await this.once(); + this.schedule(this.opts.intervalSeconds * 1000); + }, delayMs); + } +} diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index 90fea421..083f071c 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -151,3 +151,36 @@ describe('the S3 set', () => { expect(off.DORMICE_S3_FORCE_PATH_STYLE).toBe(false); }); }); + +describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { + it('defaults: no gateway (standalone), no node endpoint, 15s check-in', () => { + const config = loadConfig(TOKEN); + expect(config.DORMICE_GATEWAY_ENDPOINT).toBeUndefined(); + expect(config.DORMICE_NODE_ENDPOINT).toBeUndefined(); + expect(config.DORMICE_CHECK_IN_INTERVAL_SECONDS).toBe(15); + }); + + it('parses both endpoints as full URLs and drops a trailing slash', () => { + const config = loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677/', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80///', + DORMICE_CHECK_IN_INTERVAL_SECONDS: '5', + }); + expect(config.DORMICE_GATEWAY_ENDPOINT).toBe('http://10.0.0.5:3677'); + expect(config.DORMICE_NODE_ENDPOINT).toBe('http://10.0.0.7:80'); + expect(config.DORMICE_CHECK_IN_INTERVAL_SECONDS).toBe(5); + }); + + it('refuses an endpoint without a scheme, naming the variable', () => { + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_ENDPOINT: '10.0.0.5:3677' }), + ).toThrow(/DORMICE_GATEWAY_ENDPOINT must be a full http\(s\) URL/); + expect(() => + loadConfig({ ...TOKEN, DORMICE_NODE_ENDPOINT: 'node-7' }), + ).toThrow(/DORMICE_NODE_ENDPOINT must be a full http\(s\) URL/); + expect(() => + loadConfig({ ...TOKEN, DORMICE_CHECK_IN_INTERVAL_SECONDS: '0' }), + ).toThrow(); + }); +}); diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index 2bc2e96d..acd03501 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -169,6 +169,51 @@ const envSchema = z.object({ DORMICE_S3_REGION: z.string().default('us-east-1'), /** Path-style addressing: MinIO needs true; the clouds route by subdomain. */ DORMICE_S3_FORCE_PATH_STYLE: z.stringbool().default(false), + /** + * The gateway this daemon is a node of — its intranet address, e.g. + * http://10.0.0.5:3677. Set, the daemon checks in with it every + * DORMICE_CHECK_IN_INTERVAL_SECONDS (check-in.ts): its readings, its + * build, and where it can be reached. That check-in is the gateway's + * only source of "which nodes exist and how full are they" — no + * registration, no nodes file. Unset, the daemon is the whole platform + * by itself, as it always was, and checks in with nobody. The token it + * presents is DORMICE_API_TOKEN: gateway and nodes share one, and the + * gateway speaks to every node with the same one. + */ + DORMICE_GATEWAY_ENDPOINT: z + .url({ + protocol: /^https?$/, + error: + 'DORMICE_GATEWAY_ENDPOINT must be a full http(s) URL, e.g. http://10.0.0.5:3677', + }) + .transform((url) => url.replace(/\/+$/, '')) + .optional(), + /** + * Where the gateway reaches this node — the address it forwards to. + * Default: this daemon's own loopback address, right when gateway and + * node share a machine (the single-machine install is a fleet of one). + * On a machine of its own the daemon still binds loopback (the red + * line), so this names the front the gateway may dial — the node's + * Caddy on the intranet interface, e.g. http://10.0.0.7:80. + */ + DORMICE_NODE_ENDPOINT: z + .url({ + protocol: /^https?$/, + error: + 'DORMICE_NODE_ENDPOINT must be a full http(s) URL, e.g. http://10.0.0.7:80', + }) + .transform((url) => url.replace(/\/+$/, '')) + .optional(), + /** + * How often the node checks in with its gateway. The gateway reads two + * missed check-ins as down — the one number both ends of that wire + * share, so the node states it in every check-in. + */ + DORMICE_CHECK_IN_INTERVAL_SECONDS: z.coerce + .number() + .int() + .positive() + .default(15), }); const checkedSchema = envSchema @@ -261,6 +306,9 @@ export const CONFIG_KEYS: Record = { DORMICE_S3_SECRET_ACCESS_KEY: { sensitive: true }, DORMICE_S3_REGION: { sensitive: false }, DORMICE_S3_FORCE_PATH_STYLE: { sensitive: false }, + DORMICE_GATEWAY_ENDPOINT: { sensitive: false }, + DORMICE_NODE_ENDPOINT: { sensitive: false }, + DORMICE_CHECK_IN_INTERVAL_SECONDS: { sensitive: false }, }; export type ConfigSources = Record; diff --git a/packages/server/src/host-metrics.ts b/packages/server/src/host-metrics.ts index d629146c..5a768d7e 100644 --- a/packages/server/src/host-metrics.ts +++ b/packages/server/src/host-metrics.ts @@ -1,5 +1,6 @@ import { readFile, statfs } from 'node:fs/promises'; import os from 'node:os'; +import type { NodeReading } from '@dormice/shared'; /** * Host-side readings for getHostMetrics — the machine's own health, which @@ -158,3 +159,28 @@ export async function readDiskSpace( availableBytes: s.bavail * s.bsize, }; } + +/** + * The host half of a node's reading — what getHostMetrics answers and + * what a check-in reports, from one function so the two never disagree on + * a field: the machine's cores and CPU delta, its memory and swap, and + * the data disk that holds the sandbox disks (null until it exists). The + * CpuSampler is the caller's: a delta spans "since this instance's last + * sample", so each reader owns one (host-metrics route, metrics sampler, + * check-in) and none steals another's window. + */ +export async function readHostReading( + cpu: CpuSampler, + dataDir: string, +): Promise> { + const memory = await readHostMemory(); + const disk = await readDiskSpace(dataDir); + return { + host: { + cpuCount: os.cpus().length, + cpuUsedPct: cpu.sample(), + ...memory, + }, + dataDisk: disk ? { path: dataDir, ...disk } : null, + }; +} diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index ca800461..90b02f3d 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -3,6 +3,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { pino } from 'pino'; import { buildApp } from './app'; +import { CheckIn, readNodeReading } from './check-in'; import { Archiver } from './archive/archiver'; import { LedgerArchiveStore } from './archive/ledger-store'; import { type Config, loadConfig } from './config'; @@ -314,6 +315,36 @@ recordActivity(db, { // the daemon to the outside world is a reverse proxy's job. await app.listen({ host: '127.0.0.1', port: config.DORMICE_PORT }); +// A node of a fleet reports to its gateway; a daemon on its own reports to +// nobody. Started after listen on purpose: the check-in names where the +// gateway may forward to, and that door must be open before the gateway +// hears of it. Its CpuSampler is its own — a delta spans "since this +// instance's last sample", and the route's and the metrics ticker's +// windows must not be stolen (host-metrics.ts). +let checkIn: CheckIn | undefined; +if (config.DORMICE_GATEWAY_ENDPOINT !== undefined) { + const nodeEndpoint = + config.DORMICE_NODE_ENDPOINT ?? `http://127.0.0.1:${config.DORMICE_PORT}`; + const checkInCpu = new CpuSampler(); + checkInCpu.sample(); + checkIn = new CheckIn({ + gateway: config.DORMICE_GATEWAY_ENDPOINT, + token: config.DORMICE_API_TOKEN, + nodeId: config.DORMICE_NODE_ID, + endpoint: nodeEndpoint, + intervalSeconds: config.DORMICE_CHECK_IN_INTERVAL_SECONDS, + build, + readReading: () => readNodeReading(db, checkInCpu, config.DORMICE_DATA_DIR), + log, + }); + checkIn.start(); + log.info( + `node ${config.DORMICE_NODE_ID} checks in with gateway ${config.DORMICE_GATEWAY_ENDPOINT} every ${config.DORMICE_CHECK_IN_INTERVAL_SECONDS}s, reachable at ${nodeEndpoint}`, + ); +} else { + log.info('no gateway: standalone daemon (DORMICE_GATEWAY_ENDPOINT unset)'); +} + // systemd stops the daemon with SIGTERM. Shutdown is bounded on purpose // (shutdown.ts has the measurements): close the app — preClose ends the // long-lived streams with honest end-frames, the listener stops — give @@ -341,6 +372,7 @@ const close = async (signal: NodeJS.Signals) => { process.removeListener('SIGINT', onSigint); clearTimeout(heartbeatTimer); clearTimeout(metricsTimer); + checkIn?.stop(); watchdog.stop(); app.log.info( `${signal} received — shutting down (grace ${SHUTDOWN_GRACE_MS}ms)`, diff --git a/packages/server/src/routes/host.ts b/packages/server/src/routes/host.ts index ca733c19..d7e7d1be 100644 --- a/packages/server/src/routes/host.ts +++ b/packages/server/src/routes/host.ts @@ -1,4 +1,3 @@ -import os from 'node:os'; import { getFleetTimelineRequestSchema, getFleetTimelineResponseSchema, @@ -22,7 +21,7 @@ import { } from '../db/metrics'; import { readRuntimeSettings } from '../db/settings'; import type { Executor } from '../executor/executor'; -import { CpuSampler, readDiskSpace, readHostMemory } from '../host-metrics'; +import { CpuSampler, readHostReading } from '../host-metrics'; export interface HostRoutesOptions { config: Config; @@ -58,17 +57,8 @@ export const hostRoutes: FastifyPluginAsyncZod = async ( const rows = listSandboxes(db); const { byState, total } = countByState(rows); - const memory = await readHostMemory(); - const dataDisk = await readDiskSpace(config.DORMICE_DATA_DIR); return { - host: { - cpuCount: os.cpus().length, - cpuUsedPct: cpu.sample(), - ...memory, - }, - dataDisk: dataDisk - ? { path: config.DORMICE_DATA_DIR, ...dataDisk } - : null, + ...(await readHostReading(cpu, config.DORMICE_DATA_DIR)), sandboxes: { total, // The ledger's live knob, not the env seed — the console edits it. diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts new file mode 100644 index 00000000..11eb6cf8 --- /dev/null +++ b/packages/shared/src/gateway.ts @@ -0,0 +1,124 @@ +import { z } from 'zod'; +import { + dataDiskSchema, + hostReadingSchema, + sandboxStateCountsSchema, +} from './host'; + +/** + * The gateway's wire: the verbs between a node and the gateway that fronts + * it, and the gateway's own observation verbs. A fleet is N daemons that + * know nothing of each other behind one gateway that holds no sandbox + * state — only what the nodes tell it every few seconds and what its own + * configuration tables say. A single machine runs the same two processes + * and is a fleet of one. + */ + +/** The identity a build carries: the commit its dist was built from. Null where the dist was built outside a checkout. */ +export const buildInfoSchema = z.object({ + /** Short hash. */ + commit: z.string(), + /** The commit's subject line. */ + title: z.string(), + /** ISO 8601 UTC — the commit's time, not the build's. */ + committedAt: z.iso.datetime(), +}); + +export type BuildInfo = z.infer; + +/** + * What a node reports about itself at every check-in — everything + * placement decides on: the machine's CPU, memory and data disk, and the + * ledger's census by state. The same host reading getHostMetrics answers + * (host.ts), minus the daemon-local knobs no gateway places by. + */ +export const nodeReadingSchema = z.object({ + host: hostReadingSchema, + dataDisk: dataDiskSchema.nullable(), + sandboxes: z.object({ + total: z.number().int(), + byState: sandboxStateCountsSchema, + }), +}); + +export type NodeReading = z.infer; + +/** + * checkIn — a node reporting for duty, every DORMICE_CHECK_IN_INTERVAL_SECONDS + * (15 by default), authenticated with the token the gateway and every node + * share. The gateway learns of a node from its first check-in: there is no + * registration verb and no nodes file — "which nodes exist" has one home, + * the gateway's nodes table, written by the nodes themselves. Two missed + * check-ins read as down: no new sandbox is placed there and its sandboxes + * answer 502 until it reports again. Who knows the truth speaks: the node + * knows what it runs and where it can be reached; the gateway only listens + * and compares, and never keeps a record of what it told whom. + */ +export const checkInRequestSchema = z.object({ + /** DORMICE_NODE_ID — the node's name in every sandbox's `nodeId`. */ + nodeId: z.string().min(1), + /** Where the gateway forwards to: the node's intranet front (DORMICE_NODE_ENDPOINT). */ + endpoint: z.url({ protocol: /^https?$/ }), + /** How often this node checks in — the gateway's yardstick for "missed two in a row". */ + intervalSeconds: z.number().int().positive(), + build: buildInfoSchema.nullable(), + reading: nodeReadingSchema, +}); + +export type CheckInRequest = z.infer; + +/** Nothing yet: the configuration version the gateway will answer with arrives with the configuration authority. */ +export const checkInResponseSchema = z.object({}); + +export type CheckInResponse = z.infer; + +/** + * listNodes — every node the gateway has ever heard from, with what it + * last said. Everything here is what the gateway already holds; answering + * costs no node anything. + */ +export const listNodesRequestSchema = z.object({}); + +export const nodeViewSchema = z.object({ + id: z.string(), + endpoint: z.string(), + /** ISO 8601 UTC — the first check-in. */ + addedAt: z.iso.datetime(), + /** ISO 8601 UTC — null only right after a gateway start, before the node's next check-in. */ + lastCheckInAt: z.iso.datetime().nullable(), + intervalSeconds: z.number().int().positive().nullable(), + /** Checked in within two of its own intervals. */ + reachable: z.boolean(), + build: buildInfoSchema.nullable(), + reading: nodeReadingSchema.nullable(), + /** Sandboxes the gateway placed here since the last check-in — counted against the node until the next reading shows them. */ + placedSinceCheckIn: z.number().int().nonnegative(), +}); + +export type NodeView = z.infer; + +export const listNodesResponseSchema = z.object({ + nodes: z.array(nodeViewSchema), +}); + +export type ListNodesResponse = z.infer; + +/** + * removeNode — the operator's word that a node is gone for good: its row + * goes, its sandboxes are no longer looked for, and a name that lived only + * there is a new name again. A node that is merely down needs nothing — + * it is back the moment it checks in — and one removed by mistake re-adds + * itself the same way. + */ +export const removeNodeRequestSchema = z.object({ + id: z.string().min(1), +}); + +export type RemoveNodeRequest = z.infer; + +export const removeNodeResponseSchema = z.object({ + /** True when a row existed and was removed; false when there was none. */ + removed: z.boolean(), +}); + +export type RemoveNodeResponse = z.infer; diff --git a/packages/shared/src/host.ts b/packages/shared/src/host.ts index ad695f5a..3eeeefa7 100644 --- a/packages/shared/src/host.ts +++ b/packages/shared/src/host.ts @@ -1,68 +1,83 @@ import { z } from 'zod'; /** - * getHostMetrics() — the observation window into the machine itself: is the - * host healthy, and what do the sandboxes collectively cost it? A single - * point-in-time snapshot; for the machine's past see getHostMetricsHistory - * below. Observation never wakes a sandbox and never touches lifecycle. + * The machine's own readings — the `host` half of getHostMetrics and of a + * node's check-in (gateway.ts nodeReadingSchema), one schema so the two + * never disagree on a field. * * Readings a platform cannot produce are null, honestly — never zero, never - * invented: swap and /proc are Linux facts, and the data directory only - * exists where the docker executor runs. + * invented: swap and /proc are Linux facts. */ -export const hostMetricsResponseSchema = z.object({ - host: z.object({ - cpuCount: z.number().int().positive(), - /** - * Percent of the whole machine, 0-100. Null until the sampler has two - * samples to take a delta between — the first request after daemon - * start reports "don't know yet", not a made-up 0. - */ - cpuUsedPct: z.number().nullable(), - memTotalBytes: z.number(), - /** - * What could still be allocated without swapping — /proc/meminfo's - * MemAvailable (counts reclaimable page cache), not the naive "free". - */ - memAvailableBytes: z.number(), - /** - * Swap is the freeze mechanism's fuel: frozen sandboxes live there, and - * a full swap means "idle is free" stops being true. totalBytes of 0 is - * an honest reading of a machine with no swap configured (doctor warns - * about it); null means the platform offers no reading at all. - */ - swap: z - .object({ - totalBytes: z.number(), - usedBytes: z.number(), - }) - .nullable(), - }), +export const hostReadingSchema = z.object({ + cpuCount: z.number().int().positive(), /** - * The filesystem holding DORMICE_DATA_DIR — where sandbox disks live. A - * full data disk is the real capacity ceiling (past it, even the ledger - * cannot write). Null when the directory does not exist (fake executor, - * fresh install). + * Percent of the whole machine, 0-100. Null until the sampler has two + * samples to take a delta between — the first request after daemon + * start reports "don't know yet", not a made-up 0. */ - dataDisk: z + cpuUsedPct: z.number().nullable(), + memTotalBytes: z.number(), + /** + * What could still be allocated without swapping — /proc/meminfo's + * MemAvailable (counts reclaimable page cache), not the naive "free". + */ + memAvailableBytes: z.number(), + /** + * Swap is the freeze mechanism's fuel: frozen sandboxes live there, and + * a full swap means "idle is free" stops being true. totalBytes of 0 is + * an honest reading of a machine with no swap configured (doctor warns + * about it); null means the platform offers no reading at all. + */ + swap: z .object({ - path: z.string(), totalBytes: z.number(), usedBytes: z.number(), - availableBytes: z.number(), }) .nullable(), +}); + +export type HostReading = z.infer; + +/** + * The filesystem holding DORMICE_DATA_DIR — where sandbox disks live. A + * full data disk is the real capacity ceiling (past it, even the ledger + * cannot write). Readers get null when the directory does not exist (fake + * executor, fresh install). + */ +export const dataDiskSchema = z.object({ + path: z.string(), + totalBytes: z.number(), + usedBytes: z.number(), + availableBytes: z.number(), +}); + +export type DataDisk = z.infer; + +/** The ledger's census by lifecycle state. */ +export const sandboxStateCountsSchema = z.object({ + active: z.number().int(), + frozen: z.number().int(), + stopped: z.number().int(), + archived: z.number().int(), + restoring: z.number().int(), +}); + +export type SandboxStateCounts = z.infer; + +/** + * getHostMetrics() — the observation window into the machine itself: is the + * host healthy, and what do the sandboxes collectively cost it? A single + * point-in-time snapshot; for the machine's past see getHostMetricsHistory + * below. Observation never wakes a sandbox and never touches lifecycle. + */ +export const hostMetricsResponseSchema = z.object({ + host: hostReadingSchema, + dataDisk: dataDiskSchema.nullable(), /** Ledger aggregates: what the daemon believes it is running. */ sandboxes: z.object({ total: z.number().int(), maxSandboxes: z.number().int(), - byState: z.object({ - active: z.number().int(), - frozen: z.number().int(), - stopped: z.number().int(), - archived: z.number().int(), - restoring: z.number().int(), - }), + byState: sandboxStateCountsSchema, }), /** * What the sandbox disks cost, from the executor: nominal is the summed diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index e31684ce..4d8162f9 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -5,6 +5,7 @@ export * from './config'; export * from './destroy'; export * from './exec'; export * from './files'; +export * from './gateway'; export * from './host'; export * from './images'; export * from './ingress'; From bc0e761058ed8db4ccd150e98faa21837e1ab233 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:41:35 +0800 Subject: [PATCH 05/89] =?UTF-8?q?packages/gateway:=20the=20fleet's=20one?= =?UTF-8?q?=20door=20=E2=80=94=20nodes=20check=20in,=20new=20names=20are?= =?UTF-8?q?=20placed,=20existing=20ones=20are=20found=20by=20asking,=20eve?= =?UTF-8?q?rything=20else=20is=20forwarded=20raw?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fleet is N daemons that know nothing of each other behind one gateway. The gateway holds no sandbox state: it keeps the nodes that have checked in (one table, written by the nodes themselves — no registration verb, no nodes file), what each last reported (memory), a cache of where names were last found, and a per-name slot. When the cache has no answer it asks every node "do you hold this sandbox?" in parallel, two seconds, and routes to the one that says yes; two yeses are a 409 it refuses to guess about; no yes while a node is silent is a 503 with Retry-After, because a name that lives only on a silent node must not be built a second time elsewhere. A new name is placed on the emptiest node by active density per core, skipping nodes that are down, above the CPU limit, at the active ceiling or below the data-disk floor, with each pick counted against its node until the next reading. The forwarding plane, error dialects and placement come from the first cut by file (branch router-v1), minus the directory, the claims, the reconciler and the actor header: one token for the whole fleet, and the node trusts the gateway whole. Faces in this cut: the native sandbox verbs, the E2B control plane by name or id, envd by header, the bare signed-URL door as an honest 501, and the gateway's own checkIn / listNodes / removeNode. Daemon-addressed verbs answer 501 until the configuration authority moves here; the sandbox port proxy joins with the sandbox domain. Only the fleet token opens the door in this cut; minted keys arrive with the key table. Reverse-proved: without the name slot twenty simultaneous acquires build several copies; without the placement counter a burst lands on one node; without the 404 re-check a destroy behind the gateway's back leaves a stale entry. --- packages/gateway/drizzle.config.ts | 7 + packages/gateway/drizzle/0000_nodes.sql | 5 + .../gateway/drizzle/meta/0000_snapshot.json | 49 ++ packages/gateway/drizzle/meta/_journal.json | 13 + packages/gateway/package.json | 32 + packages/gateway/src/app.test.ts | 721 +++++++++++++++++ packages/gateway/src/app.ts | 166 ++++ packages/gateway/src/cache.ts | 77 ++ packages/gateway/src/classify.ts | 48 ++ packages/gateway/src/config.test.ts | 52 ++ packages/gateway/src/config.ts | 78 ++ packages/gateway/src/db/db.ts | 26 + packages/gateway/src/db/schema.ts | 32 + packages/gateway/src/errors.ts | 117 +++ packages/gateway/src/find.test.ts | 252 ++++++ packages/gateway/src/find.ts | 122 +++ packages/gateway/src/fleet.test.ts | 94 +++ packages/gateway/src/fleet.ts | 132 ++++ packages/gateway/src/forward.test.ts | 734 ++++++++++++++++++ packages/gateway/src/forward.ts | 367 +++++++++ packages/gateway/src/lookup.ts | 71 ++ packages/gateway/src/main.ts | 135 ++++ packages/gateway/src/placement.test.ts | 181 +++++ packages/gateway/src/placement.ts | 138 ++++ packages/gateway/src/raw.ts | 192 +++++ packages/gateway/src/routes/create.ts | 129 +++ packages/gateway/src/routes/destroy.ts | 44 ++ packages/gateway/src/routes/e2b.ts | 215 +++++ packages/gateway/src/routes/native.ts | 257 ++++++ packages/gateway/src/routes/nodes.ts | 94 +++ packages/gateway/src/routes/verdict.ts | 62 ++ packages/gateway/src/testing.ts | 60 ++ packages/gateway/src/version.ts | 23 + packages/gateway/tsconfig.json | 4 + packages/gateway/tsup.config.ts | 38 + pnpm-lock.yaml | 37 + 36 files changed, 4804 insertions(+) create mode 100644 packages/gateway/drizzle.config.ts create mode 100644 packages/gateway/drizzle/0000_nodes.sql create mode 100644 packages/gateway/drizzle/meta/0000_snapshot.json create mode 100644 packages/gateway/drizzle/meta/_journal.json create mode 100644 packages/gateway/package.json create mode 100644 packages/gateway/src/app.test.ts create mode 100644 packages/gateway/src/app.ts create mode 100644 packages/gateway/src/cache.ts create mode 100644 packages/gateway/src/classify.ts create mode 100644 packages/gateway/src/config.test.ts create mode 100644 packages/gateway/src/config.ts create mode 100644 packages/gateway/src/db/db.ts create mode 100644 packages/gateway/src/db/schema.ts create mode 100644 packages/gateway/src/errors.ts create mode 100644 packages/gateway/src/find.test.ts create mode 100644 packages/gateway/src/find.ts create mode 100644 packages/gateway/src/fleet.test.ts create mode 100644 packages/gateway/src/fleet.ts create mode 100644 packages/gateway/src/forward.test.ts create mode 100644 packages/gateway/src/forward.ts create mode 100644 packages/gateway/src/lookup.ts create mode 100644 packages/gateway/src/main.ts create mode 100644 packages/gateway/src/placement.test.ts create mode 100644 packages/gateway/src/placement.ts create mode 100644 packages/gateway/src/raw.ts create mode 100644 packages/gateway/src/routes/create.ts create mode 100644 packages/gateway/src/routes/destroy.ts create mode 100644 packages/gateway/src/routes/e2b.ts create mode 100644 packages/gateway/src/routes/native.ts create mode 100644 packages/gateway/src/routes/nodes.ts create mode 100644 packages/gateway/src/routes/verdict.ts create mode 100644 packages/gateway/src/testing.ts create mode 100644 packages/gateway/src/version.ts create mode 100644 packages/gateway/tsconfig.json create mode 100644 packages/gateway/tsup.config.ts diff --git a/packages/gateway/drizzle.config.ts b/packages/gateway/drizzle.config.ts new file mode 100644 index 00000000..807d65f7 --- /dev/null +++ b/packages/gateway/drizzle.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'drizzle-kit'; + +export default defineConfig({ + schema: './src/db/schema.ts', + out: './drizzle', + dialect: 'sqlite', +}); diff --git a/packages/gateway/drizzle/0000_nodes.sql b/packages/gateway/drizzle/0000_nodes.sql new file mode 100644 index 00000000..e7f1a6a8 --- /dev/null +++ b/packages/gateway/drizzle/0000_nodes.sql @@ -0,0 +1,5 @@ +CREATE TABLE `nodes` ( + `id` text PRIMARY KEY NOT NULL, + `endpoint` text NOT NULL, + `added_at` text NOT NULL +); diff --git a/packages/gateway/drizzle/meta/0000_snapshot.json b/packages/gateway/drizzle/meta/0000_snapshot.json new file mode 100644 index 00000000..9a496820 --- /dev/null +++ b/packages/gateway/drizzle/meta/0000_snapshot.json @@ -0,0 +1,49 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "b88172b7-a470-4a8c-addb-45c45229585f", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "nodes": { + "name": "nodes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "added_at": { + "name": "added_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/_journal.json b/packages/gateway/drizzle/meta/_journal.json new file mode 100644 index 00000000..d08827ad --- /dev/null +++ b/packages/gateway/drizzle/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "7", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "6", + "when": 1789328056388, + "tag": "0000_nodes", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/packages/gateway/package.json b/packages/gateway/package.json new file mode 100644 index 00000000..d4928d73 --- /dev/null +++ b/packages/gateway/package.json @@ -0,0 +1,32 @@ +{ + "name": "@dormice/gateway", + "version": "0.0.0", + "private": true, + "description": "Dormice gateway: the fleet's one door — places new sandboxes across nodes, finds existing ones by asking, forwards everything else", + "license": "Apache-2.0", + "type": "module", + "files": [ + "dist", + "drizzle" + ], + "scripts": { + "build": "tsup", + "typecheck": "tsc --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@dormice/server": "workspace:*", + "@dormice/shared": "workspace:*", + "better-sqlite3": "^12.11.1", + "drizzle-orm": "^0.45.2", + "fastify": "^5.10.0", + "fastify-type-provider-zod": "^7.0.0", + "pino": "^10.3.1", + "undici": "^8.7.0", + "zod": "^4.4.3" + }, + "devDependencies": { + "@types/better-sqlite3": "^7.6.13", + "drizzle-kit": "^0.31.10" + } +} diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts new file mode 100644 index 00000000..cce3d7a0 --- /dev/null +++ b/packages/gateway/src/app.test.ts @@ -0,0 +1,721 @@ +import { randomUUID } from 'node:crypto'; +import http from 'node:http'; +import net, { type AddressInfo } from 'node:net'; +import { fileURLToPath } from 'node:url'; +import { KeyedQueue } from '@dormice/server/keyed-queue'; +import { afterEach, describe, expect, it } from 'vitest'; +import { buildGatewayApp } from './app'; +import { NameCache } from './cache'; +import { loadConfig } from './config'; +import { migrateDb, openDb } from './db/db'; +import { Finder } from './find'; +import { Fleet } from './fleet'; +import { httpAskNode } from './lookup'; +import { checkInOf, type reading } from './testing'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const TOKEN = 'fleet-token-fleet-token-fleet-token-fleet'; + +/** + * A node as the gateway sees one: the daemon's wire for the handful of + * verbs the gateway touches, over a real socket. Every sandbox it holds is + * a row in `sandboxes`; every request it received is in `hits`. + */ +class FakeNode { + readonly sandboxes = new Map< + string, + { id: string; name: string; files: Map } + >(); + readonly hits: Array<{ path: string; auth: string | undefined }> = []; + creates = 0; + endpoint = ''; + private readonly server: http.Server; + + constructor(readonly id: string) { + this.server = http.createServer((req, res) => { + let text = ''; + req.on('data', (c) => { + text += c; + }); + req.on('end', () => this.answer(req, res, text)); + }); + } + + async start(): Promise { + await new Promise((r) => this.server.listen(0, '127.0.0.1', r)); + this.endpoint = `http://127.0.0.1:${(this.server.address() as AddressInfo).port}`; + return this; + } + + stop(): Promise { + return new Promise((resolve) => { + this.server.closeAllConnections(); + this.server.close(() => resolve()); + }); + } + + lookups(): number { + return this.hits.filter((h) => h.path === '/lookupSandbox').length; + } + + byId(id: string) { + return [...this.sandboxes.values()].find((s) => s.id === id); + } + + private answer( + req: http.IncomingMessage, + res: http.ServerResponse, + text: string, + ): void { + const url = req.url ?? '/'; + const path = url.split('?')[0] ?? url; + const auth = + req.headers.authorization ?? + (req.headers['x-api-key'] as string | undefined); + this.hits.push({ path, auth }); + const json = (status: number, body: unknown) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(body)); + }; + const body = text ? (JSON.parse(text) as Record) : {}; + if (path.startsWith('/e2b/envd/')) { + return json(200, { + envd: this.id, + sandboxId: req.headers['e2b-sandbox-id'], + accessToken: req.headers['x-access-token'], + url, + }); + } + if (path.startsWith('/e2b/api/')) { + if (auth !== `e2b_${TOKEN}`) + return json(401, { code: 401, message: 'invalid API key' }); + const m = path.match(/^\/e2b\/api\/sandboxes(?:\/([^/]+))?(\/.*)?$/); + if (!m) return json(404, { code: 404, message: 'not found' }); + const [, id, rest] = m; + if (id === undefined) { + if (req.method !== 'POST') + return json(404, { code: 404, message: 'not found' }); + const metadata = body.metadata as { name?: string } | undefined; + const name = metadata?.name ?? `e2b-${randomUUID()}`; + // The daemon's create is idempotent on metadata.name. + const sandbox = this.sandboxes.get(name) ?? this.create(name); + return json(201, { sandboxID: sandbox.id, templateID: 'base' }); + } + const sandbox = this.byId(id); + if (!sandbox) return json(404, { code: 404, message: 'not found' }); + if (rest) return json(200, { path: url, sandboxID: sandbox.id }); + if (req.method === 'DELETE') { + this.sandboxes.delete(sandbox.name); + res.writeHead(204); + res.end(); + return; + } + return json(200, { sandboxID: sandbox.id, alias: sandbox.name }); + } + if (auth !== `Bearer ${TOKEN}`) + return json(401, { message: 'missing or invalid API token' }); + const name = body.name as string | undefined; + const found = name === undefined ? undefined : this.sandboxes.get(name); + switch (path) { + case '/lookupSandbox': { + const sandbox = 'id' in body ? this.byId(body.id as string) : found; + return json( + 200, + sandbox + ? { + found: true, + sandbox: { + id: sandbox.id, + name: sandbox.name, + state: 'active', + }, + } + : { found: false }, + ); + } + case '/acquireSandbox': { + if (name === undefined) return json(400, { message: 'name required' }); + const sandbox = found ?? this.create(name); + return json(200, { + status: 'ready', + created: found === undefined, + sandbox: { id: sandbox.id, name, nodeId: this.id }, + }); + } + case '/destroySandbox': { + if (name !== undefined) this.sandboxes.delete(name); + return json(200, { destroyed: found !== undefined }); + } + case '/execCommand': { + if (!found) + return json(404, { + message: `no sandbox named "${name}" — acquire it first`, + }); + return json(200, { stdout: `${this.id}: ${String(body.command)}` }); + } + case '/writeFile': { + if (!found) + return json(404, { + message: `no sandbox named "${name}" — acquire it first`, + }); + found.files.set(String(body.path), String(body.content)); + return json(200, { written: true }); + } + case '/readFile': { + if (!found) + return json(404, { + message: `no sandbox named "${name}" — acquire it first`, + }); + const content = found.files.get(String(body.path)); + if (content === undefined) + return json(404, { message: `no such file: ${String(body.path)}` }); + return json(200, { content }); + } + default: + return json(404, { message: `route ${req.method} ${url} not found` }); + } + } + + private create(name: string) { + this.creates += 1; + const sandbox = { id: randomUUID(), name, files: new Map() }; + this.sandboxes.set(name, sandbox); + return sandbox; + } +} + +interface Harness { + endpoint: string; + nodes: FakeNode[]; + cache: NameCache; + fleet: Fleet; + checkIn( + node: FakeNode, + over?: Parameters[0] & { intervalSeconds?: number }, + ): Promise; + close(): Promise; +} + +const harnesses: Harness[] = []; +afterEach(async () => { + for (const h of harnesses.splice(0)) await h.close(); +}); + +async function gateway( + nodeIds: string[], + env: Record = {}, +): Promise { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const config = loadConfig({ + DORMICE_API_TOKEN: TOKEN, + DORMICE_GATEWAY_DB_PATH: ':memory:', + // A laptop running the suite is not the machine under judgment. + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '100', + ...env, + }); + const fleet = new Fleet(db); + const cache = new NameCache(); + const finder = new Finder(fleet, cache, httpAskNode(TOKEN), { + warn: () => {}, + }); + const app = buildGatewayApp({ + config, + fleet, + finder, + locks: new KeyedQueue(), + logger: false, + build: null, + }); + await app.listen({ host: '127.0.0.1', port: 0 }); + const endpoint = `http://127.0.0.1:${(app.server.address() as AddressInfo).port}`; + const nodes = await Promise.all( + nodeIds.map((id) => new FakeNode(id).start()), + ); + const harness: Harness = { + endpoint, + nodes, + cache, + fleet, + checkIn: async (node, over = {}) => { + const res = await fetch(`${endpoint}/checkIn`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(checkInOf(node.id, node.endpoint, over)), + }); + expect(res.status).toBe(200); + }, + close: async () => { + await app.close(); + for (const node of nodes) await node.stop(); + }, + }; + for (const node of nodes) await harness.checkIn(node); + harnesses.push(harness); + return harness; +} + +async function rpc( + h: Harness, + path: string, + payload: unknown = {}, + token = TOKEN, +): Promise<{ status: number; body: unknown; headers: Headers }> { + const res = await fetch(`${h.endpoint}${path}`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(payload), + }); + const text = await res.text(); + return { + status: res.status, + body: text ? JSON.parse(text) : null, + headers: res.headers, + }; +} + +const message = (r: { body: unknown }) => + (r.body as { message: string }).message; +const sandboxOf = (r: { body: unknown }) => + (r.body as { created: boolean; sandbox: { id: string; nodeId: string } }) + .sandbox; + +/** Polls until the probe answers something — cache verification runs off the request path. */ +async function until( + probe: () => Promise | T | undefined, + timeoutMs = 2_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + const value = await probe(); + if (value !== undefined) return value; + if (Date.now() > deadline) throw new Error('condition never became true'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +describe('the door', () => { + it('/healthz is open; everything else wants the fleet token, on the check-in too', async () => { + const h = await gateway([]); + const health = await fetch(`${h.endpoint}/healthz`); + expect(health.status).toBe(200); + expect(await health.json()).toEqual({ status: 'ok', build: null }); + expect((await rpc(h, '/listNodes', {}, 'w'.repeat(40))).status).toBe(401); + const bare = await fetch(`${h.endpoint}/checkIn`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(checkInOf('x', 'http://127.0.0.1:1')), + }); + expect(bare.status).toBe(401); + expect((await rpc(h, '/acquireSandbox', { name: 'x' }, 'bad')).status).toBe( + 401, + ); + }); + + it('refuses an absolute-form request target before routing', async () => { + const h = await gateway([]); + const url = new URL(h.endpoint); + const answer = await new Promise((resolve, reject) => { + let text = ''; + const socket = net.connect(Number(url.port), url.hostname, () => { + socket.write( + `POST http://evil.example/execCommand HTTP/1.1\r\nhost: gateway\r\nconnection: close\r\n\r\n`, + ); + }); + socket.on('data', (c) => { + text += c; + }); + socket.on('end', () => resolve(text)); + socket.on('error', reject); + }); + expect(answer).toContain('HTTP/1.1 400'); + expect(answer).toContain('origin-form'); + }); +}); + +describe('check-in and the node verbs', () => { + it('a check-in joins the fleet; listNodes shows what it said; a later check-in moves the endpoint; the row outlives the memory', async () => { + const h = await gateway(['b']); + const listed = (await rpc(h, '/listNodes')).body as { + nodes: Array>; + }; + expect(listed.nodes).toHaveLength(1); + expect(listed.nodes[0]).toMatchObject({ + id: 'b', + endpoint: h.nodes[0]?.endpoint, + reachable: true, + intervalSeconds: 15, + placedSinceCheckIn: 0, + build: { commit: 'abc1234' }, + }); + expect( + (listed.nodes[0]?.reading as { host: { cpuCount: number } }).host + .cpuCount, + ).toBe(8); + const moved = new FakeNode('b'); + await moved.start(); + await h.checkIn(moved, { active: 3 }); + expect(h.fleet.get('b')?.endpoint).toBe(moved.endpoint); + expect(h.fleet.get('b')?.reading?.sandboxes.byState.active).toBe(3); + await moved.stop(); + // A malformed check-in is a 400 naming the trouble, not a join. + const bad = await rpc(h, '/checkIn', { nodeId: 'z' }); + expect(bad.status).toBe(400); + expect(h.fleet.get('z')).toBeUndefined(); + }); + + it('removeNode forgets the node and everything cached on it; a removed node that checks in again re-joins', async () => { + const h = await gateway(['b', 'c']); + const created = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'x' })); + expect(h.cache.getByName('x')?.nodeId).toBe(created.nodeId); + expect((await rpc(h, '/removeNode', { id: created.nodeId })).body).toEqual({ + removed: true, + }); + expect(h.cache.getByName('x')).toBeUndefined(); + expect( + ((await rpc(h, '/listNodes')).body as { nodes: unknown[] }).nodes, + ).toHaveLength(1); + expect((await rpc(h, '/removeNode', { id: 'nobody' })).body).toEqual({ + removed: false, + }); + const back = h.nodes.find((n) => n.id === created.nodeId); + if (!back) throw new Error('node lost'); + await h.checkIn(back); + expect( + ((await rpc(h, '/listNodes')).body as { nodes: unknown[] }).nodes, + ).toHaveLength(2); + }); +}); + +describe('acquire: placing and finding', () => { + it('a new name lands on the emptiest node by active density, under the fleet token, and is cached: the second acquire asks nobody', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 10, cores: 8 }); + await h.checkIn(b, { active: 1, cores: 8 }); + const first = await rpc(h, '/acquireSandbox', { name: 'alice' }); + expect(first.status).toBe(200); + expect(sandboxOf(first).nodeId).toBe('b'); + expect(b.creates).toBe(1); + expect(a.creates).toBe(0); + expect(b.hits.find((hit) => hit.path === '/acquireSandbox')?.auth).toBe( + `Bearer ${TOKEN}`, + ); + // Both nodes were asked once — the name was new — and the answer's id + // is cached beside the name. + expect(a.lookups()).toBe(1); + expect(b.lookups()).toBe(1); + expect(h.cache.getById(sandboxOf(first).id)?.nodeId).toBe('b'); + const again = await rpc(h, '/acquireSandbox', { name: 'alice' }); + expect(sandboxOf(again).id).toBe(sandboxOf(first).id); + expect((again.body as { created: boolean }).created).toBe(false); + expect(a.lookups()).toBe(1); + expect(b.lookups()).toBe(1); + expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); + }); + + it('twenty simultaneous acquires of one new name are one create on one node; different names spread', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + const burst = await Promise.all( + Array.from({ length: 20 }, () => + rpc(h, '/acquireSandbox', { name: 'burst' }), + ), + ); + expect(new Set(burst.map((r) => sandboxOf(r).id)).size).toBe(1); + expect(a.creates + b.creates).toBe(1); + expect( + burst.filter((r) => (r.body as { created: boolean }).created), + ).toHaveLength(1); + + const spread = await Promise.all( + Array.from({ length: 6 }, (_, i) => + rpc(h, '/acquireSandbox', { name: `spread-${i}` }), + ), + ); + const landed = spread.map((r) => sandboxOf(r).nodeId); + expect(landed.filter((id) => id === 'a').length).toBeGreaterThan(0); + expect(landed.filter((id) => id === 'b').length).toBeGreaterThan(0); + }); + + it('a sandbox built behind its back is found by asking: routed at once, re-acquired where it is', async () => { + const h = await gateway(['a', 'b']); + const b = h.nodes[1] as FakeNode; + const staged = sandboxOf( + await (async () => { + const res = await fetch(`${b.endpoint}/acquireSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'staged' }), + }); + return { body: await res.json() }; + })(), + ); + const ran = await rpc(h, '/execCommand', { name: 'staged', command: 'id' }); + expect(ran.status).toBe(200); + expect((ran.body as { stdout: string }).stdout).toBe('b: id'); + const found = await rpc(h, '/acquireSandbox', { name: 'staged' }); + expect(sandboxOf(found)).toMatchObject({ id: staged.id, nodeId: 'b' }); + expect((found.body as { created: boolean }).created).toBe(false); + }); + + it('one name on two nodes is a 409 naming both, for every verb; once one copy is gone the name routes again', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + for (const node of [a, b]) { + await fetch(`${node.endpoint}/acquireSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'twin' }), + }); + } + for (const verb of ['/acquireSandbox', '/execCommand', '/destroySandbox']) { + const refused = await rpc(h, verb, { name: 'twin' }); + expect(refused.status).toBe(409); + expect(message(refused)).toContain('exists on nodes a and b'); + } + expect(h.cache.getByName('twin')).toBeUndefined(); + b.sandboxes.delete('twin'); + const healed = await rpc(h, '/acquireSandbox', { name: 'twin' }); + expect(sandboxOf(healed).nodeId).toBe('a'); + }); + + it('a node that does not answer: its cached names 502, and a new name is a 503 with Retry-After naming it — until an operator removes it', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 1 }); + await h.checkIn(b, { active: 50 }); + const onA = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'on-a' })); + expect(onA.nodeId).toBe('a'); + await a.stop(); + const cut = await rpc(h, '/execCommand', { name: 'on-a', command: 'x' }); + expect(cut.status).toBe(502); + expect(message(cut)).toMatch(/did not answer/); + const fresh = await rpc(h, '/acquireSandbox', { name: 'brand-new' }); + expect(fresh.status).toBe(503); + expect(fresh.headers.get('retry-after')).toBe('15'); + expect(message(fresh)).toContain('node a did not answer'); + expect(message(fresh)).toContain('cannot be treated as new'); + expect(b.creates).toBe(0); + expect((await rpc(h, '/removeNode', { id: 'a' })).body).toEqual({ + removed: true, + }); + const placed = await rpc(h, '/acquireSandbox', { name: 'brand-new' }); + expect(placed.status).toBe(200); + expect(sandboxOf(placed).nodeId).toBe('b'); + }); + + it('every node refusing is a 503 that names each one; no node at all says so', async () => { + const h = await gateway(['a', 'b'], { + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', + }); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 2 }); + await h.checkIn(b, { active: 1, diskAvail: 2 ** 30 }); + const refused = await rpc(h, '/acquireSandbox', { name: 'full' }); + expect(refused.status).toBe(503); + expect(message(refused)).toContain('a: 2 active sandboxes + 0 placed'); + expect(message(refused)).toContain('b: data disk has 1.0 GiB available'); + expect(refused.headers.get('retry-after')).toBe('15'); + + const empty = await gateway([]); + const nobody = await rpc(empty, '/acquireSandbox', { name: 'x' }); + expect(nobody.status).toBe(503); + expect(message(nobody)).toMatch(/no node has checked in/); + }); +}); + +describe('using, destroying, and the cache', () => { + it("files round-trip; a node's 404 for a missing file passes through and the sandbox stays cached", async () => { + const h = await gateway(['a']); + const a = h.nodes[0] as FakeNode; + await rpc(h, '/acquireSandbox', { name: 'w' }); + expect( + (await rpc(h, '/writeFile', { name: 'w', path: 'a.txt', content: 'hi' })) + .status, + ).toBe(200); + expect( + (await rpc(h, '/readFile', { name: 'w', path: 'a.txt' })).body, + ).toEqual({ content: 'hi' }); + const missing = await rpc(h, '/readFile', { name: 'w', path: 'nope' }); + expect(missing.status).toBe(404); + expect(message(missing)).toBe('no such file: nope'); + // The 404 sent one question to the node, which said yes: the entry stays. + await until(() => (a.lookups() >= 2 ? true : undefined)); + expect(h.cache.getByName('w')?.nodeId).toBe('a'); + expect( + (await rpc(h, '/execCommand', { name: 'w', command: 'ok' })).status, + ).toBe(200); + }); + + it('a destroy the gateway relays forgets the entry; a destroy behind its back is caught by the 404 re-check', async () => { + const h = await gateway(['a']); + const a = h.nodes[0] as FakeNode; + const first = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'd' })); + expect((await rpc(h, '/destroySandbox', { name: 'd' })).body).toEqual({ + destroyed: true, + }); + expect(h.cache.getByName('d')).toBeUndefined(); + expect(h.cache.getById(first.id)).toBeUndefined(); + expect((await rpc(h, '/destroySandbox', { name: 'd' })).body).toEqual({ + destroyed: false, + }); + const second = await rpc(h, '/acquireSandbox', { name: 'd' }); + expect((second.body as { created: boolean }).created).toBe(true); + expect(sandboxOf(second).id).not.toBe(first.id); + + // Behind the gateway's back: the node's own 404 is relayed as it came, + // the re-check finds the sandbox absent, and the next request asks + // the fleet afresh — a 404 in the gateway's own words. + a.sandboxes.delete('d'); + const relayed = await rpc(h, '/execCommand', { name: 'd', command: 'x' }); + expect(relayed.status).toBe(404); + await until(() => + h.cache.getByName('d') === undefined ? true : undefined, + ); + const before = a.lookups(); + const own = await rpc(h, '/execCommand', { name: 'd', command: 'x' }); + expect(own.status).toBe(404); + expect(message(own)).toBe('no sandbox named "d" — acquire it first'); + expect(a.lookups()).toBe(before + 1); + }); + + it('daemon-addressed verbs are an honest 501, a misspelled verb a 404, a body without a name a 400', async () => { + const h = await gateway(['a']); + const listed = await rpc(h, '/listSandboxes'); + expect(listed.status).toBe(501); + expect(message(listed)).toContain('call the node directly'); + expect((await rpc(h, '/createApiKey', { name: 'k' })).status).toBe(501); + expect((await rpc(h, '/acquireSandbx', { name: 'x' })).status).toBe(404); + expect((await rpc(h, '/execCommand', { command: 'x' })).status).toBe(400); + expect(h.nodes[0]?.hits).toEqual([]); + }); +}); + +describe('the E2B faces', () => { + const e2b = (h: Harness, path: string, init: RequestInit = {}) => + fetch(`${h.endpoint}/e2b/api${path}`, { + ...init, + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + ...(init.headers as Record | undefined), + }, + }); + + it('control plane: a named create is placed and cached; by-id verbs are found and forwarded under e2b_; the kill forgets the entry; an unnamed create is known by id', async () => { + const h = await gateway(['a', 'b']); + const created = await e2b(h, '/sandboxes', { + method: 'POST', + body: JSON.stringify({ metadata: { name: 'e2b-named' } }), + }); + expect(created.status).toBe(201); + const { sandboxID } = (await created.json()) as { sandboxID: string }; + const entry = h.cache.getById(sandboxID); + expect(entry?.name).toBe('e2b-named'); + const node = h.nodes.find((n) => n.id === entry?.nodeId); + if (!node) throw new Error('not placed'); + const info = await e2b(h, `/sandboxes/${sandboxID}`); + expect(info.status).toBe(200); + expect(node.hits.at(-1)?.auth).toBe(`e2b_${TOKEN}`); + const deeper = await e2b(h, `/sandboxes/${sandboxID}/metrics?x=1`, { + method: 'GET', + }); + expect(((await deeper.json()) as { path: string }).path).toBe( + `/e2b/api/sandboxes/${sandboxID}/metrics?x=1`, + ); + // The same name through the E2B face is the same sandbox (the name slot). + const again = await e2b(h, '/sandboxes', { + method: 'POST', + body: JSON.stringify({ metadata: { name: 'e2b-named' } }), + }); + expect(((await again.json()) as { sandboxID: string }).sandboxID).toBe( + sandboxID, + ); + const killed = await e2b(h, `/sandboxes/${sandboxID}`, { + method: 'DELETE', + }); + expect(killed.status).toBe(204); + expect(h.cache.getById(sandboxID)).toBeUndefined(); + const gone = await e2b(h, `/sandboxes/${sandboxID}`); + expect(gone.status).toBe(404); + expect(await gone.json()).toEqual({ + code: 404, + message: `sandbox "${sandboxID}" not found`, + }); + + const anonymous = await e2b(h, '/sandboxes', { + method: 'POST', + body: JSON.stringify({ templateID: 'base' }), + }); + expect(anonymous.status).toBe(201); + const anon = (await anonymous.json()) as { sandboxID: string }; + expect(h.cache.getById(anon.sandboxID)?.name).toBeNull(); + expect((await e2b(h, `/sandboxes/${anon.sandboxID}`)).status).toBe(200); + + expect((await e2b(h, '/v2/sandboxes')).status).toBe(501); + const wrongKey = await fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { 'x-api-key': 'e2b_wrong', 'content-type': 'application/json' }, + body: '{}', + }); + expect(wrongKey.status).toBe(401); + expect(await wrongKey.json()).toEqual({ + code: 401, + message: 'invalid API key', + }); + }); + + it("envd: routed by the E2b-Sandbox-Id header with the caller's own credentials untouched; refusals wear the connect dialect with CORS", async () => { + const h = await gateway(['a']); + const created = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'e' })); + const hit = await fetch(`${h.endpoint}/e2b/envd/files?path=/x`, { + headers: { 'e2b-sandbox-id': created.id, 'x-access-token': 'hmac-1' }, + }); + expect(hit.status).toBe(200); + expect(await hit.json()).toEqual({ + envd: 'a', + sandboxId: created.id, + accessToken: 'hmac-1', + url: '/e2b/envd/files?path=/x', + }); + const headerless = await fetch(`${h.endpoint}/e2b/envd/files`); + expect(headerless.status).toBe(401); + expect(headerless.headers.get('access-control-allow-origin')).toBe('*'); + expect(await headerless.json()).toEqual({ + code: 'unauthenticated', + message: 'missing E2b-Sandbox-Id header', + }); + const stranger = await fetch(`${h.endpoint}/e2b/envd/files`, { + headers: { 'e2b-sandbox-id': randomUUID() }, + }); + expect(stranger.status).toBe(502); + expect(((await stranger.json()) as { code: string }).code).toBe( + 'unavailable', + ); + const preflight = await fetch(`${h.endpoint}/e2b/envd/files`, { + method: 'OPTIONS', + headers: { origin: 'https://app.example' }, + }); + expect(preflight.status).toBe(204); + expect(preflight.headers.get('access-control-allow-origin')).toBe('*'); + const bare = await fetch(`${h.endpoint}/files?signature=x`); + expect(bare.status).toBe(501); + expect(bare.headers.get('access-control-allow-origin')).toBe('*'); + expect(((await bare.json()) as { code: string }).code).toBe( + 'unimplemented', + ); + }); +}); diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts new file mode 100644 index 00000000..906e8e77 --- /dev/null +++ b/packages/gateway/src/app.ts @@ -0,0 +1,166 @@ +import http from 'node:http'; +import { tokensEqual } from '@dormice/server/auth'; +import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import fastify, { type FastifyError, type FastifyServerFactory } from 'fastify'; +import { + serializerCompiler, + validatorCompiler, + type ZodTypeProvider, +} from 'fastify-type-provider-zod'; +import { type Logger, pino } from 'pino'; +import { z } from 'zod'; +import { classify, isOriginForm } from './classify'; +import type { Config } from './config'; +import { renderError } from './errors'; +import type { Finder } from './find'; +import type { Fleet } from './fleet'; +import type { PlacementKnobs } from './placement'; +import { createRawFaces } from './raw'; +import { e2bControlRoutes } from './routes/e2b'; +import { nativeRoutes } from './routes/native'; +import { nodeRoutes } from './routes/nodes'; +import { type BuildInfo, readBuildInfo } from './version'; + +export interface GatewayAppDeps { + config: Config; + fleet: Fleet; + finder: Finder; + /** One queue for the whole gateway: the create and destroy verbs of both faces share per-name slots. */ + locks: KeyedQueue; + /** A pino instance, or a boolean for the default logger (false = silent, for tests). */ + logger?: Logger | boolean; + /** The build identity /healthz reports; null when built outside a checkout. */ + build?: BuildInfo | null; +} + +/** Placement's knobs, read once from the config. */ +export function placementKnobs(config: Config): PlacementKnobs { + return { + cpuLimitPct: config.DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT, + activeLimit: config.DORMICE_GATEWAY_NODE_ACTIVE_LIMIT, + minDiskAvailableBytes: config.DORMICE_GATEWAY_NODE_MIN_DISK_GB * 2 ** 30, + }; +} + +/** + * Builds the gateway's Fastify instance — the daemon's shape (zod as + * validator and serializer, one error dialect, /healthz open), without + * the daemon's body. Building is separate from listening so tests inject + * requests without a port; the raw faces need real sockets. + */ +export function buildGatewayApp({ + config, + fleet, + finder, + locks, + logger = true, + build = readBuildInfo(), +}: GatewayAppDeps) { + const loggerInstance = + typeof logger === 'boolean' ? pino({ enabled: logger }) : logger; + const token = config.DORMICE_API_TOKEN; + + // The face keyed on a header sits in front of Fastify, exactly as the + // daemon's port proxy does (server/app.ts): refuse what is not an + // origin-form target (classify.ts isOriginForm), triage the raw + // request, hand the rest to Fastify. app.inject() bypasses the factory, + // so that face is exercised over real sockets only. + const raw = createRawFaces({ finder, token, log: loggerInstance }); + const serverFactory: FastifyServerFactory = (handler) => { + const server = http.createServer((req, res) => { + if (!isOriginForm(req)) { + renderError(res, 'native', { + status: 400, + message: + 'request target must be origin-form (a path starting with "/")', + }); + return; + } + const kind = classify(req); + if (kind.face === 'fastify') handler(req, res); + else raw.handleRequest(kind, req, res); + }); + server.on('upgrade', (req, socket) => { + raw.handleUpgrade(classify(req), req, socket); + }); + // The gateway only relays; the node's own request timeout is the one + // that should fire on a slow upload, not a second one in front of it. + server.requestTimeout = 0; + return server; + }; + const app = fastify({ + loggerInstance, + serverFactory, + }).withTypeProvider(); + app.setValidatorCompiler(validatorCompiler); + app.setSerializerCompiler(serializerCompiler); + + // The native error dialect, verbatim from the daemon: every non-2xx body + // the gateway itself produces is { message }. Errors a node produced are + // forwarded as they came, in whichever dialect that face speaks. + app.setErrorHandler((error: FastifyError, request, reply) => { + const status = error.statusCode ?? 500; + if (status >= 500) { + request.log.error(error, 'request failed'); + } + reply.code(status).send({ message: error.message }); + }); + app.setNotFoundHandler((request, reply) => { + reply + .code(404) + .send({ message: `route ${request.method} ${request.url} not found` }); + }); + + // Liveness, open by design; the build identity so an operator can tell + // which commit answers without a token. + app.get( + '/healthz', + { + schema: { + response: { + 200: z.object({ + status: z.literal('ok'), + build: z + .object({ + commit: z.string(), + title: z.string(), + committedAt: z.string(), + }) + .nullable(), + }), + }, + }, + }, + async () => ({ status: 'ok' as const, build }), + ); + + // One credential opens the door: the fleet's token, presented as a + // Bearer by callers and by nodes checking in alike. Keys minted by the + // gateway — many, expiring, revocable — arrive with its key table. + const knobs = placementKnobs(config); + app.register(async (api) => { + api.addHook('onRequest', async (request, reply) => { + const header = request.headers.authorization; + const bare = header?.startsWith('Bearer ') ? header.slice(7) : null; + if (bare === null || !tokensEqual(bare, token)) { + await reply.code(401).send({ message: 'missing or invalid API token' }); + } + }); + await api.register(nodeRoutes, { fleet, cache: finder.cache }); + // Its own sub-scope: the byte-preserving body parser it installs must + // not reach the gateway's own verbs, which keep Fastify's JSON parsing. + await api.register(nativeRoutes, { fleet, finder, locks, knobs, token }); + }); + + // The E2B control plane, its own auth and dialect, like the daemon's. + app.register(e2bControlRoutes, { + fleet, + finder, + locks, + knobs, + token, + prefix: '/e2b/api', + }); + + return app; +} diff --git a/packages/gateway/src/cache.ts b/packages/gateway/src/cache.ts new file mode 100644 index 00000000..0803928a --- /dev/null +++ b/packages/gateway/src/cache.ts @@ -0,0 +1,77 @@ +/** + * Where a sandbox was last found: name and id → node. A cache, not a + * directory — nothing here is authoritative, the ledger of the node that + * runs the sandbox is — so it is never persisted, never reconciled and + * never written from anything but a node's own answer: a lookup that said + * yes, a create that answered 2xx. It is dropped on a destroy the gateway + * relayed, on a node's 404 that a fresh lookup confirms (find.ts verify), + * and wholesale for a node an operator removed. An entry that is wrong + * costs one misrouted request, whose answer evicts it; a cache that is + * lost costs one extra round of questions per name. + */ +export interface CacheEntry { + id: string; + /** Null for a sandbox learned by id alone (an unnamed E2B create) until a lookup names it. */ + name: string | null; + nodeId: string; +} + +export class NameCache { + private readonly byName = new Map(); + private readonly byId = new Map(); + + put(entry: CacheEntry): void { + // A name that moves to a new id (destroyed and re-acquired elsewhere + // while this gateway did not see the destroy) drops the old entry + // whole, so no id points at a node that no longer holds it. + if (entry.name !== null) { + const previous = this.byName.get(entry.name); + if (previous !== undefined && previous.id !== entry.id) { + this.byId.delete(previous.id); + } + this.byName.set(entry.name, entry); + } + const known = this.byId.get(entry.id); + if ( + known?.name !== null && + known?.name !== undefined && + known.name !== entry.name + ) { + this.byName.delete(known.name); + } + this.byId.set(entry.id, entry); + } + + getByName(name: string): CacheEntry | undefined { + return this.byName.get(name); + } + + getById(id: string): CacheEntry | undefined { + return this.byId.get(id); + } + + evict(entry: CacheEntry): void { + if (entry.name !== null && this.byName.get(entry.name)?.id === entry.id) { + this.byName.delete(entry.name); + } + if (this.byId.get(entry.id)?.nodeId === entry.nodeId) { + this.byId.delete(entry.id); + } + } + + /** Everything that pointed at a node the operator removed. */ + evictNode(nodeId: string): number { + let evicted = 0; + for (const entry of [...this.byId.values()]) { + if (entry.nodeId === nodeId) { + this.evict(entry); + evicted += 1; + } + } + return evicted; + } + + get size(): number { + return this.byId.size; + } +} diff --git a/packages/gateway/src/classify.ts b/packages/gateway/src/classify.ts new file mode 100644 index 00000000..94f91adc --- /dev/null +++ b/packages/gateway/src/classify.ts @@ -0,0 +1,48 @@ +/** + * Which face a raw request belongs to — the gateway's serverFactory + * triages every request before Fastify's router sees it, because one face + * is keyed on something Fastify cannot route on (a header, on any path). + * Pure, so the order of the tests is a fact here and nowhere else: + * envd /e2b/envd/* — E2B's in-sandbox API, keyed by the + * E2b-Sandbox-Id header. + * signedRoot exactly /files at the root — the bare signed-URL form, + * which carries no sandbox id anywhere the gateway can read + * without the node's signing secret. + * fastify everything else — the native verbs, /e2b/api, /healthz, + * the gateway's own verbs. + * The sandbox port proxy (Host `-.`) joins this list + * when the sandbox domain moves into the gateway's settings. + */ +export type Classified = + | { face: 'envd' } + | { face: 'signedRoot' } + | { face: 'fastify' }; + +/** + * Only the origin-form request target (RFC 9112 §3.2.1, a path starting + * with `/`) is forwarded. The gateway routes on the path Fastify extracts + * — find-my-way strips a scheme and authority off an absolute-form target + * — and sends the target to the node byte for byte, so on an absolute + * form the two would read different requests: `POST http://8000-. + * /execCommand` is `/execCommand` to the gateway (a native verb, + * authenticated, the fleet's token attached) and, to the RFC-7230 hop in + * front of the node (Caddy: Host := the target's authority), a request to + * the node's sandbox proxy — the fleet's credential delivered into the + * tenant's own sandbox (traced 2026-09-12). Absolute form is what a + * client sends a forward proxy, which the gateway is not; every real + * client of an origin server sends origin form, so refusing the rest + * costs nothing and keeps the invariant whole: the gateway and the node + * see the same request. + */ +export function isOriginForm(req: { url?: string }): boolean { + return (req.url ?? '').startsWith('/'); +} + +export function classify(req: { url?: string }): Classified { + const url = req.url ?? ''; + const q = url.indexOf('?'); + const path = q === -1 ? url : url.slice(0, q); + if (url.startsWith('/e2b/envd/')) return { face: 'envd' }; + if (path === '/files') return { face: 'signedRoot' }; + return { face: 'fastify' }; +} diff --git a/packages/gateway/src/config.test.ts b/packages/gateway/src/config.test.ts new file mode 100644 index 00000000..186aa5cb --- /dev/null +++ b/packages/gateway/src/config.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; +import { loadConfig } from './config'; + +const TOKEN = { DORMICE_API_TOKEN: 'x'.repeat(32) }; + +describe('loadConfig', () => { + it('defaults: port 3677, an absolute database path, placement knobs 70% / 400 active / 10 GiB', () => { + const config = loadConfig(TOKEN); + expect(config.DORMICE_GATEWAY_PORT).toBe(3677); + expect(config.DORMICE_GATEWAY_DB_PATH).toBe( + '/var/lib/dormice-gateway/gateway.db', + ); + expect(config.DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT).toBe(70); + expect(config.DORMICE_GATEWAY_NODE_ACTIVE_LIMIT).toBe(400); + expect(config.DORMICE_GATEWAY_NODE_MIN_DISK_GB).toBe(10); + }); + + it('requires the fleet token, at least 32 characters, naming the variable', () => { + expect(() => loadConfig({})).toThrow(/DORMICE_API_TOKEN/); + expect(() => loadConfig({ DORMICE_API_TOKEN: 'short' })).toThrow( + /at least 32 characters/, + ); + }); + + it('refuses a relative database path and accepts :memory:', () => { + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_DB_PATH: 'data/gateway.db' }), + ).toThrow(/DORMICE_GATEWAY_DB_PATH must be an absolute path/); + expect( + loadConfig({ ...TOKEN, DORMICE_GATEWAY_DB_PATH: ':memory:' }) + .DORMICE_GATEWAY_DB_PATH, + ).toBe(':memory:'); + }); + + it('parses the placement knobs and refuses nonsense', () => { + const config = loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '85', + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', + DORMICE_GATEWAY_NODE_MIN_DISK_GB: '0.5', + }); + expect(config.DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT).toBe(85); + expect(config.DORMICE_GATEWAY_NODE_ACTIVE_LIMIT).toBe(2); + expect(config.DORMICE_GATEWAY_NODE_MIN_DISK_GB).toBe(0.5); + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '101' }), + ).toThrow(); + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '0' }), + ).toThrow(); + }); +}); diff --git a/packages/gateway/src/config.ts b/packages/gateway/src/config.ts new file mode 100644 index 00000000..94facb04 --- /dev/null +++ b/packages/gateway/src/config.ts @@ -0,0 +1,78 @@ +import { isAbsolute } from 'node:path'; +import { z } from 'zod'; + +/** + * The gateway's configuration: environment variables, validated once at + * startup so a bad value fails loudly here — the daemon's discipline + * (packages/server/src/config.ts). The gateway's own knobs carry the + * DORMICE_GATEWAY_ prefix so a gateway and a node can share one machine's + * /etc/dormice without colliding; the token deliberately does not — it is + * the very same DORMICE_API_TOKEN every node has, one string for the whole + * fleet, written into both env files by the same hand. + */ +const envSchema = z.object({ + DORMICE_GATEWAY_PORT: z.coerce.number().int().min(1).max(65535).default(3677), + /** + * Absolute, like the daemon's in docker mode and for the same reason: a + * relative path depends on the start directory, and under systemd (no + * WorkingDirectory) that is `/` — a gateway started without this variable + * would keep its tables in /data, silently, until the day someone sets + * the path and every node it ever knew is gone. + */ + DORMICE_GATEWAY_DB_PATH: z + .string() + .default('/var/lib/dormice-gateway/gateway.db') + .refine((path) => path === ':memory:' || isAbsolute(path), { + error: + 'DORMICE_GATEWAY_DB_PATH must be an absolute path, e.g. /var/lib/dormice-gateway/gateway.db', + }), + /** + * The one token of the fleet. Callers present it to the gateway, nodes + * present it when they check in, and the gateway presents it to nodes + * when it forwards. Required, no default: loopback-only is not + * authentication. + */ + DORMICE_API_TOKEN: z.string().min(32, { + error: + 'DORMICE_API_TOKEN must be at least 32 characters — generate one with: openssl rand -hex 32', + }), + /** + * Placement: a node whose last whole-machine CPU reading is above this + * takes no new sandboxes. 70 leaves the headroom a wake burst needs — + * the 2026-09-11 Beijing incident saturated a 128-core host from a + * reading well under 100 in one burst of cold starts. + */ + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: z.coerce + .number() + .min(0) + .max(100) + .default(70), + /** + * Placement: a node already running this many active sandboxes (plus + * those placed on it since its last check-in) takes no more. Active + * only — frozen sandboxes cost swap, not CPU or dockerd attention, and + * a node holds thousands of them; the ceiling this guards is running + * containers per host. + */ + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: z.coerce + .number() + .int() + .positive() + .default(400), + /** + * Placement: a node whose data disk has less than this free takes no new + * sandboxes. Ten GiB is one default sandbox disk: below that, a single + * new sandbox writing its disk full would fill the host, and a full data + * disk is the one failure that stops every sandbox on the node at once + * (the ledger itself cannot write). The node's own create still answers + * its honest 500 past this point; the gate is what keeps the next ten + * names from landing on the same full box. + */ + DORMICE_GATEWAY_NODE_MIN_DISK_GB: z.coerce.number().nonnegative().default(10), +}); + +export type Config = z.infer; + +export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config { + return envSchema.parse(env); +} diff --git a/packages/gateway/src/db/db.ts b/packages/gateway/src/db/db.ts new file mode 100644 index 00000000..f0865403 --- /dev/null +++ b/packages/gateway/src/db/db.ts @@ -0,0 +1,26 @@ +import { mkdirSync } from 'node:fs'; +import { dirname } from 'node:path'; +import Database from 'better-sqlite3'; +import { drizzle } from 'drizzle-orm/better-sqlite3'; +import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; +import * as schema from './schema'; + +/** + * Opens the gateway's database — the daemon's shape (WAL: readers never + * block the single writer, a crash mid-write cannot corrupt the file). + */ +export function openDb(path: string) { + if (path !== ':memory:') { + mkdirSync(dirname(path), { recursive: true }); + } + const sqlite = new Database(path); + sqlite.pragma('journal_mode = WAL'); + return drizzle(sqlite, { schema }); +} + +export type Db = ReturnType; + +/** Applies pending migrations (drizzle-kit output, committed) at every start. */ +export function migrateDb(db: Db, migrationsFolder: string) { + migrate(db, { migrationsFolder }); +} diff --git a/packages/gateway/src/db/schema.ts b/packages/gateway/src/db/schema.ts new file mode 100644 index 00000000..227d9d5b --- /dev/null +++ b/packages/gateway/src/db/schema.ts @@ -0,0 +1,32 @@ +import { sqliteTable, text } from 'drizzle-orm/sqlite-core'; + +/** + * The gateway's tables are "how the fleet is configured and who may + * enter" — never a sandbox's state, which lives in the ledger of the node + * that runs it and is asked for when needed (find.ts). One table today; + * api_keys, settings, templates and console_account arrive here with the + * configuration authority. + */ + +/** + * Every node that has ever checked in (routes/nodes.ts): its id, the + * address the gateway forwards to, and when it first appeared. Written by + * the nodes themselves at their first check-in — there is no registration + * verb and no nodes file, so "which nodes exist" has exactly one home — + * and deleted only by an operator's removeNode. Persistent, not memory, + * for one reason: a node that is down must still be known after a gateway + * restart, or a name that lives only there would be placed anew elsewhere + * and come back as a conflict when the node returns. Everything the node + * last reported (its reading, build, check-in time) is memory: fifteen + * seconds later it is reported again. + */ +export const nodes = sqliteTable('nodes', { + /** DORMICE_NODE_ID as the node states it — the `nodeId` in every sandbox answer. */ + id: text('id').primaryKey(), + /** Where the gateway forwards to; updated when a check-in states a new one. */ + endpoint: text('endpoint').notNull(), + /** ISO 8601 UTC — the first check-in. */ + addedAt: text('added_at').notNull(), +}); + +export type NodeRow = typeof nodes.$inferSelect; diff --git a/packages/gateway/src/errors.ts b/packages/gateway/src/errors.ts new file mode 100644 index 00000000..b310db13 --- /dev/null +++ b/packages/gateway/src/errors.ts @@ -0,0 +1,117 @@ +import type http from 'node:http'; +import { UnreachableError } from './forward'; + +/** What relay needs of a logger — pino's and Fastify's request logger both fit. */ +export interface ErrorLog { + error(obj: unknown, msg?: string): void; +} + +/** + * Every face the gateway fronts has its own error dialect, and an error + * the gateway itself produces must wear the dialect of the face it was + * asked on — a client library parses what it expects: + * native { message } the daemon's API + * control { code: , message } E2B's control plane (openapi-fetch checks error.code === 404) + * connect { code: '', message } E2B's envd (Connect RPC's string codes) + * Errors a node produced are forwarded verbatim and never re-dressed. + */ +export type Dialect = 'native' | 'control' | 'connect'; + +export interface RenderedError { + status: number; + message: string; + /** Connect's string code, used by the connect dialect only. */ + connectCode?: string; + /** Browser-consumable faces carry CORS on errors too, or the refusal is unreadable. */ + cors?: boolean; + /** Seconds — on a 503 whose cause the caller may simply outwait. */ + retryAfterSeconds?: number; +} + +export function errorBody(dialect: Dialect, error: RenderedError): string { + switch (dialect) { + case 'native': + return JSON.stringify({ message: error.message }); + case 'control': + return JSON.stringify({ code: error.status, message: error.message }); + case 'connect': + return JSON.stringify({ + code: error.connectCode ?? 'unknown', + message: error.message, + }); + } +} + +/** Writes an error the gateway produced, on a raw response, in the face's dialect. */ +export function renderError( + res: http.ServerResponse, + dialect: Dialect, + error: RenderedError, +): void { + if (res.headersSent || res.destroyed) { + res.destroy(); + return; + } + const body = errorBody(dialect, error); + const headers: http.OutgoingHttpHeaders = { + 'content-type': 'application/json', + 'content-length': String(Buffer.byteLength(body)), + }; + if (error.cors) headers['access-control-allow-origin'] = '*'; + if (error.retryAfterSeconds !== undefined) { + headers['retry-after'] = String(error.retryAfterSeconds); + } + res.writeHead(error.status, headers); + res.end(body); +} + +/** The preflight answer of the browser-consumable faces — the daemon's cors.ts shape. */ +export function sendPreflight( + req: http.IncomingMessage, + res: http.ServerResponse, +): void { + const requested = req.headers['access-control-request-headers']; + res.writeHead(204, { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET, POST, OPTIONS', + 'access-control-allow-headers': + typeof requested === 'string' && requested !== '' ? requested : '*', + 'access-control-max-age': '7200', + }); + res.end(); +} + +/** + * Runs one forwarding step on a response the gateway owns (a hijacked + * reply, a raw face) and turns its failure into an answer in the face's + * dialect: the node's transport failure is a 502 rendered by `unreachable` + * (what the caller may do next), anything else a 500 that sends the + * operator to the log. Never silence: Fastify writes nothing for a + * hijacked reply whose handler throws, the gateway has no request timeout + * of its own, and a raw face has no framework behind it at all — so a + * step that threw after the node had answered would leave the caller + * waiting forever with the sandbox already built (found by review, + * 2026-09-12). + */ +export async function relay( + res: http.ServerResponse, + dialect: Dialect, + log: ErrorLog, + step: () => Promise, + unreachable: (error: UnreachableError) => RenderedError, +): Promise { + try { + await step(); + } catch (error) { + if (error instanceof UnreachableError) { + renderError(res, dialect, unreachable(error)); + return; + } + log.error(error, 'forwarding failed on the gateway'); + renderError(res, dialect, { + status: 500, + message: 'the gateway failed while forwarding — see its log', + cors: dialect === 'connect', + }); + } +} diff --git a/packages/gateway/src/find.test.ts b/packages/gateway/src/find.test.ts new file mode 100644 index 00000000..f09805af --- /dev/null +++ b/packages/gateway/src/find.test.ts @@ -0,0 +1,252 @@ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { fileURLToPath } from 'node:url'; +import { afterEach, describe, expect, it } from 'vitest'; +import { NameCache } from './cache'; +import { migrateDb, openDb } from './db/db'; +import { Finder } from './find'; +import { Fleet } from './fleet'; +import { type AskNode, httpAskNode, type LookupAnswer } from './lookup'; +import { checkInOf } from './testing'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const TOKEN = 'shared-token-shared-token-shared-token'; +const NOW = new Date('2026-09-14T12:00:00.000Z'); + +function fleetOf(...ids: string[]) { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const fleet = new Fleet(db); + for (const id of ids) fleet.checkIn(checkInOf(id, `http://${id}:80`), NOW); + return fleet; +} + +const silentLog = { warn: () => {} }; + +/** An asker scripted per node: what each node says to any question, and how often it was asked. */ +function scripted(script: Record) { + const asked: string[] = []; + const ask: AskNode = async (node) => { + asked.push(node.id); + return script[node.id] ?? { kind: 'absent' }; + }; + return { ask, asked }; +} + +describe('Finder', () => { + it('asks every node in parallel; exactly one yes wins and is cached by name and id', async () => { + const fleet = fleetOf('a', 'b', 'c'); + const { ask, asked } = scripted({ + b: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + }); + const cache = new NameCache(); + const finder = new Finder(fleet, cache, ask, silentLog); + const found = await finder.byName('alice'); + expect(found).toMatchObject({ kind: 'one', id: 'sb-1', name: 'alice' }); + expect(found.kind === 'one' && found.node.id).toBe('b'); + expect(asked.sort()).toEqual(['a', 'b', 'c']); + // The cache answers the next questions, by either handle, without a + // single node asked. + asked.length = 0; + expect((await finder.byName('alice')).kind).toBe('one'); + expect((await finder.byId('sb-1')).kind).toBe('one'); + expect(asked).toEqual([]); + }); + + it('one yes wins even while another node is silent: the sandbox is where it says it is', async () => { + const fleet = fleetOf('a', 'b'); + const { ask } = scripted({ + a: { kind: 'silent', why: 'ECONNREFUSED' }, + b: { kind: 'found', id: 'sb-1', name: 'alice', state: 'frozen' }, + }); + const found = await new Finder( + fleet, + new NameCache(), + ask, + silentLog, + ).byName('alice'); + expect(found.kind === 'one' && found.node.id).toBe('b'); + }); + + it('two yeses are a conflict naming both nodes, and nothing is cached', async () => { + const fleet = fleetOf('a', 'b', 'c'); + const { ask } = scripted({ + c: { kind: 'found', id: 'sb-2', name: 'alice', state: 'active' }, + a: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + }); + const cache = new NameCache(); + const found = await new Finder(fleet, cache, ask, silentLog).byName( + 'alice', + ); + expect(found).toEqual({ kind: 'conflict', nodeIds: ['a', 'c'] }); + expect(cache.size).toBe(0); + }); + + it('every node says no: the name is new; a silent node among the noes: unsure, naming it', async () => { + const fleet = fleetOf('a', 'b'); + expect( + await new Finder( + fleet, + new NameCache(), + scripted({}).ask, + silentLog, + ).byName('nobody'), + ).toEqual({ kind: 'none' }); + const warned: unknown[] = []; + const unsure = await new Finder( + fleet, + new NameCache(), + scripted({ b: { kind: 'silent', why: 'timeout' } }).ask, + { warn: (obj) => warned.push(obj) }, + ).byName('nobody'); + expect(unsure).toEqual({ + kind: 'unsure', + silent: [{ nodeId: 'b', why: 'timeout' }], + }); + expect(warned).toHaveLength(1); + }); + + it('an empty fleet finds nothing and asks nobody', async () => { + const { ask, asked } = scripted({}); + expect( + await new Finder(fleetOf(), new NameCache(), ask, silentLog).byName('x'), + ).toEqual({ kind: 'none' }); + expect(asked).toEqual([]); + }); + + it('a cached entry whose node was removed is dropped and the fleet asked afresh', async () => { + const fleet = fleetOf('a', 'b'); + const { ask, asked } = scripted({ + a: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + }); + const cache = new NameCache(); + const finder = new Finder(fleet, cache, ask, silentLog); + await finder.byName('alice'); + fleet.remove('a'); + asked.length = 0; + expect(await finder.byName('alice')).toEqual({ kind: 'none' }); + expect(asked).toEqual(['b']); + expect(cache.size).toBe(0); + }); + + it('verify: a node that says absent loses the entry; a silent node keeps it; a removed node loses it', async () => { + const fleet = fleetOf('a', 'b'); + const script: Record = { + a: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + }; + const cache = new NameCache(); + const finder = new Finder( + fleet, + cache, + async (node) => script[node.id] ?? { kind: 'absent' }, + silentLog, + ); + await finder.byName('alice'); + const entry = { id: 'sb-1', name: 'alice', nodeId: 'a' }; + script.a = { kind: 'silent', why: 'timeout' }; + await finder.verify(entry); + expect(cache.getByName('alice')).toEqual(entry); + script.a = { kind: 'absent' }; + await finder.verify(entry); + expect(cache.getByName('alice')).toBeUndefined(); + expect(cache.getById('sb-1')).toBeUndefined(); + + cache.put(entry); + fleet.remove('a'); + await finder.verify(entry); + expect(cache.size).toBe(0); + }); +}); + +describe('NameCache', () => { + it('a name that moved to a new id drops the old id; an id that gained a name is reachable by both', () => { + const cache = new NameCache(); + cache.put({ id: 'old', name: 'alice', nodeId: 'a' }); + cache.put({ id: 'new', name: 'alice', nodeId: 'b' }); + expect(cache.getById('old')).toBeUndefined(); + expect(cache.getByName('alice')?.nodeId).toBe('b'); + cache.put({ id: 'anon', name: null, nodeId: 'a' }); + cache.put({ id: 'anon', name: 'e2b-1', nodeId: 'a' }); + expect(cache.getByName('e2b-1')?.id).toBe('anon'); + expect(cache.size).toBe(2); + expect(cache.evictNode('a')).toBe(1); + expect(cache.getById('anon')).toBeUndefined(); + expect(cache.size).toBe(1); + }); +}); + +describe('httpAskNode', () => { + const servers: http.Server[] = []; + afterEach(async () => { + await Promise.all( + servers + .splice(0) + .map((s) => new Promise((resolve) => s.close(() => resolve()))), + ); + }); + + async function node( + handler: ( + body: unknown, + headers: http.IncomingHttpHeaders, + ) => { status: number; body: string }, + ) { + const server = http.createServer((req, res) => { + let text = ''; + req.on('data', (c) => { + text += c; + }); + req.on('end', () => { + const answer = handler(JSON.parse(text), req.headers); + res.writeHead(answer.status, { 'content-type': 'application/json' }); + res.end(answer.body); + }); + }); + servers.push(server); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + return { + id: 'n', + endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + }; + } + + it('posts the question under the fleet token and reads found / absent; a non-200 or a refused connection is silent with the reason', async () => { + const seen: unknown[] = []; + const yes = await node((body, headers) => { + seen.push({ body, auth: headers.authorization }); + return { + status: 200, + body: JSON.stringify({ + found: true, + sandbox: { id: 'sb-1', name: 'alice', state: 'active' }, + }), + }; + }); + const ask = httpAskNode(TOKEN); + expect(await ask(yes, { name: 'alice' })).toEqual({ + kind: 'found', + id: 'sb-1', + name: 'alice', + state: 'active', + }); + expect(seen).toEqual([ + { body: { name: 'alice' }, auth: `Bearer ${TOKEN}` }, + ]); + const no = await node(() => ({ + status: 200, + body: JSON.stringify({ found: false }), + })); + expect(await ask(no, { id: 'x' })).toEqual({ kind: 'absent' }); + const refusing = await node(() => ({ + status: 401, + body: '{"message":"missing or invalid API token"}', + })); + expect(await ask(refusing, { id: 'x' })).toMatchObject({ + kind: 'silent', + why: expect.stringMatching(/answered 401/), + }); + expect( + await ask({ id: 'gone', endpoint: 'http://127.0.0.1:9' }, { id: 'x' }), + ).toMatchObject({ kind: 'silent', why: 'ECONNREFUSED' }); + }); +}); diff --git a/packages/gateway/src/find.ts b/packages/gateway/src/find.ts new file mode 100644 index 00000000..d6f2ef99 --- /dev/null +++ b/packages/gateway/src/find.ts @@ -0,0 +1,122 @@ +import type { CacheEntry, NameCache } from './cache'; +import type { Fleet, NodeState } from './fleet'; +import type { AskNode, LookupQuery } from './lookup'; + +/** + * Where a sandbox is, adjudicated once for every face: + * one exactly one node holds it — route there + * conflict several nodes hold it — refused, never guessed about (an + * operator destroys one copy directly on its node) + * none every node answered, none holds it — the name is new + * unsure no node holds it as far as anyone answered, but a node did + * not answer, so "new" cannot be proven: a name that lives + * only on a silent node must not be built a second time + * elsewhere (design record #28: a 503 with Retry-After, the + * retry is the application's) + */ +export type Found = + | { kind: 'one'; node: NodeState; id: string; name: string | null } + | { kind: 'conflict'; nodeIds: string[] } + | { kind: 'none' } + | { kind: 'unsure'; silent: Array<{ nodeId: string; why: string }> }; + +export interface FinderLog { + warn(obj: unknown, msg: string): void; +} + +/** + * Finds a sandbox by asking. The cache answers first; on a miss every + * node in the fleet is asked in parallel (lookup.ts), and exactly one + * "yes" wins — the sandbox is wherever it says it is, whether or not some + * other node was slow to say no. Down nodes are asked like any other: + * the reading a node last reported says where NOT to place, never where + * a sandbox is; a node that has missed its check-ins but still answers a + * lookup still holds its sandboxes. + */ +export class Finder { + constructor( + private readonly fleet: Fleet, + readonly cache: NameCache, + private readonly ask: AskNode, + private readonly log: FinderLog, + ) {} + + byName(name: string): Promise { + return this.find(this.cache.getByName(name), { name }); + } + + byId(id: string): Promise { + return this.find(this.cache.getById(id), { id }); + } + + private async find( + cached: CacheEntry | undefined, + query: LookupQuery, + ): Promise { + if (cached !== undefined) { + const node = this.fleet.get(cached.nodeId); + // A node the operator removed while the entry was cached: the + // entry is stale by definition, and the fleet is asked afresh. + if (node !== undefined) { + return { kind: 'one', node, id: cached.id, name: cached.name }; + } + this.cache.evict(cached); + } + const members = this.fleet.all(); + const answers = await Promise.all( + members.map(async (node) => ({ + node, + answer: await this.ask(node, query), + })), + ); + const found = answers.flatMap(({ node, answer }) => + answer.kind === 'found' ? [{ node, answer }] : [], + ); + const first = found[0]; + if (found.length === 1 && first !== undefined) { + const entry = { + id: first.answer.id, + name: first.answer.name, + nodeId: first.node.id, + }; + this.cache.put(entry); + return { kind: 'one', node: first.node, id: entry.id, name: entry.name }; + } + if (found.length > 1) { + return { + kind: 'conflict', + nodeIds: found.map((f) => f.node.id).sort(), + }; + } + const silent = answers.flatMap(({ node, answer }) => + answer.kind === 'silent' ? [{ nodeId: node.id, why: answer.why }] : [], + ); + if (silent.length > 0) { + this.log.warn( + { query, silent }, + 'lookup: a node did not answer; a name it may hold cannot be treated as new', + ); + return { kind: 'unsure', silent }; + } + return { kind: 'none' }; + } + + /** + * After a cached node answered 404 for a name: is the sandbox really + * gone from it? A node's 404 is not "no such sandbox" by itself — + * readFile answers 404 for a missing path too — so the node is asked + * the one question that means exactly that, and only a plain "absent" + * evicts. Silence keeps the entry: the node may be down, and its + * sandboxes are still there. Off the request path (the 404 has already + * been relayed); the next request for the name asks the fleet afresh. + */ + async verify(entry: CacheEntry): Promise { + const node = this.fleet.get(entry.nodeId); + if (node === undefined) { + this.cache.evict(entry); + return; + } + const answer = await this.ask(node, { id: entry.id }); + if (answer.kind === 'absent') this.cache.evict(entry); + } +} diff --git a/packages/gateway/src/fleet.test.ts b/packages/gateway/src/fleet.test.ts new file mode 100644 index 00000000..18e4b523 --- /dev/null +++ b/packages/gateway/src/fleet.test.ts @@ -0,0 +1,94 @@ +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { migrateDb, openDb } from './db/db'; +import { downReason, Fleet } from './fleet'; +import { checkInOf } from './testing'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const NOW = new Date('2026-09-14T12:00:00.000Z'); + +function db() { + const handle = openDb(':memory:'); + migrateDb(handle, MIGRATIONS); + return handle; +} + +describe('Fleet', () => { + it('a first check-in joins the node and persists its row; a gateway restart still knows it, unreached until it checks in again', () => { + const handle = db(); + const fleet = new Fleet(handle); + expect(fleet.all()).toEqual([]); + const { node, joined } = fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), + NOW, + ); + expect(joined).toBe(true); + expect(node.addedAt).toBe(NOW.toISOString()); + expect(node.lastCheckInAt).toBe(NOW); + expect(node.reading?.host.cpuCount).toBe(8); + expect(downReason(node, NOW)).toBeNull(); + + // The same database after a restart: the row is there, the memory is not. + const restarted = new Fleet(handle); + const known = restarted.get('node-b'); + expect(known?.endpoint).toBe('http://10.0.0.7:80'); + expect(known?.addedAt).toBe(NOW.toISOString()); + expect(known?.reading).toBeNull(); + expect(downReason(known as NonNullable, NOW)).toBe( + 'has not checked in since the gateway started', + ); + }); + + it('a later check-in is not a join; a changed endpoint is written through; the placement counter restarts', () => { + const handle = db(); + const fleet = new Fleet(handle); + const first = fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW); + first.node.placedSinceCheckIn = 3; + const second = fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.8:80', { active: 12 }), + new Date(NOW.getTime() + 15_000), + ); + expect(second.joined).toBe(false); + expect(second.node).toBe(first.node); + expect(second.node.endpoint).toBe('http://10.0.0.8:80'); + expect(second.node.reading?.sandboxes.byState.active).toBe(12); + expect(second.node.placedSinceCheckIn).toBe(0); + expect(new Fleet(handle).get('node-b')?.endpoint).toBe( + 'http://10.0.0.8:80', + ); + }); + + it('downReason: fresh within two of its own intervals, down past them', () => { + const fleet = new Fleet(db()); + const { node } = fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), + NOW, + ); + expect(downReason(node, new Date(NOW.getTime() + 29_000))).toBeNull(); + expect(downReason(node, new Date(NOW.getTime() + 31_000))).toBe( + 'has not checked in for 31s', + ); + // A one-second node (the exam's) is judged by its own interval. + const quick = fleet.checkIn( + checkInOf('node-c', 'http://10.0.0.9:80', { intervalSeconds: 1 }), + NOW, + ).node; + expect(downReason(quick, new Date(NOW.getTime() + 1_500))).toBeNull(); + expect(downReason(quick, new Date(NOW.getTime() + 2_500))).toBe( + 'has not checked in for 3s', + ); + }); + + it('remove forgets the node and its row; removing an unknown id is false; a node that checks in again re-joins', () => { + const handle = db(); + const fleet = new Fleet(handle); + fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW); + expect(fleet.remove('node-b')).toBe(true); + expect(fleet.all()).toEqual([]); + expect(new Fleet(handle).all()).toEqual([]); + expect(fleet.remove('node-b')).toBe(false); + expect( + fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW).joined, + ).toBe(true); + }); +}); diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts new file mode 100644 index 00000000..42967ada --- /dev/null +++ b/packages/gateway/src/fleet.ts @@ -0,0 +1,132 @@ +import type { BuildInfo, CheckInRequest, NodeReading } from '@dormice/shared'; +import { eq } from 'drizzle-orm'; +import type { Db } from './db/db'; +import { nodes } from './db/schema'; + +/** + * A node as the gateway knows it: the persistent row (id, endpoint, + * addedAt) and what it last reported (memory — reported again at the next + * check-in, gone with the process and rightly so). placedSinceCheckIn + * counts the sandboxes the gateway sent here since the last reading, so a + * burst inside one interval is counted against the node before its next + * reading shows it (placement.ts). + */ +export interface NodeState { + readonly id: string; + endpoint: string; + readonly addedAt: string; + lastCheckInAt: Date | null; + intervalSeconds: number | null; + build: BuildInfo | null; + reading: NodeReading | null; + placedSinceCheckIn: number; +} + +/** + * Why a node is not to be placed on right now, or null when it is fine: + * never checked in since this gateway started, or silent for two of its + * own intervals — the interval it stated in its last check-in, so the + * exam's one-second nodes and production's fifteen-second nodes are judged + * by the same rule. Two, not one: a check-in delayed by a busy event loop + * or a slow reading is normal; two in a row missing is a node in trouble. + * The same word answers listNodes' `reachable`. + */ +export function downReason(node: NodeState, now: Date): string | null { + if (node.lastCheckInAt === null || node.intervalSeconds === null) { + return 'has not checked in since the gateway started'; + } + const silentMs = now.getTime() - node.lastCheckInAt.getTime(); + if (silentMs > 2 * node.intervalSeconds * 1000) { + return `has not checked in for ${Math.round(silentMs / 1000)}s`; + } + return null; +} + +/** + * The fleet: every node that has ever checked in. Rows come from the + * database at start (a node that is down must still be known — its names + * are not new names); everything else fills in as the nodes report. + */ +export class Fleet { + private readonly members = new Map(); + + constructor(private readonly db: Db) { + for (const row of db.select().from(nodes).all()) { + this.members.set(row.id, { + id: row.id, + endpoint: row.endpoint, + addedAt: row.addedAt, + lastCheckInAt: null, + intervalSeconds: null, + build: null, + reading: null, + placedSinceCheckIn: 0, + }); + } + } + + all(): NodeState[] { + return [...this.members.values()]; + } + + get(id: string): NodeState | undefined { + return this.members.get(id); + } + + /** + * A node reporting for duty. A first check-in adds the node (`joined` + * says so, for the log); a changed endpoint is written through — the + * node states where it lives, the gateway does not remember better. The + * placement counter restarts at zero: what was placed before this + * reading is in it now, and what is still in flight on the node (its + * row is written after the container is up) is in neither figure until + * the next reading — one interval of slack, self-correcting, the same + * as before. + */ + checkIn( + report: CheckInRequest, + now = new Date(), + ): { node: NodeState; joined: boolean } { + let node = this.members.get(report.nodeId); + let joined = false; + if (node === undefined) { + const addedAt = now.toISOString(); + this.db + .insert(nodes) + .values({ id: report.nodeId, endpoint: report.endpoint, addedAt }) + .run(); + node = { + id: report.nodeId, + endpoint: report.endpoint, + addedAt, + lastCheckInAt: null, + intervalSeconds: null, + build: null, + reading: null, + placedSinceCheckIn: 0, + }; + this.members.set(node.id, node); + joined = true; + } else if (node.endpoint !== report.endpoint) { + this.db + .update(nodes) + .set({ endpoint: report.endpoint }) + .where(eq(nodes.id, report.nodeId)) + .run(); + node.endpoint = report.endpoint; + } + node.lastCheckInAt = now; + node.intervalSeconds = report.intervalSeconds; + node.build = report.build; + node.reading = report.reading; + node.placedSinceCheckIn = 0; + return { node, joined }; + } + + /** The operator's word that the node is gone for good; a node still running re-adds itself at its next check-in. */ + remove(id: string): boolean { + const existed = this.members.delete(id); + this.db.delete(nodes).where(eq(nodes.id, id)).run(); + return existed; + } +} diff --git a/packages/gateway/src/forward.test.ts b/packages/gateway/src/forward.test.ts new file mode 100644 index 00000000..48457edd --- /dev/null +++ b/packages/gateway/src/forward.test.ts @@ -0,0 +1,734 @@ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import net from 'node:net'; +import { request } from 'undici'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + forwardCapture, + forwardStream, + forwardUpgrade, + replay, + UnreachableError, +} from './forward'; + +const TOKEN = 'node-token-node-token-node-token-node'; +const servers: http.Server[] = []; +// Every accepted socket, upgraded ones included: http.Server#close waits +// for sockets it no longer tracks after an upgrade (the daemon's +// shutdown.ts has the reference), so teardown cuts them by hand. +const sockets = new Set(); +afterEach(async () => { + for (const socket of sockets) socket.destroy(); + sockets.clear(); + await Promise.all( + servers + .splice(0) + .map((s) => new Promise((resolve) => s.close(() => resolve()))), + ); +}); + +async function listen(server: http.Server): Promise { + servers.push(server); + server.on('connection', (socket) => { + sockets.add(socket); + socket.once('close', () => sockets.delete(socket)); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +/** A gateway-shaped front: every request is forwarded to `endpoint` with the given options. */ +async function front( + endpoint: string, + options: Partial[2]> = {}, +): Promise { + const server = http.createServer((req, res) => { + void forwardStream(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + ...options, + }).catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + res.writeHead(502, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ message })); + }); + }); + return listen(server); +} + +describe('forwardStream', () => { + it('streams the answer frame by frame: the first frame reaches the client before the node writes the second', async () => { + let releaseSecond: () => void = () => {}; + const gate = new Promise((resolve) => { + releaseSecond = resolve; + }); + const node = http.createServer((_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain' }); + res.write('first\n'); + void gate.then(() => res.end('second\n')); + }); + const router = await front(await listen(node)); + + const res = await request(`${router}/execCommand`, { method: 'POST' }); + const chunks: string[] = []; + const reader = res.body[Symbol.asyncIterator](); + const first = await reader.next(); + chunks.push(String(first.value)); + // Proof of streaming: the client holds the first frame while the node + // is still parked before its second. A buffering front could not get here. + expect(chunks).toEqual(['first\n']); + releaseSecond(); + for await (const chunk of { [Symbol.asyncIterator]: () => reader }) { + chunks.push(String(chunk)); + } + expect(chunks.join('')).toBe('first\nsecond\n'); + expect(res.headers['transfer-encoding']).toBe('chunked'); + }); + + it("resolves with the node's status once the answer is relayed — the named verbs read a 404 off it", async () => { + const node = http.createServer((_req, res) => { + res.writeHead(404, { 'content-type': 'application/json' }); + res.end('{"message":"no sandbox named \\"x\\""}'); + }); + const endpoint = await listen(node); + let resolved: number | null | undefined; + const front = http.createServer((req, res) => { + void forwardStream(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + }).then((status) => { + resolved = status; + }); + }); + const url = await listen(front); + const res = await request(`${url}/execCommand`, { method: 'POST' }); + expect(res.statusCode).toBe(404); + await res.body.text(); + const deadline = Date.now() + 1_000; + while (resolved === undefined && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(resolved).toBe(404); + }); + + it('keeps a fixed-length answer fixed-length and passes response headers through — pagination, cookies, CORS', async () => { + const node = http.createServer((_req, res) => { + res.writeHead(200, { + 'content-type': 'application/json', + 'content-length': '5', + 'x-next-token': 'abc', + 'set-cookie': ['a=1', 'b=2'], + 'access-control-allow-origin': '*', + }); + res.end('hello'); + }); + const router = await front(await listen(node)); + const res = await request(`${router}/listSandboxes`, { method: 'POST' }); + expect(res.statusCode).toBe(200); + expect(res.headers['content-length']).toBe('5'); + expect(res.headers['transfer-encoding']).toBeUndefined(); + expect(res.headers['x-next-token']).toBe('abc'); + expect(res.headers['set-cookie']).toEqual(['a=1', 'b=2']); + expect(res.headers['access-control-allow-origin']).toBe('*'); + expect(await res.body.text()).toBe('hello'); + }); + + it('a client that goes away mid-stream takes the node connection down with it', async () => { + let nodeSawClose: () => void = () => {}; + const closed = new Promise((resolve) => { + nodeSawClose = resolve; + }); + const node = http.createServer((req, res) => { + res.writeHead(200); + res.write('tick\n'); + const timer = setInterval(() => res.write('tick\n'), 20); + req.on('close', () => { + clearInterval(timer); + nodeSawClose(); + }); + }); + const router = await front(await listen(node)); + const res = await request(`${router}/stream`, { method: 'POST' }); + const reader = res.body[Symbol.asyncIterator](); + await reader.next(); + res.body.destroy(); + await closed; + }); + + it('forwards the request verbatim — path, query, body — with the credential swapped and the Host renamed to the node unless the face keeps it', async () => { + const seen: Array<{ + url?: string; + headers: http.IncomingHttpHeaders; + body: string; + }> = []; + const node = http.createServer((req, res) => { + let body = ''; + req.on('data', (c) => { + body += c; + }); + req.on('end', () => { + seen.push({ url: req.url, headers: req.headers, body }); + res.writeHead(200); + res.end(); + }); + }); + const endpoint = await listen(node); + const router = await front(endpoint, { credential: 'bearer' }); + await request(`${router}/readFile?x=1`, { + method: 'POST', + headers: { + host: '8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test', + authorization: 'Bearer caller-secret', + 'content-type': 'application/json', + connection: 'keep-alive', + }, + body: '{"name":"alice"}', + }); + expect(seen).toHaveLength(1); + expect(seen[0]?.url).toBe('/readFile?x=1'); + expect(seen[0]?.body).toBe('{"name":"alice"}'); + // The Host named the gateway; the node is asked as itself, so a Caddy + // in front of it that binds the gateway's domain never answers a 308. + expect(seen[0]?.headers.host).toBe(new URL(endpoint).host); + expect(seen[0]?.headers.authorization).toBe(`Bearer ${TOKEN}`); + expect(seen[0]?.headers['content-type']).toBe('application/json'); + + // The E2B face swaps the other header; the credential-less faces touch none. + const asE2b = await front(endpoint, { credential: 'x-api-key' }); + await request(`${asE2b}/sandboxes`, { + method: 'POST', + headers: { 'x-api-key': 'e2b_caller' }, + }); + expect(seen[1]?.headers['x-api-key']).toBe(`e2b_${TOKEN}`); + const asEnvd = await front(endpoint, { credential: 'none' }); + await request(`${asEnvd}/e2b/envd/files`, { + method: 'GET', + headers: { authorization: 'Basic dXNlcjo=', 'x-access-token': 'hmac' }, + }); + expect(seen[2]?.headers.authorization).toBe('Basic dXNlcjo='); + expect(seen[2]?.headers['x-access-token']).toBe('hmac'); + + // The proxy face is the one that keeps the Host: it is the routing + // key on the node as well. + const asProxy = await front(endpoint, { + credential: 'none', + preserveHost: true, + }); + await request(`${asProxy}/`, { + method: 'GET', + headers: { host: '8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test' }, + }); + expect(seen[3]?.headers.host).toBe( + '8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test', + ); + }); + + it('sends the request target byte for byte — dot segments included — instead of a URL it resolved itself', async () => { + const seen: string[] = []; + const node = http.createServer((req, res) => { + seen.push(req.url ?? ''); + res.writeHead(200); + res.end(); + }); + const endpoint = await listen(node); + const router = await front(endpoint); + const url = new URL(router); + // A client library would collapse the segments before they left; the + // socket carries them as an attacker would. + const path = '/e2b/api/sandboxes/some-id/../../sandboxes?x=1'; + await new Promise((resolve, reject) => { + const socket = net.connect(Number(url.port), url.hostname, () => { + socket.write( + `GET ${path} HTTP/1.1\r\nhost: router\r\nconnection: close\r\n\r\n`, + ); + }); + socket.on('data', () => {}); + socket.on('end', resolve); + socket.on('error', reject); + }); + expect(seen).toEqual([path]); + }); + + it('drops the Expect header: the 100-continue handshake ended at the gateway, and undici refuses to carry it', async () => { + const seen: http.IncomingHttpHeaders[] = []; + const node = http.createServer((req, res) => { + req.resume(); + req.on('end', () => { + seen.push(req.headers); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end('{"ok":true}'); + }); + }); + const endpoint = await listen(node); + const router = await front(endpoint); + // curl adds this to every body over 1024 bytes (undici's own client + // refuses to send it, so the exam speaks node:http): Node's server + // answered the 100 before forwardStream saw the request. + const body = JSON.stringify({ name: 'big', content: 'x'.repeat(2048) }); + const res = await new Promise<{ status: number; text: string }>( + (resolve, reject) => { + const req = http.request( + `${router}/writeFile`, + { + method: 'POST', + headers: { + expect: '100-continue', + 'content-type': 'application/json', + 'content-length': String(Buffer.byteLength(body)), + }, + }, + (answer) => { + let text = ''; + answer.on('data', (c) => { + text += c; + }); + answer.on('end', () => + resolve({ status: answer.statusCode ?? 0, text }), + ); + }, + ); + req.on('continue', () => req.end(body)); + req.on('error', reject); + }, + ); + expect(res.status).toBe(200); + expect(JSON.parse(res.text)).toEqual({ ok: true }); + expect(seen[0]?.expect).toBeUndefined(); + }); + + it('a node that does not answer is an UnreachableError, tried exactly once', async () => { + let hits = 0; + const node = http.createServer((req) => { + hits += 1; + req.socket.destroy(); + }); + const endpoint = await listen(node); + const router = await front(endpoint); + const cut = await request(`${router}/acquireSandbox`, { method: 'POST' }); + expect(cut.statusCode).toBe(502); + expect((await cut.body.json()) as { message: string }).toMatchObject({ + message: expect.stringMatching( + /^node http:\/\/127\.0\.0\.1:\d+ did not answer: /, + ), + }); + expect(hits).toBe(1); + + const refused = await front('http://127.0.0.1:9'); + const res = await request(`${refused}/acquireSandbox`, { method: 'POST' }); + expect(res.statusCode).toBe(502); + expect(((await res.body.json()) as { message: string }).message).toMatch( + /did not answer: ECONNREFUSED/, + ); + }); +}); + +describe('forwardCapture + replay', () => { + it('collects the answer whole and replays it verbatim, asks for it uncompressed, and reports a refused connection as unreachable', async () => { + const encodings: Array = []; + const node = http.createServer((req, res) => { + encodings.push(req.headers['accept-encoding']); + res.writeHead(201, { + 'content-type': 'application/json', + 'x-extra': 'y', + }); + res.end('{"sandboxID":"s-1"}'); + }); + const endpoint = await listen(node); + const server = http.createServer((req, res) => { + void forwardCapture(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'x-api-key', + }).then( + (answer) => { + if (answer === null) throw new Error('the client did not leave'); + expect(answer.status).toBe(201); + expect(JSON.parse(answer.body.toString())).toEqual({ + sandboxID: 's-1', + }); + replay(res, answer); + }, + (error: unknown) => { + res.writeHead(502); + res.end(error instanceof Error ? error.name : 'other'); + }, + ); + }); + const router = await listen(server); + const res = await request(`${router}/sandboxes`, { + method: 'POST', + headers: { 'accept-encoding': 'gzip, br' }, + }); + expect(res.statusCode).toBe(201); + expect(res.headers['x-extra']).toBe('y'); + expect(res.headers['content-length']).toBe('19'); + expect(await res.body.json()).toEqual({ sandboxID: 's-1' }); + // A compressing hop between router and node would hand back bytes + // the gateway cannot read the id out of. + expect(encodings).toEqual(['identity']); + + const orphan = new http.IncomingMessage(new net.Socket()); + await expect( + forwardCapture( + Object.assign(orphan, { url: '/x', method: 'POST', headers: {} }), + new http.ServerResponse(orphan), + { + target: { endpoint: 'http://127.0.0.1:9', token: TOKEN }, + credential: 'bearer', + body: Buffer.from('{}'), + }, + ), + ).rejects.toBeInstanceOf(UnreachableError); + }); +}); + +describe('forwardCapture + replay — bodiless answers', () => { + it('replays a 204 without inventing a content-length', async () => { + const node = http.createServer((_req, res) => { + res.writeHead(204); + res.end(); + }); + const endpoint = await listen(node); + const front = http.createServer((req, res) => { + void forwardCapture(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'x-api-key', + }).then((answer) => answer && replay(res, answer)); + }); + const url = await listen(front); + const res = await request(`${url}/e2b/api/sandboxes/x`, { + method: 'DELETE', + }); + expect(res.statusCode).toBe(204); + expect(res.headers['content-length']).toBeUndefined(); + await res.body.text(); + }); +}); + +describe('forwardStream — a client that leaves', () => { + it('withdraws the request from the node when the client leaves before the node answered', async () => { + let nodeSawClose: number | null = null; + const node = http.createServer((req, res) => { + const at = Date.now(); + req.on('close', () => { + nodeSawClose = Date.now() - at; + }); + // Answers late — the client will be gone by then. + setTimeout(() => { + if (!res.destroyed) { + res.writeHead(200); + res.end('late'); + } + }, 1500); + }); + const endpoint = await listen(node); + const routerUrl = await front(endpoint); + const url = new URL(routerUrl); + const client = http.request({ + host: url.hostname, + port: url.port, + path: '/execCommand', + method: 'POST', + headers: { 'content-type': 'application/json' }, + }); + client.on('error', () => {}); + client.end('{"name":"x"}'); + await new Promise((resolve) => setTimeout(resolve, 100)); + client.destroy(); + const deadline = Date.now() + 1_000; + while (nodeSawClose === null && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + // Well before the node's own answer at 1.5s: the gateway aborted it. + expect(nodeSawClose).not.toBeNull(); + expect(nodeSawClose as unknown as number).toBeLessThan(1_000); + }); +}); + +describe('forwardCapture — a client that leaves', () => { + it('withdraws the request from the node when the client leaves before the node answered, and answers null; a client already gone is not sent at all', async () => { + let nodeSawClose: number | null = null; + let requests = 0; + const node = http.createServer((req, res) => { + requests += 1; + const at = Date.now(); + req.on('close', () => { + nodeSawClose = Date.now() - at; + }); + setTimeout(() => { + if (!res.destroyed) { + res.writeHead(200); + res.end('{}'); + } + }, 1500); + }); + const endpoint = await listen(node); + const outcomes: Array<'null' | 'answer' | 'error'> = []; + const front = http.createServer((req, res) => { + void forwardCapture(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + }).then( + (answer) => { + outcomes.push(answer === null ? 'null' : 'answer'); + if (answer !== null) replay(res, answer); + }, + () => outcomes.push('error'), + ); + }); + const url = new URL(await listen(front)); + const client = http.request({ + host: url.hostname, + port: url.port, + path: '/acquireSandbox', + method: 'POST', + headers: { 'content-type': 'application/json' }, + }); + client.on('error', () => {}); + client.end('{"name":"x"}'); + await new Promise((resolve) => setTimeout(resolve, 100)); + client.destroy(); + const deadline = Date.now() + 1_000; + while (outcomes.length === 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + // Well before the node's own answer at 1.5s: the gateway aborted it + // and told the caller there is nothing to send. + expect(outcomes).toEqual(['null']); + expect(nodeSawClose).not.toBeNull(); + expect(nodeSawClose as unknown as number).toBeLessThan(1_000); + + // A verb whose client left while it waited for its slot: the response + // is already gone when the capture begins, and the node hears nothing. + const gone = new http.IncomingMessage(new net.Socket()); + const res = new http.ServerResponse(gone); + res.destroy(); + expect( + await forwardCapture( + Object.assign(gone, { url: '/x', method: 'POST', headers: {} }), + res, + { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + body: Buffer.from('{}'), + }, + ), + ).toBeNull(); + expect(requests).toBe(1); + }); +}); + +describe('forwardCapture — the head arrived, then the client left', () => { + it('reads the small body to the end and answers it, so a 2xx that did land is still learned; replay writes nothing to nobody', async () => { + let finishBody: () => void = () => {}; + let headOut: () => void = () => {}; + const headWritten = new Promise((resolve) => { + headOut = resolve; + }); + const node = http.createServer((_req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.write('{"sandbox":{"id":"sb-1"', headOut); + finishBody = () => res.end('}}'); + }); + const endpoint = await listen(node); + let answer: Awaited> | undefined; + let clientGone: () => void = () => {}; + const gone = new Promise((resolve) => { + clientGone = resolve; + }); + const front = http.createServer((req, res) => { + res.once('close', () => clientGone()); + void forwardCapture(req, res, { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + }).then((a) => { + answer = a; + if (a !== null) replay(res, a); + }); + }); + const url = new URL(await listen(front)); + const client = http.request({ + host: url.hostname, + port: url.port, + path: '/acquireSandbox', + method: 'POST', + }); + client.on('error', () => {}); + client.end('{"name":"x"}'); + // The node's head is out; the body is not. The client gives up. + await headWritten; + client.destroy(); + await gone; + finishBody(); + const deadline = Date.now() + 2_000; + while (answer === undefined && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(answer).toBeDefined(); + expect(answer).not.toBeNull(); + expect(answer?.status).toBe(200); + expect(JSON.parse(answer?.body.toString('utf8') ?? '')).toEqual({ + sandbox: { id: 'sb-1' }, + }); + }); +}); + +describe('forwardUpgrade', () => { + it('a node behind TLS is refused with a 502 before any dial — upgrades are plain TCP here', async () => { + const router = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + router.on('upgrade', (req, socket, head) => + forwardUpgrade(req, socket, head, { + endpoint: 'https://node.example:443', + token: TOKEN, + }), + ); + const routerUrl = await listen(router); + const client = net.connect(Number(new URL(routerUrl).port), '127.0.0.1'); + const received: string[] = []; + client.on('data', (chunk) => received.push(String(chunk))); + const closed = new Promise((resolve) => client.on('close', resolve)); + client.write( + 'GET /ws HTTP/1.1\r\nHost: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + await closed; + expect(received.join('')).toContain('HTTP/1.1 502 Bad Gateway'); + expect(received.join('')).toContain('not forwarded over TLS'); + expect(received.join('')).not.toContain('node.example'); + }); + + it('a node that dies after the handshake cuts the client without writing an HTTP status into the upgraded stream', async () => { + const node = http.createServer((_req, res) => { + res.writeHead(426); + res.end(); + }); + node.on('upgrade', (req, socket) => { + socket.write( + `HTTP/1.1 101 Switching Protocols\r\nUpgrade: ${req.headers.upgrade}\r\nConnection: Upgrade\r\n\r\n`, + ); + // RST from the node side: the router's upstream sees an error, not a FIN. + socket.on('data', () => (socket as net.Socket).resetAndDestroy()); + }); + const endpoint = await listen(node); + const router = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + router.on('upgrade', (req, socket, head) => + forwardUpgrade(req, socket, head, { endpoint, token: TOKEN }), + ); + const routerUrl = await listen(router); + const client = net.connect(Number(new URL(routerUrl).port), '127.0.0.1'); + const received: string[] = []; + client.on('data', (chunk) => received.push(String(chunk))); + const closed = new Promise((resolve) => client.on('close', resolve)); + client.write( + 'GET /ws HTTP/1.1\r\nHost: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + await new Promise((resolve) => { + const check = () => { + if (received.join('').includes('101')) resolve(); + else client.once('data', check); + }; + check(); + }); + client.write('ping'); + await closed; + expect(received.join('')).not.toContain('502'); + }); + + it("a node that black-holes the dial is a 502 within the connect deadline, not the kernel's SYN budget", async () => { + const router = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + // TEST-NET-1 (RFC 5737) is never routed: the SYN is dropped, or the + // network answers unreachable at once — a 502 either way, and without + // the deadline the dropped case held this test for 75s on macOS. + router.on('upgrade', (req, socket, head) => + forwardUpgrade( + req, + socket, + head, + { endpoint: 'http://192.0.2.1:80', token: TOKEN }, + 500, + ), + ); + const routerUrl = await listen(router); + const client = net.connect(Number(new URL(routerUrl).port), '127.0.0.1'); + const received: string[] = []; + client.on('data', (chunk) => received.push(String(chunk))); + const closed = new Promise((resolve) => client.on('close', resolve)); + const started = Date.now(); + client.write( + 'GET /ws HTTP/1.1\r\nHost: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + await closed; + expect(received.join('')).toContain('HTTP/1.1 502 Bad Gateway'); + expect(received.join('')).toContain('did not answer the upgrade'); + expect(Date.now() - started).toBeLessThan(4000); + }); + + it('replays the upgrade handshake to the node and pipes both ways', async () => { + const node = http.createServer((_req, res) => { + res.writeHead(426); + res.end(); + }); + let nodeSideClosed: () => void = () => {}; + const nodeClosed = new Promise((resolve) => { + nodeSideClosed = resolve; + }); + node.on('upgrade', (req, socket) => { + socket.write( + `HTTP/1.1 101 Switching Protocols\r\nUpgrade: ${req.headers.upgrade}\r\nConnection: Upgrade\r\nX-Seen-Host: ${req.headers.host}\r\n\r\n`, + ); + socket.on('data', (chunk) => socket.write(`echo:${chunk}`)); + // A real upgraded peer (a WebSocket server in the sandbox) ends its + // side when the other side does; http sockets allow half-open, so + // without this the chain would hang on a peer that never answers FIN. + socket.on('end', () => socket.end()); + socket.on('close', nodeSideClosed); + }); + const endpoint = await listen(node); + const router = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + router.on('upgrade', (req, socket, head) => + forwardUpgrade(req, socket, head, { endpoint, token: TOKEN }), + ); + const routerUrl = await listen(router); + const port = Number(new URL(routerUrl).port); + + const client = net.connect(port, '127.0.0.1'); + const received: string[] = []; + const done = new Promise((resolve) => { + client.on('data', (chunk) => { + received.push(String(chunk)); + if (received.join('').includes('echo:ping')) resolve(); + }); + }); + client.write( + 'GET /ws HTTP/1.1\r\nHost: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + await new Promise((resolve) => { + const check = () => { + if (received.join('').includes('101')) resolve(); + else client.once('data', check); + }; + check(); + }); + client.write('ping'); + await done; + const all = received.join(''); + expect(all).toContain('HTTP/1.1 101 Switching Protocols'); + expect(all).toContain( + 'X-Seen-Host: 8000-2d5c6f0e-1111-4222-8333-444455556666.sbx.test', + ); + expect(all).toContain('echo:ping'); + // The client leaving takes the node-side socket down with it. + client.destroy(); + await nodeClosed; + }); +}); diff --git a/packages/gateway/src/forward.ts b/packages/gateway/src/forward.ts new file mode 100644 index 00000000..c025cfda --- /dev/null +++ b/packages/gateway/src/forward.ts @@ -0,0 +1,367 @@ +import type http from 'node:http'; +import net from 'node:net'; +import type { Duplex } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { Agent } from 'undici'; + +/** + * The one place the gateway talks to a node on a caller's behalf. Bytes + * in, bytes out: the request body is streamed (or handed over as the + * buffer Fastify already read), the response is written head-first and + * piped — never buffered, never re-parsed, never re-framed. Every + * long-lived shape a node produces (an exec that answers after an hour, + * an envd process stream, a proxied SSE) rides through untouched. + * + * The dispatcher switches undici's hidden clocks off, for the same reason + * the SDK's does (packages/sdk/src/client.ts): a node legitimately takes + * hours to answer an exec, and a default dispatcher gives up on headers + * after 300s. Connecting keeps a short deadline — a node that will not + * even accept the connection in ten seconds is unreachable now. + * + * Never retried, never re-routed. A create that did not answer may have + * created; sending it again elsewhere is how one name ends up on two + * nodes. The caller gets the honest 502, and its retry finds the sandbox + * by asking (find.ts) wherever it landed. + */ +const CONNECT_TIMEOUT_MS = 10_000; +const agent = new Agent({ + headersTimeout: 0, + bodyTimeout: 0, + connect: { timeout: CONNECT_TIMEOUT_MS }, +}); + +export interface ForwardTarget { + endpoint: string; + /** The fleet's one token: what the gateway presents to every node. */ + token: string; +} + +/** + * How the node is authenticated to: `bearer` replaces the Authorization + * header with the fleet token (native verbs), `x-api-key` replaces + * X-API-KEY with `e2b_` (E2B control plane), `none` touches no + * credential header at all — envd traffic carries its own (an access + * token minted by the node itself) and the gateway has nothing to add. + * No header names the real caller: the gateway is the fleet's one door + * and the node trusts it whole (design record #3). + */ +export type Credential = 'bearer' | 'x-api-key' | 'none'; + +export interface ForwardOptions { + target: ForwardTarget; + credential: Credential; + /** The request body when Fastify already consumed the stream; omit to stream req itself. */ + body?: Buffer | undefined; + /** + * Keep the caller's Host header. Only a face keyed on the Host wants + * this (the sandbox port proxy, once it routes through the gateway). + * Everywhere else the Host names the gateway, and carrying it to a node + * whose Caddy binds that very domain gets a 308 to https instead of the + * daemon — so the default lets undici name the node's own endpoint. + */ + preserveHost?: boolean; +} + +/** The node did not answer at all — no headers came back. */ +export class UnreachableError extends Error { + constructor( + readonly node: string, + /** The transport's word for why (ECONNREFUSED, a timeout, a cut body). */ + readonly why: string, + ) { + super(`node ${node} did not answer: ${why}`); + this.name = 'UnreachableError'; + } +} + +// Hop-by-hop headers describe one connection, not the message; a proxy +// that forwards them lies to the next hop about its framing. `expect` is +// one too: the 100-continue handshake is between the client and the +// first server, and Node's http answered it before this code ran (a +// server with no checkContinue listener sends 100 and reads the body). +// undici refuses to forward it outright (NotSupportedError), and curl +// adds it to every body over 1024 bytes — so without this line a 1.5MB +// writeFile through the front was a 502 (measured 2026-09-12). +const HOP_BY_HOP = new Set([ + 'connection', + 'expect', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade', +]); + +function outboundHeaders( + req: http.IncomingMessage, + options: ForwardOptions, +): Record { + const headers: Record = {}; + for (const [name, value] of Object.entries(req.headers)) { + if (value === undefined || HOP_BY_HOP.has(name)) continue; + // A buffered body is re-framed by undici; the client's length may + // describe a chunked encoding that no longer exists. + if (name === 'content-length' && options.body !== undefined) continue; + if (name === 'host' && options.preserveHost !== true) continue; + headers[name] = value; + } + switch (options.credential) { + case 'bearer': + headers.authorization = `Bearer ${options.target.token}`; + break; + case 'x-api-key': + headers['x-api-key'] = `e2b_${options.target.token}`; + break; + case 'none': + break; + } + return headers; +} + +function inboundHeaders( + headers: Record, +): http.OutgoingHttpHeaders { + const out: http.OutgoingHttpHeaders = {}; + for (const [name, value] of Object.entries(headers)) { + if (value === undefined || HOP_BY_HOP.has(name)) continue; + out[name] = value; + } + return out; +} + +function causeOf(error: unknown): string { + const e = error as { code?: string; message?: string; cause?: unknown }; + const cause = e.cause as { code?: string; message?: string } | undefined; + return cause?.code ?? cause?.message ?? e.code ?? e.message ?? String(error); +} + +/** + * The request target is sent to the node exactly as the caller wrote it — + * origin and path handed to undici separately, never joined into a URL. + * A URL parser resolves dot segments: `/e2b/api/sandboxes//../../ + * sandboxes` is a sub-route of a known id to Fastify, which routed it to + * the id's node, and `/e2b/api/sandboxes` to the parser — so a caller + * holding any live id could reach the node's create, under the fleet's + * credential, past placement and the name slot (reproduced 2026-09-12). + * The node judges the path it is sent; the gateway's routing and the + * node's must see the same bytes. + */ +async function dispatch( + req: http.IncomingMessage, + options: ForwardOptions, + override: Record = {}, + signal?: AbortSignal, +) { + try { + return await agent.request({ + origin: options.target.endpoint, + path: req.url ?? '/', + method: req.method as 'GET', + headers: { ...outboundHeaders(req, options), ...override }, + body: options.body ?? req, + signal, + }); + } catch (error) { + throw new UnreachableError(options.target.endpoint, causeOf(error)); + } +} + +/** + * Forwards the request and streams the node's answer back as it arrives; + * resolves with the node's status once the answer has been relayed (the + * named verbs read a 404 off it to re-check the cache — find.ts verify). + * Throws UnreachableError only before any byte of the answer was written; + * once the head is out, a failure mid-stream can only be a cut connection + * — there is no honest status left to send. Resolves null when the client + * left before the node answered: the node-side request is aborted and + * nothing is rendered (the response is gone). Otherwise an abandoned exec + * against a slow node would hold a gateway→node socket until the node + * answered — on a hung node, until its TCP died. forwardCapture follows + * the same rule. + */ +export async function forwardStream( + req: http.IncomingMessage, + res: http.ServerResponse, + options: ForwardOptions, +): Promise { + const gone = new AbortController(); + const onClose = () => gone.abort(); + res.once('close', onClose); + let upstream: Awaited>; + try { + upstream = await dispatch(req, options, {}, gone.signal); + } catch (error) { + if (gone.signal.aborted) return null; + throw error; + } finally { + res.off('close', onClose); + } + res.writeHead(upstream.statusCode, inboundHeaders(upstream.headers)); + try { + // pipeline destroys both ends on failure: a client that went away + // aborts the node's response, a node that died cuts the client. + await pipeline(upstream.body, res); + } catch { + res.destroy(); + } + return upstream.statusCode; +} + +export interface CapturedResponse { + status: number; + headers: http.OutgoingHttpHeaders; + body: Buffer; +} + +/** + * Forwards and collects the whole answer, for the few verbs whose answer + * the gateway must read before passing it on: a create (to learn the id + * the node minted, for the cache) and a destroy (to forget the entry). + * Their answers are small JSON; everything else streams. + * + * The same abort rule as forwardStream, and for the same reason: a client + * that leaves before the node has answered takes its request with it, and + * `null` says so (the caller writes nothing — the response is gone). These + * verbs hold the name's queue slot, and a request nobody is waiting for + * must not hold it: a node that accepts the connection and never answers + * (a daemon with a hung event loop — 2026-08-13 and 2026-09-11 shapes) + * would otherwise pin the name until its TCP died, every retry queueing + * behind the abandoned attempt and firing at the node together when it + * came back. A queued verb whose client left while it waited for the slot + * is not sent at all. Once the node's head has arrived the small body is + * read to the end regardless, so a 2xx that did land is still learned. + */ +export async function forwardCapture( + req: http.IncomingMessage, + res: http.ServerResponse, + options: ForwardOptions, +): Promise { + if (res.destroyed) return null; + const gone = new AbortController(); + const onClose = () => gone.abort(); + res.once('close', onClose); + let upstream: Awaited>; + try { + // The gateway is the reader of this answer, so it states its own + // preference: identity. A caller's accept-encoding would let a hop + // that compresses (a Caddy with `encode` in front of the node) hand + // back bytes the gateway cannot read the id out of. + upstream = await dispatch( + req, + options, + { 'accept-encoding': 'identity' }, + gone.signal, + ); + } catch (error) { + if (gone.signal.aborted) return null; + throw error; + } finally { + res.off('close', onClose); + } + let body: Buffer; + try { + body = Buffer.from(await upstream.body.arrayBuffer()); + } catch (error) { + throw new UnreachableError( + options.target.endpoint, + `answer cut mid-body: ${causeOf(error)}`, + ); + } + return { + status: upstream.statusCode, + headers: inboundHeaders(upstream.headers), + body, + }; +} + +/** Writes a captured answer through, verbatim. */ +export function replay( + res: http.ServerResponse, + answer: CapturedResponse, +): void { + // The client left while the small body was being read: nothing to say + // to nobody (what the answer taught the cache is kept regardless). + if (res.destroyed) return; + // A 204, a 304 or a 1xx carries no body and must not claim a length + // (RFC 9110 §8.6): the node sent none, and a `content-length: 0` added + // here would change the framing this file promises to keep. + const bodiless = + answer.status === 204 || answer.status === 304 || answer.status < 200; + res.writeHead( + answer.status, + bodiless + ? answer.headers + : { ...answer.headers, 'content-length': String(answer.body.length) }, + ); + res.end(bodiless ? undefined : answer.body); +} + +/** + * The upgrade path (sandbox WebSockets, once the port proxy routes + * through the gateway): the daemon's own replay (sandbox-proxy.ts + * handleUpgrade) — dial the node, write the request line and rawHeaders + * verbatim, then pipe both ways. Plain TCP: node endpoints are + * private-network http, and a TLS node endpoint is refused here rather + * than half-supported. + */ +export function forwardUpgrade( + req: http.IncomingMessage, + socket: Duplex, + head: Buffer, + target: ForwardTarget, + connectTimeoutMs = CONNECT_TIMEOUT_MS, +): void { + socket.on('error', () => socket.destroy()); + const url = new URL(target.endpoint); + if (url.protocol !== 'http:') { + // An operator's configuration, refused without naming the address: + // this face is unauthenticated (raw.ts). + socket.end( + `HTTP/1.1 502 Bad Gateway\r\ncontent-type: application/json\r\nconnection: close\r\n\r\n${JSON.stringify({ message: "the sandbox's node is not a plain http endpoint — upgrades are not forwarded over TLS" })}`, + ); + return; + } + // A URL keeps the brackets of an IPv6 literal in hostname; a dialer + // wants them off. + const host = url.hostname.replace(/^\[(.*)\]$/, '$1'); + let replayed = false; + // The HTTP faces' connect deadline (the dispatcher above), here by hand: + // a node whose host is down without an RST black-holes the SYN, and a + // bare net.connect would hold the client for the OS's own retry budget + // (75s on macOS, about two minutes on Linux — measured 2026-09-12 with + // nc against 192.0.2.1) before the 502 the HTTP faces give in ten. + // Idle after the handshake is legitimate (a quiet WebSocket), so the + // timer is switched off the moment the connection is up. + const upstream = net.connect( + { port: Number(url.port || 80), host, timeout: connectTimeoutMs }, + () => { + upstream.setTimeout(0); + replayed = true; + let raw = `${req.method} ${req.url} HTTP/1.1\r\n`; + for (let i = 0; i < req.rawHeaders.length; i += 2) { + raw += `${req.rawHeaders[i]}: ${req.rawHeaders[i + 1]}\r\n`; + } + upstream.write(`${raw}\r\n`); + if (head.length > 0) upstream.write(head); + socket.pipe(upstream); + upstream.pipe(socket); + }, + ); + upstream.on('timeout', () => upstream.destroy(new Error('connect timeout'))); + upstream.on('error', () => { + // An HTTP status is honest only before the handshake was replayed; + // after it the client is inside an upgraded stream (WebSocket frames), + // where a 502 line would be a corrupt frame, not an answer. + if (!replayed && !socket.writableEnded && socket.writable) { + socket.end( + `HTTP/1.1 502 Bad Gateway\r\ncontent-type: application/json\r\nconnection: close\r\n\r\n${JSON.stringify({ message: "the sandbox's node did not answer the upgrade — retry" })}`, + ); + } else { + socket.destroy(); + } + }); + socket.on('close', () => upstream.destroy()); + upstream.on('close', () => socket.destroy()); +} diff --git a/packages/gateway/src/lookup.ts b/packages/gateway/src/lookup.ts new file mode 100644 index 00000000..11476283 --- /dev/null +++ b/packages/gateway/src/lookup.ts @@ -0,0 +1,71 @@ +import { + type LookupSandboxRequest, + lookupSandboxResponseSchema, + type SandboxState, +} from '@dormice/shared'; +import { request } from 'undici'; + +/** + * Asking one node "do you hold this sandbox?" — the daemon's lookupSandbox + * verb, the one question the gateway puts to a node on its own account + * (everything else it sends is a caller's request, forwarded raw). Two + * seconds, not more: a node that cannot answer a ledger read in two + * seconds is a node in trouble, and the caller is waiting on the whole + * round. There is no second, slower deadline — slow is down (design + * record #35). + */ +export const LOOKUP_TIMEOUT_MS = 2_000; + +export type LookupQuery = LookupSandboxRequest; + +export type LookupAnswer = + | { kind: 'found'; id: string; name: string; state: SandboxState } + | { kind: 'absent' } + /** The node did not answer: no connection, a timeout, a non-200, an unreadable body. `why` is the transport's word. */ + | { kind: 'silent'; why: string }; + +export interface AskedNode { + id: string; + endpoint: string; +} + +export type AskNode = ( + node: AskedNode, + query: LookupQuery, +) => Promise; + +export function causeOf(error: unknown): string { + const e = error as { code?: string; message?: string; cause?: unknown }; + const cause = e.cause as { code?: string; message?: string } | undefined; + return cause?.code ?? cause?.message ?? e.code ?? e.message ?? String(error); +} + +/** The production asker: HTTP to the node's endpoint under the fleet's token. */ +export function httpAskNode(token: string): AskNode { + return async (node, query) => { + try { + const res = await request(`${node.endpoint}/lookupSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(query), + signal: AbortSignal.timeout(LOOKUP_TIMEOUT_MS), + }); + if (res.statusCode !== 200) { + const text = await res.body.text(); + return { + kind: 'silent', + why: `lookupSandbox answered ${res.statusCode}: ${text.slice(0, 200)}`, + }; + } + const answer = lookupSandboxResponseSchema.parse(await res.body.json()); + return answer.found + ? { kind: 'found', ...answer.sandbox } + : { kind: 'absent' }; + } catch (error) { + return { kind: 'silent', why: causeOf(error) }; + } + }; +} diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts new file mode 100644 index 00000000..be302d6c --- /dev/null +++ b/packages/gateway/src/main.ts @@ -0,0 +1,135 @@ +import { fileURLToPath } from 'node:url'; +import { KeyedQueue } from '@dormice/server/keyed-queue'; +import { acquireSingleWriterLock } from '@dormice/server/lock'; +import { closeWithGrace, trackConnections } from '@dormice/server/shutdown'; +import { pino } from 'pino'; +import { z } from 'zod'; +import { buildGatewayApp } from './app'; +import { NameCache } from './cache'; +import { loadConfig } from './config'; +import { migrateDb, openDb } from './db/db'; +import { Finder } from './find'; +import { Fleet } from './fleet'; +import { httpAskNode } from './lookup'; +import { readBuildInfo } from './version'; + +const log = pino(); + +/** An operator mistake, not a bug: one honest line, no stack trace. */ +function fatal(error: unknown): never { + log.fatal( + error instanceof z.ZodError + ? z.prettifyError(error) + : error instanceof Error + ? error.message + : String(error), + ); + process.exit(1); +} + +const config = (() => { + try { + return loadConfig(); + } catch (error) { + return fatal(error); + } +})(); + +// One database file, one gateway — enforced, not assumed (the daemon's own +// lock over the gateway's file). The handle is kept for the life of the +// process on purpose: better-sqlite3 closes a handle when its object is +// garbage collected, and a closed handle drops the file lock (measured +// 2026-09-11 on the daemon with a discarded return value). +let lock: ReturnType | undefined; +if (config.DORMICE_GATEWAY_DB_PATH !== ':memory:') { + try { + lock = acquireSingleWriterLock( + config.DORMICE_GATEWAY_DB_PATH, + `another gateway is already running against ${config.DORMICE_GATEWAY_DB_PATH} — one database, one gateway. Stop the other instance, or point this one at its own DORMICE_GATEWAY_DB_PATH.`, + ); + } catch (error) { + fatal(error); + } +} + +// Migrate on every boot; a fresh install needs no separate setup step. +const db = openDb(config.DORMICE_GATEWAY_DB_PATH); +migrateDb(db, fileURLToPath(new URL('../drizzle', import.meta.url))); + +// The fleet from the nodes table (a node that is down is still a node); +// the cache and the readings fill in as nodes report and callers ask. +const fleet = new Fleet(db); +const finder = new Finder( + fleet, + new NameCache(), + httpAskNode(config.DORMICE_API_TOKEN), + log, +); + +// One queue for the whole gateway: the native acquire/destroy and the E2B +// create/kill must share per-name slots or the serialization means nothing. +const locks = new KeyedQueue(); + +const build = readBuildInfo(); +log.info( + build + ? `dormice-gateway build ${build.commit} (${build.title})` + : 'dormice-gateway build: no version identity (built outside a git checkout)', +); + +const app = buildGatewayApp({ + config, + fleet, + finder, + locks, + logger: log, + build, +}); + +// Same red line as the daemon: loopback only, host not configurable — the +// public face is a reverse proxy's job. Nothing is awaited before the door +// opens: the nodes report themselves within one interval, and until then +// a new name is refused with placement's honest 503 while every existing +// sandbox is found by asking. +await app.listen({ host: '127.0.0.1', port: config.DORMICE_GATEWAY_PORT }); +const known = fleet.all(); +log.info( + known.length === 0 + ? 'no node has checked in yet; nodes join the fleet at their first check-in' + : `fronting ${known.length} node(s) from the last run: ${known.map((n) => `${n.id} at ${n.endpoint}`).join(', ')} — each is placed on again after its first check-in`, +); + +// Bounded shutdown, the daemon's (packages/server/src/shutdown.ts has the +// measurements): close the listener, give in-flight short work the grace, +// cut what is still connected — a forwarded exec may legitimately live for +// hours — and exit explicitly. +const SHUTDOWN_GRACE_MS = 10_000; +const connections = trackConnections(app.server); +let closing = false; +const close = async (signal: NodeJS.Signals) => { + if (closing) return; + closing = true; + process.removeListener('SIGTERM', onSigterm); + process.removeListener('SIGINT', onSigint); + app.log.info( + `${signal} received — shutting down (grace ${SHUTDOWN_GRACE_MS}ms)`, + ); + try { + const cut = await closeWithGrace(app, connections, SHUTDOWN_GRACE_MS); + if (cut > 0) { + app.log.warn( + { cut }, + 'connections still open at the end of the grace period were cut', + ); + } + } catch (error) { + app.log.error(error, `graceful shutdown after ${signal} failed`); + process.exitCode = 1; + } + lock?.close(); + process.exit(process.exitCode ?? 0); +}; +const onSigterm = () => void close('SIGTERM'); +const onSigint = () => void close('SIGINT'); +process.once('SIGTERM', onSigterm); +process.once('SIGINT', onSigint); diff --git a/packages/gateway/src/placement.test.ts b/packages/gateway/src/placement.test.ts new file mode 100644 index 00000000..c9833ee2 --- /dev/null +++ b/packages/gateway/src/placement.test.ts @@ -0,0 +1,181 @@ +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { type Db, migrateDb, openDb } from './db/db'; +import { Fleet, type NodeState } from './fleet'; +import { type PlacementKnobs, pick, refusalMessage } from './placement'; +import { checkInOf, type reading } from './testing'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const NOW = new Date('2026-09-14T12:00:00.000Z'); +const KNOBS: PlacementKnobs = { + cpuLimitPct: 70, + activeLimit: 400, + minDiskAvailableBytes: 10 * 2 ** 30, +}; + +function fleet(): { db: Db; fleet: Fleet } { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + return { db, fleet: new Fleet(db) }; +} + +/** A node that checked in at NOW (or when told) with the given reading. */ +function node( + f: Fleet, + id: string, + over: Parameters[0] & { + intervalSeconds?: number; + placed?: number; + checkedInAt?: Date; + } = {}, +): NodeState { + const { node } = f.checkIn( + checkInOf(id, `http://${id}:80`, over), + over.checkedInAt ?? NOW, + ); + node.placedSinceCheckIn = over.placed ?? 0; + return node; +} + +describe('pick', () => { + it('cpu above the limit refuses, at the limit passes, unknown passes', () => { + const f = fleet().fleet; + expect(pick([node(f, 'a', { cpu: 71 })], KNOBS, NOW).node).toBeNull(); + expect(pick([node(f, 'b', { cpu: 70 })], KNOBS, NOW).node?.id).toBe('b'); + // A freshly restarted node's first reading has no delta: not saturated. + expect(pick([node(f, 'c', { cpu: null })], KNOBS, NOW).node?.id).toBe('c'); + }); + + it('active at the limit refuses; frozen sandboxes are not counted', () => { + const f = fleet().fleet; + expect(pick([node(f, 'a', { active: 400 })], KNOBS, NOW).node).toBeNull(); + expect(pick([node(f, 'b', { active: 399 })], KNOBS, NOW).node?.id).toBe( + 'b', + ); + // Thousands of frozen sandboxes are the normal shape of a node. + expect( + pick([node(f, 'c', { active: 1, frozen: 14_000 })], KNOBS, NOW).node?.id, + ).toBe('c'); + }); + + it('counts what was placed since the reading, so a burst inside one interval cannot overfill a node', () => { + const f = fleet().fleet; + const a = node(f, 'a', { active: 390 }); + expect(pick([a], KNOBS, NOW).node?.id).toBe('a'); + a.placedSinceCheckIn = 10; + expect(pick([a], KNOBS, NOW).node).toBeNull(); + expect(pick([a], KNOBS, NOW).refused[0]?.reason).toMatch( + /390 active sandboxes \+ 10 placed since the reading reach the 400 limit/, + ); + }); + + it('a data disk below the floor refuses, naming the GiB; a missing disk reading passes', () => { + const f = fleet().fleet; + const full = node(f, 'full', { diskAvail: 9 * 2 ** 30, memAvail: 30e9 }); + const room = node(f, 'room', { diskAvail: 50 * 2 ** 30, memAvail: 20e9 }); + expect(pick([full, room], KNOBS, NOW).node?.id).toBe('room'); + expect(pick([full], KNOBS, NOW).refused).toEqual([ + { + nodeId: 'full', + reason: 'data disk has 9.0 GiB available, below the 10.0 GiB floor', + }, + ]); + expect( + pick([node(f, 'unknown', { diskAvail: null })], KNOBS, NOW).node?.id, + ).toBe('unknown'); + }); + + it('scores by active density — (active + placed) per core — then by available memory, then id', () => { + const f = fleet().fleet; + // Fewer sandboxes per core beats more free memory: memory is a + // fifteen-second-old reading, the count moves with every pick. + const a = node(f, 'a', { memAvail: 8e9, active: 5 }); + const b = node(f, 'b', { memAvail: 16e9, active: 50 }); + expect(pick([a, b], KNOBS, NOW).node?.id).toBe('a'); + // Equal density: the most available memory, then the id. + const c = node(f, 'c', { memAvail: 16e9, active: 5 }); + expect(pick([a, b, c], KNOBS, NOW).node?.id).toBe('c'); + const d = node(f, 'd', { memAvail: 16e9, active: 5 }); + expect(pick([d, c], KNOBS, NOW).node?.id).toBe('c'); + // Per core: 40 sandboxes on 64 cores is emptier than 6 on 8. + const big = node(f, 'big', { cores: 64, active: 40, memAvail: 1e9 }); + const small = node(f, 'small', { cores: 8, active: 6, memAvail: 64e9 }); + expect(pick([small, big], KNOBS, NOW).node?.id).toBe('big'); + }); + + it('a burst inside one reading spreads: each pick counts against the node it chose, so the next pick sees it fuller', () => { + const f = fleet().fleet; + const a = node(f, 'a', { memAvail: 17e9, active: 350 }); + const b = node(f, 'b', { memAvail: 16e9, active: 10 }); + const landed = { a: 0, b: 0 }; + for (let i = 0; i < 100; i++) { + const chosen = pick([a, b], KNOBS, NOW).node; + if (chosen === null) throw new Error('every node refused'); + chosen.placedSinceCheckIn += 1; + landed[chosen.id as 'a' | 'b'] += 1; + } + expect(landed).toEqual({ a: 0, b: 100 }); + // And with equal readings the two alternate. + const c = node(f, 'c', { active: 10 }); + const d = node(f, 'd', { active: 10 }); + const alternating: string[] = []; + for (let i = 0; i < 6; i++) { + const chosen = pick([c, d], KNOBS, NOW).node; + if (chosen === null) throw new Error('every node refused'); + chosen.placedSinceCheckIn += 1; + alternating.push(chosen.id); + } + expect(alternating).toEqual(['c', 'd', 'c', 'd', 'c', 'd']); + }); + + it('refuses a node that never checked in since the start, or is silent for two of its intervals', () => { + const { db, fleet: f } = fleet(); + const stale = node(f, 'stale', { + intervalSeconds: 15, + checkedInAt: new Date(NOW.getTime() - 31_000), + }); + const fresh = node(f, 'fresh', { + intervalSeconds: 15, + checkedInAt: new Date(NOW.getTime() - 29_000), + }); + // The same row after a gateway restart: known, but nothing reported yet. + const restarted = new Fleet(db).get('fresh'); + if (!restarted) throw new Error('row lost'); + const result = pick([stale, restarted], KNOBS, NOW); + expect(result.node).toBeNull(); + expect( + Object.fromEntries(result.refused.map((r) => [r.nodeId, r.reason])), + ).toEqual({ + stale: 'has not checked in for 31s', + fresh: 'has not checked in since the gateway started', + }); + expect(pick([fresh], KNOBS, NOW).node?.id).toBe('fresh'); + }); + + it('when every node refuses, each refusal names the node and its reason for the 503; no node at all says so', () => { + const f = fleet().fleet; + const result = pick( + [ + node(f, 'a', { cpu: 90, active: 100, memAvail: 1e9 }), + node(f, 'b', { cpu: 20, active: 400, memAvail: 2e9 }), + ], + KNOBS, + NOW, + ); + expect(result.node).toBeNull(); + expect(result.refused).toEqual([ + { nodeId: 'a', reason: 'cpu 90% is above the 70% limit' }, + { + nodeId: 'b', + reason: + '400 active sandboxes + 0 placed since the reading reach the 400 limit', + }, + ]); + expect(refusalMessage(result)).toBe( + 'no node can take a new sandbox right now — a: cpu 90% is above the 70% limit; b: 400 active sandboxes + 0 placed since the reading reach the 400 limit', + ); + expect(refusalMessage(pick([], KNOBS, NOW))).toMatch( + /no node has checked in/, + ); + }); +}); diff --git a/packages/gateway/src/placement.ts b/packages/gateway/src/placement.ts new file mode 100644 index 00000000..456e26ec --- /dev/null +++ b/packages/gateway/src/placement.ts @@ -0,0 +1,138 @@ +import { downReason, type NodeState } from './fleet'; + +export interface PlacementKnobs { + /** A reading above this refuses the node. */ + cpuLimitPct: number; + /** Active sandboxes plus placements since the reading at or above this refuse the node. */ + activeLimit: number; + /** A data disk with less than this available refuses the node. */ + minDiskAvailableBytes: number; +} + +/** One node's refusal, in the words the 503 repeats: `: `. */ +export interface NodeRefusal { + nodeId: string; + reason: string; +} + +export interface Placement { + /** The node to create on, or null when every node refused. */ + node: NodeState | null; + /** Every node that refused and why — the 503's body when node is null. */ + refused: NodeRefusal[]; +} + +/** + * Where a new sandbox goes: four gates, then one score. Pure — it reads + * what the nodes last reported and never asks a node. + * + * A node that is down (fleet.ts downReason: no check-in since the gateway + * started, or two of its own intervals silent) is refused: placing blind + * is how a gateway puts the twenty-first sandbox on the box that just + * fell over. A node whose whole-machine CPU is above the limit is + * refused; a null reading passes — the first check-in after a node + * restart has no delta to report, and "don't know yet" is not + * "saturated". A node at its active ceiling is refused, counting what + * this gateway placed there since the reading: frozen sandboxes are not + * counted, on purpose — they cost swap, not CPU or dockerd attention, a + * node holds thousands of them, and counting them would shut the gate + * from the first minute of a cut-over and never open it again. A node + * whose data disk is below the floor is refused (design record #36): a + * full data disk stops every sandbox on the node at once, and the node's + * own create answering 500 would otherwise be re-chosen — scored emptiest, + * because nothing it fails to build ever counts — for every new name + * until someone noticed. A missing disk reading passes, like the CPU: + * unknown is not full. + * + * Among the nodes that pass, the emptiest by active density wins: active + * sandboxes plus what this gateway placed there since the reading, per + * core. Per core because the one density ever measured is per core (535 + * sandboxes healthy and 882 dead on 128 cores, 2026-09-11) and a 64-core + * node with 40 sandboxes has more room than an 8-core node with 6; + * counting the in-flight placements is what spreads a burst — a reading + * is fifteen seconds old, a hundred creates arrive inside that window, + * and a score that does not move with each pick sends all hundred to the + * node the reading favoured (reproduced twice, 2026-09-12). Ties go to the + * most available memory — the resource the freeze/swap design overcommits + * and an overloaded host runs out of first — then to the id, so the + * choice is stable and explainable. Memory does not gate placement: a + * node with room per core and little memory left is still chosen, and + * memory pressure is judged where it is felt, by the node's own admission + * (design record #27, after the cluster), not guessed from a + * fifteen-second-old figure the picks do not move. + */ +export function pick( + nodes: readonly NodeState[], + knobs: PlacementKnobs, + now: Date, +): Placement { + const refused: NodeRefusal[] = []; + const candidates: Array<{ + node: NodeState; + /** (active + placed since the reading) per core. */ + load: number; + memAvailableBytes: number; + }> = []; + for (const node of nodes) { + const refuse = (reason: string) => + refused.push({ nodeId: node.id, reason }); + const down = downReason(node, now); + if (down !== null) { + refuse(down); + continue; + } + const reading = node.reading; + if (reading === null) { + refuse('has not reported a reading'); + continue; + } + const cpuUsedPct = reading.host.cpuUsedPct; + if (cpuUsedPct !== null && cpuUsedPct > knobs.cpuLimitPct) { + refuse( + `cpu ${Math.round(cpuUsedPct)}% is above the ${knobs.cpuLimitPct}% limit`, + ); + continue; + } + const active = reading.sandboxes.byState.active; + const placed = node.placedSinceCheckIn; + if (active + placed >= knobs.activeLimit) { + refuse( + `${active} active sandboxes + ${placed} placed since the reading reach the ${knobs.activeLimit} limit`, + ); + continue; + } + const disk = reading.dataDisk; + if (disk !== null && disk.availableBytes < knobs.minDiskAvailableBytes) { + refuse( + `data disk has ${gib(disk.availableBytes)} GiB available, below the ${gib(knobs.minDiskAvailableBytes)} GiB floor`, + ); + continue; + } + candidates.push({ + node, + load: (active + placed) / Math.max(1, reading.host.cpuCount), + memAvailableBytes: reading.host.memAvailableBytes, + }); + } + candidates.sort( + (a, b) => + a.load - b.load || + b.memAvailableBytes - a.memAvailableBytes || + (a.node.id < b.node.id ? -1 : a.node.id > b.node.id ? 1 : 0), + ); + return { node: candidates[0]?.node ?? null, refused }; +} + +function gib(bytes: number): string { + return (bytes / 2 ** 30).toFixed(1); +} + +/** The 503's sentence when every node refused: each node and its reason — or that there is no node at all. */ +export function refusalMessage(placement: Placement): string { + if (placement.refused.length === 0) { + return 'no node can take a new sandbox: no node has checked in with this gateway yet'; + } + return `no node can take a new sandbox right now — ${placement.refused + .map((r) => `${r.nodeId}: ${r.reason}`) + .join('; ')}`; +} diff --git a/packages/gateway/src/raw.ts b/packages/gateway/src/raw.ts new file mode 100644 index 00000000..a5611263 --- /dev/null +++ b/packages/gateway/src/raw.ts @@ -0,0 +1,192 @@ +import type http from 'node:http'; +import type { Duplex } from 'node:stream'; +import type { Logger } from 'pino'; +import type { Classified } from './classify'; +import { relay, renderError, sendPreflight } from './errors'; +import type { Finder, Found } from './find'; +import { forwardStream } from './forward'; + +export interface RawFacesDeps { + finder: Finder; + token: string; + log: Logger; +} + +/** How long a caller refused with "a node did not answer" may wait before asking again — one check-in interval. */ +export const RETRY_AFTER_SECONDS = 15; + +/** + * The faces Fastify never sees — keyed on a header on any path, judged on + * the raw request the serverFactory hands over: + * envd E2B's in-sandbox API, keyed by E2b-Sandbox-Id; forwarded + * with no credential change (the access token is the node's + * own HMAC). Preflights are answered here: the node answers + * them without auth, and a preflight that came back 401 + * without CORS would fail every browser-direct upload. + * signedRoot the bare signed-URL form: no sandbox id anywhere the + * gateway can read without the node's signing secret, so an + * honest 501. + * + * Nobody has authenticated to the gateway on these faces — the node + * judges the credential, after the gateway has picked it — so what the + * gateway says here describes the sandbox's state and never the fleet: no + * node id, no endpoint. A sandbox id is handed to browsers in every + * getHost URL, and the daemon's own proxy says only "not found" for one + * it lacks. The detail an operator needs goes to the log; the + * authenticated faces and listNodes name nodes freely. + */ +export function createRawFaces({ finder, token, log }: RawFacesDeps) { + /** The one sentence per finding for a sandbox id on these faces — generic on purpose (above). */ + function refusal( + res: http.ServerResponse, + id: string, + found: Exclude, + ): void { + switch (found.kind) { + case 'conflict': + renderError(res, 'connect', { + status: 502, + connectCode: 'unavailable', + message: `sandbox "${id}" is held by more than one node — routing resumes once an operator destroys one copy (listNodes and the gateway log name them)`, + cors: true, + }); + return; + case 'none': + renderError(res, 'connect', { + status: 502, + connectCode: 'unavailable', + message: `sandbox "${id}" is on no node — it may have been destroyed`, + cors: true, + }); + return; + case 'unsure': + renderError(res, 'connect', { + status: 503, + connectCode: 'unavailable', + message: `sandbox "${id}": a node did not answer, so its whereabouts cannot be settled — retry`, + cors: true, + retryAfterSeconds: RETRY_AFTER_SECONDS, + }); + return; + } + } + + /** + * Finds the id and forwards, or answers the refusal — the one path + * every keyed face takes. Nothing here may throw past this point: no + * framework stands behind a raw face, so an escaped rejection would be + * the process's, not the request's (errors.ts relay answers instead). + */ + async function route( + req: http.IncomingMessage, + res: http.ServerResponse, + id: string, + ): Promise { + let found: Found; + try { + found = await finder.byId(id); + } catch (error) { + log.error(error, 'envd face: the lookup itself failed'); + renderError(res, 'connect', { + status: 500, + connectCode: 'internal', + message: 'the gateway failed while locating the sandbox — see its log', + cors: true, + }); + return; + } + if (found.kind !== 'one') { + refusal(res, id, found); + return; + } + const node = found.node; + await relay( + res, + 'connect', + log, + async () => { + await forwardStream(req, res, { + target: { endpoint: node.endpoint, token }, + credential: 'none', + }); + }, + (error) => { + log.warn( + { + sandboxId: id, + nodeId: node.id, + endpoint: node.endpoint, + why: error.why, + }, + "envd face: the sandbox's node did not answer", + ); + return { + status: 502, + connectCode: 'unavailable', + message: `sandbox "${id}": its node did not answer (${error.why}) — retry`, + cors: true, + }; + }, + ); + } + + return { + handleRequest( + kind: Exclude, + req: http.IncomingMessage, + res: http.ServerResponse, + ): void { + switch (kind.face) { + case 'envd': { + if (req.method === 'OPTIONS') { + sendPreflight(req, res); + return; + } + const header = req.headers['e2b-sandbox-id']; + const id = Array.isArray(header) ? header[0] : header; + if (!id) { + renderError(res, 'connect', { + status: 401, + connectCode: 'unauthenticated', + message: 'missing E2b-Sandbox-Id header', + cors: true, + }); + return; + } + void route(req, res, id); + return; + } + case 'signedRoot': { + if (req.method === 'OPTIONS') { + sendPreflight(req, res); + return; + } + renderError(res, 'connect', { + status: 501, + connectCode: 'unimplemented', + message: + 'the bare signed-URL form is not routed by the gateway yet — use the sandbox host form, or the node directly', + cors: true, + }); + return; + } + } + }, + + handleUpgrade( + _kind: Classified, + _req: http.IncomingMessage, + socket: Duplex, + ): void { + // First, before anything else: http.Server drops its own error + // listener from the socket before emitting 'upgrade', so a client + // that resets here would raise an uncaught ECONNRESET and take the + // whole gateway down — from an unauthenticated face (the daemon's + // sandbox-proxy.ts handleUpgrade has the same first line). + socket.on('error', () => socket.destroy()); + // No face of this build takes upgrades: sandbox WebSockets ride the + // port proxy, which joins the gateway with the sandbox domain. + socket.destroy(); + }, + }; +} diff --git a/packages/gateway/src/routes/create.ts b/packages/gateway/src/routes/create.ts new file mode 100644 index 00000000..f6e3d174 --- /dev/null +++ b/packages/gateway/src/routes/create.ts @@ -0,0 +1,129 @@ +import type http from 'node:http'; +import type { FastifyReply, FastifyRequest } from 'fastify'; +import { z } from 'zod'; +import type { NameCache } from '../cache'; +import type { Fleet, NodeState } from '../fleet'; +import { + type CapturedResponse, + type Credential, + forwardCapture, +} from '../forward'; +import { type Placement, type PlacementKnobs, pick } from '../placement'; + +/** + * What a create looks like on each face the gateway creates through: the + * credential it forwards under, and where the node's answer keeps the id + * it minted — all the gateway ever reads of that answer. + */ +export interface CreateFace { + credential: Credential; + idOf(answer: unknown): string | null; +} + +const acquireAnswerSchema = z.object({ sandbox: z.object({ id: z.string() }) }); +const e2bAnswerSchema = z.object({ sandboxID: z.string() }); + +export const NATIVE_CREATE: CreateFace = { + credential: 'bearer', + idOf: (answer) => { + const parsed = acquireAnswerSchema.safeParse(answer); + return parsed.success ? parsed.data.sandbox.id : null; + }, +}; + +export const E2B_CREATE: CreateFace = { + credential: 'x-api-key', + idOf: (answer) => { + const parsed = e2bAnswerSchema.safeParse(answer); + return parsed.success ? parsed.data.sandboxID : null; + }, +}; + +/** + * Where a new sandbox goes: placement's pick, counted against the node at + * once — the next acquire in this same interval must see this one as + * already there. Nothing is recorded anywhere else: if the answer is + * lost, the caller's retry asks the fleet (find.ts) and the node that + * built it answers "yes" from inside the name's slot. + */ +export function place( + fleet: Fleet, + knobs: PlacementKnobs, + now: Date, +): Placement { + const placement = pick(fleet.all(), knobs, now); + if (placement.node !== null) placement.node.placedSinceCheckIn += 1; + return placement; +} + +/** + * A create that waited for its name's slot behind a slow destroy and whose + * client left meanwhile: nothing is placed or counted for it — the + * response is gone, forwardCapture would send nothing. The reply is + * hijacked so Fastify writes nothing to the dead socket either. + */ +export function clientGone(reply: FastifyReply): boolean { + if (!reply.raw.destroyed) return false; + reply.hijack(); + return true; +} + +export interface CreateOptions { + target: NodeState; + token: string; + name: string | null; + body: Buffer | undefined; + face: CreateFace; +} + +/** + * Forwards a create and learns from the node's answer: a 2xx with a + * readable id goes into the cache (name and id → this node), so the next + * request for the sandbox skips the round of questions. A 2xx without a + * readable id is logged — the sandbox exists on the node and the next + * lookup finds it there, never silently. Any other answer teaches + * nothing: a 4xx or the node's own 500 means the node holds nothing under + * the name and the next attempt may be placed elsewhere; a 502/503/504 + * from a hop in front of the node, or no answer at all, leaves the + * question to the next lookup — the node that may have built it answers + * from inside the name's slot. + */ +export async function forwardCreate( + cache: NameCache, + request: FastifyRequest, + res: http.ServerResponse, + { target, token, name, body, face }: CreateOptions, +): Promise { + const answer = await forwardCapture(request.raw, res, { + target: { endpoint: target.endpoint, token }, + credential: face.credential, + body, + }); + if (answer === null) return null; + if (answer.status >= 200 && answer.status < 300) { + const id = face.idOf(parseJson(answer.body)); + if (id !== null) { + cache.put({ id, name, nodeId: target.id }); + } else { + request.log.warn( + { node: target.id, name, status: answer.status }, + 'create answered 2xx without a readable sandbox id; the next lookup finds it on the node', + ); + } + } else if (answer.status >= 500) { + request.log.warn( + { node: target.id, name, status: answer.status }, + 'create was refused by the node or a hop in front of it; nothing cached', + ); + } + return answer; +} + +export function parseJson(body: Buffer | undefined): unknown { + if (body === undefined || body.length === 0) return undefined; + try { + return JSON.parse(body.toString('utf8')); + } catch { + return undefined; + } +} diff --git a/packages/gateway/src/routes/destroy.ts b/packages/gateway/src/routes/destroy.ts new file mode 100644 index 00000000..189105c3 --- /dev/null +++ b/packages/gateway/src/routes/destroy.ts @@ -0,0 +1,44 @@ +import type http from 'node:http'; +import type { FastifyRequest } from 'fastify'; +import type { CacheEntry, NameCache } from '../cache'; +import type { NodeState } from '../fleet'; +import { + type CapturedResponse, + type Credential, + forwardCapture, +} from '../forward'; + +export interface DestroyOptions { + target: NodeState; + token: string; + /** The cache entry the verb addresses — forgotten on the node's yes. */ + entry: CacheEntry; + body: Buffer | undefined; + credential: Credential; +} + +/** + * The one destroy path, the mirror of create.ts: the native destroySandbox + * and the E2B kill both forward, wait for the node's whole answer and + * learn from a 2xx — the cache entry goes. Only the node's yes teaches + * anything: its 404 is not "gone" (the daemon answers 404 for a sandbox + * past its kill deadline that its scanner has not torn down yet, a row a + * lookup would still answer yes for), and a 5xx or no answer leaves the + * cache as it was. The cache is a cache: an entry kept one destroy too + * long costs the next request one 404 and a re-check (find.ts verify). + */ +export async function forwardDestroy( + cache: NameCache, + request: FastifyRequest, + res: http.ServerResponse, + { target, token, entry, body, credential }: DestroyOptions, +): Promise { + const answer = await forwardCapture(request.raw, res, { + target: { endpoint: target.endpoint, token }, + credential, + body, + }); + if (answer === null) return null; + if (answer.status >= 200 && answer.status < 300) cache.evict(entry); + return answer; +} diff --git a/packages/gateway/src/routes/e2b.ts b/packages/gateway/src/routes/e2b.ts new file mode 100644 index 00000000..ee5d0e51 --- /dev/null +++ b/packages/gateway/src/routes/e2b.ts @@ -0,0 +1,215 @@ +import { tokensEqual } from '@dormice/server/auth'; +import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import type { FastifyError, FastifyReply, FastifyRequest } from 'fastify'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { relay } from '../errors'; +import type { Finder } from '../find'; +import type { Fleet, NodeState } from '../fleet'; +import { forwardStream, replay } from '../forward'; +import { type PlacementKnobs, refusalMessage } from '../placement'; +import { RETRY_AFTER_SECONDS } from '../raw'; +import { + clientGone, + E2B_CREATE, + forwardCreate, + parseJson, + place, +} from './create'; +import { forwardDestroy } from './destroy'; +import { refuse, verdict } from './verdict'; + +export interface E2bRoutesOptions { + fleet: Fleet; + finder: Finder; + locks: KeyedQueue; + knobs: PlacementKnobs; + token: string; +} + +/** + * The E2B control plane in front of several nodes: what the official SDK + * calls api.e2b.app for, mounted at /e2b/api like the daemon's. Creates + * are placed (by metadata.name when the SDK gave one — the same name slot + * the native acquire takes — else unnamed, known by id alone until a + * lookup names it); everything addressed by id is found and forwarded. + * The dialect is E2B's: { code: , message }. + */ +export const e2bControlRoutes: FastifyPluginAsyncZod = async ( + app, + { fleet, finder, locks, knobs, token }, +) => { + app.addContentTypeParser( + 'application/json', + { parseAs: 'buffer' }, + (_request, body, done) => done(null, body), + ); + app.setErrorHandler((error: FastifyError, request, reply) => { + const status = error.statusCode ?? 500; + if (status >= 500) request.log.error(error, 'request failed'); + reply.code(status).send({ code: status, message: error.message }); + }); + app.setNotFoundHandler((request, reply) => { + reply.code(404).send({ + code: 404, + message: `route ${request.method} ${request.url} not found`, + }); + }); + + // The daemon's own X-API-KEY convention (`e2b_`), over the + // fleet's one token. + app.addHook('onRequest', async (request, reply) => { + const presented = request.headers['x-api-key']; + const key = Array.isArray(presented) ? presented[0] : presented; + const bare = key?.startsWith('e2b_') ? key.slice(4) : key; + if (bare === undefined || !tokensEqual(bare, token)) { + await reply.code(401).send({ code: 401, message: 'invalid API key' }); + } + }); + + const dialect = (message: string) => ({ code: 0, message }); + const send = (reply: FastifyReply, code: number, message: string) => + reply.code(code).send({ code, message }); + + /** From the hijack on the raw response is the gateway's to finish, in E2B's dialect (errors.ts relay). */ + const forwarded = ( + request: FastifyRequest, + reply: FastifyReply, + tail: string, + step: () => Promise, + ) => { + reply.hijack(); + return relay(reply.raw, 'control', request.log, step, (error) => ({ + status: 502, + message: `${error.message}${tail}`, + })); + }; + + const RETRY_FINDS_IT = + ' — retry: if the sandbox was built, the node answers for it'; + + function create( + request: FastifyRequest, + reply: FastifyReply, + target: NodeState, + name: string | null, + body: Buffer | undefined, + ) { + return forwarded(request, reply, RETRY_FINDS_IT, async () => { + const answer = await forwardCreate(finder.cache, request, reply.raw, { + target, + token, + name, + body, + face: E2B_CREATE, + }); + if (answer !== null) replay(reply.raw, answer); + }); + } + + function placed(reply: FastifyReply): NodeState | null { + if (clientGone(reply)) return null; + const placement = place(fleet, knobs, new Date()); + if (placement.node === null) { + reply.header('retry-after', String(RETRY_AFTER_SECONDS)); + send(reply, 503, refusalMessage(placement)); + return null; + } + return placement.node; + } + + app.post('/sandboxes', async (request, reply) => { + const body = request.body as Buffer | undefined; + const parsed = parseJson(body) as + | { metadata?: { name?: unknown } } + | undefined; + const name = parsed?.metadata?.name; + if (typeof name === 'string' && name.length > 0) { + return locks.run(name, async () => { + const judged = verdict(await finder.byName(name), `sandbox "${name}"`); + if (judged.kind === 'refuse') { + return refuse(reply, judged, (message) => ({ + ...dialect(message), + code: judged.status, + })); + } + const target = judged.kind === 'node' ? judged.node : placed(reply); + if (target === null) return reply; + return create(request, reply, target, name, body); + }); + } + const target = placed(reply); + if (target === null) return reply; + return create(request, reply, target, null, body); + }); + + app.get('/v2/sandboxes', async (_request, reply) => + send( + reply, + 501, + 'listing sandboxes is not routed by the gateway yet — call the node directly', + ), + ); + + const byId = async (request: FastifyRequest, reply: FastifyReply) => { + const { id } = request.params as { id: string }; + const judged = verdict(await finder.byId(id), `sandbox "${id}"`); + if (judged.kind === 'refuse') { + return refuse(reply, judged, (message) => ({ + ...dialect(message), + code: judged.status, + })); + } + if (judged.kind === 'none') { + return send(reply, 404, `sandbox "${id}" not found`); + } + const body = request.body as Buffer | undefined; + const { node, name } = judged; + // Only the kill itself — DELETE /sandboxes/, nothing deeper — is + // captured and learned from (routes/destroy.ts has what is learned). + // A DELETE on a sub-route the node lacks answers the same 404 shape as + // a kill of an unknown id (measured 2026-09-12 with + // DELETE /sandboxes//bogus). The kill takes the name's slot like + // the native destroy (the daemon's own kill does too). + if (request.method === 'DELETE' && isKill(request)) { + return locks.run(name ?? id, () => + forwarded(request, reply, ' — retry', async () => { + const answer = await forwardDestroy( + finder.cache, + request, + reply.raw, + { + target: node, + token, + entry: { id, name, nodeId: node.id }, + body, + credential: 'x-api-key', + }, + ); + if (answer !== null) replay(reply.raw, answer); + }), + ); + } + return forwarded(request, reply, ' — retry', async () => { + const status = await forwardStream(request.raw, reply.raw, { + target: { endpoint: node.endpoint, token }, + credential: 'x-api-key', + body, + }); + if (status === 404) void finder.verify({ id, name, nodeId: node.id }); + }); + }; + // Every method, not the ones this build of the daemon happens to serve: + // the node answers for its own routes (e2b 2.31 already PUTs + // /sandboxes/:id/network, which the daemon 404s — its 404, not the + // gateway's, is the honest one). The path reaches the node exactly as + // written (forward.ts dispatch), so a sub-route is a sub-route there + // too, never a create resolved out of dot segments. + const methods = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE'] as const; + app.route({ method: [...methods], url: '/sandboxes/:id', handler: byId }); + app.route({ method: [...methods], url: '/sandboxes/:id/*', handler: byId }); +}; + +/** The kill route exactly: `/sandboxes/` with nothing after the id. */ +function isKill(request: FastifyRequest): boolean { + return (request.params as { '*'?: string })['*'] === undefined; +} diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts new file mode 100644 index 00000000..fd8d3fed --- /dev/null +++ b/packages/gateway/src/routes/native.ts @@ -0,0 +1,257 @@ +import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import { WRITE_FILES_BODY_LIMIT_BYTES } from '@dormice/shared'; +import type { FastifyReply, FastifyRequest } from 'fastify'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { relay } from '../errors'; +import type { Finder } from '../find'; +import type { Fleet } from '../fleet'; +import { forwardStream, replay } from '../forward'; +import { type PlacementKnobs, refusalMessage } from '../placement'; +import { RETRY_AFTER_SECONDS } from '../raw'; +import { + clientGone, + forwardCreate, + NATIVE_CREATE, + parseJson, + place, +} from './create'; +import { forwardDestroy } from './destroy'; +import { refuse, verdict } from './verdict'; + +/** + * The native verbs the gateway routes: every verb addressed to one sandbox + * by `name`. Registered one by one — never as `/:verb` — so a misspelled + * verb is a 404 like on the daemon, and nothing else under the root is + * swallowed. + */ +export const NAMED_VERBS = [ + 'acquireSandbox', + 'execCommand', + 'writeFile', + 'writeFiles', + 'readFile', + 'readFiles', + 'rebuildSandbox', + 'updatePolicy', + 'updateSpec', + 'updateTemplate', + 'expandDisk', + 'updateMetadata', + 'destroySandbox', + 'getSandboxMetrics', + 'getSandboxMetricsHistory', +] as const; + +/** + * The verbs that address the daemon, not a sandbox: fleet lists, host + * readings, templates, settings, ingress, upgrade, keys. Asking every + * node and merging, or answering from the gateway's own tables, comes + * with the configuration authority; until then each answers an honest + * 501 naming the alternative, instead of a misleading answer from + * whichever node the gateway happened to pick. + */ +export const UNNAMED_VERBS = [ + 'listSandboxes', + 'listSandboxMetrics', + 'listSandboxImages', + 'listActivity', + 'getFleetTimeline', + 'getHostMetrics', + 'getHostMetricsHistory', + 'getConfig', + 'checkUpgrade', + 'applyUpgrade', + 'getUpgradeStatus', + 'getIngress', + 'setIngress', + 'registerTemplate', + 'listTemplates', + 'removeTemplate', + 'updateSettings', + 'createApiKey', + 'listApiKeys', + 'updateApiKey', + 'revokeApiKey', +] as const; + +export interface NativeRoutesOptions { + fleet: Fleet; + finder: Finder; + locks: KeyedQueue; + knobs: PlacementKnobs; + token: string; +} + +export const nativeRoutes: FastifyPluginAsyncZod = async ( + app, + { fleet, finder, locks, knobs, token }, +) => { + // Bodies stay bytes: the gateway reads one field (`name`) and forwards + // the bytes it received, never a re-serialization. Scoped to this + // plugin — the gateway's own verbs keep Fastify's JSON parsing. + app.addContentTypeParser( + 'application/json', + { parseAs: 'buffer' }, + (_request, body, done) => done(null, body), + ); + + for (const verb of UNNAMED_VERBS) { + app.post(`/${verb}`, async (_request, reply) => + reply.code(501).send({ + message: `${verb} is not routed by the gateway yet — call the node directly (this version routes only sandbox-addressed verbs)`, + }), + ); + } + + for (const verb of NAMED_VERBS) { + const bodyLimit = + verb === 'writeFile' || verb === 'writeFiles' + ? WRITE_FILES_BODY_LIMIT_BYTES + : undefined; + app.post( + `/${verb}`, + bodyLimit === undefined ? {} : { bodyLimit }, + async (request, reply) => { + const body = request.body as Buffer | undefined; + const name = nameOf(body); + if (name === null) { + return reply + .code(400) + .send({ message: 'name is required and must be a string' }); + } + // Only the two verbs that create or remove take the name's slot — + // the daemon's own discipline (its other verbs run unserialized + // too). The slot is what keeps twenty simultaneous acquires of a + // new name from each placing their own copy: the first finds + // nothing and places, the rest find its cache entry. + if (verb === 'acquireSandbox') { + return locks.run(name, () => acquire(request, reply, name, body)); + } + if (verb === 'destroySandbox') { + return locks.run(name, () => destroy(request, reply, name, body)); + } + return forwardNamed(request, reply, name, body); + }, + ); + } + + /** + * From the hijack on, the raw response is the gateway's to finish: the + * node's answer is replayed, or the gateway's own 502/500 rendered, in + * the native dialect (errors.ts relay). `tail` is what the caller may + * do after a node that did not answer. + */ + const forwarded = ( + request: FastifyRequest, + reply: FastifyReply, + tail: string, + step: () => Promise, + ) => { + reply.hijack(); + return relay(reply.raw, 'native', request.log, step, (error) => ({ + status: 502, + message: `${error.message}${tail}`, + })); + }; + + /** A create that did not answer: the sandbox may exist; the retry asks the fleet and lands on the node that built it. */ + const RETRY_FINDS_IT = + ' — retry: if the sandbox was built, the node answers for it'; + + async function acquire( + request: FastifyRequest, + reply: FastifyReply, + name: string, + body: Buffer | undefined, + ) { + const judged = verdict(await finder.byName(name), `sandbox "${name}"`); + if (judged.kind === 'refuse') return refuse(reply, judged); + let target = judged.kind === 'node' ? judged.node : null; + if (target === null) { + if (clientGone(reply)) return; + const placement = place(fleet, knobs, new Date()); + if (placement.node === null) { + reply.header('retry-after', String(RETRY_AFTER_SECONDS)); + return reply.code(503).send({ message: refusalMessage(placement) }); + } + target = placement.node; + } + const chosen = target; + return forwarded(request, reply, RETRY_FINDS_IT, async () => { + const answer = await forwardCreate(finder.cache, request, reply.raw, { + target: chosen, + token, + name, + body, + face: NATIVE_CREATE, + }); + if (answer !== null) replay(reply.raw, answer); + }); + } + + async function destroy( + request: FastifyRequest, + reply: FastifyReply, + name: string, + body: Buffer | undefined, + ) { + const judged = verdict(await finder.byName(name), `sandbox "${name}"`); + if (judged.kind === 'refuse') return refuse(reply, judged); + // Nothing to forward: every node answered and none holds the name, + // which is exactly the daemon's own idempotent answer. + if (judged.kind === 'none') + return reply.code(200).send({ destroyed: false }); + const { node, id } = judged; + return forwarded(request, reply, ' — retry', async () => { + const answer = await forwardDestroy(finder.cache, request, reply.raw, { + target: node, + token, + entry: { id, name, nodeId: node.id }, + body, + credential: 'bearer', + }); + if (answer !== null) replay(reply.raw, answer); + }); + } + + /** + * Every other named verb: find the node, stream the answer through as + * it came — status, headers, body. A node's 404 is not "no such + * sandbox" by itself (readFile answers 404 for a missing path too), so + * it is relayed as it came and the cache entry is re-checked off the + * request path (find.ts verify): only a node that says "absent" to the + * one question that means it loses the entry. + */ + async function forwardNamed( + request: FastifyRequest, + reply: FastifyReply, + name: string, + body: Buffer | undefined, + ) { + const judged = verdict(await finder.byName(name), `sandbox "${name}"`); + if (judged.kind === 'refuse') return refuse(reply, judged); + if (judged.kind === 'none') { + return reply + .code(404) + .send({ message: `no sandbox named "${name}" — acquire it first` }); + } + const { node, id } = judged; + return forwarded(request, reply, ' — retry', async () => { + const status = await forwardStream(request.raw, reply.raw, { + target: { endpoint: node.endpoint, token }, + credential: 'bearer', + body, + }); + if (status === 404) { + void finder.verify({ id, name, nodeId: node.id }); + } + }); + } +}; + +/** The one field the gateway reads from a native body. */ +export function nameOf(body: Buffer | undefined): string | null { + const parsed = parseJson(body) as { name?: unknown } | undefined; + const name = parsed?.name; + return typeof name === 'string' && name.length > 0 ? name : null; +} diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts new file mode 100644 index 00000000..83ddd779 --- /dev/null +++ b/packages/gateway/src/routes/nodes.ts @@ -0,0 +1,94 @@ +import { + checkInRequestSchema, + checkInResponseSchema, + listNodesRequestSchema, + listNodesResponseSchema, + removeNodeRequestSchema, + removeNodeResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import type { NameCache } from '../cache'; +import { downReason, type Fleet } from '../fleet'; + +export interface NodeRoutesOptions { + fleet: Fleet; + cache: NameCache; +} + +/** + * The gateway's own verbs about its nodes: the check-in the nodes send + * (RULES/协议.md「网关」), and what an operator reads and does about them. + * Everything listNodes answers is what the gateway already holds, so + * answering costs no node anything. + */ +export const nodeRoutes: FastifyPluginAsyncZod = async ( + app, + { fleet, cache }, +) => { + app.post( + '/checkIn', + { + schema: { + body: checkInRequestSchema, + response: { 200: checkInResponseSchema }, + }, + }, + async (request) => { + const { node, joined } = fleet.checkIn(request.body); + if (joined) { + request.log.info( + { nodeId: node.id, endpoint: node.endpoint }, + 'a node checked in for the first time and joined the fleet', + ); + } + return {}; + }, + ); + + app.post( + '/listNodes', + { + schema: { + body: listNodesRequestSchema, + response: { 200: listNodesResponseSchema }, + }, + }, + async () => { + const now = new Date(); + return { + nodes: fleet.all().map((node) => ({ + id: node.id, + endpoint: node.endpoint, + addedAt: node.addedAt, + lastCheckInAt: node.lastCheckInAt?.toISOString() ?? null, + intervalSeconds: node.intervalSeconds, + reachable: downReason(node, now) === null, + build: node.build, + reading: node.reading, + placedSinceCheckIn: node.placedSinceCheckIn, + })), + }; + }, + ); + + app.post( + '/removeNode', + { + schema: { + body: removeNodeRequestSchema, + response: { 200: removeNodeResponseSchema }, + }, + }, + async (request) => { + const removed = fleet.remove(request.body.id); + const evicted = cache.evictNode(request.body.id); + if (removed) { + request.log.warn( + { nodeId: request.body.id, evicted }, + 'a node was removed by an operator: its sandboxes are no longer looked for, and names that lived only there are new names again', + ); + } + return { removed }; + }, + ); +}; diff --git a/packages/gateway/src/routes/verdict.ts b/packages/gateway/src/routes/verdict.ts new file mode 100644 index 00000000..e4636633 --- /dev/null +++ b/packages/gateway/src/routes/verdict.ts @@ -0,0 +1,62 @@ +import type { FastifyReply } from 'fastify'; +import type { Found } from '../find'; +import type { NodeState } from '../fleet'; +import { RETRY_AFTER_SECONDS } from '../raw'; + +/** + * A finding turned into a routing decision for an authenticated face: the + * node to forward to (with what the cache or the lookup knows of the + * sandbox), nothing (every node answered and none holds it — a create + * places, a destroy gives its idempotent answer, everything else a 404), + * or a refusal with its status and sentence, which the face sends in its + * own dialect. A value, so every verb reads the decision the same way. + */ +export type Verdict = + | { kind: 'node'; node: NodeState; id: string; name: string | null } + | { kind: 'none' } + | { + kind: 'refuse'; + status: number; + message: string; + retryAfterSeconds?: number; + }; + +export function verdict(found: Found, what: string): Verdict { + switch (found.kind) { + case 'one': + return { kind: 'node', node: found.node, id: found.id, name: found.name }; + case 'none': + return { kind: 'none' }; + case 'conflict': + // The gateway refuses every verb for this name with this very 409, + // destroy included — it will not guess which copy the caller means. + return { + kind: 'refuse', + status: 409, + message: `${what} exists on nodes ${found.nodeIds.join(' and ')} — destroy one copy directly on its node before routing can resume`, + }; + case 'unsure': + return { + kind: 'refuse', + status: 503, + message: `${what}: ${found.silent + .map((s) => `node ${s.nodeId} did not answer (${s.why})`) + .join( + ', ', + )} — it cannot be treated as new while a node that may hold it is silent; retry, or remove the node if it is gone for good`, + retryAfterSeconds: RETRY_AFTER_SECONDS, + }; + } +} + +/** Sends a refusal in the native dialect, with Retry-After where the verdict carries one. */ +export function refuse( + reply: FastifyReply, + refusal: Extract, + body: (message: string) => unknown = (message) => ({ message }), +) { + if (refusal.retryAfterSeconds !== undefined) { + reply.header('retry-after', String(refusal.retryAfterSeconds)); + } + return reply.code(refusal.status).send(body(refusal.message)); +} diff --git a/packages/gateway/src/testing.ts b/packages/gateway/src/testing.ts new file mode 100644 index 00000000..083a1be6 --- /dev/null +++ b/packages/gateway/src/testing.ts @@ -0,0 +1,60 @@ +import type { CheckInRequest, NodeReading } from '@dormice/shared'; + +/** + * Test scaffolding shared by the gateway's suites: a node's reading and + * check-in with a few knobs turned. Not shipped — nothing under src/ but + * main.ts is bundled (tsup.config.ts). + */ +export function reading( + over: { + cpu?: number | null; + cores?: number; + active?: number; + frozen?: number; + memAvail?: number; + diskAvail?: number | null; + } = {}, +): NodeReading { + const frozen = over.frozen ?? 0; + const active = over.active ?? 10; + return { + host: { + cpuCount: over.cores ?? 8, + cpuUsedPct: over.cpu === undefined ? 10 : over.cpu, + memTotalBytes: 32e9, + memAvailableBytes: over.memAvail ?? 16e9, + swap: null, + }, + dataDisk: + over.diskAvail === null + ? null + : { + path: '/var/lib/dormice', + totalBytes: 1e12, + usedBytes: 1e12 - (over.diskAvail ?? 5e11), + availableBytes: over.diskAvail ?? 5e11, + }, + sandboxes: { + total: active + frozen, + byState: { active, frozen, stopped: 0, archived: 0, restoring: 0 }, + }, + }; +} + +export function checkInOf( + nodeId: string, + endpoint: string, + over: Parameters[0] & { intervalSeconds?: number } = {}, +): CheckInRequest { + return { + nodeId, + endpoint, + intervalSeconds: over.intervalSeconds ?? 15, + build: { + commit: 'abc1234', + title: 'a commit', + committedAt: '2026-09-14T00:00:00.000Z', + }, + reading: reading(over), + }; +} diff --git a/packages/gateway/src/version.ts b/packages/gateway/src/version.ts new file mode 100644 index 00000000..62ed4335 --- /dev/null +++ b/packages/gateway/src/version.ts @@ -0,0 +1,23 @@ +/** + * The identity tsup baked into this build (tsup.config.ts): the commit the + * dist was built from. In the built daemon these `process.env` reads are + * compile-time literals; running from source (tests, tsx) they fall + * through to the real environment and come back empty — an unbuilt tree + * has no build identity, and null says so honestly. + */ +export interface BuildInfo { + /** Short hash. */ + commit: string; + /** The commit's subject line. */ + title: string; + /** ISO 8601 UTC — the commit's time, not the build's. */ + committedAt: string; +} + +export function readBuildInfo(): BuildInfo | null { + const commit = process.env.DORMICE_BUILD_COMMIT; + const title = process.env.DORMICE_BUILD_COMMIT_TITLE; + const committedAt = process.env.DORMICE_BUILD_COMMIT_AT; + if (!commit || !title || !committedAt) return null; + return { commit, title, committedAt }; +} diff --git a/packages/gateway/tsconfig.json b/packages/gateway/tsconfig.json new file mode 100644 index 00000000..564a5990 --- /dev/null +++ b/packages/gateway/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src"] +} diff --git a/packages/gateway/tsup.config.ts b/packages/gateway/tsup.config.ts new file mode 100644 index 00000000..d607d033 --- /dev/null +++ b/packages/gateway/tsup.config.ts @@ -0,0 +1,38 @@ +import { execSync } from 'node:child_process'; +import { defineConfig } from 'tsup'; + +/** + * Same build identity as the daemon (packages/server/tsup.config.ts has + * the reasoning): the commit the dist was built from, baked in as + * literals that src/version.ts reads back — what the process IS, not what + * the checkout says after a pull. + */ +function git(args: string): string { + try { + return execSync(`git ${args}`, { + stdio: ['ignore', 'pipe', 'ignore'], + }) + .toString() + .trim(); + } catch { + return ''; + } +} + +const commitTime = git('log -1 --format=%cI'); + +export default defineConfig({ + // A service, not a library: main.ts is the one entry and nothing imports + // the package (e2e boots dist/main.js as a process), so no index and no + // declarations. + entry: ['src/main.ts'], + format: ['esm'], + clean: true, + env: { + DORMICE_BUILD_COMMIT: git('rev-parse --short HEAD'), + DORMICE_BUILD_COMMIT_TITLE: git('log -1 --format=%s'), + DORMICE_BUILD_COMMIT_AT: commitTime + ? new Date(commitTime).toISOString() + : '', + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3eeb1852..44238dfe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -177,6 +177,43 @@ importers: specifier: ^8.1.3 version: 8.1.3(@types/node@26.1.0)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.0)(yaml@2.9.0) + packages/gateway: + dependencies: + '@dormice/server': + specifier: workspace:* + version: link:../server + '@dormice/shared': + specifier: workspace:* + version: link:../shared + better-sqlite3: + specifier: ^12.11.1 + version: 12.11.1 + drizzle-orm: + specifier: ^0.45.2 + version: 0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.11.1)(kysely@0.28.17) + fastify: + specifier: ^5.10.0 + version: 5.10.0 + fastify-type-provider-zod: + specifier: ^7.0.0 + version: 7.0.0(@fastify/swagger@9.7.0)(fastify@5.10.0)(openapi-types@12.1.3)(zod@4.4.3) + pino: + specifier: ^10.3.1 + version: 10.3.1 + undici: + specifier: ^8.7.0 + version: 8.7.0 + zod: + specifier: ^4.4.3 + version: 4.4.3 + devDependencies: + '@types/better-sqlite3': + specifier: ^7.6.13 + version: 7.6.13 + drizzle-kit: + specifier: ^0.31.10 + version: 0.31.10 + packages/sdk: dependencies: '@dormice/shared': From 7b560eff7024c461d2cda9a3c8a1a32c41fe2df8 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:48:51 +0800 Subject: [PATCH 06/89] A sandbox placed and destroyed inside one check-in interval no longer holds a placement slot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The placement counter exists so a burst inside one interval is counted against a node before its next reading shows it. It was only ever reset by the reading, so a sandbox placed and destroyed within the same interval — a short job, or the exam's churn — kept holding a slot the reading would never show; with the exam's two-per-node gate that shut placement for a whole interval after a handful of fast tests. Each node now remembers the ids its creates answered with since the last reading; a destroy of one of them takes the placement off the count, and a create the node itself refused is uncounted at once. A hop's 502/503/504, or no answer, still counts until the reading: the sandbox may exist. --- packages/gateway/src/app.test.ts | 7 +++++- packages/gateway/src/fleet.ts | 9 ++++++- packages/gateway/src/routes/create.ts | 34 ++++++++++++++++++-------- packages/gateway/src/routes/destroy.ts | 11 +++++++-- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index cce3d7a0..63e03031 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -62,11 +62,13 @@ class FakeNode { return [...this.sandboxes.values()].find((s) => s.id === id); } + // Inferred return type on purpose: the early `return json(...)` exits + // read as statements, and an explicit void would flag each one. private answer( req: http.IncomingMessage, res: http.ServerResponse, text: string, - ): void { + ) { const url = req.url ?? '/'; const path = url.split('?')[0] ?? url; const auth = @@ -569,6 +571,9 @@ describe('using, destroying, and the cache', () => { }); expect(h.cache.getByName('d')).toBeUndefined(); expect(h.cache.getById(first.id)).toBeUndefined(); + // Placed and destroyed inside one interval: the placement no longer + // counts against the node — the reading will never show it. + expect(h.fleet.get('a')?.placedSinceCheckIn).toBe(0); expect((await rpc(h, '/destroySandbox', { name: 'd' })).body).toEqual({ destroyed: false, }); diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 42967ada..1e3611d1 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -9,7 +9,10 @@ import { nodes } from './db/schema'; * check-in, gone with the process and rightly so). placedSinceCheckIn * counts the sandboxes the gateway sent here since the last reading, so a * burst inside one interval is counted against the node before its next - * reading shows it (placement.ts). + * reading shows it (placement.ts); placedIds are the ones among them whose + * create answered with an id, so a sandbox placed and destroyed inside one + * interval — a short job, the exam's churn — is taken off the count again + * instead of holding a slot the reading will never show (routes/destroy.ts). */ export interface NodeState { readonly id: string; @@ -20,6 +23,7 @@ export interface NodeState { build: BuildInfo | null; reading: NodeReading | null; placedSinceCheckIn: number; + placedIds: Set; } /** @@ -61,6 +65,7 @@ export class Fleet { build: null, reading: null, placedSinceCheckIn: 0, + placedIds: new Set(), }); } } @@ -104,6 +109,7 @@ export class Fleet { build: null, reading: null, placedSinceCheckIn: 0, + placedIds: new Set(), }; this.members.set(node.id, node); joined = true; @@ -120,6 +126,7 @@ export class Fleet { node.build = report.build; node.reading = report.reading; node.placedSinceCheckIn = 0; + node.placedIds.clear(); return { node, joined }; } diff --git a/packages/gateway/src/routes/create.ts b/packages/gateway/src/routes/create.ts index f6e3d174..8448458c 100644 --- a/packages/gateway/src/routes/create.ts +++ b/packages/gateway/src/routes/create.ts @@ -79,15 +79,19 @@ export interface CreateOptions { /** * Forwards a create and learns from the node's answer: a 2xx with a * readable id goes into the cache (name and id → this node), so the next - * request for the sandbox skips the round of questions. A 2xx without a - * readable id is logged — the sandbox exists on the node and the next - * lookup finds it there, never silently. Any other answer teaches - * nothing: a 4xx or the node's own 500 means the node holds nothing under - * the name and the next attempt may be placed elsewhere; a 502/503/504 - * from a hop in front of the node, or no answer at all, leaves the - * question to the next lookup — the node that may have built it answers - * from inside the name's slot. + * request for the sandbox skips the round of questions, and into the + * node's placedIds, so a destroy inside the same interval can take the + * placement off the count again. A 2xx without a readable id is logged — + * the sandbox exists on the node and the next lookup finds it there, + * never silently. A 4xx or the node's own 500 means the node built + * nothing: the placement is taken off the count at once (it would have + * held a slot for a whole interval otherwise) and the next attempt may be + * placed elsewhere. A 502/503/504 from a hop in front of the node, or no + * answer at all, leaves the count as it is and the question to the next + * lookup — the node that may have built it answers from inside the + * name's slot. */ +const HOP_STATUSES = new Set([502, 503, 504]); export async function forwardCreate( cache: NameCache, request: FastifyRequest, @@ -104,16 +108,26 @@ export async function forwardCreate( const id = face.idOf(parseJson(answer.body)); if (id !== null) { cache.put({ id, name, nodeId: target.id }); + target.placedIds.add(id); } else { request.log.warn( { node: target.id, name, status: answer.status }, 'create answered 2xx without a readable sandbox id; the next lookup finds it on the node', ); } - } else if (answer.status >= 500) { + } else if (!HOP_STATUSES.has(answer.status)) { + // The node itself answered no: nothing was built there. + target.placedSinceCheckIn = Math.max(0, target.placedSinceCheckIn - 1); + if (answer.status >= 500) { + request.log.warn( + { node: target.id, name, status: answer.status }, + 'the node itself failed the create; nothing cached, the placement is uncounted', + ); + } + } else { request.log.warn( { node: target.id, name, status: answer.status }, - 'create was refused by the node or a hop in front of it; nothing cached', + 'a hop in front of the node answered the create; nothing cached, the placement stays counted until the next reading', ); } return answer; diff --git a/packages/gateway/src/routes/destroy.ts b/packages/gateway/src/routes/destroy.ts index 189105c3..e501a489 100644 --- a/packages/gateway/src/routes/destroy.ts +++ b/packages/gateway/src/routes/destroy.ts @@ -20,7 +20,9 @@ export interface DestroyOptions { /** * The one destroy path, the mirror of create.ts: the native destroySandbox * and the E2B kill both forward, wait for the node's whole answer and - * learn from a 2xx — the cache entry goes. Only the node's yes teaches + * learn from a 2xx — the cache entry goes, and a sandbox placed since the + * node's last reading comes off its placement count (fleet.ts placedIds). + * Only the node's yes teaches * anything: its 404 is not "gone" (the daemon answers 404 for a sandbox * past its kill deadline that its scanner has not torn down yet, a row a * lookup would still answer yes for), and a 5xx or no answer leaves the @@ -39,6 +41,11 @@ export async function forwardDestroy( body, }); if (answer === null) return null; - if (answer.status >= 200 && answer.status < 300) cache.evict(entry); + if (answer.status >= 200 && answer.status < 300) { + cache.evict(entry); + if (target.placedIds.delete(entry.id)) { + target.placedSinceCheckIn = Math.max(0, target.placedSinceCheckIn - 1); + } + } return answer; } From 2fc3ea9f4909d7a252959013bb25418284d82a32 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:48:51 +0800 Subject: [PATCH 07/89] =?UTF-8?q?e2e:=20the=20gateway=20exam=20=E2=80=94?= =?UTF-8?q?=20two=20daemons=20behind=20a=20real=20gateway,=20a=20third=20t?= =?UTF-8?q?hat=20joins=20and=20dies,=20all=20driven=20over=20the=20wire?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The setup boots node A exactly as before (every existing suite is untouched) and, outside docker mode, a gateway plus nodes B and C that share its token and check in every second. The exam speaks only the SDK, the official e2b package and plain fetch; direct calls to a node stage what the gateway must then find: a sandbox built behind its back is routed at once (no reconcile to wait for), a name on two nodes is a 409 naming both, a destroy behind its back is caught by the 404 re-check. Placement, the active gate and its Retry-After, five simultaneous acquires, streaming through two hops, envd and the bare signed door are graded; a node booted by the test itself joins at its first check-in, and when it dies its sandboxes 502 and a new name is a 503 naming it until removeNode. A second gateway on the same database file dies at boot naming the conflict, as does a second daemon on node A's ledger. --- e2e/package.json | 1 + e2e/src/gateway.test.ts | 559 ++++++++++++++++++++++++++++++++++++++++ e2e/src/native.test.ts | 37 +++ e2e/src/setup/daemon.ts | 281 +++++++++++++++----- pnpm-lock.yaml | 3 + 5 files changed, 813 insertions(+), 68 deletions(-) create mode 100644 e2e/src/gateway.test.ts diff --git a/e2e/package.json b/e2e/package.json index 9d104a0d..7c574ba9 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -14,6 +14,7 @@ "e2b": "^2.31.0" }, "devDependencies": { + "@dormice/gateway": "workspace:*", "@dormice/server": "workspace:*" } } diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts new file mode 100644 index 00000000..31c2f481 --- /dev/null +++ b/e2e/src/gateway.test.ts @@ -0,0 +1,559 @@ +import { spawn } from 'node:child_process'; +import { randomBytes, randomUUID } from 'node:crypto'; +import { mkdtemp, rm } from 'node:fs/promises'; +import net, { type AddressInfo } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Dormice } from '@dormice/sdk'; +import { Sandbox } from 'e2b'; +import { describe, expect, inject, it } from 'vitest'; + +// The gateway exam: two real daemons behind a real gateway, all three +// booted the production way and driven only over the wire — the SDK, the +// official e2b package and plain fetch. Direct calls to a node exist only +// to stage what the gateway must then find (a sandbox built behind its +// back, a name on two nodes, a destroy it did not see). Skipped in docker +// mode, where the setup boots node A alone. +const skip = inject('dormiceGatewayEndpoint') === null; + +const gateway = () => inject('dormiceGatewayEndpoint') as string; +const token = () => inject('dormiceGatewayToken') as string; +const nodes = () => + inject('dormiceGatewayNodes') as Array<{ id: string; endpoint: string }>; +const viaGateway = () => new Dormice({ endpoint: gateway(), token: token() }); +function direct(id: string) { + const node = nodes().find((n) => n.id === id); + if (!node) throw new Error(`no node ${id} in the exam fleet`); + return new Dormice({ endpoint: node.endpoint, token: token() }); +} +const other = (id: string) => (id === 'node-b' ? 'node-c' : 'node-b'); + +/** Polls until the probe answers something — nodes check in on their own clock, not ours. */ +async function until( + probe: () => Promise, + timeoutMs = 10_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + const value = await probe(); + if (value !== undefined) return value; + if (Date.now() > deadline) throw new Error('condition never became true'); + await new Promise((resolve) => setTimeout(resolve, 100)); + } +} + +async function rpc( + path: string, + payload: unknown = {}, + bearer = token(), + endpoint = gateway(), +): Promise<{ status: number; body: unknown; headers: Headers }> { + const res = await fetch(`${endpoint}${path}`, { + method: 'POST', + headers: { + authorization: `Bearer ${bearer}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(payload), + }); + const text = await res.text(); + return { + status: res.status, + body: text ? JSON.parse(text) : null, + headers: res.headers, + }; +} + +interface ListedNode { + id: string; + endpoint: string; + reachable: boolean; + lastCheckInAt: string | null; + build: { commit: string } | null; + reading: { sandboxes: { byState: { active: number } } } | null; + placedSinceCheckIn: number; +} +async function listNodes(): Promise { + const { status, body } = await rpc('/listNodes'); + expect(status).toBe(200); + return (body as { nodes: ListedNode[] }).nodes; +} + +const status = (error: unknown) => (error as { status?: number }).status; +const message = (r: { body: unknown }) => + (r.body as { message: string }).message; + +describe.skipIf(skip)('the gateway in front of two daemons', () => { + it('/healthz is open and names the build; a wrong token is refused', async () => { + const health = (await (await fetch(`${gateway()}/healthz`)).json()) as { + status: string; + build: { commit: string } | null; + }; + expect(health.status).toBe('ok'); + expect(health.build?.commit).toMatch(/^[0-9a-f]{7,}$/); + expect((await rpc('/listNodes', {}, 'x'.repeat(64))).status).toBe(401); + }); + + it('both nodes checked in: reachable, with a reading and the build they run', async () => { + const listed = await until(async () => { + const seen = await listNodes(); + return seen.length === 2 && seen.every((n) => n.reachable) + ? seen + : undefined; + }); + expect(listed.map((n) => n.id).sort()).toEqual(['node-b', 'node-c']); + for (const node of listed) { + expect(node.reading?.sandboxes.byState.active).toBeGreaterThanOrEqual(0); + expect(node.build?.commit).toMatch(/^[0-9a-f]{7,}$/); + expect(node.endpoint).toBe( + nodes().find((n) => n.id === node.id)?.endpoint, + ); + } + }); + + it('acquire is placed on one node and lands in exactly one ledger; re-acquire finds it there', async () => { + const created = await viaGateway().acquireSandbox('gw-place'); + try { + expect(created.status).toBe('ready'); + expect(created.created).toBe(true); + expect(['node-b', 'node-c']).toContain(created.sandbox.nodeId); + const here = await direct(created.sandbox.nodeId).listSandboxes(); + expect(here.some((s) => s.id === created.sandbox.id)).toBe(true); + const there = await direct(other(created.sandbox.nodeId)).listSandboxes(); + expect(there.some((s) => s.name === 'gw-place')).toBe(false); + + const again = await viaGateway().acquireSandbox('gw-place'); + expect(again.created).toBe(false); + expect(again.sandbox.id).toBe(created.sandbox.id); + expect(again.sandbox.nodeId).toBe(created.sandbox.nodeId); + } finally { + await viaGateway().destroySandbox('gw-place'); + } + }); + + it("files and commands round-trip; a node's 404 for a missing file passes through and the sandbox stays routable", async () => { + await viaGateway().acquireSandbox('gw-work'); + try { + await viaGateway().writeFile('gw-work', 'hello.txt', 'via gateway'); + const read = await viaGateway().readFile('gw-work', 'hello.txt'); + expect(new TextDecoder().decode(read.content)).toBe('via gateway'); + const ran = await viaGateway().execCommand('gw-work', 'echo through'); + expect(ran.stdout).toBe('through\n'); + + await expect( + viaGateway().readFile('gw-work', 'missing.txt'), + ).rejects.toMatchObject({ name: 'DormiceApiError', status: 404 }); + expect( + (await viaGateway().execCommand('gw-work', 'echo still-here')).stdout, + ).toBe('still-here\n'); + } finally { + await viaGateway().destroySandbox('gw-work'); + } + }); + + it('five simultaneous acquires of a new name are one sandbox on one node', async () => { + const results = await Promise.all( + Array.from({ length: 5 }, () => viaGateway().acquireSandbox('gw-burst')), + ); + try { + expect(new Set(results.map((r) => r.sandbox.id)).size).toBe(1); + expect(new Set(results.map((r) => r.sandbox.nodeId)).size).toBe(1); + expect(results.filter((r) => r.created)).toHaveLength(1); + } finally { + await viaGateway().destroySandbox('gw-burst'); + } + }); + + it('a sandbox built behind its back is found by asking — at once, no reconcile to wait for', async () => { + const staged = await direct('node-b').acquireSandbox('gw-staged'); + try { + const ran = await viaGateway().execCommand('gw-staged', 'echo found'); + expect(ran.stdout).toBe('found\n'); + const found = await viaGateway().acquireSandbox('gw-staged'); + expect(found.created).toBe(false); + expect(found.sandbox.id).toBe(staged.sandbox.id); + expect(found.sandbox.nodeId).toBe('node-b'); + } finally { + await viaGateway().destroySandbox('gw-staged'); + } + }); + + it('one name on two nodes is refused with a 409 naming both, and routable again once one copy is destroyed', async () => { + await direct('node-b').acquireSandbox('gw-twin'); + await direct('node-c').acquireSandbox('gw-twin'); + try { + await expect( + viaGateway().acquireSandbox('gw-twin'), + ).rejects.toMatchObject({ + status: 409, + message: expect.stringMatching(/node-b and node-c/), + }); + await expect( + viaGateway().execCommand('gw-twin', 'true'), + ).rejects.toMatchObject({ status: 409 }); + await direct('node-c').destroySandbox('gw-twin'); + const healed = await viaGateway().acquireSandbox('gw-twin'); + expect(healed.created).toBe(false); + expect(healed.sandbox.nodeId).toBe('node-b'); + } finally { + await direct('node-b').destroySandbox('gw-twin'); + await direct('node-c').destroySandbox('gw-twin'); + } + }); + + it('a destroy through the gateway is final; a destroy behind its back is caught on the next use and the name is asked for afresh', async () => { + const created = await viaGateway().acquireSandbox('gw-gone'); + expect(await viaGateway().destroySandbox('gw-gone')).toEqual({ + destroyed: true, + }); + await expect( + viaGateway().execCommand('gw-gone', 'true'), + ).rejects.toMatchObject({ + status: 404, + message: expect.stringMatching(/no sandbox named "gw-gone"/), + }); + expect(await viaGateway().destroySandbox('gw-gone')).toEqual({ + destroyed: false, + }); + const reborn = await viaGateway().acquireSandbox('gw-gone'); + expect(reborn.created).toBe(true); + expect(reborn.sandbox.id).not.toBe(created.sandbox.id); + try { + // Destroyed directly on its node, then rebuilt directly on the other: + // the gateway's first use relays the node's own 404, re-checks the + // cache off the request path, and the next use finds the new home. + // Without the re-check every use would keep going to the old node. + await direct(reborn.sandbox.nodeId).destroySandbox('gw-gone'); + await expect( + viaGateway().execCommand('gw-gone', 'true'), + ).rejects.toMatchObject({ status: 404 }); + const moved = await direct(other(reborn.sandbox.nodeId)).acquireSandbox( + 'gw-gone', + ); + await until(async () => { + try { + return await viaGateway().execCommand('gw-gone', 'echo moved'); + } catch (error) { + if (status(error) === 404) return undefined; + throw error; + } + }); + const found = await viaGateway().acquireSandbox('gw-gone'); + expect(found.created).toBe(false); + expect(found.sandbox.id).toBe(moved.sandbox.id); + } finally { + await viaGateway().destroySandbox('gw-gone'); + } + }); + + it('the active gate (2 per node here) refuses with a 503 naming every node, and reopens when a sandbox is destroyed', async () => { + const baseline = new Map( + (await listNodes()).map((n) => [ + n.id, + n.reading?.sandboxes.byState.active ?? 0, + ]), + ); + const names: string[] = []; + try { + const refusal = await until(async () => { + if (names.length > 8) throw new Error('gate never closed'); + const name = `gw-fill-${names.length}`; + const answer = await rpc('/acquireSandbox', { name }); + if (answer.status === 200) { + names.push(name); + return undefined; + } + return answer.status === 503 ? answer : undefined; + }, 20_000); + expect(message(refusal)).toContain('node-b'); + expect(message(refusal)).toContain('node-c'); + expect(message(refusal)).toContain('reach the 2 limit'); + expect(refusal.headers.get('retry-after')).toBe('15'); + + const freed = names.pop(); + if (freed) await viaGateway().destroySandbox(freed); + const reopened = await until(async () => { + try { + return await viaGateway().acquireSandbox('gw-fill-late'); + } catch (error) { + if (status(error) === 503) return undefined; + throw error; + } + }); + names.push('gw-fill-late'); + expect(reopened.status).toBe('ready'); + } finally { + for (const name of names) await viaGateway().destroySandbox(name); + // Leave the fleet as found: the gate reads the nodes' readings, which + // catch up with the destroys at their next check-in — the next test + // must not inherit a closed gate. + await until(async () => + (await listNodes()).every( + (n) => + n.reading?.sandboxes.byState.active === baseline.get(n.id) && + n.placedSinceCheckIn === 0, + ) + ? true + : undefined, + ); + } + }); + + it('daemon-addressed verbs are an honest 501; a misspelled verb is a 404', async () => { + await expect(viaGateway().listSandboxes()).rejects.toMatchObject({ + status: 501, + message: expect.stringMatching(/call the node directly/), + }); + const e2bList = await fetch(`${gateway()}/e2b/api/v2/sandboxes`, { + headers: { 'x-api-key': `e2b_${token()}` }, + }); + expect(e2bList.status).toBe(501); + expect(((await e2bList.json()) as { code: number }).code).toBe(501); + expect((await rpc('/acquireSandbx', { name: 'x' })).status).toBe(404); + }); + + it('the official e2b package works through the gateway: create, live streaming, files, kill; an unnamed create routes by id', async () => { + const connection = { + apiKey: `e2b_${token()}`, + apiUrl: `${gateway()}/e2b/api`, + sandboxUrl: `${gateway()}/e2b/envd`, + }; + const sbx = await Sandbox.create({ + ...connection, + metadata: { name: 'gw-e2b' }, + }); + try { + const chunks: Array<{ text: string; at: number }> = []; + const result = await sbx.commands.run( + 'echo first; sleep 1; echo second', + { + onStdout: (text) => { + chunks.push({ text, at: Date.now() }); + }, + }, + ); + expect(result.stdout).toBe('first\nsecond\n'); + // Streaming through two hops: a real gap between the frames, as in + // e2b.test.ts against the daemon alone — a buffering hop would + // deliver both at once. + const at = chunks.map((c) => c.at); + expect(at.length).toBeGreaterThanOrEqual(2); + expect((at.at(-1) ?? 0) - (at[0] ?? 0)).toBeGreaterThanOrEqual(500); + + await sbx.files.write('/home/user/gateway.txt', 'through the gateway'); + expect(await sbx.files.read('/home/user/gateway.txt')).toBe( + 'through the gateway', + ); + // The same name through the native face is the same sandbox. + const found = await viaGateway().acquireSandbox('gw-e2b'); + expect(found.created).toBe(false); + expect(found.sandbox.id).toBe(sbx.sandboxId); + } finally { + await sbx.kill(); + } + expect(await viaGateway().destroySandbox('gw-e2b')).toEqual({ + destroyed: false, + }); + + const anonymous = await Sandbox.create(connection); + try { + const info = await fetch( + `${connection.apiUrl}/sandboxes/${anonymous.sandboxId}`, + { headers: { 'x-api-key': connection.apiKey } }, + ); + expect(info.status).toBe(200); + } finally { + await anonymous.kill(); + } + }); + + it('envd preflights are answered without a header; the bare signed-URL door is a 501 with CORS', async () => { + const preflight = await fetch(`${gateway()}/e2b/envd/files`, { + method: 'OPTIONS', + headers: { origin: 'https://app.example' }, + }); + expect(preflight.status).toBe(204); + expect(preflight.headers.get('access-control-allow-origin')).toBe('*'); + const bare = await fetch(`${gateway()}/files?signature=x&path=/y`); + expect(bare.status).toBe(501); + expect(bare.headers.get('access-control-allow-origin')).toBe('*'); + expect(((await bare.json()) as { code: string }).code).toBe( + 'unimplemented', + ); + const stranger = await fetch(`${gateway()}/e2b/envd/files`, { + headers: { 'e2b-sandbox-id': randomUUID() }, + }); + expect(stranger.status).toBe(502); + expect(((await stranger.json()) as { code: string }).code).toBe( + 'unavailable', + ); + }); + + it('a third node joins at its first check-in; when it dies its sandboxes 502 and new names are a 503 naming it, until an operator removes it', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dormice-e2e-node-d-')); + const port = await freePort(); + const child = spawn('node', [inject('dormiceDaemonMain')], { + env: { + PATH: process.env.PATH ?? '', + DORMICE_NODE_ID: 'node-d', + DORMICE_PORT: String(port), + DORMICE_DATA_DIR: dir, + DORMICE_DB_PATH: join(dir, 'dormice.db'), + DORMICE_API_TOKEN: token(), + DORMICE_GATEWAY_ENDPOINT: gateway(), + DORMICE_CHECK_IN_INTERVAL_SECONDS: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stderr.on('data', (chunk) => { + output += chunk; + }); + const exited = new Promise((resolve) => + child.on('exit', () => resolve()), + ); + const d = new Dormice({ + endpoint: `http://127.0.0.1:${port}`, + token: token(), + }); + try { + await until(async () => + (await listNodes()).some((n) => n.id === 'node-d' && n.reachable) + ? true + : undefined, + ).catch((error) => { + throw new Error(`${String(error)}\n${output}`); + }); + const staged = await d.acquireSandbox('gw-on-d'); + expect( + (await viaGateway().execCommand('gw-on-d', 'echo on-d')).stdout, + ).toBe('on-d\n'); + + child.kill(); + await exited; + await expect( + viaGateway().execCommand('gw-on-d', 'true'), + ).rejects.toMatchObject({ + status: 502, + message: expect.stringMatching(/did not answer/), + }); + const refused = await rpc('/acquireSandbox', { name: 'gw-while-d-down' }); + expect(refused.status).toBe(503); + expect(message(refused)).toContain('node node-d did not answer'); + expect(refused.headers.get('retry-after')).toBe('15'); + // Down is also what listNodes says, on the node's own interval. + await until(async () => + (await listNodes()).some((n) => n.id === 'node-d' && !n.reachable) + ? true + : undefined, + ); + + expect((await rpc('/removeNode', { id: 'node-d' })).body).toEqual({ + removed: true, + }); + expect((await listNodes()).some((n) => n.id === 'node-d')).toBe(false); + const placed = await viaGateway().acquireSandbox('gw-while-d-down'); + try { + expect(['node-b', 'node-c']).toContain(placed.sandbox.nodeId); + } finally { + await viaGateway().destroySandbox('gw-while-d-down'); + } + expect(staged.sandbox.nodeId).toBe('node-d'); + } finally { + child.kill(); + await exited; + await rm(dir, { recursive: true, force: true }); + } + }); + + it("a gateway's boot, black-box: a second gateway on the same database file dies naming the conflict", async () => { + const dir = await mkdtemp(join(tmpdir(), 'dormice-e2e-gateway-boot-')); + try { + const env = async () => ({ + PATH: process.env.PATH ?? '', + DORMICE_GATEWAY_PORT: String(await freePort()), + DORMICE_GATEWAY_DB_PATH: join(dir, 'gateway.db'), + DORMICE_API_TOKEN: randomBytes(32).toString('hex'), + }); + const first = await bootGateway(await env()); + expect(first.outcome, first.output).toBe('healthy'); + try { + const second = await bootGateway(await env()); + expect(second.outcome, second.output).toEqual({ exitCode: 1 }); + expect(second.output).toMatch( + /another gateway is already running against .*gateway\.db/, + ); + expect(second.output).toContain('DORMICE_GATEWAY_DB_PATH'); + } finally { + await first.kill(); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); + +/** A port the OS just handed out and released — for a process this test boots itself. */ +function freePort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + const { port } = server.address() as AddressInfo; + server.close(() => resolve(port)); + }); + }); +} + +/** + * Boots a gateway the production way (`node dist/main.js` + environment) + * and reports how the boot ended: healthy at its port, or the exit code + * it died with — its whole output kept for the assertion either way. + */ +async function bootGateway(env: Record): Promise<{ + outcome: 'healthy' | { exitCode: number | null }; + output: string; + kill: () => Promise; +}> { + const child = spawn('node', [inject('dormiceGatewayMain')], { + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stderr.on('data', (chunk) => { + output += chunk; + }); + let exited: number | null | undefined; + const exit = new Promise((resolve) => { + child.on('exit', (code) => { + exited = code; + resolve(); + }); + }); + const endpoint = `http://127.0.0.1:${env.DORMICE_GATEWAY_PORT}`; + const deadline = Date.now() + 15_000; + while (Date.now() < deadline) { + if (exited !== undefined) { + return { outcome: { exitCode: exited }, output, kill: async () => {} }; + } + const healthy = await fetch(`${endpoint}/healthz`) + .then((r) => r.ok) + .catch(() => false); + if (healthy) break; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + return { + outcome: exited === undefined ? 'healthy' : { exitCode: exited }, + output, + kill: async () => { + child.kill(); + await exit; + }, + }; +} diff --git a/e2e/src/native.test.ts b/e2e/src/native.test.ts index 65635d55..cd0c2b48 100644 --- a/e2e/src/native.test.ts +++ b/e2e/src/native.test.ts @@ -1,3 +1,4 @@ +import { spawn } from 'node:child_process'; import { DEFAULT_LIFECYCLE_POLICY, Dormice } from '@dormice/sdk'; import { describe, expect, inject, it } from 'vitest'; @@ -111,6 +112,42 @@ describe('native API over a real daemon', () => { ).resolves.toBe(400); }); + it('a second daemon on the same ledger dies at boot naming the conflict — one ledger, one daemon', async () => { + // The same environment as the running daemon, another port: the port + // is not what must refuse it, the ledger lock is (measured 2026-09-11: + // with the lock handle garbage-collected, this second daemon started). + const child = spawn('node', [inject('dormiceDaemonMain')], { + env: { + ...inject('dormiceNodeAEnv'), + DORMICE_PORT: String( + Number(new URL(inject('dormiceEndpoint')).port) + 1000, + ), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + child.stdout.on('data', (chunk) => { + output += chunk; + }); + child.stderr.on('data', (chunk) => { + output += chunk; + }); + const code = await new Promise((resolve) => { + const timer = setTimeout(() => { + child.kill(); + resolve(null); + }, 10_000); + child.on('exit', (exitCode) => { + clearTimeout(timer); + resolve(exitCode); + }); + }); + expect(code, output).toBe(1); + expect(output).toMatch( + /another daemon is already running against .*dormice\.db/, + ); + }); + it('destroys a sandbox: gone, forgotten, idempotent', async () => { const created = await client().acquireSandbox('destroy-key'); diff --git a/e2e/src/setup/daemon.ts b/e2e/src/setup/daemon.ts index 075aa6c9..41b11a4c 100644 --- a/e2e/src/setup/daemon.ts +++ b/e2e/src/setup/daemon.ts @@ -1,4 +1,4 @@ -import { spawn } from 'node:child_process'; +import { type ChildProcess, spawn } from 'node:child_process'; import { randomBytes } from 'node:crypto'; import { existsSync } from 'node:fs'; import { mkdtemp, rm } from 'node:fs/promises'; @@ -8,12 +8,31 @@ import { fileURLToPath } from 'node:url'; import { startMiniS3 } from '@dormice/server/mini-s3'; import type { TestProject } from 'vitest/node'; +export interface FleetNodeHandle { + id: string; + endpoint: string; +} + declare module 'vitest' { export interface ProvidedContext { dormiceEndpoint: string; dormiceToken: string; /** The Caddy config file the exam daemon owns — an operator-visible artifact. */ dormiceIngressFile: string; + /** The built daemon entry, for tests that boot a daemon of their own. */ + dormiceDaemonMain: string; + /** Node A's exact environment — a second daemon on the same ledger must refuse to start. */ + dormiceNodeAEnv: Record; + /** The gateway fronting nodes B and C; null in docker mode, where only node A runs. */ + dormiceGatewayEndpoint: string | null; + /** The fleet's one token: the gateway's, and every fronted node's. */ + dormiceGatewayToken: string | null; + /** The fronted nodes, reachable directly — to stage what the gateway must then find. */ + dormiceGatewayNodes: FleetNodeHandle[] | null; + /** The exam's S3, for a node a test boots itself. */ + dormiceMiniS3Url: string; + /** The built gateway entry, for tests that boot a gateway of their own (its boot refusals). */ + dormiceGatewayMain: string; } } @@ -25,26 +44,23 @@ declare module 'vitest' { const MAIN = fileURLToPath( new URL('../../../packages/server/dist/main.js', import.meta.url), ); +const GATEWAY_MAIN = fileURLToPath( + new URL('../../../packages/gateway/dist/main.js', import.meta.url), +); -export default async function setup(project: TestProject) { - if (!existsSync(MAIN)) { - throw new Error( - `daemon build not found at ${MAIN} — run \`pnpm build\` first`, - ); - } - - const token = randomBytes(32).toString('hex'); - const dataDir = await mkdtemp(join(tmpdir(), 'dormice-e2e-')); - // Random high port: never collides with a locally running daemon on 3676. - const port = 20000 + Math.floor(Math.random() * 20000); - const endpoint = `http://127.0.0.1:${port}`; - - // The exam's own S3 (in-process, test-only): the archive lifecycle runs - // black-box in every mode — the fake executor exports real files, the - // store speaks real HTTP. It also mimics OSS's checksum strictness, so a - // daemon that regressed pit #8 fails here too. - const miniS3 = await startMiniS3(); +interface DaemonSpec { + /** DORMICE_NODE_ID; omitted keeps the daemon's default, as node A always has. */ + nodeId?: string; + port: number; + token: string; + dataDir: string; + miniS3Url: string; + extraEnv?: Record; +} +/** Boots one daemon the production way and waits for /healthz. */ +async function bootDaemon(spec: DaemonSpec) { + const endpoint = `http://127.0.0.1:${spec.port}`; // An explicit allowlist instead of inheriting the whole environment: // whatever DORMICE_* knobs happen to be exported in the developer's shell // must not silently reconfigure the daemon under test. The three docker @@ -63,82 +79,211 @@ export default async function setup(project: TestProject) { inherited[name] = value; } } + const env: Record = { + // Exam disks evaporate with the exam: without this default, a docker + // run without an exported DORMICE_DATA_DIR drops its sandbox disks + // into /var/lib/dormice — the resident daemon's data dir, whose + // startup guard then refuses to start (measured 2026-07-10). An + // exported value still wins through `inherited` below. + DORMICE_DATA_DIR: spec.dataDir, + ...inherited, + ...(spec.nodeId === undefined ? {} : { DORMICE_NODE_ID: spec.nodeId }), + DORMICE_PORT: String(spec.port), + DORMICE_DB_PATH: join(spec.dataDir, 'dormice.db'), + DORMICE_API_TOKEN: spec.token, + // Sweep every second so lifecycle tests run on second-scale policies + // instead of the production default of days. + DORMICE_SCAN_INTERVAL_SECONDS: '1', + // Sample every second so history verbs have rows to answer with + // inside a test's lifetime. + DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS: '1', + // A wildcard sandbox domain so getHost() and the port proxy are + // exercised — no DNS needed, tests spoof the Host header locally. + // A first-boot seed since the setting moved into the ledger: every + // exam starts on a fresh DB, so the seed lands every run, and + // settings-hot.test.ts exercises the live edit on top of it. + DORMICE_SANDBOX_DOMAIN: 'sbx.dormice.test', + // The archiver, pointed at the exam's mini S3 — likewise a + // first-boot seed for the ledger's S3 settings. Deliberately NOT in + // the inherited allowlist: a developer's real DORMICE_S3_* exports + // must never leak an exam's archives into a production bucket. + DORMICE_S3_ENDPOINT: spec.miniS3Url, + DORMICE_S3_BUCKET: 'e2e-archive', + DORMICE_S3_ACCESS_KEY_ID: 'e2e-key', + DORMICE_S3_SECRET_ACCESS_KEY: 'e2e-secret', + DORMICE_S3_FORCE_PATH_STYLE: 'true', + // A managed ingress so the domain-binding verbs run black-box. The + // reload command is a no-op: the exam grades what the daemon writes + // and answers, not Caddy — Caddy's side is real-machine acceptance. + DORMICE_INGRESS_FILE: join(spec.dataDir, 'Caddyfile'), + DORMICE_INGRESS_RELOAD_CMD: 'true', + ...spec.extraEnv, + }; const child = spawn('node', [MAIN], { - env: { - // Exam disks evaporate with the exam: without this default, a docker - // run without an exported DORMICE_DATA_DIR drops its sandbox disks - // into /var/lib/dormice — the resident daemon's data dir, whose - // startup guard then refuses to start (measured 2026-07-10). An - // exported value still wins through `inherited` below. - DORMICE_DATA_DIR: dataDir, - ...inherited, - DORMICE_PORT: String(port), - DORMICE_DB_PATH: join(dataDir, 'dormice.db'), - DORMICE_API_TOKEN: token, - // Sweep every second so lifecycle tests run on second-scale policies - // instead of the production default of days. - DORMICE_SCAN_INTERVAL_SECONDS: '1', - // Sample every second so history verbs have rows to answer with - // inside a test's lifetime. - DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS: '1', - // A wildcard sandbox domain so getHost() and the port proxy are - // exercised — no DNS needed, tests spoof the Host header locally. - // A first-boot seed since the setting moved into the ledger: every - // exam starts on a fresh DB, so the seed lands every run, and - // settings-hot.test.ts exercises the live edit on top of it. - DORMICE_SANDBOX_DOMAIN: 'sbx.dormice.test', - // The archiver, pointed at the exam's mini S3 — likewise a - // first-boot seed for the ledger's S3 settings. Deliberately NOT in - // the inherited allowlist: a developer's real DORMICE_S3_* exports - // must never leak an exam's archives into a production bucket. - DORMICE_S3_ENDPOINT: miniS3.url, - DORMICE_S3_BUCKET: 'e2e-archive', - DORMICE_S3_ACCESS_KEY_ID: 'e2e-key', - DORMICE_S3_SECRET_ACCESS_KEY: 'e2e-secret', - DORMICE_S3_FORCE_PATH_STYLE: 'true', - // A managed ingress so the domain-binding verbs run black-box. The - // reload command is a no-op: the exam grades what the daemon writes - // and answers, not Caddy — Caddy's side is real-machine acceptance. - DORMICE_INGRESS_FILE: join(dataDir, 'Caddyfile'), - DORMICE_INGRESS_RELOAD_CMD: 'true', - }, + env, stdio: ['ignore', 'pipe', 'pipe'], }); + await waitHealthy(child, endpoint, `daemon ${spec.nodeId ?? 'A'}`); + return { + endpoint, + token: spec.token, + env, + ingressFile: join(spec.dataDir, 'Caddyfile'), + kill: () => child.kill(), + }; +} + +async function waitHealthy( + child: ChildProcess, + endpoint: string, + what: string, +): Promise { let output = ''; - child.stdout.on('data', (chunk) => { + child.stdout?.on('data', (chunk) => { output += chunk; }); - child.stderr.on('data', (chunk) => { + child.stderr?.on('data', (chunk) => { output += chunk; }); - const deadline = Date.now() + 10_000; for (;;) { if (child.exitCode !== null) { - throw new Error(`daemon exited during startup:\n${output}`); + throw new Error(`${what} exited during startup:\n${output}`); } try { const res = await fetch(`${endpoint}/healthz`); if (res.ok) { - break; + return; } } catch { // Not listening yet; keep probing until the deadline. } if (Date.now() > deadline) { child.kill(); - throw new Error(`daemon did not come up within 10s:\n${output}`); + throw new Error(`${what} did not come up within 10s:\n${output}`); } await new Promise((resolve) => setTimeout(resolve, 100)); } +} + +export default async function setup(project: TestProject) { + if (!existsSync(MAIN)) { + throw new Error( + `daemon build not found at ${MAIN} — run \`pnpm build\` first`, + ); + } + + const dataDir = await mkdtemp(join(tmpdir(), 'dormice-e2e-')); + // Random high base port: never collides with a locally running daemon on + // 3676 or a gateway on 3677; the fleet takes the next three. Below + // 32768, where Linux hands out ephemeral ports — a fleet port already + // taken by some outbound connection would be a boot failure for nothing. + const base = 20000 + Math.floor(Math.random() * 12000); - project.provide('dormiceEndpoint', endpoint); - project.provide('dormiceToken', token); - project.provide('dormiceIngressFile', join(dataDir, 'Caddyfile')); + // The exam's own S3 (in-process, test-only): the archive lifecycle runs + // black-box in every mode — the fake executor exports real files, the + // store speaks real HTTP. It also mimics OSS's checksum strictness, so a + // daemon that regressed pit #8 fails here too. + const miniS3 = await startMiniS3(); + + // Node A: the daemon every existing suite talks to, exactly as before — + // a standalone daemon that checks in with nobody. + const a = await bootDaemon({ + port: base, + token: randomBytes(32).toString('hex'), + dataDir, + miniS3Url: miniS3.url, + }); + project.provide('dormiceEndpoint', a.endpoint); + project.provide('dormiceToken', a.token); + project.provide('dormiceIngressFile', a.ingressFile); + project.provide('dormiceDaemonMain', MAIN); + project.provide('dormiceNodeAEnv', a.env); + project.provide('dormiceMiniS3Url', miniS3.url); + project.provide('dormiceGatewayMain', GATEWAY_MAIN); + + // The gateway exam: two more daemons behind a gateway, sharing A's mini + // S3 bucket the way a real fleet shares one, and sharing one token with + // the gateway the way a real fleet does. Fake mode only — in docker mode + // the startup guard judges containers by label across the whole machine, + // so two daemons on one docker would refuse each other. + const fleet: Array<{ kill: () => void }> = []; + const dirs: string[] = [dataDir]; + // Whatever came up before a boot failed is killed here: vitest never + // calls the teardown of a setup that threw, and a spawned daemon does + // not die with its parent. + const abandon = async (error: unknown) => { + for (const process of fleet.reverse()) process.kill(); + a.kill(); + await miniS3.close(); + throw error; + }; + if (process.env.DORMICE_EXECUTOR !== 'docker') { + if (!existsSync(GATEWAY_MAIN)) { + await abandon( + new Error( + `gateway build not found at ${GATEWAY_MAIN} — run \`pnpm build\` first`, + ), + ); + } + const gatewayPort = base + 3; + const gatewayEndpoint = `http://127.0.0.1:${gatewayPort}`; + const fleetToken = randomBytes(32).toString('hex'); + // The gateway first, so the nodes' first check-in lands; a gateway + // that comes up after its nodes learns them at their next check-in + // anyway, but the exam should not start on that slack. + const gateway = spawn('node', [GATEWAY_MAIN], { + env: { + PATH: process.env.PATH ?? '', + DORMICE_GATEWAY_PORT: String(gatewayPort), + DORMICE_GATEWAY_DB_PATH: join(dataDir, 'gateway.db'), + DORMICE_API_TOKEN: fleetToken, + // A tiny active limit so the gate can be reached with a handful of + // sandboxes; the CPU gate is opened wide — a laptop running the + // suite is not the machine under judgment; the disk floor is off + // for the same reason. + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '100', + DORMICE_GATEWAY_NODE_MIN_DISK_GB: '0', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + fleet.push({ kill: () => gateway.kill() }); + await waitHealthy(gateway, gatewayEndpoint, 'gateway').catch(abandon); + const nodes: FleetNodeHandle[] = []; + for (const [index, id] of (['node-b', 'node-c'] as const).entries()) { + const nodeDir = await mkdtemp(join(tmpdir(), `dormice-e2e-${id}-`)); + dirs.push(nodeDir); + const node = await bootDaemon({ + nodeId: id, + port: base + 1 + index, + token: fleetToken, + dataDir: nodeDir, + miniS3Url: miniS3.url, + extraEnv: { + DORMICE_GATEWAY_ENDPOINT: gatewayEndpoint, + // Second-scale check-ins so a node's readings and its absence + // show inside a test. + DORMICE_CHECK_IN_INTERVAL_SECONDS: '1', + }, + }).catch(abandon); + fleet.push(node); + nodes.push({ id, endpoint: node.endpoint }); + } + project.provide('dormiceGatewayEndpoint', gatewayEndpoint); + project.provide('dormiceGatewayToken', fleetToken); + project.provide('dormiceGatewayNodes', nodes); + } else { + project.provide('dormiceGatewayEndpoint', null); + project.provide('dormiceGatewayToken', null); + project.provide('dormiceGatewayNodes', null); + } return async () => { - child.kill(); + // The nodes first (they stop checking in), then the gateway. + for (const process of fleet.reverse()) process.kill(); + a.kill(); await miniS3.close(); - await rm(dataDir, { recursive: true, force: true }); + for (const dir of dirs) await rm(dir, { recursive: true, force: true }); }; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 44238dfe..c520a471 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -36,6 +36,9 @@ importers: specifier: ^2.31.0 version: 2.31.0 devDependencies: + '@dormice/gateway': + specifier: workspace:* + version: link:../packages/gateway '@dormice/server': specifier: workspace:* version: link:../packages/server From 4c697dd44760e2b3d47bda0ae4cfe89aaa1b29f5 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:48:51 +0800 Subject: [PATCH 08/89] The gateway's systemd unit, its row in the README, and the shared changeset for the fleet wire The unit is copied into place by hand until install.sh learns the two roles; it carries no docker dependency, because the gateway never touches a container. --- .changeset/gateway-check-in-and-lookup.md | 5 +++++ README.md | 1 + deploy/dormice-gateway.service | 24 +++++++++++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 .changeset/gateway-check-in-and-lookup.md create mode 100644 deploy/dormice-gateway.service diff --git a/.changeset/gateway-check-in-and-lookup.md b/.changeset/gateway-check-in-and-lookup.md new file mode 100644 index 00000000..35d961a0 --- /dev/null +++ b/.changeset/gateway-check-in-and-lookup.md @@ -0,0 +1,5 @@ +--- +"@dormice/shared": minor +--- + +Wire schemas for a fleet behind a gateway: `lookupSandbox` (a node answers "do you hold this sandbox?" by name or id), the node check-in (`checkInRequestSchema`, readings, build), and the gateway's `listNodes` / `removeNode`. The host-metrics schema is split into named parts (`hostReadingSchema`, `dataDiskSchema`, `sandboxStateCountsSchema`) that `hostMetricsResponseSchema` still composes unchanged. diff --git a/README.md b/README.md index 4ea0e84a..8ca14744 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,7 @@ pnpm monorepo: | `packages/sdk` | `@dormice/sdk` — TypeScript client for the native API | | `packages/cli` | `dormice` command-line tool (`dor` for short) | | `packages/console` | Web console: React SPA, served by the daemon at `/console` | +| `packages/gateway` | The fleet's one door in front of one or more daemons: places new sandboxes, finds existing ones by asking the nodes, forwards everything else | | `e2e` | Black-box suite: boots the built daemon, drives it over the wire | | `examples` | Runnable demos: the native SDK, the official `e2b` package, a resident agent | diff --git a/deploy/dormice-gateway.service b/deploy/dormice-gateway.service new file mode 100644 index 00000000..e8369bbc --- /dev/null +++ b/deploy/dormice-gateway.service @@ -0,0 +1,24 @@ +# Dormice gateway: the fleet's one door in front of one or more daemons. +# Configuration lives in /etc/dormice/gateway.env (full-line comments only +# there — systemd's EnvironmentFile treats an inline comment as part of the +# value). install.sh does not install this unit yet: the two-role install +# (gateway + node on one machine, `--role node` elsewhere) is a later step; +# until then it is copied into place by hand (docs/测试机搭建手册.md). +[Unit] +Description=Dormice gateway (fleet front door) +Wants=network-online.target +After=network-online.target + +[Service] +ExecStart=/usr/local/bin/node /opt/dormice/packages/gateway/dist/main.js +EnvironmentFile=/etc/dormice/gateway.env +# Crash-only: the gateway holds no sandbox state — nodes check in again +# within seconds and the cache refills by asking — so restarting it is +# always safe. +Restart=always +RestartSec=3 +# One process per database file. The SQLite lock enforces this; never wrap +# the gateway in a process manager that forks workers. + +[Install] +WantedBy=multi-user.target From f2df652d685a6de2296199032f56c680005ec347 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 03:49:10 +0800 Subject: [PATCH 09/89] main.ts: import order, as biome sorts it --- packages/server/src/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index 90b02f3d..bd9fbe9f 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -3,9 +3,9 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { pino } from 'pino'; import { buildApp } from './app'; -import { CheckIn, readNodeReading } from './check-in'; import { Archiver } from './archive/archiver'; import { LedgerArchiveStore } from './archive/ledger-store'; +import { CheckIn, readNodeReading } from './check-in'; import { type Config, loadConfig } from './config'; import { recordActivity } from './db/activity'; import { migrateDb, openDb } from './db/db'; From fceecefaf58808b8051e8d9547c234a45f772056 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 04:21:13 +0800 Subject: [PATCH 10/89] Gateway review: a wake is not a placement, an invalid name is refused at the door, and a check-in that moves or shares an endpoint is said in the log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit forwardCreate took every 2xx id into placedIds and paid every node-side 4xx back to placedSinceCheckIn, wakes included. A wake was never counted in the first place, so destroying one inside the same interval uncounted a real placement and let the active gate admit one sandbox more than the node's reading allows. The callers now say whether pick() chose the node (a placement) or the name was found on it (a wake); only a placement moves the count either way. Reverse-proved: without the flag the new test reads b's count as 0 with a placement still in flight. nameOf and the E2B metadata.name are judged by the shared sandboxNameSchema before any node is asked. A node's 400 to lookupSandbox read as silence, so a 129-character name came back as a 503 with Retry-After — retry forever — instead of the 400 it deserves. Fleet.checkIn reports the endpoint a node moved from; the check-in route warns on a move and when two nodes report one endpoint. Both are misconfigurations (two machines sharing DORMICE_NODE_ID, whose default is node-1; a DORMICE_NODE_ENDPOINT naming the wrong machine) whose symptom downstream is a 409 on every name, and the check-in is the only place that sees them. e2b.ts: the `dialect` helper that existed only to be overridden is gone. --- packages/gateway/src/app.test.ts | 77 +++++++++++++++++++++++++++ packages/gateway/src/fleet.test.ts | 2 + packages/gateway/src/fleet.ts | 10 ++-- packages/gateway/src/routes/create.ts | 41 +++++++++----- packages/gateway/src/routes/e2b.ts | 46 +++++++++++----- packages/gateway/src/routes/native.ts | 42 +++++++++++---- packages/gateway/src/routes/nodes.ts | 29 +++++++++- 7 files changed, 206 insertions(+), 41 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 63e03031..f4490e9c 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -288,6 +288,19 @@ const sandboxOf = (r: { body: unknown }) => (r.body as { created: boolean; sandbox: { id: string; nodeId: string } }) .sandbox; +/** Builds a sandbox directly on a node, behind the gateway's back. */ +async function stage(node: FakeNode, name: string) { + const res = await fetch(`${node.endpoint}/acquireSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name }), + }); + return sandboxOf({ body: await res.json() }); +} + /** Polls until the probe answers something — cache verification runs off the request path. */ async function until( probe: () => Promise | T | undefined, @@ -470,6 +483,45 @@ describe('acquire: placing and finding', () => { expect((found.body as { created: boolean }).created).toBe(false); }); + it('a wake is not a placement: it neither counts against the node nor pays a placement back when destroyed — on either face', async () => { + const h = await gateway(['a', 'b']); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 10 }); + await h.checkIn(b, { active: 1 }); + const placedOnB = () => h.fleet.get('b')?.placedSinceCheckIn; + // Sandboxes that already live on b: the gateway finds and wakes them + // there, and b's reading already counts them. + await stage(b, 'sleepy'); + await stage(b, 'dozy'); + expect( + sandboxOf(await rpc(h, '/acquireSandbox', { name: 'sleepy' })).nodeId, + ).toBe('b'); + expect(placedOnB()).toBe(0); + const e2bWake = await fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ metadata: { name: 'dozy' } }), + }); + expect(e2bWake.status).toBe(201); + expect(placedOnB()).toBe(0); + // A new name placed on b (the emptiest) is counted once. + expect( + sandboxOf(await rpc(h, '/acquireSandbox', { name: 'fresh' })).nodeId, + ).toBe('b'); + expect(placedOnB()).toBe(1); + // Destroying the woken sandboxes must not pay back a placement that + // never was: b would be judged one sandbox emptier than it is. + await rpc(h, '/destroySandbox', { name: 'sleepy' }); + expect(placedOnB()).toBe(1); + await rpc(h, '/destroySandbox', { name: 'dozy' }); + expect(placedOnB()).toBe(1); + await rpc(h, '/destroySandbox', { name: 'fresh' }); + expect(placedOnB()).toBe(0); + }); + it('one name on two nodes is a 409 naming both, for every verb; once one copy is gone the name routes again', async () => { const h = await gateway(['a', 'b']); const [a, b] = h.nodes as [FakeNode, FakeNode]; @@ -607,6 +659,31 @@ describe('using, destroying, and the cache', () => { expect((await rpc(h, '/execCommand', { command: 'x' })).status).toBe(400); expect(h.nodes[0]?.hits).toEqual([]); }); + + it('a name the wire refuses is a 400 at the door on both faces, and no node is asked', async () => { + const h = await gateway(['a']); + const long = 'x'.repeat(129); + const native = await rpc(h, '/acquireSandbox', { name: long }); + expect(native.status).toBe(400); + expect(message(native)).toMatch(/^invalid name: /); + expect( + (await rpc(h, '/execCommand', { name: 7, command: 'x' })).status, + ).toBe(400); + const e2b = await fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ metadata: { name: long } }), + }); + expect(e2b.status).toBe(400); + expect(await e2b.json()).toMatchObject({ + code: 400, + message: expect.stringMatching(/^invalid metadata\.name: /), + }); + expect(h.nodes[0]?.hits).toEqual([]); + }); }); describe('the E2B faces', () => { diff --git a/packages/gateway/src/fleet.test.ts b/packages/gateway/src/fleet.test.ts index 18e4b523..97e3ab8b 100644 --- a/packages/gateway/src/fleet.test.ts +++ b/packages/gateway/src/fleet.test.ts @@ -49,6 +49,8 @@ describe('Fleet', () => { new Date(NOW.getTime() + 15_000), ); expect(second.joined).toBe(false); + expect(first.movedFrom).toBeNull(); + expect(second.movedFrom).toBe('http://10.0.0.7:80'); expect(second.node).toBe(first.node); expect(second.node.endpoint).toBe('http://10.0.0.8:80'); expect(second.node.reading?.sandboxes.byState.active).toBe(12); diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 1e3611d1..69e02140 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -81,7 +81,9 @@ export class Fleet { /** * A node reporting for duty. A first check-in adds the node (`joined` * says so, for the log); a changed endpoint is written through — the - * node states where it lives, the gateway does not remember better. The + * node states where it lives, the gateway does not remember better — + * and `movedFrom` names the old one, for the log: a node that moves at + * every check-in is two machines sharing one DORMICE_NODE_ID. The * placement counter restarts at zero: what was placed before this * reading is in it now, and what is still in flight on the node (its * row is written after the container is up) is in neither figure until @@ -91,9 +93,10 @@ export class Fleet { checkIn( report: CheckInRequest, now = new Date(), - ): { node: NodeState; joined: boolean } { + ): { node: NodeState; joined: boolean; movedFrom: string | null } { let node = this.members.get(report.nodeId); let joined = false; + let movedFrom: string | null = null; if (node === undefined) { const addedAt = now.toISOString(); this.db @@ -119,6 +122,7 @@ export class Fleet { .set({ endpoint: report.endpoint }) .where(eq(nodes.id, report.nodeId)) .run(); + movedFrom = node.endpoint; node.endpoint = report.endpoint; } node.lastCheckInAt = now; @@ -127,7 +131,7 @@ export class Fleet { node.reading = report.reading; node.placedSinceCheckIn = 0; node.placedIds.clear(); - return { node, joined }; + return { node, joined, movedFrom }; } /** The operator's word that the node is gone for good; a node still running re-adds itself at its next check-in. */ diff --git a/packages/gateway/src/routes/create.ts b/packages/gateway/src/routes/create.ts index 8448458c..d9f700b1 100644 --- a/packages/gateway/src/routes/create.ts +++ b/packages/gateway/src/routes/create.ts @@ -74,21 +74,34 @@ export interface CreateOptions { name: string | null; body: Buffer | undefined; face: CreateFace; + /** + * True when place() chose the target for a name no node held — the + * placement was counted against it in placedSinceCheckIn. False for a + * wake: the name was found on the target, its sandbox is in the reading + * already, and nothing was counted that could be uncounted. + */ + placed: boolean; } /** * Forwards a create and learns from the node's answer: a 2xx with a * readable id goes into the cache (name and id → this node), so the next - * request for the sandbox skips the round of questions, and into the - * node's placedIds, so a destroy inside the same interval can take the - * placement off the count again. A 2xx without a readable id is logged — - * the sandbox exists on the node and the next lookup finds it there, - * never silently. A 4xx or the node's own 500 means the node built - * nothing: the placement is taken off the count at once (it would have - * held a slot for a whole interval otherwise) and the next attempt may be - * placed elsewhere. A 502/503/504 from a hop in front of the node, or no - * answer at all, leaves the count as it is and the question to the next - * lookup — the node that may have built it answers from inside the + * request for the sandbox skips the round of questions. A 2xx without a + * readable id is logged — the sandbox exists on the node and the next + * lookup finds it there, never silently. + * + * The placement count moves only for a placement (`placed`): a 2xx id + * also goes into the node's placedIds, so a destroy inside the same + * interval takes the placement off the count again; a 4xx or the node's + * own 500 means the node built nothing, and the placement comes off the + * count at once (it would have held a slot for a whole interval + * otherwise), so the next attempt may be placed elsewhere. A wake — a + * name found on the node — moves nothing either way: its sandbox is in + * the reading already, and paying back a placement that never was would + * open the gate one sandbox wider than the reading allows (found by + * review, 2026-09-14). A 502/503/504 from a hop in front of the node, or + * no answer at all, leaves the count as it is and the question to the + * next lookup — the node that may have built it answers from inside the * name's slot. */ const HOP_STATUSES = new Set([502, 503, 504]); @@ -96,7 +109,7 @@ export async function forwardCreate( cache: NameCache, request: FastifyRequest, res: http.ServerResponse, - { target, token, name, body, face }: CreateOptions, + { target, token, name, body, face, placed }: CreateOptions, ): Promise { const answer = await forwardCapture(request.raw, res, { target: { endpoint: target.endpoint, token }, @@ -108,7 +121,7 @@ export async function forwardCreate( const id = face.idOf(parseJson(answer.body)); if (id !== null) { cache.put({ id, name, nodeId: target.id }); - target.placedIds.add(id); + if (placed) target.placedIds.add(id); } else { request.log.warn( { node: target.id, name, status: answer.status }, @@ -117,7 +130,9 @@ export async function forwardCreate( } } else if (!HOP_STATUSES.has(answer.status)) { // The node itself answered no: nothing was built there. - target.placedSinceCheckIn = Math.max(0, target.placedSinceCheckIn - 1); + if (placed) { + target.placedSinceCheckIn = Math.max(0, target.placedSinceCheckIn - 1); + } if (answer.status >= 500) { request.log.warn( { node: target.id, name, status: answer.status }, diff --git a/packages/gateway/src/routes/e2b.ts b/packages/gateway/src/routes/e2b.ts index ee5d0e51..a80443e9 100644 --- a/packages/gateway/src/routes/e2b.ts +++ b/packages/gateway/src/routes/e2b.ts @@ -1,5 +1,6 @@ import { tokensEqual } from '@dormice/server/auth'; import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import { sandboxNameSchema } from '@dormice/shared'; import type { FastifyError, FastifyReply, FastifyRequest } from 'fastify'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { relay } from '../errors'; @@ -66,7 +67,6 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( } }); - const dialect = (message: string) => ({ code: 0, message }); const send = (reply: FastifyReply, code: number, message: string) => reply.code(code).send({ code, message }); @@ -87,12 +87,14 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( const RETRY_FINDS_IT = ' — retry: if the sandbox was built, the node answers for it'; + /** `placed`: pick() chose the node for a new name (counted there) — false for a name found on it, a wake. */ function create( request: FastifyRequest, reply: FastifyReply, target: NodeState, name: string | null, body: Buffer | undefined, + placed: boolean, ) { return forwarded(request, reply, RETRY_FINDS_IT, async () => { const answer = await forwardCreate(finder.cache, request, reply.raw, { @@ -101,12 +103,14 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( name, body, face: E2B_CREATE, + placed, }); if (answer !== null) replay(reply.raw, answer); }); } - function placed(reply: FastifyReply): NodeState | null { + /** Places a new sandbox, or sends the 503 and answers null (null too for a client that already left). */ + function placeOrRefuse(reply: FastifyReply): NodeState | null { if (clientGone(reply)) return null; const placement = place(fleet, knobs, new Date()); if (placement.node === null) { @@ -122,24 +126,38 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( const parsed = parseJson(body) as | { metadata?: { name?: unknown } } | undefined; - const name = parsed?.metadata?.name; - if (typeof name === 'string' && name.length > 0) { + if (parsed?.metadata?.name !== undefined) { + // Judged by the wire's own rule before any node is asked (native.ts + // nameOf has why); the node's stricter E2B pattern still answers its + // own 400, relayed as it came. + const judged = sandboxNameSchema.safeParse(parsed.metadata.name); + if (!judged.success) { + return send( + reply, + 400, + `invalid metadata.name: ${judged.error.issues[0]?.message ?? 'refused by the wire'}`, + ); + } + const name = judged.data; return locks.run(name, async () => { - const judged = verdict(await finder.byName(name), `sandbox "${name}"`); - if (judged.kind === 'refuse') { - return refuse(reply, judged, (message) => ({ - ...dialect(message), - code: judged.status, + const found = verdict(await finder.byName(name), `sandbox "${name}"`); + if (found.kind === 'refuse') { + return refuse(reply, found, (message) => ({ + code: found.status, + message, })); } - const target = judged.kind === 'node' ? judged.node : placed(reply); + if (found.kind === 'node') { + return create(request, reply, found.node, name, body, false); + } + const target = placeOrRefuse(reply); if (target === null) return reply; - return create(request, reply, target, name, body); + return create(request, reply, target, name, body, true); }); } - const target = placed(reply); + const target = placeOrRefuse(reply); if (target === null) return reply; - return create(request, reply, target, null, body); + return create(request, reply, target, null, body, true); }); app.get('/v2/sandboxes', async (_request, reply) => @@ -155,8 +173,8 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( const judged = verdict(await finder.byId(id), `sandbox "${id}"`); if (judged.kind === 'refuse') { return refuse(reply, judged, (message) => ({ - ...dialect(message), code: judged.status, + message, })); } if (judged.kind === 'none') { diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index fd8d3fed..17cfafa3 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -1,5 +1,8 @@ import type { KeyedQueue } from '@dormice/server/keyed-queue'; -import { WRITE_FILES_BODY_LIMIT_BYTES } from '@dormice/shared'; +import { + sandboxNameSchema, + WRITE_FILES_BODY_LIMIT_BYTES, +} from '@dormice/shared'; import type { FastifyReply, FastifyRequest } from 'fastify'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { relay } from '../errors'; @@ -113,12 +116,11 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( bodyLimit === undefined ? {} : { bodyLimit }, async (request, reply) => { const body = request.body as Buffer | undefined; - const name = nameOf(body); - if (name === null) { - return reply - .code(400) - .send({ message: 'name is required and must be a string' }); + const named = nameOf(body); + if ('refusal' in named) { + return reply.code(400).send({ message: named.refusal }); } + const { name } = named; // Only the two verbs that create or remove take the name's slot — // the daemon's own discipline (its other verbs run unserialized // too). The slot is what keeps twenty simultaneous acquires of a @@ -167,6 +169,9 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( const judged = verdict(await finder.byName(name), `sandbox "${name}"`); if (judged.kind === 'refuse') return refuse(reply, judged); let target = judged.kind === 'node' ? judged.node : null; + // Found nowhere: a placement, counted against the node it lands on. + // Found somewhere: a wake, already in that node's reading. + const placed = target === null; if (target === null) { if (clientGone(reply)) return; const placement = place(fleet, knobs, new Date()); @@ -184,6 +189,7 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( name, body, face: NATIVE_CREATE, + placed, }); if (answer !== null) replay(reply.raw, answer); }); @@ -249,9 +255,25 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( } }; -/** The one field the gateway reads from a native body. */ -export function nameOf(body: Buffer | undefined): string | null { +/** + * The one field the gateway reads from a native body, judged by the wire's + * own rule (shared sandboxNameSchema — what lookupSandbox validates + * against) so no node is ever asked a question it would refuse: a node's + * 400 to the lookup reads as silence, and the caller would get a 503 with + * Retry-After for a name that can never be valid (found by review, + * 2026-09-14). + */ +export function nameOf( + body: Buffer | undefined, +): { name: string } | { refusal: string } { const parsed = parseJson(body) as { name?: unknown } | undefined; - const name = parsed?.name; - return typeof name === 'string' && name.length > 0 ? name : null; + if (parsed?.name === undefined) { + return { refusal: 'name is required and must be a string' }; + } + const judged = sandboxNameSchema.safeParse(parsed.name); + return judged.success + ? { name: judged.data } + : { + refusal: `invalid name: ${judged.error.issues[0]?.message ?? 'refused by the wire'}`, + }; } diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index 83ddd779..1240e56a 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -34,13 +34,40 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( }, }, async (request) => { - const { node, joined } = fleet.checkIn(request.body); + const { node, joined, movedFrom } = fleet.checkIn(request.body); if (joined) { request.log.info( { nodeId: node.id, endpoint: node.endpoint }, 'a node checked in for the first time and joined the fleet', ); } + // Two misconfigurations show only here, so they are said here. A + // node whose endpoint moves at every check-in is two machines + // sharing one DORMICE_NODE_ID (the daemon's default is `node-1`). + // Two nodes reporting one endpoint is a DORMICE_NODE_ENDPOINT that + // names the wrong machine — the daemon refuses the loopback default + // when its gateway is remote, but a hand-written value still can. + // Either way the symptom downstream is a 409 on every name (both + // "nodes" answer the lookup) or sandboxes on the wrong machine. + if (movedFrom !== null) { + request.log.warn( + { nodeId: node.id, from: movedFrom, to: node.endpoint }, + 'a node checked in from a new endpoint; the gateway forwards there from now on', + ); + } + const twins = fleet + .all() + .filter((n) => n.id !== node.id && n.endpoint === node.endpoint); + if (twins.length > 0) { + request.log.warn( + { + nodeId: node.id, + endpoint: node.endpoint, + alsoReportedBy: twins.map((n) => n.id), + }, + 'two nodes report the same endpoint — check DORMICE_NODE_ID and DORMICE_NODE_ENDPOINT on both; their sandboxes will be found twice (409) or land on the wrong machine', + ); + } return {}; }, ); From 6701d5983fe51e2b821f68656409407b773f4df0 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 04:21:13 +0800 Subject: [PATCH 11/89] A node whose gateway is on another machine must set DORMICE_NODE_ENDPOINT: the loopback default names the gateway's own machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Left unset, the node reports http://127.0.0.1: as where it can be reached, and a remote gateway dials its own daemon (or nothing) for this node: every sandbox placed "here" lands there, and every lookup then finds it twice — a 409 on every name, from one forgotten variable. Refused at boot, naming the variable and the address to write, whenever DORMICE_GATEWAY_ENDPOINT is not loopback. An explicit value is the operator's word and is taken as written. The loopback test tolerates an unparsable URL: zod runs the object-level refinements even when a field failed, and the field's own error ("must be a full http(s) URL") must be the one that shows, not an Invalid URL thrown from inside the rule. --- packages/server/src/config.test.ts | 28 +++++++++++++++++++ packages/server/src/config.ts | 44 +++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index 083f071c..963e89cb 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -183,4 +183,32 @@ describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { loadConfig({ ...TOKEN, DORMICE_CHECK_IN_INTERVAL_SECONDS: '0' }), ).toThrow(); }); + + it('a gateway on another machine requires the node endpoint, naming why; a loopback gateway does not', () => { + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + }), + ).toThrow( + /DORMICE_NODE_ENDPOINT is required when DORMICE_GATEWAY_ENDPOINT is not loopback/, + ); + expect( + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80', + }).DORMICE_NODE_ENDPOINT, + ).toBe('http://10.0.0.7:80'); + for (const local of [ + 'http://127.0.0.1:3677', + 'http://localhost:3677', + 'http://[::1]:3677', + ]) { + expect( + loadConfig({ ...TOKEN, DORMICE_GATEWAY_ENDPOINT: local }) + .DORMICE_NODE_ENDPOINT, + ).toBeUndefined(); + } + }); }); diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index acd03501..c31d7289 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -194,7 +194,11 @@ const envSchema = z.object({ * node share a machine (the single-machine install is a fleet of one). * On a machine of its own the daemon still binds loopback (the red * line), so this names the front the gateway may dial — the node's - * Caddy on the intranet interface, e.g. http://10.0.0.7:80. + * Caddy on the intranet interface, e.g. http://10.0.0.7:80 — and is + * required there (checkedSchema below): left at the default, the node + * would tell a remote gateway to dial 127.0.0.1, an address on the + * gateway's own machine, and every sandbox placed "here" would land on + * whatever daemon lives there and be found twice. */ DORMICE_NODE_ENDPOINT: z .url({ @@ -216,6 +220,27 @@ const envSchema = z.object({ .default(15), }); +/** + * Loopback as an operator writes it: 127.0.0.0/8, ::1, localhost. Null + * when the URL does not parse — its own field has already said so, and an + * object-level rule must not throw over it (zod runs the refinements even + * when a field failed). + */ +function isLoopbackUrl(url: string): boolean | null { + let host: string; + try { + host = new URL(url).hostname; + } catch { + return null; + } + return ( + host === 'localhost' || + host === '::1' || + host === '[::1]' || + host.startsWith('127.') + ); +} + const checkedSchema = envSchema .refine( (cfg) => cfg.DORMICE_EXECUTOR !== 'docker' || !!cfg.DORMICE_BASE_IMAGE, @@ -248,6 +273,23 @@ const checkedSchema = envSchema path: ['DORMICE_DATA_DIR'], }, ) + // A node whose gateway is on another machine must say where it is. The + // check-in's default endpoint is this daemon's loopback address, which + // on the gateway's machine names the gateway's own daemon (or nothing): + // a remote gateway dialing 127.0.0.1 for this node would place sandboxes + // on the wrong machine and then find every one of them twice (409). An + // explicit value is the operator's word and is taken as written. + .refine( + (cfg) => + cfg.DORMICE_GATEWAY_ENDPOINT === undefined || + isLoopbackUrl(cfg.DORMICE_GATEWAY_ENDPOINT) !== false || + cfg.DORMICE_NODE_ENDPOINT !== undefined, + { + message: + "DORMICE_NODE_ENDPOINT is required when DORMICE_GATEWAY_ENDPOINT is not loopback: the gateway is on another machine, and without it this node would report http://127.0.0.1: — an address on the gateway's machine, not this one. Name this node's address on the network, e.g. http://10.0.0.7:80", + path: ['DORMICE_NODE_ENDPOINT'], + }, + ) // All-or-none: a half-configured store would make the archiver's // existence ambiguous, and ambiguity here decides real policy defaults. .superRefine((cfg, ctx) => { From 042b26541008ad280046665e67ec77db5b8e38bc Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 04:21:13 +0800 Subject: [PATCH 12/89] lookupSandbox takes the name's slot in one step: locks.run already answers at once when the slot is free MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tryRun-then-run did the same thing in three lines. run() executes the task immediately when nothing holds the key and queues it behind the holder otherwise — exactly "a plain no when the slot is free, wait for the acquire when it is not". SKIPPED is no longer imported here. --- packages/server/src/routes/sandboxes.ts | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/packages/server/src/routes/sandboxes.ts b/packages/server/src/routes/sandboxes.ts index 739f0e9c..7488481d 100644 --- a/packages/server/src/routes/sandboxes.ts +++ b/packages/server/src/routes/sandboxes.ts @@ -84,7 +84,7 @@ import { NotAFileError, } from '../executor/executor'; import { httpError } from '../http-error'; -import { type KeyedQueue, SKIPPED } from '../keyed-queue'; +import type { KeyedQueue } from '../keyed-queue'; import { destroySandbox, rebuildSandbox, wakeSandbox } from '../lifecycle'; import { ArchiveDisabledError, resolvePolicy } from '../policy'; import { resolveSpec } from '../spec'; @@ -1119,17 +1119,17 @@ export const sandboxRoutes: FastifyPluginAsyncZod< // The gateway's one question on its own account: does this node hold // the sandbox? Read-only — never wakes, never touches the idle clock — - // and truthful about a create in flight, in three steps. A row that + // and truthful about a create in flight, in two steps. A row that // exists answers at once, whatever its state: a restoring sandbox has a // row, and waiting for its slot would hold the answer for the whole // restore, long past the gateway's two-second patience — the gateway - // would read a live sandbox as a node that did not answer. No row while - // the name's slot is busy means an acquire may be writing the row right - // now (create first, row second, both under the slot), so the answer - // waits its turn behind it and looks again. No row and a free slot is a - // plain no. By id there is no slot to wait on (slots are keyed by name), - // and none is needed: nobody can ask about an id before the create that - // minted it has answered. + // would read a live sandbox as a node that did not answer. No row means + // an acquire may be writing it right now (create first, row second, + // both under the name's slot), so the answer takes the slot itself and + // looks again: at once when the slot is free (a plain no), behind the + // acquire when it is not. By id there is no slot to wait on (slots are + // keyed by name), and none is needed: nobody can ask about an id before + // the create that minted it has answered. app.post( '/lookupSandbox', { @@ -1151,8 +1151,6 @@ export const sandboxRoutes: FastifyPluginAsyncZod< : { found: false as const }; const now = look(); if (now !== undefined || !('name' in query)) return answer(now); - const unheld = await locks.tryRun(query.name, async () => look()); - if (unheld !== SKIPPED) return answer(unheld); return answer(await locks.run(query.name, async () => look())); }, ); From 1c4f1a250f27f18f0f95f525186c49042b7b3522 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 04:49:41 +0800 Subject: [PATCH 13/89] A node endpoint is an origin, and a redirected check-in is reported instead of followed The gateway hands a node's endpoint to undici as the request's origin when it forwards, and undici refuses an origin that carries a path (UND_ERR_INVALID_ARG, measured). Its own lookup joins `/lookupSandbox` as a string and works with a path. So a node reporting http://10.0.0.7:80/dormice was found by every lookup and reached by no forward. The shared wire now has `endpointSchema`: a trailing slash is dropped (the two ends must agree byte for byte, and `//lookupSandbox` is a 404 the gateway reads as silence), a path or query is a 400 at /checkIn, and DORMICE_NODE_ENDPOINT refuses the same at the daemon's boot, where the operator is looking. The node's check-in used fetch's default redirect handling. A front that answers plain http with a 308 to https (a Caddy binding the gateway's domain) would be followed across origins, and the Fetch standard drops Authorization on the way, so the gateway answered 401 and the log said "wrong token" where the address was wrong. The check-in no longer follows: it reports the 3xx and its Location, and names the variable to fix. Found by the second review pass of 2026-09-14; each fix reverse-proved (the new test goes red with the fix removed). --- .changeset/gateway-check-in-and-lookup.md | 2 +- packages/gateway/src/app.test.ts | 15 +++++++++ packages/server/src/check-in.test.ts | 35 +++++++++++++++++++-- packages/server/src/check-in.ts | 13 +++++++- packages/server/src/config.test.ts | 22 +++++++++++++ packages/server/src/config.ts | 13 +++++++- packages/shared/src/gateway.ts | 38 +++++++++++++++++++++-- 7 files changed, 130 insertions(+), 8 deletions(-) diff --git a/.changeset/gateway-check-in-and-lookup.md b/.changeset/gateway-check-in-and-lookup.md index 35d961a0..7561e255 100644 --- a/.changeset/gateway-check-in-and-lookup.md +++ b/.changeset/gateway-check-in-and-lookup.md @@ -2,4 +2,4 @@ "@dormice/shared": minor --- -Wire schemas for a fleet behind a gateway: `lookupSandbox` (a node answers "do you hold this sandbox?" by name or id), the node check-in (`checkInRequestSchema`, readings, build), and the gateway's `listNodes` / `removeNode`. The host-metrics schema is split into named parts (`hostReadingSchema`, `dataDiskSchema`, `sandboxStateCountsSchema`) that `hostMetricsResponseSchema` still composes unchanged. +Wire schemas for a fleet behind a gateway: `lookupSandbox` (a node answers "do you hold this sandbox?" by name or id), the node check-in (`checkInRequestSchema`, readings, build; its `endpoint` is an origin — `endpointSchema` drops a trailing slash and refuses a path), and the gateway's `listNodes` / `removeNode`. The host-metrics schema is split into named parts (`hostReadingSchema`, `dataDiskSchema`, `sandboxStateCountsSchema`) that `hostMetricsResponseSchema` still composes unchanged. diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index f4490e9c..573eda1f 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -383,6 +383,21 @@ describe('check-in and the node verbs', () => { const bad = await rpc(h, '/checkIn', { nodeId: 'z' }); expect(bad.status).toBe(400); expect(h.fleet.get('z')).toBeUndefined(); + // An endpoint the gateway could not use is refused at the wire (shared + // endpointSchema has the measurement): undici takes the endpoint as + // the request's origin and throws on a path — a node every lookup + // would find and no forward could reach. A trailing slash is merely + // dropped; the two ends must agree byte for byte. + const pathy = await rpc( + h, + '/checkIn', + checkInOf('p', 'http://10.0.0.9:80/dormice'), + ); + expect(pathy.status).toBe(400); + expect(message(pathy)).toMatch(/an endpoint is an origin/); + expect(h.fleet.get('p')).toBeUndefined(); + await rpc(h, '/checkIn', checkInOf('s', 'http://10.0.0.9:80/')); + expect(h.fleet.get('s')?.endpoint).toBe('http://10.0.0.9:80'); }); it('removeNode forgets the node and everything cached on it; a removed node that checks in again re-joins', async () => { diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index 6cbcc69a..f4636b82 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -21,7 +21,13 @@ afterEach(async () => { }); /** A gateway-shaped listener: records every check-in, answers what the test says. */ -async function gateway(answer: () => { status: number; body: string }) { +async function gateway( + answer: () => { + status: number; + body: string; + headers?: Record; + }, +) { const seen: Array<{ headers: http.IncomingHttpHeaders; body: unknown }> = []; const server = http.createServer((req, res) => { let text = ''; @@ -31,7 +37,10 @@ async function gateway(answer: () => { status: number; body: string }) { req.on('end', () => { seen.push({ headers: req.headers, body: JSON.parse(text) }); const a = answer(); - res.writeHead(a.status, { 'content-type': 'application/json' }); + res.writeHead(a.status, { + 'content-type': 'application/json', + ...a.headers, + }); res.end(a.body); }); }); @@ -46,12 +55,17 @@ async function gateway(answer: () => { status: number; body: string }) { function logSpy() { const infos: string[] = []; const warns: string[] = []; + const details: unknown[] = []; return { infos, warns, + details, log: { info: (msg: string) => infos.push(msg), - warn: (_obj: unknown, msg: string) => warns.push(msg), + warn: (obj: unknown, msg: string) => { + warns.push(msg); + details.push(obj); + }, }, }; } @@ -154,6 +168,21 @@ describe('CheckIn', () => { expect(warns).toHaveLength(2); }); + it('a front that redirects is reported as a wrong address, not followed with the token stripped', async () => { + const gw = await gateway(() => ({ + status: 308, + body: '', + headers: { location: 'https://gateway.example/checkIn' }, + })); + const { log, warns, details } = logSpy(); + await new CheckIn(options(gw.endpoint, log)).once(); + expect(gw.seen).toHaveLength(1); + expect(warns).toEqual([expect.stringMatching(/check-in failed/)]); + expect((details[0] as { error: string }).error).toMatch( + /gateway answered 308 redirecting to https:\/\/gateway\.example\/checkIn — DORMICE_GATEWAY_ENDPOINT must be the gateway's own address/, + ); + }); + it('a gateway that is not there is a logged failure, never a throw', async () => { const { log, warns } = logSpy(); const checkIn = new CheckIn(options('http://127.0.0.1:9', log)); diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index 1fda3f3e..d5acc4da 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -101,11 +101,22 @@ export class CheckIn { }, body: JSON.stringify(body), signal: AbortSignal.timeout(CHECK_IN_TIMEOUT_MS), + // A front that redirects (a Caddy binding the gateway's domain + // answers plain http with a 308 to https) is reported as what it + // is. Followed, the redirect would cross origins and fetch would + // drop the Authorization header on the way (the Fetch standard's + // rule), so the gateway would answer 401 — and the operator would + // read a wrong token where there is a wrong address (found by + // review, 2026-09-14). + redirect: 'manual', }); if (res.status !== 200) { const text = await res.text(); + const location = res.headers.get('location'); throw new Error( - `gateway answered ${res.status}: ${text.slice(0, 200)}`, + location === null + ? `gateway answered ${res.status}: ${text.slice(0, 200)}` + : `gateway answered ${res.status} redirecting to ${location} — DORMICE_GATEWAY_ENDPOINT must be the gateway's own address, not a front that redirects`, ); } checkInResponseSchema.parse(await res.json()); diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index 963e89cb..d07cebfa 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -184,6 +184,28 @@ describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { ).toThrow(); }); + it('a node endpoint with a path or a query is refused at boot, naming why; a trailing slash is still just dropped', () => { + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80/dormice', + }), + ).toThrow( + /DORMICE_NODE_ENDPOINT must name the node's front without a path/, + ); + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80/?x=1', + }), + ).toThrow(/without a path/); + expect( + loadConfig({ ...TOKEN, DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80/' }) + .DORMICE_NODE_ENDPOINT, + ).toBe('http://10.0.0.7:80'); + }); + it('a gateway on another machine requires the node endpoint, naming why; a loopback gateway does not', () => { expect(() => loadConfig({ diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index c31d7289..ddfac547 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -1,5 +1,9 @@ import { isAbsolute } from 'node:path'; -import { bareHostnameRegex, PIDS_LIMIT_MIN } from '@dormice/shared'; +import { + bareHostnameRegex, + isOriginUrl, + PIDS_LIMIT_MIN, +} from '@dormice/shared'; import { z } from 'zod'; import type { S3Settings } from './archive/s3-store'; @@ -207,6 +211,13 @@ const envSchema = z.object({ 'DORMICE_NODE_ENDPOINT must be a full http(s) URL, e.g. http://10.0.0.7:80', }) .transform((url) => url.replace(/\/+$/, '')) + // An origin, nothing more (shared endpointSchema has the measurement): + // the gateway would answer this node's every check-in with a 400 + // otherwise — refused here, at boot, where the operator is looking. + .refine(isOriginUrl, { + error: + "DORMICE_NODE_ENDPOINT must name the node's front without a path — scheme, host and port only, e.g. http://10.0.0.7:80 (the gateway dials /)", + }) .optional(), /** * How often the node checks in with its gateway. The gateway reads two diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts index 11eb6cf8..12d2c6b7 100644 --- a/packages/shared/src/gateway.ts +++ b/packages/shared/src/gateway.ts @@ -26,6 +26,40 @@ export const buildInfoSchema = z.object({ export type BuildInfo = z.infer; +/** + * Whether a URL is an origin and nothing more: scheme, host and port — no + * path, query or fragment. False for a string that is not a URL at all. + */ +export function isOriginUrl(url: string): boolean { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return false; + } + return parsed.pathname === '/' && parsed.search === '' && parsed.hash === ''; +} + +/** + * Where a process can be dialled — an origin. The gateway joins + * `/` for its own lookup and hands the endpoint to undici + * as the request's origin when it forwards, and undici refuses an origin + * that carries a path (UND_ERR_INVALID_ARG, measured 2026-09-14): a node + * reporting `http://10.0.0.7:80/dormice` would be found by every lookup + * and reached by no forward. So a path is refused at the wire. A trailing + * slash is dropped rather than refused — the same address to everyone but + * a string compare, and the two ends must agree byte for byte (the join + * above would otherwise ask for `//lookupSandbox`, a 404 the gateway reads + * as silence). + */ +export const endpointSchema = z + .url({ protocol: /^https?$/ }) + .transform((url) => url.replace(/\/+$/, '')) + .refine(isOriginUrl, { + error: + 'an endpoint is an origin — scheme, host and port only, no path or query, e.g. http://10.0.0.7:80', + }); + /** * What a node reports about itself at every check-in — everything * placement decides on: the machine's CPU, memory and data disk, and the @@ -57,8 +91,8 @@ export type NodeReading = z.infer; export const checkInRequestSchema = z.object({ /** DORMICE_NODE_ID — the node's name in every sandbox's `nodeId`. */ nodeId: z.string().min(1), - /** Where the gateway forwards to: the node's intranet front (DORMICE_NODE_ENDPOINT). */ - endpoint: z.url({ protocol: /^https?$/ }), + /** Where the gateway forwards to: the node's intranet front (DORMICE_NODE_ENDPOINT) — an origin, endpointSchema has why. */ + endpoint: endpointSchema, /** How often this node checks in — the gateway's yardstick for "missed two in a row". */ intervalSeconds: z.number().int().positive(), build: buildInfoSchema.nullable(), From 918073765ace14888562b5ada4a91ad183041bc0 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 04:49:41 +0800 Subject: [PATCH 14/89] forwardStream sends nothing for a client that already left MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit forwardCapture opened with `if (res.destroyed) return null`; forwardStream did not. A caller of a streamed verb (execCommand) that hangs up during the lookup round — up to two seconds when a node is slow — has a destroyed response whose 'close' has already fired, so the abort wired to 'close' would never come, and the node would run the command to its end for nobody. Same first line now, and the doc comment that already claimed the two followed one rule is true. Found by the second review pass of 2026-09-14; reverse-proved. --- packages/gateway/src/forward.test.ts | 32 ++++++++++++++++++++++++++++ packages/gateway/src/forward.ts | 10 +++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/packages/gateway/src/forward.test.ts b/packages/gateway/src/forward.test.ts index 48457edd..58ef946d 100644 --- a/packages/gateway/src/forward.test.ts +++ b/packages/gateway/src/forward.test.ts @@ -155,6 +155,38 @@ describe('forwardStream', () => { await closed; }); + it('a client already gone when the forward begins is not sent to the node at all, and answers null', async () => { + let requests = 0; + const node = http.createServer((_req, res) => { + requests += 1; + res.end('{}'); + }); + const endpoint = await listen(node); + // The shape forwardNamed produces: the lookup round took a while and + // the caller hung up meanwhile — the response is destroyed before the + // forward starts, so its 'close' has already fired and no abort would + // ever follow. + const gone = new http.IncomingMessage(new net.Socket()); + const res = new http.ServerResponse(gone); + res.destroy(); + expect( + await forwardStream( + Object.assign(gone, { + url: '/execCommand', + method: 'POST', + headers: {}, + }), + res, + { + target: { endpoint, token: TOKEN }, + credential: 'bearer', + body: Buffer.from('{"name":"x","command":"sleep 3600"}'), + }, + ), + ).toBeNull(); + expect(requests).toBe(0); + }); + it('forwards the request verbatim — path, query, body — with the credential swapped and the Host renamed to the node unless the face keeps it', async () => { const seen: Array<{ url?: string; diff --git a/packages/gateway/src/forward.ts b/packages/gateway/src/forward.ts index c025cfda..5fdc09a4 100644 --- a/packages/gateway/src/forward.ts +++ b/packages/gateway/src/forward.ts @@ -175,8 +175,9 @@ async function dispatch( * Throws UnreachableError only before any byte of the answer was written; * once the head is out, a failure mid-stream can only be a cut connection * — there is no honest status left to send. Resolves null when the client - * left before the node answered: the node-side request is aborted and - * nothing is rendered (the response is gone). Otherwise an abandoned exec + * left before the node answered: the node-side request is aborted — or, + * for a client already gone, never sent — and nothing is rendered (the + * response is gone). Otherwise an abandoned exec * against a slow node would hold a gateway→node socket until the node * answered — on a hung node, until its TCP died. forwardCapture follows * the same rule. @@ -186,6 +187,11 @@ export async function forwardStream( res: http.ServerResponse, options: ForwardOptions, ): Promise { + // The client left while its sandbox was being found (a lookup round is + // up to two seconds): 'close' has fired already, the abort below would + // never come, and the node would run an exec to its end for nobody — + // forwardCapture's first line, missing here (found by review, 2026-09-14). + if (res.destroyed) return null; const gone = new AbortController(); const onClose = () => gone.abort(); res.once('close', onClose); From a9b735bad3fe0f0627c34294346b623b44796106 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 05:14:43 +0800 Subject: [PATCH 15/89] A creator confirms its cached node before waking there; removeNode and a second reporter under one id are refused while the node is live MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three ways the fleet could end up with one name on two nodes, or a placement past every gate, found by the second review pass of 2026-09-14. The cache is a hint for every verb but the two creators: a stale entry costs a reader one 404 that evicts it, but acquireSandbox and the E2B named create are create-or-wake on the node, so a cache hit forwarded as a wake to a node that has since deleted the row (an E2B deadline kill is the scanner's routine, five minutes by default) rebuilt the sandbox there — past placement's four gates, uncounted in placedSinceCheckIn, every re-create of an expired name pinned to its first node. The creators now confirm a cache hit with that one node by id (Finder.byName confirm): found is a wake, absent evicts and asks the fleet, silent is the same 503 a silent stranger earns. One RTT per warm acquire. removeNode on a node still checking in deleted its row; a name of its acquired before its next check-in was placed elsewhere, then the node re-added itself and the name was on two nodes — a 409 an operator clears by hand. It is now a 409 up front, naming what to do: stop the daemon, wait two of its intervals (what "down" means), then remove. A check-in that changed a node's endpoint inside the previous reporter's own interval was written through. Two daemons sharing one DORMICE_NODE_ID (the default is node-1) flipped the endpoint at every check-in; a lookup asked whichever was current, a name on the other read as new and was built again, on two nodes, with no 409 ever. Such a check-in is now refused (409) naming both addresses; the first reporter keeps the id; a node that really moved is taken an interval later. The 503 for a silent node now says the node may be busy building that very name (its lookup waits for the name slot, and a cold create outlasts the gateway's two seconds), and the shared lookup comment no longer promises that a retry finds a sandbox still being built. --- packages/gateway/src/app.test.ts | 86 +++++++++++++++++++++++++- packages/gateway/src/find.ts | 50 ++++++++++++--- packages/gateway/src/fleet.test.ts | 58 ++++++++++++----- packages/gateway/src/fleet.ts | 36 ++++++++--- packages/gateway/src/placement.test.ts | 4 +- packages/gateway/src/routes/e2b.ts | 7 ++- packages/gateway/src/routes/native.ts | 10 ++- packages/gateway/src/routes/nodes.ts | 36 ++++++++++- packages/gateway/src/routes/verdict.ts | 2 +- packages/shared/src/gateway.ts | 5 +- packages/shared/src/lookup.ts | 7 ++- 11 files changed, 260 insertions(+), 41 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 573eda1f..080bee39 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -375,6 +375,20 @@ describe('check-in and the node verbs', () => { ).toBe(8); const moved = new FakeNode('b'); await moved.start(); + // Inside the first reporter's interval a different address is a second + // daemon under one DORMICE_NODE_ID, refused with why; the first keeps + // the id. An interval later the same report is a move. + const twin = await rpc( + h, + '/checkIn', + checkInOf('b', moved.endpoint, { active: 3 }), + ); + expect(twin.status).toBe(409); + expect(message(twin)).toMatch(/two daemons share one DORMICE_NODE_ID/); + expect(h.fleet.get('b')?.endpoint).toBe(h.nodes[0]?.endpoint); + const first = h.fleet.get('b'); + if (!first) throw new Error('node lost'); + first.lastCheckInAt = new Date(Date.now() - 16_000); await h.checkIn(moved, { active: 3 }); expect(h.fleet.get('b')?.endpoint).toBe(moved.endpoint); expect(h.fleet.get('b')?.reading?.sandboxes.byState.active).toBe(3); @@ -404,6 +418,17 @@ describe('check-in and the node verbs', () => { const h = await gateway(['b', 'c']); const created = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'x' })); expect(h.cache.getByName('x')?.nodeId).toBe(created.nodeId); + // Still checking in: refused, with what to do instead — its names + // would be placed elsewhere before its next check-in and come back on + // two nodes. + const live = await rpc(h, '/removeNode', { id: created.nodeId }); + expect(live.status).toBe(409); + expect(message(live)).toMatch(/checked in \ds ago — it is running/); + expect(h.fleet.get(created.nodeId)).toBeDefined(); + // Silent for two of its intervals: down, and removable. + const silent = h.fleet.get(created.nodeId); + if (!silent) throw new Error('node lost'); + silent.lastCheckInAt = new Date(Date.now() - 31_000); expect((await rpc(h, '/removeNode', { id: created.nodeId })).body).toEqual({ removed: true, }); @@ -424,7 +449,7 @@ describe('check-in and the node verbs', () => { }); describe('acquire: placing and finding', () => { - it('a new name lands on the emptiest node by active density, under the fleet token, and is cached: the second acquire asks nobody', async () => { + it('a new name lands on the emptiest node by active density, under the fleet token, and is cached: the second acquire asks only its node, once', async () => { const h = await gateway(['a', 'b']); const [a, b] = h.nodes as [FakeNode, FakeNode]; await h.checkIn(a, { active: 10, cores: 8 }); @@ -445,8 +470,58 @@ describe('acquire: placing and finding', () => { const again = await rpc(h, '/acquireSandbox', { name: 'alice' }); expect(sandboxOf(again).id).toBe(sandboxOf(first).id); expect((again.body as { created: boolean }).created).toBe(false); + // A creator confirms its cache hit with that one node (by id) before + // trusting it; the other node hears nothing. expect(a.lookups()).toBe(1); - expect(b.lookups()).toBe(1); + expect(b.lookups()).toBe(2); + expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); + }); + + it('a name whose sandbox its node has since removed on its own is a placement again: the cached node is asked first, the gate judges afresh, the count moves — on either face', async () => { + const h = await gateway(['a', 'b'], { + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', + }); + const [a, b] = h.nodes as [FakeNode, FakeNode]; + await h.checkIn(a, { active: 0 }); + await h.checkIn(b, { active: 1 }); + const born = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'ttl' })); + expect(born.nodeId).toBe('a'); + expect(h.cache.getByName('ttl')?.nodeId).toBe('a'); + // The node reaps it on its own (an E2B deadline kill is the scanner's + // routine): the gateway hears nothing, the entry is stale. Meanwhile a + // fills up and b empties — the gate must judge afresh, not wake. + a.sandboxes.delete('ttl'); + await h.checkIn(a, { active: 2 }); + await h.checkIn(b, { active: 0 }); + const again = await rpc(h, '/acquireSandbox', { name: 'ttl' }); + expect(again.status).toBe(200); + expect(sandboxOf(again).nodeId).toBe('b'); + expect((again.body as { created: boolean }).created).toBe(true); + // Nothing was rebuilt on the full node; the placement is counted where + // it landed and the cache follows. + expect(a.creates).toBe(1); + expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); + expect(h.cache.getByName('ttl')?.nodeId).toBe('b'); + + // The E2B face takes the same slot and the same confirmation. + const create = (name: string) => + fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ metadata: { name } }), + }); + await h.checkIn(a, { active: 0 }); + await h.checkIn(b, { active: 2 }); + expect((await create('ttl-e2b')).status).toBe(201); + expect(h.cache.getByName('ttl-e2b')?.nodeId).toBe('a'); + a.sandboxes.delete('ttl-e2b'); + await h.checkIn(a, { active: 2 }); + await h.checkIn(b, { active: 0 }); + expect((await create('ttl-e2b')).status).toBe(201); + expect(h.cache.getByName('ttl-e2b')?.nodeId).toBe('b'); expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); }); @@ -578,6 +653,13 @@ describe('acquire: placing and finding', () => { expect(message(fresh)).toContain('node a did not answer'); expect(message(fresh)).toContain('cannot be treated as new'); expect(b.creates).toBe(0); + // Its socket is dead but its check-in was seconds ago: not yet "down", + // and removeNode says so. Two of its intervals of silence later it is. + const early = await rpc(h, '/removeNode', { id: 'a' }); + expect(early.status).toBe(409); + const silent = h.fleet.get('a'); + if (!silent) throw new Error('node lost'); + silent.lastCheckInAt = new Date(Date.now() - 31_000); expect((await rpc(h, '/removeNode', { id: 'a' })).body).toEqual({ removed: true, }); diff --git a/packages/gateway/src/find.ts b/packages/gateway/src/find.ts index d6f2ef99..8cd34a1f 100644 --- a/packages/gateway/src/find.ts +++ b/packages/gateway/src/find.ts @@ -41,26 +41,62 @@ export class Finder { private readonly log: FinderLog, ) {} - byName(name: string): Promise { - return this.find(this.cache.getByName(name), { name }); + /** + * `confirm`: a creator's cache hit is checked with the cached node before + * it is trusted. For every other verb a stale entry costs one misrouted + * request whose 404 evicts it (verify below); a create forwarded as a + * wake to a node that no longer holds the name *builds* — the daemon's + * acquire and E2B create are create-or-wake — on a node placement never + * judged and never counted, and the daemon deletes rows on its own (an + * E2B deadline kill is the scanner's routine, five minutes by default), + * so every re-create of an expired name would pin to its first node + * past every gate. One question to one node, by id so no name slot is + * waited on; "absent" evicts and the fleet is asked afresh (found by + * review, 2026-09-14). + */ + byName(name: string, options: { confirm?: boolean } = {}): Promise { + return this.find( + this.cache.getByName(name), + { name }, + options.confirm === true, + ); } byId(id: string): Promise { - return this.find(this.cache.getById(id), { id }); + return this.find(this.cache.getById(id), { id }, false); } private async find( cached: CacheEntry | undefined, query: LookupQuery, + confirm: boolean, ): Promise { if (cached !== undefined) { const node = this.fleet.get(cached.nodeId); - // A node the operator removed while the entry was cached: the - // entry is stale by definition, and the fleet is asked afresh. - if (node !== undefined) { + if (node === undefined) { + // A node the operator removed while the entry was cached: the + // entry is stale by definition, and the fleet is asked afresh. + this.cache.evict(cached); + } else if (!confirm) { return { kind: 'one', node, id: cached.id, name: cached.name }; + } else { + const answer = await this.ask(node, { id: cached.id }); + if (answer.kind === 'found') { + return { kind: 'one', node, id: answer.id, name: answer.name }; + } + if (answer.kind === 'silent') { + // The one node that may hold it did not answer: not new, not + // known to be there — the same refusal a silent stranger earns. + const silent = [{ nodeId: node.id, why: answer.why }]; + this.log.warn( + { query, silent }, + 'lookup: the cached node did not confirm; the name cannot be treated as new', + ); + return { kind: 'unsure', silent }; + } + // The node itself says the sandbox is gone: the entry was stale. + this.cache.evict(cached); } - this.cache.evict(cached); } const members = this.fleet.all(); const answers = await Promise.all( diff --git a/packages/gateway/src/fleet.test.ts b/packages/gateway/src/fleet.test.ts index 97e3ab8b..287e184e 100644 --- a/packages/gateway/src/fleet.test.ts +++ b/packages/gateway/src/fleet.test.ts @@ -1,12 +1,18 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; import { migrateDb, openDb } from './db/db'; -import { downReason, Fleet } from './fleet'; +import { type CheckInOutcome, downReason, Fleet } from './fleet'; import { checkInOf } from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); const NOW = new Date('2026-09-14T12:00:00.000Z'); +/** A check-in the test expects to be taken. */ +function taken(outcome: CheckInOutcome) { + if ('refused' in outcome) throw new Error(outcome.refused); + return outcome; +} + function db() { const handle = openDb(':memory:'); migrateDb(handle, MIGRATIONS); @@ -18,9 +24,11 @@ describe('Fleet', () => { const handle = db(); const fleet = new Fleet(handle); expect(fleet.all()).toEqual([]); - const { node, joined } = fleet.checkIn( - checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), - NOW, + const { node, joined } = taken( + fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), + NOW, + ), ); expect(joined).toBe(true); expect(node.addedAt).toBe(NOW.toISOString()); @@ -39,14 +47,27 @@ describe('Fleet', () => { ); }); - it('a later check-in is not a join; a changed endpoint is written through; the placement counter restarts', () => { + it('a later check-in is not a join; a changed endpoint inside the interval is refused as a second daemon, an interval later written through as a move; the placement counter restarts', () => { const handle = db(); const fleet = new Fleet(handle); - const first = fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW); + const first = taken( + fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW), + ); first.node.placedSinceCheckIn = 3; - const second = fleet.checkIn( - checkInOf('node-b', 'http://10.0.0.8:80', { active: 12 }), - new Date(NOW.getTime() + 15_000), + const twin = fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.8:80'), + new Date(NOW.getTime() + 7_000), + ); + expect('refused' in twin && twin.refused).toMatch( + /^node node-b checked in from http:\/\/10\.0\.0\.7:80 7s ago and now from http:\/\/10\.0\.0\.8:80 — two daemons share one DORMICE_NODE_ID/, + ); + expect(first.node.endpoint).toBe('http://10.0.0.7:80'); + expect(first.node.placedSinceCheckIn).toBe(3); + const second = taken( + fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.8:80', { active: 12 }), + new Date(NOW.getTime() + 15_000), + ), ); expect(second.joined).toBe(false); expect(first.movedFrom).toBeNull(); @@ -62,18 +83,22 @@ describe('Fleet', () => { it('downReason: fresh within two of its own intervals, down past them', () => { const fleet = new Fleet(db()); - const { node } = fleet.checkIn( - checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), - NOW, + const { node } = taken( + fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), + NOW, + ), ); expect(downReason(node, new Date(NOW.getTime() + 29_000))).toBeNull(); expect(downReason(node, new Date(NOW.getTime() + 31_000))).toBe( 'has not checked in for 31s', ); // A one-second node (the exam's) is judged by its own interval. - const quick = fleet.checkIn( - checkInOf('node-c', 'http://10.0.0.9:80', { intervalSeconds: 1 }), - NOW, + const quick = taken( + fleet.checkIn( + checkInOf('node-c', 'http://10.0.0.9:80', { intervalSeconds: 1 }), + NOW, + ), ).node; expect(downReason(quick, new Date(NOW.getTime() + 1_500))).toBeNull(); expect(downReason(quick, new Date(NOW.getTime() + 2_500))).toBe( @@ -90,7 +115,8 @@ describe('Fleet', () => { expect(new Fleet(handle).all()).toEqual([]); expect(fleet.remove('node-b')).toBe(false); expect( - fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW).joined, + taken(fleet.checkIn(checkInOf('node-b', 'http://10.0.0.7:80'), NOW)) + .joined, ).toBe(true); }); }); diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 69e02140..079700bc 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -46,6 +46,11 @@ export function downReason(node: NodeState, now: Date): string | null { return null; } +/** What a check-in came to: taken (and whether it joined or moved), or refused with the sentence the node is told (routes/nodes.ts answers 409). */ +export type CheckInOutcome = + | { node: NodeState; joined: boolean; movedFrom: string | null } + | { refused: string }; + /** * The fleet: every node that has ever checked in. Rows come from the * database at start (a node that is down must still be known — its names @@ -82,18 +87,22 @@ export class Fleet { * A node reporting for duty. A first check-in adds the node (`joined` * says so, for the log); a changed endpoint is written through — the * node states where it lives, the gateway does not remember better — - * and `movedFrom` names the old one, for the log: a node that moves at - * every check-in is two machines sharing one DORMICE_NODE_ID. The + * and `movedFrom` names the old one, for the log. Except inside the + * previous reporter's own interval: a different address that soon is a + * second machine with the same DORMICE_NODE_ID (the daemon's default is + * node-1), not a move, and is refused — written through, the two would + * flip the endpoint at every check-in, a lookup would ask whichever is + * current, a name on the other would read as new and be built again, on + * two nodes, with no 409 ever (traced by review, 2026-09-14). The first + * reporter keeps the id; the second is told why. A node that really + * moved is taken at its next check-in, one interval on. The * placement counter restarts at zero: what was placed before this * reading is in it now, and what is still in flight on the node (its * row is written after the container is up) is in neither figure until * the next reading — one interval of slack, self-correcting, the same * as before. */ - checkIn( - report: CheckInRequest, - now = new Date(), - ): { node: NodeState; joined: boolean; movedFrom: string | null } { + checkIn(report: CheckInRequest, now = new Date()): CheckInOutcome { let node = this.members.get(report.nodeId); let joined = false; let movedFrom: string | null = null; @@ -117,6 +126,19 @@ export class Fleet { this.members.set(node.id, node); joined = true; } else if (node.endpoint !== report.endpoint) { + if ( + node.lastCheckInAt !== null && + node.intervalSeconds !== null && + now.getTime() - node.lastCheckInAt.getTime() < + node.intervalSeconds * 1000 + ) { + const ago = Math.round( + (now.getTime() - node.lastCheckInAt.getTime()) / 1000, + ); + return { + refused: `node ${report.nodeId} checked in from ${node.endpoint} ${ago}s ago and now from ${report.endpoint} — two daemons share one DORMICE_NODE_ID (give this one its own), or the node just moved (then its next check-in, an interval later, is taken)`, + }; + } this.db .update(nodes) .set({ endpoint: report.endpoint }) @@ -134,7 +156,7 @@ export class Fleet { return { node, joined, movedFrom }; } - /** The operator's word that the node is gone for good; a node still running re-adds itself at its next check-in. */ + /** The operator's word that the node is gone for good (routes/nodes.ts refuses it for a node still checking in); one removed while briefly silent re-adds itself at its next check-in. */ remove(id: string): boolean { const existed = this.members.delete(id); this.db.delete(nodes).where(eq(nodes.id, id)).run(); diff --git a/packages/gateway/src/placement.test.ts b/packages/gateway/src/placement.test.ts index c9833ee2..165cf6de 100644 --- a/packages/gateway/src/placement.test.ts +++ b/packages/gateway/src/placement.test.ts @@ -29,10 +29,12 @@ function node( checkedInAt?: Date; } = {}, ): NodeState { - const { node } = f.checkIn( + const outcome = f.checkIn( checkInOf(id, `http://${id}:80`, over), over.checkedInAt ?? NOW, ); + if ('refused' in outcome) throw new Error(outcome.refused); + const { node } = outcome; node.placedSinceCheckIn = over.placed ?? 0; return node; } diff --git a/packages/gateway/src/routes/e2b.ts b/packages/gateway/src/routes/e2b.ts index a80443e9..3af2dcf8 100644 --- a/packages/gateway/src/routes/e2b.ts +++ b/packages/gateway/src/routes/e2b.ts @@ -140,7 +140,12 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( } const name = judged.data; return locks.run(name, async () => { - const found = verdict(await finder.byName(name), `sandbox "${name}"`); + // Confirmed with the cached node first (find.ts byName has why): + // the daemon's create builds what it does not find. + const found = verdict( + await finder.byName(name, { confirm: true }), + `sandbox "${name}"`, + ); if (found.kind === 'refuse') { return refuse(reply, found, (message) => ({ code: found.status, diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index 17cfafa3..1aec5452 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -166,11 +166,17 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( name: string, body: Buffer | undefined, ) { - const judged = verdict(await finder.byName(name), `sandbox "${name}"`); + // A creator confirms a cache hit with its node first (find.ts byName + // has why): the daemon's acquire builds what it does not find. + const judged = verdict( + await finder.byName(name, { confirm: true }), + `sandbox "${name}"`, + ); if (judged.kind === 'refuse') return refuse(reply, judged); let target = judged.kind === 'node' ? judged.node : null; // Found nowhere: a placement, counted against the node it lands on. - // Found somewhere: a wake, already in that node's reading. + // Found somewhere — and confirmed there: a wake, already in that + // node's reading. const placed = target === null; if (target === null) { if (clientGone(reply)) return; diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index 1240e56a..bf6987dc 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -15,6 +15,11 @@ export interface NodeRoutesOptions { cache: NameCache; } +/** A refusal in the native dialect, rendered by the app's error handler as `{ message }` under its status. */ +function refusal(statusCode: number, message: string): Error { + return Object.assign(new Error(message), { statusCode }); +} + /** * The gateway's own verbs about its nodes: the check-in the nodes send * (RULES/协议.md「网关」), and what an operator reads and does about them. @@ -34,7 +39,15 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( }, }, async (request) => { - const { node, joined, movedFrom } = fleet.checkIn(request.body); + const outcome = fleet.checkIn(request.body); + if ('refused' in outcome) { + request.log.warn( + { nodeId: request.body.nodeId, endpoint: request.body.endpoint }, + outcome.refused, + ); + throw refusal(409, outcome.refused); + } + const { node, joined, movedFrom } = outcome; if (joined) { request.log.info( { nodeId: node.id, endpoint: node.endpoint }, @@ -107,6 +120,27 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( }, }, async (request) => { + // "Gone for good" is refused for a node that is still checking in: + // its row would go, its names would be new names, and any of them + // acquired in the seconds before its next check-in would be built + // elsewhere — then the node re-adds itself and every such name is on + // two nodes, a 409 an operator clears by hand. Stop the daemon + // first; two of its intervals of silence is what "down" means + // (fleet.ts downReason), and a down node is removable (found by + // review, 2026-09-14). + const node = fleet.get(request.body.id); + if (node !== undefined) { + const now = new Date(); + if (downReason(node, now) === null && node.lastCheckInAt !== null) { + const ago = Math.round( + (now.getTime() - node.lastCheckInAt.getTime()) / 1000, + ); + throw refusal( + 409, + `node ${node.id} checked in ${ago}s ago — it is running, and its names would be placed elsewhere before it checked in again and come back as a 409 on two nodes; stop its daemon, wait two of its intervals (${node.intervalSeconds}s each), then remove it`, + ); + } + } const removed = fleet.remove(request.body.id); const evicted = cache.evictNode(request.body.id); if (removed) { diff --git a/packages/gateway/src/routes/verdict.ts b/packages/gateway/src/routes/verdict.ts index e4636633..f7eabd36 100644 --- a/packages/gateway/src/routes/verdict.ts +++ b/packages/gateway/src/routes/verdict.ts @@ -43,7 +43,7 @@ export function verdict(found: Found, what: string): Verdict { .map((s) => `node ${s.nodeId} did not answer (${s.why})`) .join( ', ', - )} — it cannot be treated as new while a node that may hold it is silent; retry, or remove the node if it is gone for good`, + )} — it cannot be treated as new while a node that may hold it is silent: that node may be down, or busy building this very name (its lookup waits for that); retry after Retry-After, and remove the node only if it is gone for good`, retryAfterSeconds: RETRY_AFTER_SECONDS, }; } diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts index 12d2c6b7..7022acfc 100644 --- a/packages/shared/src/gateway.ts +++ b/packages/shared/src/gateway.ts @@ -142,7 +142,10 @@ export type ListNodesResponse = z.infer; * goes, its sandboxes are no longer looked for, and a name that lived only * there is a new name again. A node that is merely down needs nothing — * it is back the moment it checks in — and one removed by mistake re-adds - * itself the same way. + * itself the same way. Refused (409) while the node is still checking in: + * a name of its acquired in the seconds before its next check-in would be + * built elsewhere and come back on two nodes. Stop the daemon, wait two of + * its intervals (that is "down"), then remove. */ export const removeNodeRequestSchema = z.object({ id: z.string().min(1), diff --git a/packages/shared/src/lookup.ts b/packages/shared/src/lookup.ts index c449b0af..8eb0a281 100644 --- a/packages/shared/src/lookup.ts +++ b/packages/shared/src/lookup.ts @@ -15,8 +15,11 @@ import { SANDBOX_STATES } from './states'; * row that exists answers at once, whatever its state; no row while an * acquire of that name is in flight waits for the acquire (the daemon * creates first and writes the row second, both under the slot) and looks - * again — so a gateway retrying a create whose answer was lost finds the - * sandbox on the node that built it, and never places a second copy. + * again — so a gateway retrying a create whose answer was lost never + * places a second copy: while the build is still running the gateway's + * two-second patience runs out first and the caller is told to retry + * (503 with Retry-After); once the row is written, the retry finds the + * sandbox on the node that built it. */ export const lookupSandboxRequestSchema = z.union([ z.object({ name: sandboxNameSchema }), From d3cc1309c3e182bcaf3890b83644dc013eb6f9af Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 05:14:43 +0800 Subject: [PATCH 16/89] forwardStream flushes the node's head as soon as it arrives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node holds a written head until the first body byte. A node stream that opens and then waits for its first event looked, through the gateway, like a node that had not answered — measured: a 300ms pause before the first byte held the status line for 300ms. The head now goes out when the node's did. --- packages/gateway/src/forward.test.ts | 23 +++++++++++++++++++++++ packages/gateway/src/forward.ts | 6 ++++++ 2 files changed, 29 insertions(+) diff --git a/packages/gateway/src/forward.test.ts b/packages/gateway/src/forward.test.ts index 58ef946d..60e5e24a 100644 --- a/packages/gateway/src/forward.test.ts +++ b/packages/gateway/src/forward.test.ts @@ -85,6 +85,29 @@ describe('forwardStream', () => { expect(res.headers['transfer-encoding']).toBe('chunked'); }); + it("the node's head reaches the client when it arrives, not with the first body byte", async () => { + let nodeEnded = false; + const node = http.createServer((_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain' }); + res.flushHeaders(); + setTimeout(() => { + nodeEnded = true; + res.end('late'); + }, 800); + }); + const endpoint = await listen(node); + const res = await request(`${await front(endpoint)}/execCommand`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: '{"name":"x"}', + }); + // undici resolves on the head; the node has not written a body byte. + expect(res.statusCode).toBe(200); + expect(nodeEnded).toBe(false); + expect(await res.body.text()).toBe('late'); + expect(nodeEnded).toBe(true); + }); + it("resolves with the node's status once the answer is relayed — the named verbs read a 404 off it", async () => { const node = http.createServer((_req, res) => { res.writeHead(404, { 'content-type': 'application/json' }); diff --git a/packages/gateway/src/forward.ts b/packages/gateway/src/forward.ts index 5fdc09a4..6b9917bf 100644 --- a/packages/gateway/src/forward.ts +++ b/packages/gateway/src/forward.ts @@ -205,6 +205,12 @@ export async function forwardStream( res.off('close', onClose); } res.writeHead(upstream.statusCode, inboundHeaders(upstream.headers)); + // Node holds a written head until the first body byte; the node's head + // has arrived, so it goes out now — a stream that opens and then waits + // (a process stream before its first event) must look open to the + // caller, not like a node that has not answered (found by review, + // 2026-09-14). + res.flushHeaders(); try { // pipeline destroys both ends on failure: a client that went away // aborts the node's response, a node that died cuts the client. From 4bb011a0bed301f65f3babb4b4c86a8b4c4ee24f Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 05:14:43 +0800 Subject: [PATCH 17/89] The check-in's first CPU reading is an honest null, not a percentage over a few milliseconds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sampler was primed right before the first check-in, so its first delta spanned the sliver between the two calls and read near 0 or near 100 by luck; a freshly restarted node could sit out its first interval on a number that meant nothing. Unprimed, the first reading is null — "no interval yet" — which placement lets through as unknown, as it was written to. --- packages/server/src/main.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index bd9fbe9f..6bc55694 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -325,8 +325,13 @@ let checkIn: CheckIn | undefined; if (config.DORMICE_GATEWAY_ENDPOINT !== undefined) { const nodeEndpoint = config.DORMICE_NODE_ENDPOINT ?? `http://127.0.0.1:${config.DORMICE_PORT}`; + // Not primed: the first check-in then reports cpuUsedPct null — "no + // interval yet" — which placement lets through as unknown. A sample a + // few milliseconds before it would make that first reading a percentage + // over the sliver in between, near 0 or near 100 by luck, and a freshly + // restarted node could sit out its first interval on a number that + // meant nothing (found by review, 2026-09-14). const checkInCpu = new CpuSampler(); - checkInCpu.sample(); checkIn = new CheckIn({ gateway: config.DORMICE_GATEWAY_ENDPOINT, token: config.DORMICE_API_TOKEN, From 00f6e458c845fbe2efacc38482e22aa676aa96e8 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 12:46:03 +0800 Subject: [PATCH 18/89] The lookup gives up in two seconds on a black-holed node too, and says why in words MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit undici's request() ignores an abort signal while the socket is still connecting, so a node whose host drops the SYN (a deleted VM, a closed security group — the case removeNode exists for) held every question for the dispatcher's 10s connect timeout, not the promised two seconds; and the creator's confirmation runs inside the name's slot, so queued acquires of a name cached there paid that in series. fetch honours the signal mid-connect (measured: request 10 500ms, fetch 2 001ms against 192.0.2.1); it follows no redirect here either. causeOf now prefers string codes: a DOMException's numeric legacy code rendered a timed-out node as "did not answer (23)". One causeOf, in lookup.ts, shared with forward.ts. The refused-connection test moved off port 9, which fetch refuses as a "bad port" without dialling. --- packages/gateway/src/find.test.ts | 34 ++++++++++++++++++++++-- packages/gateway/src/forward.ts | 7 +---- packages/gateway/src/lookup.ts | 44 ++++++++++++++++++++++++------- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/packages/gateway/src/find.test.ts b/packages/gateway/src/find.test.ts index f09805af..dcb4b260 100644 --- a/packages/gateway/src/find.test.ts +++ b/packages/gateway/src/find.test.ts @@ -6,7 +6,12 @@ import { NameCache } from './cache'; import { migrateDb, openDb } from './db/db'; import { Finder } from './find'; import { Fleet } from './fleet'; -import { type AskNode, httpAskNode, type LookupAnswer } from './lookup'; +import { + type AskNode, + httpAskNode, + LOOKUP_TIMEOUT_MS, + type LookupAnswer, +} from './lookup'; import { checkInOf } from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); @@ -245,8 +250,33 @@ describe('httpAskNode', () => { kind: 'silent', why: expect.stringMatching(/answered 401/), }); + // A port nobody listens on any more: the OS hands one out and it is + // released before the question is asked. (Not port 9 — fetch refuses + // the Fetch standard's "bad ports" without dialling.) + const released = http.createServer(); + await new Promise((r) => released.listen(0, '127.0.0.1', r)); + const port = (released.address() as AddressInfo).port; + await new Promise((r) => released.close(() => r())); expect( - await ask({ id: 'gone', endpoint: 'http://127.0.0.1:9' }, { id: 'x' }), + await ask( + { id: 'gone', endpoint: `http://127.0.0.1:${port}` }, + { id: 'x' }, + ), ).toMatchObject({ kind: 'silent', why: 'ECONNREFUSED' }); }); + + it('a node whose host swallows the connection is silent after the two seconds, not after a connect timeout — and the why is a word, not a number', async () => { + // 192.0.2.1 (TEST-NET-1) is routed nowhere: the SYN is dropped, the + // connect hangs. Where a network refuses it outright instead + // (ENETUNREACH), the answer is immediate and the assertion still holds. + const started = Date.now(); + const answer = await httpAskNode(TOKEN)( + { id: 'hole', endpoint: 'http://192.0.2.1:80' }, + { id: 'x' }, + ); + expect(Date.now() - started).toBeLessThan(LOOKUP_TIMEOUT_MS + 1_500); + expect(answer.kind).toBe('silent'); + expect(answer.kind === 'silent' && typeof answer.why).toBe('string'); + expect(answer.kind === 'silent' && answer.why).not.toMatch(/^\d+$/); + }, 15_000); }); diff --git a/packages/gateway/src/forward.ts b/packages/gateway/src/forward.ts index 6b9917bf..a867546d 100644 --- a/packages/gateway/src/forward.ts +++ b/packages/gateway/src/forward.ts @@ -3,6 +3,7 @@ import net from 'node:net'; import type { Duplex } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { Agent } from 'undici'; +import { causeOf } from './lookup'; /** * The one place the gateway talks to a node on a caller's behalf. Bytes @@ -131,12 +132,6 @@ function inboundHeaders( return out; } -function causeOf(error: unknown): string { - const e = error as { code?: string; message?: string; cause?: unknown }; - const cause = e.cause as { code?: string; message?: string } | undefined; - return cause?.code ?? cause?.message ?? e.code ?? e.message ?? String(error); -} - /** * The request target is sent to the node exactly as the caller wrote it — * origin and path handed to undici separately, never joined into a URL. diff --git a/packages/gateway/src/lookup.ts b/packages/gateway/src/lookup.ts index 11476283..ad471898 100644 --- a/packages/gateway/src/lookup.ts +++ b/packages/gateway/src/lookup.ts @@ -3,7 +3,6 @@ import { lookupSandboxResponseSchema, type SandboxState, } from '@dormice/shared'; -import { request } from 'undici'; /** * Asking one node "do you hold this sandbox?" — the daemon's lookupSandbox @@ -34,17 +33,41 @@ export type AskNode = ( query: LookupQuery, ) => Promise; +/** + * The transport's word for a failure, for a log line or a 503's sentence: + * a system code where there is one (ECONNREFUSED, UND_ERR_CONNECT_TIMEOUT), + * else the message. Only string codes count — a DOMException carries a + * numeric legacy `code` (TimeoutError is 23), and "node b did not answer + * (23)" tells an operator nothing (found by review, 2026-09-14). + */ export function causeOf(error: unknown): string { - const e = error as { code?: string; message?: string; cause?: unknown }; - const cause = e.cause as { code?: string; message?: string } | undefined; - return cause?.code ?? cause?.message ?? e.code ?? e.message ?? String(error); + const e = error as { code?: unknown; message?: string; cause?: unknown }; + const cause = e.cause as { code?: unknown; message?: string } | undefined; + if (typeof cause?.code === 'string') return cause.code; + if (cause?.message !== undefined) return cause.message; + if (typeof e.code === 'string') return e.code; + return e.message ?? String(error); } -/** The production asker: HTTP to the node's endpoint under the fleet's token. */ +/** + * The production asker: HTTP to the node's endpoint under the fleet's + * token. `fetch`, not undici's `request`, for the deadline: `request` + * ignores an abort signal while the socket is still connecting, so a node + * whose host drops the SYN (a VM deleted, a security group closed — the + * machine-gone case removeNode exists for) held every question for + * undici's 10s connect timeout, not two seconds — and the creator's + * confirmation runs inside the name's slot, so twenty queued acquires of a + * name cached there would have waited 3.5 minutes for the last (measured + * 2026-09-14: request 10 500ms, fetch 2 001ms, against 192.0.2.1). No + * redirect is followed: a front that redirects is not a node, and fetch + * would drop the Authorization header across origins on the way. One + * quirk comes with fetch: it refuses the Fetch standard's "bad ports" + * (9, 22, 25, 6000 …) without dialling — no node's front lives on one. + */ export function httpAskNode(token: string): AskNode { return async (node, query) => { try { - const res = await request(`${node.endpoint}/lookupSandbox`, { + const res = await fetch(`${node.endpoint}/lookupSandbox`, { method: 'POST', headers: { authorization: `Bearer ${token}`, @@ -52,15 +75,16 @@ export function httpAskNode(token: string): AskNode { }, body: JSON.stringify(query), signal: AbortSignal.timeout(LOOKUP_TIMEOUT_MS), + redirect: 'manual', }); - if (res.statusCode !== 200) { - const text = await res.body.text(); + if (res.status !== 200) { + const text = await res.text(); return { kind: 'silent', - why: `lookupSandbox answered ${res.statusCode}: ${text.slice(0, 200)}`, + why: `lookupSandbox answered ${res.status}: ${text.slice(0, 200)}`, }; } - const answer = lookupSandboxResponseSchema.parse(await res.body.json()); + const answer = lookupSandboxResponseSchema.parse(await res.json()); return answer.found ? { kind: 'found', ...answer.sandbox } : { kind: 'absent' }; From bafb6021122d486b377ca3f53b5fde8772f19667 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 12:46:52 +0800 Subject: [PATCH 19/89] A creator whose client left while it waited for the slot asks no node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clientGone was checked only at placement, after the lookup round: a creator queued behind a slow create or destroy whose client had given up still confirmed with the cached node — a question inside the slot, paid in series by every abandoned request behind a stuck name. It is now the first thing inside the slot on both faces, and still checked after the round of questions. The fake node learned a slow destroy for the test. --- packages/gateway/src/app.test.ts | 49 +++++++++++++++++++++++++-- packages/gateway/src/routes/create.ts | 10 +++--- packages/gateway/src/routes/e2b.ts | 3 ++ packages/gateway/src/routes/native.ts | 7 ++++ 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 080bee39..5820b2ea 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -29,6 +29,8 @@ class FakeNode { readonly hits: Array<{ path: string; auth: string | undefined }> = []; creates = 0; endpoint = ''; + /** How long a destroy takes to answer — a slow node holding the name's slot. */ + destroyTakesMs = 0; private readonly server: http.Server; constructor(readonly id: string) { @@ -145,8 +147,15 @@ class FakeNode { }); } case '/destroySandbox': { - if (name !== undefined) this.sandboxes.delete(name); - return json(200, { destroyed: found !== undefined }); + const destroy = () => { + if (name !== undefined) this.sandboxes.delete(name); + json(200, { destroyed: found !== undefined }); + }; + if (this.destroyTakesMs > 0) { + setTimeout(destroy, this.destroyTakesMs); + return; + } + return destroy(); } case '/execCommand': { if (!found) @@ -525,6 +534,42 @@ describe('acquire: placing and finding', () => { expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); }); + it('an acquire whose client left while it waited for the name\'s slot asks no node and builds nothing', async () => { + const h = await gateway(['a']); + const a = h.nodes[0] as FakeNode; + await rpc(h, '/acquireSandbox', { name: 'q' }); + // A destroy holds the slot; an acquire queues behind it, and its + // client gives up while queued. + a.destroyTakesMs = 400; + const destroying = rpc(h, '/destroySandbox', { name: 'q' }); + await until(() => + a.hits.some((hit) => hit.path === '/destroySandbox') ? true : undefined, + ); + const asked = a.lookups(); + const left = new AbortController(); + const abandoned = fetch(`${h.endpoint}/acquireSandbox`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'q' }), + signal: left.signal, + }).then( + () => 'answered', + () => 'left', + ); + await new Promise((r) => setTimeout(r, 50)); + left.abort(); + expect(await abandoned).toBe('left'); + expect((await destroying).body).toEqual({ destroyed: true }); + // The slot is free; the abandoned acquire ran and did nothing. + await new Promise((r) => setTimeout(r, 100)); + expect(a.lookups()).toBe(asked); + expect(a.creates).toBe(1); + expect(a.sandboxes.has('q')).toBe(false); + }); + it('twenty simultaneous acquires of one new name are one create on one node; different names spread', async () => { const h = await gateway(['a', 'b']); const [a, b] = h.nodes as [FakeNode, FakeNode]; diff --git a/packages/gateway/src/routes/create.ts b/packages/gateway/src/routes/create.ts index d9f700b1..9dd3ca31 100644 --- a/packages/gateway/src/routes/create.ts +++ b/packages/gateway/src/routes/create.ts @@ -57,10 +57,12 @@ export function place( } /** - * A create that waited for its name's slot behind a slow destroy and whose - * client left meanwhile: nothing is placed or counted for it — the - * response is gone, forwardCapture would send nothing. The reply is - * hijacked so Fastify writes nothing to the dead socket either. + * A create whose client has already left: nothing is asked, placed or + * counted for it — the response is gone, forwardCapture would send + * nothing. Checked first thing inside the slot (a creator that waited + * behind a slow create or destroy) and again after the lookup round (up + * to two seconds). The reply is hijacked so Fastify writes nothing to the + * dead socket either. */ export function clientGone(reply: FastifyReply): boolean { if (!reply.raw.destroyed) return false; diff --git a/packages/gateway/src/routes/e2b.ts b/packages/gateway/src/routes/e2b.ts index 3af2dcf8..4b5ad57e 100644 --- a/packages/gateway/src/routes/e2b.ts +++ b/packages/gateway/src/routes/e2b.ts @@ -140,6 +140,9 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( } const name = judged.data; return locks.run(name, async () => { + // A client that left while waiting for the slot asks nobody + // (native.ts acquire has why). + if (clientGone(reply)) return reply; // Confirmed with the cached node first (find.ts byName has why): // the daemon's create builds what it does not find. const found = verdict( diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index 1aec5452..9c61107a 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -166,6 +166,12 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( name: string, body: Buffer | undefined, ) { + // First, before any node is asked: a creator that waited for the slot + // behind a slow create or destroy and whose client left meanwhile + // asks nobody — the confirmation below is a question to a node inside + // the slot, and twenty abandoned acquires of one name would otherwise + // hold the slot for twenty answers nobody reads (create.ts clientGone). + if (clientGone(reply)) return; // A creator confirms a cache hit with its node first (find.ts byName // has why): the daemon's acquire builds what it does not find. const judged = verdict( @@ -179,6 +185,7 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( // node's reading. const placed = target === null; if (target === null) { + // Asked again: the round of questions took up to two seconds. if (clientGone(reply)) return; const placement = place(fleet, knobs, new Date()); if (placement.node === null) { From 42f9bd635abcbfb7dd464a19126e8b64072e098d Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 12:52:31 +0800 Subject: [PATCH 20/89] install.sh builds the gateway with the daemon and restarts a hand-installed gateway unit first A machine that runs both (the single-machine install is a fleet of one) upgraded with install.sh rebuilt only the server, CLI and console: the gateway's dist stayed on the older commit and ran it at its next restart, against a daemon whose check-in it may no longer parse. The gateway is now in the build filter, and a dormice-gateway unit that is running or enabled is restarted before the daemon so its first check-in lands on the new one. --- deploy/dormice-gateway.service | 4 +++- deploy/install.sh | 24 +++++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/deploy/dormice-gateway.service b/deploy/dormice-gateway.service index e8369bbc..45f3eded 100644 --- a/deploy/dormice-gateway.service +++ b/deploy/dormice-gateway.service @@ -3,7 +3,9 @@ # there — systemd's EnvironmentFile treats an inline comment as part of the # value). install.sh does not install this unit yet: the two-role install # (gateway + node on one machine, `--role node` elsewhere) is a later step; -# until then it is copied into place by hand (docs/测试机搭建手册.md). +# until then it is copied into place by hand (docs/测试机搭建手册.md). Once it +# is enabled, install.sh re-runs rebuild the gateway with the daemon and +# restart this unit before the daemon's, so both run one commit. [Unit] Description=Dormice gateway (fleet front door) Wants=network-online.target diff --git a/deploy/install.sh b/deploy/install.sh index 858f8b52..8b49fa30 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -520,11 +520,15 @@ build_repo() { else pnpm install --frozen-lockfile fi - # Build only what a daemon host runs: the server (plus its workspace deps), - # the CLI, and the console SPA. The website package is the project's Next.js - # marketing site — building it here would cost minutes and import a frontend - # toolchain's failure modes into an installer whose job is the daemon. - pnpm --filter "@dormice/server..." --filter "@dormice/cli..." --filter "@dormice/console" build + # Build only what a Dormice host runs: the server (plus its workspace deps), + # the CLI, the console SPA — and the gateway, whose dist must move with the + # daemon's on a machine that runs both (the single-machine install is a + # fleet of one): a gateway left on an older dist runs it at its next + # restart, against a daemon whose check-in it may no longer parse. Seconds + # to build. The website package is the project's Next.js marketing site — + # building it here would cost minutes and import a frontend toolchain's + # failure modes into an installer whose job is the daemon. + pnpm --filter "@dormice/server..." --filter "@dormice/gateway..." --filter "@dormice/cli..." --filter "@dormice/console" build } log "Dormice code ($INSTALL_DIR)" @@ -728,6 +732,16 @@ log 'systemd service' cp "$INSTALL_DIR/deploy/dormice.service" /etc/systemd/system/dormice.service systemctl daemon-reload systemctl enable dormice >/dev/null 2>&1 +# A gateway installed by hand on this machine (deploy/dormice-gateway.service; +# install.sh does not install it yet) was just rebuilt with the daemon and is +# restarted first, so the daemon's first check-in lands on the new one: the two +# processes of a fleet of one run one commit, never two. Running or enabled — +# a unit started by hand and never enabled is running the old dist just the +# same. +if systemctl is-active -q dormice-gateway 2>/dev/null || systemctl is-enabled -q dormice-gateway 2>/dev/null; then + systemctl restart dormice-gateway + note 'restarted dormice-gateway (hand-installed unit, rebuilt with the daemon)' +fi # Restart, not start: a re-run just built fresh code, and the daemon is # crash-only by design — restarting it is always safe. systemctl restart dormice From fb058e8d2fb1ccbaf27bf27f175de803d5bce3d1 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 12:52:45 +0800 Subject: [PATCH 21/89] A check-in refusal reaches the log whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The non-200 branch cut the gateway's answer at 200 characters. The gateway's longest refusal, the 409 naming both endpoints of a shared node id, runs to about 260 — the operator saw the diagnosis and lost the remedy. Cut at 400 now: whole enough for every sentence the gateway writes, short enough that a front's HTML error page does not flood the log. --- packages/server/src/check-in.test.ts | 14 ++++++++++++++ packages/server/src/check-in.ts | 7 ++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index f4636b82..5b162e7b 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -183,6 +183,20 @@ describe('CheckIn', () => { ); }); + it("a refusal's whole sentence reaches the log — the 409 for a shared node id names two endpoints and a remedy past the 200th character", async () => { + const refusal = + 'node node-7 checked in from http://10.0.0.7:80 3s ago and now from http://10.0.0.8:80 — two daemons share one DORMICE_NODE_ID (give this one its own), or the node just moved (then its next check-in, an interval later, is taken)'; + const gw = await gateway(() => ({ + status: 409, + body: JSON.stringify({ message: refusal }), + })); + const { log, details } = logSpy(); + await new CheckIn(options(gw.endpoint, log)).once(); + expect((details[0] as { error: string }).error).toBe( + `gateway answered 409: ${JSON.stringify({ message: refusal })}`, + ); + }); + it('a gateway that is not there is a logged failure, never a throw', async () => { const { log, warns } = logSpy(); const checkIn = new CheckIn(options('http://127.0.0.1:9', log)); diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index d5acc4da..c81c741e 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -113,9 +113,14 @@ export class CheckIn { if (res.status !== 200) { const text = await res.text(); const location = res.headers.get('location'); + // Whole enough for the gateway's own refusals (its longest, the + // 409 naming both endpoints of a shared node id, runs to about 260 + // characters — cut at 200 it lost its remedy), short enough that + // a front's HTML error page does not flood the log. + const body = text.slice(0, 400); throw new Error( location === null - ? `gateway answered ${res.status}: ${text.slice(0, 200)}` + ? `gateway answered ${res.status}: ${body}` : `gateway answered ${res.status} redirecting to ${location} — DORMICE_GATEWAY_ENDPOINT must be the gateway's own address, not a front that redirects`, ); } From 4996ab25f5a0c5d335793c8fdf9d3d879e3f3977 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 13:00:39 +0800 Subject: [PATCH 22/89] The slot comment in the native face tells the truth about the daemon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It claimed the daemon runs its other verbs unserialized; the daemon holds a slot per name for thirteen of the fifteen. The gateway's own reason for taking the slot only for acquire and destroy — the two verbs whose outcome it acts on — stands on its own and is now the one stated. --- packages/gateway/src/routes/native.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index 9c61107a..161d2f85 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -121,11 +121,14 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( return reply.code(400).send({ message: named.refusal }); } const { name } = named; - // Only the two verbs that create or remove take the name's slot — - // the daemon's own discipline (its other verbs run unserialized - // too). The slot is what keeps twenty simultaneous acquires of a - // new name from each placing their own copy: the first finds - // nothing and places, the rest find its cache entry. + // The gateway takes the name's slot for the two verbs whose + // outcome it acts on — a create it may have to place, a destroy it + // must forget — so twenty simultaneous acquires of a new name + // place once: the first finds nothing and places, the rest find + // its cache entry. Every other verb is relayed unserialized: the + // daemon holds its own slot per name for the ones that touch the + // sandbox (routes/sandboxes.ts locks.run), and a relay in front of + // it has nothing to add to their order. if (verb === 'acquireSandbox') { return locks.run(name, () => acquire(request, reply, name, body)); } From 7b914769553093b5ba4f32fa461d278d703a7068 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 13:00:39 +0800 Subject: [PATCH 23/89] The name cache is bounded, least recently used first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node deletes rows on its own (an E2B deadline kill is the scanner's routine) and tells no gateway, so the cache kept an entry for every sandbox ever created through the process for as long as it lived — an unnamed E2B create a minute is half a million dead entries a year. A hundred thousand entries, generous next to a fleet's live population; past it the entry nobody asked about for longest goes, and a hit moves an entry to the young end. A wrongly evicted entry costs the one round of questions any miss costs. --- packages/gateway/src/cache.test.ts | 60 ++++++++++++++++++++++++++++++ packages/gateway/src/cache.ts | 34 ++++++++++++++++- 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 packages/gateway/src/cache.test.ts diff --git a/packages/gateway/src/cache.test.ts b/packages/gateway/src/cache.test.ts new file mode 100644 index 00000000..bd1e464a --- /dev/null +++ b/packages/gateway/src/cache.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; +import { NameCache } from './cache'; + +const on = (id: string, name: string | null, nodeId = 'a') => ({ + id, + name, + nodeId, +}); + +describe('NameCache', () => { + it('answers by name and by id; a name that moves to a new id drops the old id whole; an id that changes name frees the old name', () => { + const cache = new NameCache(); + cache.put(on('sb-1', 'alice')); + expect(cache.getByName('alice')?.id).toBe('sb-1'); + expect(cache.getById('sb-1')?.name).toBe('alice'); + cache.put(on('sb-2', 'alice', 'b')); + expect(cache.getById('sb-1')).toBeUndefined(); + expect(cache.getByName('alice')?.nodeId).toBe('b'); + cache.put(on('sb-2', 'alicia', 'b')); + expect(cache.getByName('alice')).toBeUndefined(); + expect(cache.getByName('alicia')?.id).toBe('sb-2'); + expect(cache.size).toBe(1); + }); + + it('evict drops one entry, both handles; evictNode drops every entry of a node and counts them', () => { + const cache = new NameCache(); + cache.put(on('sb-1', 'alice')); + cache.put(on('sb-2', null)); + cache.put(on('sb-3', 'carol', 'b')); + cache.evict(on('sb-1', 'alice')); + expect(cache.getByName('alice')).toBeUndefined(); + expect(cache.getById('sb-1')).toBeUndefined(); + expect(cache.evictNode('a')).toBe(1); + expect(cache.getById('sb-2')).toBeUndefined(); + expect(cache.getByName('carol')?.id).toBe('sb-3'); + expect(cache.size).toBe(1); + }); + + it('is bounded: past the limit the entry least recently asked about goes, both handles; one just read stays', () => { + const cache = new NameCache(3); + cache.put(on('sb-1', 'alice')); + cache.put(on('sb-2', 'bob')); + cache.put(on('sb-3', null)); + // alice is the oldest write, but she was just asked about. + expect(cache.getByName('alice')).toBeDefined(); + cache.put(on('sb-4', 'dora')); + expect(cache.size).toBe(3); + expect(cache.getByName('bob')).toBeUndefined(); + expect(cache.getById('sb-2')).toBeUndefined(); + expect(cache.getByName('alice')?.id).toBe('sb-1'); + expect(cache.getById('sb-3')).toBeDefined(); + expect(cache.getByName('dora')).toBeDefined(); + // A re-put of a known id is a use too, not a second entry. + cache.put(on('sb-3', 'cathy')); + expect(cache.size).toBe(3); + cache.put(on('sb-5', 'eve')); + expect(cache.getByName('alice')).toBeUndefined(); + expect(cache.getByName('cathy')?.id).toBe('sb-3'); + }); +}); diff --git a/packages/gateway/src/cache.ts b/packages/gateway/src/cache.ts index 0803928a..a9736694 100644 --- a/packages/gateway/src/cache.ts +++ b/packages/gateway/src/cache.ts @@ -8,6 +8,16 @@ * and wholesale for a node an operator removed. An entry that is wrong * costs one misrouted request, whose answer evicts it; a cache that is * lost costs one extra round of questions per name. + * + * Bounded, least recently used first. A node deletes rows on its own — an + * E2B deadline kill is the scanner's routine — and tells no gateway, so + * an unbounded map would keep an entry for every sandbox ever created + * through this process, for as long as it lived: an unnamed E2B create + * every minute is half a million dead entries a year. The bound is + * generous next to a fleet's live population (Beijing holds some fifteen + * thousand rows, 2026-09), and past it the entry nobody has asked about + * for longest goes; asked about again, it costs the one round of + * questions any miss costs. */ export interface CacheEntry { id: string; @@ -16,10 +26,15 @@ export interface CacheEntry { nodeId: string; } +export const CACHE_LIMIT = 100_000; + export class NameCache { private readonly byName = new Map(); + /** Insertion order is recency: a hit re-inserts, and the first key is the least recently used. */ private readonly byId = new Map(); + constructor(private readonly limit = CACHE_LIMIT) {} + put(entry: CacheEntry): void { // A name that moves to a new id (destroyed and re-acquired elsewhere // while this gateway did not see the destroy) drops the old entry @@ -39,15 +54,30 @@ export class NameCache { ) { this.byName.delete(known.name); } + this.byId.delete(entry.id); this.byId.set(entry.id, entry); + if (this.byId.size > this.limit) { + const oldest = this.byId.values().next().value; + if (oldest !== undefined) this.evict(oldest); + } } getByName(name: string): CacheEntry | undefined { - return this.byName.get(name); + const entry = this.byName.get(name); + if (entry !== undefined) this.touch(entry); + return entry; } getById(id: string): CacheEntry | undefined { - return this.byId.get(id); + const entry = this.byId.get(id); + if (entry !== undefined) this.touch(entry); + return entry; + } + + /** Marks the entry as just used: back to the end of the recency order. */ + private touch(entry: CacheEntry): void { + this.byId.delete(entry.id); + this.byId.set(entry.id, entry); } evict(entry: CacheEntry): void { From e1790906979f988ec6006c05e70cc140459219c8 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 13:00:39 +0800 Subject: [PATCH 24/89] A 409 for two node ids at one endpoint says they are one daemon, and names the way out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node whose DORMICE_NODE_ID changed checks in as a new node while its old id keeps its row; both are asked, both answer from the same address, and every name there was a 409 telling the operator to destroy one copy — of a sandbox that exists once. The finding now carries the endpoints, and when they are one the sentence says so: remove the id that no longer checks in (or correct a DORMICE_NODE_ENDPOINT that names the wrong machine). Nothing is collapsed or healed; the fleet's list is wrong and the operator is told exactly how. --- packages/gateway/src/app.test.ts | 37 +++++++++++++++++++++++++- packages/gateway/src/find.test.ts | 36 ++++++++++++++++++++++++- packages/gateway/src/find.ts | 8 ++++-- packages/gateway/src/routes/verdict.ts | 20 ++++++++++++-- 4 files changed, 95 insertions(+), 6 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 5820b2ea..3d13bea4 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -534,7 +534,7 @@ describe('acquire: placing and finding', () => { expect(h.fleet.get('b')?.placedSinceCheckIn).toBe(1); }); - it('an acquire whose client left while it waited for the name\'s slot asks no node and builds nothing', async () => { + it("an acquire whose client left while it waited for the name's slot asks no node and builds nothing", async () => { const h = await gateway(['a']); const a = h.nodes[0] as FakeNode; await rpc(h, '/acquireSandbox', { name: 'q' }); @@ -681,6 +681,41 @@ describe('acquire: placing and finding', () => { expect(sandboxOf(healed).nodeId).toBe('a'); }); + it("a node that checked in again under a new id answers every lookup twice: the 409 says the two ids are one endpoint and names the way out, not 'destroy one copy'", async () => { + const h = await gateway(['a', 'b']); + const [a] = h.nodes as [FakeNode, FakeNode]; + expect( + sandboxOf(await rpc(h, '/acquireSandbox', { name: 'kept' })).nodeId, + ).toBe('a'); + // The operator renamed DORMICE_NODE_ID on a's machine; the old row stays. + const renamed = await fetch(`${h.endpoint}/checkIn`, { + method: 'POST', + headers: { + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(checkInOf('a-renamed', a.endpoint)), + }); + expect(renamed.status).toBe(200); + h.cache.evict({ + id: sandboxOf(await rpc(h, '/acquireSandbox', { name: 'kept' })).id, + name: 'kept', + nodeId: 'a', + }); + const refused = await rpc(h, '/execCommand', { + name: 'kept', + command: 'true', + }); + expect(refused.status).toBe(409); + expect(message(refused)).toContain( + `nodes a and a-renamed, which are one endpoint (${a.endpoint})`, + ); + expect(message(refused)).toContain( + 'removeNode the id that no longer checks in', + ); + expect(message(refused)).not.toContain('destroy one copy'); + }); + it('a node that does not answer: its cached names 502, and a new name is a 503 with Retry-After naming it — until an operator removes it', async () => { const h = await gateway(['a', 'b']); const [a, b] = h.nodes as [FakeNode, FakeNode]; diff --git a/packages/gateway/src/find.test.ts b/packages/gateway/src/find.test.ts index dcb4b260..1b8a6023 100644 --- a/packages/gateway/src/find.test.ts +++ b/packages/gateway/src/find.test.ts @@ -83,10 +83,44 @@ describe('Finder', () => { const found = await new Finder(fleet, cache, ask, silentLog).byName( 'alice', ); - expect(found).toEqual({ kind: 'conflict', nodeIds: ['a', 'c'] }); + expect(found).toEqual({ + kind: 'conflict', + nodes: [ + { id: 'a', endpoint: 'http://a:80' }, + { id: 'c', endpoint: 'http://c:80' }, + ], + }); expect(cache.size).toBe(0); }); + it('two ids checked in from one endpoint answer twice for every name there: a conflict whose nodes share the endpoint', async () => { + const fleet = fleetOf('a', 'b'); + // The node at a's address checks in again under a new id: a rename. + fleet.checkIn(checkInOf('a-renamed', 'http://a:80'), NOW); + const { ask } = scripted({ + a: { kind: 'found', id: 'sb-1', name: 'alice', state: 'active' }, + 'a-renamed': { + kind: 'found', + id: 'sb-1', + name: 'alice', + state: 'active', + }, + }); + const found = await new Finder( + fleet, + new NameCache(), + ask, + silentLog, + ).byName('alice'); + expect(found).toEqual({ + kind: 'conflict', + nodes: [ + { id: 'a', endpoint: 'http://a:80' }, + { id: 'a-renamed', endpoint: 'http://a:80' }, + ], + }); + }); + it('every node says no: the name is new; a silent node among the noes: unsure, naming it', async () => { const fleet = fleetOf('a', 'b'); expect( diff --git a/packages/gateway/src/find.ts b/packages/gateway/src/find.ts index 8cd34a1f..bc6191ea 100644 --- a/packages/gateway/src/find.ts +++ b/packages/gateway/src/find.ts @@ -16,7 +16,7 @@ import type { AskNode, LookupQuery } from './lookup'; */ export type Found = | { kind: 'one'; node: NodeState; id: string; name: string | null } - | { kind: 'conflict'; nodeIds: string[] } + | { kind: 'conflict'; nodes: Array<{ id: string; endpoint: string }> } | { kind: 'none' } | { kind: 'unsure'; silent: Array<{ nodeId: string; why: string }> }; @@ -119,9 +119,13 @@ export class Finder { return { kind: 'one', node: first.node, id: entry.id, name: entry.name }; } if (found.length > 1) { + // The endpoints ride along: two ids answering from one endpoint is + // one daemon under two names, which verdict.ts diagnoses as such. return { kind: 'conflict', - nodeIds: found.map((f) => f.node.id).sort(), + nodes: found + .map((f) => ({ id: f.node.id, endpoint: f.node.endpoint })) + .sort((x, y) => x.id.localeCompare(y.id)), }; } const silent = answers.flatMap(({ node, answer }) => diff --git a/packages/gateway/src/routes/verdict.ts b/packages/gateway/src/routes/verdict.ts index f7eabd36..038c09f6 100644 --- a/packages/gateway/src/routes/verdict.ts +++ b/packages/gateway/src/routes/verdict.ts @@ -27,14 +27,30 @@ export function verdict(found: Found, what: string): Verdict { return { kind: 'node', node: found.node, id: found.id, name: found.name }; case 'none': return { kind: 'none' }; - case 'conflict': + case 'conflict': { // The gateway refuses every verb for this name with this very 409, // destroy included — it will not guess which copy the caller means. + const ids = found.nodes.map((n) => n.id).join(' and '); + const endpoints = new Set(found.nodes.map((n) => n.endpoint)); + if (endpoints.size === 1) { + // Two ids, one endpoint: not two copies but one daemon answering + // twice — a node whose DORMICE_NODE_ID changed (its old id keeps + // its row until removed), or two nodes whose DORMICE_NODE_ENDPOINT + // name the same machine. Nothing to destroy; the fleet's list of + // nodes is wrong, and the check-in log said so when it happened + // (routes/nodes.ts). + return { + kind: 'refuse', + status: 409, + message: `${what} was answered for by nodes ${ids}, which are one endpoint (${[...endpoints][0]}) — one daemon under two node ids: a node whose DORMICE_NODE_ID changed and whose old id still has its row (removeNode the id that no longer checks in; listNodes shows which), or two nodes whose DORMICE_NODE_ENDPOINT name the same machine (correct the wrong one)`, + }; + } return { kind: 'refuse', status: 409, - message: `${what} exists on nodes ${found.nodeIds.join(' and ')} — destroy one copy directly on its node before routing can resume`, + message: `${what} exists on nodes ${ids} — destroy one copy directly on its node before routing can resume`, }; + } case 'unsure': return { kind: 'refuse', From 18024fc44d68e70f02ad7ce7b78be1a509f33985 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 13:03:29 +0800 Subject: [PATCH 25/89] The check-in log says why in the transport's word, and speaks again when the failure changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetch says "fetch failed" and keeps ECONNREFUSED, ENOTFOUND or the TLS error in cause; the log carried only the first half. And a failure was logged once per streak: a gateway that was unreachable and then, up again, refused this node as a twin of another (409) never made the log — the streak had already been announced. What is wrong is now compared with the numbers blanked, so a change of failure is one more line and a 409 that says 3s ago, then 4s ago, is still one. --- packages/server/src/check-in.test.ts | 37 ++++++++++++++++++++++++--- packages/server/src/check-in.ts | 38 ++++++++++++++++++++-------- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index 5b162e7b..666cfa82 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -197,11 +197,42 @@ describe('CheckIn', () => { ); }); - it('a gateway that is not there is a logged failure, never a throw', async () => { - const { log, warns } = logSpy(); - const checkIn = new CheckIn(options('http://127.0.0.1:9', log)); + it('a failure that changes is logged again — unreachable, then refused, are two events — while the same refusal with another number in it is not', async () => { + let answer = { status: 500, body: '{"message":"boom"}' }; + const gw = await gateway(() => answer); + const { log, warns, details } = logSpy(); + const checkIn = new CheckIn(options(gw.endpoint, log)); + await checkIn.once(); + await checkIn.once(); + expect(warns).toHaveLength(1); + const refusal = (ago: number) => + JSON.stringify({ + message: `node node-7 checked in from http://10.0.0.7:80 ${ago}s ago and now from http://10.0.0.8:80 — two daemons share one DORMICE_NODE_ID`, + }); + answer = { status: 409, body: refusal(3) }; + await checkIn.once(); + answer = { status: 409, body: refusal(4) }; + await checkIn.once(); + expect(warns).toHaveLength(2); + expect(warns[1]).toMatch(/still failing, differently/); + expect((details[1] as { error: string }).error).toMatch( + /gateway answered 409: .*3s ago/, + ); + }); + + it("a gateway that is not there is a logged failure, never a throw, and the log says why in the transport's word", async () => { + // A port the OS just released: dialling it is refused, not black-holed. + const probe = http.createServer(); + await new Promise((resolve) => probe.listen(0, '127.0.0.1', resolve)); + const port = (probe.address() as AddressInfo).port; + await new Promise((resolve) => probe.close(() => resolve())); + const { log, warns, details } = logSpy(); + const checkIn = new CheckIn(options(`http://127.0.0.1:${port}`, log)); await expect(checkIn.once()).resolves.toBeUndefined(); expect(warns).toHaveLength(1); + expect((details[0] as { error: string }).error).toMatch( + /fetch failed \(ECONNREFUSED\)/, + ); }); it('ticks on its interval from start() and stops on stop()', async () => { diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index c81c741e..a208d1a0 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -59,16 +59,31 @@ const CHECK_IN_TIMEOUT_MS = 10_000; * Chained setTimeout, the daemon's discipline: the next tick is scheduled * when this one is done, so a slow gateway never has ticks pile up. * Failures are logged on the change — once when the gateway stops - * answering, once when it answers again — never every tick: a gateway - * down for an hour is one event, not two hundred and forty lines. Never - * fatal: the gateway is the fleet's front door and configuration - * authority, not the node's reason to live; the node keeps running its - * sandboxes and keeps trying. + * answering, once more when what is wrong changes (a gateway that was + * unreachable and now refuses this node is news), once when it answers + * again — never every tick: a gateway down for an hour is one event, not + * two hundred and forty lines. Never fatal: the gateway is the fleet's + * front door and configuration authority, not the node's reason to live; + * the node keeps running its sandboxes and keeps trying. */ +/** + * A failure in the operator's words. fetch says "fetch failed" and keeps + * the reason (ECONNREFUSED, ENOTFOUND, a TLS error) in `cause`; a + * timeout is a DOMException whose only code is a legacy number. The + * transport's word is the one the operator acts on, so it is appended. + */ +function describe(error: unknown): string { + const e = error as { message?: string; cause?: unknown }; + const cause = e.cause as { code?: unknown; message?: string } | undefined; + const message = e.message ?? String(error); + const why = typeof cause?.code === 'string' ? cause.code : cause?.message; + return why === undefined ? message : `${message} (${why})`; +} + export class CheckIn { private timer: NodeJS.Timeout | undefined; private closing = false; - /** The failure the gateway is currently in, or null while it answers. */ + /** The failure the gateway is currently in (its sentence with the numbers blanked, so a 409 that says "3s ago" and then "4s ago" is one failure), or null while it answers. */ private failing: string | null = null; constructor(private readonly opts: CheckInOptions) {} @@ -130,14 +145,17 @@ export class CheckIn { this.failing = null; } } catch (error) { - const message = error instanceof Error ? error.message : String(error); - if (this.failing === null) { + const message = describe(error); + const failure = message.replace(/\d+/g, '#'); + if (failure !== this.failing) { opts.log.warn( { gateway: opts.gateway, error: message }, - 'check-in failed; the gateway places nothing here and forwards no new names to this node until it answers again — retrying every interval', + this.failing === null + ? 'check-in failed; the gateway places nothing here and forwards no new names to this node until it answers again — retrying every interval' + : 'check-in still failing, differently — retrying every interval', ); } - this.failing = message; + this.failing = failure; } } From 63dc74f3f58eb37e850cf114dbf4ec133478e9ce Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 13:03:29 +0800 Subject: [PATCH 26/89] A node whose gateway is on another machine must state its own DORMICE_NODE_ID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway tells nodes apart by it, and node-1 is what every other unconfigured node says too: the second to check in was refused as a twin at every check-in, or — when the first had been silent for an interval — taken for it having moved, and the first's names placed again elsewhere. The same rule as DORMICE_NODE_ENDPOINT, for the same reason: refused at boot, where the operator is looking. Beside its gateway, or alone, the default still serves. --- packages/server/src/config.test.ts | 29 +++++++++++++++++++++++++++++ packages/server/src/config.ts | 26 ++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index d07cebfa..c9a76efd 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -165,6 +165,7 @@ describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { ...TOKEN, DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677/', DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80///', + DORMICE_NODE_ID: 'node-7', DORMICE_CHECK_IN_INTERVAL_SECONDS: '5', }); expect(config.DORMICE_GATEWAY_ENDPOINT).toBe('http://10.0.0.5:3677'); @@ -220,6 +221,7 @@ describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { ...TOKEN, DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80', + DORMICE_NODE_ID: 'node-7', }).DORMICE_NODE_ENDPOINT, ).toBe('http://10.0.0.7:80'); for (const local of [ @@ -233,4 +235,31 @@ describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { ).toBeUndefined(); } }); + + it('a gateway on another machine requires a node id of its own, naming why; beside its gateway the default serves', () => { + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80', + }), + ).toThrow( + /DORMICE_NODE_ID is required when DORMICE_GATEWAY_ENDPOINT is not loopback/, + ); + expect( + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://10.0.0.5:3677', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80', + DORMICE_NODE_ID: 'bj-7', + }).DORMICE_NODE_ID, + ).toBe('bj-7'); + expect( + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_ENDPOINT: 'http://127.0.0.1:3677', + }).DORMICE_NODE_ID, + ).toBe('node-1'); + expect(loadConfig(TOKEN).DORMICE_NODE_ID).toBe('node-1'); + }); }); diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index ddfac547..59a0e184 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -20,8 +20,13 @@ import type { S3Settings } from './archive/s3-store'; const envSchema = z.object({ DORMICE_PORT: z.coerce.number().int().min(1).max(65535).default(3676), DORMICE_DB_PATH: z.string().default('data/dormice.db'), - /** Identifies this machine in the ledger. Single-machine today; the field keeps the ledger shardable. */ - DORMICE_NODE_ID: z.string().default('node-1'), + /** + * This node's name: in its ledger's rows, and to its gateway, which + * tells nodes apart by it. The default serves a daemon that is the + * whole platform, or shares its machine with its gateway; a node whose + * gateway is elsewhere must state its own (checkedSchema below). + */ + DORMICE_NODE_ID: z.string().min(1).default('node-1'), /** How often the idle scanner sweeps the ledger. */ DORMICE_SCAN_INTERVAL_SECONDS: z.coerce.number().int().positive().default(60), /** @@ -301,6 +306,23 @@ const checkedSchema = envSchema path: ['DORMICE_NODE_ENDPOINT'], }, ) + // ...and who it is. The gateway tells nodes apart by DORMICE_NODE_ID, + // and node-1 is what every other unconfigured node says too: the + // second node-1 to check in is refused as a twin (409) at every + // check-in — or, when the first has been silent for an interval, taken + // for it having moved, and the first's names are placed again + // elsewhere. Refused here, at boot, where the operator is looking. + .refine( + (cfg) => + cfg.DORMICE_GATEWAY_ENDPOINT === undefined || + isLoopbackUrl(cfg.DORMICE_GATEWAY_ENDPOINT) !== false || + cfg.DORMICE_NODE_ID !== 'node-1', + { + message: + 'DORMICE_NODE_ID is required when DORMICE_GATEWAY_ENDPOINT is not loopback: the gateway tells nodes apart by it, and node-1 (the default) is what every other unconfigured node says — the second to check in is refused as a twin. Give this node a name of its own, e.g. its hostname', + path: ['DORMICE_NODE_ID'], + }, + ) // All-or-none: a half-configured store would make the archiver's // existence ambiguous, and ambiguity here decides real policy defaults. .superRefine((cfg, ctx) => { From aa362bd606ef49a0590699c1284e37470d60d8bc Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 15:01:09 +0800 Subject: [PATCH 27/89] A node the restarted gateway has not heard from yet cannot be removed for thirty seconds, and a shared endpoint is warned about once Two leftovers from the third review. After a gateway restart every node is silent so far, the running ones included, and downReason alone let an operator remove a live node in the first interval; the Fleet now records when it started and removeNode refuses a never-heard node for two default check-in intervals, naming the wait. Two nodes reporting one endpoint were warned about at every check-in, two hundred and forty lines an hour; the route now remembers what it last said per node and speaks when the situation arises, changes or ends. --- packages/gateway/src/app.test.ts | 84 +++++++++++++++++++++++++++- packages/gateway/src/fleet.test.ts | 7 ++- packages/gateway/src/fleet.ts | 22 +++++++- packages/gateway/src/routes/nodes.ts | 48 +++++++++++++--- 4 files changed, 148 insertions(+), 13 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 3d13bea4..fd6b2715 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -3,6 +3,7 @@ import http from 'node:http'; import net, { type AddressInfo } from 'node:net'; import { fileURLToPath } from 'node:url'; import { KeyedQueue } from '@dormice/server/keyed-queue'; +import { pino } from 'pino'; import { afterEach, describe, expect, it } from 'vitest'; import { buildGatewayApp } from './app'; import { NameCache } from './cache'; @@ -215,6 +216,12 @@ afterEach(async () => { async function gateway( nodeIds: string[], env: Record = {}, + opts: { + /** When the gateway "started" — the removeNode startup grace is judged against it. */ + startedAt?: Date; + /** Collects the gateway's own log lines (JSON, one per entry) when a test asserts on what it says. */ + logs?: string[]; + } = {}, ): Promise { const db = openDb(':memory:'); migrateDb(db, MIGRATIONS); @@ -225,17 +232,21 @@ async function gateway( DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '100', ...env, }); - const fleet = new Fleet(db); + const fleet = new Fleet(db, opts.startedAt); const cache = new NameCache(); const finder = new Finder(fleet, cache, httpAskNode(TOKEN), { warn: () => {}, }); + const logs = opts.logs; const app = buildGatewayApp({ config, fleet, finder, locks: new KeyedQueue(), - logger: false, + logger: + logs === undefined + ? false + : pino({ level: 'info' }, { write: (line: string) => logs.push(line) }), build: null, }); await app.listen({ host: '127.0.0.1', port: 0 }); @@ -455,6 +466,75 @@ describe('check-in and the node verbs', () => { ((await rpc(h, '/listNodes')).body as { nodes: unknown[] }).nodes, ).toHaveLength(2); }); + + it('two nodes on one endpoint are warned about when it arises or changes, not at every check-in; and once more when it stops', async () => { + const logs: string[] = []; + const h = await gateway(['b'], {}, { logs }); + const shared = h.nodes[0]?.endpoint ?? ''; + const warned = () => + logs.filter((l) => l.includes('two nodes report the same endpoint')); + // Three check-ins from c at b's address: one warning, not three. + for (let i = 0; i < 3; i += 1) { + await rpc(h, '/checkIn', checkInOf('c', shared)); + } + expect(warned()).toHaveLength(1); + expect(JSON.parse(warned()[0] ?? '{}')).toMatchObject({ + nodeId: 'c', + alsoReportedBy: ['b'], + }); + // A third node at the address is news; c's next check-in is news + // again, because the set it shares with changed. + await rpc(h, '/checkIn', checkInOf('d', shared)); + expect(warned()).toHaveLength(2); + await rpc(h, '/checkIn', checkInOf('c', shared)); + expect(warned()).toHaveLength(3); + expect(JSON.parse(warned()[2] ?? '{}')).toMatchObject({ + nodeId: 'c', + alsoReportedBy: ['b', 'd'], + }); + // c moves to an address of its own (an interval later, so the move is + // taken): said once, as the end of the situation. + const c = h.fleet.get('c'); + if (!c) throw new Error('node lost'); + c.lastCheckInAt = new Date(Date.now() - 16_000); + await rpc(h, '/checkIn', checkInOf('c', 'http://10.0.0.99:80')); + await rpc(h, '/checkIn', checkInOf('c', 'http://10.0.0.99:80')); + expect( + logs.filter((l) => l.includes('no longer shares its endpoint')), + ).toHaveLength(1); + expect(warned()).toHaveLength(3); + }); + + it('right after a gateway start a node not yet heard from cannot be removed; past two default intervals it can', async () => { + // A restart: the rows are known, nothing has checked in yet. + const fresh = await gateway(['b']); + const b = fresh.fleet.get('b'); + if (!b) throw new Error('node lost'); + b.lastCheckInAt = null; + b.intervalSeconds = null; + const early = await rpc(fresh, '/removeNode', { id: 'b' }); + expect(early.status).toBe(409); + expect(message(early)).toMatch( + /^the gateway started \ds ago and has not heard from node b yet/, + ); + expect(fresh.fleet.get('b')).toBeDefined(); + // The same silence thirty-one seconds into the gateway's life is a + // node that is down. + const settled = await gateway( + ['b'], + {}, + { + startedAt: new Date(Date.now() - 31_000), + }, + ); + const quiet = settled.fleet.get('b'); + if (!quiet) throw new Error('node lost'); + quiet.lastCheckInAt = null; + quiet.intervalSeconds = null; + expect((await rpc(settled, '/removeNode', { id: 'b' })).body).toEqual({ + removed: true, + }); + }); }); describe('acquire: placing and finding', () => { diff --git a/packages/gateway/src/fleet.test.ts b/packages/gateway/src/fleet.test.ts index 287e184e..f6f5ae53 100644 --- a/packages/gateway/src/fleet.test.ts +++ b/packages/gateway/src/fleet.test.ts @@ -22,8 +22,13 @@ function db() { describe('Fleet', () => { it('a first check-in joins the node and persists its row; a gateway restart still knows it, unreached until it checks in again', () => { const handle = db(); - const fleet = new Fleet(handle); + const fleet = new Fleet(handle, NOW); expect(fleet.all()).toEqual([]); + expect(fleet.startedAt).toBe(NOW); + // Left unsaid, the start is now. + expect(Date.now() - new Fleet(handle).startedAt.getTime()).toBeLessThan( + 5_000, + ); const { node, joined } = taken( fleet.checkIn( checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 079700bc..cb5a93ab 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -26,6 +26,19 @@ export interface NodeState { placedIds: Set; } +/** + * How long after a gateway start a node that has not checked in yet is + * still presumed alive. A restarted gateway knows its nodes from their + * rows and nothing else: every lastCheckInAt is null, and "has not checked + * in since the gateway started" is true of a healthy node for up to one + * of its intervals. Judged by downReason alone, removeNode would let an + * operator delete a running node in that window (found by review, + * 2026-09-14). Two of the daemon's default intervals + * (DORMICE_CHECK_IN_INTERVAL_SECONDS, 15): the gateway cannot know a + * node's own interval before it has heard from it once. + */ +export const STARTUP_GRACE_MS = 2 * 15 * 1000; + /** * Why a node is not to be placed on right now, or null when it is fine: * never checked in since this gateway started, or silent for two of its @@ -59,7 +72,14 @@ export type CheckInOutcome = export class Fleet { private readonly members = new Map(); - constructor(private readonly db: Db) { + /** When this gateway process started — the yardstick for STARTUP_GRACE_MS. */ + readonly startedAt: Date; + + constructor( + private readonly db: Db, + startedAt: Date = new Date(), + ) { + this.startedAt = startedAt; for (const row of db.select().from(nodes).all()) { this.members.set(row.id, { id: row.id, diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index bf6987dc..76401a88 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -8,7 +8,7 @@ import { } from '@dormice/shared'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import type { NameCache } from '../cache'; -import { downReason, type Fleet } from '../fleet'; +import { downReason, type Fleet, STARTUP_GRACE_MS } from '../fleet'; export interface NodeRoutesOptions { fleet: Fleet; @@ -30,6 +30,16 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( app, { fleet, cache }, ) => { + /** + * Per node, the ids it was last reported to share an endpoint with + * (sorted, joined) — so the warning below is said when the situation + * arises or changes, not at every check-in: two nodes on one endpoint + * checking in every fifteen seconds is one misconfiguration, not two + * hundred and forty log lines an hour (the daemon's check-in log keeps + * the same discipline, server/check-in.ts). + */ + const twinsWarned = new Map(); + app.post( '/checkIn', { @@ -70,15 +80,24 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( } const twins = fleet .all() - .filter((n) => n.id !== node.id && n.endpoint === node.endpoint); + .filter((n) => n.id !== node.id && n.endpoint === node.endpoint) + .map((n) => n.id) + .sort(); + const before = twinsWarned.get(node.id); if (twins.length > 0) { - request.log.warn( - { - nodeId: node.id, - endpoint: node.endpoint, - alsoReportedBy: twins.map((n) => n.id), - }, - 'two nodes report the same endpoint — check DORMICE_NODE_ID and DORMICE_NODE_ENDPOINT on both; their sandboxes will be found twice (409) or land on the wrong machine', + const now = twins.join(','); + if (now !== before) { + twinsWarned.set(node.id, now); + request.log.warn( + { nodeId: node.id, endpoint: node.endpoint, alsoReportedBy: twins }, + 'two nodes report the same endpoint — check DORMICE_NODE_ID and DORMICE_NODE_ENDPOINT on both; their sandboxes will be found twice (409) or land on the wrong machine', + ); + } + } else if (before !== undefined) { + twinsWarned.delete(node.id); + request.log.info( + { nodeId: node.id, endpoint: node.endpoint }, + 'the node no longer shares its endpoint with another', ); } return {}; @@ -131,6 +150,17 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( const node = fleet.get(request.body.id); if (node !== undefined) { const now = new Date(); + // Right after a gateway start every node is silent so far, the + // running ones included: they are heard from within one interval. + // Until two default intervals have passed, "not heard from" is + // not "down" (fleet.ts STARTUP_GRACE_MS). + const sinceStart = now.getTime() - fleet.startedAt.getTime(); + if (node.lastCheckInAt === null && sinceStart < STARTUP_GRACE_MS) { + throw refusal( + 409, + `the gateway started ${Math.round(sinceStart / 1000)}s ago and has not heard from node ${node.id} yet — a running node checks in within its interval, so silence this early proves nothing; wait ${STARTUP_GRACE_MS / 1000}s from the gateway's start, then remove it`, + ); + } if (downReason(node, now) === null && node.lastCheckInAt !== null) { const ago = Math.round( (now.getTime() - node.lastCheckInAt.getTime()) / 1000, From f5aa90bcbfce4a02362d40d313a7f327ede3c27d Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 15:24:32 +0800 Subject: [PATCH 28/89] The activity ring is gone: lifecycle moves are one structured log line each, not rows in a table nobody queried MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design record #16 (2026-09-13). The bounded activity table, the listActivity verb, the SDK method, the shared ACTIVITY_KINDS and actor vocabulary, the console's activity page and the workbench's activity card (with its message domain in all ten locales) are deleted, and migration 0023 drops the table. What used to be an event is now a line in the daemon's own log where a logger already is: the routes log the sandbox-addressed changes, the key management and the settings, ingress and upgrade writes through the request log; main.ts logs the start and the disk growth; the heartbeat's reconcile and scan summaries were already logged. The lifecycle engine, the reconciler and the archiver stay silent and no longer carry an actor parameter — attribution existed only to feed the ring, so the auth hooks now answer yes or no and the request carries no identity. A shell death is still written to the row as lastExit, which the wire and the console read; the FakeExecutor records shell removals so the tests that used to count 'rebuilt' events still see a swap happen, or not. --- e2e/src/ingress.test.ts | 13 - e2e/src/native.test.ts | 35 - packages/console/messages/de/activity.json | 50 -- packages/console/messages/de/shell.json | 1 - packages/console/messages/de/workbench.json | 3 - packages/console/messages/en/activity.json | 50 -- packages/console/messages/en/shell.json | 1 - packages/console/messages/en/workbench.json | 3 - packages/console/messages/es/activity.json | 50 -- packages/console/messages/es/shell.json | 1 - packages/console/messages/es/workbench.json | 3 - packages/console/messages/fr/activity.json | 50 -- packages/console/messages/fr/shell.json | 1 - packages/console/messages/fr/workbench.json | 3 - packages/console/messages/ja/activity.json | 50 -- packages/console/messages/ja/shell.json | 1 - packages/console/messages/ja/workbench.json | 3 - packages/console/messages/ko/activity.json | 50 -- packages/console/messages/ko/shell.json | 1 - packages/console/messages/ko/workbench.json | 3 - packages/console/messages/pt-BR/activity.json | 50 -- packages/console/messages/pt-BR/shell.json | 1 - .../console/messages/pt-BR/workbench.json | 3 - packages/console/messages/ru/activity.json | 50 -- packages/console/messages/ru/shell.json | 1 - packages/console/messages/ru/workbench.json | 3 - packages/console/messages/zh-CN/activity.json | 50 -- packages/console/messages/zh-CN/shell.json | 1 - .../console/messages/zh-CN/workbench.json | 3 - packages/console/messages/zh-TW/activity.json | 50 -- packages/console/messages/zh-TW/shell.json | 1 - .../console/messages/zh-TW/workbench.json | 3 - packages/console/project.inlang/settings.json | 1 - packages/console/src/components/nav.ts | 2 - .../console/src/features/activity/actors.ts | 32 - .../features/activity/hooks/useActivity.ts | 18 - .../console/src/features/activity/kinds.ts | 121 --- .../features/activity/pages/ActivityPage.tsx | 247 ------ .../src/features/auth/pages/LoginPage.tsx | 2 +- .../components/workbench/MonitorRail.tsx | 75 -- packages/console/src/lib/api.ts | 6 - packages/console/src/lib/mock.ts | 4 +- packages/console/src/routeTree.gen.ts | 21 - packages/console/src/routes/_app/activity.tsx | 15 - packages/console/vite.config.ts | 1 - packages/gateway/src/routes/native.ts | 1 - packages/sdk/src/client.test.ts | 8 - packages/sdk/src/client.ts | 14 - .../server/drizzle/0023_drop-activity.sql | 1 + .../server/drizzle/meta/0023_snapshot.json | 773 ++++++++++++++++++ packages/server/drizzle/meta/_journal.json | 7 + packages/server/src/app.test.ts | 229 +----- packages/server/src/app.ts | 40 +- packages/server/src/archive/archiver.ts | 31 +- packages/server/src/auth.ts | 43 +- packages/server/src/check-in.test.ts | 1 - packages/server/src/db/activity.ts | 60 -- packages/server/src/db/api-keys.ts | 59 +- packages/server/src/db/ledger.ts | 16 - packages/server/src/db/schema.ts | 35 +- packages/server/src/e2b/compat.test.ts | 14 - packages/server/src/e2b/control.ts | 61 +- packages/server/src/e2b/deps.ts | 7 +- packages/server/src/e2b/envd/shared.ts | 16 +- packages/server/src/e2b/protocol.ts | 15 +- packages/server/src/executor/fake.ts | 8 + packages/server/src/ingress.test.ts | 14 - packages/server/src/lifecycle.ts | 159 +--- packages/server/src/main.ts | 31 +- packages/server/src/reconciler.test.ts | 24 +- packages/server/src/reconciler.ts | 44 +- packages/server/src/routes/activity.ts | 34 - packages/server/src/routes/api-keys.ts | 28 +- packages/server/src/routes/ingress.ts | 14 +- .../server/src/routes/observability.test.ts | 105 +-- packages/server/src/routes/sandboxes.ts | 108 +-- packages/server/src/routes/settings.test.ts | 73 +- packages/server/src/routes/settings.ts | 16 +- packages/server/src/routes/spec.test.ts | 46 +- packages/server/src/routes/upgrade.ts | 19 +- packages/server/src/sandbox-proxy.ts | 2 +- packages/server/src/scanner.ts | 24 +- packages/shared/src/activity.ts | 137 ---- packages/shared/src/index.ts | 1 - skills/dormice/SKILL.md | 2 +- website/content/docs/console.mdx | 11 +- website/content/docs/http-api.mdx | 22 +- website/content/docs/troubleshooting.mdx | 19 +- 88 files changed, 1056 insertions(+), 2449 deletions(-) delete mode 100644 packages/console/messages/de/activity.json delete mode 100644 packages/console/messages/en/activity.json delete mode 100644 packages/console/messages/es/activity.json delete mode 100644 packages/console/messages/fr/activity.json delete mode 100644 packages/console/messages/ja/activity.json delete mode 100644 packages/console/messages/ko/activity.json delete mode 100644 packages/console/messages/pt-BR/activity.json delete mode 100644 packages/console/messages/ru/activity.json delete mode 100644 packages/console/messages/zh-CN/activity.json delete mode 100644 packages/console/messages/zh-TW/activity.json delete mode 100644 packages/console/src/features/activity/actors.ts delete mode 100644 packages/console/src/features/activity/hooks/useActivity.ts delete mode 100644 packages/console/src/features/activity/kinds.ts delete mode 100644 packages/console/src/features/activity/pages/ActivityPage.tsx delete mode 100644 packages/console/src/routes/_app/activity.tsx create mode 100644 packages/server/drizzle/0023_drop-activity.sql create mode 100644 packages/server/drizzle/meta/0023_snapshot.json delete mode 100644 packages/server/src/db/activity.ts delete mode 100644 packages/server/src/routes/activity.ts delete mode 100644 packages/shared/src/activity.ts diff --git a/e2e/src/ingress.test.ts b/e2e/src/ingress.test.ts index 08f5737c..7f0fb538 100644 --- a/e2e/src/ingress.test.ts +++ b/e2e/src/ingress.test.ts @@ -91,17 +91,4 @@ describe('ingress domain binding over a real daemon', () => { writeFileSync(file, ours); expect((await client().getIngress()).domains).toEqual([]); }); - - it('records binds in the activity window', async () => { - await client().setIngress(['activity.dormice-e2e.test']); - await client().setIngress([]); - const events = await client().listActivity(); - const details = events - .filter((event) => event.kind === 'ingress-updated') - .map((event) => event.detail); - expect(details.length).toBeGreaterThanOrEqual(2); - expect(details.some((d) => d.includes('activity.dormice-e2e.test'))).toBe( - true, - ); - }); }); diff --git a/e2e/src/native.test.ts b/e2e/src/native.test.ts index cd0c2b48..f8f8ba1d 100644 --- a/e2e/src/native.test.ts +++ b/e2e/src/native.test.ts @@ -540,20 +540,6 @@ describe('native API over a real daemon', () => { const mine = listed.find((k) => k.name === 'rotation'); expect(mine?.lastUsedAt).not.toBeNull(); - // Attribution answers the blast-radius question: the key's work carries - // its id, and the mint (done above on the env token) says who minted. - const story = await client().listActivity({ limit: 500 }); - const keyWork = story.filter((e) => e.sandboxName === 'rotation-key'); - expect(keyWork.map((e) => e.actor)).toEqual([ - `apikey:${apiKey.id}`, - `apikey:${apiKey.id}`, - ]); - expect( - story.find( - (e) => e.kind === 'apikey-created' && e.detail.includes('"rotation"'), - )?.actor, - ).toBe('env-token'); - // Key-manages-key is refused with the honest 403 — a leaked key must // not be able to mint itself an unrevoked successor. await expect(keyed.listApiKeys()).rejects.toMatchObject({ @@ -689,16 +675,6 @@ describe('native API over a real daemon', () => { ), ).toMatchObject({ image: 'img:swap-v2', upgradable: false }); - // Both halves of the move made the audit trail, in order. - const story = (await client().listActivity({ limit: 500 })).filter( - (e) => e.sandboxName === 'swap-key', - ); - const kinds = story.map((e) => e.kind); - expect(kinds.slice(0, 2)).toEqual(['woken', 'rebuilt']); - expect(story[1]?.detail).toBe( - 'stale shell swapped at wake: img:swap-v1 -> img:swap-v2', - ); - await client().destroySandbox('swap-key'); await client().removeTemplate('swap-tpl'); }, @@ -849,17 +825,6 @@ describe('the observability verbs over a real daemon', () => { ).rejects.toMatchObject({ name: 'DormiceApiError', status: 404 }); }); - it("listActivity tells one sandbox's story, newest first", async () => { - await client().acquireSandbox('obs-story-key'); - await client().destroySandbox('obs-story-key'); - const events = await client().listActivity({ limit: 500 }); - const mine = events.filter((e) => e.sandboxName === 'obs-story-key'); - expect(mine.map((e) => e.kind)).toEqual(['destroyed', 'created']); - expect(mine[1]?.detail).toContain('acquireSandbox'); - // Attribution: this suite runs on the env token, and the events say so. - expect(mine.map((e) => e.actor)).toEqual(['env-token', 'env-token']); - }); - it('getSandboxMetricsHistory fills up as the sampler ticks, and 404s after destroy', async () => { await client().acquireSandbox('obs-history-key'); // The exam daemon samples every second — wait for the first row to diff --git a/packages/console/messages/de/activity.json b/packages/console/messages/de/activity.json deleted file mode 100644 index 7134f3b4..00000000 --- a/packages/console/messages/de/activity.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://inlang.com/schema/inlang-message-format", - "activity_kind_created": "Erstellt", - "activity_kind_woken": "Aufgeweckt", - "activity_kind_frozen": "Eingefroren", - "activity_kind_stopped": "Gestoppt", - "activity_kind_rebuilt": "Neu aufgebaut", - "activity_kind_destroyed": "Zerstört", - "activity_kind_expired_killed": "Abgelaufen", - "activity_kind_archived": "Archiviert", - "activity_kind_restore_started": "Wiederherstellung gestartet", - "activity_kind_restored": "Wiederhergestellt", - "activity_kind_restore_failed": "Wiederherstellung fehlgeschlagen", - "activity_kind_reconciled": "Abgeglichen", - "activity_kind_policy_changed": "Richtlinie geändert", - "activity_kind_metadata_changed": "Labels geändert", - "activity_kind_spec_changed": "Spezifikation geändert", - "activity_kind_template_changed": "Template geändert", - "activity_kind_disk_expanded": "Datenträger erweitert", - "activity_kind_host_disk_grown": "Datendisk automatisch erweitert", - "activity_kind_daemon_started": "Daemon gestartet", - "activity_kind_ingress_updated": "Domain aktualisiert", - "activity_kind_settings_updated": "Einstellungen geändert", - "activity_kind_apikey_created": "Schlüssel erstellt", - "activity_kind_apikey_updated": "Schlüssel geändert", - "activity_kind_apikey_disabled": "Schlüssel deaktiviert", - "activity_kind_apikey_enabled": "Schlüssel aktiviert", - "activity_kind_apikey_revoked": "Schlüssel widerrufen", - "activity_kind_upgrade_started": "Upgrade gestartet", - "activity_actor_system": "System", - "activity_actor_env_token": "Bootstrap-Token", - "activity_actor_console": "Konsole", - "activity_actor_apikey": "Schlüssel {name}", - "activity_page_title": "Aktivität", - "activity_search_placeholder": "Nach Name suchen", - "activity_filter_kind": "Ereignis", - "activity_filter_actor": "Akteur", - "activity_count": "{filtered} / {total}", - "activity_loading": "Aktivität wird geladen", - "activity_load_failed": "Laden fehlgeschlagen", - "activity_empty_title": "Noch keine Aktivität", - "activity_empty_description": "Erstellen Sie eine Sandbox, und ihr ganzer Lebenszyklus erscheint hier.", - "activity_no_match_title": "Keine passenden Ereignisse", - "activity_no_match_description": "Versuchen Sie ein anderes Stichwort oder einen anderen Ereignistyp.", - "activity_col_time": "Zeit", - "activity_col_kind": "Ereignis", - "activity_col_name": "Name", - "activity_col_actor": "Akteur", - "activity_col_detail": "Details" -} diff --git a/packages/console/messages/de/shell.json b/packages/console/messages/de/shell.json index 6837e0c5..1d46cdb4 100644 --- a/packages/console/messages/de/shell.json +++ b/packages/console/messages/de/shell.json @@ -6,7 +6,6 @@ "shell_nav_dashboard": "Dashboard", "shell_nav_sandboxes": "Sandboxes", "shell_nav_templates": "Vorlagen", - "shell_nav_activity": "Aktivität", "shell_nav_api_keys": "API-Schlüssel", "shell_nav_domains": "Domains", "shell_nav_doctor": "Doctor", diff --git a/packages/console/messages/de/workbench.json b/packages/console/messages/de/workbench.json index 6375305a..fda37fb0 100644 --- a/packages/console/messages/de/workbench.json +++ b/packages/console/messages/de/workbench.json @@ -60,9 +60,6 @@ "workbench_process_actions": "Aktionen für Prozess {pid}", "workbench_send_sigterm": "SIGTERM senden", "workbench_send_sigkill": "SIGKILL senden", - "workbench_recent_activity": "Letzte Aktivität", - "workbench_view_all": "Alle anzeigen", - "workbench_activity_empty": "Keine jüngsten Ereignisse zu dieser Sandbox — der Aktivitätsring behält nur die letzten 1000 globalen Einträge.", "workbench_sandbox_info": "Sandbox-Info", "workbench_copy_id": "Sandbox-ID kopieren", "workbench_copied_id": "Sandbox-ID kopiert", diff --git a/packages/console/messages/en/activity.json b/packages/console/messages/en/activity.json deleted file mode 100644 index 07c2f577..00000000 --- a/packages/console/messages/en/activity.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://inlang.com/schema/inlang-message-format", - "activity_kind_created": "Created", - "activity_kind_woken": "Woken", - "activity_kind_frozen": "Frozen", - "activity_kind_stopped": "Stopped", - "activity_kind_rebuilt": "Rebuilt", - "activity_kind_destroyed": "Destroyed", - "activity_kind_expired_killed": "Expired", - "activity_kind_archived": "Archived", - "activity_kind_restore_started": "Restore started", - "activity_kind_restored": "Restored", - "activity_kind_restore_failed": "Restore failed", - "activity_kind_reconciled": "Reconciled", - "activity_kind_policy_changed": "Policy changed", - "activity_kind_metadata_changed": "Labels changed", - "activity_kind_spec_changed": "Spec changed", - "activity_kind_template_changed": "Template changed", - "activity_kind_disk_expanded": "Disk expanded", - "activity_kind_host_disk_grown": "Data disk auto-grown", - "activity_kind_daemon_started": "Daemon started", - "activity_kind_ingress_updated": "Domain updated", - "activity_kind_settings_updated": "Settings changed", - "activity_kind_apikey_created": "Key created", - "activity_kind_apikey_updated": "Key updated", - "activity_kind_apikey_disabled": "Key disabled", - "activity_kind_apikey_enabled": "Key enabled", - "activity_kind_apikey_revoked": "Key revoked", - "activity_kind_upgrade_started": "Upgrade started", - "activity_actor_system": "System", - "activity_actor_env_token": "Bootstrap token", - "activity_actor_console": "Console", - "activity_actor_apikey": "Key {name}", - "activity_page_title": "Activity", - "activity_search_placeholder": "Search by name", - "activity_filter_kind": "Event", - "activity_filter_actor": "Actor", - "activity_count": "{filtered} / {total}", - "activity_loading": "Loading activity", - "activity_load_failed": "Load failed", - "activity_empty_title": "No activity yet", - "activity_empty_description": "Create a sandbox and its whole lifecycle will show up here.", - "activity_no_match_title": "No matching events", - "activity_no_match_description": "Try a different keyword or event type.", - "activity_col_time": "Time", - "activity_col_kind": "Event", - "activity_col_name": "Name", - "activity_col_actor": "Actor", - "activity_col_detail": "Detail" -} diff --git a/packages/console/messages/en/shell.json b/packages/console/messages/en/shell.json index c9bed3df..a830791a 100644 --- a/packages/console/messages/en/shell.json +++ b/packages/console/messages/en/shell.json @@ -6,7 +6,6 @@ "shell_nav_dashboard": "Dashboard", "shell_nav_sandboxes": "Sandboxes", "shell_nav_templates": "Templates", - "shell_nav_activity": "Activity", "shell_nav_api_keys": "API Keys", "shell_nav_domains": "Domains", "shell_nav_doctor": "Doctor", diff --git a/packages/console/messages/en/workbench.json b/packages/console/messages/en/workbench.json index f577bd1d..47b44755 100644 --- a/packages/console/messages/en/workbench.json +++ b/packages/console/messages/en/workbench.json @@ -60,9 +60,6 @@ "workbench_process_actions": "Actions for process {pid}", "workbench_send_sigterm": "Send SIGTERM", "workbench_send_sigkill": "Send SIGKILL", - "workbench_recent_activity": "Recent activity", - "workbench_view_all": "View all", - "workbench_activity_empty": "No recent events for this sandbox — the activity ring keeps only the latest 1000 global events.", "workbench_sandbox_info": "Sandbox info", "workbench_copy_id": "Copy sandbox ID", "workbench_copied_id": "Sandbox ID copied", diff --git a/packages/console/messages/es/activity.json b/packages/console/messages/es/activity.json deleted file mode 100644 index 23ed92fc..00000000 --- a/packages/console/messages/es/activity.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://inlang.com/schema/inlang-message-format", - "activity_kind_created": "Creado", - "activity_kind_woken": "Reactivado", - "activity_kind_frozen": "Congelado", - "activity_kind_stopped": "Detenido", - "activity_kind_rebuilt": "Reconstruido", - "activity_kind_destroyed": "Destruido", - "activity_kind_expired_killed": "Vencido", - "activity_kind_archived": "Archivado", - "activity_kind_restore_started": "Restauración iniciada", - "activity_kind_restored": "Restaurado", - "activity_kind_restore_failed": "Error al restaurar", - "activity_kind_reconciled": "Reconciliado", - "activity_kind_policy_changed": "Política modificada", - "activity_kind_metadata_changed": "Etiquetas modificadas", - "activity_kind_spec_changed": "Especificación modificada", - "activity_kind_template_changed": "Plantilla cambiada", - "activity_kind_disk_expanded": "Disco ampliado", - "activity_kind_host_disk_grown": "Disco de datos ampliado automáticamente", - "activity_kind_daemon_started": "Daemon iniciado", - "activity_kind_ingress_updated": "Dominio actualizado", - "activity_kind_settings_updated": "Ajustes modificados", - "activity_kind_apikey_created": "Clave creada", - "activity_kind_apikey_updated": "Clave modificada", - "activity_kind_apikey_disabled": "Clave desactivada", - "activity_kind_apikey_enabled": "Clave activada", - "activity_kind_apikey_revoked": "Clave revocada", - "activity_kind_upgrade_started": "Actualización iniciada", - "activity_actor_system": "Sistema", - "activity_actor_env_token": "Token de arranque", - "activity_actor_console": "Consola", - "activity_actor_apikey": "Clave {name}", - "activity_page_title": "Actividad", - "activity_search_placeholder": "Buscar por nombre", - "activity_filter_kind": "Evento", - "activity_filter_actor": "Actor", - "activity_count": "{filtered} / {total}", - "activity_loading": "Cargando actividad", - "activity_load_failed": "Error al cargar", - "activity_empty_title": "Aún no hay actividad", - "activity_empty_description": "Crea un sandbox y todo su ciclo de vida aparecerá aquí.", - "activity_no_match_title": "Sin eventos coincidentes", - "activity_no_match_description": "Prueba con otra palabra clave o tipo de evento.", - "activity_col_time": "Hora", - "activity_col_kind": "Evento", - "activity_col_name": "Nombre", - "activity_col_actor": "Actor", - "activity_col_detail": "Detalle" -} diff --git a/packages/console/messages/es/shell.json b/packages/console/messages/es/shell.json index 02dd30e7..f178eae3 100644 --- a/packages/console/messages/es/shell.json +++ b/packages/console/messages/es/shell.json @@ -6,7 +6,6 @@ "shell_nav_dashboard": "Panel", "shell_nav_sandboxes": "Sandboxes", "shell_nav_templates": "Plantillas", - "shell_nav_activity": "Actividad", "shell_nav_api_keys": "Claves de API", "shell_nav_domains": "Dominios", "shell_nav_doctor": "Doctor", diff --git a/packages/console/messages/es/workbench.json b/packages/console/messages/es/workbench.json index 678991da..ad7ff1c7 100644 --- a/packages/console/messages/es/workbench.json +++ b/packages/console/messages/es/workbench.json @@ -60,9 +60,6 @@ "workbench_process_actions": "Acciones para el proceso {pid}", "workbench_send_sigterm": "Enviar SIGTERM", "workbench_send_sigkill": "Enviar SIGKILL", - "workbench_recent_activity": "Actividad reciente", - "workbench_view_all": "Ver todo", - "workbench_activity_empty": "No hay eventos recientes de este sandbox — el anillo de actividad guarda solo los últimos 1000 eventos globales.", "workbench_sandbox_info": "Información del sandbox", "workbench_copy_id": "Copiar el ID del sandbox", "workbench_copied_id": "ID del sandbox copiado", diff --git a/packages/console/messages/fr/activity.json b/packages/console/messages/fr/activity.json deleted file mode 100644 index 69c2a4ab..00000000 --- a/packages/console/messages/fr/activity.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://inlang.com/schema/inlang-message-format", - "activity_kind_created": "Création", - "activity_kind_woken": "Réveil", - "activity_kind_frozen": "Gel", - "activity_kind_stopped": "Arrêt", - "activity_kind_rebuilt": "Reconstruction", - "activity_kind_destroyed": "Destruction", - "activity_kind_expired_killed": "Expiration", - "activity_kind_archived": "Archivage", - "activity_kind_restore_started": "Restauration lancée", - "activity_kind_restored": "Restauration terminée", - "activity_kind_restore_failed": "Échec de restauration", - "activity_kind_reconciled": "Réconciliation", - "activity_kind_policy_changed": "Politique modifiée", - "activity_kind_metadata_changed": "Étiquettes modifiées", - "activity_kind_spec_changed": "Spécification modifiée", - "activity_kind_template_changed": "Modèle modifié", - "activity_kind_disk_expanded": "Disque agrandi", - "activity_kind_host_disk_grown": "Disque de données agrandi automatiquement", - "activity_kind_daemon_started": "Daemon démarré", - "activity_kind_ingress_updated": "Domaine mis à jour", - "activity_kind_settings_updated": "Paramètres modifiés", - "activity_kind_apikey_created": "Clé créée", - "activity_kind_apikey_updated": "Clé modifiée", - "activity_kind_apikey_disabled": "Clé désactivée", - "activity_kind_apikey_enabled": "Clé activée", - "activity_kind_apikey_revoked": "Clé révoquée", - "activity_kind_upgrade_started": "Mise à niveau lancée", - "activity_actor_system": "Système", - "activity_actor_env_token": "Token d'amorçage", - "activity_actor_console": "Console", - "activity_actor_apikey": "Clé {name}", - "activity_page_title": "Activité", - "activity_search_placeholder": "Rechercher par nom", - "activity_filter_kind": "Événement", - "activity_filter_actor": "Acteur", - "activity_count": "{filtered} / {total}", - "activity_loading": "Chargement de l'activité", - "activity_load_failed": "Échec du chargement", - "activity_empty_title": "Aucune activité pour l'instant", - "activity_empty_description": "Créez une sandbox et tout son cycle de vie apparaîtra ici.", - "activity_no_match_title": "Aucun événement correspondant", - "activity_no_match_description": "Essayez un autre mot-clé ou type d'événement.", - "activity_col_time": "Heure", - "activity_col_kind": "Événement", - "activity_col_name": "Nom", - "activity_col_actor": "Acteur", - "activity_col_detail": "Détail" -} diff --git a/packages/console/messages/fr/shell.json b/packages/console/messages/fr/shell.json index ec0eb5f4..b4ab5557 100644 --- a/packages/console/messages/fr/shell.json +++ b/packages/console/messages/fr/shell.json @@ -6,7 +6,6 @@ "shell_nav_dashboard": "Tableau de bord", "shell_nav_sandboxes": "Sandbox", "shell_nav_templates": "Modèles", - "shell_nav_activity": "Activité", "shell_nav_api_keys": "Clés API", "shell_nav_domains": "Domaines", "shell_nav_doctor": "Doctor", diff --git a/packages/console/messages/fr/workbench.json b/packages/console/messages/fr/workbench.json index 9cc5c667..cc9d61a3 100644 --- a/packages/console/messages/fr/workbench.json +++ b/packages/console/messages/fr/workbench.json @@ -60,9 +60,6 @@ "workbench_process_actions": "Actions pour le processus {pid}", "workbench_send_sigterm": "Envoyer SIGTERM", "workbench_send_sigkill": "Envoyer SIGKILL", - "workbench_recent_activity": "Activité récente", - "workbench_view_all": "Tout voir", - "workbench_activity_empty": "Aucun événement récent pour cette sandbox — le journal d'activité ne conserve que les 1000 derniers événements globaux.", "workbench_sandbox_info": "Infos sandbox", "workbench_copy_id": "Copier l'ID de la sandbox", "workbench_copied_id": "ID de la sandbox copié", diff --git a/packages/console/messages/ja/activity.json b/packages/console/messages/ja/activity.json deleted file mode 100644 index b281b3a5..00000000 --- a/packages/console/messages/ja/activity.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://inlang.com/schema/inlang-message-format", - "activity_kind_created": "作成", - "activity_kind_woken": "ウェイクアップ", - "activity_kind_frozen": "凍結", - "activity_kind_stopped": "停止", - "activity_kind_rebuilt": "再構築", - "activity_kind_destroyed": "破棄", - "activity_kind_expired_killed": "期限切れ破棄", - "activity_kind_archived": "アーカイブ", - "activity_kind_restore_started": "復元開始", - "activity_kind_restored": "復元完了", - "activity_kind_restore_failed": "復元失敗", - "activity_kind_reconciled": "整合性修復", - "activity_kind_policy_changed": "ポリシー変更", - "activity_kind_metadata_changed": "ラベル変更", - "activity_kind_spec_changed": "スペック変更", - "activity_kind_template_changed": "テンプレート変更", - "activity_kind_disk_expanded": "ディスク拡張", - "activity_kind_host_disk_grown": "データディスク自動拡張", - "activity_kind_daemon_started": "daemon 起動", - "activity_kind_ingress_updated": "ドメイン更新", - "activity_kind_settings_updated": "設定変更", - "activity_kind_apikey_created": "キー作成", - "activity_kind_apikey_updated": "キー更新", - "activity_kind_apikey_disabled": "キー無効化", - "activity_kind_apikey_enabled": "キー有効化", - "activity_kind_apikey_revoked": "キー取り消し", - "activity_kind_upgrade_started": "アップグレード開始", - "activity_actor_system": "システム", - "activity_actor_env_token": "ブートストラップ認証情報", - "activity_actor_console": "コンソール", - "activity_actor_apikey": "キー {name}", - "activity_page_title": "アクティビティ", - "activity_search_placeholder": "名前で検索", - "activity_filter_kind": "イベント", - "activity_filter_actor": "実行者", - "activity_count": "{filtered} / {total} 件", - "activity_loading": "アクティビティを読み込み中", - "activity_load_failed": "読み込みに失敗しました", - "activity_empty_title": "アクティビティはまだありません", - "activity_empty_description": "サンドボックスを作成すると、そのライフサイクル全体がここに表示されます。", - "activity_no_match_title": "一致するイベントがありません", - "activity_no_match_description": "キーワードやイベント種別を変えてお試しください。", - "activity_col_time": "時刻", - "activity_col_kind": "イベント", - "activity_col_name": "名前", - "activity_col_actor": "実行者", - "activity_col_detail": "詳細" -} diff --git a/packages/console/messages/ja/shell.json b/packages/console/messages/ja/shell.json index e6a62869..a41402ad 100644 --- a/packages/console/messages/ja/shell.json +++ b/packages/console/messages/ja/shell.json @@ -6,7 +6,6 @@ "shell_nav_dashboard": "ダッシュボード", "shell_nav_sandboxes": "サンドボックス", "shell_nav_templates": "テンプレート", - "shell_nav_activity": "アクティビティ", "shell_nav_api_keys": "API キー", "shell_nav_domains": "ドメイン", "shell_nav_doctor": "診断", diff --git a/packages/console/messages/ja/workbench.json b/packages/console/messages/ja/workbench.json index da4b1c23..203d6390 100644 --- a/packages/console/messages/ja/workbench.json +++ b/packages/console/messages/ja/workbench.json @@ -60,9 +60,6 @@ "workbench_process_actions": "プロセス {pid} の操作", "workbench_send_sigterm": "SIGTERM を送信", "workbench_send_sigkill": "SIGKILL を送信", - "workbench_recent_activity": "最近のアクティビティ", - "workbench_view_all": "すべて表示", - "workbench_activity_empty": "このサンドボックスの最近のイベントはありません — アクティビティリングは全体で直近 1000 件のみ保持します。", "workbench_sandbox_info": "サンドボックス情報", "workbench_copy_id": "サンドボックス ID をコピー", "workbench_copied_id": "サンドボックス ID をコピーしました", diff --git a/packages/console/messages/ko/activity.json b/packages/console/messages/ko/activity.json deleted file mode 100644 index 7647ddb0..00000000 --- a/packages/console/messages/ko/activity.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://inlang.com/schema/inlang-message-format", - "activity_kind_created": "생성", - "activity_kind_woken": "깨움", - "activity_kind_frozen": "동결", - "activity_kind_stopped": "중지", - "activity_kind_rebuilt": "재구축", - "activity_kind_destroyed": "삭제", - "activity_kind_expired_killed": "만료 삭제", - "activity_kind_archived": "아카이브", - "activity_kind_restore_started": "복원 시작", - "activity_kind_restored": "복원 완료", - "activity_kind_restore_failed": "복원 실패", - "activity_kind_reconciled": "정합 복구", - "activity_kind_policy_changed": "정책 변경", - "activity_kind_metadata_changed": "레이블 변경", - "activity_kind_spec_changed": "사양 변경", - "activity_kind_template_changed": "템플릿 변경", - "activity_kind_disk_expanded": "디스크 확장", - "activity_kind_host_disk_grown": "데이터 디스크 자동 확장", - "activity_kind_daemon_started": "daemon 시작", - "activity_kind_ingress_updated": "도메인 변경", - "activity_kind_settings_updated": "설정 변경", - "activity_kind_apikey_created": "키 생성", - "activity_kind_apikey_updated": "키 변경", - "activity_kind_apikey_disabled": "키 비활성화", - "activity_kind_apikey_enabled": "키 활성화", - "activity_kind_apikey_revoked": "키 폐기", - "activity_kind_upgrade_started": "업그레이드 시작", - "activity_actor_system": "시스템", - "activity_actor_env_token": "부트스트랩 토큰", - "activity_actor_console": "콘솔", - "activity_actor_apikey": "키 {name}", - "activity_page_title": "활동", - "activity_search_placeholder": "이름으로 검색", - "activity_filter_kind": "이벤트", - "activity_filter_actor": "수행자", - "activity_count": "{filtered} / {total}건", - "activity_loading": "활동 읽는 중", - "activity_load_failed": "읽기 실패", - "activity_empty_title": "아직 활동이 없습니다", - "activity_empty_description": "샌드박스를 하나 만들면 그 수명 주기 전체가 여기에 나타납니다.", - "activity_no_match_title": "일치하는 이벤트가 없습니다", - "activity_no_match_description": "다른 키워드나 이벤트 유형으로 시도해 보세요.", - "activity_col_time": "시간", - "activity_col_kind": "이벤트", - "activity_col_name": "이름", - "activity_col_actor": "수행자", - "activity_col_detail": "상세" -} diff --git a/packages/console/messages/ko/shell.json b/packages/console/messages/ko/shell.json index 05f92484..ace12051 100644 --- a/packages/console/messages/ko/shell.json +++ b/packages/console/messages/ko/shell.json @@ -6,7 +6,6 @@ "shell_nav_dashboard": "대시보드", "shell_nav_sandboxes": "샌드박스", "shell_nav_templates": "템플릿", - "shell_nav_activity": "활동", "shell_nav_api_keys": "API 키", "shell_nav_domains": "도메인", "shell_nav_doctor": "진단", diff --git a/packages/console/messages/ko/workbench.json b/packages/console/messages/ko/workbench.json index 1040d222..280d78c2 100644 --- a/packages/console/messages/ko/workbench.json +++ b/packages/console/messages/ko/workbench.json @@ -60,9 +60,6 @@ "workbench_process_actions": "프로세스 {pid}의 작업", "workbench_send_sigterm": "SIGTERM 전송", "workbench_send_sigkill": "SIGKILL 전송", - "workbench_recent_activity": "최근 활동", - "workbench_view_all": "전체 보기", - "workbench_activity_empty": "최근 이 샌드박스의 이벤트가 없습니다 — 활동 링은 전역 최근 1000건만 보관합니다.", "workbench_sandbox_info": "샌드박스 정보", "workbench_copy_id": "샌드박스 ID 복사", "workbench_copied_id": "샌드박스 ID 복사됨", diff --git a/packages/console/messages/pt-BR/activity.json b/packages/console/messages/pt-BR/activity.json deleted file mode 100644 index cfb0ea24..00000000 --- a/packages/console/messages/pt-BR/activity.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://inlang.com/schema/inlang-message-format", - "activity_kind_created": "Criado", - "activity_kind_woken": "Acordado", - "activity_kind_frozen": "Congelado", - "activity_kind_stopped": "Parado", - "activity_kind_rebuilt": "Reconstruído", - "activity_kind_destroyed": "Destruído", - "activity_kind_expired_killed": "Expirado", - "activity_kind_archived": "Arquivado", - "activity_kind_restore_started": "Restauração iniciada", - "activity_kind_restored": "Restaurado", - "activity_kind_restore_failed": "Falha na restauração", - "activity_kind_reconciled": "Reconciliado", - "activity_kind_policy_changed": "Política alterada", - "activity_kind_metadata_changed": "Rótulos alterados", - "activity_kind_spec_changed": "Especificação alterada", - "activity_kind_template_changed": "Modelo alterado", - "activity_kind_disk_expanded": "Disco expandido", - "activity_kind_host_disk_grown": "Disco de dados expandido automaticamente", - "activity_kind_daemon_started": "Daemon iniciado", - "activity_kind_ingress_updated": "Domínio atualizado", - "activity_kind_settings_updated": "Configurações alteradas", - "activity_kind_apikey_created": "Chave criada", - "activity_kind_apikey_updated": "Chave atualizada", - "activity_kind_apikey_disabled": "Chave desativada", - "activity_kind_apikey_enabled": "Chave ativada", - "activity_kind_apikey_revoked": "Chave revogada", - "activity_kind_upgrade_started": "Atualização iniciada", - "activity_actor_system": "Sistema", - "activity_actor_env_token": "Token de bootstrap", - "activity_actor_console": "Console", - "activity_actor_apikey": "Chave {name}", - "activity_page_title": "Atividade", - "activity_search_placeholder": "Buscar por nome", - "activity_filter_kind": "Evento", - "activity_filter_actor": "Autor", - "activity_count": "{filtered} / {total}", - "activity_loading": "Carregando atividade", - "activity_load_failed": "Falha ao carregar", - "activity_empty_title": "Nenhuma atividade ainda", - "activity_empty_description": "Crie um sandbox e todo o ciclo de vida dele aparecerá aqui.", - "activity_no_match_title": "Nenhum evento corresponde", - "activity_no_match_description": "Tente outra palavra-chave ou tipo de evento.", - "activity_col_time": "Hora", - "activity_col_kind": "Evento", - "activity_col_name": "Nome", - "activity_col_actor": "Autor", - "activity_col_detail": "Detalhes" -} diff --git a/packages/console/messages/pt-BR/shell.json b/packages/console/messages/pt-BR/shell.json index 8656e1eb..6072febf 100644 --- a/packages/console/messages/pt-BR/shell.json +++ b/packages/console/messages/pt-BR/shell.json @@ -6,7 +6,6 @@ "shell_nav_dashboard": "Painel", "shell_nav_sandboxes": "Sandboxes", "shell_nav_templates": "Templates", - "shell_nav_activity": "Atividade", "shell_nav_api_keys": "Chaves de API", "shell_nav_domains": "Domínios", "shell_nav_doctor": "Diagnóstico", diff --git a/packages/console/messages/pt-BR/workbench.json b/packages/console/messages/pt-BR/workbench.json index a8118f4d..5de6c17e 100644 --- a/packages/console/messages/pt-BR/workbench.json +++ b/packages/console/messages/pt-BR/workbench.json @@ -60,9 +60,6 @@ "workbench_process_actions": "Ações do processo {pid}", "workbench_send_sigterm": "Enviar SIGTERM", "workbench_send_sigkill": "Enviar SIGKILL", - "workbench_recent_activity": "Atividade recente", - "workbench_view_all": "Ver tudo", - "workbench_activity_empty": "Nenhum evento recente deste sandbox — o anel de atividade guarda apenas os últimos 1000 eventos globais.", "workbench_sandbox_info": "Informações do sandbox", "workbench_copy_id": "Copiar ID do sandbox", "workbench_copied_id": "ID do sandbox copiado", diff --git a/packages/console/messages/ru/activity.json b/packages/console/messages/ru/activity.json deleted file mode 100644 index 9ea7b6eb..00000000 --- a/packages/console/messages/ru/activity.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://inlang.com/schema/inlang-message-format", - "activity_kind_created": "Создание", - "activity_kind_woken": "Пробуждение", - "activity_kind_frozen": "Заморозка", - "activity_kind_stopped": "Остановка", - "activity_kind_rebuilt": "Пересборка", - "activity_kind_destroyed": "Уничтожение", - "activity_kind_expired_killed": "Уничтожение по сроку", - "activity_kind_archived": "Архивация", - "activity_kind_restore_started": "Начало восстановления", - "activity_kind_restored": "Восстановление завершено", - "activity_kind_restore_failed": "Сбой восстановления", - "activity_kind_reconciled": "Сверка", - "activity_kind_policy_changed": "Изменение политики", - "activity_kind_metadata_changed": "Изменение меток", - "activity_kind_spec_changed": "Изменение спецификации", - "activity_kind_template_changed": "Изменение шаблона", - "activity_kind_disk_expanded": "Расширение диска", - "activity_kind_host_disk_grown": "Авторасширение диска данных", - "activity_kind_daemon_started": "Запуск daemon", - "activity_kind_ingress_updated": "Настройка домена", - "activity_kind_settings_updated": "Изменение настроек", - "activity_kind_apikey_created": "Создание ключа", - "activity_kind_apikey_updated": "Изменение ключа", - "activity_kind_apikey_disabled": "Отключение ключа", - "activity_kind_apikey_enabled": "Включение ключа", - "activity_kind_apikey_revoked": "Отзыв ключа", - "activity_kind_upgrade_started": "Запуск обновления", - "activity_actor_system": "Система", - "activity_actor_env_token": "Bootstrap-токен", - "activity_actor_console": "Консоль", - "activity_actor_apikey": "Ключ {name}", - "activity_page_title": "Активность", - "activity_search_placeholder": "Поиск по имени", - "activity_filter_kind": "Событие", - "activity_filter_actor": "Инициатор", - "activity_count": "{filtered} / {total}", - "activity_loading": "Загрузка активности", - "activity_load_failed": "Не удалось загрузить", - "activity_empty_title": "Активности пока нет", - "activity_empty_description": "Создайте песочницу — и весь её жизненный цикл появится здесь.", - "activity_no_match_title": "Нет подходящих событий", - "activity_no_match_description": "Попробуйте другое ключевое слово или тип события.", - "activity_col_time": "Время", - "activity_col_kind": "Событие", - "activity_col_name": "Имя", - "activity_col_actor": "Инициатор", - "activity_col_detail": "Детали" -} diff --git a/packages/console/messages/ru/shell.json b/packages/console/messages/ru/shell.json index 8e40f8e2..2dc6d702 100644 --- a/packages/console/messages/ru/shell.json +++ b/packages/console/messages/ru/shell.json @@ -6,7 +6,6 @@ "shell_nav_dashboard": "Дашборд", "shell_nav_sandboxes": "Песочницы", "shell_nav_templates": "Шаблоны", - "shell_nav_activity": "Активность", "shell_nav_api_keys": "API-ключи", "shell_nav_domains": "Домены", "shell_nav_doctor": "Диагностика", diff --git a/packages/console/messages/ru/workbench.json b/packages/console/messages/ru/workbench.json index b1532ac0..4b7893a3 100644 --- a/packages/console/messages/ru/workbench.json +++ b/packages/console/messages/ru/workbench.json @@ -60,9 +60,6 @@ "workbench_process_actions": "Действия для процесса {pid}", "workbench_send_sigterm": "Отправить SIGTERM", "workbench_send_sigkill": "Отправить SIGKILL", - "workbench_recent_activity": "Последняя активность", - "workbench_view_all": "Показать все", - "workbench_activity_empty": "Недавних событий этой песочницы нет — кольцо активности хранит только 1000 последних глобальных событий.", "workbench_sandbox_info": "О песочнице", "workbench_copy_id": "Копировать ID песочницы", "workbench_copied_id": "ID песочницы скопирован", diff --git a/packages/console/messages/zh-CN/activity.json b/packages/console/messages/zh-CN/activity.json deleted file mode 100644 index 0747d728..00000000 --- a/packages/console/messages/zh-CN/activity.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://inlang.com/schema/inlang-message-format", - "activity_kind_created": "创建", - "activity_kind_woken": "唤醒", - "activity_kind_frozen": "冻结", - "activity_kind_stopped": "停止", - "activity_kind_rebuilt": "重建", - "activity_kind_destroyed": "销毁", - "activity_kind_expired_killed": "到期销毁", - "activity_kind_archived": "归档", - "activity_kind_restore_started": "开始恢复", - "activity_kind_restored": "恢复完成", - "activity_kind_restore_failed": "恢复失败", - "activity_kind_reconciled": "对账修复", - "activity_kind_policy_changed": "策略调整", - "activity_kind_metadata_changed": "标签调整", - "activity_kind_spec_changed": "规格调整", - "activity_kind_template_changed": "模板切换", - "activity_kind_disk_expanded": "磁盘扩容", - "activity_kind_host_disk_grown": "数据盘自动扩容", - "activity_kind_daemon_started": "daemon 启动", - "activity_kind_ingress_updated": "域名配置", - "activity_kind_settings_updated": "设置调整", - "activity_kind_apikey_created": "密钥创建", - "activity_kind_apikey_updated": "密钥调整", - "activity_kind_apikey_disabled": "密钥停用", - "activity_kind_apikey_enabled": "密钥启用", - "activity_kind_apikey_revoked": "密钥吊销", - "activity_kind_upgrade_started": "发起升级", - "activity_actor_system": "系统", - "activity_actor_env_token": "引导凭证", - "activity_actor_console": "控制台", - "activity_actor_apikey": "密钥 {name}", - "activity_page_title": "活动", - "activity_search_placeholder": "按名称搜索", - "activity_filter_kind": "事件", - "activity_filter_actor": "操作者", - "activity_count": "{filtered} / {total} 条", - "activity_loading": "读取活动", - "activity_load_failed": "读取失败", - "activity_empty_title": "还没有活动", - "activity_empty_description": "创建一个沙箱,它的整个生命周期就会出现在这里。", - "activity_no_match_title": "没有匹配的事件", - "activity_no_match_description": "换个关键词或事件类型试试。", - "activity_col_time": "时间", - "activity_col_kind": "事件", - "activity_col_name": "名称", - "activity_col_actor": "操作者", - "activity_col_detail": "详情" -} diff --git a/packages/console/messages/zh-CN/shell.json b/packages/console/messages/zh-CN/shell.json index aeead08b..4d70037d 100644 --- a/packages/console/messages/zh-CN/shell.json +++ b/packages/console/messages/zh-CN/shell.json @@ -6,7 +6,6 @@ "shell_nav_dashboard": "仪表盘", "shell_nav_sandboxes": "沙箱", "shell_nav_templates": "模板", - "shell_nav_activity": "活动", "shell_nav_api_keys": "API 密钥", "shell_nav_domains": "域名", "shell_nav_doctor": "体检", diff --git a/packages/console/messages/zh-CN/workbench.json b/packages/console/messages/zh-CN/workbench.json index b57d82ab..dd7f1bdf 100644 --- a/packages/console/messages/zh-CN/workbench.json +++ b/packages/console/messages/zh-CN/workbench.json @@ -60,9 +60,6 @@ "workbench_process_actions": "进程 {pid} 的操作", "workbench_send_sigterm": "发送 SIGTERM", "workbench_send_sigkill": "发送 SIGKILL", - "workbench_recent_activity": "最近活动", - "workbench_view_all": "查看全部", - "workbench_activity_empty": "最近没有这个沙箱的事件 — 活动环只保留全局最近 1000 条。", "workbench_sandbox_info": "沙箱信息", "workbench_copy_id": "复制沙箱 ID", "workbench_copied_id": "已复制沙箱 ID", diff --git a/packages/console/messages/zh-TW/activity.json b/packages/console/messages/zh-TW/activity.json deleted file mode 100644 index 34739b4f..00000000 --- a/packages/console/messages/zh-TW/activity.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://inlang.com/schema/inlang-message-format", - "activity_kind_created": "建立", - "activity_kind_woken": "喚醒", - "activity_kind_frozen": "凍結", - "activity_kind_stopped": "停止", - "activity_kind_rebuilt": "重建", - "activity_kind_destroyed": "銷毀", - "activity_kind_expired_killed": "到期銷毀", - "activity_kind_archived": "封存", - "activity_kind_restore_started": "開始還原", - "activity_kind_restored": "還原完成", - "activity_kind_restore_failed": "還原失敗", - "activity_kind_reconciled": "對帳修復", - "activity_kind_policy_changed": "策略調整", - "activity_kind_metadata_changed": "標籤調整", - "activity_kind_spec_changed": "規格調整", - "activity_kind_template_changed": "模板切換", - "activity_kind_disk_expanded": "磁碟擴容", - "activity_kind_host_disk_grown": "資料碟自動擴容", - "activity_kind_daemon_started": "daemon 啟動", - "activity_kind_ingress_updated": "網域設定", - "activity_kind_settings_updated": "設定調整", - "activity_kind_apikey_created": "金鑰建立", - "activity_kind_apikey_updated": "金鑰調整", - "activity_kind_apikey_disabled": "金鑰停用", - "activity_kind_apikey_enabled": "金鑰啟用", - "activity_kind_apikey_revoked": "金鑰吊銷", - "activity_kind_upgrade_started": "發起升級", - "activity_actor_system": "系統", - "activity_actor_env_token": "引導憑證", - "activity_actor_console": "主控台", - "activity_actor_apikey": "金鑰 {name}", - "activity_page_title": "活動", - "activity_search_placeholder": "依名稱搜尋", - "activity_filter_kind": "事件", - "activity_filter_actor": "操作者", - "activity_count": "{filtered} / {total} 筆", - "activity_loading": "讀取活動", - "activity_load_failed": "讀取失敗", - "activity_empty_title": "還沒有活動", - "activity_empty_description": "建立一個沙箱,它的整個生命週期就會出現在這裡。", - "activity_no_match_title": "沒有符合的事件", - "activity_no_match_description": "換個關鍵字或事件類型試試。", - "activity_col_time": "時間", - "activity_col_kind": "事件", - "activity_col_name": "名稱", - "activity_col_actor": "操作者", - "activity_col_detail": "詳情" -} diff --git a/packages/console/messages/zh-TW/shell.json b/packages/console/messages/zh-TW/shell.json index 49c8d603..66fd135a 100644 --- a/packages/console/messages/zh-TW/shell.json +++ b/packages/console/messages/zh-TW/shell.json @@ -6,7 +6,6 @@ "shell_nav_dashboard": "儀表板", "shell_nav_sandboxes": "沙箱", "shell_nav_templates": "範本", - "shell_nav_activity": "活動", "shell_nav_api_keys": "API 金鑰", "shell_nav_domains": "網域", "shell_nav_doctor": "健檢", diff --git a/packages/console/messages/zh-TW/workbench.json b/packages/console/messages/zh-TW/workbench.json index 15bc0ad7..9a74ea2f 100644 --- a/packages/console/messages/zh-TW/workbench.json +++ b/packages/console/messages/zh-TW/workbench.json @@ -60,9 +60,6 @@ "workbench_process_actions": "程序 {pid} 的操作", "workbench_send_sigterm": "傳送 SIGTERM", "workbench_send_sigkill": "傳送 SIGKILL", - "workbench_recent_activity": "最近活動", - "workbench_view_all": "查看全部", - "workbench_activity_empty": "最近沒有這個沙箱的事件 — 活動環只保留全域最近 1000 筆。", "workbench_sandbox_info": "沙箱資訊", "workbench_copy_id": "複製沙箱 ID", "workbench_copied_id": "已複製沙箱 ID", diff --git a/packages/console/project.inlang/settings.json b/packages/console/project.inlang/settings.json index d58e5080..b491f50e 100644 --- a/packages/console/project.inlang/settings.json +++ b/packages/console/project.inlang/settings.json @@ -22,7 +22,6 @@ "./messages/{locale}/overview.json", "./messages/{locale}/sandboxes.json", "./messages/{locale}/workbench.json", - "./messages/{locale}/activity.json", "./messages/{locale}/apikeys.json", "./messages/{locale}/templates.json", "./messages/{locale}/domains.json", diff --git a/packages/console/src/components/nav.ts b/packages/console/src/components/nav.ts index 0ca56335..482e4004 100644 --- a/packages/console/src/components/nav.ts +++ b/packages/console/src/components/nav.ts @@ -1,5 +1,4 @@ import { - Activity01Icon, DashboardSquare01Icon, GitCommitIcon, Globe02Icon, @@ -45,7 +44,6 @@ export const NAV_GROUPS: Array<{ id: 'ops', label: m.shell_nav_group_ops, items: [ - { to: '/activity', label: m.shell_nav_activity, icon: Activity01Icon }, { to: '/api-keys', label: m.shell_nav_api_keys, icon: Key01Icon }, { to: '/domains', label: m.shell_nav_domains, icon: Globe02Icon }, { diff --git a/packages/console/src/features/activity/actors.ts b/packages/console/src/features/activity/actors.ts deleted file mode 100644 index 8ac1c7b9..00000000 --- a/packages/console/src/features/activity/actors.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { - type ApiKey, - apiKeyActorId, - CONSOLE_ACTOR, - ENV_TOKEN_ACTOR, -} from '@dormice/shared'; -import { m } from '@/paraglide/messages'; - -/** - * 操作者的显示翻译 — kinds.ts 是 kind 的唯一翻译点,这里是 actor 的。 - * 词表在 shared/activity.ts:'env-token' / 'console' / 'apikey:' / - * null(=daemon 自动作业:闲置扫描、对账、归档,或不带账本凭证的数据面 - * 唤醒)。key 按 id 归因(名字可改、id 才稳定),显示时经密钥列表翻译回 - * 当前名 — 吊销是软删除、行永不消失,所以 id 永远翻得回来;密钥列表还 - * 没到手时退回截断 id,不装懂。词表之外的字符串原样示人,不翻译也不吞。 - */ -export function actorLabel( - actor: string | null, - apiKeys: Pick[] | undefined, -): string { - if (actor === null) return m.activity_actor_system(); - if (actor === ENV_TOKEN_ACTOR) return m.activity_actor_env_token(); - if (actor === CONSOLE_ACTOR) return m.activity_actor_console(); - const keyId = apiKeyActorId(actor); - if (keyId !== null) { - const key = apiKeys?.find((k) => k.id === keyId); - return m.activity_actor_apikey({ - name: key ? key.name : keyId.slice(0, 8), - }); - } - return actor; -} diff --git a/packages/console/src/features/activity/hooks/useActivity.ts b/packages/console/src/features/activity/hooks/useActivity.ts deleted file mode 100644 index 89c714db..00000000 --- a/packages/console/src/features/activity/hooks/useActivity.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { useQuery } from '@tanstack/react-query'; -import { listActivity } from '@/lib/api'; - -/** - * 活动流是观察窗:5 秒轮询即可 — 事件写在动作发生处,读只是翻账本, - * 比沙箱列表的 2 秒松一档(历史不像状态那样急着看)。 - * - * limit 缺省用 wire 默认(200);详情页历史 tab 要按 name 过滤, - * 传环形表上限(1000)把一个沙箱的事件尽量捞全。 - */ -export function useActivity(limit?: number) { - return useQuery({ - queryKey: ['activity', limit ?? null], - queryFn: () => listActivity(limit), - refetchInterval: 5000, - retry: false, - }); -} diff --git a/packages/console/src/features/activity/kinds.ts b/packages/console/src/features/activity/kinds.ts deleted file mode 100644 index 4819d7c9..00000000 --- a/packages/console/src/features/activity/kinds.ts +++ /dev/null @@ -1,121 +0,0 @@ -import type { ActivityKind } from '@dormice/shared'; -import { m } from '@/paraglide/messages'; - -/** - * 事件的显示名与徽章配色 — 与 wire 上的 kind 一比一,这里是唯一的翻译点。 - * 活动页与沙箱工作台共用,不各自翻译。函数形式(而非常量表)是刻意的: - * 消息要在渲染时按当前 locale 取值。 - */ -export function activityKindLabel(kind: ActivityKind): string { - switch (kind) { - case 'created': - return m.activity_kind_created(); - case 'woken': - return m.activity_kind_woken(); - case 'frozen': - return m.activity_kind_frozen(); - case 'stopped': - return m.activity_kind_stopped(); - case 'rebuilt': - return m.activity_kind_rebuilt(); - case 'destroyed': - return m.activity_kind_destroyed(); - case 'expired-killed': - return m.activity_kind_expired_killed(); - case 'archived': - return m.activity_kind_archived(); - case 'restore-started': - return m.activity_kind_restore_started(); - case 'restored': - return m.activity_kind_restored(); - case 'restore-failed': - return m.activity_kind_restore_failed(); - case 'reconciled': - return m.activity_kind_reconciled(); - case 'policy-changed': - return m.activity_kind_policy_changed(); - case 'metadata-changed': - return m.activity_kind_metadata_changed(); - case 'spec-changed': - return m.activity_kind_spec_changed(); - case 'template-changed': - return m.activity_kind_template_changed(); - case 'disk-expanded': - return m.activity_kind_disk_expanded(); - case 'host-disk-grown': - return m.activity_kind_host_disk_grown(); - case 'daemon-started': - return m.activity_kind_daemon_started(); - case 'ingress-updated': - return m.activity_kind_ingress_updated(); - case 'settings-updated': - return m.activity_kind_settings_updated(); - case 'apikey-created': - return m.activity_kind_apikey_created(); - case 'apikey-updated': - return m.activity_kind_apikey_updated(); - case 'apikey-disabled': - return m.activity_kind_apikey_disabled(); - case 'apikey-enabled': - return m.activity_kind_apikey_enabled(); - case 'apikey-revoked': - return m.activity_kind_apikey_revoked(); - case 'upgrade-started': - return m.activity_kind_upgrade_started(); - } -} - -// 事件色与沙箱状态徽章同一色系:落到哪个状态就穿哪个颜色; -// 配置类事件(策略、域名)统一紫色。 -export const ACTIVITY_KIND_STYLES: Record = { - created: - 'border-emerald-500/40 bg-emerald-500/10 text-emerald-600 dark:text-emerald-400', - woken: - 'border-emerald-500/40 bg-emerald-500/10 text-emerald-600 dark:text-emerald-400', - frozen: 'border-sky-500/40 bg-sky-500/10 text-sky-600 dark:text-sky-400', - stopped: 'border-border bg-muted text-muted-foreground', - rebuilt: - 'border-violet-500/40 bg-violet-500/10 text-violet-600 dark:text-violet-400', - destroyed: 'border-red-500/40 bg-red-500/10 text-red-600 dark:text-red-400', - 'expired-killed': - 'border-red-500/40 bg-red-500/10 text-red-600 dark:text-red-400', - archived: - 'border-indigo-500/40 bg-indigo-500/10 text-indigo-600 dark:text-indigo-400', - 'restore-started': - 'border-indigo-500/40 bg-indigo-500/10 text-indigo-600 dark:text-indigo-400', - restored: - 'border-emerald-500/40 bg-emerald-500/10 text-emerald-600 dark:text-emerald-400', - 'restore-failed': - 'border-red-500/40 bg-red-500/10 text-red-600 dark:text-red-400', - reconciled: - 'border-amber-500/40 bg-amber-500/10 text-amber-600 dark:text-amber-400', - 'policy-changed': - 'border-violet-500/40 bg-violet-500/10 text-violet-600 dark:text-violet-400', - 'metadata-changed': - 'border-violet-500/40 bg-violet-500/10 text-violet-600 dark:text-violet-400', - 'spec-changed': - 'border-violet-500/40 bg-violet-500/10 text-violet-600 dark:text-violet-400', - 'template-changed': - 'border-violet-500/40 bg-violet-500/10 text-violet-600 dark:text-violet-400', - 'disk-expanded': - 'border-violet-500/40 bg-violet-500/10 text-violet-600 dark:text-violet-400', - // 系统自主动作穿 daemon 灰,不穿配置紫:没有操作者按过任何按钮。 - 'host-disk-grown': 'border-border bg-muted text-muted-foreground', - 'daemon-started': 'border-border bg-muted text-muted-foreground', - 'ingress-updated': - 'border-violet-500/40 bg-violet-500/10 text-violet-600 dark:text-violet-400', - 'settings-updated': - 'border-violet-500/40 bg-violet-500/10 text-violet-600 dark:text-violet-400', - 'apikey-created': - 'border-violet-500/40 bg-violet-500/10 text-violet-600 dark:text-violet-400', - 'apikey-updated': - 'border-violet-500/40 bg-violet-500/10 text-violet-600 dark:text-violet-400', - 'apikey-disabled': - 'border-amber-500/40 bg-amber-500/10 text-amber-600 dark:text-amber-400', - 'apikey-enabled': - 'border-emerald-500/40 bg-emerald-500/10 text-emerald-600 dark:text-emerald-400', - 'apikey-revoked': - 'border-red-500/40 bg-red-500/10 text-red-600 dark:text-red-400', - 'upgrade-started': - 'border-violet-500/40 bg-violet-500/10 text-violet-600 dark:text-violet-400', -}; diff --git a/packages/console/src/features/activity/pages/ActivityPage.tsx b/packages/console/src/features/activity/pages/ActivityPage.tsx deleted file mode 100644 index 409f9a4c..00000000 --- a/packages/console/src/features/activity/pages/ActivityPage.tsx +++ /dev/null @@ -1,247 +0,0 @@ -import { ACTIVITY_KINDS, type ActivityKind } from '@dormice/shared'; -import { Search01Icon } from '@hugeicons/core-free-icons'; -import { HugeiconsIcon } from '@hugeicons/react'; -import { Link, useSearch } from '@tanstack/react-router'; -import { useMemo, useState } from 'react'; -import { DataTable } from '@/components/DataTable'; -import { FilterMenu } from '@/components/FilterMenu'; -import { paginate, TablePager } from '@/components/TablePager'; -import { Badge } from '@/components/ui/badge'; -import { - Empty, - EmptyDescription, - EmptyHeader, - EmptyTitle, -} from '@/components/ui/empty'; -import { - InputGroup, - InputGroupAddon, - InputGroupInput, -} from '@/components/ui/input-group'; -import { Spinner } from '@/components/ui/spinner'; -import { - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from '@/components/ui/table'; -import { useApiKeys } from '@/features/api-keys/hooks/useApiKeys'; -import { ago } from '@/features/sandboxes/format'; -import { formatDateTime } from '@/lib/datetime'; -import { cn } from '@/lib/utils'; -import { m } from '@/paraglide/messages'; -import { actorLabel } from '../actors'; -import { useActivity } from '../hooks/useActivity'; -import { ACTIVITY_KIND_STYLES, activityKindLabel } from '../kinds'; - -const PAGE_SIZE = 50; - -/** - * 筛选器里 actor=null(系统)的哨兵值。词表是封闭的('env-token' / - * 'console' / 'apikey:'),裸串 'system' 永不与真实 actor 相撞。 - */ -const SYSTEM_ACTOR_FILTER = 'system'; - -/** - * 「我不在的时候发生了什么」:daemon 每一次生命周期动作(创建、降温、 - * 唤醒、销毁)和对账修复的有界环形记录 — 事件写在动作发生处,这里只读。 - * 筛选是纯前端的:环一共就 1000 条,全在手里,没必要为过滤发明服务端 - * 参数。操作者列回答事故响应的第一问(「这把 key 干了什么」):按 kind - * 之外再按 actor 筛,就是那把 key 的爆炸半径。 - */ -export function ActivityPage() { - const { data, isPending, isError, error } = useActivity(); - // 只为把 apikey: 翻译回名字;密钥页共用一份缓存。 - const apiKeys = useApiKeys().data?.apiKeys; - // ?sandbox= 是沙箱工作台「查看全部」带来的预筛,只当搜索框的一次性 - // 种子 — 落地后搜索框归用户,不做双向同步。 - const { sandbox: sandboxParam } = useSearch({ from: '/_app/activity' }); - const [search, setSearch] = useState(() => sandboxParam ?? ''); - const [kindFilter, setKindFilter] = useState<'all' | ActivityKind>('all'); - const [actorFilter, setActorFilter] = useState(''); - const [page, setPage] = useState(1); - - const events = data?.events ?? []; - const filtered = useMemo( - () => - events.filter( - (event) => - (kindFilter === 'all' || event.kind === kindFilter) && - (actorFilter === '' || - (actorFilter === SYSTEM_ACTOR_FILTER - ? event.actor === null - : event.actor === actorFilter)) && - (search === '' || - (event.sandboxName ?? '') - .toLowerCase() - .includes(search.toLowerCase())), - ), - [events, kindFilter, actorFilter, search], - ); - // 选项来自数据里实际出现过的操作者 — 不虚构没干过活的候选。 - const actorOptions = useMemo(() => { - const seen = new Set(events.map((event) => event.actor)); - return [...seen].map((actor) => ({ - value: actor ?? SYSTEM_ACTOR_FILTER, - label: actorLabel(actor, apiKeys), - })); - }, [events, apiKeys]); - const { rows, safePage, pageCount } = paginate(filtered, page, PAGE_SIZE); - - return ( - // openasi 列表页版式(2026-07-16 用户拍板):限宽居中、表格吃掉剩余 - // 高度框内滚、分页条钉底。 -
-
-

{m.activity_page_title()}

-
- -
- - - - - { - setSearch(event.target.value); - setPage(1); - }} - placeholder={m.activity_search_placeholder()} - /> - - ({ - value: kind, - label: activityKindLabel(kind), - }))} - onChange={(value) => { - setKindFilter(value === '' ? 'all' : (value as ActivityKind)); - setPage(1); - }} - /> - { - setActorFilter(value); - setPage(1); - }} - /> - - {m.activity_count({ - filtered: filtered.length, - total: events.length, - })} - -
- - {isPending ? ( -
- {m.activity_loading()} -
- ) : isError ? ( - - - {m.activity_load_failed()} - {error.message} - - - ) : events.length === 0 ? ( - - - {m.activity_empty_title()} - - {m.activity_empty_description()} - - - - ) : filtered.length === 0 ? ( - - - {m.activity_no_match_title()} - - {m.activity_no_match_description()} - - - - ) : ( - // 环形记录上限 1000 条,是全站最长的表 — fill 框内滚,表头吸顶。 - - - - {m.activity_col_time()} - {m.activity_col_kind()} - {m.activity_col_name()} - {m.activity_col_actor()} - {m.activity_col_detail()} - - - - {rows.map((event) => ( - - - {ago(event.at)} - - - - {activityKindLabel(event.kind)} - - - - {event.sandboxName ? ( - - {event.sandboxName} - - ) : ( - — - )} - - - {actorLabel(event.actor, apiKeys)} - - - {event.detail} - - - ))} - - - )} - - {!isPending && !isError && filtered.length > 0 && ( - - )} -
- ); -} diff --git a/packages/console/src/features/auth/pages/LoginPage.tsx b/packages/console/src/features/auth/pages/LoginPage.tsx index 3c61389f..c2580b58 100644 --- a/packages/console/src/features/auth/pages/LoginPage.tsx +++ b/packages/console/src/features/auth/pages/LoginPage.tsx @@ -140,7 +140,7 @@ export function LoginPage() { } // The wire speaks English by design; the Chinese is UI copy, translated at -// the edge — the same rule the activity page follows. +// the edge — the same rule every page follows. function errorText(error: unknown, invalidCredential: string): string { if (error instanceof ApiError) { if (error.status === 401) return invalidCredential; diff --git a/packages/console/src/features/sandboxes/components/workbench/MonitorRail.tsx b/packages/console/src/features/sandboxes/components/workbench/MonitorRail.tsx index c592b0d1..b0019e51 100644 --- a/packages/console/src/features/sandboxes/components/workbench/MonitorRail.tsx +++ b/packages/console/src/features/sandboxes/components/workbench/MonitorRail.tsx @@ -8,7 +8,6 @@ import { RamMemoryIcon, } from '@hugeicons/core-free-icons'; import { HugeiconsIcon, type HugeiconsProps } from '@hugeicons/react'; -import { Link } from '@tanstack/react-router'; import { useState } from 'react'; import { toast } from 'sonner'; import { Meter } from '@/components/Meter'; @@ -28,18 +27,10 @@ import { DropdownMenuItem, DropdownMenuTrigger, } from '@/components/ui/dropdown-menu'; -import { actorLabel } from '@/features/activity/actors'; -import { useActivity } from '@/features/activity/hooks/useActivity'; -import { - ACTIVITY_KIND_STYLES, - activityKindLabel, -} from '@/features/activity/kinds'; -import { useApiKeys } from '@/features/api-keys/hooks/useApiKeys'; import { Sparkline } from '@/features/overview/components/Sparkline'; import { copyText } from '@/lib/copy'; import { formatDateTime } from '@/lib/datetime'; import { formatBytes, pctOf } from '@/lib/format'; -import { cn } from '@/lib/utils'; import { m } from '@/paraglide/messages'; import { ago, policyLine } from '../../format'; import { useEnvdAuth, useKillProcess, useProcesses } from '../../hooks/useEnvd'; @@ -313,71 +304,6 @@ function ProcessesCard({ sandbox }: { sandbox: Sandbox }) { ); } -/** 右栏活动卡列几条最近的;全量(带筛选与详情列)在活动页。 */ -const ACTIVITY_ROWS = 8; - -function ActivityCard({ sandbox }: { sandbox: Sandbox }) { - const { data } = useActivity(1000); - const apiKeys = useApiKeys().data?.apiKeys; - const events = (data?.events ?? []).filter( - (event) => event.sandboxName === sandbox.name, - ); - - return ( - } - > - {m.workbench_view_all()} - - } - > - {events.length === 0 ? ( -

- {m.workbench_activity_empty()} -

- ) : ( -
- {events.slice(0, ACTIVITY_ROWS).map((event) => ( -
- - {activityKindLabel(event.kind)} - - - {actorLabel(event.actor, apiKeys)} - - - {ago(event.at)} - -
- ))} -
- )} - - ); -} - function InfoRow({ label, children, @@ -481,7 +407,6 @@ export function MonitorRail({ sandbox }: { sandbox: Sandbox }) { - ); diff --git a/packages/console/src/lib/api.ts b/packages/console/src/lib/api.ts index f314f33a..8ef558e0 100644 --- a/packages/console/src/lib/api.ts +++ b/packages/console/src/lib/api.ts @@ -14,7 +14,6 @@ import type { GetUpgradeStatusResponse, HostMetricsResponse, LifecyclePolicyOverride, - ListActivityResponse, ListSandboxImagesResponse, ListSandboxMetricsResponse, RegisterTemplateResponse, @@ -190,11 +189,6 @@ export const getHostMetricsHistory = (start: string, end: string) => export const listSandboxImages = () => rpc('/listSandboxImages'); -// The ledger's recent history, newest first — a bounded ring, not an audit -// log. The daemon records at the moves themselves; this only reads. -export const listActivity = (limit?: number) => - rpc('/listActivity', limit ? { limit } : {}); - // Effective configuration. Secrets come back as "set", never as their // value; archive.enabled is the daemon's own adjudication. The env entries // stay read-only; `settings` 是账本里的运营旋钮,写入走 updateSettings。 diff --git a/packages/console/src/lib/mock.ts b/packages/console/src/lib/mock.ts index 7de43754..32af415b 100644 --- a/packages/console/src/lib/mock.ts +++ b/packages/console/src/lib/mock.ts @@ -1,7 +1,7 @@ /** * The mock gate for pages whose server side does not exist yet — today only - * the doctor page (activity, settings and per-sandbox metrics graduated to - * real endpoints 2026-07-11). In dev it renders typed sample data so the UI + * the doctor page (settings and per-sandbox metrics graduated to real + * endpoints 2026-07-11). In dev it renders typed sample data so the UI * can be designed first; a production build hides it entirely — a real * deployment must never show a fake all-green health report. The fixtures' * TYPES are the proposed wire shapes: when the server side lands, they diff --git a/packages/console/src/routeTree.gen.ts b/packages/console/src/routeTree.gen.ts index 74f27301..2441d7eb 100644 --- a/packages/console/src/routeTree.gen.ts +++ b/packages/console/src/routeTree.gen.ts @@ -19,7 +19,6 @@ import { Route as AppDomainsRouteImport } from './routes/_app/domains' import { Route as AppDoctorRouteImport } from './routes/_app/doctor' import { Route as AppConnectRouteImport } from './routes/_app/connect' import { Route as AppApiKeysRouteImport } from './routes/_app/api-keys' -import { Route as AppActivityRouteImport } from './routes/_app/activity' import { Route as AppSandboxesIndexRouteImport } from './routes/_app/sandboxes/index' import { Route as AppTempSplatRouteImport } from './routes/_app/temp.$' import { Route as AppSandboxesNameRouteImport } from './routes/_app/sandboxes/$name' @@ -73,11 +72,6 @@ const AppApiKeysRoute = AppApiKeysRouteImport.update({ path: '/api-keys', getParentRoute: () => AppRoute, } as any) -const AppActivityRoute = AppActivityRouteImport.update({ - id: '/activity', - path: '/activity', - getParentRoute: () => AppRoute, -} as any) const AppSandboxesIndexRoute = AppSandboxesIndexRouteImport.update({ id: '/sandboxes/', path: '/sandboxes/', @@ -97,7 +91,6 @@ const AppSandboxesNameRoute = AppSandboxesNameRouteImport.update({ export interface FileRoutesByFullPath { '/': typeof AppIndexRoute '/login': typeof LoginRoute - '/activity': typeof AppActivityRoute '/api-keys': typeof AppApiKeysRoute '/connect': typeof AppConnectRoute '/doctor': typeof AppDoctorRoute @@ -111,7 +104,6 @@ export interface FileRoutesByFullPath { } export interface FileRoutesByTo { '/login': typeof LoginRoute - '/activity': typeof AppActivityRoute '/api-keys': typeof AppApiKeysRoute '/connect': typeof AppConnectRoute '/doctor': typeof AppDoctorRoute @@ -128,7 +120,6 @@ export interface FileRoutesById { __root__: typeof rootRouteImport '/_app': typeof AppRouteWithChildren '/login': typeof LoginRoute - '/_app/activity': typeof AppActivityRoute '/_app/api-keys': typeof AppApiKeysRoute '/_app/connect': typeof AppConnectRoute '/_app/doctor': typeof AppDoctorRoute @@ -146,7 +137,6 @@ export interface FileRouteTypes { fullPaths: | '/' | '/login' - | '/activity' | '/api-keys' | '/connect' | '/doctor' @@ -160,7 +150,6 @@ export interface FileRouteTypes { fileRoutesByTo: FileRoutesByTo to: | '/login' - | '/activity' | '/api-keys' | '/connect' | '/doctor' @@ -176,7 +165,6 @@ export interface FileRouteTypes { | '__root__' | '/_app' | '/login' - | '/_app/activity' | '/_app/api-keys' | '/_app/connect' | '/_app/doctor' @@ -267,13 +255,6 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AppApiKeysRouteImport parentRoute: typeof AppRoute } - '/_app/activity': { - id: '/_app/activity' - path: '/activity' - fullPath: '/activity' - preLoaderRoute: typeof AppActivityRouteImport - parentRoute: typeof AppRoute - } '/_app/sandboxes/': { id: '/_app/sandboxes/' path: '/sandboxes' @@ -299,7 +280,6 @@ declare module '@tanstack/react-router' { } interface AppRouteChildren { - AppActivityRoute: typeof AppActivityRoute AppApiKeysRoute: typeof AppApiKeysRoute AppConnectRoute: typeof AppConnectRoute AppDoctorRoute: typeof AppDoctorRoute @@ -314,7 +294,6 @@ interface AppRouteChildren { } const AppRouteChildren: AppRouteChildren = { - AppActivityRoute: AppActivityRoute, AppApiKeysRoute: AppApiKeysRoute, AppConnectRoute: AppConnectRoute, AppDoctorRoute: AppDoctorRoute, diff --git a/packages/console/src/routes/_app/activity.tsx b/packages/console/src/routes/_app/activity.tsx deleted file mode 100644 index 34ace34e..00000000 --- a/packages/console/src/routes/_app/activity.tsx +++ /dev/null @@ -1,15 +0,0 @@ -import { createFileRoute } from '@tanstack/react-router'; -import { ActivityPage } from '@/features/activity/pages/ActivityPage'; - -/** - * 可选的 `?sandbox=`:沙箱工作台的「查看全部活动」带着名字跳过来, - * 落地即预筛。只做一次性种子(灌进页内搜索框的初值),此后搜索框归 - * 用户 — 不做双向同步。 - */ -export const Route = createFileRoute('/_app/activity')({ - validateSearch: (search: Record): { sandbox?: string } => - typeof search.sandbox === 'string' && search.sandbox !== '' - ? { sandbox: search.sandbox } - : {}, - component: ActivityPage, -}); diff --git a/packages/console/vite.config.ts b/packages/console/vite.config.ts index 84f42656..cc380302 100644 --- a/packages/console/vite.config.ts +++ b/packages/console/vite.config.ts @@ -56,7 +56,6 @@ export default defineConfig({ '/listSandboxMetrics': 'http://127.0.0.1:3676', '/listSandboxImages': 'http://127.0.0.1:3676', '/getFleetTimeline': 'http://127.0.0.1:3676', - '/listActivity': 'http://127.0.0.1:3676', '/getConfig': 'http://127.0.0.1:3676', '/checkUpgrade': 'http://127.0.0.1:3676', '/applyUpgrade': 'http://127.0.0.1:3676', diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index 161d2f85..088fbc6f 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -57,7 +57,6 @@ export const UNNAMED_VERBS = [ 'listSandboxes', 'listSandboxMetrics', 'listSandboxImages', - 'listActivity', 'getFleetTimeline', 'getHostMetrics', 'getHostMetricsHistory', diff --git a/packages/sdk/src/client.test.ts b/packages/sdk/src/client.test.ts index 33f5674f..4d05847e 100644 --- a/packages/sdk/src/client.test.ts +++ b/packages/sdk/src/client.test.ts @@ -504,12 +504,4 @@ describe('the observability verbs over real HTTP', () => { const after = await client.listSandboxMetrics(); expect(after.filter((s) => s.sandboxName.startsWith('fleet-'))).toEqual([]); }); - - it('listActivity tells the story just written, newest first', async () => { - await client.acquireSandbox('story-sdk'); - await client.destroySandbox('story-sdk'); - const log = await client.listActivity({ limit: 10 }); - const mine = log.filter((e) => e.sandboxName === 'story-sdk'); - expect(mine.map((e) => e.kind)).toEqual(['destroyed', 'created']); - }); }); diff --git a/packages/sdk/src/client.ts b/packages/sdk/src/client.ts index b2011d07..6f0e5818 100644 --- a/packages/sdk/src/client.ts +++ b/packages/sdk/src/client.ts @@ -1,6 +1,5 @@ import { type AcquireResponse, - type ActivityEvent, type ApiKey, type ApplyUpgradeResponse, acquireResponseSchema, @@ -34,7 +33,6 @@ import { type LifecyclePolicyOverride, type ListSandboxImagesResponse, type ListSandboxMetricsResponse, - listActivityResponseSchema, listApiKeysResponseSchema, listSandboxesResponseSchema, listSandboxImagesResponseSchema, @@ -303,18 +301,6 @@ export class Dormice { return listSandboxImagesResponseSchema.parse(data).images; } - /** - * The daemon's recent history, newest first: who was created, cooled, - * woken, destroyed, and what reconciliation repaired. A bounded ring — - * an explanation window, not an audit log. - */ - async listActivity(options?: { limit?: number }): Promise { - const data = await this.rpc('listActivity', { - limit: options?.limit, - }); - return listActivityResponseSchema.parse(data).events; - } - /** * The daemon's effective configuration, read-only: every knob, the value * in force, and whether it came from the environment or a default. diff --git a/packages/server/drizzle/0023_drop-activity.sql b/packages/server/drizzle/0023_drop-activity.sql new file mode 100644 index 00000000..729d6579 --- /dev/null +++ b/packages/server/drizzle/0023_drop-activity.sql @@ -0,0 +1 @@ +DROP TABLE `activity`; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0023_snapshot.json b/packages/server/drizzle/meta/0023_snapshot.json new file mode 100644 index 00000000..7eb158af --- /dev/null +++ b/packages/server/drizzle/meta/0023_snapshot.json @@ -0,0 +1,773 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "d8435ca2-c0e2-44ec-b4c5-9928f5bcd2e4", + "prevId": "787b4cbc-6107-4ccb-865a-1b61e5dbc5f3", + "tables": { + "api_keys": { + "name": "api_keys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + "key_hash" + ], + "isUnique": true + }, + "api_keys_active_name_idx": { + "name": "api_keys_active_name_idx", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"api_keys\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "console_account": { + "name": "console_account", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_secret": { + "name": "session_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "daemon_secrets": { + "name": "daemon_secrets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "envd_signing_secret": { + "name": "envd_signing_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "fleet_snapshots": { + "name": "fleet_snapshots", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "active": { + "name": "active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "frozen": { + "name": "frozen", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stopped": { + "name": "stopped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "restoring": { + "name": "restoring", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "host_metrics_samples": { + "name": "host_metrics_samples", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "cpu_used_pct": { + "name": "cpu_used_pct", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_available_bytes": { + "name": "mem_available_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_total_bytes": { + "name": "swap_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_used_bytes": { + "name": "swap_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_total_bytes": { + "name": "disk_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_used_bytes": { + "name": "disk_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_available_bytes": { + "name": "disk_available_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "runtime_settings": { + "name": "runtime_settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "max_sandboxes": { + "name": "max_sandboxes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_cpus": { + "name": "sandbox_cpus", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_memory_gb": { + "name": "sandbox_memory_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_disk_gb": { + "name": "sandbox_disk_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_freeze_after_seconds": { + "name": "default_freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_stop_after_seconds": { + "name": "default_stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_archive_after_seconds": { + "name": "default_archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_gb": { + "name": "swap_gb", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "s3_endpoint": { + "name": "s3_endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_access_key_id": { + "name": "s3_access_key_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_secret_access_key": { + "name": "s3_secret_access_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_region": { + "name": "s3_region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_force_path_style": { + "name": "s3_force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain": { + "name": "sandbox_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain_aliases": { + "name": "sandbox_domain_aliases", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pids_limit": { + "name": "pids_limit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sandbox_metrics_samples": { + "name": "sandbox_metrics_samples", + "columns": { + "sandbox_id": { + "name": "sandbox_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "at": { + "name": "at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cpu_count": { + "name": "cpu_count", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cpu_used_pct": { + "name": "cpu_used_pct", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_used_bytes": { + "name": "mem_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_cache_bytes": { + "name": "mem_cache_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_used_bytes": { + "name": "swap_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_total_bytes": { + "name": "swap_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_used_bytes": { + "name": "disk_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "disk_total_bytes": { + "name": "disk_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "sandbox_metrics_samples_sandbox_at_idx": { + "name": "sandbox_metrics_samples_sandbox_at_idx", + "columns": [ + "sandbox_id", + "at" + ], + "isUnique": false + }, + "sandbox_metrics_samples_at_idx": { + "name": "sandbox_metrics_samples_at_idx", + "columns": [ + "at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sandboxes": { + "name": "sandboxes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "freeze_after_seconds": { + "name": "freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stop_after_seconds": { + "name": "stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "archive_after_seconds": { + "name": "archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpus": { + "name": "cpus", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "memory_gb": { + "name": "memory_gb", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_gb": { + "name": "disk_gb", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_active_at": { + "name": "last_active_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_exit_at": { + "name": "last_exit_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_exit_code": { + "name": "last_exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_exit_cause": { + "name": "last_exit_cause", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "envs": { + "name": "envs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "on_deadline": { + "name": "on_deadline", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "paused_by_user": { + "name": "paused_by_user", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "sandboxes_name_unique": { + "name": "sandboxes_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index c072f61f..b60be5f6 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -162,6 +162,13 @@ "when": 1788964773753, "tag": "0022_sandbox-swap", "breakpoints": true + }, + { + "idx": 23, + "version": "6", + "when": 1789370106697, + "tag": "0023_drop-activity", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/server/src/app.test.ts b/packages/server/src/app.test.ts index 555e2b8f..6db7558c 100644 --- a/packages/server/src/app.test.ts +++ b/packages/server/src/app.test.ts @@ -511,17 +511,6 @@ describe('acquire finds the shell dead under an active row', () => { cause: 'runtime-died', }); expect(again.sandbox.lastExit.at).toMatch(/^\d{4}-/); - - const kinds = (await rpc(app, '/listActivity')) - .json() - .events.map((e: { kind: string; detail: string }) => [e.kind, e.detail]); - expect(kinds).toContainEqual([ - 'reconciled', - "container is stopped — state active corrected to stopped (exit 2, not an OOM kill — gVisor's sentry itself died, the signature a pids-cap hit leaves; see the sandbox pids cap in settings), found dead at wake", - ]); - // The restart is its own event, recorded once it has happened — the - // death record never claims it ahead of time. - expect(kinds[0]).toEqual(['woken', 'cold start from the surviving disk']); }); it('lastExit is sticky history: a later idle stop and wake keep the last death readable', async () => { @@ -1009,13 +998,6 @@ describe('POST /updatePolicy', () => { }); // Adjusting a knob is not activity: the idle countdown keeps running. expect(res.json().sandbox.lastActiveAt).toBe(created.sandbox.lastActiveAt); - - const events = (await rpc(app, '/listActivity')).json().events; - expect(events[0]).toMatchObject({ - kind: 'policy-changed', - sandboxName: 'alice', - detail: 'freeze 600s -> 120s', - }); }); it('promotes a frozen sandbox to never-stop without waking it', async () => { @@ -1080,7 +1062,7 @@ describe('POST /updatePolicy', () => { expect(res.json().message).toMatch(/acquire it first/); }); - it('treats a no-change patch as the goal state and writes no history', async () => { + it('treats a no-change patch as the goal state', async () => { const { app } = testApp(); await acquire(app, { name: 'alice' }); const res = await rpc(app, '/updatePolicy', { @@ -1090,10 +1072,6 @@ describe('POST /updatePolicy', () => { }, }); expect(res.statusCode).toBe(200); - const events = (await rpc(app, '/listActivity')).json().events; - expect( - events.some((e: { kind: string }) => e.kind === 'policy-changed'), - ).toBe(false); }); }); @@ -1116,16 +1094,9 @@ describe('POST /updateMetadata', () => { expect(res.json().sandbox.metadata).toEqual({ app: 'assistant' }); // Relabeling is not activity: the idle countdown keeps running. expect(res.json().sandbox.lastActiveAt).toBe(created.sandbox.lastActiveAt); - - const events = (await rpc(app, '/listActivity')).json().events; - expect(events[0]).toMatchObject({ - kind: 'metadata-changed', - sandboxName: 'alice', - detail: 'app=assistant', - }); }); - it('clears every label with {} and says so in the history', async () => { + it('clears every label with {}', async () => { const { app } = testApp(); await acquire(app, { name: 'alice', metadata: { app: 'crawler' } }); const res = await rpc(app, '/updateMetadata', { @@ -1134,11 +1105,6 @@ describe('POST /updateMetadata', () => { }); expect(res.statusCode).toBe(200); expect(res.json().sandbox.metadata).toEqual({}); - const events = (await rpc(app, '/listActivity')).json().events; - expect(events[0]).toMatchObject({ - kind: 'metadata-changed', - detail: 'cleared', - }); }); it('relabels a frozen sandbox without waking it — a pure ledger write', async () => { @@ -1173,7 +1139,7 @@ describe('POST /updateMetadata', () => { expect(res.json().message).toMatch(/acquire it first/); }); - it('treats a no-change replacement as the goal state and writes no history', async () => { + it('treats a no-change replacement as the goal state', async () => { const { app } = testApp(); await acquire(app, { name: 'alice', metadata: { app: 'crawler' } }); const res = await rpc(app, '/updateMetadata', { @@ -1181,15 +1147,11 @@ describe('POST /updateMetadata', () => { metadata: { app: 'crawler' }, }); expect(res.statusCode).toBe(200); - const events = (await rpc(app, '/listActivity')).json().events; - expect( - events.some((e: { kind: string }) => e.kind === 'metadata-changed'), - ).toBe(false); }); }); describe('POST /updateTemplate', () => { - it('re-homes the sandbox, does not refresh the idle clock, and records the move', async () => { + it('re-homes the sandbox and does not refresh the idle clock', async () => { const { app } = testApp(); await rpc(app, '/registerTemplate', { name: 'py-a', image: 'img-a' }); await rpc(app, '/registerTemplate', { name: 'py-b', image: 'img-b' }); @@ -1205,13 +1167,6 @@ describe('POST /updateTemplate', () => { expect(res.json().sandbox.template).toBe('py-b'); // Re-homing is not activity: the idle countdown keeps running. expect(res.json().sandbox.lastActiveAt).toBe(created.sandbox.lastActiveAt); - - const events = (await rpc(app, '/listActivity')).json().events; - expect(events[0]).toMatchObject({ - kind: 'template-changed', - sandboxName: 'alice', - detail: 'template py-a -> py-b; applies at the next cold wake', - }); }); it('a frozen sandbox stays frozen; the next wake swaps the shell onto the new template, data intact', async () => { @@ -1274,11 +1229,6 @@ describe('POST /updateTemplate', () => { }); expect(res.statusCode).toBe(200); expect(res.json().sandbox.template).toBeNull(); - const events = (await rpc(app, '/listActivity')).json().events; - expect(events[0]).toMatchObject({ - kind: 'template-changed', - detail: 'template py-a -> base image; applies at the next cold wake', - }); // With no rows referencing it, the old template can now be removed — // the migration story this verb exists for. expect( @@ -1305,7 +1255,7 @@ describe('POST /updateTemplate', () => { expect(nobody.json().message).toMatch(/acquire it first/); }); - it('treats a same-template update as the goal state and writes no history', async () => { + it('treats a same-template update as the goal state', async () => { const { app } = testApp(); await rpc(app, '/registerTemplate', { name: 'py-a', image: 'img-a' }); await acquire(app, { name: 'alice', template: 'py-a' }); @@ -1314,10 +1264,6 @@ describe('POST /updateTemplate', () => { template: 'py-a', }); expect(res.statusCode).toBe(200); - const events = (await rpc(app, '/listActivity')).json().events; - expect( - events.some((e: { kind: string }) => e.kind === 'template-changed'), - ).toBe(false); }); }); @@ -1777,17 +1723,8 @@ describe('cold wakes converge onto the current image', () => { expect(Buffer.from(read.json().contentBase64, 'base64').toString()).toBe( 'survives', ); - // The audit trail names both halves of the move. - const kinds = (await rpc(app, '/listActivity')) - .json() - .events.map((e: { kind: string }) => e.kind); - expect(kinds.slice(0, 2)).toEqual(['woken', 'rebuilt']); - const rebuilt = (await rpc(app, '/listActivity')) - .json() - .events.find((e: { kind: string }) => e.kind === 'rebuilt'); - expect(rebuilt.detail).toBe( - 'stale shell swapped at wake: img-v1 -> img-v2', - ); + // The old shell was removed — a swap, not a plain unpause. + expect(executor.removedShells).toEqual([created.id]); }); it('frozen + fresh: a plain unpause, no shell removed', async () => { @@ -1806,10 +1743,7 @@ describe('cold wakes converge onto the current image', () => { const woken = (await acquire(app, { name: 'alice' })).json().sandbox; expect(woken.state).toBe('active'); expect(await executor.imageOf(created.id)).toBe('img-v1'); - const kinds = (await rpc(app, '/listActivity')) - .json() - .events.map((e: { kind: string }) => e.kind); - expect(kinds).not.toContain('rebuilt'); + expect(executor.removedShells).toEqual([]); }); it('stopped + stale: the same convergence — stop kept the old shell, the wake replaces it', async () => { @@ -1832,10 +1766,7 @@ describe('cold wakes converge onto the current image', () => { const woken = (await acquire(app, { name: 'alice' })).json().sandbox; expect(woken.state).toBe('active'); expect(await executor.imageOf(created.id)).toBe('img-v2'); - const kinds = (await rpc(app, '/listActivity')) - .json() - .events.map((e: { kind: string }) => e.kind); - expect(kinds.slice(0, 2)).toEqual(['woken', 'rebuilt']); + expect(executor.removedShells).toEqual([created.id]); }); it('a template-less sandbox is judged against the base image — fresh, so untouched', async () => { @@ -1851,10 +1782,7 @@ describe('cold wakes converge onto the current image', () => { const woken = (await acquire(app, { name: 'alice' })).json().sandbox; expect(woken.state).toBe('active'); expect(await executor.imageOf(created.id)).toBe(executor.baseImage); - const kinds = (await rpc(app, '/listActivity')) - .json() - .events.map((e: { kind: string }) => e.kind); - expect(kinds).not.toContain('rebuilt'); + expect(executor.removedShells).toEqual([]); }); it('a vanished shell is not judged stale — the start builds from the current image by itself', async () => { @@ -1877,10 +1805,7 @@ describe('cold wakes converge onto the current image', () => { // Converged all the same, but through start's own rebuild — no shell // was removed, so no 'rebuilt' entry claims one was. expect(await executor.imageOf(created.id)).toBe('img-v2'); - const kinds = (await rpc(app, '/listActivity')) - .json() - .events.map((e: { kind: string }) => e.kind); - expect(kinds).not.toContain('rebuilt'); + expect(executor.removedShells).toEqual([]); }); }); @@ -2080,16 +2005,24 @@ describe('API keys', () => { expect(second.lastUsedAt).toBe(first.lastUsedAt); }); - it('records mint and revoke in the activity ring, token nowhere in sight', async () => { + it('a console session passes the admin gate too: a key minted from the console', async () => { const { app } = testApp(); - const { id, token } = await mint(app, 'ci'); - await rpc(app, '/revokeApiKey', { id }); - - const events = (await rpc(app, '/listActivity')).json().events; - const kinds = events.map((e: { kind: string }) => e.kind); - expect(kinds).toContain('apikey-created'); - expect(kinds).toContain('apikey-revoked'); - expect(JSON.stringify(events)).not.toContain(token); + const setup = await app.inject({ + method: 'POST', + url: '/console/auth/setup', + payload: { token: TOKEN, username: 'operator', password: 'horse pass' }, + }); + const cookie = setup.cookies.find((c) => c.name === SESSION_COOKIE); + const minted = await app.inject({ + method: 'POST', + url: '/createApiKey', + headers: { [CONSOLE_HEADER]: '1' }, + cookies: { [SESSION_COOKIE]: (cookie as { value: string }).value }, + payload: { name: 'by-console' }, + }); + expect(minted.statusCode).toBe(200); + expect(minted.json().apiKey.name).toBe('by-console'); + expect((await useKey(app, minted.json().token)).statusCode).toBe(200); }); it('disable parks the key reversibly: 401 while disabled, 200 again after enable', async () => { @@ -2114,14 +2047,6 @@ describe('API keys', () => { ).json().apiKey; expect(enabled.disabledAt).toBeNull(); expect((await useKey(app, token)).statusCode).toBe(200); - - const kinds = (await rpc(app, '/listActivity')) - .json() - .events.map((e: { kind: string }) => e.kind); - expect(kinds.filter((k: string) => k === 'apikey-disabled')).toHaveLength( - 1, - ); - expect(kinds.filter((k: string) => k === 'apikey-enabled')).toHaveLength(1); }); it('expiry closes the door: a past expiresAt is 401, clearing it reopens', async () => { @@ -2181,10 +2106,10 @@ describe('API keys', () => { expect(edited.json().message).toMatch(/rotation history/); }); - it('a no-op patch changes nothing and records nothing', async () => { + it('a no-op patch changes nothing', async () => { const { app } = testApp(); const { id } = await mint(app, 'ci'); - const before = (await rpc(app, '/listActivity')).json().events.length; + const before = (await rpc(app, '/listApiKeys')).json().apiKeys; const res = await rpc(app, '/updateApiKey', { id, @@ -2193,9 +2118,7 @@ describe('API keys', () => { }); expect(res.statusCode).toBe(200); expect(res.json().apiKey.name).toBe('ci'); - - const after = (await rpc(app, '/listActivity')).json().events.length; - expect(after).toBe(before); + expect((await rpc(app, '/listApiKeys')).json().apiKeys).toEqual(before); }); it('carries expiresAt from mint into the list', async () => { @@ -2269,96 +2192,6 @@ describe('API keys', () => { }); }); -describe('activity attribution', () => { - /** - * The newest event of this kind (listActivity is newest-first), - * optionally narrowed to one sandbox. Actor strings are asserted as - * literals on purpose: they are wire vocabulary, and a drifted constant - * must fail here, not ride through. - */ - const eventOf = async ( - app: ReturnType['app'], - kind: string, - sandboxName?: string, - ) => { - const events = (await rpc(app, '/listActivity')).json().events as Array<{ - kind: string; - sandboxName: string | null; - actor: string | null; - }>; - return events.find( - (e) => - e.kind === kind && - (sandboxName === undefined || e.sandboxName === sandboxName), - ); - }; - - it('lifecycle verbs name their credential: env token and API key are distinct actors', async () => { - const { app } = testApp(); - const minted = (await rpc(app, '/createApiKey', { name: 'agent' })).json(); - const asKey = { authorization: `Bearer ${minted.token}` }; - - await acquire(app, { name: 'mine' }); - await acquire(app, { name: 'theirs' }, asKey); - expect((await eventOf(app, 'created', 'mine'))?.actor).toBe('env-token'); - expect((await eventOf(app, 'created', 'theirs'))?.actor).toBe( - `apikey:${minted.apiKey.id}`, - ); - - // The blast-radius question a leak raises: which key destroyed this? - await app.inject({ - method: 'POST', - url: '/destroySandbox', - headers: asKey, - payload: { name: 'mine' }, - }); - expect((await eventOf(app, 'destroyed', 'mine'))?.actor).toBe( - `apikey:${minted.apiKey.id}`, - ); - }); - - it('daemon moves stay null; the wake that follows names its caller', async () => { - const { app, db, executor, locks } = testApp(); - const created = (await acquire(app, { name: 'alice' })).json(); - - await scanOnce( - db, - executor, - locks, - after( - created.sandbox.lastActiveAt, - DEFAULT_LIFECYCLE_POLICY.freezeAfterSeconds, - ), - ); - // The idle scanner froze it: no credential asked, so no actor. - expect((await eventOf(app, 'frozen', 'alice'))?.actor).toBeNull(); - - await acquire(app, { name: 'alice' }); - expect((await eventOf(app, 'woken', 'alice'))?.actor).toBe('env-token'); - }); - - it('apikey management events name the administrator: env token or console, never a key', async () => { - const { app } = testApp(); - await rpc(app, '/createApiKey', { name: 'by-env' }); - expect((await eventOf(app, 'apikey-created'))?.actor).toBe('env-token'); - - const setup = await app.inject({ - method: 'POST', - url: '/console/auth/setup', - payload: { token: TOKEN, username: 'operator', password: 'horse pass' }, - }); - const cookie = setup.cookies.find((c) => c.name === SESSION_COOKIE); - await app.inject({ - method: 'POST', - url: '/createApiKey', - headers: { [CONSOLE_HEADER]: '1' }, - cookies: { [SESSION_COOKIE]: (cookie as { value: string }).value }, - payload: { name: 'by-console' }, - }); - expect((await eventOf(app, 'apikey-created'))?.actor).toBe('console'); - }); -}); - describe('POST /lookupSandbox', () => { it('answers by name and by id with the state, without waking or touching the idle clock', async () => { const { app, db } = testApp(); diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 35a4fcb5..f1ba8da0 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -1,6 +1,5 @@ import http from 'node:http'; import nodePath from 'node:path'; -import { apiKeyActor, ENV_TOKEN_ACTOR } from '@dormice/shared'; import fastifyCookie from '@fastify/cookie'; import fastify, { type FastifyError, type FastifyServerFactory } from 'fastify'; import { @@ -25,7 +24,6 @@ import { WatcherTable } from './e2b/watcher-table'; import type { Executor } from './executor/executor'; import type { Ingress } from './ingress'; import type { KeyedQueue } from './keyed-queue'; -import { activityRoutes } from './routes/activity'; import { apiKeyRoutes } from './routes/api-keys'; import { configRoutes } from './routes/config'; import { consoleRoutes } from './routes/console'; @@ -230,33 +228,22 @@ export function buildApp({ // cookie on the native routes, the /console surface mints and clears it. app.register(fastifyCookie); - // Attribution rides the request from the auth hook that admitted it into - // every recordActivity the handler reaches. Declared once so the property - // shape is stable; null is what unauthenticated surfaces keep. - app.decorateRequest('actor', null); - - // The one adjudication of "does this bare credential open the door" — - // and of who it is (the two are the same act, so identity is captured - // here, not re-derived later): the env token (constant-time compare — - // the bootstrap credential, always valid) or any active ledger API key - // (sha256 indexed lookup, judged per request so a mint or revoke takes - // effect on the very next call). Both faces — the native Bearer header - // and the E2B X-API-KEY hook — feed this same closure: one truth, two - // dialects. - const identifyCredential = (bare: string): string | null => { - if (tokensEqual(bare, config.DORMICE_API_TOKEN)) { - return ENV_TOKEN_ACTOR; - } - const keyId = verifyApiKeyToken(db, bare); - return keyId === null ? null : apiKeyActor(keyId); - }; + // The one adjudication of "does this bare credential open the door": + // the env token (constant-time compare — the bootstrap credential, + // always valid) or any active ledger API key (sha256 indexed lookup, + // judged per request so a mint or revoke takes effect on the very next + // call). Both faces — the native Bearer header and the E2B X-API-KEY + // hook — feed this same closure: one truth, two dialects. + const isCredential = (bare: string): boolean => + tokensEqual(bare, config.DORMICE_API_TOKEN) || + verifyApiKeyToken(db, bare) !== null; // Built once, used by every guarded surface. The secret getter reads the // ledger per request because setup can replace the account (and void its // sessions) while the daemon runs — a captured value would keep dead // sessions alive until restart. const apiAuth = requireApiAuth( - identifyCredential, + isCredential, () => getConsoleAccount(db)?.sessionSecret ?? null, ); @@ -287,15 +274,14 @@ export function buildApp({ }); await api.register(templateRoutes, { db }); await api.register(hostRoutes, { config, db, executor }); - await api.register(activityRoutes, { db }); - await api.register(ingressRoutes, { db, ingress }); + await api.register(ingressRoutes, { ingress }); await api.register(configRoutes, { config, db, sources, swap, }); - await api.register(upgradeRoutes, { updater, db }); + await api.register(upgradeRoutes, { updater }); }); // The apiKey management verbs and updateSettings sit behind the stricter @@ -340,7 +326,7 @@ export function buildApp({ watchers, archiver, envdSigningSecret, - identifyCredential, + isCredential, }); }); diff --git a/packages/server/src/archive/archiver.ts b/packages/server/src/archive/archiver.ts index f59d4e65..94f2909e 100644 --- a/packages/server/src/archive/archiver.ts +++ b/packages/server/src/archive/archiver.ts @@ -1,6 +1,5 @@ import { mkdir, rm, stat } from 'node:fs/promises'; import path from 'node:path'; -import { recordActivity } from '../db/activity'; import type { Db } from '../db/db'; import { findById, setPausedByUser, touch, transition } from '../db/ledger'; import type { SandboxRow } from '../db/schema'; @@ -201,13 +200,7 @@ export class Archiver { await rm(tmp, { force: true }); } const seconds = ((Date.now() - startedAt) / 1000).toFixed(1); - recordActivity(this.db, { - kind: 'archived', - sandboxName: row.name, - sandboxId: row.id, - detail: `disk shipped to S3 in ${seconds}s; local copy freed`, - }); - this.log(`archived ${row.id} in ${seconds}s`); + this.log(`archived ${row.id} (${row.name}) in ${seconds}s`); } /** @@ -224,12 +217,7 @@ export class Archiver { ); } transition(this.db, row.id, 'restoring'); - recordActivity(this.db, { - kind: 'restore-started', - sandboxName: row.name, - sandboxId: row.id, - detail: 'restore from S3 began', - }); + this.log(`restoring ${row.id} (${row.name}) from S3`); // Captured here, kept for the whole task: a settings edit mid-restore // must not switch clients under a running download (the moving-store // guard in updateSettings means only credentials can change here, and @@ -341,12 +329,7 @@ export class Archiver { } transition(this.db, sandboxId, 'active'); touch(this.db, sandboxId); - recordActivity(this.db, { - kind: 'restored', - sandboxName: name, - sandboxId, - detail: 'disk back from S3, sandbox active; archive object deleted', - }); + this.log(`restored ${sandboxId} (${name}): disk back from S3, active`); }); // The object's job is done — the disk is local again, so from here // "an object exists" means "the row is archived", and destroy never @@ -362,14 +345,6 @@ export class Archiver { const fresh = findById(this.db, sandboxId); if (fresh?.state === 'restoring') { transition(this.db, sandboxId, 'archived'); - recordActivity(this.db, { - kind: 'restore-failed', - sandboxName: name, - sandboxId, - detail: `back to archived, S3 object intact — ${ - err instanceof Error ? err.message : String(err) - }`.slice(0, 300), - }); } }); throw err; diff --git a/packages/server/src/auth.ts b/packages/server/src/auth.ts index d1b341a5..5e6c4abd 100644 --- a/packages/server/src/auth.ts +++ b/packages/server/src/auth.ts @@ -6,23 +6,8 @@ import { scrypt, timingSafeEqual, } from 'node:crypto'; -import { CONSOLE_ACTOR, ENV_TOKEN_ACTOR } from '@dormice/shared'; import type { FastifyRequest, onRequestAsyncHookHandler } from 'fastify'; -declare module 'fastify' { - interface FastifyRequest { - /** - * Attribution: who this request is, in the shared actor vocabulary - * ('env-token' | 'console' | 'apikey:'). Set by whichever auth hook - * admitted the request — authentication is the one moment identity is - * adjudicated, so it is captured there and threaded into recordActivity, - * never re-derived. Null only on unauthenticated surfaces (/healthz, - * console setup/login), which record no activity. - */ - actor: string | null; - } -} - // Hand-rolled instead of util.promisify: promisify picks the overload // without the options argument, and the cost parameters live there. function scryptAsync( @@ -182,35 +167,27 @@ function sessionCookieValid( * The single arbiter of who may call the API (/healthz stays open — * liveness probes have no secrets). Two credentials open the same door: * a Bearer credential (SDK, CLI, curl — the env token or any live API - * key, adjudicated by identifyCredential) and the web console's session + * key, adjudicated by isCredential) and the web console's session * cookie (which additionally requires the console header, see above). A * second route surface with its own auth would be a second truth. * - * identifyCredential judges bare tokens and answers who they are (the - * shared actor vocabulary; null = not a credential), so both faces (this - * Bearer header and the E2B X-API-KEY hook) feed it the same canonical - * form — one closure, one truth, two dialects. Admission and attribution - * are the same adjudication: the actor rides the request from here into - * every recordActivity the handler reaches. The 'Bearer ' prefix is public - * framing, not a secret, so stripping it needs no constant time; the - * secret comparisons live inside identifyCredential. + * isCredential judges bare tokens, so both faces (this Bearer header and + * the E2B X-API-KEY hook) feed it the same canonical form — one closure, + * one truth, two dialects. The 'Bearer ' prefix is public framing, not a + * secret, so stripping it needs no constant time; the secret comparisons + * live inside isCredential. */ export function requireApiAuth( - identifyCredential: (bareToken: string) => string | null, + isCredential: (bareToken: string) => boolean, getSessionSecret: () => string | null, ): onRequestAsyncHookHandler { return async (request, reply) => { const header = request.headers.authorization; const bare = header?.startsWith('Bearer ') ? header.slice(7) : null; - if (bare !== null) { - const actor = identifyCredential(bare); - if (actor !== null) { - request.actor = actor; - return; - } + if (bare !== null && isCredential(bare)) { + return; } if (sessionCookieValid(request, getSessionSecret)) { - request.actor = CONSOLE_ACTOR; return; } await reply.code(401).send({ message: 'missing or invalid API token' }); @@ -245,11 +222,9 @@ export function requireAdminAuth( const header = request.headers.authorization; const bare = header?.startsWith('Bearer ') ? header.slice(7) : null; if (bare !== null && isEnvToken(bare)) { - request.actor = ENV_TOKEN_ACTOR; return; } if (sessionCookieValid(request, getSessionSecret)) { - request.actor = CONSOLE_ACTOR; return; } if (bare !== null && isLiveApiKey(bare)) { diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index 666cfa82..fc00c959 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -135,7 +135,6 @@ describe('CheckIn', () => { template: null, metadata: null, spec: undefined, - actor: null, }); } const reading = await readNodeReading(db, new CpuSampler(), '/tmp'); diff --git a/packages/server/src/db/activity.ts b/packages/server/src/db/activity.ts deleted file mode 100644 index 3f3845bf..00000000 --- a/packages/server/src/db/activity.ts +++ /dev/null @@ -1,60 +0,0 @@ -import type { ActivityKind } from '@dormice/shared'; -import { desc, lte } from 'drizzle-orm'; -import type { Db } from './db'; -import { type ActivityRow, activity } from './schema'; - -/** - * How much history the ring keeps. Not a knob: nobody sizes an explanation - * window, and a bound this generous covers days of a busy single machine — - * anyone needing more than "what just happened" needs a real audit trail, - * which this deliberately is not. - */ -export const ACTIVITY_KEEP = 1000; - -export interface ActivityInput { - kind: ActivityKind; - /** The owning sandbox's name/id — prefixed: they reference another entity. */ - sandboxName?: string | null; - sandboxId?: string | null; - /** - * Which credential asked (shared/activity.ts vocabulary), threaded from - * request.actor by the route that took the request. Absent = null = the - * daemon's own doing — the honest default for the scanner, reconciler - * and archiver, which pass nothing. - */ - actor?: string | null; - detail: string; -} - -/** - * Appends one event and prunes the ring in the same breath. Synchronous - * like every ledger write, and deliberately unguarded: if the ledger can - * record state, it can record history — a failure here is the same disk - * catastrophe that would fail the transition itself. - */ -export function recordActivity(db: Db, input: ActivityInput): void { - const inserted = db - .insert(activity) - .values({ - at: new Date().toISOString(), - kind: input.kind, - sandboxName: input.sandboxName ?? null, - sandboxId: input.sandboxId ?? null, - actor: input.actor ?? null, - detail: input.detail, - }) - .run(); - db.delete(activity) - .where(lte(activity.id, Number(inserted.lastInsertRowid) - ACTIVITY_KEEP)) - .run(); -} - -/** Newest first — the question is always "what just happened". */ -export function listActivityEvents(db: Db, limit: number): ActivityRow[] { - return db - .select() - .from(activity) - .orderBy(desc(activity.id)) - .limit(limit) - .all(); -} diff --git a/packages/server/src/db/api-keys.ts b/packages/server/src/db/api-keys.ts index 546bc08d..0ddc1d7a 100644 --- a/packages/server/src/db/api-keys.ts +++ b/packages/server/src/db/api-keys.ts @@ -1,6 +1,5 @@ import { createHash, randomBytes, randomUUID } from 'node:crypto'; import { and, desc, eq, gt, isNull, lt, or, sql } from 'drizzle-orm'; -import { recordActivity } from './activity'; import type { Db } from './db'; import { type ApiKeyRow, apiKeys } from './schema'; @@ -43,7 +42,6 @@ export function createApiKey( db: Db, name: string, expiresAt: string | undefined, - actor: string | null, ): { row: ApiKeyRow; token: string } { const token = randomBytes(32).toString('hex'); const row: ApiKeyRow = { @@ -58,15 +56,6 @@ export function createApiKey( revokedAt: null, }; db.insert(apiKeys).values(row).run(); - recordActivity(db, { - kind: 'apikey-created', - // The admin gate means this actor can only be env-token or console — - // a key can never appear as the minter of another key. - actor, - detail: - `API key "${name}" (prefix ${row.prefix}) minted` + - (row.expiresAt ? `, expires ${row.expiresAt}` : ''), - }); return { row, token }; } @@ -105,11 +94,7 @@ export function listApiKeys(db: Db): ApiKeyRow[] { * or is already revoked — the desired end state was already true. The row * survives as history; the name is immediately free for a new key. */ -export function revokeApiKey( - db: Db, - id: string, - actor: string | null, -): boolean { +export function revokeApiKey(db: Db, id: string): boolean { const row = findApiKeyById(db, id); if (!row || row.revokedAt !== null) { return false; @@ -118,11 +103,6 @@ export function revokeApiKey( .set({ revokedAt: new Date().toISOString() }) .where(eq(apiKeys.id, id)) .run(); - recordActivity(db, { - kind: 'apikey-revoked', - actor, - detail: `API key "${row.name}" revoked`, - }); return true; } @@ -132,62 +112,35 @@ export function revokeApiKey( * only computes the changed-field set against the row it was handed and * writes once. A field equal to its current value is not a change (the * updatePolicy idiom: a no-op patch is the goal state, not an error), so - * disabling an already-disabled key keeps its original disabledAt and - * records nothing. One request can still yield two activity events — a - * disable that also renames is two facts, each separately filterable. + * disabling an already-disabled key keeps its original disabledAt. The + * returned row carries what changed; the route logs it. */ export function updateApiKey( db: Db, row: ApiKeyRow, patch: { name?: string; expiresAt?: string | null; disabled?: boolean }, - actor: string | null, ): ApiKeyRow { const changes: Partial = {}; - const facts: { - kind: 'apikey-updated' | 'apikey-disabled' | 'apikey-enabled'; - detail: string; - }[] = []; - const updated: string[] = []; - if (patch.name !== undefined && patch.name !== row.name) { changes.name = patch.name; - updated.push(`renamed to "${patch.name}"`); } if (patch.expiresAt !== undefined) { const next = patch.expiresAt === null ? null : normalizeIso(patch.expiresAt); if (next !== row.expiresAt) { changes.expiresAt = next; - updated.push(`expires ${row.expiresAt ?? 'never'} -> ${next ?? 'never'}`); } } - if (updated.length > 0) { - facts.push({ - kind: 'apikey-updated', - detail: `API key "${row.name}" ${updated.join(', ')}`, - }); - } if (patch.disabled === true && row.disabledAt === null) { changes.disabledAt = new Date().toISOString(); - facts.push({ - kind: 'apikey-disabled', - detail: `API key "${row.name}" disabled`, - }); } else if (patch.disabled === false && row.disabledAt !== null) { changes.disabledAt = null; - facts.push({ - kind: 'apikey-enabled', - detail: `API key "${row.name}" enabled`, - }); } if (Object.keys(changes).length === 0) { return row; } db.update(apiKeys).set(changes).where(eq(apiKeys.id, row.id)).run(); - for (const fact of facts) { - recordActivity(db, { ...fact, actor }); - } return { ...row, ...changes }; } @@ -232,10 +185,8 @@ export function isLiveApiKey(db: Db, bareToken: string): boolean { * sha256(key), which preimage resistance makes worthless to an attacker * (the argument GitHub token storage rests on). * - * A hit answers the key's id (attribution's raw material — the auth hook - * dresses it as an actor and rides it on the request) and stamps lastUsedAt - * — only a hit: verification is the one moment a credential was actually - * honored. Throttled to LAST_USED_GRANULARITY_MS so a polling client does + * A hit answers the key's id and stamps lastUsedAt — only a hit: + * verification is the one moment a credential was actually honored. Throttled to LAST_USED_GRANULARITY_MS so a polling client does * not write the ledger per request. ISO strings compare lexicographically * as timestamps, so the cutoff is a plain string <. */ diff --git a/packages/server/src/db/ledger.ts b/packages/server/src/db/ledger.ts index ba791e3e..7a70b87b 100644 --- a/packages/server/src/db/ledger.ts +++ b/packages/server/src/db/ledger.ts @@ -5,7 +5,6 @@ import { type SandboxState, } from '@dormice/shared'; import { count, eq } from 'drizzle-orm'; -import { recordActivity } from './activity'; import type { Db } from './db'; import { type SandboxRow, sandboxes } from './schema'; @@ -60,11 +59,6 @@ export interface CreateSandboxInput { deadlineAt: string; onDeadline: 'kill' | 'pause'; }; - /** - * Who asked (request.actor) — every creation is request-caused, so both - * faces pass this; absent only in tests that fabricate rows directly. - */ - actor?: string | null; } /** Inserts a new sandbox row in `active` state. Throws if the name is taken. */ @@ -94,16 +88,6 @@ export function createSandbox(db: Db, input: CreateSandboxInput): SandboxRow { pausedByUser: false, }; db.insert(sandboxes).values(row).run(); - // The one place every creation passes through, whichever face asked. - recordActivity(db, { - kind: 'created', - sandboxName: row.name, - sandboxId: row.id, - actor: input.actor, - detail: `${input.e2b ? 'via E2B create' : 'via acquireSandbox'}${ - input.template ? `, template ${input.template}` : '' - }`, - }); return row; } diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 031846e8..14c529c1 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -1,8 +1,4 @@ -import { - ACTIVITY_KINDS, - SANDBOX_STATES, - SHELL_EXIT_CAUSES, -} from '@dormice/shared'; +import { SANDBOX_STATES, SHELL_EXIT_CAUSES } from '@dormice/shared'; import { sql } from 'drizzle-orm'; import { index, @@ -107,35 +103,6 @@ export const templates = sqliteTable('templates', { export type TemplateRow = typeof templates.$inferSelect; -/** - * The activity ring: the ledger's recent history, one row per lifecycle - * event (created, cooled, woken, destroyed, repaired). Bounded by count — - * recordActivity prunes past the newest N — so it answers "what just - * happened" without ever becoming a second database to babysit. The - * autoincrement id is the ring position AND the newest-first sort key; - * unlike sandbox ids it never leaves this machine, so the UUID rule does - * not apply. - */ -export const activity = sqliteTable('activity', { - id: integer('id').primaryKey({ autoIncrement: true }), - /** ISO 8601 UTC. */ - at: text('at').notNull(), - kind: text('kind', { enum: ACTIVITY_KINDS }).notNull(), - /** Null for events with no owning sandbox (orphan sweeps, daemon start). */ - sandboxName: text('sandbox_name'), - sandboxId: text('sandbox_id'), - /** - * Which credential asked — the closed vocabulary in shared/activity.ts - * ('env-token' | 'console' | 'apikey:'). Null = no credential did: - * the daemon's own actors, plus rows from before attribution existed - * (the ring prunes those away within days). - */ - actor: text('actor'), - detail: text('detail').notNull(), -}); - -export type ActivityRow = typeof activity.$inferSelect; - /** * Per-sandbox metrics history, written by the background sampler every * DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS for each measurable (running or diff --git a/packages/server/src/e2b/compat.test.ts b/packages/server/src/e2b/compat.test.ts index 4f995315..41a0dce6 100644 --- a/packages/server/src/e2b/compat.test.ts +++ b/packages/server/src/e2b/compat.test.ts @@ -339,20 +339,6 @@ describe('E2B control plane', () => { payload: {}, }); expect(created.statusCode).toBe(201); - - // Both faces feed the same identity closure, so the created event names - // the key — the same attribution the native Bearer face gets. - const events = ( - await t.app.inject({ - method: 'POST', - url: '/listActivity', - headers: { authorization: `Bearer ${TOKEN}` }, - payload: {}, - }) - ).json().events as Array<{ kind: string; actor: string | null }>; - expect(events.find((e) => e.kind === 'created')?.actor).toBe( - `apikey:${minted.apiKey.id}`, - ); }); it('creates a fresh sandbox per call — E2B semantics, no key given', async () => { diff --git a/packages/server/src/e2b/control.ts b/packages/server/src/e2b/control.ts index 4d044198..5dacc236 100644 --- a/packages/server/src/e2b/control.ts +++ b/packages/server/src/e2b/control.ts @@ -34,7 +34,7 @@ import { apiError, E2bError, ENVD_VERSION, - identifyApiKey, + isApiKey, mintEnvdToken, } from './protocol'; import { e2bView } from './view'; @@ -102,7 +102,7 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( watchers, archiver, envdSigningSecret, - identifyCredential, + isCredential, }, ) => { /** @@ -134,12 +134,9 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( app.addHook('onRequest', async (request, reply) => { const presented = request.headers['x-api-key']; const key = Array.isArray(presented) ? presented[0] : presented; - const actor = identifyApiKey(identifyCredential, key); - if (actor === null) { + if (!isApiKey(isCredential, key)) { await reply.code(401).send({ code: 401, message: 'invalid API key' }); - return; } - request.actor = actor; }); // The E2B error dialect: every non-2xx body is { code, message } — the @@ -312,13 +309,7 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( // E2B clothes. Stored metadata/envs stay (same principle as the // native policy's "override applies at creation only"); the // deadline is extended like a connect. - const awake = await wakeSandbox( - db, - executor, - existing, - request.actor, - watchers, - ); + const awake = await wakeSandbox(db, executor, existing, watchers); extendDeadline(awake, timeoutSeconds); return touch(db, awake.id); } @@ -330,13 +321,12 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( executor, existing.id, archiver?.currentStore() ?? null, - { - kind: 'destroyed', - cause: 'protocol-dead row reaped by E2B create', - actor: request.actor, - }, watchers, ); + request.log.info( + { sandbox: name, id: existing.id }, + 'protocol-dead sandbox reaped by E2B create', + ); } const maxSandboxes = readRuntimeSettings(db).maxSandboxes; @@ -360,7 +350,6 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( archiveEnabled(db), ), template, - actor: request.actor, metadata: body.metadata && Object.keys(body.metadata).length > 0 ? JSON.stringify(body.metadata) @@ -398,13 +387,7 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( if (!fresh || e2bView(fresh, new Date()) === 'dead') { throw notFound(id); } - const awake = await wakeSandbox( - db, - executor, - fresh, - request.actor, - watchers, - ); + const awake = await wakeSandbox(db, executor, fresh, watchers); extendDeadline(awake, request.body.timeout ?? DEFAULT_TIMEOUT_SECONDS); return touch(db, awake.id); }); @@ -544,13 +527,12 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( executor, fresh.id, archiver?.currentStore() ?? null, - { - kind: 'destroyed', - cause: 'via E2B kill', - actor: request.actor, - }, watchers, ); + request.log.info( + { sandbox: fresh.name, id: fresh.id }, + 'sandbox destroyed via E2B kill', + ); }); return reply.code(204).send(); }); @@ -590,25 +572,12 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( if (!fresh) throw notFound(id); let current = fresh; if (current.state === 'active') { - current = await freezeSandbox( - db, - executor, - current.id, - 'paused via E2B', - request.actor, - ); + current = await freezeSandbox(db, executor, current.id); } // keepMemory:false maps to stopped: filesystem only, cold boot on // resume — physically exactly what E2B promises for it. if (request.body?.memory === false && current.state === 'frozen') { - await stopSandbox( - db, - executor, - current.id, - 'paused via E2B (memory discarded)', - request.actor, - watchers, - ); + await stopSandbox(db, executor, current.id, watchers); } setPausedByUser(db, fresh.id, true); }); diff --git a/packages/server/src/e2b/deps.ts b/packages/server/src/e2b/deps.ts index 3d27217d..7540459e 100644 --- a/packages/server/src/e2b/deps.ts +++ b/packages/server/src/e2b/deps.ts @@ -32,9 +32,8 @@ export interface E2bDeps { envdSigningSecret: string; /** * buildApp's one adjudication of "does this bare credential open the - * door" and who it is (the shared actor vocabulary; null = not a - * credential) — the same closure the native Bearer face uses, consulted - * here by the X-API-KEY hook, which rides the answer on request.actor. + * door" — the same closure the native Bearer face uses, consulted here + * by the X-API-KEY hook. */ - identifyCredential: (bareToken: string) => string | null; + isCredential: (bareToken: string) => boolean; } diff --git a/packages/server/src/e2b/envd/shared.ts b/packages/server/src/e2b/envd/shared.ts index dca4011a..0e64309f 100644 --- a/packages/server/src/e2b/envd/shared.ts +++ b/packages/server/src/e2b/envd/shared.ts @@ -222,13 +222,7 @@ export function createEnvdContext(deps: E2bDeps): EnvdContext { await joinRestore(sandboxId); return locks.run(before.name, async () => { const fresh = requireRunningRow(sandboxId); - const awake = await wakeSandbox( - db, - executor, - fresh, - undefined, - deps.watchers, - ); + const awake = await wakeSandbox(db, executor, fresh, deps.watchers); return touch(db, awake.id); }); } @@ -241,13 +235,7 @@ export function createEnvdContext(deps: E2bDeps): EnvdContext { await joinRestore(sandboxId); return locks.run(before.name, async () => { const fresh = requireRunningRow(sandboxId); - const awake = await wakeSandbox( - db, - executor, - fresh, - undefined, - deps.watchers, - ); + const awake = await wakeSandbox(db, executor, fresh, deps.watchers); const row = touch(db, awake.id); try { return await work(row); diff --git a/packages/server/src/e2b/protocol.ts b/packages/server/src/e2b/protocol.ts index 33ce28cd..00e1e668 100644 --- a/packages/server/src/e2b/protocol.ts +++ b/packages/server/src/e2b/protocol.ts @@ -86,17 +86,16 @@ export function verifyEnvdToken( * X-API-KEY check. The official SDK formats keys as `e2b_` and our * hex credentials are compliant as `e2b_`; the bare token is * accepted too — the prefix is the SDK's convention, not a secret. What - * opens the door — and who it is — is adjudicated by identifyCredential - * (buildApp's closure: env token or any active ledger API key), the same - * truth the native Bearer face consults; the answer is the actor (null = - * refused). + * opens the door is adjudicated by isCredential (buildApp's closure: env + * token or any active ledger API key), the same truth the native Bearer + * face consults. */ -export function identifyApiKey( - identifyCredential: (bareToken: string) => string | null, +export function isApiKey( + isCredential: (bareToken: string) => boolean, presented: string | undefined, -): string | null { +): boolean { const bare = presented?.startsWith('e2b_') ? presented.slice(4) : presented; - return bare === undefined ? null : identifyCredential(bare); + return bare !== undefined && isCredential(bare); } /** Connect streaming envelope flags: 0x00 = message, 0x02 = end of stream. */ diff --git a/packages/server/src/executor/fake.ts b/packages/server/src/executor/fake.ts index 7bc4b060..045fbff0 100644 --- a/packages/server/src/executor/fake.ts +++ b/packages/server/src/executor/fake.ts @@ -247,6 +247,13 @@ export class FakeExecutor implements Executor { } private readonly containers = new Map(); private readonly disks = new Set(); + /** + * Every removeContainer, in order: the shell swaps a wake made (the one + * way a live sandbox loses its container but keeps its disk). Public so + * a test can assert a rebuild happened — or, on the fast path, that + * none did — now that no activity ring records it. + */ + readonly removedShells: string[] = []; /** * The image each shell was born from. Keyed like containers, not disks: * an image is a property of the shell, set at its birth and gone with it @@ -490,6 +497,7 @@ export class FakeExecutor implements Executor { if (!hadContainer && !this.disks.has(sandboxId)) { throw new Error(`container ${sandboxId} is absent, cannot remove`); } + this.removedShells.push(sandboxId); this.images.delete(sandboxId); this.limits.delete(sandboxId); this.exits.delete(sandboxId); diff --git a/packages/server/src/ingress.test.ts b/packages/server/src/ingress.test.ts index 60736556..daa2ebed 100644 --- a/packages/server/src/ingress.test.ts +++ b/packages/server/src/ingress.test.ts @@ -243,20 +243,6 @@ describe('ingress routes', () => { const cleared = await rpc(app, '/setIngress', { domains: [] }); expect(cleared.statusCode).toBe(200); expect(cleared.json()).toEqual({ domains: [] }); - - const activity = await rpc(app, '/listActivity'); - const details = ( - activity.json().events as Array<{ kind: string; detail: string }> - ) - .filter((event) => event.kind === 'ingress-updated') - .map((event) => event.detail); - // Newest first: clear, drop, the double bind. - expect(details).toHaveLength(3); - expect(details[2]).toContain('bound console.example.com'); - expect(details[2]).toContain('bound api.example.com'); - expect(details[1]).toContain('unbound console.example.com'); - expect(details[1]).toContain('now serving api.example.com'); - expect(details[0]).toContain('plain-HTTP IP access only'); }); it('rejects a domain with a scheme at the schema gate', async () => { diff --git a/packages/server/src/lifecycle.ts b/packages/server/src/lifecycle.ts index ce0d2314..3cb675b5 100644 --- a/packages/server/src/lifecycle.ts +++ b/packages/server/src/lifecycle.ts @@ -1,6 +1,5 @@ import type { ShellExitCause } from '@dormice/shared'; import { type ArchiveStore, objectKey } from './archive/store'; -import { recordActivity } from './db/activity'; import type { Db } from './db/db'; import { deleteSandbox, @@ -29,51 +28,31 @@ import { resolveSpec, shellSpecOf } from './spec'; */ /** - * The lifecycle verbs also feed the activity ring here, after the ledger - * write — history is recorded where reality and ledger already move - * together, so no caller can forget it. `cause` is the caller's one line of - * context ("why"); `actor` is who asked (request.actor's vocabulary) — the - * daemon's own callers (scanner, reconciler) pass neither, and the honest - * defaults name the bare move and no credential. + * The verbs return the row they left behind and say nothing themselves: + * the caller with a logger (a route's request log, the heartbeat's + * summary in main.ts) is the one that speaks. The activity ring that once + * recorded every move here went with design record #16 (2026-09-13): a + * bounded SQLite history nobody queried, replaced by the daemon's own + * structured log in journald. */ export async function freezeSandbox( db: Db, executor: Executor, sandboxId: string, - cause?: string, - actor?: string | null, ): Promise { await executor.freeze(sandboxId); - const row = transition(db, sandboxId, 'frozen'); - recordActivity(db, { - kind: 'frozen', - sandboxName: row.name, - sandboxId, - actor, - detail: cause ?? 'memory squeezed into swap', - }); - return row; + return transition(db, sandboxId, 'frozen'); } export async function stopSandbox( db: Db, executor: Executor, sandboxId: string, - cause?: string, - actor?: string | null, watchers?: WatcherTable, ): Promise { await executor.stop(sandboxId); watchers?.disposeSandbox(sandboxId); - const row = transition(db, sandboxId, 'stopped'); - recordActivity(db, { - kind: 'stopped', - sandboxName: row.name, - sandboxId, - actor, - detail: cause ?? 'container torn down, disk kept', - }); - return row; + return transition(db, sandboxId, 'stopped'); } /** @@ -94,14 +73,6 @@ export async function destroySandbox( executor: Executor, sandboxId: string, store: ArchiveStore | null, - activity: { - kind: 'destroyed' | 'expired-killed'; - cause: string; - actor?: string | null; - } = { - kind: 'destroyed', - cause: 'via destroySandbox', - }, watchers?: WatcherTable, ): Promise { const row = findById(db, sandboxId); @@ -117,28 +88,12 @@ export async function destroySandbox( // With the disk gone its metrics history has no owner; fleet snapshots // belong to no sandbox and stay. deleteSandboxMetricsSamples(db, sandboxId); - recordActivity(db, { - kind: activity.kind, - sandboxName: row.name, - sandboxId, - actor: activity.actor, - detail: `${activity.cause}; archive object deleted`, - }); return; } await executor.destroy(sandboxId); watchers?.disposeSandbox(sandboxId); deleteSandbox(db, sandboxId); deleteSandboxMetricsSamples(db, sandboxId); - if (row) { - recordActivity(db, { - kind: activity.kind, - sandboxName: row.name, - sandboxId, - actor: activity.actor, - detail: activity.cause, - }); - } } /** @@ -154,39 +109,24 @@ export function causeOfExit(exit: ShellExit): ShellExitCause { return 'exited'; } -/** The same three verdicts in the words the activity feed uses. */ -export function describeExit(exit: ShellExit): string { - const cause = causeOfExit(exit); - if (cause === 'oom-killed') { - return ` (exit ${exit.exitCode}, OOM-killed by the kernel's memory cgroup)`; - } - if (cause === 'runtime-died') { - return ` (exit ${exit.exitCode}, not an OOM kill — gVisor's sentry itself died, the signature a pids-cap hit leaves; see the sandbox pids cap in settings)`; - } - return ` (exit ${exit.exitCode}, not an OOM kill)`; -} - /** * A shell that stopped under a row that never ordered a stop — a death. The * one place it is recorded, whoever noticed: the reconciler's heartbeat * (an idle sandbox nobody touches) or a wake that found the shell dead (a - * busy one, whose caller is about to use it). Both write the same three - * facts — state stopped, lastExit, a `reconciled` event carrying the - * exit — so the console, the wire and the activity feed tell one story. - * Watchers are disposed here too: a dead container has ended every - * inotifywait it hosted. `noticed` names the observer in the detail, the - * only thing that differs between the two — and only the observation is - * recorded here: the cold start a wake goes on to attempt is its own - * `woken` event once it has actually happened, never a claim made ahead - * of it. lastExit.at is the runtime's record of the exit (the death - * itself), which is why the row can say when a sandbox died even when the - * reconciler only found it a heartbeat later. + * busy one, whose caller is about to use it). Both write the same two + * facts — state stopped and lastExit — so the console and the wire tell + * one story. Watchers are disposed here too: a dead container has ended + * every inotifywait it hosted. Only the observation is recorded: the cold + * start a wake goes on to attempt is its own move once it has actually + * happened, never a claim made ahead of it. lastExit.at is the runtime's + * record of the exit (the death itself), which is why the row can say + * when a sandbox died even when the reconciler only found it a heartbeat + * later. */ export function recordShellDeath( db: Db, row: SandboxRow, exit: ShellExit, - noticed: 'by the reconciler' | 'at wake', watchers?: WatcherTable, ): void { watchers?.disposeSandbox(row.id); @@ -195,12 +135,6 @@ export function recordShellDeath( exitCode: exit.exitCode, cause: causeOfExit(exit), }); - recordActivity(db, { - kind: 'reconciled', - sandboxName: row.name, - sandboxId: row.id, - detail: `container is stopped — state ${row.state} corrected to stopped${describeExit(exit)}${noticed === 'at wake' ? ', found dead at wake' : ''}`, - }); } /** @@ -218,21 +152,10 @@ export async function rebuildSandbox( db: Db, executor: Executor, row: SandboxRow, - actor?: string | null, - detail?: string, watchers?: WatcherTable, ): Promise { await executor.removeContainer(row.id); watchers?.disposeSandbox(row.id); - recordActivity(db, { - kind: 'rebuilt', - sandboxName: row.name, - sandboxId: row.id, - actor, - detail: - detail ?? - 'shell removed, disk kept — next wake builds from the current image', - }); if (row.state === 'stopped') { return row; } @@ -270,7 +193,6 @@ export async function wakeSandbox( db: Db, executor: Executor, row: SandboxRow, - actor?: string | null, watchers?: WatcherTable, ): Promise { switch (row.state) { @@ -299,12 +221,12 @@ export async function wakeSandbox( await watchers?.reapDeferred(row.id); return row; } - recordShellDeath(db, row, exit, 'at wake', watchers); + recordShellDeath(db, row, exit, watchers); const dead = findById(db, row.id); if (dead === undefined) { throw new Error(`sandbox ${row.id} vanished while recording its death`); } - return wakeSandbox(db, executor, dead, actor, watchers); + return wakeSandbox(db, executor, dead, watchers); } case 'frozen': case 'stopped': { @@ -319,27 +241,18 @@ export async function wakeSandbox( // image is swapped regardless of what limits it was born with. const limits = born !== null && born === next ? await executor.limitsOf(row.id) : null; - const staleCause = - born !== null && born !== next - ? `stale shell swapped at wake: ${born} -> ${next}` - : limits !== null && - (limits.nanoCpus !== wantNanoCpus || - limits.memoryBytes !== wantMemoryBytes) - ? `stale shell swapped at wake: limits ${limits.nanoCpus / 1e9} cpus / ${limits.memoryBytes / 1024 ** 3} GiB -> ${spec.cpus} cpus / ${spec.memoryGb} GiB` - : null; - const fresh = - staleCause !== null - ? await rebuildSandbox(db, executor, row, actor, staleCause, watchers) - : row; + const stale = + (born !== null && born !== next) || + (limits !== null && + (limits.nanoCpus !== wantNanoCpus || + limits.memoryBytes !== wantMemoryBytes)); + const fresh = stale + ? await rebuildSandbox(db, executor, row, watchers) + : row; if (fresh.state === 'frozen') { await executor.unfreeze(fresh.id); await watchers?.reapDeferred(fresh.id); - return awaken( - db, - fresh, - 'from frozen (memory back out of swap)', - actor, - ); + return awaken(db, fresh); } // If no container object exists (pruned away, or the stale shell was // just removed), start rebuilds it from the current image and the @@ -349,7 +262,7 @@ export async function wakeSandbox( ...shellSpecOf(fresh), }); await watchers?.reapDeferred(fresh.id); - return awaken(db, fresh, 'cold start from the surviving disk', actor); + return awaken(db, fresh); } case 'archived': case 'restoring': @@ -367,22 +280,10 @@ export async function wakeSandbox( * so any explicit E2B pause mark is cleared along with the transition — * ledger honesty, not an E2B-surface concern leaking in. */ -function awaken( - db: Db, - row: SandboxRow, - how: string, - actor?: string | null, -): SandboxRow { +function awaken(db: Db, row: SandboxRow): SandboxRow { if (row.pausedByUser) { setPausedByUser(db, row.id, false); } const awake = transition(db, row.id, 'active'); - recordActivity(db, { - kind: 'woken', - sandboxName: row.name, - sandboxId: row.id, - actor, - detail: how, - }); return { ...awake, pausedByUser: false }; } diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index 6bc55694..edc0c972 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -7,7 +7,6 @@ import { Archiver } from './archive/archiver'; import { LedgerArchiveStore } from './archive/ledger-store'; import { CheckIn, readNodeReading } from './check-in'; import { type Config, loadConfig } from './config'; -import { recordActivity } from './db/activity'; import { migrateDb, openDb } from './db/db'; import { listSandboxes } from './db/ledger'; import { acquireSingleWriterLock } from './db/lock'; @@ -298,17 +297,15 @@ app.log.info(repaired, 'startup reconcile'); // for their next wake would have cost each of them one more death. const swept = await sweepPidsLimit(db, executor, locks, beat); app.log.info(swept, 'startup pids cap sweep'); -recordActivity(db, { - kind: 'daemon-started', - detail: - `executor ${config.DORMICE_EXECUTOR}; startup reconcile: ` + - `${repaired.repairedStates} states repaired, ${repaired.deletedRows} rows deleted, ` + - `${repaired.destroyedOrphans} orphan containers destroyed, ${repaired.removedDisks} disks removed; ` + - `pids cap: ${swept.updated} running shells brought to ${readRuntimeSettings(db).pidsLimit}` + - (swept.failures.length > 0 - ? `, ${swept.failures.length} refused (daemon log has the names)` - : ''), -}); +app.log.info( + { + executor: config.DORMICE_EXECUTOR, + reconcile: repaired, + pidsLimit: readRuntimeSettings(db).pidsLimit, + pidsSweep: { updated: swept.updated, refused: swept.failures.length }, + }, + 'daemon started', +); // Red line: the daemon binds to loopback only, and the host is deliberately // not configurable — a knob would be one typo away from 0.0.0.0. Exposing @@ -494,12 +491,10 @@ async function metricsTick() { // itself stays pure observation. const growth = await diskGrower?.check(); if (growth?.outcome === 'grown') { - recordActivity(db, { - kind: 'host-disk-grown', - detail: - `data disk device grew; filesystem resized ` + - `${gib(growth.fromBytes)} GiB -> ${gib(growth.toBytes)} GiB`, - }); + app.log.info( + { fromGib: gib(growth.fromBytes), toGib: gib(growth.toBytes) }, + 'data disk device grew; filesystem resized to fill it', + ); } } catch (error) { app.log.error(error, 'metrics sampler tick failed'); diff --git a/packages/server/src/reconciler.test.ts b/packages/server/src/reconciler.test.ts index 68ab1924..c9b01e28 100644 --- a/packages/server/src/reconciler.test.ts +++ b/packages/server/src/reconciler.test.ts @@ -2,7 +2,6 @@ import { randomUUID } from 'node:crypto'; import { fileURLToPath } from 'node:url'; import { DEFAULT_LIFECYCLE_POLICY } from '@dormice/shared'; import { describe, expect, it } from 'vitest'; -import { listActivityEvents } from './db/activity'; import { type Db, migrateDb, openDb } from './db/db'; import { createSandbox, findByName, transition } from './db/ledger'; import type { SandboxRow } from './db/schema'; @@ -88,19 +87,8 @@ describe('startup reconcile', () => { const result = await reconcile(db, executor, locks); expect(result).toEqual({ ...NONE, repairedStates: 2 }); - const details = listActivityEvents(db, 10) - .filter((e) => e.kind === 'reconciled') - .map((e) => [e.sandboxName, e.detail]); - expect(details).toContainEqual([ - 'alice', - "container is stopped — state active corrected to stopped (exit 137, OOM-killed by the kernel's memory cgroup)", - ]); - expect(details).toContainEqual([ - 'bob', - "container is stopped — state active corrected to stopped (exit 2, not an OOM kill — gVisor's sentry itself died, the signature a pids-cap hit leaves; see the sandbox pids cap in settings)", - ]); - // The same verdict lands on the row as lastExit — the wire's copy of - // the death, for the client whose stream just ended in EOF. + // The verdict lands on the row as lastExit — the wire's copy of the + // death, for the client whose stream just ended in EOF. expect(findByName(db, 'alice')).toMatchObject({ state: 'stopped', lastExitCode: 137, @@ -149,9 +137,11 @@ describe('startup reconcile', () => { await executor.stop(row.id); await reconcile(db, executor, locks); - expect(listActivityEvents(db, 1)[0]?.detail).toBe( - 'container is stopped — state frozen corrected to stopped (exit 137, not an OOM kill)', - ); + expect(findByName(db, 'alice')).toMatchObject({ + state: 'stopped', + lastExitCode: 137, + lastExitCause: 'exited', + }); }); it('repairs across rungs and drops ownership when reality is stopped', async () => { diff --git a/packages/server/src/reconciler.ts b/packages/server/src/reconciler.ts index 6392ec19..11ae7a1c 100644 --- a/packages/server/src/reconciler.ts +++ b/packages/server/src/reconciler.ts @@ -1,5 +1,4 @@ import type { SandboxState } from '@dormice/shared'; -import { recordActivity } from './db/activity'; import type { Db } from './db/db'; import { deleteSandbox, @@ -138,16 +137,6 @@ export async function reconcile( } }); - // Every applied repair is history worth explaining: reconciliation is - // the actor an operator least expects, so its moves go on the record. - const note = (row: SandboxRow, detail: string) => - recordActivity(db, { - kind: 'reconciled', - sandboxName: row.name, - sandboxId: row.id, - detail, - }); - for (const row of rows) { onProgress?.(); const observed = containers.get(row.id); @@ -158,13 +147,11 @@ export async function reconcile( await repairUnderLock(row, async () => { await executor.destroy(row.id); result.archivedSwept += 1; - note(row, 'leftover container of an archived sandbox destroyed'); }); } else if (disks.has(row.id)) { await repairUnderLock(row, async () => { await executor.removeDisk(row.id); result.archivedSwept += 1; - note(row, 'leftover disk of an archived sandbox removed'); }); } continue; @@ -178,20 +165,12 @@ export async function reconcile( await repairUnderLock(row, () => { overwriteState(db, row.id, LEDGER_STATE[observed]); result.repairedStates += 1; - note( - row, - `crashed restore had finished — recorded ${LEDGER_STATE[observed]}`, - ); }); } else { await repairUnderLock(row, async () => { await executor.removeDisk(row.id); overwriteState(db, row.id, 'archived'); result.repairedStates += 1; - note( - row, - 'crashed restore: half-built disk removed, back to archived', - ); }); } continue; @@ -206,21 +185,17 @@ export async function reconcile( // the shell (the other reader of deaths) is seen as the live // container it left, not repaired over from this pass's stale // snapshot. recordShellDeath writes what the wake would have: - // state, lastExit and the reconciled event, one story. + // state and lastExit, one story. if (observed === 'stopped') { const exit = await executor.exitOf(row.id); // Revived under us (or gone — the next pass sees that shape). if (exit === null) return; - recordShellDeath(db, row, exit, 'by the reconciler', watchers); + recordShellDeath(db, row, exit, watchers); result.repairedStates += 1; return; } overwriteState(db, row.id, LEDGER_STATE[observed]); result.repairedStates += 1; - note( - row, - `container is ${observed} — state ${row.state} corrected to ${LEDGER_STATE[observed]}`, - ); }); } } else if (disks.has(row.id)) { @@ -230,10 +205,6 @@ export async function reconcile( watchers?.disposeSandbox(row.id); overwriteState(db, row.id, 'stopped'); result.repairedStates += 1; - note( - row, - `container gone but disk survives (docker prune?) — recorded stopped, was ${row.state}`, - ); }); } } else { @@ -241,7 +212,6 @@ export async function reconcile( watchers?.disposeSandbox(row.id); deleteSandbox(db, row.id); result.deletedRows += 1; - note(row, 'container and disk both gone — row deleted, key freed'); }); } } @@ -254,11 +224,6 @@ export async function reconcile( if (priorSuspects === undefined || priorSuspects.has(sandboxId)) { await executor.destroy(sandboxId); result.destroyedOrphans += 1; - recordActivity(db, { - kind: 'reconciled', - sandboxId, - detail: 'unowned container destroyed — no ledger row points at it', - }); } else { result.suspects.push(sandboxId); } @@ -271,11 +236,6 @@ export async function reconcile( if (priorSuspects === undefined || priorSuspects.has(sandboxId)) { await executor.removeDisk(sandboxId); result.removedDisks += 1; - recordActivity(db, { - kind: 'reconciled', - sandboxId, - detail: 'unowned disk removed — neither a row nor a container owns it', - }); } else { result.suspects.push(sandboxId); } diff --git a/packages/server/src/routes/activity.ts b/packages/server/src/routes/activity.ts deleted file mode 100644 index cba6ee36..00000000 --- a/packages/server/src/routes/activity.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { - listActivityRequestSchema, - listActivityResponseSchema, -} from '@dormice/shared'; -import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; -import { listActivityEvents } from '../db/activity'; -import type { Db } from '../db/db'; - -export interface ActivityRoutesOptions { - db: Db; -} - -/** - * The ledger's recent history — the explanation window beside the - * observation windows (listSandboxes for now, getHostMetrics for the - * machine). Read-only over the activity ring; recording happens where the - * moves themselves happen. - */ -export const activityRoutes: FastifyPluginAsyncZod< - ActivityRoutesOptions -> = async (app, { db }) => { - app.post( - '/listActivity', - { - schema: { - body: listActivityRequestSchema, - response: { 200: listActivityResponseSchema }, - }, - }, - async (request) => ({ - events: listActivityEvents(db, request.body.limit), - }), - ); -}; diff --git a/packages/server/src/routes/api-keys.ts b/packages/server/src/routes/api-keys.ts index a75f476e..c086e528 100644 --- a/packages/server/src/routes/api-keys.ts +++ b/packages/server/src/routes/api-keys.ts @@ -45,7 +45,7 @@ function view(row: ApiKeyRow): ApiKey { * requireAdminAuth (env token or console session; a live key gets an * honest 403), because a credential must not manage the credential ledger * it lives in. Verification itself lives in db/api-keys.ts and is - * consulted by buildApp's identifyCredential closure, not here. + * consulted by buildApp's isCredential closure, not here. */ export const apiKeyRoutes: FastifyPluginAsyncZod = async ( app, @@ -71,7 +71,12 @@ export const apiKeyRoutes: FastifyPluginAsyncZod = async ( `an active API key named '${name}' already exists — revoke it first or pick another name`, ); } - const { row, token } = createApiKey(db, name, expiresAt, request.actor); + const { row, token } = createApiKey(db, name, expiresAt); + // The token itself never reaches the log. + request.log.info( + { apiKey: row.id, name, prefix: row.prefix, expiresAt: row.expiresAt }, + 'API key minted', + ); return { apiKey: view(row), token }; }, ); @@ -119,7 +124,14 @@ export const apiKeyRoutes: FastifyPluginAsyncZod = async ( ); } } - return { apiKey: view(updateApiKey(db, row, patch, request.actor)) }; + const updated = updateApiKey(db, row, patch); + if (updated !== row) { + request.log.info( + { apiKey: row.id, name: updated.name, patch }, + 'API key updated', + ); + } + return { apiKey: view(updated) }; }, ); @@ -131,8 +143,12 @@ export const apiKeyRoutes: FastifyPluginAsyncZod = async ( response: { 200: revokeApiKeyResponseSchema }, }, }, - async (request) => ({ - revoked: revokeApiKey(db, request.body.id, request.actor), - }), + async (request) => { + const revoked = revokeApiKey(db, request.body.id); + if (revoked) { + request.log.info({ apiKey: request.body.id }, 'API key revoked'); + } + return { revoked }; + }, ); }; diff --git a/packages/server/src/routes/ingress.ts b/packages/server/src/routes/ingress.ts index 478d6725..6dcf06aa 100644 --- a/packages/server/src/routes/ingress.ts +++ b/packages/server/src/routes/ingress.ts @@ -4,13 +4,10 @@ import { setIngressResponseSchema, } from '@dormice/shared'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; -import { recordActivity } from '../db/activity'; -import type { Db } from '../db/db'; import { httpError } from '../http-error'; import { type Ingress, UnmanagedIngressFileError } from '../ingress'; export interface IngressRoutesOptions { - db: Db; /** Present exactly when DORMICE_INGRESS_FILE is set (the archiver precedent). */ ingress?: Ingress; } @@ -23,7 +20,7 @@ export interface IngressRoutesOptions { */ export const ingressRoutes: FastifyPluginAsyncZod< IngressRoutesOptions -> = async (app, { db, ingress }) => { +> = async (app, { ingress }) => { app.post( '/getIngress', { @@ -72,13 +69,12 @@ export const ingressRoutes: FastifyPluginAsyncZod< .filter((domain) => !domains.includes(domain)) .map((domain) => `unbound ${domain}`), ]; - recordActivity(db, { - kind: 'ingress-updated', - actor: request.actor, - detail: `${changes.join(', ') || 'domains unchanged'} — now serving ${ + request.log.info( + { changes, domains }, + `ingress updated: ${changes.join(', ') || 'domains unchanged'} — now serving ${ domains.length ? domains.join(', ') : 'plain-HTTP IP access only' }`, - }); + ); return { domains }; }, ); diff --git a/packages/server/src/routes/observability.test.ts b/packages/server/src/routes/observability.test.ts index 0f15f875..541e6fc0 100644 --- a/packages/server/src/routes/observability.test.ts +++ b/packages/server/src/routes/observability.test.ts @@ -3,39 +3,32 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { - type ActivityEvent, type ConfigEntry, getConfigResponseSchema, getFleetTimelineResponseSchema, getHostMetricsHistoryResponseSchema, getSandboxMetricsHistoryResponseSchema, getSandboxMetricsResponseSchema, - listActivityResponseSchema, listSandboxImagesResponseSchema, listSandboxMetricsResponseSchema, } from '@dormice/shared'; -import { count } from 'drizzle-orm'; import { describe, expect, it } from 'vitest'; import { buildApp } from '../app'; import { Archiver } from '../archive/archiver'; import { MemStore } from '../archive/mem-store'; import { CONFIG_KEYS, type ConfigSources, loadConfig } from '../config'; -import { ACTIVITY_KEEP, recordActivity } from '../db/activity'; import { migrateDb, openDb } from '../db/db'; import { insertMetricsTick, MAX_POINTS } from '../db/metrics'; -import { activity } from '../db/schema'; import { FAKE_BASE_IMAGE, FakeExecutor } from '../executor/fake'; import { CpuSampler, type HostSample } from '../host-metrics'; import { KeyedQueue } from '../keyed-queue'; import { freezeSandbox, stopSandbox } from '../lifecycle'; import { sampleOnce } from '../metrics-sampler'; import { ARCHIVE_DEFAULT_SECONDS } from '../policy'; -import { reconcile } from '../reconciler'; -import { scanOnce } from '../scanner'; -// The three observability verbs, app-level: getConfig, listActivity, -// getSandboxMetrics — the console's food, so the tests eat exactly what a -// browser would. +// The observability verbs, app-level: getConfig, getSandboxMetrics and +// the history windows — the console's food, so the tests eat exactly what +// a browser would. const MIGRATIONS = fileURLToPath(new URL('../../drizzle', import.meta.url)); const TOKEN = 'test-token-test-token-test-token'; @@ -75,12 +68,6 @@ function rpc(app: App, url: string, payload: Record = {}) { return app.inject({ method: 'POST', url, headers: authed, payload }); } -async function events(app: App): Promise { - const res = await rpc(app, '/listActivity'); - expect(res.statusCode).toBe(200); - return listActivityResponseSchema.parse(res.json()).events; -} - // One tick's non-sandbox inputs. A fresh CpuSampler per call is fine: its // delta-less first reading is an honest null; the data dir doesn't exist, // so disk is null too. @@ -192,92 +179,6 @@ describe('getConfig', () => { }); }); -describe('listActivity', () => { - it('records create, wake, cooling and release, newest first', async () => { - const { app, db, executor, locks } = testApp(); - const res = await rpc(app, '/acquireSandbox', { - name: 'story', - policy: { freezeAfterSeconds: 5, stopAfterSeconds: 10 }, - }); - expect(res.statusCode).toBe(200); - const created = res.json().sandbox; - - // Cool it two rungs by time travel, then wake it back through acquire. - await scanOnce( - db, - executor, - locks, - new Date(Date.parse(created.lastActiveAt) + 6_000), - ); - await scanOnce( - db, - executor, - locks, - new Date(Date.parse(created.lastActiveAt) + 11_000), - ); - await rpc(app, '/acquireSandbox', { name: 'story' }); - await rpc(app, '/destroySandbox', { name: 'story' }); - - const log = await events(app); - expect(log.map((e) => e.kind)).toEqual([ - 'destroyed', - 'woken', - 'stopped', - 'frozen', - 'created', - ]); - // Every event names its sandbox, and the scanner names its threshold. - expect(new Set(log.map((e) => e.sandboxName))).toEqual(new Set(['story'])); - expect(log.find((e) => e.kind === 'frozen')?.detail).toContain('scanner'); - expect(log.find((e) => e.kind === 'created')?.detail).toContain( - 'acquireSandbox', - ); - }); - - it('records what reconciliation repaired', async () => { - const { app, db, executor, locks } = testApp(); - await rpc(app, '/acquireSandbox', { name: 'doomed' }); - const { sandboxes } = (await rpc(app, '/listSandboxes')).json(); - // Reality loses both container and disk behind the ledger's back. - await executor.destroy(sandboxes[0].id); - await reconcile(db, executor, locks); - - const log = await events(app); - expect(log[0]).toMatchObject({ kind: 'reconciled', sandboxName: 'doomed' }); - expect(log[0]?.detail).toContain('row deleted'); - }); - - it('honors the limit and keeps the ring bounded', async () => { - const { app, db } = testApp(); - for (let i = 0; i < ACTIVITY_KEEP + 50; i += 1) { - recordActivity(db, { kind: 'daemon-started', detail: `tick ${i}` }); - } - const page = listActivityResponseSchema.parse( - (await rpc(app, '/listActivity', { limit: 3 })).json(), - ).events; - expect(page).toHaveLength(3); - expect(page[0]?.detail).toBe(`tick ${ACTIVITY_KEEP + 49}`); - - // The bound must live in the TABLE, not in the page clamp: a missing - // prune with limit=1000 would return the same page — count the rows. - const total = db.select({ n: count() }).from(activity).get() as { - n: number; - }; - expect(total.n).toBe(ACTIVITY_KEEP); - const all = listActivityResponseSchema.parse( - (await rpc(app, '/listActivity', { limit: 1000 })).json(), - ).events; - // The oldest 50 fell off the ring. - expect(all.at(-1)?.detail).toBe('tick 50'); - }); - - it('rejects an out-of-range limit', async () => { - const { app } = testApp(); - const res = await rpc(app, '/listActivity', { limit: 0 }); - expect(res.statusCode).toBe(400); - }); -}); - describe('getSandboxMetrics', () => { it('answers a single sample for a running sandbox', async () => { const { app } = testApp(); diff --git a/packages/server/src/routes/sandboxes.ts b/packages/server/src/routes/sandboxes.ts index 7488481d..98890722 100644 --- a/packages/server/src/routes/sandboxes.ts +++ b/packages/server/src/routes/sandboxes.ts @@ -51,7 +51,6 @@ import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { ZodError, z } from 'zod'; import type { Archiver, RestoreProgress } from '../archive/archiver'; import type { Config } from '../config'; -import { recordActivity } from '../db/activity'; import type { Db } from '../db/db'; import { countSandboxes, @@ -184,7 +183,6 @@ export const sandboxRoutes: FastifyPluginAsyncZod< template: string | null, metadata: string | null, spec: SandboxSpecOverride | undefined, - actor: string | null, ): Promise { const existing = findByName(db, name); if (existing?.state === 'archived' || existing?.state === 'restoring') { @@ -222,7 +220,7 @@ export const sandboxRoutes: FastifyPluginAsyncZod< // Requested template and metadata are not applied: like policy, they // take effect only when this acquire creates the sandbox (metadata // has its own update verb, updateMetadata). - const awake = await wakeSandbox(db, executor, existing, actor, watchers); + const awake = await wakeSandbox(db, executor, existing, watchers); return { status: 'ready', created: false, row: touch(db, awake.id) }; } @@ -262,7 +260,6 @@ export const sandboxRoutes: FastifyPluginAsyncZod< template, metadata, spec, - actor, }), }; } @@ -272,10 +269,7 @@ export const sandboxRoutes: FastifyPluginAsyncZod< // is more likely a typo than an intent to build a sandbox as a side // effect — then wake whatever cold state the sandbox is in and refresh // its idle clock. Must be called while holding the key's queue slot. - async function wakeForUse( - name: string, - actor: string | null, - ): Promise { + async function wakeForUse(name: string): Promise { const existing = findByName(db, name); if (!existing) { throw httpError(404, `no sandbox named "${name}" — acquire it first`); @@ -288,7 +282,7 @@ export const sandboxRoutes: FastifyPluginAsyncZod< `sandbox "${name}" is ${existing.state} — call acquireSandbox and poll until it is ready`, ); } - const awake = await wakeSandbox(db, executor, existing, actor, watchers); + const awake = await wakeSandbox(db, executor, existing, watchers); return touch(db, awake.id); } @@ -356,7 +350,6 @@ export const sandboxRoutes: FastifyPluginAsyncZod< template ?? null, serializeMetadata(metadata), spec, - request.actor, ), ); if (outcome.status === 'restoring') { @@ -569,7 +562,7 @@ export const sandboxRoutes: FastifyPluginAsyncZod< // whole duration. The heartbeat keeps the scanner away; a concurrent // destroy mid-exec removes the container and this exec fails with // the executor's honest error — accepted, not defended against. - const row = await locks.run(name, () => wakeForUse(name, request.actor)); + const row = await locks.run(name, () => wakeForUse(name)); const stopHeartbeat = startExecHeartbeat( db, @@ -623,7 +616,7 @@ export const sandboxRoutes: FastifyPluginAsyncZod< async (request) => { const { name, files } = request.body; return locks.run(name, async () => { - const row = await wakeForUse(name, request.actor); + const row = await wakeForUse(name); try { await executor.writeFiles( row.id, @@ -658,7 +651,7 @@ export const sandboxRoutes: FastifyPluginAsyncZod< async (request) => { const { name, path, contentBase64 } = request.body; return locks.run(name, async () => { - const row = await wakeForUse(name, request.actor); + const row = await wakeForUse(name); try { await executor.writeFiles(row.id, [ { path, content: Buffer.from(contentBase64, 'base64') }, @@ -683,7 +676,7 @@ export const sandboxRoutes: FastifyPluginAsyncZod< async (request) => { const { name, path } = request.body; return locks.run(name, async () => { - const row = await wakeForUse(name, request.actor); + const row = await wakeForUse(name); let content: Buffer; try { content = await executor.readFile(row.id, path); @@ -713,7 +706,7 @@ export const sandboxRoutes: FastifyPluginAsyncZod< async (request) => { const { name, paths } = request.body; return locks.run(name, async () => { - const row = await wakeForUse(name, request.actor); + const row = await wakeForUse(name); const files: { path: string; contentBase64: string }[] = []; let totalBytes = 0; for (const path of paths) { @@ -769,14 +762,8 @@ export const sandboxRoutes: FastifyPluginAsyncZod< `sandbox "${name}" is ${existing.state} — it has no container to rebuild`, ); } - const row = await rebuildSandbox( - db, - executor, - existing, - request.actor, - undefined, - watchers, - ); + const row = await rebuildSandbox(db, executor, existing, watchers); + request.log.info({ sandbox: name, id: row.id }, 'sandbox rebuilt'); return { sandbox: view(row) }; }); }, @@ -855,18 +842,15 @@ export const sandboxRoutes: FastifyPluginAsyncZod< const row = updatePolicy(db, existing.id, merged.data); const fmt = (seconds: number | null) => seconds === null ? 'never' : `${seconds}s`; - recordActivity(db, { - kind: 'policy-changed', - sandboxName: name, - sandboxId: row.id, - actor: request.actor, - detail: changed + request.log.info( + { sandbox: name, id: row.id, policy: merged.data }, + `sandbox policy changed: ${changed .map( (knob) => `${knob.replace('AfterSeconds', '')} ${fmt(before[knob])} -> ${fmt(merged.data[knob])}`, ) - .join(', '), - }); + .join(', ')}`, + ); return { row }; }, ); @@ -925,13 +909,10 @@ export const sandboxRoutes: FastifyPluginAsyncZod< ? [`memoryGb ${fmt(existing.memoryGb)} -> ${fmt(merged.memoryGb)}`] : []), ]; - recordActivity(db, { - kind: 'spec-changed', - sandboxName: name, - sandboxId: updated.id, - actor: request.actor, - detail: `${changed.join(', ')}; applies at the next cold wake`, - }); + request.log.info( + { sandbox: name, id: updated.id, spec: merged }, + `sandbox spec changed: ${changed.join(', ')}; applies at the next cold wake`, + ); return updated; }); return { sandbox: view(row) }; @@ -979,13 +960,10 @@ export const sandboxRoutes: FastifyPluginAsyncZod< } const updated = updateTemplate(db, existing.id, template); const fmt = (t: string | null) => (t === null ? 'base image' : t); - recordActivity(db, { - kind: 'template-changed', - sandboxName: name, - sandboxId: updated.id, - actor: request.actor, - detail: `template ${fmt(existing.template)} -> ${fmt(template)}; applies at the next cold wake`, - }); + request.log.info( + { sandbox: name, id: updated.id, template }, + `sandbox template changed: ${fmt(existing.template)} -> ${fmt(template)}; applies at the next cold wake`, + ); return updated; }); return { sandbox: view(row) }; @@ -1049,15 +1027,12 @@ export const sandboxRoutes: FastifyPluginAsyncZod< await executor.growDisk(existing.id, diskGb); } const row = setDiskGb(db, existing.id, diskGb); - recordActivity(db, { - kind: 'disk-expanded', - sandboxName: name, - sandboxId: row.id, - actor: request.actor, - // The equal-size pin deserves its own words: "10 GiB -> 10 GiB" - // reads like nothing happened, but something did — the sandbox - // left the fleet-wide knob for a size of its own. - detail: `${ + // The equal-size pin deserves its own words: "10 GiB -> 10 GiB" + // reads like nothing happened, but something did — the sandbox + // left the fleet-wide knob for a size of its own. + request.log.info( + { sandbox: name, id: row.id, diskGb }, + `sandbox ${ diskGb === current ? `disk pinned at ${diskGb} GiB (was following the global default)` : `disk ${current} GiB -> ${diskGb} GiB` @@ -1066,7 +1041,7 @@ export const sandboxRoutes: FastifyPluginAsyncZod< ? ' (ledger only — the restore opens the disk at the recorded size)' : '' }`, - }); + ); return { row }; }, ); @@ -1101,16 +1076,10 @@ export const sandboxRoutes: FastifyPluginAsyncZod< return existing; } const updated = updateMetadata(db, existing.id, serialized); - recordActivity(db, { - kind: 'metadata-changed', - sandboxName: name, - sandboxId: updated.id, - actor: request.actor, - detail: - Object.entries(metadata) - .map(([key, value]) => `${key}=${value}`) - .join(', ') || 'cleared', - }); + request.log.info( + { sandbox: name, id: updated.id, metadata }, + 'sandbox metadata replaced', + ); return updated; }); return { sandbox: view(row) }; @@ -1187,13 +1156,12 @@ export const sandboxRoutes: FastifyPluginAsyncZod< executor, existing.id, archiver?.currentStore() ?? null, - { - kind: 'destroyed', - cause: 'via destroySandbox', - actor: request.actor, - }, watchers, ); + request.log.info( + { sandbox: name, id: existing.id }, + 'sandbox destroyed', + ); return { destroyed: true }; }); }, diff --git a/packages/server/src/routes/settings.test.ts b/packages/server/src/routes/settings.test.ts index 3aab0e07..a33d395f 100644 --- a/packages/server/src/routes/settings.test.ts +++ b/packages/server/src/routes/settings.test.ts @@ -2,7 +2,6 @@ import { fileURLToPath } from 'node:url'; import { DEFAULT_LIFECYCLE_POLICY, getConfigResponseSchema, - listActivityResponseSchema, updateSettingsResponseSchema, } from '@dormice/shared'; import { sql } from 'drizzle-orm'; @@ -278,16 +277,8 @@ describe('updateSettings', () => { expect( (await rpc(app, '/updateSettings', { pidsLimit: 0 })).statusCode, ).toBe(400); - - const events = listActivityResponseSchema.parse( - (await rpc(app, '/listActivity')).json(), - ).events; - expect(events[0]).toMatchObject({ - kind: 'settings-updated', - detail: 'pidsLimit=256', - }); }); - it('sets the pids cap live, floors it, and records the change', async () => { + it('sets the pids cap live and floors it', async () => { const app = appOn(freshDb()); const set = await rpc(app, '/updateSettings', { pidsLimit: 8192 }); expect(set.statusCode).toBe(200); @@ -295,13 +286,6 @@ describe('updateSettings', () => { updateSettingsResponseSchema.parse(set.json()).settings.pidsLimit, ).toBe(8192); expect((await settingsOf(app)).pidsLimit).toBe(8192); - const events = listActivityResponseSchema.parse( - (await rpc(app, '/listActivity')).json(), - ).events; - expect(events[0]).toMatchObject({ - kind: 'settings-updated', - detail: 'pidsLimit=8192', - }); // Below the floor a sandbox cannot boot its own runtime — refused, and // the ledger keeps the value in force. "Unlimited" has no spelling. @@ -331,10 +315,6 @@ describe('updateSettings', () => { expect( updateSettingsResponseSchema.parse(raised.json()).settings.pidsLimit, ).toBe(4096); - const events = listActivityResponseSchema.parse( - (await rpc(upgraded, '/listActivity')).json(), - ).events; - expect(events[0]?.detail).toBe('pidsLimit=4096'); // The ledger has spoken: a later env edit is ignored. const later = appOn(db, { DORMICE_SANDBOX_PIDS_LIMIT: '999' }); @@ -540,14 +520,6 @@ describe('updateSettings', () => { expect(tooLow.statusCode).toBe(400); expect(tooLow.json().message).toMatch(/at least 256/); expect((await settingsOf(app)).pidsLimit).toBe(4096); - - const events = listActivityResponseSchema.parse( - (await rpc(app, '/listActivity')).json(), - ).events; - expect(events[0]).toMatchObject({ - kind: 'settings-updated', - detail: 'pidsLimit=4096', - }); }); it('pidsLimit: running sandboxes follow the write in place, frozen ones at their wake', async () => { @@ -628,19 +600,6 @@ describe('updateSettings', () => { expect(saved.pidsLimit).toBe(2048); }); - it('records the change in the activity ring with its actor', async () => { - const app = appOn(freshDb()); - await rpc(app, '/updateSettings', { maxSandboxes: 3 }); - const events = listActivityResponseSchema.parse( - (await rpc(app, '/listActivity')).json(), - ).events; - expect(events[0]).toMatchObject({ - kind: 'settings-updated', - actor: 'env-token', - detail: 'maxSandboxes=3', - }); - }); - it('is admin-only: an API key gets an honest 403', async () => { const app = appOn(freshDb()); const minted = await rpc(app, '/createApiKey', { name: 'robot' }); @@ -792,20 +751,10 @@ describe('updateSettings: the S3 archive store', () => { expect((await settingsOf(app)).s3?.bucket).toBe('patched-bucket'); }); - it('names the store, never the keys, in the activity ring', async () => { + it('never echoes the keys: the write answers with the view shape', async () => { const app = appOn(freshDb()); - await rpc(app, '/updateSettings', { s3: S3_PATCH }); - await rpc(app, '/updateSettings', { s3: null }); - const res = await rpc(app, '/listActivity'); - const events = listActivityResponseSchema.parse(res.json()).events; - expect(events[0]).toMatchObject({ - kind: 'settings-updated', - detail: 's3=cleared', - }); - expect(events[1]).toMatchObject({ - kind: 'settings-updated', - detail: 's3=http://127.0.0.1:9000/patched-bucket', - }); + const res = await rpc(app, '/updateSettings', { s3: S3_PATCH }); + expect(res.statusCode).toBe(200); expect(res.body).not.toContain('patch-secret-never-on-the-wire'); expect(res.body).not.toContain('patch-key'); }); @@ -826,12 +775,6 @@ describe('updateSettings: the sandbox domain', () => { const cleared = await rpc(app, '/updateSettings', { sandboxDomain: null }); expect(cleared.statusCode).toBe(200); expect((await settingsOf(app)).sandboxDomain).toBeNull(); - - const events = listActivityResponseSchema.parse( - (await rpc(app, '/listActivity')).json(), - ).events; - expect(events[0]?.detail).toBe('sandboxDomain=cleared'); - expect(events[1]?.detail).toBe('sandboxDomain=sbx.example.com'); }); it('refuses anything but a bare hostname', async () => { @@ -874,14 +817,6 @@ describe('updateSettings: the sandbox domain', () => { }); expect(cleared.statusCode).toBe(200); expect((await settingsOf(app)).sandboxDomainAliases).toEqual([]); - - const events = listActivityResponseSchema.parse( - (await rpc(app, '/listActivity')).json(), - ).events; - expect(events[0]?.detail).toBe('sandboxDomainAliases=cleared'); - expect(events[1]?.detail).toBe( - 'sandboxDomainAliases=a.example.com/b.example.com', - ); }); it('refuses alias lists that contradict the post-patch state, honestly', async () => { diff --git a/packages/server/src/routes/settings.ts b/packages/server/src/routes/settings.ts index 92110ab2..b1fef8c1 100644 --- a/packages/server/src/routes/settings.ts +++ b/packages/server/src/routes/settings.ts @@ -6,7 +6,6 @@ import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { z } from 'zod'; import { probeS3 as defaultProbeS3, S3ProbeError } from '../archive/probe'; import type { S3Settings } from '../archive/s3-store'; -import { recordActivity } from '../db/activity'; import type { Db } from '../db/db'; import { countByState, listSandboxes } from '../db/ledger'; import { @@ -190,10 +189,9 @@ export const settingsRoutes: FastifyPluginAsyncZod< } } const settings = writeRuntimeSettings(db, patch, new Date()); - recordActivity(db, { - kind: 'settings-updated', - actor: request.actor, - detail: [ + request.log.info( + { settings }, + `runtime settings updated: ${[ ...(patch.maxSandboxes !== undefined ? [`maxSandboxes=${patch.maxSandboxes}`] : []), @@ -208,8 +206,8 @@ export const settingsRoutes: FastifyPluginAsyncZod< ] : []), ...(patch.swapGb !== undefined ? [`swapGb=${patch.swapGb}`] : []), - // Endpoint and bucket only — the keys never reach the activity - // feed, the same "value never crosses" rule as the wire's. + // Endpoint and bucket only — the keys never reach the log, the + // same "value never crosses" rule as the wire's. ...(patch.s3 !== undefined ? [ patch.s3 === null @@ -231,8 +229,8 @@ export const settingsRoutes: FastifyPluginAsyncZod< ...(patch.pidsLimit !== undefined ? [`pidsLimit=${patch.pidsLimit}`] : []), - ].join(', '), - }); + ].join(', ')}`, + ); // Reconcile the host after the write — each knob with a reality out // there on its own, neither's failure sparing the other: the ledger // holds both targets now, and a swapfile that would not grow says diff --git a/packages/server/src/routes/spec.test.ts b/packages/server/src/routes/spec.test.ts index 9fe06f5a..de3efead 100644 --- a/packages/server/src/routes/spec.test.ts +++ b/packages/server/src/routes/spec.test.ts @@ -91,13 +91,6 @@ function acquire( return rpc(app, '/acquireSandbox', payload); } -async function activityKinds(app: ReturnType['app']) { - const events = (await rpc(app, '/listActivity')).json().events as Array<{ - kind: string; - }>; - return events.map((event) => event.kind); -} - /** Time travel for the scanner, app.test.ts's helper. */ function after(iso: string, seconds: number): Date { return new Date(Date.parse(iso) + seconds * 1000); @@ -177,7 +170,6 @@ describe('POST /updateSpec', () => { // A pure ledger write: state untouched, idle clock NOT refreshed. expect(res.json().sandbox.state).toBe('active'); expect(res.json().sandbox.lastActiveAt).toBe(before.lastActiveAt); - expect(await activityKinds(app)).toContain('spec-changed'); }); it('omitted knobs keep their values; null pins back to the global default', async () => { @@ -191,11 +183,16 @@ describe('POST /updateSpec', () => { expect(findByName(db, 'alice')?.cpus).toBeNull(); }); - it('a no-op patch writes no history', async () => { - const { app } = testApp(); + it('a no-op patch is the goal state: 200, nothing rewritten', async () => { + const { app, db } = testApp(); await acquire(app, { name: 'alice', spec: { cpus: 2 } }); - await rpc(app, '/updateSpec', { name: 'alice', spec: { cpus: 2 } }); - expect(await activityKinds(app)).not.toContain('spec-changed'); + const before = findByName(db, 'alice'); + const res = await rpc(app, '/updateSpec', { + name: 'alice', + spec: { cpus: 2 }, + }); + expect(res.statusCode).toBe(200); + expect(findByName(db, 'alice')).toEqual(before); }); it('answers 404 for an unknown key — updateSpec is not a creator', async () => { @@ -233,17 +230,14 @@ describe('POST /updateSpec', () => { nanoCpus: 2e9, memoryBytes: 2 * 1024 ** 3, }); - expect(await activityKinds(app)).toContain('rebuilt'); + expect(executor.removedShells).toEqual([id]); // Freeze again with the spec unchanged: the wake must NOT rebuild — // the millisecond unpause path stays untouched. const again = (await rpc(app, '/listSandboxes')).json().sandboxes[0]; await scanOnce(db, executor, locks, after(again.lastActiveAt, 1)); await acquire(app, { name: 'alice' }); - const rebuilds = (await activityKinds(app)).filter( - (kind) => kind === 'rebuilt', - ); - expect(rebuilds).toHaveLength(1); + expect(executor.removedShells).toEqual([id]); }); }); @@ -298,7 +292,7 @@ describe('a global default edit through the cold-wake convergence', () => { nanoCpus: 2e9, memoryBytes: 4 * 1024 ** 3, }); - expect(await activityKinds(app)).toContain('rebuilt'); + expect(executor.removedShells).toEqual([id]); // The regression this test exists for: the fresh shell was born from // the LIVE defaults, so the second wake must not rebuild again — an @@ -307,10 +301,7 @@ describe('a global default edit through the cold-wake convergence', () => { const again = (await rpc(app, '/listSandboxes')).json().sandboxes[0]; await scanOnce(db, executor, locks, after(again.lastActiveAt, 1)); await acquire(app, { name: 'alice' }); - const rebuilds = (await activityKinds(app)).filter( - (kind) => kind === 'rebuilt', - ); - expect(rebuilds).toHaveLength(1); + expect(executor.removedShells).toEqual([id]); }); }); @@ -324,7 +315,6 @@ describe('POST /expandDisk', () => { expect((await executor.metrics(created.sandbox.id)).diskTotalBytes).toBe( 20 * 1024 ** 3, ); - expect(await activityKinds(app)).toContain('disk-expanded'); }); it('refuses to shrink with a 400', async () => { @@ -335,17 +325,15 @@ describe('POST /expandDisk', () => { expect(res.json().message).toMatch(/only grows/); }); - it('asking for the pinned size again is a no-op success, no history', async () => { - const { app } = testApp(); + it('asking for the pinned size again is a no-op success', async () => { + const { app, db } = testApp(); await acquire(app, { name: 'alice' }); await rpc(app, '/expandDisk', { name: 'alice', diskGb: 20 }); + const before = findByName(db, 'alice'); const res = await rpc(app, '/expandDisk', { name: 'alice', diskGb: 20 }); expect(res.statusCode).toBe(200); expect(res.json().sandbox.spec.diskGb).toBe(20); - const expansions = (await activityKinds(app)).filter( - (kind) => kind === 'disk-expanded', - ); - expect(expansions).toHaveLength(1); + expect(findByName(db, 'alice')).toEqual(before); }); it('asking for the default size on an unpinned row pins it', async () => { diff --git a/packages/server/src/routes/upgrade.ts b/packages/server/src/routes/upgrade.ts index 6998f91b..07bbd45b 100644 --- a/packages/server/src/routes/upgrade.ts +++ b/packages/server/src/routes/upgrade.ts @@ -7,13 +7,10 @@ import { getUpgradeStatusResponseSchema, } from '@dormice/shared'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; -import { recordActivity } from '../db/activity'; -import type { Db } from '../db/db'; import type { Updater } from '../updater'; export interface UpgradeRoutesOptions { updater: Updater; - db: Db; } /** @@ -22,12 +19,12 @@ export interface UpgradeRoutesOptions { * reaches the network exactly when asked — no background phone-home — and * a server-side cache keeps repeats cheap. Applying hands install.sh to a * systemd transient unit that outlives the daemon's own restart; only the - * launch is recorded in the activity ring, because the daemon that would - * record "finished" is the one being replaced. + * launch is logged here, because the daemon that would log "finished" is + * the one being replaced. */ export const upgradeRoutes: FastifyPluginAsyncZod< UpgradeRoutesOptions -> = async (app, { updater, db }) => { +> = async (app, { updater }) => { app.post( '/checkUpgrade', { @@ -49,12 +46,10 @@ export const upgradeRoutes: FastifyPluginAsyncZod< }, async (request) => { await updater.apply(); - const current = updater.current; - recordActivity(db, { - kind: 'upgrade-started', - actor: request.actor, - detail: `one-click upgrade launched${current ? ` from ${current.commit}` : ''} (systemd unit dormice-upgrade)`, - }); + request.log.info( + { from: updater.current?.commit ?? null }, + 'one-click upgrade launched (systemd unit dormice-upgrade)', + ); return { started: true as const }; }, ); diff --git a/packages/server/src/sandbox-proxy.ts b/packages/server/src/sandbox-proxy.ts index d33f0523..a7eb6fcb 100644 --- a/packages/server/src/sandbox-proxy.ts +++ b/packages/server/src/sandbox-proxy.ts @@ -147,7 +147,7 @@ export function createSandboxProxy(deps: SandboxProxyDeps): SandboxProxy { const before = liveRow(parsed.sandboxId); const row = await locks.run(before.name, async () => { const fresh = liveRow(parsed.sandboxId); - const awake = await wakeSandbox(db, executor, fresh, undefined, watchers); + const awake = await wakeSandbox(db, executor, fresh, watchers); return touch(db, awake.id); }); const target = await executor.resolvePortTarget(row.id, parsed.port); diff --git a/packages/server/src/scanner.ts b/packages/server/src/scanner.ts index 2800c4dd..52a0b1a1 100644 --- a/packages/server/src/scanner.ts +++ b/packages/server/src/scanner.ts @@ -152,40 +152,22 @@ export async function scanOnce( executor, fresh.id, archiver?.currentStore() ?? null, - { kind: 'expired-killed', cause: 'E2B deadline (kill) reached' }, watchers, ); result.expiredKilled += 1; return; } if (deadline === 'pause') { - await freezeSandbox( - db, - executor, - fresh.id, - 'E2B deadline reached (pause)', - ); + await freezeSandbox(db, executor, fresh.id); result.frozen += 1; return; } const freshDue = dueTransition(fresh, now); if (freshDue === 'freeze') { - await freezeSandbox( - db, - executor, - fresh.id, - `idle ${fresh.freezeAfterSeconds}s reached — memory squeezed into swap (scanner)`, - ); + await freezeSandbox(db, executor, fresh.id); result.frozen += 1; } else if (freshDue === 'stop') { - await stopSandbox( - db, - executor, - fresh.id, - `idle ${fresh.stopAfterSeconds}s reached — container torn down, disk kept (scanner)`, - undefined, - watchers, - ); + await stopSandbox(db, executor, fresh.id, watchers); result.stopped += 1; } }); diff --git a/packages/shared/src/activity.ts b/packages/shared/src/activity.ts deleted file mode 100644 index 6081943b..00000000 --- a/packages/shared/src/activity.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { z } from 'zod'; - -/** - * listActivity() — the ledger's recent history: who was created, cooled, - * woken, destroyed, and what reconciliation repaired. Events come from the - * daemon's own actors (lifecycle moves, the idle scanner, the reconciler, - * the archiver) and live in a bounded ring in SQLite — the newest N are - * kept, older ones fall off. This is an explanation window, not an audit - * log and not a monitoring system: it answers "why is my sandbox stopped", - * not "prove nobody touched it". - */ -export const ACTIVITY_KINDS = [ - 'created', - 'woken', - 'frozen', - 'stopped', - 'rebuilt', - 'destroyed', - /** An E2B deadline with the kill action passed: destroyed, disk and all. */ - 'expired-killed', - 'archived', - 'restore-started', - 'restored', - 'restore-failed', - /** The reconciler corrected the ledger (or reality) to match the other. */ - 'reconciled', - /** updatePolicy rewrote a sandbox's lifecycle thresholds (ledger-only). */ - 'policy-changed', - /** updateMetadata replaced a sandbox's label set (ledger-only). */ - 'metadata-changed', - /** updateSpec rewrote a sandbox's CPU/memory spec (ledger-only; the next cold wake converges the shell). */ - 'spec-changed', - /** updateTemplate re-homed a sandbox onto another template (ledger-only; the next cold wake converges the shell). */ - 'template-changed', - /** expandDisk grew a sandbox's disk — the one sanctioned resize, grow-only. */ - 'disk-expanded', - /** The daemon grew the HOST data filesystem to fill its expanded device (resize2fs, grow-only). */ - 'host-disk-grown', - 'daemon-started', - /** An operator bound or cleared the console domain through setIngress. */ - 'ingress-updated', - /** updateSettings rewrote runtime settings; the detail names the groups that moved. */ - 'settings-updated', - /** An operator minted an API key (createApiKey). The token itself is never logged. */ - 'apikey-created', - /** updateApiKey changed a key's name or expiry; the detail names what moved. */ - 'apikey-updated', - /** An operator parked an API key (updateApiKey disabled) — reversible, unlike revoke. */ - 'apikey-disabled', - /** An operator resumed a parked API key — the credential opens doors again. */ - 'apikey-enabled', - /** An operator revoked an API key (revokeApiKey) — the credential died here. */ - 'apikey-revoked', - /** - * An operator launched the one-click self-upgrade (applyUpgrade). Only - * the launch is recorded — the outcome lives in getUpgradeStatus, and - * the daemon that would record "finished" is the one being replaced. - */ - 'upgrade-started', -] as const; - -export const activityKindSchema = z.enum(ACTIVITY_KINDS); -export type ActivityKind = z.infer; - -/** - * Attribution: which credential asked for the recorded action. A closed - * vocabulary, built and parsed only here so the string shapes cannot drift - * between the daemon (which writes them) and the console (which displays - * them): - * - * 'env-token' — the bootstrap credential (DORMICE_API_TOKEN). - * 'console' — a console session; the human at the web console. - * 'apikey:' — a ledger API key. By id, not name: names became - * renameable, ids are the stable handle (the sandbox - * name/id doctrine), and revoked rows are never deleted, - * so an id always resolves back to a display name. - * null — no credential asked: the daemon's own actors (idle - * scanner, reconciler, archiver, startup) and data-plane - * wakes that carry no ledger credential (the sandbox - * port proxy, envd operations). Rows written before - * attribution existed are also null; the ring prunes - * them away within days. - * - * This attributes the explanation window, nothing more: lifecycle verbs - * name their actor, but exec and file traffic never enter the ring (they - * would flush it in minutes), so "which key ran what command" needs a real - * audit log, which this deliberately is not. - */ -export const ENV_TOKEN_ACTOR = 'env-token'; -export const CONSOLE_ACTOR = 'console'; -const APIKEY_ACTOR_PREFIX = 'apikey:'; - -export function apiKeyActor(id: string): string { - return `${APIKEY_ACTOR_PREFIX}${id}`; -} - -/** The id inside an 'apikey:' actor, null for every other actor shape. */ -export function apiKeyActorId(actor: string | null): string | null { - return actor?.startsWith(APIKEY_ACTOR_PREFIX) - ? actor.slice(APIKEY_ACTOR_PREFIX.length) - : null; -} - -export const activityEventSchema = z.object({ - /** Ring position; monotonically increasing, newest is largest. */ - id: z.number().int(), - /** ISO 8601 UTC. */ - at: z.string(), - kind: activityKindSchema, - /** - * Null for events with no owning sandbox (orphan sweeps, daemon start). - * Prefixed because they reference another entity: a bare `name` here - * would read as the event's own name (`id` above already is its own). - */ - sandboxName: z.string().nullable(), - sandboxId: z.string().nullable(), - /** Who asked — the closed actor vocabulary above; null = the daemon itself. */ - actor: z.string().nullable(), - /** One short line of context: which threshold, what was repaired, what changed. */ - detail: z.string(), -}); - -export type ActivityEvent = z.infer; - -export const listActivityRequestSchema = z.object({ - /** Newest-first page size; the ring never holds more than its bound anyway. */ - limit: z.number().int().min(1).max(1000).default(200), -}); - -export type ListActivityRequest = z.input; - -export const listActivityResponseSchema = z.object({ - /** Newest first. */ - events: z.array(activityEventSchema), -}); - -export type ListActivityResponse = z.infer; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 4d8162f9..296c5f5b 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1,5 +1,4 @@ export * from './acquire'; -export * from './activity'; export * from './api-keys'; export * from './config'; export * from './destroy'; diff --git a/skills/dormice/SKILL.md b/skills/dormice/SKILL.md index 1d1ea750..1801fa96 100644 --- a/skills/dormice/SKILL.md +++ b/skills/dormice/SKILL.md @@ -126,7 +126,7 @@ disable switch; the create response shows the key once, never again; these four verbs accept only the env token), `getHostMetrics`, `getSandboxMetrics` / `listSandboxMetrics` (live resource samples; never wake anything), `listSandboxImages` (who still -runs an old template image), `listActivity` (recent daemon history), +runs an old template image), `getConfig` (effective config, secrets redacted), `getIngress` / `setIngress` (bind domains on the daemon's managed reverse proxy). `execCommand` takes diff --git a/website/content/docs/console.mdx b/website/content/docs/console.mdx index 8aaea6ae..987ee9fd 100644 --- a/website/content/docs/console.mdx +++ b/website/content/docs/console.mdx @@ -102,17 +102,10 @@ editing `/etc/dormice/env`, and it (or a console session) is also the only credential the key-management actions themselves accept: keys cannot manage keys. -## Activity, domains, settings, and version +## Domains, settings, and version -Four more pages round out the operator view: +Three more pages round out the operator view: -- **Activity** — the daemon's recent history (who was created, frozen, - destroyed, what the reconciler repaired, which API key was minted or - revoked), each event attributed to the credential that asked — an API - key by name, the env token, a console session, or "system" for the - daemon's own moves. Filterable by event kind, actor, and sandbox - name: filtering by a key is how you read a leaked key's blast radius - before revoking it. A bounded window, not an audit log. - **Domains** — two sections. Console domains bind to the daemon's front door (the managed reverse proxy), with a copyable DNS record guide and a live per-domain probe that shows DNS and certificate diff --git a/website/content/docs/http-api.mdx b/website/content/docs/http-api.mdx index 21939edd..c04e7889 100644 --- a/website/content/docs/http-api.mdx +++ b/website/content/docs/http-api.mdx @@ -54,7 +54,6 @@ The [E2B compatibility surface](/docs/e2b-sdks) is a separate wire under | `POST /getSandboxMetrics` | one sandbox's live CPU/memory/disk sample; `sample` is `null` when nothing is running | 404 unknown name | | `POST /listSandboxMetrics` | every measurable sandbox's sample in one answer | — | | `POST /listSandboxImages` | each sandbox's born image vs its template's current one | — | -| `POST /listActivity` | the daemon's recent history with per-event [attribution](#who-did-what), newest first — a bounded ring, not an audit log | — | | `POST /getConfig` | effective configuration: env knobs (read-only; secrets reported present-or-absent, value never sent) plus the live runtime `settings` | — | | `POST /updateSettings` | rewrite the runtime settings (capacity cap, new-sandbox defaults, default policy, managed swap, the S3 archive store, the sandbox domain, the per-sandbox pids cap) — immediate effect, no restart; each provided group replaces that group whole. Managed swap (`swapGb`) grows immediately; shrinking waits for the next host reboot (an active swapfile is never unmounted — that would drag every frozen sandbox's memory back into RAM). The `s3` group takes all six fields every time (keys included — they are never read back) and is probed with a real write-read-delete round trip before anything is saved; `s3: null` turns archiving off. `sandboxDomain` takes a bare hostname or `null`; `sandboxDomainAliases` takes the full list of inbound-only alias hostnames (`[]` clears) — swapping the canonical domain is one call carrying both fields, see [Exposing ports](/docs/ports). `pidsLimit` (floor 256, never unlimited) reaches new containers at birth, running sandboxes in place before the call returns, and frozen or stopped ones at their next wake — no rebuild | 400 empty patch, a `pidsLimit` below 256, a default policy that asks to archive with no store in force, `swapGb` on a host that cannot manage swap, a failed probe (S3 answered 4xx), clearing/moving the store while sandboxes are archived in it (named by count), or an alias list that contradicts the post-patch state (duplicates, the canonical domain listed as an alias, or aliases left standing with no canonical domain); 500 when a value was saved but the host would not follow — `swapGb`'s grow failed, or a running sandbox refused the new `pidsLimit` (named; it follows at its next wake); a patch carrying both knobs gets both verdicts in one message; 502 when the probed store is unreachable | | `POST /getIngress` | domains bound on the daemon's managed reverse proxy, with live DNS and certificate probes | — | @@ -87,7 +86,7 @@ runtime itself died: under gVisor, exit 2 with no OOM flag — the signature of a [pids-cap hit](/docs/troubleshooting#a-sandbox-vanished-with-exit-code-2-and-no-oom)) or `exited` (any other exit; the code is all that is known). Stops the daemon ordered — the idle policy, a rebuild, a destroy — are not deaths -and never appear here; they are the activity feed's business. The field +and never appear here. The field is history, not state: a wake does not clear it, only the next death overwrites it. `at` is when the container's init exited as the runtime recorded it (Docker's `State.FinishedAt`) — the death itself, not the @@ -154,21 +153,10 @@ the row as rotation history, and is the only one that frees the name for reuse. Each key row carries `lastUsedAt` (written with 60-second granularity), and revoked rows stay in `listApiKeys` forever. -## Who did what - -Every `listActivity` event carries an `actor`: `"env-token"` (the -bootstrap credential), `"console"` (a console session), `"apikey:"` -(that key — by `id`, since names are renameable; resolve it through -`listApiKeys`, where the row lives forever), or `null` when no -credential asked — the idle scanner freezing, the reconciler repairing, -the archiver archiving. - -This is what turns key hygiene into incident response: suspect a leak, -disable the key to stop the bleeding, filter the activity ring by that -key to see what it created and destroyed, then revoke and rotate. The -ring attributes lifecycle verbs only — commands and file traffic never -enter it (they would flush its 1000 entries in minutes), so "which key -ran what" needs a real audit system, which this deliberately is not. +Suspect a leak: disable the key to stop the bleeding, then revoke and +rotate. Dormice keeps no per-key audit trail — lifecycle moves land in +the daemon's own log (journald on a systemd host), attributed to the +request, not to the credential. ## Commands and files on the wire diff --git a/website/content/docs/troubleshooting.mdx b/website/content/docs/troubleshooting.mdx index c7ee73c9..7f7ee0b9 100644 --- a/website/content/docs/troubleshooting.mdx +++ b/website/content/docs/troubleshooting.mdx @@ -76,14 +76,12 @@ Out-of-memory *inside* the sandbox takes down the whole gVisor container next acquire revives the sandbox. Raise the limit if the workload honestly needs more. -The activity feed (`listActivity`, or the console's activity page) says -which it was: the `reconciled` event that recorded the death carries the -exit code and whether the kernel's memory cgroup OOM-killed the -container. `OOM-killed` is the kernel's verdict; a plain `exit 137` is a -SIGKILL from elsewhere. The same verdict rides on the sandbox object as -[`lastExit`](/docs/http-api#the-sandbox-object) — `cause: "oom-killed"` -here — so a client that saw its stream end can read it straight from -the `acquireSandbox` that revives the sandbox. +The sandbox object says which it was: +[`lastExit`](/docs/http-api#the-sandbox-object) carries the exit code +and whether the kernel's memory cgroup OOM-killed the container — +`cause: "oom-killed"` is the kernel's verdict; a plain `exit 137` is a +SIGKILL from elsewhere — so a client that saw its stream end can read it +straight from the `acquireSandbox` that revives the sandbox. The verdict is read two ways, and one of them depends on the host. A wake that arrives while the container's cgroup still exists reads the @@ -114,9 +112,8 @@ The sandbox hit its pids cap (the `pidsLimit` setting; that cap bounds the sandbox's host-side threads and processes, not a count the sandbox can see — so nothing inside gets an error; the sandbox's kernel simply dies, exit 2, and the host's own kernel log -shows one line: `fork rejected by pids controller`. The `reconciled` -activity event names this signature, and the sandbox object's -[`lastExit`](/docs/http-api#the-sandbox-object) reads +shows one line: `fork rejected by pids controller`. The sandbox object's +[`lastExit`](/docs/http-api#the-sandbox-object) names this signature: `cause: "runtime-died"`. The disk survives and the next acquire revives the sandbox — immediately, even inside the heartbeat's blind spot (see the entry above). The default is 4096 (it was 512, runc's From 82b8cbe3ab5453dfe86fbf7147b9bebdf40f7f36 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 15:31:22 +0800 Subject: [PATCH 29/89] The sandbox count cap is gone: a ledger row is not a resource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design record #23 (2026-09-13). DORMICE_MAX_SANDBOXES, the maxSandboxes settings column and knob, the two count-based 429 gates at acquire and E2B create, the capacity figure in getHostMetrics, the console's capacity dialog and the overview's total-against-cap card are deleted; migration 0024 drops the column. The number counted every row — active, frozen, stopped, archived — and none of those is a physical ceiling: stopped rows cost only their disk, archived ones nothing local. The ceilings that exist each have their own reading: the data disk's free space, the host's CPU and memory, which the gateway places by and the operator watches. Both production machines had already set the cap to 100000 on 2026-07-19 to take it out of the way. The overview's third card now shows the total with the three cold-state counts under it; the SDK, e2e and settings tests that used the knob as their "any knob" example move to pidsLimit. --- e2e/src/native.test.ts | 11 +- packages/console/messages/de/overview.json | 4 +- packages/console/messages/de/settings.json | 7 - packages/console/messages/en/overview.json | 4 +- packages/console/messages/en/settings.json | 7 - packages/console/messages/es/overview.json | 4 +- packages/console/messages/es/settings.json | 7 - packages/console/messages/fr/overview.json | 4 +- packages/console/messages/fr/settings.json | 7 - packages/console/messages/ja/overview.json | 4 +- packages/console/messages/ja/settings.json | 7 - packages/console/messages/ko/overview.json | 4 +- packages/console/messages/ko/settings.json | 7 - packages/console/messages/pt-BR/overview.json | 4 +- packages/console/messages/pt-BR/settings.json | 7 - packages/console/messages/ru/overview.json | 4 +- packages/console/messages/ru/settings.json | 7 - packages/console/messages/zh-CN/overview.json | 4 +- packages/console/messages/zh-CN/settings.json | 7 - packages/console/messages/zh-TW/overview.json | 4 +- packages/console/messages/zh-TW/settings.json | 7 - .../overview/components/FleetStatCards.tsx | 25 +- .../components/RuntimeSettingsCard.tsx | 68 -- .../features/settings/pages/SettingsPage.tsx | 1 - packages/sdk/src/client.test.ts | 12 +- .../drizzle/0024_drop-max-sandboxes.sql | 1 + .../server/drizzle/meta/0024_snapshot.json | 766 ++++++++++++++++++ packages/server/drizzle/meta/_journal.json | 7 + packages/server/src/app.test.ts | 19 - packages/server/src/config.ts | 9 - packages/server/src/db/ledger.ts | 7 +- packages/server/src/db/schema.ts | 1 - packages/server/src/db/settings.ts | 5 - packages/server/src/e2b/control.ts | 8 - packages/server/src/e2b/signing.ts | 2 +- packages/server/src/routes/host.ts | 3 - .../server/src/routes/observability.test.ts | 4 +- packages/server/src/routes/sandboxes.ts | 18 +- packages/server/src/routes/settings.test.ts | 46 +- packages/server/src/routes/settings.ts | 22 +- packages/shared/src/host.ts | 1 - packages/shared/src/settings.ts | 8 +- website/content/docs/configuration.mdx | 4 +- website/content/docs/console.mdx | 4 +- website/content/docs/http-api.mdx | 2 +- website/content/docs/metrics.mdx | 2 +- website/content/docs/troubleshooting.mdx | 9 - 47 files changed, 848 insertions(+), 327 deletions(-) create mode 100644 packages/server/drizzle/0024_drop-max-sandboxes.sql create mode 100644 packages/server/drizzle/meta/0024_snapshot.json diff --git a/e2e/src/native.test.ts b/e2e/src/native.test.ts index f8f8ba1d..3ecc559c 100644 --- a/e2e/src/native.test.ts +++ b/e2e/src/native.test.ts @@ -747,7 +747,6 @@ describe('native API over a real daemon', () => { expect(metrics.host.cpuCount).toBeGreaterThan(0); expect(metrics.host.memTotalBytes).toBeGreaterThan(0); expect(metrics.sandboxes.total).toBeGreaterThanOrEqual(1); - expect(metrics.sandboxes.maxSandboxes).toBeGreaterThan(0); expect(metrics.sandboxDisks.count).toBeGreaterThanOrEqual(1); expect(metrics.sandboxDisks.actualBytes).toBeGreaterThan(0); // Disks are sparse: the fleet is promised more than it occupies — @@ -777,15 +776,15 @@ describe('the observability verbs over a real daemon', () => { it('updateSettings moves a ledger knob with immediate effect', async () => { const before = (await client().getConfig()).settings; const { settings } = await client().updateSettings({ - maxSandboxes: before.maxSandboxes + 1, + pidsLimit: before.pidsLimit + 1, }); - expect(settings.maxSandboxes).toBe(before.maxSandboxes + 1); + expect(settings.pidsLimit).toBe(before.pidsLimit + 1); expect(settings.updatedAt).not.toBeNull(); - expect((await client().getConfig()).settings.maxSandboxes).toBe( - before.maxSandboxes + 1, + expect((await client().getConfig()).settings.pidsLimit).toBe( + before.pidsLimit + 1, ); // Restore: the exam daemon is shared by every suite in this run. - await client().updateSettings({ maxSandboxes: before.maxSandboxes }); + await client().updateSettings({ pidsLimit: before.pidsLimit }); }); it('the swap knob follows getConfig: refused where unmanageable, accepted where real', async () => { diff --git a/packages/console/messages/de/overview.json b/packages/console/messages/de/overview.json index 0d41a71f..047ae4d7 100644 --- a/packages/console/messages/de/overview.json +++ b/packages/console/messages/de/overview.json @@ -13,8 +13,8 @@ "overview_stat_peak_none": "Noch keine Messwerte im Fenster", "overview_stat_peak_at": "Um {time}", "overview_stat_total_label": "Sandboxes gesamt", - "overview_stat_total_hint": "Kapazität {max}", - "overview_stat_total_sub": "{pct}% belegt", + "overview_stat_total_hint": "Alle Sandboxes im Ledger", + "overview_stat_total_sub": "eingefroren {frozen} · gestoppt {stopped} · archiviert {archived}", "overview_disks_label": "Sandbox-Datenträger", "overview_disks_hint": "Zugesagt {size}", "overview_disks_sub": "{count} Datenträger · {pct}% tatsächlich", diff --git a/packages/console/messages/de/settings.json b/packages/console/messages/de/settings.json index 7f331924..5b65f88f 100644 --- a/packages/console/messages/de/settings.json +++ b/packages/console/messages/de/settings.json @@ -7,7 +7,6 @@ "settings_hint_executor": "Executor: docker sind echte Sandboxes, fake ist ein In-Memory-Stub (für Entwicklung/Tests)", "settings_hint_base_image": "Standard-Image für Sandboxes; im docker-Modus Pflicht", "settings_hint_data_dir": "Zuhause der Sandbox-Datenträger (disks/*.img); auch temporäre Archivdateien liegen hier", - "settings_hint_max_sandboxes": "Erststart-Saatwert für das Kapazitätslimit — der wirksame Wert steht oben bei den Betriebsreglern", "settings_hint_scan_interval": "Leerlauf-Scan-Intervall: Jede Runde kühlt Sandboxes an der Schwelle eine Stufe ab", "settings_hint_metrics_sample_interval": "Messintervall: die Auflösung der Verlaufskurven; Übersichtstrends und Messwerthistorie werden in diesem Takt gespeichert", "settings_hint_metrics_retention": "Aufbewahrungsdauer der Messwerte je Sandbox; die Flotten-Zustandszählungen werden unabhängig davon immer 30 Tage behalten", @@ -48,8 +47,6 @@ "settings_knobs_desc": "Wohnen im Ledger; Änderungen wirken sofort, ohne Neustart — die gleichnamigen Umgebungsvariablen unten sind nur Saatwerte für den ersten Start.", "settings_knobs_last_modified": " Zuletzt geändert: {time}.", "settings_knobs_never_modified": " Nie geändert; noch die Saatwerte.", - "settings_row_max_sandboxes": "Sandbox-Kapazitätslimit", - "settings_row_max_value": "{n} (blockiert nur Neuerstellung; Aufwachen ist nie begrenzt)", "settings_row_defaults": "Standardquoten neuer Sandboxes", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB Speicher · {disk} GiB Datenträger", "settings_row_policy": "Standard-Lebenszyklus-Richtlinie", @@ -60,10 +57,6 @@ "settings_swap_line_shrink": "Ziel {target} GiB · eingehängt {active} GiB — Verkleinern greift beim nächsten Neustart des Hosts", "settings_swap_line_grow": "Ziel {target} GiB · eingehängt {active} GiB — Vergrößern unvollständig, siehe Daemon-Logs", "settings_swap_line_ok": "{target} GiB (System-Swap nicht mitgezählt)", - "settings_max_dialog_title": "Sandbox-Kapazitätslimit anpassen", - "settings_max_dialog_desc": "Blockiert nur Neuerstellung (beim Erreichen gibt es 429); Aufwachen ist nie begrenzt. Das ist eine Sicherung gegen unkontrolliertes Wachstum — steht sie zu hoch, ist der physische Datenträger der einzige Rückhalt; behalten Sie dann den Pegel des Datenlaufwerks auf der Übersichtsseite im Blick.", - "settings_max_saved": "Kapazitätslimit geändert auf {value}", - "settings_max_label": "Maximal gleichzeitig existierende Sandboxes", "settings_defaults_dialog_title": "Standardquoten neuer Sandboxes anpassen", "settings_defaults_dialog_desc": "CPU/Speicher greifen, wenn das nächste Mal ein Container geboren wird (auch bei Bestands-Sandboxes nach einem Kaltstart); die Datenträgergröße wird bei der Geburt des Datenträgers festgelegt (Ersterstellung und Archiv-Wiederherstellung) — der Datenträger ist die Sandbox selbst und wird nie an Ort und Stelle umdimensioniert. Vor dem Verkleinern prüfen, dass der Wert nicht unter dem echten Inhalt archivierter Sandboxes liegt.", "settings_defaults_saved": "Standardquoten neuer Sandboxes aktualisiert", diff --git a/packages/console/messages/en/overview.json b/packages/console/messages/en/overview.json index af5ae065..d6fdbb58 100644 --- a/packages/console/messages/en/overview.json +++ b/packages/console/messages/en/overview.json @@ -13,8 +13,8 @@ "overview_stat_peak_none": "No samples yet", "overview_stat_peak_at": "At {time}", "overview_stat_total_label": "Total sandboxes", - "overview_stat_total_hint": "Capacity {max}", - "overview_stat_total_sub": "{pct}% used", + "overview_stat_total_hint": "Every sandbox in the ledger", + "overview_stat_total_sub": "frozen {frozen} · stopped {stopped} · archived {archived}", "overview_disks_label": "Sandbox disks", "overview_disks_hint": "Promised {size}", "overview_disks_sub": "{count} disks · {pct}% real", diff --git a/packages/console/messages/en/settings.json b/packages/console/messages/en/settings.json index 7aea74e8..1eab2d21 100644 --- a/packages/console/messages/en/settings.json +++ b/packages/console/messages/en/settings.json @@ -7,7 +7,6 @@ "settings_hint_executor": "Executor: docker is real sandboxes, fake is an in-memory stub (for dev/testing)", "settings_hint_base_image": "Default sandbox image; required in docker mode", "settings_hint_data_dir": "Home of sandbox disk images (disks/*.img); archive temp files live here too", - "settings_hint_max_sandboxes": "First-boot seed for the capacity cap — the value in force lives in the knobs card above", "settings_hint_scan_interval": "Idle scan interval: each round cools sandboxes that hit a threshold by one step", "settings_hint_metrics_sample_interval": "Metrics sampling interval: the resolution of history curves; overview trends and metric history are stored at this cadence", "settings_hint_metrics_retention": "Retention for per-sandbox metric samples; fleet state counts are always kept 30 days regardless", @@ -48,8 +47,6 @@ "settings_knobs_desc": "Stored in the ledger; changes take effect immediately with no restart — the same-named env vars below are only first-boot seed values.", "settings_knobs_last_modified": " Last modified {time}.", "settings_knobs_never_modified": " Never changed; still the seed values.", - "settings_row_max_sandboxes": "Sandbox capacity limit", - "settings_row_max_value": "{n} (only blocks creation; waking is never limited)", "settings_row_defaults": "New sandbox default quotas", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB memory · {disk} GiB disk", "settings_row_policy": "Default lifecycle policy", @@ -60,10 +57,6 @@ "settings_swap_line_shrink": "Target {target} GiB · mounted {active} GiB — shrinking takes effect on the next host reboot", "settings_swap_line_grow": "Target {target} GiB · mounted {active} GiB — growth incomplete, see daemon logs", "settings_swap_line_ok": "{target} GiB (system swap not included)", - "settings_max_dialog_title": "Adjust sandbox capacity limit", - "settings_max_dialog_desc": "Only blocks new creation (returns 429 when hit); waking is never limited. This is a fuse against runaway growth — set it too high and the physical disk becomes the only backstop, so watch the data disk level on the overview page.", - "settings_max_saved": "Capacity limit changed to {value}", - "settings_max_label": "Maximum concurrent sandboxes", "settings_defaults_dialog_title": "Adjust default quotas for new sandboxes", "settings_defaults_dialog_desc": "CPU/memory take effect the next time a container is born (including existing sandboxes cold-started after a stop); disk size is fixed when the disk is born (first creation and archive restore) — the disk is the sandbox itself and is never resized in place, so before lowering it make sure it is not smaller than an archived sandbox's real content.", "settings_defaults_saved": "New sandbox default quotas updated", diff --git a/packages/console/messages/es/overview.json b/packages/console/messages/es/overview.json index a1425710..012dbdd4 100644 --- a/packages/console/messages/es/overview.json +++ b/packages/console/messages/es/overview.json @@ -13,8 +13,8 @@ "overview_stat_peak_none": "Aún no hay muestras", "overview_stat_peak_at": "A las {time}", "overview_stat_total_label": "Sandboxes en total", - "overview_stat_total_hint": "Capacidad {max}", - "overview_stat_total_sub": "{pct}% en uso", + "overview_stat_total_hint": "Todos los sandboxes del libro mayor", + "overview_stat_total_sub": "congelados {frozen} · detenidos {stopped} · archivados {archived}", "overview_disks_label": "Discos de sandboxes", "overview_disks_hint": "{size} prometidos", "overview_disks_sub": "{count} discos · {pct}% real", diff --git a/packages/console/messages/es/settings.json b/packages/console/messages/es/settings.json index fef07c53..39137890 100644 --- a/packages/console/messages/es/settings.json +++ b/packages/console/messages/es/settings.json @@ -7,7 +7,6 @@ "settings_hint_executor": "Ejecutor: docker son sandboxes reales, fake es un sustituto en memoria (para desarrollo y pruebas)", "settings_hint_base_image": "Imagen predeterminada de los sandboxes; obligatoria en modo docker", "settings_hint_data_dir": "Hogar de las imágenes de disco de los sandboxes (disks/*.img); los archivos temporales de archivado también viven aquí", - "settings_hint_max_sandboxes": "Valor semilla del primer arranque para el límite de capacidad — el valor efectivo está en los parámetros de arriba", "settings_hint_scan_interval": "Periodo del escaneo de inactividad: cada ronda enfría un paso los sandboxes que alcanzan un umbral", "settings_hint_metrics_sample_interval": "Periodo de muestreo de métricas: la resolución de las curvas históricas; las tendencias del panel y el historial de métricas se guardan con esta cadencia", "settings_hint_metrics_retention": "Retención de las muestras de métricas por sandbox; los recuentos de estado de la flota se guardan siempre 30 días al margen de esto", @@ -48,8 +47,6 @@ "settings_knobs_desc": "Viven en el libro de registro; los cambios se aplican de inmediato sin reiniciar — las variables de entorno del mismo nombre de abajo son solo valores semilla del primer arranque.", "settings_knobs_last_modified": " Última modificación: {time}.", "settings_knobs_never_modified": " Nunca se cambiaron; siguen los valores semilla.", - "settings_row_max_sandboxes": "Límite de capacidad de sandboxes", - "settings_row_max_value": "{n} (solo bloquea la creación; despertar nunca se limita)", "settings_row_defaults": "Cuotas predeterminadas de los sandboxes nuevos", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB de memoria · {disk} GiB de disco", "settings_row_policy": "Política de ciclo de vida predeterminada", @@ -60,10 +57,6 @@ "settings_swap_line_shrink": "Objetivo {target} GiB · montado {active} GiB — la reducción se aplica al próximo reinicio del host", "settings_swap_line_grow": "Objetivo {target} GiB · montado {active} GiB — la ampliación quedó incompleta, revisa los registros del daemon", "settings_swap_line_ok": "{target} GiB (sin contar el swap del sistema)", - "settings_max_dialog_title": "Ajustar el límite de capacidad de sandboxes", - "settings_max_dialog_desc": "Solo bloquea la creación de nuevos sandboxes (devuelve 429 al alcanzarlo); despertar nunca se limita. Es un fusible contra el crecimiento descontrolado — si lo subes demasiado, el disco físico queda como única red de seguridad, así que vigila el nivel del disco de datos en el panel.", - "settings_max_saved": "El límite de capacidad cambió a {value}", - "settings_max_label": "Máximo de sandboxes simultáneos", "settings_defaults_dialog_title": "Ajustar las cuotas predeterminadas de los sandboxes nuevos", "settings_defaults_dialog_desc": "La CPU y la memoria se aplican en el próximo nacimiento de un contenedor (incluidos los sandboxes existentes que arrancan en frío tras una parada); el tamaño del disco queda fijado al nacer el disco (primera creación y restauración de archivado) — el disco es el sandbox en sí y nunca se redimensiona en el sitio, así que antes de bajarlo asegúrate de que no quede por debajo del contenido real de un sandbox archivado.", "settings_defaults_saved": "Cuotas predeterminadas de los sandboxes nuevos actualizadas", diff --git a/packages/console/messages/fr/overview.json b/packages/console/messages/fr/overview.json index 17bea740..b7eff2a6 100644 --- a/packages/console/messages/fr/overview.json +++ b/packages/console/messages/fr/overview.json @@ -13,8 +13,8 @@ "overview_stat_peak_none": "Aucun échantillon sur cette fenêtre", "overview_stat_peak_at": "À {time}", "overview_stat_total_label": "Total des sandbox", - "overview_stat_total_hint": "Capacité {max}", - "overview_stat_total_sub": "{pct} % utilisés", + "overview_stat_total_hint": "Tous les sandboxes du registre", + "overview_stat_total_sub": "gelés {frozen} · arrêtés {stopped} · archivés {archived}", "overview_disks_label": "Disques des sandbox", "overview_disks_hint": "{size} promis", "overview_disks_sub": "{count} disques · {pct} % réels", diff --git a/packages/console/messages/fr/settings.json b/packages/console/messages/fr/settings.json index 85dab173..ae306160 100644 --- a/packages/console/messages/fr/settings.json +++ b/packages/console/messages/fr/settings.json @@ -7,7 +7,6 @@ "settings_hint_executor": "Exécuteur : docker = vraies sandbox, fake = simulacre en mémoire (dev/tests)", "settings_hint_base_image": "Image par défaut des sandbox ; obligatoire en mode docker", "settings_hint_data_dir": "Foyer des images disque des sandbox (disks/*.img) ; les fichiers temporaires d'archivage vivent ici aussi", - "settings_hint_max_sandboxes": "Valeur d'amorçage du plafond de capacité — la valeur effective est dans les réglages ci-dessus", "settings_hint_scan_interval": "Période du balayage d'inactivité : à chaque tour, les sandbox ayant atteint un seuil descendent d'un cran", "settings_hint_metrics_sample_interval": "Période d'échantillonnage des métriques : la résolution des courbes d'historique ; tendances du tableau de bord et historiques de métriques sont stockés à cette cadence", "settings_hint_metrics_retention": "Durée de rétention des échantillons de métriques par sandbox ; les comptages d'états de la flotte sont toujours conservés 30 jours, indépendamment", @@ -48,8 +47,6 @@ "settings_knobs_desc": "Stockés dans le registre ; les changements prennent effet immédiatement, sans redémarrage — les variables d'environnement homonymes ci-dessous ne sont que des valeurs d'amorçage du premier démarrage.", "settings_knobs_last_modified": " Dernière modification le {time}.", "settings_knobs_never_modified": " Jamais modifiés ; toujours les valeurs d'amorçage.", - "settings_row_max_sandboxes": "Plafond de capacité des sandbox", - "settings_row_max_value": "{n} (ne bloque que la création ; le réveil n'est jamais limité)", "settings_row_defaults": "Quotas par défaut des nouvelles sandbox", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB de mémoire · {disk} GiB de disque", "settings_row_policy": "Politique de cycle de vie par défaut", @@ -60,10 +57,6 @@ "settings_swap_line_shrink": "Cible {target} GiB · monté {active} GiB — la réduction prend effet au prochain redémarrage de l'hôte", "settings_swap_line_grow": "Cible {target} GiB · monté {active} GiB — extension incomplète, voir les journaux du daemon", "settings_swap_line_ok": "{target} GiB (hors swap système)", - "settings_max_dialog_title": "Ajuster le plafond de capacité des sandbox", - "settings_max_dialog_desc": "Ne bloque que la création (429 en cas de dépassement) ; le réveil n'est jamais limité. C'est un fusible contre l'emballement — placé trop haut, le disque physique devient le seul garde-fou : surveillez le niveau du disque de données sur le tableau de bord.", - "settings_max_saved": "Plafond de capacité changé à {value}", - "settings_max_label": "Nombre maximal de sandbox simultanées", "settings_defaults_dialog_title": "Ajuster les quotas par défaut des nouvelles sandbox", "settings_defaults_dialog_desc": "CPU et mémoire prennent effet à la prochaine naissance d'un conteneur (y compris les sandbox existantes redémarrées à froid après un arrêt) ; la taille du disque est figée à la naissance du disque (première création et restauration d'archive) — le disque est la sandbox elle-même et n'est jamais redimensionné en place ; avant de la réduire, vérifiez qu'elle n'est pas inférieure au contenu réel d'une sandbox archivée.", "settings_defaults_saved": "Quotas par défaut des nouvelles sandbox mis à jour", diff --git a/packages/console/messages/ja/overview.json b/packages/console/messages/ja/overview.json index 41cf288d..d96b94c2 100644 --- a/packages/console/messages/ja/overview.json +++ b/packages/console/messages/ja/overview.json @@ -13,8 +13,8 @@ "overview_stat_peak_none": "この期間にはまだサンプルがありません", "overview_stat_peak_at": "{time} に記録", "overview_stat_total_label": "サンドボックス総数", - "overview_stat_total_hint": "容量上限 {max}", - "overview_stat_total_sub": "使用率 {pct}%", + "overview_stat_total_hint": "台帳にある全サンドボックス", + "overview_stat_total_sub": "凍結 {frozen} · 停止 {stopped} · アーカイブ {archived}", "overview_disks_label": "サンドボックスディスク", "overview_disks_hint": "割り当て合計 {size}", "overview_disks_sub": "{count} 台のディスク · 実使用 {pct}%", diff --git a/packages/console/messages/ja/settings.json b/packages/console/messages/ja/settings.json index 7a7bfbea..78d1b957 100644 --- a/packages/console/messages/ja/settings.json +++ b/packages/console/messages/ja/settings.json @@ -7,7 +7,6 @@ "settings_hint_executor": "エグゼキューター:docker は本物のサンドボックス、fake はメモリ内のスタブです(開発・テスト用)", "settings_hint_base_image": "サンドボックスの既定イメージ。docker モードでは必須です", "settings_hint_data_dir": "サンドボックスのディスクイメージの置き場所(disks/*.img)。アーカイブの一時ファイルもここに置かれます", - "settings_hint_max_sandboxes": "容量上限の初回起動シード値 — 有効値は上の運用設定カードにあります", "settings_hint_scan_interval": "アイドルスキャンの周期:1 回のスキャンごとに、しきい値に達したサンドボックスを 1 段階冷却します", "settings_hint_metrics_sample_interval": "メトリクスのサンプリング周期:履歴曲線の解像度で、概要の推移もメトリクス履歴もこの間隔で保存されます", "settings_hint_metrics_retention": "サンドボックス別メトリクスサンプルの保持期間。フリートの状態カウントは常に 30 日保持され、この値には従いません", @@ -48,8 +47,6 @@ "settings_knobs_desc": "台帳に保存され、変更は再起動なしで即時に反映されます — 下表の同名環境変数は初回起動時のシード値にすぎません。", "settings_knobs_last_modified": "最終変更:{time}。", "settings_knobs_never_modified": "変更されたことはなく、シード値のままです。", - "settings_row_max_sandboxes": "サンドボックス容量上限", - "settings_row_max_value": "{n} 台(新規作成のみ制限、ウェイクアップは無制限)", "settings_row_defaults": "新規サンドボックスの既定クォータ", "settings_row_defaults_value": "{cpus} CPU · メモリ {memory} GiB · ディスク {disk} GiB", "settings_row_policy": "既定のライフサイクルポリシー", @@ -60,10 +57,6 @@ "settings_swap_line_shrink": "目標 {target} GiB · 現在のマウント {active} GiB — 縮小は次回のホスト再起動時に反映されます", "settings_swap_line_grow": "目標 {target} GiB · 現在のマウント {active} GiB — 拡張が未完了です。詳細は daemon のログを参照してください", "settings_swap_line_ok": "{target} GiB(システム標準の swap は別枠)", - "settings_max_dialog_title": "サンドボックス容量上限の調整", - "settings_max_dialog_desc": "制限されるのは新規作成だけで(超過時は 429)、ウェイクアップは無制限です。これは暴走を防ぐヒューズです — 高くしすぎると物理ディスクだけが最後の砦になるため、概要ページのデータディスク残量に注意してください。", - "settings_max_saved": "容量上限を {value} に変更しました", - "settings_max_label": "同時に存在できるサンドボックスの最大数", "settings_defaults_dialog_title": "新規サンドボックスの既定クォータの調整", "settings_defaults_dialog_desc": "CPU とメモリは次にコンテナが生成されるときに反映されます(停止後にコールドスタートする既存サンドボックスを含む)。ディスクはディスク作成時に確定します(初回作成とアーカイブからの復元)— ディスクはサンドボックスの本体であり、その場でのサイズ変更は決して行われないため、縮小する前にアーカイブ済みサンドボックスの実際の内容量を下回らないか確認してください。", "settings_defaults_saved": "新規サンドボックスの既定クォータを更新しました", diff --git a/packages/console/messages/ko/overview.json b/packages/console/messages/ko/overview.json index c709a91f..f6722baf 100644 --- a/packages/console/messages/ko/overview.json +++ b/packages/console/messages/ko/overview.json @@ -13,8 +13,8 @@ "overview_stat_peak_none": "이 기간에는 아직 샘플이 없음", "overview_stat_peak_at": "{time}에 발생", "overview_stat_total_label": "샌드박스 총수", - "overview_stat_total_hint": "용량 상한 {max}", - "overview_stat_total_sub": "{pct}% 사용", + "overview_stat_total_hint": "원장에 있는 모든 샌드박스", + "overview_stat_total_sub": "동결 {frozen} · 중지 {stopped} · 보관 {archived}", "overview_disks_label": "샌드박스 디스크", "overview_disks_hint": "총 약정 {size}", "overview_disks_sub": "디스크 {count}개 · 실사용 {pct}%", diff --git a/packages/console/messages/ko/settings.json b/packages/console/messages/ko/settings.json index 2cd33921..16aeeba4 100644 --- a/packages/console/messages/ko/settings.json +++ b/packages/console/messages/ko/settings.json @@ -7,7 +7,6 @@ "settings_hint_executor": "실행기: docker는 실제 샌드박스, fake는 메모리 내 가짜 실행기(개발/테스트용)", "settings_hint_base_image": "샌드박스의 기본 이미지. docker 모드에서는 필수", "settings_hint_data_dir": "샌드박스 디스크 이미지의 집(disks/*.img). 아카이브 임시 파일도 여기에 있습니다", - "settings_hint_max_sandboxes": "용량 상한의 첫 부팅 시드 값 — 유효 값은 위의 운영 노브 카드에 있습니다", "settings_hint_scan_interval": "유휴 스캔 주기: 매 라운드마다 임계값에 도달한 샌드박스의 온도를 한 단계 낮춥니다", "settings_hint_metrics_sample_interval": "메트릭 샘플링 주기: 히스토리 곡선의 해상도이며, 개요 추이와 메트릭 히스토리가 이 주기로 저장됩니다", "settings_hint_metrics_retention": "샌드박스별 메트릭 샘플의 보관 기간. 함대 상태 카운트는 이와 무관하게 항상 30일 보관", @@ -48,8 +47,6 @@ "settings_knobs_desc": "장부에 저장되어 변경 즉시 적용되며 재시작이 필요 없습니다 — 아래 표의 같은 이름 환경 변수는 첫 부팅의 시드 값일 뿐입니다.", "settings_knobs_last_modified": " 마지막 수정: {time}.", "settings_knobs_never_modified": " 변경 이력이 없어 아직 시드 값 그대로입니다.", - "settings_row_max_sandboxes": "샌드박스 용량 상한", - "settings_row_max_value": "{n}개(신규 생성만 차단, 깨우기는 제한 없음)", "settings_row_defaults": "새 샌드박스 기본 할당량", "settings_row_defaults_value": "{cpus} CPU · 메모리 {memory} GiB · 디스크 {disk} GiB", "settings_row_policy": "기본 수명 주기 정책", @@ -60,10 +57,6 @@ "settings_swap_line_shrink": "목표 {target} GiB · 현재 마운트 {active} GiB — 축소는 다음 호스트 재부팅 시 적용", "settings_swap_line_grow": "목표 {target} GiB · 현재 마운트 {active} GiB — 확장 미완료, daemon 로그 참조", "settings_swap_line_ok": "{target} GiB(시스템 자체 swap 별도)", - "settings_max_dialog_title": "샌드박스 용량 상한 조정", - "settings_max_dialog_desc": "신규 생성만 차단하며(도달 시 429 반환) 깨우기는 절대 제한받지 않습니다. 폭주를 막는 퓨즈입니다 — 너무 높이 올리면 물리 디스크가 유일한 안전판이 되니, 개요 페이지의 데이터 디스크 수위를 지켜보세요.", - "settings_max_saved": "용량 상한이 {value}(으)로 변경되었습니다", - "settings_max_label": "동시에 존재할 수 있는 최대 샌드박스 수", "settings_defaults_dialog_title": "새 샌드박스의 기본 할당량 조정", "settings_defaults_dialog_desc": "CPU/메모리는 다음에 컨테이너가 태어날 때 적용됩니다(중지 후 콜드 스타트되는 기존 샌드박스 포함). 디스크는 디스크가 태어날 때 확정됩니다(최초 생성과 아카이브 복원) — 디스크는 샌드박스의 본체라서 절대 제자리에서 크기를 바꾸지 않으니, 줄이기 전에 아카이브된 샌드박스의 실제 내용보다 작아지지 않는지 확인하세요.", "settings_defaults_saved": "새 샌드박스 기본 할당량이 업데이트되었습니다", diff --git a/packages/console/messages/pt-BR/overview.json b/packages/console/messages/pt-BR/overview.json index 8ccb578e..e5d48fbc 100644 --- a/packages/console/messages/pt-BR/overview.json +++ b/packages/console/messages/pt-BR/overview.json @@ -13,8 +13,8 @@ "overview_stat_peak_none": "Ainda sem amostras", "overview_stat_peak_at": "Em {time}", "overview_stat_total_label": "Total de sandboxes", - "overview_stat_total_hint": "Capacidade {max}", - "overview_stat_total_sub": "{pct}% usado", + "overview_stat_total_hint": "Todos os sandboxes no livro-razão", + "overview_stat_total_sub": "congelados {frozen} · parados {stopped} · arquivados {archived}", "overview_disks_label": "Discos dos sandboxes", "overview_disks_hint": "Prometido {size}", "overview_disks_sub": "{count} discos · {pct}% real", diff --git a/packages/console/messages/pt-BR/settings.json b/packages/console/messages/pt-BR/settings.json index 443df2fc..c2c39cf5 100644 --- a/packages/console/messages/pt-BR/settings.json +++ b/packages/console/messages/pt-BR/settings.json @@ -7,7 +7,6 @@ "settings_hint_executor": "Executor: docker é sandbox de verdade, fake é um stub em memória (para dev/teste)", "settings_hint_base_image": "Imagem padrão dos sandboxes; obrigatória no modo docker", "settings_hint_data_dir": "Casa das imagens de disco dos sandboxes (disks/*.img); os arquivos temporários de arquivamento também moram aqui", - "settings_hint_max_sandboxes": "Semente de primeira inicialização para o limite de capacidade — o valor efetivo está nos controles acima", "settings_hint_scan_interval": "Intervalo da varredura de ociosidade: cada rodada esfria em um degrau os sandboxes que atingiram um limite", "settings_hint_metrics_sample_interval": "Intervalo de amostragem de métricas: a resolução das curvas de histórico; as tendências do painel e o histórico de métricas são gravados nessa cadência", "settings_hint_metrics_retention": "Retenção das amostras de métricas por sandbox; as contagens de estado da frota são sempre mantidas por 30 dias, independentemente", @@ -48,8 +47,6 @@ "settings_knobs_desc": "Moram no ledger; mudanças entram em vigor imediatamente, sem reiniciar — as variáveis de ambiente homônimas abaixo são apenas sementes da primeira inicialização.", "settings_knobs_last_modified": " Última modificação em {time}.", "settings_knobs_never_modified": " Nunca alterados; ainda são os valores-semente.", - "settings_row_max_sandboxes": "Limite de capacidade de sandboxes", - "settings_row_max_value": "{n} (só bloqueia criação; acordar nunca é limitado)", "settings_row_defaults": "Cotas padrão de novos sandboxes", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB de memória · {disk} GiB de disco", "settings_row_policy": "Política padrão de ciclo de vida", @@ -60,10 +57,6 @@ "settings_swap_line_shrink": "Alvo {target} GiB · montado {active} GiB — a redução entra em vigor no próximo reinício do host", "settings_swap_line_grow": "Alvo {target} GiB · montado {active} GiB — crescimento incompleto, veja os logs do daemon", "settings_swap_line_ok": "{target} GiB (swap do sistema não incluído)", - "settings_max_dialog_title": "Ajustar limite de capacidade de sandboxes", - "settings_max_dialog_desc": "Só bloqueia novas criações (retorna 429 ao bater no limite); acordar nunca é limitado. É um fusível contra crescimento descontrolado — alto demais, e o disco físico vira o único anteparo, então acompanhe o nível do disco de dados no painel.", - "settings_max_saved": "Limite de capacidade alterado para {value}", - "settings_max_label": "Máximo de sandboxes simultâneos", "settings_defaults_dialog_title": "Ajustar cotas padrão de novos sandboxes", "settings_defaults_dialog_desc": "CPU/memória entram em vigor no próximo nascimento de contêiner (inclusive sandboxes existentes que fazem cold start após parar); o tamanho do disco é fixado quando o disco nasce (primeira criação e restauração de arquivamento) — o disco é o próprio sandbox e nunca é redimensionado no lugar; antes de reduzir, garanta que não fique menor que o conteúdo real de um sandbox arquivado.", "settings_defaults_saved": "Cotas padrão de novos sandboxes atualizadas", diff --git a/packages/console/messages/ru/overview.json b/packages/console/messages/ru/overview.json index 07a8c2bb..03fc6e67 100644 --- a/packages/console/messages/ru/overview.json +++ b/packages/console/messages/ru/overview.json @@ -13,8 +13,8 @@ "overview_stat_peak_none": "Замеров в этом окне пока нет", "overview_stat_peak_at": "В {time}", "overview_stat_total_label": "Всего песочниц", - "overview_stat_total_hint": "Лимит {max}", - "overview_stat_total_sub": "Занято {pct}%", + "overview_stat_total_hint": "Все песочницы в реестре", + "overview_stat_total_sub": "заморожено {frozen} · остановлено {stopped} · в архиве {archived}", "overview_disks_label": "Диски песочниц", "overview_disks_hint": "Обещано {size}", "overview_disks_sub": "Дисков: {count} · фактически {pct}%", diff --git a/packages/console/messages/ru/settings.json b/packages/console/messages/ru/settings.json index 861b142e..0bcf1e76 100644 --- a/packages/console/messages/ru/settings.json +++ b/packages/console/messages/ru/settings.json @@ -7,7 +7,6 @@ "settings_hint_executor": "Исполнитель: docker — настоящие песочницы, fake — заглушка в памяти (для разработки и тестов)", "settings_hint_base_image": "Образ песочниц по умолчанию; в режиме docker обязателен", "settings_hint_data_dir": "Дом дисковых образов песочниц (disks/*.img); здесь же временные файлы архивации", - "settings_hint_max_sandboxes": "Стартовое значение лимита ёмкости — действующее значение в настройках выше", "settings_hint_scan_interval": "Период сканирования простоя: каждый цикл переводит достигшие порога песочницы на ступень холоднее", "settings_hint_metrics_sample_interval": "Период съёма метрик: разрешение исторических графиков; с этим шагом сохраняются тренды обзора и история метрик", "settings_hint_metrics_retention": "Срок хранения замеров метрик по песочницам; счётчики состояний парка всегда хранятся 30 дней и от него не зависят", @@ -48,8 +47,6 @@ "settings_knobs_desc": "Хранятся в реестре; изменения действуют сразу, без перезапуска — одноимённые переменные окружения ниже лишь задают стартовые значения при первом запуске.", "settings_knobs_last_modified": " Последнее изменение: {time}.", "settings_knobs_never_modified": " Не менялись; всё ещё стартовые значения.", - "settings_row_max_sandboxes": "Лимит числа песочниц", - "settings_row_max_value": "{n} (ограничивает только создание; пробуждение без лимита)", "settings_row_defaults": "Квоты новых песочниц по умолчанию", "settings_row_defaults_value": "{cpus} CPU · {memory} ГиБ памяти · {disk} ГиБ диска", "settings_row_policy": "Политика жизненного цикла по умолчанию", @@ -60,10 +57,6 @@ "settings_swap_line_shrink": "Цель {target} ГиБ · подключено {active} ГиБ — уменьшение вступит в силу после следующей перезагрузки хоста", "settings_swap_line_grow": "Цель {target} ГиБ · подключено {active} ГиБ — увеличение не завершено, подробности в логах daemon", "settings_swap_line_ok": "{target} ГиБ (системный swap не считается)", - "settings_max_dialog_title": "Лимит числа песочниц", - "settings_max_dialog_desc": "Ограничивает только создание новых (при превышении — 429); пробуждение без лимита. Это предохранитель от неконтролируемого роста — если поднять его слишком высоко, последней страховкой останется физический диск: следите за уровнем диска данных на дашборде.", - "settings_max_saved": "Лимит изменён на {value}", - "settings_max_label": "Максимум одновременно существующих песочниц", "settings_defaults_dialog_title": "Квоты новых песочниц по умолчанию", "settings_defaults_dialog_desc": "CPU и память применяются при следующем рождении контейнера (включая холодный старт существующих песочниц после остановки); размер диска фиксируется при рождении диска (первое создание и восстановление из архива) — диск и есть песочница, на месте он никогда не меняется, поэтому перед уменьшением убедитесь, что новый размер не меньше реального содержимого архивированных песочниц.", "settings_defaults_saved": "Квоты новых песочниц по умолчанию обновлены", diff --git a/packages/console/messages/zh-CN/overview.json b/packages/console/messages/zh-CN/overview.json index 6195a2e5..601c21d1 100644 --- a/packages/console/messages/zh-CN/overview.json +++ b/packages/console/messages/zh-CN/overview.json @@ -13,8 +13,8 @@ "overview_stat_peak_none": "窗口内还没有采样", "overview_stat_peak_at": "出现于 {time}", "overview_stat_total_label": "沙箱总数", - "overview_stat_total_hint": "容量上限 {max}", - "overview_stat_total_sub": "已用 {pct}%", + "overview_stat_total_hint": "账本里的全部沙箱", + "overview_stat_total_sub": "冻结 {frozen} · 停止 {stopped} · 归档 {archived}", "overview_disks_label": "沙箱磁盘", "overview_disks_hint": "共许诺 {size}", "overview_disks_sub": "{count} 块盘 · 实占 {pct}%", diff --git a/packages/console/messages/zh-CN/settings.json b/packages/console/messages/zh-CN/settings.json index bce2b908..73910de9 100644 --- a/packages/console/messages/zh-CN/settings.json +++ b/packages/console/messages/zh-CN/settings.json @@ -7,7 +7,6 @@ "settings_hint_executor": "执行器:docker 是真沙箱,fake 是内存假执行器(开发/测试用)", "settings_hint_base_image": "沙箱的默认镜像;docker 模式必填", "settings_hint_data_dir": "沙箱磁盘镜像的家(disks/*.img);归档临时文件也在这里", - "settings_hint_max_sandboxes": "容量上限的首启种子值 — 生效值在上方运营旋钮里", "settings_hint_scan_interval": "空闲扫描周期:每一轮把到阈值的沙箱降一格温度", "settings_hint_metrics_sample_interval": "指标采样周期:历史曲线的分辨率,总览走势与指标历史都按它落库", "settings_hint_metrics_retention": "逐沙箱指标样本的保留时长;舰队状态计数恒保 30 天,不随它走", @@ -48,8 +47,6 @@ "settings_knobs_desc": "住在账本里,改了立即生效,不用重启 — 下表同名环境变量只是首次启动的种子值。", "settings_knobs_last_modified": "最后修改于 {time}。", "settings_knobs_never_modified": "从未改过,仍是种子值。", - "settings_row_max_sandboxes": "沙箱容量上限", - "settings_row_max_value": "{n} 个(只挡新建,唤醒永不受限)", "settings_row_defaults": "新沙箱默认配额", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB 内存 · {disk} GiB 磁盘", "settings_row_policy": "默认生命周期策略", @@ -60,10 +57,6 @@ "settings_swap_line_shrink": "目标 {target} GiB · 当前挂载 {active} GiB — 缩容在下次重启宿主时生效", "settings_swap_line_grow": "目标 {target} GiB · 当前挂载 {active} GiB — 增容未完成,详见 daemon 日志", "settings_swap_line_ok": "{target} GiB(系统自带 swap 另计)", - "settings_max_dialog_title": "调整沙箱容量上限", - "settings_max_dialog_desc": "只挡新建(撞上回 429),唤醒永不受限。这是一根防失控的保险丝 — 拔太高之后,物理磁盘就是唯一兜底,记得盯总览页的数据盘水位。", - "settings_max_saved": "容量上限已改为 {value}", - "settings_max_label": "最多同时存在的沙箱数", "settings_defaults_dialog_title": "调整新沙箱的默认配额", "settings_defaults_dialog_desc": "CPU/内存在下一次容器出生时生效(含停止后冷启动的存量沙箱);磁盘在磁盘出生时定型(首次创建与归档恢复) — 磁盘是沙箱的本体,永不原地改尺寸,调小前注意别小于归档沙箱的真实内容。", "settings_defaults_saved": "新沙箱默认配额已更新", diff --git a/packages/console/messages/zh-TW/overview.json b/packages/console/messages/zh-TW/overview.json index 0fbcdbf1..4d9db046 100644 --- a/packages/console/messages/zh-TW/overview.json +++ b/packages/console/messages/zh-TW/overview.json @@ -13,8 +13,8 @@ "overview_stat_peak_none": "此區間內還沒有取樣", "overview_stat_peak_at": "出現於 {time}", "overview_stat_total_label": "沙箱總數", - "overview_stat_total_hint": "容量上限 {max}", - "overview_stat_total_sub": "已用 {pct}%", + "overview_stat_total_hint": "帳本裡的全部沙箱", + "overview_stat_total_sub": "凍結 {frozen} · 停止 {stopped} · 歸檔 {archived}", "overview_disks_label": "沙箱磁碟", "overview_disks_hint": "共承諾 {size}", "overview_disks_sub": "{count} 顆磁碟 · 實佔 {pct}%", diff --git a/packages/console/messages/zh-TW/settings.json b/packages/console/messages/zh-TW/settings.json index 8adaced7..dd4d3250 100644 --- a/packages/console/messages/zh-TW/settings.json +++ b/packages/console/messages/zh-TW/settings.json @@ -7,7 +7,6 @@ "settings_hint_executor": "執行器:docker 是真沙箱,fake 是記憶體假執行器(開發/測試用)", "settings_hint_base_image": "沙箱的預設映像檔;docker 模式必填", "settings_hint_data_dir": "沙箱磁碟映像檔的家(disks/*.img);封存暫存檔也在這裡", - "settings_hint_max_sandboxes": "容量上限的首次啟動種子值 — 生效值在上方維運旋鈕裡", "settings_hint_scan_interval": "閒置掃描週期:每一輪把到門檻的沙箱降一格溫度", "settings_hint_metrics_sample_interval": "指標取樣週期:歷史曲線的解析度,總覽走勢與指標歷史都按它寫入資料庫", "settings_hint_metrics_retention": "逐沙箱指標樣本的保留時長;艦隊狀態計數恆保 30 天,不隨它走", @@ -48,8 +47,6 @@ "settings_knobs_desc": "住在帳本裡,改了立即生效,不用重啟 — 下表同名環境變數只是首次啟動的種子值。", "settings_knobs_last_modified": "最後修改於 {time}。", "settings_knobs_never_modified": "從未改過,仍是種子值。", - "settings_row_max_sandboxes": "沙箱容量上限", - "settings_row_max_value": "{n} 個(只擋新建,喚醒永不受限)", "settings_row_defaults": "新沙箱預設配額", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB 記憶體 · {disk} GiB 磁碟", "settings_row_policy": "預設生命週期策略", @@ -60,10 +57,6 @@ "settings_swap_line_shrink": "目標 {target} GiB · 目前掛載 {active} GiB — 縮容在下次重啟主機時生效", "settings_swap_line_grow": "目標 {target} GiB · 目前掛載 {active} GiB — 增容未完成,詳見 daemon 日誌", "settings_swap_line_ok": "{target} GiB(系統內建 swap 另計)", - "settings_max_dialog_title": "調整沙箱容量上限", - "settings_max_dialog_desc": "只擋新建(撞上回 429),喚醒永不受限。這是一根防失控的保險絲 — 拉太高之後,物理磁碟就是唯一兜底,記得盯總覽頁的資料磁碟水位。", - "settings_max_saved": "容量上限已改為 {value}", - "settings_max_label": "最多同時存在的沙箱數", "settings_defaults_dialog_title": "調整新沙箱的預設配額", "settings_defaults_dialog_desc": "CPU/記憶體在下一次容器出生時生效(含停止後冷啟動的既有沙箱);磁碟在磁碟出生時定型(首次建立與封存還原)— 磁碟是沙箱的本體,永不原地改尺寸,調小前注意別小於已封存沙箱的真實內容。", "settings_defaults_saved": "新沙箱預設配額已更新", diff --git a/packages/console/src/features/overview/components/FleetStatCards.tsx b/packages/console/src/features/overview/components/FleetStatCards.tsx index 121d1f09..6f8542e3 100644 --- a/packages/console/src/features/overview/components/FleetStatCards.tsx +++ b/packages/console/src/features/overview/components/FleetStatCards.tsx @@ -1,6 +1,4 @@ -import { Meter } from '@/components/Meter'; import { Alert, AlertDescription } from '@/components/ui/alert'; -import { pctOf } from '@/lib/format'; import { m } from '@/paraglide/messages'; import { fullClock } from '../format'; import { @@ -15,8 +13,9 @@ import { StatCard, StatCardSkeleton } from './StatCard'; /** * 舰队四卡(openasi 顶排版式,2026-07-16 沙箱磁盘上顶):当前活跃 - * (5 秒一刷的快照 + 窗口内活跃数 sparkline)、窗口峰值、总数/容量、 - * 沙箱磁盘账单。当前值来自 /getHostMetrics;峰值与 sparkline 来自 + * (5 秒一刷的快照 + 窗口内活跃数 sparkline)、窗口峰值、总数、 + * 沙箱磁盘账单。容量上限随讨论稿 #23 删(2026-09-14):账本行数不是 + * 资源,数据盘水位才是——它有自己的卡。当前值来自 /getHostMetrics;峰值与 sparkline 来自 * /getFleetTimeline — daemon 采样器 30 秒落一行,峰值由原始行现算, * 分桶抹不掉它。档位由页头的全局切换器驱动。 */ @@ -50,9 +49,6 @@ export function FleetStatCards({ range }: { range: TimelineRangeKey }) { const { sandboxes } = host.data; const { points, peak } = timeline.data; const activeSeries = points.map((p) => p.byState.active); - const capacityPct = Math.round( - pctOf(sandboxes.total, sandboxes.maxSandboxes), - ); return (
@@ -81,14 +77,13 @@ export function FleetStatCards({ range }: { range: TimelineRangeKey }) { /> - -
- } + value={String(sandboxes.total)} + hint={m.overview_stat_total_hint()} + sub={m.overview_stat_total_sub({ + frozen: sandboxes.byState.frozen, + stopped: sandboxes.byState.stopped, + archived: sandboxes.byState.archived, + })} to="/sandboxes" /> diff --git a/packages/console/src/features/settings/components/RuntimeSettingsCard.tsx b/packages/console/src/features/settings/components/RuntimeSettingsCard.tsx index 90a15226..27f18ddf 100644 --- a/packages/console/src/features/settings/components/RuntimeSettingsCard.tsx +++ b/packages/console/src/features/settings/components/RuntimeSettingsCard.tsx @@ -78,69 +78,6 @@ function EditTrigger() { ); } -function MaxSandboxesDialog({ settings }: { settings: RuntimeSettings }) { - const [open, setOpen] = useState(false); - const [value, setValue] = useState(''); - const { pending, error, setError, submit } = useUpdateSettings(() => - setOpen(false), - ); - - const valid = - value.trim() !== '' && Number.isInteger(Number(value)) && Number(value) > 0; - - return ( - { - setOpen(next); - if (next) { - setValue(String(settings.maxSandboxes)); - setError(null); - } - }} - > - - - - {m.settings_max_dialog_title()} - {m.settings_max_dialog_desc()} - -
{ - event.preventDefault(); - void submit( - { maxSandboxes: Number(value) }, - m.settings_max_saved({ value: Number(value) }), - ); - }} - > - - - - {m.settings_max_label()} - - setValue(event.target.value)} - /> - - {error && {error}} - - - - -
- - - ); -} - function SandboxDefaultsDialog({ settings }: { settings: RuntimeSettings }) { const [open, setOpen] = useState(false); const [cpus, setCpus] = useState(''); @@ -579,11 +516,6 @@ export function RuntimeSettingsCard({ data }: { data: GetConfigResponse }) {

- } - /> string> = { DORMICE_EXECUTOR: m.settings_hint_executor, DORMICE_BASE_IMAGE: m.settings_hint_base_image, DORMICE_DATA_DIR: m.settings_hint_data_dir, - DORMICE_MAX_SANDBOXES: m.settings_hint_max_sandboxes, DORMICE_SCAN_INTERVAL_SECONDS: m.settings_hint_scan_interval, DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS: m.settings_hint_metrics_sample_interval, diff --git a/packages/sdk/src/client.test.ts b/packages/sdk/src/client.test.ts index 4d05847e..73d4d07a 100644 --- a/packages/sdk/src/client.test.ts +++ b/packages/sdk/src/client.test.ts @@ -436,22 +436,22 @@ describe('runtime settings over real HTTP', () => { it('updates a knob and reads it back through getConfig', async () => { const before = (await client.getConfig()).settings; const { settings } = await client.updateSettings({ - maxSandboxes: before.maxSandboxes + 1, + pidsLimit: before.pidsLimit + 1, }); - expect(settings.maxSandboxes).toBe(before.maxSandboxes + 1); + expect(settings.pidsLimit).toBe(before.pidsLimit + 1); expect(settings.updatedAt).not.toBeNull(); - expect((await client.getConfig()).settings.maxSandboxes).toBe( - before.maxSandboxes + 1, + expect((await client.getConfig()).settings.pidsLimit).toBe( + before.pidsLimit + 1, ); // Restore: other suites share this daemon's ledger. - await client.updateSettings({ maxSandboxes: before.maxSandboxes }); + await client.updateSettings({ pidsLimit: before.pidsLimit }); }); it('is admin-only, like the apiKey verbs', async () => { const { apiKey, token } = await client.createApiKey('sdk-settings'); const keyed = new Dormice({ endpoint, token }); await expect( - keyed.updateSettings({ maxSandboxes: 12345 }), + keyed.updateSettings({ pidsLimit: 12345 }), ).rejects.toMatchObject({ status: 403, message: expect.stringMatching(/cannot manage API keys or settings/), diff --git a/packages/server/drizzle/0024_drop-max-sandboxes.sql b/packages/server/drizzle/0024_drop-max-sandboxes.sql new file mode 100644 index 00000000..a6c2c08c --- /dev/null +++ b/packages/server/drizzle/0024_drop-max-sandboxes.sql @@ -0,0 +1 @@ +ALTER TABLE `runtime_settings` DROP COLUMN `max_sandboxes`; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0024_snapshot.json b/packages/server/drizzle/meta/0024_snapshot.json new file mode 100644 index 00000000..bc1855ce --- /dev/null +++ b/packages/server/drizzle/meta/0024_snapshot.json @@ -0,0 +1,766 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "e20f0c46-e1c7-4734-ace9-9dc7f88d40a6", + "prevId": "d8435ca2-c0e2-44ec-b4c5-9928f5bcd2e4", + "tables": { + "api_keys": { + "name": "api_keys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + "key_hash" + ], + "isUnique": true + }, + "api_keys_active_name_idx": { + "name": "api_keys_active_name_idx", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"api_keys\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "console_account": { + "name": "console_account", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_secret": { + "name": "session_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "daemon_secrets": { + "name": "daemon_secrets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "envd_signing_secret": { + "name": "envd_signing_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "fleet_snapshots": { + "name": "fleet_snapshots", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "active": { + "name": "active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "frozen": { + "name": "frozen", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stopped": { + "name": "stopped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "restoring": { + "name": "restoring", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "host_metrics_samples": { + "name": "host_metrics_samples", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "cpu_used_pct": { + "name": "cpu_used_pct", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_available_bytes": { + "name": "mem_available_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_total_bytes": { + "name": "swap_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_used_bytes": { + "name": "swap_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_total_bytes": { + "name": "disk_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_used_bytes": { + "name": "disk_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_available_bytes": { + "name": "disk_available_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "runtime_settings": { + "name": "runtime_settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "sandbox_cpus": { + "name": "sandbox_cpus", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_memory_gb": { + "name": "sandbox_memory_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_disk_gb": { + "name": "sandbox_disk_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_freeze_after_seconds": { + "name": "default_freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_stop_after_seconds": { + "name": "default_stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_archive_after_seconds": { + "name": "default_archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_gb": { + "name": "swap_gb", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "s3_endpoint": { + "name": "s3_endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_access_key_id": { + "name": "s3_access_key_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_secret_access_key": { + "name": "s3_secret_access_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_region": { + "name": "s3_region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_force_path_style": { + "name": "s3_force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain": { + "name": "sandbox_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain_aliases": { + "name": "sandbox_domain_aliases", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pids_limit": { + "name": "pids_limit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sandbox_metrics_samples": { + "name": "sandbox_metrics_samples", + "columns": { + "sandbox_id": { + "name": "sandbox_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "at": { + "name": "at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cpu_count": { + "name": "cpu_count", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cpu_used_pct": { + "name": "cpu_used_pct", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_used_bytes": { + "name": "mem_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_cache_bytes": { + "name": "mem_cache_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_used_bytes": { + "name": "swap_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_total_bytes": { + "name": "swap_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_used_bytes": { + "name": "disk_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "disk_total_bytes": { + "name": "disk_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "sandbox_metrics_samples_sandbox_at_idx": { + "name": "sandbox_metrics_samples_sandbox_at_idx", + "columns": [ + "sandbox_id", + "at" + ], + "isUnique": false + }, + "sandbox_metrics_samples_at_idx": { + "name": "sandbox_metrics_samples_at_idx", + "columns": [ + "at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sandboxes": { + "name": "sandboxes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "freeze_after_seconds": { + "name": "freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stop_after_seconds": { + "name": "stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "archive_after_seconds": { + "name": "archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpus": { + "name": "cpus", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "memory_gb": { + "name": "memory_gb", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_gb": { + "name": "disk_gb", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_active_at": { + "name": "last_active_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_exit_at": { + "name": "last_exit_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_exit_code": { + "name": "last_exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_exit_cause": { + "name": "last_exit_cause", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "envs": { + "name": "envs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "on_deadline": { + "name": "on_deadline", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "paused_by_user": { + "name": "paused_by_user", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "sandboxes_name_unique": { + "name": "sandboxes_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index b60be5f6..8b0882e6 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -169,6 +169,13 @@ "when": 1789370106697, "tag": "0023_drop-activity", "breakpoints": true + }, + { + "idx": 24, + "version": "6", + "when": 1789370887387, + "tag": "0024_drop-max-sandboxes", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/server/src/app.test.ts b/packages/server/src/app.test.ts index 6db7558c..906876cc 100644 --- a/packages/server/src/app.test.ts +++ b/packages/server/src/app.test.ts @@ -238,24 +238,6 @@ describe('error shape', () => { }); }); -describe('sandbox capacity', () => { - it('caps creation at maxSandboxes with an honest 429', async () => { - // The env variable seeds the ledger's runtime settings at first boot. - const { app } = testApp(undefined, { DORMICE_MAX_SANDBOXES: '1' }); - expect((await acquire(app, { name: 'alice' })).statusCode).toBe(200); - - const capped = await acquire(app, { name: 'bob' }); - expect(capped.statusCode).toBe(429); - expect(capped.json().message).toMatch(/maxSandboxes=1/); - - // Existing sandboxes always wake — the cap only guards creation. - expect((await acquire(app, { name: 'alice' })).statusCode).toBe(200); - // Releasing frees the slot. - await rpc(app, '/destroySandbox', { name: 'alice' }); - expect((await acquire(app, { name: 'bob' })).statusCode).toBe(200); - }); -}); - describe('concurrent acquires', () => { /** create() takes seconds under real Docker; 20ms makes two in-flight * requests overlap deterministically. */ @@ -1834,7 +1816,6 @@ describe('POST /getHostMetrics', () => { expect(body.dataDisk?.totalBytes).toBeGreaterThan(0); expect(body.sandboxes).toEqual({ total: 0, - maxSandboxes: 100, byState: { active: 0, frozen: 0, stopped: 0, archived: 0, restoring: 0 }, }); expect(body.sandboxDisks).toEqual({ diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index 59a0e184..528a4be6 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -50,14 +50,6 @@ const envSchema = z.object({ .int() .positive() .default(168), - /** - * How many sandboxes may exist at once. The binding resource is disk: - * every sandbox holds a disk image, and an unbounded acquire loop fills - * the host until the ledger itself can no longer write — the daemon dying - * of its own success. Past the cap, acquire answers an honest 429; wakes - * of existing sandboxes are never blocked. - */ - DORMICE_MAX_SANDBOXES: z.coerce.number().int().positive().default(100), /** * Required, no default: loopback-only is not authentication — any local * process could otherwise drive the daemon. @@ -363,7 +355,6 @@ export const CONFIG_KEYS: Record = { DORMICE_EXECUTOR: { sensitive: false }, DORMICE_BASE_IMAGE: { sensitive: false }, DORMICE_DATA_DIR: { sensitive: false }, - DORMICE_MAX_SANDBOXES: { sensitive: false }, DORMICE_SCAN_INTERVAL_SECONDS: { sensitive: false }, DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS: { sensitive: false }, DORMICE_METRICS_RETENTION_HOURS: { sensitive: false }, diff --git a/packages/server/src/db/ledger.ts b/packages/server/src/db/ledger.ts index 7a70b87b..4a043261 100644 --- a/packages/server/src/db/ledger.ts +++ b/packages/server/src/db/ledger.ts @@ -4,7 +4,7 @@ import { SANDBOX_STATES, type SandboxState, } from '@dormice/shared'; -import { count, eq } from 'drizzle-orm'; +import { eq } from 'drizzle-orm'; import type { Db } from './db'; import { type SandboxRow, sandboxes } from './schema'; @@ -129,11 +129,6 @@ export function countByState(rows: SandboxRow[]): { return { byState, total: rows.length }; } -/** How many sandboxes exist, for the capacity check at acquire. */ -export function countSandboxes(db: Db): number { - return db.select({ n: count() }).from(sandboxes).get()?.n ?? 0; -} - /** * Removes the row entirely. Release is legal from any state, so deletion * does not go through ALLOWED_TRANSITIONS — that table governs moves between diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 14c529c1..a9b5c107 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -268,7 +268,6 @@ export type DaemonSecretsRow = typeof daemonSecrets.$inferSelect; */ export const runtimeSettings = sqliteTable('runtime_settings', { id: integer('id').primaryKey(), - maxSandboxes: integer('max_sandboxes').notNull(), sandboxCpus: real('sandbox_cpus').notNull(), sandboxMemoryGb: real('sandbox_memory_gb').notNull(), sandboxDiskGb: real('sandbox_disk_gb').notNull(), diff --git a/packages/server/src/db/settings.ts b/packages/server/src/db/settings.ts index dc2d4301..65e7e93d 100644 --- a/packages/server/src/db/settings.ts +++ b/packages/server/src/db/settings.ts @@ -44,7 +44,6 @@ export function ensureRuntimeSettings(db: Db, config: Config): void { db.insert(runtimeSettings) .values({ id: SETTINGS_ROW_ID, - maxSandboxes: config.DORMICE_MAX_SANDBOXES, sandboxCpus: config.DORMICE_SANDBOX_CPUS, sandboxMemoryGb: config.DORMICE_SANDBOX_MEMORY_GB, sandboxDiskGb: config.DORMICE_SANDBOX_DISK_GB, @@ -142,7 +141,6 @@ function toView(row: RuntimeSettingsRow): RuntimeSettings { } if (row.pidsLimit === null) throw virginError('pids_limit'); return { - maxSandboxes: row.maxSandboxes, sandboxDefaults: { cpus: row.sandboxCpus, memoryGb: row.sandboxMemoryGb, @@ -241,9 +239,6 @@ export function writeRuntimeSettings( const row = db .update(runtimeSettings) .set({ - ...(patch.maxSandboxes !== undefined - ? { maxSandboxes: patch.maxSandboxes } - : {}), ...(patch.sandboxDefaults !== undefined ? { sandboxCpus: patch.sandboxDefaults.cpus, diff --git a/packages/server/src/e2b/control.ts b/packages/server/src/e2b/control.ts index 5dacc236..9f819da0 100644 --- a/packages/server/src/e2b/control.ts +++ b/packages/server/src/e2b/control.ts @@ -3,7 +3,6 @@ import type { FastifyError } from 'fastify'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { z } from 'zod'; import { - countSandboxes, createSandbox, findById, findByName, @@ -329,13 +328,6 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( ); } - const maxSandboxes = readRuntimeSettings(db).maxSandboxes; - if (countSandboxes(db) >= maxSandboxes) { - throw apiError( - 429, - `sandbox limit reached (maxSandboxes=${maxSandboxes}) — destroy a sandbox or raise the limit in settings`, - ); - } const id = randomUUID(); await executor.create(id, { image: resolveImage(db, template), diff --git a/packages/server/src/e2b/signing.ts b/packages/server/src/e2b/signing.ts index 30649b84..42ff50f0 100644 --- a/packages/server/src/e2b/signing.ts +++ b/packages/server/src/e2b/signing.ts @@ -56,7 +56,7 @@ function matchesToken( * `/files?...` and browsers add nothing) — but every sandbox's * access token is different, so the signature itself binds the sandbox: * compute the expected signature per live ledger row and take the match. - * At most DORMICE_MAX_SANDBOXES hashes per request — microseconds. + * One hash per live ledger row — microseconds even at thousands. * * This is the single-domain answer to what real E2B solves with one * subdomain per sandbox; the deliberate divergence is documented in the diff --git a/packages/server/src/routes/host.ts b/packages/server/src/routes/host.ts index d7e7d1be..86f9c1f5 100644 --- a/packages/server/src/routes/host.ts +++ b/packages/server/src/routes/host.ts @@ -19,7 +19,6 @@ import { resolveBucketSeconds, resolveWindow, } from '../db/metrics'; -import { readRuntimeSettings } from '../db/settings'; import type { Executor } from '../executor/executor'; import { CpuSampler, readHostReading } from '../host-metrics'; @@ -61,8 +60,6 @@ export const hostRoutes: FastifyPluginAsyncZod = async ( ...(await readHostReading(cpu, config.DORMICE_DATA_DIR)), sandboxes: { total, - // The ledger's live knob, not the env seed — the console edits it. - maxSandboxes: readRuntimeSettings(db).maxSandboxes, byState, }, sandboxDisks: await executor.diskUsage(), diff --git a/packages/server/src/routes/observability.test.ts b/packages/server/src/routes/observability.test.ts index 541e6fc0..6c1d1e33 100644 --- a/packages/server/src/routes/observability.test.ts +++ b/packages/server/src/routes/observability.test.ts @@ -98,7 +98,7 @@ function hostReading(cpuUsedPct: number | null): HostSample { describe('getConfig', () => { it('reports every knob with value and source, and validates', async () => { - const { app } = testApp({ DORMICE_MAX_SANDBOXES: '7' }); + const { app } = testApp({ DORMICE_SANDBOX_DISK_GB: '7' }); const res = await rpc(app, '/getConfig'); expect(res.statusCode).toBe(200); const body = getConfigResponseSchema.parse(res.json()); @@ -106,7 +106,7 @@ describe('getConfig', () => { const byKey = new Map(body.entries.map((e: ConfigEntry) => [e.key, e])); // Complete: one entry per knob the config schema knows. expect(body.entries).toHaveLength(Object.keys(CONFIG_KEYS).length); - expect(byKey.get('DORMICE_MAX_SANDBOXES')).toMatchObject({ + expect(byKey.get('DORMICE_SANDBOX_DISK_GB')).toMatchObject({ value: '7', source: 'env', }); diff --git a/packages/server/src/routes/sandboxes.ts b/packages/server/src/routes/sandboxes.ts index 98890722..21db1901 100644 --- a/packages/server/src/routes/sandboxes.ts +++ b/packages/server/src/routes/sandboxes.ts @@ -53,7 +53,6 @@ import type { Archiver, RestoreProgress } from '../archive/archiver'; import type { Config } from '../config'; import type { Db } from '../db/db'; import { - countSandboxes, createSandbox, findById, findByName, @@ -224,18 +223,11 @@ export const sandboxRoutes: FastifyPluginAsyncZod< return { status: 'ready', created: false, row: touch(db, awake.id) }; } - // The capacity check lives at the only verb that creates — wakes of - // existing sandboxes are never blocked. Disk is the real ceiling: every - // sandbox holds a disk image, and unbounded creation fills the host - // until the ledger itself can no longer write. Read live from the - // ledger: a console edit applies to the very next create. - const maxSandboxes = readRuntimeSettings(db).maxSandboxes; - if (countSandboxes(db) >= maxSandboxes) { - throw httpError( - 429, - `sandbox limit reached (maxSandboxes=${maxSandboxes}) — destroy a sandbox or raise the limit in settings`, - ); - } + // No count-based cap: a ledger row is not a resource (design record + // #23 — stopped rows cost only disk, archived ones nothing local), and + // the physical ceilings each have their own reading: the data disk's + // free space and the host's CPU and memory, which the gateway places + // by and the operator watches through getHostMetrics. // Reality first, ledger second: bring the container up, then record // it. If create fails, no row was written — the next acquire retries diff --git a/packages/server/src/routes/settings.test.ts b/packages/server/src/routes/settings.test.ts index a33d395f..00677733 100644 --- a/packages/server/src/routes/settings.test.ts +++ b/packages/server/src/routes/settings.test.ts @@ -122,12 +122,8 @@ function seedArchivedRow(db: ReturnType, name: string) { describe('runtime settings: seeding', () => { it('seeds from the env at first boot, defaults where the env is silent', async () => { - const app = appOn(freshDb(), { - DORMICE_MAX_SANDBOXES: '7', - DORMICE_SANDBOX_DISK_GB: '20', - }); + const app = appOn(freshDb(), { DORMICE_SANDBOX_DISK_GB: '20' }); expect(await settingsOf(app)).toEqual({ - maxSandboxes: 7, sandboxDefaults: { cpus: 1, memoryGb: 2, diskGb: 20 }, // No S3 seed in this env, so the seeded default never archives. defaultPolicy: { ...DEFAULT_LIFECYCLE_POLICY, archiveAfterSeconds: null }, @@ -171,17 +167,17 @@ describe('runtime settings: seeding', () => { it('the ledger wins over a later env edit — seeds are read once', async () => { const db = freshDb(); - appOn(db, { DORMICE_MAX_SANDBOXES: '5' }); + appOn(db, { DORMICE_SANDBOX_DISK_GB: '5' }); // Same ledger, "restarted" with a different env: the row already // exists, so the new env value is deliberately ignored... - const rebooted = appOn(db, { DORMICE_MAX_SANDBOXES: '9' }); - expect((await settingsOf(rebooted)).maxSandboxes).toBe(5); + const rebooted = appOn(db, { DORMICE_SANDBOX_DISK_GB: '9' }); + expect((await settingsOf(rebooted)).sandboxDefaults.diskGb).toBe(5); // ...while getConfig still reports what the env says, as an entry. const body = getConfigResponseSchema.parse( (await rpc(rebooted, '/getConfig')).json(), ); expect( - body.entries.find((e) => e.key === 'DORMICE_MAX_SANDBOXES')?.value, + body.entries.find((e) => e.key === 'DORMICE_SANDBOX_DISK_GB')?.value, ).toBe('9'); }); @@ -320,30 +316,6 @@ describe('updateSettings', () => { const later = appOn(db, { DORMICE_SANDBOX_PIDS_LIMIT: '999' }); expect((await settingsOf(later)).pidsLimit).toBe(4096); }); - it('raises maxSandboxes with immediate effect on the acquire gate', async () => { - const app = appOn(freshDb(), { DORMICE_MAX_SANDBOXES: '1' }); - expect((await rpc(app, '/acquireSandbox', { name: 'a' })).statusCode).toBe( - 200, - ); - expect((await rpc(app, '/acquireSandbox', { name: 'b' })).statusCode).toBe( - 429, - ); - - const res = await rpc(app, '/updateSettings', { maxSandboxes: 2 }); - expect(res.statusCode).toBe(200); - expect( - updateSettingsResponseSchema.parse(res.json()).settings.maxSandboxes, - ).toBe(2); - - // No restart, no re-read of the env: the very next create passes. - expect((await rpc(app, '/acquireSandbox', { name: 'b' })).statusCode).toBe( - 200, - ); - // And the observation window reports the new capacity. - const host = await rpc(app, '/getHostMetrics'); - expect(host.json().sandboxes.maxSandboxes).toBe(2); - }); - it('a new default policy applies to the next acquire, not existing sandboxes', async () => { const app = appOn(freshDb()); const before = await rpc(app, '/acquireSandbox', { name: 'old' }); @@ -373,9 +345,9 @@ describe('updateSettings', () => { it('replaces provided groups whole and leaves the rest untouched', async () => { const app = appOn(freshDb(), { DORMICE_SANDBOX_MEMORY_GB: '4' }); - await rpc(app, '/updateSettings', { maxSandboxes: 50 }); + await rpc(app, '/updateSettings', { pidsLimit: 512 }); const settings = await settingsOf(app); - expect(settings.maxSandboxes).toBe(50); + expect(settings.pidsLimit).toBe(512); expect(settings.sandboxDefaults.memoryGb).toBe(4); expect(settings.updatedAt).not.toBeNull(); }); @@ -483,7 +455,7 @@ describe('updateSettings', () => { ); expect(swap.reconciled).toEqual([32]); // A patch without swapGb must not re-trigger the block juggler. - await rpc(app, '/updateSettings', { maxSandboxes: 9 }); + await rpc(app, '/updateSettings', { pidsLimit: 512 }); expect(swap.reconciled).toEqual([32]); const body = getConfigResponseSchema.parse( (await rpc(app, '/getConfig')).json(), @@ -609,7 +581,7 @@ describe('updateSettings', () => { const refused = await rpc( app, '/updateSettings', - { maxSandboxes: 999 }, + { pidsLimit: 999 }, { authorization: `Bearer ${keyToken}` }, ); expect(refused.statusCode).toBe(403); diff --git a/packages/server/src/routes/settings.ts b/packages/server/src/routes/settings.ts index b1fef8c1..2ae2f44f 100644 --- a/packages/server/src/routes/settings.ts +++ b/packages/server/src/routes/settings.ts @@ -44,17 +44,14 @@ export interface SettingsRoutesOptions { * session only, like the apiKey verbs — a leaked automation key must not * be able to raise the very limits that contain it. * - * A ledger write with immediate effect: the consumers read live - * (acquire's capacity gate, the executor's births, resolvePolicy's - * defaults, the archiver's store, the sandbox proxy's domain, the - * executor's pids cap at each birth and wake), so nothing here restarts or - * wakes a sandbox. Two knobs have a reality on the host that the write - * alone does not move, and each is reconciled right after it: managed swap - * (a swapfile) and the pids cap on the shells running right now (a cgroup - * write their processes never notice). Lowering maxSandboxes below the - * current total is deliberately legal — the gate only blocks creation, and - * refusing would leave an operator unable to say "no more" during an - * incident. + * A ledger write with immediate effect: the consumers read live (the + * executor's births, resolvePolicy's defaults, the archiver's store, the + * sandbox proxy's domain, the executor's pids cap at each birth and + * wake), so nothing here restarts or wakes a sandbox. Two knobs have a + * reality on the host that the write alone does not move, and each is + * reconciled right after it: managed swap (a swapfile) and the pids cap + * on the shells running right now (a cgroup write their processes never + * notice). */ export const settingsRoutes: FastifyPluginAsyncZod< SettingsRoutesOptions @@ -192,9 +189,6 @@ export const settingsRoutes: FastifyPluginAsyncZod< request.log.info( { settings }, `runtime settings updated: ${[ - ...(patch.maxSandboxes !== undefined - ? [`maxSandboxes=${patch.maxSandboxes}`] - : []), ...(patch.sandboxDefaults !== undefined ? [ `sandboxDefaults=${patch.sandboxDefaults.cpus}cpu/${patch.sandboxDefaults.memoryGb}GiB/${patch.sandboxDefaults.diskGb}GiB`, diff --git a/packages/shared/src/host.ts b/packages/shared/src/host.ts index 3eeeefa7..c6b9ecfd 100644 --- a/packages/shared/src/host.ts +++ b/packages/shared/src/host.ts @@ -76,7 +76,6 @@ export const hostMetricsResponseSchema = z.object({ /** Ledger aggregates: what the daemon believes it is running. */ sandboxes: z.object({ total: z.number().int(), - maxSandboxes: z.number().int(), byState: sandboxStateCountsSchema, }), /** diff --git a/packages/shared/src/settings.ts b/packages/shared/src/settings.ts index 6c42b26b..c43f339d 100644 --- a/packages/shared/src/settings.ts +++ b/packages/shared/src/settings.ts @@ -5,7 +5,7 @@ import { lifecyclePolicySchema } from './policy'; * Runtime settings — the operator knobs that live in the ledger, not the * environment. The dividing line (2026-07-19): a knob belongs here exactly * when changing it is an operations decision that must not require shell - * access and a restart (capacity, what new sandboxes get); it stays an env + * access and a restart (what new sandboxes get); it stays an env * variable when changing it makes a different daemon (port, token, * executor, data dir). * @@ -99,8 +99,6 @@ export type S3ArchiveView = z.infer; export const PIDS_LIMIT_MIN = 256; export const runtimeSettingsSchema = z.object({ - /** How many sandboxes may exist at once; past it, creation answers 429. Wakes are never blocked. */ - maxSandboxes: z.number().int().positive(), sandboxDefaults: sandboxResourceDefaultsSchema, /** What acquire() gives a sandbox that asks for nothing. Existing sandboxes keep theirs. */ defaultPolicy: lifecyclePolicySchema, @@ -167,7 +165,6 @@ export type RuntimeSettings = z.infer; */ export const updateSettingsRequestSchema = z .object({ - maxSandboxes: z.number().int().positive().optional(), sandboxDefaults: sandboxResourceDefaultsSchema.optional(), defaultPolicy: lifecyclePolicySchema.optional(), swapGb: z.number().int().nonnegative().optional(), @@ -203,7 +200,6 @@ export const updateSettingsRequestSchema = z .refine( (patch) => patch.pidsLimit !== undefined || - patch.maxSandboxes !== undefined || patch.sandboxDefaults !== undefined || patch.defaultPolicy !== undefined || patch.swapGb !== undefined || @@ -212,7 +208,7 @@ export const updateSettingsRequestSchema = z patch.sandboxDomainAliases !== undefined, { message: - 'updateSettings needs at least one of maxSandboxes, sandboxDefaults, defaultPolicy, swapGb, s3, sandboxDomain, sandboxDomainAliases, pidsLimit', + 'updateSettings needs at least one of sandboxDefaults, defaultPolicy, swapGb, s3, sandboxDomain, sandboxDomainAliases, pidsLimit', }, ); diff --git a/website/content/docs/configuration.mdx b/website/content/docs/configuration.mdx index 4f665a8b..cede20be 100644 --- a/website/content/docs/configuration.mdx +++ b/website/content/docs/configuration.mdx @@ -55,8 +55,7 @@ daemon's first start they are written into its database as runtime settings, and from then on the database value is what's in force — edit it live from the console's settings page (or the `updateSettings` verb), no restart needed. Once seeded, changing these variables in the -environment has no effect. The same applies to `DORMICE_MAX_SANDBOXES` -below. +environment has no effect. | Variable | Default | What it does | | --- | --- | --- | @@ -70,7 +69,6 @@ below. | Variable | Default | What it does | | --- | --- | --- | | `DORMICE_SCAN_INTERVAL_SECONDS` | `60` | How often the idle scanner runs. Each sweep moves an idle sandbox down at most one state — see [Sandbox lifecycle](/docs/lifecycle). | -| `DORMICE_MAX_SANDBOXES` | `100` | How many sandboxes may exist at once. Past the cap, *creation* answers 429; wakes of existing sandboxes are never blocked. First-boot seed — after that, the live value is edited in the console's settings page. | | `DORMICE_RECLAIM_TIMEOUT_SECONDS` | `45` | Upper bound on the memory-reclaim step of a freeze. Hitting it is expected on stubborn workloads, not a failure. | ## Sandbox network diff --git a/website/content/docs/console.mdx b/website/content/docs/console.mdx index 987ee9fd..71bb5424 100644 --- a/website/content/docs/console.mdx +++ b/website/content/docs/console.mdx @@ -116,8 +116,8 @@ Three more pages round out the operator view: URL speaks, plus inbound-only aliases with a "make canonical" action for zero-downtime domain switches; a copyable wildcard DNS record guide, applied live, no dependency on the managed reverse proxy. -- **Settings** — two halves. The operational knobs (sandbox capacity - cap, default resources for new sandboxes, default lifecycle policy, +- **Settings** — two halves. The operational knobs (default resources + for new sandboxes, default lifecycle policy, extra managed [swap](/docs/core-concepts#swap-and-freezing), and the S3 [archive store](/docs/archiving)) live in the daemon's database and are edited right here, taking effect diff --git a/website/content/docs/http-api.mdx b/website/content/docs/http-api.mdx index c04e7889..6c6ef862 100644 --- a/website/content/docs/http-api.mdx +++ b/website/content/docs/http-api.mdx @@ -55,7 +55,7 @@ The [E2B compatibility surface](/docs/e2b-sdks) is a separate wire under | `POST /listSandboxMetrics` | every measurable sandbox's sample in one answer | — | | `POST /listSandboxImages` | each sandbox's born image vs its template's current one | — | | `POST /getConfig` | effective configuration: env knobs (read-only; secrets reported present-or-absent, value never sent) plus the live runtime `settings` | — | -| `POST /updateSettings` | rewrite the runtime settings (capacity cap, new-sandbox defaults, default policy, managed swap, the S3 archive store, the sandbox domain, the per-sandbox pids cap) — immediate effect, no restart; each provided group replaces that group whole. Managed swap (`swapGb`) grows immediately; shrinking waits for the next host reboot (an active swapfile is never unmounted — that would drag every frozen sandbox's memory back into RAM). The `s3` group takes all six fields every time (keys included — they are never read back) and is probed with a real write-read-delete round trip before anything is saved; `s3: null` turns archiving off. `sandboxDomain` takes a bare hostname or `null`; `sandboxDomainAliases` takes the full list of inbound-only alias hostnames (`[]` clears) — swapping the canonical domain is one call carrying both fields, see [Exposing ports](/docs/ports). `pidsLimit` (floor 256, never unlimited) reaches new containers at birth, running sandboxes in place before the call returns, and frozen or stopped ones at their next wake — no rebuild | 400 empty patch, a `pidsLimit` below 256, a default policy that asks to archive with no store in force, `swapGb` on a host that cannot manage swap, a failed probe (S3 answered 4xx), clearing/moving the store while sandboxes are archived in it (named by count), or an alias list that contradicts the post-patch state (duplicates, the canonical domain listed as an alias, or aliases left standing with no canonical domain); 500 when a value was saved but the host would not follow — `swapGb`'s grow failed, or a running sandbox refused the new `pidsLimit` (named; it follows at its next wake); a patch carrying both knobs gets both verdicts in one message; 502 when the probed store is unreachable | +| `POST /updateSettings` | rewrite the runtime settings (new-sandbox defaults, default policy, managed swap, the S3 archive store, the sandbox domain, the per-sandbox pids cap) — immediate effect, no restart; each provided group replaces that group whole. Managed swap (`swapGb`) grows immediately; shrinking waits for the next host reboot (an active swapfile is never unmounted — that would drag every frozen sandbox's memory back into RAM). The `s3` group takes all six fields every time (keys included — they are never read back) and is probed with a real write-read-delete round trip before anything is saved; `s3: null` turns archiving off. `sandboxDomain` takes a bare hostname or `null`; `sandboxDomainAliases` takes the full list of inbound-only alias hostnames (`[]` clears) — swapping the canonical domain is one call carrying both fields, see [Exposing ports](/docs/ports). `pidsLimit` (floor 256, never unlimited) reaches new containers at birth, running sandboxes in place before the call returns, and frozen or stopped ones at their next wake — no rebuild | 400 empty patch, a `pidsLimit` below 256, a default policy that asks to archive with no store in force, `swapGb` on a host that cannot manage swap, a failed probe (S3 answered 4xx), clearing/moving the store while sandboxes are archived in it (named by count), or an alias list that contradicts the post-patch state (duplicates, the canonical domain listed as an alias, or aliases left standing with no canonical domain); 500 when a value was saved but the host would not follow — `swapGb`'s grow failed, or a running sandbox refused the new `pidsLimit` (named; it follows at its next wake); a patch carrying both knobs gets both verdicts in one message; 502 when the probed store is unreachable | | `POST /getIngress` | domains bound on the daemon's managed reverse proxy, with live DNS and certificate probes | — | | `POST /setIngress` | rewrite the managed proxy config to exactly this domain list (empty list unbinds all) | 400 when the daemon manages no proxy | diff --git a/website/content/docs/metrics.mdx b/website/content/docs/metrics.mdx index 672e72fb..1050d324 100644 --- a/website/content/docs/metrics.mdx +++ b/website/content/docs/metrics.mdx @@ -78,7 +78,7 @@ The snapshot has four sections: | --- | --- | | `host` | CPU usage, available memory, and **swap** — the freeze mechanism's fuel. Frozen sandboxes live in swap; a full swap means "idle is free" is ending. | | `dataDisk` | The filesystem holding sandbox disks (`DORMICE_DATA_DIR`). This is the real capacity ceiling: past full, even the daemon's database cannot write. | -| `sandboxes` | Totals from the daemon's records: sandbox count, the capacity cap (seeded from `DORMICE_MAX_SANDBOXES`, edited live in the console's settings page), and a count per lifecycle state. | +| `sandboxes` | Totals from the daemon's records: sandbox count and a count per lifecycle state. There is no count-based cap — a record is not a resource; the data disk and the host's CPU and memory are the ceilings, and they are the readings above. | | `sandboxDisks` | What the sparse disk images *promise* (nominal) versus what they *occupy* (actual). The gap is your disk overcommit — watch it, because nothing else caps it. Measured on the test host: a sandbox promising 10 GiB actually occupied 68 MiB. | **Readings are honest about their limits:** diff --git a/website/content/docs/troubleshooting.mdx b/website/content/docs/troubleshooting.mdx index 7f7ee0b9..72b9572f 100644 --- a/website/content/docs/troubleshooting.mdx +++ b/website/content/docs/troubleshooting.mdx @@ -133,15 +133,6 @@ The sandbox image predates `inotify-tools`. Rebuild the image from [rebuild](/docs/upgrading) the sandboxes that need watching. Doctor's image probe warns about exactly this. -## `acquireSandbox` answers 429 - -The sandbox limit (default 100, seeded from `DORMICE_MAX_SANDBOXES`) -gates *creation* only — waking an existing sandbox is never blocked. -Raise it live in the console's settings page (or the `updateSettings` -verb); no restart needed. Before raising it, look at the disk-overcommit -figure in [`getHostMetrics`](/docs/metrics): disk is the real ceiling, -and past full even the daemon's database cannot write. - ## Creating from a template fails: "image … is not on this host" By design: [registration is configuration](/docs/templates) — the image From 3338349610c23da1bce29cc4622fc676feacae75 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 15:38:44 +0800 Subject: [PATCH 30/89] The gateway grows its configuration tables: fleet settings with a version, templates, API keys, the console account MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first half of moving the configuration authority off the daemon (design record #22). The gateway's database gains the settings row — seeded once from the env in the daemon's own variable names, with a version that counts every change the nodes must hear about — the templates, the API keys and the console account, and the nodes table gains the one per-node knob, the managed swap target. Migration 0001. Nothing serves them yet: the verbs, the gates and the console arrive in the next commits, the node pulls the bundle after that. The daemon exports its S3 store as a subpath so the gateway's probe can use the real client, and the archive default moves to the shared policy module, where both seeds read it. --- .../drizzle/0001_configuration-authority.sql | 52 +++ .../gateway/drizzle/meta/0001_snapshot.json | 372 ++++++++++++++++++ packages/gateway/drizzle/meta/_journal.json | 7 + packages/gateway/src/config.ts | 170 +++++++- packages/gateway/src/db/account.ts | 51 +++ packages/gateway/src/db/api-keys.ts | 210 ++++++++++ packages/gateway/src/db/db.ts | 8 + packages/gateway/src/db/schema.ts | 182 ++++++++- packages/gateway/src/db/settings.test.ts | 159 ++++++++ packages/gateway/src/db/settings.ts | 208 ++++++++++ packages/gateway/src/db/templates.ts | 64 +++ packages/gateway/src/main.ts | 3 + packages/server/package.json | 4 + packages/server/src/policy.ts | 12 +- packages/server/tsup.config.ts | 1 + packages/shared/src/policy.ts | 10 + 16 files changed, 1491 insertions(+), 22 deletions(-) create mode 100644 packages/gateway/drizzle/0001_configuration-authority.sql create mode 100644 packages/gateway/drizzle/meta/0001_snapshot.json create mode 100644 packages/gateway/src/db/account.ts create mode 100644 packages/gateway/src/db/api-keys.ts create mode 100644 packages/gateway/src/db/settings.test.ts create mode 100644 packages/gateway/src/db/settings.ts create mode 100644 packages/gateway/src/db/templates.ts diff --git a/packages/gateway/drizzle/0001_configuration-authority.sql b/packages/gateway/drizzle/0001_configuration-authority.sql new file mode 100644 index 00000000..57a904b3 --- /dev/null +++ b/packages/gateway/drizzle/0001_configuration-authority.sql @@ -0,0 +1,52 @@ +CREATE TABLE `api_keys` ( + `id` text PRIMARY KEY NOT NULL, + `name` text NOT NULL, + `key_hash` text NOT NULL, + `prefix` text NOT NULL, + `created_at` text NOT NULL, + `last_used_at` text, + `expires_at` text, + `disabled_at` text, + `revoked_at` text +); +--> statement-breakpoint +CREATE UNIQUE INDEX `api_keys_key_hash_unique` ON `api_keys` (`key_hash`);--> statement-breakpoint +CREATE UNIQUE INDEX `api_keys_active_name_idx` ON `api_keys` (`name`) WHERE "api_keys"."revoked_at" IS NULL;--> statement-breakpoint +CREATE TABLE `console_account` ( + `id` integer PRIMARY KEY NOT NULL, + `username` text NOT NULL, + `password_hash` text NOT NULL, + `session_secret` text NOT NULL, + `created_at` text NOT NULL, + `updated_at` text NOT NULL +); +--> statement-breakpoint +CREATE TABLE `settings` ( + `id` integer PRIMARY KEY NOT NULL, + `version` integer NOT NULL, + `sandbox_cpus` real NOT NULL, + `sandbox_memory_gb` real NOT NULL, + `sandbox_disk_gb` real NOT NULL, + `default_freeze_after_seconds` integer NOT NULL, + `default_stop_after_seconds` integer, + `default_archive_after_seconds` integer, + `s3_endpoint` text, + `s3_bucket` text, + `s3_access_key_id` text, + `s3_secret_access_key` text, + `s3_region` text, + `s3_force_path_style` integer, + `sandbox_domain` text, + `sandbox_domain_aliases` text NOT NULL, + `pids_limit` integer NOT NULL, + `updated_at` text +); +--> statement-breakpoint +CREATE TABLE `templates` ( + `name` text PRIMARY KEY NOT NULL, + `image` text NOT NULL, + `created_at` text NOT NULL, + `updated_at` text NOT NULL +); +--> statement-breakpoint +ALTER TABLE `nodes` ADD `swap_gb` integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/0001_snapshot.json b/packages/gateway/drizzle/meta/0001_snapshot.json new file mode 100644 index 00000000..0632006d --- /dev/null +++ b/packages/gateway/drizzle/meta/0001_snapshot.json @@ -0,0 +1,372 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "2821eca7-9ec2-43c6-a788-77c858d4360d", + "prevId": "b88172b7-a470-4a8c-addb-45c45229585f", + "tables": { + "api_keys": { + "name": "api_keys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + "key_hash" + ], + "isUnique": true + }, + "api_keys_active_name_idx": { + "name": "api_keys_active_name_idx", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"api_keys\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "console_account": { + "name": "console_account", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_secret": { + "name": "session_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "nodes": { + "name": "nodes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "added_at": { + "name": "added_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_gb": { + "name": "swap_gb", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_cpus": { + "name": "sandbox_cpus", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_memory_gb": { + "name": "sandbox_memory_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_disk_gb": { + "name": "sandbox_disk_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_freeze_after_seconds": { + "name": "default_freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_stop_after_seconds": { + "name": "default_stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_archive_after_seconds": { + "name": "default_archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_endpoint": { + "name": "s3_endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_access_key_id": { + "name": "s3_access_key_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_secret_access_key": { + "name": "s3_secret_access_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_region": { + "name": "s3_region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_force_path_style": { + "name": "s3_force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain": { + "name": "sandbox_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain_aliases": { + "name": "sandbox_domain_aliases", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pids_limit": { + "name": "pids_limit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/_journal.json b/packages/gateway/drizzle/meta/_journal.json index d08827ad..b37272a1 100644 --- a/packages/gateway/drizzle/meta/_journal.json +++ b/packages/gateway/drizzle/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1789328056388, "tag": "0000_nodes", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1789371415369, + "tag": "0001_configuration-authority", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/gateway/src/config.ts b/packages/gateway/src/config.ts index 94facb04..55e60027 100644 --- a/packages/gateway/src/config.ts +++ b/packages/gateway/src/config.ts @@ -1,4 +1,6 @@ import { isAbsolute } from 'node:path'; +import type { S3Settings } from '@dormice/server/s3-store'; +import { bareHostnameRegex, PIDS_LIMIT_MIN } from '@dormice/shared'; import { z } from 'zod'; /** @@ -9,6 +11,15 @@ import { z } from 'zod'; * /etc/dormice without colliding; the token deliberately does not — it is * the very same DORMICE_API_TOKEN every node has, one string for the whole * fleet, written into both env files by the same hand. + * + * The fleet's operator knobs (new-sandbox defaults, the pids cap, the S3 + * store, the sandbox domain) are first-boot seeds only: the gateway's + * first start writes them into its settings table (db/settings.ts), and + * from then on the table is the one truth, edited from the console or + * updateSettings; a later env edit of these is deliberately ignored — two + * live sources for one knob is a standing ambiguity. They keep the + * daemon's old names so an operator moving a single machine onto the + * gateway copies the lines across, nothing more. */ const envSchema = z.object({ DORMICE_GATEWAY_PORT: z.coerce.number().int().min(1).max(65535).default(3677), @@ -69,10 +80,167 @@ const envSchema = z.object({ * names from landing on the same full box. */ DORMICE_GATEWAY_NODE_MIN_DISK_GB: z.coerce.number().nonnegative().default(10), + // ---- first-boot seeds of the settings table, in the daemon's words ---- + DORMICE_SANDBOX_DISK_GB: z.coerce.number().positive().default(10), + DORMICE_SANDBOX_CPUS: z.coerce.number().positive().default(1), + DORMICE_SANDBOX_MEMORY_GB: z.coerce.number().positive().default(2), + /** + * The pids cgroup cap on each sandbox's container, fleet-wide. Under + * gVisor it caps the sandbox's host-side footprint, not a count the + * guest can see, and hitting it kills the whole sandbox (exit 2, no OOM + * flag; measured 2026-09-08). 4096 is ~8x the peak a real agent sandbox + * was seen at. Floored at the wire's PIDS_LIMIT_MIN: the settings view + * promises that floor, so a lower seed would leave getConfig unable to + * serialize its own settings — refused here, at boot, named. + */ + DORMICE_SANDBOX_PIDS_LIMIT: z.coerce + .number() + .int() + .min(PIDS_LIMIT_MIN, { + error: `DORMICE_SANDBOX_PIDS_LIMIT must be at least ${PIDS_LIMIT_MIN} — below that a sandbox cannot boot its own runtime`, + }) + .default(4096), + /** + * The sandbox wildcard domain behind getHost() and port previews. A bare + * hostname — the same regex the wire validates against. + */ + DORMICE_SANDBOX_DOMAIN: z + .string() + .regex(bareHostnameRegex, { + error: + 'DORMICE_SANDBOX_DOMAIN must be a bare hostname like sbx.example.com — no scheme, no port, no leading/trailing dots', + }) + .optional(), + /** + * The S3-compatible object store behind every node's archiver (AWS, R2, + * MinIO, OSS in S3-compat mode). The four core variables come as a set + * — a half-configured seed refuses to boot; none of them seeds + * "archiving off", which the console can turn on at any time. + */ + DORMICE_S3_ENDPOINT: z + .url({ + protocol: /^https?$/, + error: + 'DORMICE_S3_ENDPOINT must be a full http(s) URL, e.g. https://s3.example.com or http://127.0.0.1:9000', + }) + .optional(), + DORMICE_S3_BUCKET: z.string().min(1).optional(), + DORMICE_S3_ACCESS_KEY_ID: z.string().min(1).optional(), + DORMICE_S3_SECRET_ACCESS_KEY: z.string().min(1).optional(), + DORMICE_S3_REGION: z.string().default('us-east-1'), + /** Path-style addressing: MinIO needs true; the clouds route by subdomain. */ + DORMICE_S3_FORCE_PATH_STYLE: z.stringbool().default(false), + /** + * The Caddy config file the gateway owns — the switch for web-based + * domain binding (setIngress rewrites the file, reloads Caddy, Caddy + * handles the certificate). Unset, the gateway never touches any proxy + * config and setIngress is refused — the feature is honestly absent. + * Absolute: a system file must not move with the start directory. + */ + DORMICE_INGRESS_FILE: z + .string() + .refine(isAbsolute, { + error: + 'DORMICE_INGRESS_FILE must be an absolute path, e.g. /etc/caddy/Caddyfile', + }) + .optional(), + /** + * How the gateway tells the running proxy to re-read its config after a + * bind. Defaults to `caddy reload --config `; an + * operator whose own Caddyfile imports a Dormice-owned fragment points + * this at the outer file instead. + */ + DORMICE_INGRESS_RELOAD_CMD: z.string().min(1).optional(), +}); + +// All-or-none: a half-configured store would make "is archiving on" +// ambiguous, and that answer decides real policy defaults. +const checkedSchema = envSchema.superRefine((cfg, ctx) => { + const wanted = [ + 'DORMICE_S3_ENDPOINT', + 'DORMICE_S3_BUCKET', + 'DORMICE_S3_ACCESS_KEY_ID', + 'DORMICE_S3_SECRET_ACCESS_KEY', + ] as const; + const missing = wanted.filter((name) => cfg[name] === undefined); + const first = missing[0]; + if (first !== undefined && missing.length < wanted.length) { + ctx.addIssue({ + code: 'custom', + message: `the DORMICE_S3_* variables come as a set: ${missing.join(', ')} ${missing.length === 1 ? 'is' : 'are'} missing — set all four to seed the archive store, or none to leave archiving off`, + path: [first], + }); + } }); export type Config = z.infer; export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config { - return envSchema.parse(env); + return checkedSchema.parse(env); +} + +/** + * Every knob the gateway has, in display order, with its secrecy flag — + * the single adjudication of "what getConfig reports". A Record over keyof + * Config so the compiler refuses a new env variable until it is listed + * here too: a knob that exists but is invisible would be a silent lie. + */ +export const CONFIG_KEYS: Record = { + DORMICE_GATEWAY_PORT: { sensitive: false }, + DORMICE_GATEWAY_DB_PATH: { sensitive: false }, + DORMICE_API_TOKEN: { sensitive: true }, + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: { sensitive: false }, + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: { sensitive: false }, + DORMICE_GATEWAY_NODE_MIN_DISK_GB: { sensitive: false }, + DORMICE_SANDBOX_DISK_GB: { sensitive: false }, + DORMICE_SANDBOX_CPUS: { sensitive: false }, + DORMICE_SANDBOX_MEMORY_GB: { sensitive: false }, + DORMICE_SANDBOX_PIDS_LIMIT: { sensitive: false }, + DORMICE_SANDBOX_DOMAIN: { sensitive: false }, + DORMICE_S3_ENDPOINT: { sensitive: false }, + DORMICE_S3_BUCKET: { sensitive: false }, + DORMICE_S3_ACCESS_KEY_ID: { sensitive: true }, + DORMICE_S3_SECRET_ACCESS_KEY: { sensitive: true }, + DORMICE_S3_REGION: { sensitive: false }, + DORMICE_S3_FORCE_PATH_STYLE: { sensitive: false }, + DORMICE_INGRESS_FILE: { sensitive: false }, + DORMICE_INGRESS_RELOAD_CMD: { sensitive: false }, +}; + +export type ConfigSources = Record; + +/** + * Which knobs the operator set explicitly versus which fell back to + * defaults. Read off the raw environment at load time — the parsed config + * cannot tell the two apart once defaults are applied. + */ +export function configSources( + env: NodeJS.ProcessEnv = process.env, +): ConfigSources { + return Object.fromEntries( + (Object.keys(CONFIG_KEYS) as Array).map((key) => [ + key, + env[key] !== undefined ? 'env' : 'default', + ]), + ) as ConfigSources; +} + +/** The S3 first-boot seed: null unless the whole DORMICE_S3_* set is present (a partial set never gets past the schema). */ +export function s3Seed(config: Config): S3Settings | null { + if ( + config.DORMICE_S3_ENDPOINT === undefined || + config.DORMICE_S3_BUCKET === undefined || + config.DORMICE_S3_ACCESS_KEY_ID === undefined || + config.DORMICE_S3_SECRET_ACCESS_KEY === undefined + ) { + return null; + } + return { + endpoint: config.DORMICE_S3_ENDPOINT, + bucket: config.DORMICE_S3_BUCKET, + accessKeyId: config.DORMICE_S3_ACCESS_KEY_ID, + secretAccessKey: config.DORMICE_S3_SECRET_ACCESS_KEY, + region: config.DORMICE_S3_REGION, + forcePathStyle: config.DORMICE_S3_FORCE_PATH_STYLE, + }; } diff --git a/packages/gateway/src/db/account.ts b/packages/gateway/src/db/account.ts new file mode 100644 index 00000000..bdabf813 --- /dev/null +++ b/packages/gateway/src/db/account.ts @@ -0,0 +1,51 @@ +import { eq } from 'drizzle-orm'; +import type { Db } from './db'; +import { type ConsoleAccountRow, consoleAccount } from './schema'; + +/** + * The console's single human account (see the schema comment for why it is + * a singleton). The fixed id makes setup an upsert: presenting the API + * token overwrites whatever is there — account creation, password change + * and forgot-password are all the same verb. + */ +const ACCOUNT_ID = 1; + +export function getConsoleAccount(db: Db): ConsoleAccountRow | undefined { + return db + .select() + .from(consoleAccount) + .where(eq(consoleAccount.id, ACCOUNT_ID)) + .get(); +} + +export function setConsoleAccount( + db: Db, + input: { username: string; passwordHash: string; sessionSecret: string }, +): ConsoleAccountRow { + const now = new Date().toISOString(); + const row: ConsoleAccountRow = { + id: ACCOUNT_ID, + username: input.username, + passwordHash: input.passwordHash, + sessionSecret: input.sessionSecret, + createdAt: now, + updatedAt: now, + }; + db.insert(consoleAccount) + .values(row) + .onConflictDoUpdate({ + target: consoleAccount.id, + set: { + username: input.username, + passwordHash: input.passwordHash, + sessionSecret: input.sessionSecret, + updatedAt: now, + }, + }) + .run(); + const stored = getConsoleAccount(db); + if (!stored) { + throw new Error('console account vanished mid-setup'); + } + return stored; +} diff --git a/packages/gateway/src/db/api-keys.ts b/packages/gateway/src/db/api-keys.ts new file mode 100644 index 00000000..0ddc1d7a --- /dev/null +++ b/packages/gateway/src/db/api-keys.ts @@ -0,0 +1,210 @@ +import { createHash, randomBytes, randomUUID } from 'node:crypto'; +import { and, desc, eq, gt, isNull, lt, or, sql } from 'drizzle-orm'; +import type { Db } from './db'; +import { type ApiKeyRow, apiKeys } from './schema'; + +/** + * lastUsedAt write granularity. A hot-polling client authenticates many + * times a second; writing the ledger on every hit would turn the WAL into + * an EKG for zero information. One write a minute answers the only question + * the column exists for — "is this key still alive, roughly since when". + */ +const LAST_USED_GRANULARITY_MS = 60_000; + +/** sha256 hex of the bare key material — the only form the ledger stores. */ +export function hashApiKeyToken(token: string): string { + return createHash('sha256').update(token).digest('hex'); +} + +/** + * The one write-side shape for timestamps that later feed string + * comparisons. Wire ISO input has variable precision ("…00Z" sorts after + * "…00.500Z" while being chronologically earlier), so every expiresAt is + * normalized to exact toISOString() output before it touches the ledger — + * after that, plain string compare is chronologically sound (the same + * argument the lastUsedAt throttle rests on). + */ +function normalizeIso(value: string): string { + return new Date(value).toISOString(); +} + +/** + * Mints a key and returns the material exactly once — the row keeps only + * the hash, so this return value is the caller's single chance to see it. + * Pure hex, no brand prefix: the official Python E2B SDK validates + * `e2b_[0-9a-f]+` client-side, so anything non-hex could never be used on + * the X-API-KEY face at all. + * + * The caller (route) has already refused a duplicate active name with a + * 409; the partial unique index backstops that check as a schema fact. + */ +export function createApiKey( + db: Db, + name: string, + expiresAt: string | undefined, +): { row: ApiKeyRow; token: string } { + const token = randomBytes(32).toString('hex'); + const row: ApiKeyRow = { + id: randomUUID(), + name, + keyHash: hashApiKeyToken(token), + prefix: token.slice(0, 8), + createdAt: new Date().toISOString(), + lastUsedAt: null, + expiresAt: expiresAt ? normalizeIso(expiresAt) : null, + disabledAt: null, + revokedAt: null, + }; + db.insert(apiKeys).values(row).run(); + return { row, token }; +} + +export function findActiveApiKeyByName( + db: Db, + name: string, +): ApiKeyRow | undefined { + return db + .select() + .from(apiKeys) + .where(and(eq(apiKeys.name, name), isNull(apiKeys.revokedAt))) + .get(); +} + +export function findApiKeyById(db: Db, id: string): ApiKeyRow | undefined { + return db.select().from(apiKeys).where(eq(apiKeys.id, id)).get(); +} + +/** + * Every key ever minted, revoked ones included — the rotation history. + * createdAt has millisecond granularity, so two keys minted back-to-back + * can tie; rowid breaks the tie by insertion order, and it is trustworthy + * here because api_keys rows are never deleted (revoke is soft), so rowids + * are never reused. + */ +export function listApiKeys(db: Db): ApiKeyRow[] { + return db + .select() + .from(apiKeys) + .orderBy(desc(apiKeys.createdAt), desc(sql`rowid`)) + .all(); +} + +/** + * Soft-revokes the key with this id. Returns false when it does not exist + * or is already revoked — the desired end state was already true. The row + * survives as history; the name is immediately free for a new key. + */ +export function revokeApiKey(db: Db, id: string): boolean { + const row = findApiKeyById(db, id); + if (!row || row.revokedAt !== null) { + return false; + } + db.update(apiKeys) + .set({ revokedAt: new Date().toISOString() }) + .where(eq(apiKeys.id, id)) + .run(); + return true; +} + +/** + * Edits a non-revoked key in place. The route has already adjudicated the + * 404 (unknown id), the 409s (revoked row, name collision) — this function + * only computes the changed-field set against the row it was handed and + * writes once. A field equal to its current value is not a change (the + * updatePolicy idiom: a no-op patch is the goal state, not an error), so + * disabling an already-disabled key keeps its original disabledAt. The + * returned row carries what changed; the route logs it. + */ +export function updateApiKey( + db: Db, + row: ApiKeyRow, + patch: { name?: string; expiresAt?: string | null; disabled?: boolean }, +): ApiKeyRow { + const changes: Partial = {}; + if (patch.name !== undefined && patch.name !== row.name) { + changes.name = patch.name; + } + if (patch.expiresAt !== undefined) { + const next = + patch.expiresAt === null ? null : normalizeIso(patch.expiresAt); + if (next !== row.expiresAt) { + changes.expiresAt = next; + } + } + if (patch.disabled === true && row.disabledAt === null) { + changes.disabledAt = new Date().toISOString(); + } else if (patch.disabled === false && row.disabledAt !== null) { + changes.disabledAt = null; + } + + if (Object.keys(changes).length === 0) { + return row; + } + db.update(apiKeys).set(changes).where(eq(apiKeys.id, row.id)).run(); + return { ...row, ...changes }; +} + +/** + * The single liveness adjudication: revoked, disabled and expired all close + * the door, in one WHERE. Pure read — it never stamps lastUsedAt — so the + * admin gate can consult it for its honest 403 without a refused request + * leaving "recently used" fingerprints. The expiry compare is a plain + * string > against toISOString(now), sound because expiresAt is normalized + * on write (see normalizeIso). + */ +export function findLiveApiKeyByHash( + db: Db, + hash: string, +): { id: string; lastUsedAt: string | null } | undefined { + return db + .select({ id: apiKeys.id, lastUsedAt: apiKeys.lastUsedAt }) + .from(apiKeys) + .where( + and( + eq(apiKeys.keyHash, hash), + isNull(apiKeys.revokedAt), + isNull(apiKeys.disabledAt), + or( + isNull(apiKeys.expiresAt), + gt(apiKeys.expiresAt, new Date().toISOString()), + ), + ), + ) + .get(); +} + +/** findLiveApiKeyByHash for callers holding the bare token — hashing stays in this module. */ +export function isLiveApiKey(db: Db, bareToken: string): boolean { + return findLiveApiKeyByHash(db, hashApiKeyToken(bareToken)) !== undefined; +} + +/** + * The ledger leg of credential verification: does this bare token match a + * live key? An indexed exact-match lookup on sha256(token) — not a + * timing-safe scan, deliberately: the comparison can at worst leak bytes of + * sha256(key), which preimage resistance makes worthless to an attacker + * (the argument GitHub token storage rests on). + * + * A hit answers the key's id and stamps lastUsedAt — only a hit: + * verification is the one moment a credential was actually honored. Throttled to LAST_USED_GRANULARITY_MS so a polling client does + * not write the ledger per request. ISO strings compare lexicographically + * as timestamps, so the cutoff is a plain string <. + */ +export function verifyApiKeyToken(db: Db, bareToken: string): string | null { + const row = findLiveApiKeyByHash(db, hashApiKeyToken(bareToken)); + if (!row) { + return null; + } + const now = Date.now(); + const cutoff = new Date(now - LAST_USED_GRANULARITY_MS).toISOString(); + db.update(apiKeys) + .set({ lastUsedAt: new Date(now).toISOString() }) + .where( + and( + eq(apiKeys.id, row.id), + or(isNull(apiKeys.lastUsedAt), lt(apiKeys.lastUsedAt, cutoff)), + ), + ) + .run(); + return row.id; +} diff --git a/packages/gateway/src/db/db.ts b/packages/gateway/src/db/db.ts index f0865403..7b1fbb12 100644 --- a/packages/gateway/src/db/db.ts +++ b/packages/gateway/src/db/db.ts @@ -20,6 +20,14 @@ export function openDb(path: string) { export type Db = ReturnType; +/** + * What a write-side helper needs of the database — satisfied by the + * database itself and by the transaction handle inside db.transaction(), + * so a helper can run inside a caller's transaction (db/templates.ts + * writes a row and counts the configuration version up as one). + */ +export type Writer = Pick; + /** Applies pending migrations (drizzle-kit output, committed) at every start. */ export function migrateDb(db: Db, migrationsFolder: string) { migrate(db, { migrationsFolder }); diff --git a/packages/gateway/src/db/schema.ts b/packages/gateway/src/db/schema.ts index 227d9d5b..e6e993fe 100644 --- a/packages/gateway/src/db/schema.ts +++ b/packages/gateway/src/db/schema.ts @@ -1,24 +1,35 @@ -import { sqliteTable, text } from 'drizzle-orm/sqlite-core'; +import { sql } from 'drizzle-orm'; +import { + integer, + real, + sqliteTable, + text, + uniqueIndex, +} from 'drizzle-orm/sqlite-core'; /** * The gateway's tables are "how the fleet is configured and who may * enter" — never a sandbox's state, which lives in the ledger of the node - * that runs it and is asked for when needed (find.ts). One table today; - * api_keys, settings, templates and console_account arrive here with the - * configuration authority. + * that runs it and is asked for when needed (find.ts). Five tables: the + * nodes that have ever checked in, the fleet-wide settings row, the + * templates, the API keys and the console account. The last four moved + * here from the daemon with the configuration authority (design record + * #22, 2026-09-13): one authority, one edit, every node pulls it at its + * next check-in and keeps a copy in its own ledger. */ /** * Every node that has ever checked in (routes/nodes.ts): its id, the - * address the gateway forwards to, and when it first appeared. Written by - * the nodes themselves at their first check-in — there is no registration - * verb and no nodes file, so "which nodes exist" has exactly one home — - * and deleted only by an operator's removeNode. Persistent, not memory, - * for one reason: a node that is down must still be known after a gateway - * restart, or a name that lives only there would be placed anew elsewhere - * and come back as a conflict when the node returns. Everything the node - * last reported (its reading, build, check-in time) is memory: fifteen - * seconds later it is reported again. + * address the gateway forwards to, when it first appeared, and the one + * per-node setting — how much swap its daemon manages on its own disk. + * Written by the nodes themselves at their first check-in — there is no + * registration verb and no nodes file, so "which nodes exist" has exactly + * one home — and deleted only by an operator's removeNode. Persistent, not + * memory, for one reason: a node that is down must still be known after a + * gateway restart, or a name that lives only there would be placed anew + * elsewhere and come back as a conflict when the node returns. Everything + * the node last reported (its reading, build, check-in time) is memory: + * fifteen seconds later it is reported again. */ export const nodes = sqliteTable('nodes', { /** DORMICE_NODE_ID as the node states it — the `nodeId` in every sandbox answer. */ @@ -27,6 +38,151 @@ export const nodes = sqliteTable('nodes', { endpoint: text('endpoint').notNull(), /** ISO 8601 UTC — the first check-in. */ addedAt: text('added_at').notNull(), + /** + * Managed swap the node's daemon keeps on its data disk, GiB, on top of + * the host's own — the one setting that is a machine's, not the fleet's + * (a 29 GB test box and a 243 GB production box want different numbers). + * Set by updateNodeSettings; the node applies it at its next check-in. + * 0 = manage none, the only value that fits every host at birth. + */ + swapGb: integer('swap_gb').notNull().default(0), }); export type NodeRow = typeof nodes.$inferSelect; + +/** + * The fleet-wide settings: one row, fixed id — the operator knobs whose + * change is an operations decision, never a machine's identity (shared + * settings.ts draws the line). `version` counts every configuration + * change the nodes must hear about — this row, a node's row, the + * templates — and rides the check-in wire: a node reports the version it + * applied, the gateway answers the current one and, when they differ, the + * whole bundle. Two states per optional group (NULL = off), no sentinel: + * the row is born whole from the env seeds at the gateway's first start, + * never adopted column by column as the daemon's once was. + * + * s3SecretAccessKey is stored plaintext, like the daemon's own secrets: a + * credential presented verbatim to S3 cannot be hashed. It leaves this + * table on exactly one path — the check-in bundle to a node, on the + * intranet, under the fleet token — and never on the observation wire. + */ +export const settings = sqliteTable('settings', { + id: integer('id').primaryKey(), + version: integer('version').notNull(), + sandboxCpus: real('sandbox_cpus').notNull(), + sandboxMemoryGb: real('sandbox_memory_gb').notNull(), + sandboxDiskGb: real('sandbox_disk_gb').notNull(), + defaultFreezeAfterSeconds: integer('default_freeze_after_seconds').notNull(), + /** NULL = new sandboxes default to never stopping. */ + defaultStopAfterSeconds: integer('default_stop_after_seconds'), + /** NULL = never archive — forced while the fleet has no store. */ + defaultArchiveAfterSeconds: integer('default_archive_after_seconds'), + /** The S3 archive store; all six NULL = archiving is off. */ + s3Endpoint: text('s3_endpoint'), + s3Bucket: text('s3_bucket'), + s3AccessKeyId: text('s3_access_key_id'), + s3SecretAccessKey: text('s3_secret_access_key'), + s3Region: text('s3_region'), + s3ForcePathStyle: integer('s3_force_path_style', { mode: 'boolean' }), + /** The canonical sandbox wildcard domain; NULL = the port proxy and domain fields are off. */ + sandboxDomain: text('sandbox_domain'), + /** Inbound-only alias domains, a JSON string array; '[]' = none. */ + sandboxDomainAliases: text('sandbox_domain_aliases').notNull(), + /** The pids cgroup cap on every sandbox container, fleet-wide. */ + pidsLimit: integer('pids_limit').notNull(), + /** Null until the first updateSettings: "still exactly the seed" is information. */ + updatedAt: text('updated_at'), +}); + +export type SettingsRow = typeof settings.$inferSelect; + +/** + * Templates: a name for an image, fleet-wide. Registered here, carried to + * every node in the bundle (a cold wake of a template sandbox resolves + * name → image on the node, with or without a gateway present). Removal + * asks every node whether a sandbox still uses the name (routes/templates.ts). + */ +export const templates = sqliteTable('templates', { + name: text('name').primaryKey(), + image: text('image').notNull(), + createdAt: text('created_at').notNull(), + /** Bumped only when the image actually changes — see db/templates.ts. */ + updatedAt: text('updated_at').notNull(), +}); + +export type TemplateRow = typeof templates.$inferSelect; + +/** + * Gateway-minted API keys: the credentials callers present at the fleet's + * one door, peers of DORMICE_API_TOKEN over the sandbox verbs. They exist + * so a client's secret can rotate without an env edit and a restart on + * every machine of the fleet (design record #20): mint a new key, move the + * client over, revoke the old one, a minute's work and no downtime. The + * env token itself never lives here — it stays the bootstrap/recovery + * credential, checked from config, and the one credential nodes accept. + * + * keyHash is sha256 of the key material, not scrypt: a key is 256 random + * bits, not a human password, so offline brute force is moot and a slow + * KDF would only tax every authenticated request. Verification is an + * indexed exact-match lookup on the hash. + * + * Revocation is soft (revokedAt) — the row stays as rotation history and + * keeps lastUsedAt readable after the credential dies. "At most one ACTIVE + * key per name" is a schema fact via the partial unique index below; a + * revoked name is free for reuse. + */ +export const apiKeys = sqliteTable( + 'api_keys', + { + /** UUID, never an autoincrement — ids must stay unique across machines. */ + id: text('id').primaryKey(), + name: text('name').notNull(), + /** sha256 hex of the bare 64-hex key material. The key itself is never stored. */ + keyHash: text('key_hash').notNull().unique(), + /** First 8 hex chars of the key, for display — 32 bits, no meaningful entropy. */ + prefix: text('prefix').notNull(), + createdAt: text('created_at').notNull(), + /** Null = never used. Written with 60s granularity, not per request. */ + lastUsedAt: text('last_used_at'), + /** + * Null = never expires. Always written through normalizeIso (exact + * toISOString shape) so the liveness filter's string comparison against + * "now" is chronologically sound — wire input has variable precision. + */ + expiresAt: text('expires_at'), + /** + * Null = enabled. The reversible half of revocation: set/cleared by + * updateApiKey, and the name stays held while disabled — only revoke + * frees a name. + */ + disabledAt: text('disabled_at'), + /** Null = active. Set once by revokeApiKey; never cleared. */ + revokedAt: text('revoked_at'), + }, + (table) => [ + uniqueIndex('api_keys_active_name_idx') + .on(table.name) + .where(sql`${table.revokedAt} IS NULL`), + ], +); + +export type ApiKeyRow = typeof apiKeys.$inferSelect; + +/** + * The web console's single human account, one row with a fixed id: the + * fleet has one console (design record #24) and the console has one + * operator. Setup with the env token overwrites the row — creation, + * password change and forgot-password are one verb (routes/console.ts). + */ +export const consoleAccount = sqliteTable('console_account', { + id: integer('id').primaryKey(), + username: text('username').notNull(), + /** Self-describing scrypt string: scrypt$N$r$p$$. */ + passwordHash: text('password_hash').notNull(), + /** The HMAC key of every session cookie; a new one voids every session. */ + sessionSecret: text('session_secret').notNull(), + createdAt: text('created_at').notNull(), + updatedAt: text('updated_at').notNull(), +}); + +export type ConsoleAccountRow = typeof consoleAccount.$inferSelect; diff --git a/packages/gateway/src/db/settings.test.ts b/packages/gateway/src/db/settings.test.ts new file mode 100644 index 00000000..e4c831d5 --- /dev/null +++ b/packages/gateway/src/db/settings.test.ts @@ -0,0 +1,159 @@ +import { fileURLToPath } from 'node:url'; +import { + ARCHIVE_DEFAULT_SECONDS, + DEFAULT_LIFECYCLE_POLICY, +} from '@dormice/shared'; +import { describe, expect, it } from 'vitest'; +import { loadConfig } from '../config'; +import { migrateDb, openDb } from './db'; +import { + bumpConfigVersion, + ensureSettings, + readConfigVersion, + readS3Settings, + readSettings, + writeSettings, +} from './settings'; +import { + findTemplate, + listTemplates, + registerTemplate, + removeTemplate, +} from './templates'; + +const MIGRATIONS = fileURLToPath(new URL('../../drizzle', import.meta.url)); +const TOKEN = 'fleet-token-fleet-token-fleet-token-fleet'; +const NOW = new Date('2026-09-14T12:00:00.000Z'); + +const S3_ENV = { + DORMICE_S3_ENDPOINT: 'http://127.0.0.1:9000', + DORMICE_S3_BUCKET: 'seed-bucket', + DORMICE_S3_ACCESS_KEY_ID: 'seed-key', + DORMICE_S3_SECRET_ACCESS_KEY: 'seed-secret-never-on-the-wire', +}; + +function seeded(env: Record = {}) { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + // Through loadConfig on purpose: defaults are adjudicated once, in the + // schema — a hand-written literal here would drift as knobs are added. + const config = loadConfig({ DORMICE_API_TOKEN: TOKEN, ...env }); + ensureSettings(db, config); + return { db, config }; +} + +describe('the settings row', () => { + it('is seeded from the env at the first start, defaults where the env is silent, at version 1', () => { + const { db } = seeded({ DORMICE_SANDBOX_DISK_GB: '20' }); + expect(readSettings(db)).toEqual({ + sandboxDefaults: { cpus: 1, memoryGb: 2, diskGb: 20 }, + // No S3 seed, so the seeded default never archives. + defaultPolicy: { ...DEFAULT_LIFECYCLE_POLICY, archiveAfterSeconds: null }, + s3: null, + sandboxDomain: null, + sandboxDomainAliases: [], + pidsLimit: 4096, + updatedAt: null, + }); + expect(readConfigVersion(db)).toBe(1); + expect(readS3Settings(db)).toBeNull(); + }); + + it('an S3 seed turns the archive default on and keeps the keys for the bundle, never for the view', () => { + const { db } = seeded({ + ...S3_ENV, + DORMICE_SANDBOX_DOMAIN: 'sbx.example.com', + }); + const view = readSettings(db); + expect(view.defaultPolicy.archiveAfterSeconds).toBe( + ARCHIVE_DEFAULT_SECONDS, + ); + expect(view.s3).toEqual({ + endpoint: 'http://127.0.0.1:9000', + bucket: 'seed-bucket', + region: 'us-east-1', + forcePathStyle: false, + }); + expect(JSON.stringify(view)).not.toContain('seed-key'); + expect(JSON.stringify(view)).not.toContain('seed-secret'); + expect(readS3Settings(db)).toMatchObject({ + accessKeyId: 'seed-key', + secretAccessKey: 'seed-secret-never-on-the-wire', + }); + expect(view.sandboxDomain).toBe('sbx.example.com'); + }); + + it('the table wins over a later env edit: the seed is read once', () => { + const { db, config } = seeded({ DORMICE_SANDBOX_PIDS_LIMIT: '512' }); + expect(readSettings(db).pidsLimit).toBe(512); + ensureSettings(db, { ...config, DORMICE_SANDBOX_PIDS_LIMIT: 8192 }); + expect(readSettings(db).pidsLimit).toBe(512); + expect(readConfigVersion(db)).toBe(1); + }); + + it('a write replaces the provided groups whole, leaves the rest, counts the version up and stamps updatedAt', () => { + const { db } = seeded(S3_ENV); + const after = writeSettings( + db, + { + pidsLimit: 2048, + sandboxDomain: 'sbx.example.com', + sandboxDomainAliases: ['a.example.com'], + }, + NOW, + ); + expect(after.pidsLimit).toBe(2048); + expect(after.sandboxDomain).toBe('sbx.example.com'); + expect(after.sandboxDomainAliases).toEqual(['a.example.com']); + expect(after.sandboxDefaults).toEqual({ cpus: 1, memoryGb: 2, diskGb: 10 }); + expect(after.s3?.bucket).toBe('seed-bucket'); + expect(after.updatedAt).toBe(NOW.toISOString()); + expect(readConfigVersion(db)).toBe(2); + // Clearing the store is all six columns at once; the keys go with it. + writeSettings( + db, + { s3: null, defaultPolicy: DEFAULT_LIFECYCLE_POLICY }, + NOW, + ); + expect(readSettings(db).s3).toBeNull(); + expect(readS3Settings(db)).toBeNull(); + expect(readConfigVersion(db)).toBe(3); + }); + + it('bumpConfigVersion counts up by one and answers the new version', () => { + const { db } = seeded(); + expect(bumpConfigVersion(db)).toBe(2); + expect(bumpConfigVersion(db)).toBe(3); + expect(readConfigVersion(db)).toBe(3); + }); +}); + +describe('templates', () => { + it('register is an upsert that counts the version up only when the image changes', () => { + const { db } = seeded(); + const born = registerTemplate(db, { name: 'py', image: 'img-v1' }); + expect(born.createdAt).toBe(born.updatedAt); + expect(readConfigVersion(db)).toBe(2); + // Same image again: nothing written, nothing to tell the nodes. + expect(registerTemplate(db, { name: 'py', image: 'img-v1' })).toEqual(born); + expect(readConfigVersion(db)).toBe(2); + const moved = registerTemplate(db, { name: 'py', image: 'img-v2' }); + expect(moved.image).toBe('img-v2'); + expect(moved.createdAt).toBe(born.createdAt); + expect(readConfigVersion(db)).toBe(3); + expect(findTemplate(db, 'py')?.image).toBe('img-v2'); + registerTemplate(db, { name: 'a-first', image: 'x' }); + expect(listTemplates(db).map((t) => t.name)).toEqual(['a-first', 'py']); + }); + + it('remove answers whether a row went, and counts the version up only then', () => { + const { db } = seeded(); + registerTemplate(db, { name: 'py', image: 'img-v1' }); + expect(readConfigVersion(db)).toBe(2); + expect(removeTemplate(db, 'py')).toBe(true); + expect(readConfigVersion(db)).toBe(3); + expect(removeTemplate(db, 'py')).toBe(false); + expect(readConfigVersion(db)).toBe(3); + expect(listTemplates(db)).toEqual([]); + }); +}); diff --git a/packages/gateway/src/db/settings.ts b/packages/gateway/src/db/settings.ts new file mode 100644 index 00000000..62a088df --- /dev/null +++ b/packages/gateway/src/db/settings.ts @@ -0,0 +1,208 @@ +import type { S3Settings } from '@dormice/server/s3-store'; +import { + ARCHIVE_DEFAULT_SECONDS, + DEFAULT_LIFECYCLE_POLICY, + type RuntimeSettings, + type UpdateSettingsRequest, +} from '@dormice/shared'; +import { eq, sql } from 'drizzle-orm'; +import { type Config, s3Seed } from '../config'; +import type { Db, Writer } from './db'; +import { type SettingsRow, settings } from './schema'; + +/** The console_account fixed-id pattern: "at most one row" as a schema fact. */ +const SETTINGS_ROW_ID = 1; + +/** + * The fleet-wide settings as the wire shows them. The daemon's settings + * view once carried the managed-swap target too; on the gateway that knob + * is a node's (nodes.swapGb), so it is absent here. + */ +export type FleetSettings = Omit; + +/** + * Seeds the settings row from the env at the gateway's first start — + * insert-or-nothing, so every later start finds the row and leaves it + * alone: the table is the one truth from then on, and a later env edit of + * a seed is deliberately ignored (the daemon's discipline since 2026-07-19, + * shared/settings.ts has the line). The archive default is adjudicated + * here: an S3 seed present means new sandboxes archive after a week, + * absent means never. Version 1 is the seed; every change counts up from + * there, and the nodes compare against it at each check-in. + */ +export function ensureSettings(db: Db, config: Config): void { + const s3 = s3Seed(config); + db.insert(settings) + .values({ + id: SETTINGS_ROW_ID, + version: 1, + sandboxCpus: config.DORMICE_SANDBOX_CPUS, + sandboxMemoryGb: config.DORMICE_SANDBOX_MEMORY_GB, + sandboxDiskGb: config.DORMICE_SANDBOX_DISK_GB, + defaultFreezeAfterSeconds: DEFAULT_LIFECYCLE_POLICY.freezeAfterSeconds, + defaultStopAfterSeconds: DEFAULT_LIFECYCLE_POLICY.stopAfterSeconds, + defaultArchiveAfterSeconds: s3 ? ARCHIVE_DEFAULT_SECONDS : null, + ...s3Columns(s3), + sandboxDomain: config.DORMICE_SANDBOX_DOMAIN ?? null, + sandboxDomainAliases: '[]', + pidsLimit: config.DORMICE_SANDBOX_PIDS_LIMIT, + updatedAt: null, + }) + .onConflictDoNothing() + .run(); +} + +/** The six S3 columns as one unit: a store, or all NULL = off. */ +function s3Columns(s3: S3Settings | null) { + return { + s3Endpoint: s3?.endpoint ?? null, + s3Bucket: s3?.bucket ?? null, + s3AccessKeyId: s3?.accessKeyId ?? null, + s3SecretAccessKey: s3?.secretAccessKey ?? null, + s3Region: s3?.region ?? null, + s3ForcePathStyle: s3?.forcePathStyle ?? null, + }; +} + +function readRow(db: Db): SettingsRow { + const row = db + .select() + .from(settings) + .where(eq(settings.id, SETTINGS_ROW_ID)) + .get(); + if (!row) { + throw new Error('settings row missing — ensureSettings must run at boot'); + } + return row; +} + +function toView(row: SettingsRow): FleetSettings { + return { + sandboxDefaults: { + cpus: row.sandboxCpus, + memoryGb: row.sandboxMemoryGb, + diskGb: row.sandboxDiskGb, + }, + defaultPolicy: { + freezeAfterSeconds: row.defaultFreezeAfterSeconds, + stopAfterSeconds: row.defaultStopAfterSeconds, + archiveAfterSeconds: row.defaultArchiveAfterSeconds, + }, + s3: + row.s3Endpoint === null + ? null + : { + endpoint: row.s3Endpoint, + // biome-ignore-start lint/style/noNonNullAssertion: the six columns write as one unit (s3Columns) + bucket: row.s3Bucket!, + region: row.s3Region!, + forcePathStyle: row.s3ForcePathStyle!, + // biome-ignore-end lint/style/noNonNullAssertion: the six columns write as one unit (s3Columns) + }, + sandboxDomain: row.sandboxDomain, + // The one writer JSON.stringifies an array; a corrupt value should + // throw right here, not read as "no aliases". + sandboxDomainAliases: JSON.parse(row.sandboxDomainAliases) as string[], + pidsLimit: row.pidsLimit, + updatedAt: row.updatedAt, + }; +} + +/** The knobs in force, read fresh at each use — a point read costs microseconds and makes a console edit apply to the very next request. */ +export function readSettings(db: Db): FleetSettings { + return toView(readRow(db)); +} + +/** The configuration version the nodes compare against: bumped by every write here, in db/templates.ts and by updateNodeSettings. */ +export function readConfigVersion(db: Db): number { + return readRow(db).version; +} + +/** + * The S3 store in force, keys included — for the probe that guards a + * settings write and for the bundle a node pulls, never for the + * observation wire (readSettings withholds both keys). + */ +export function readS3Settings(db: Db): S3Settings | null { + const row = readRow(db); + if (row.s3Endpoint === null) return null; + return { + endpoint: row.s3Endpoint, + // biome-ignore-start lint/style/noNonNullAssertion: the six columns write as one unit (s3Columns) + bucket: row.s3Bucket!, + accessKeyId: row.s3AccessKeyId!, + secretAccessKey: row.s3SecretAccessKey!, + region: row.s3Region!, + forcePathStyle: row.s3ForcePathStyle!, + // biome-ignore-end lint/style/noNonNullAssertion: the six columns write as one unit (s3Columns) + }; +} + +/** + * Counts a configuration change the nodes must hear about. Callers that + * change something outside this row (a template, a node's swap target) + * run their own write and this bump inside one transaction, so a node can + * never see the new version with the old content or the reverse. + */ +export function bumpConfigVersion(db: Writer): number { + const row = db + .update(settings) + .set({ version: sql`${settings.version} + 1` }) + .where(eq(settings.id, SETTINGS_ROW_ID)) + .returning({ version: settings.version }) + .get(); + if (!row) { + throw new Error('settings row missing — ensureSettings must run at boot'); + } + return row.version; +} + +/** + * Applies an updateSettings patch: each provided group replaces that group + * whole, absent groups keep their stored values (shared/settings.ts is the + * arbiter of that contract), and the version counts up with the write. + * Validation — the archive-without-store refusal, the alias rules, the + * moving-store guard, the S3 probe — happened at the route; this is the + * pure write. + */ +export function writeSettings( + db: Db, + patch: Omit, + now: Date, +): FleetSettings { + const row = db + .update(settings) + .set({ + ...(patch.sandboxDefaults !== undefined + ? { + sandboxCpus: patch.sandboxDefaults.cpus, + sandboxMemoryGb: patch.sandboxDefaults.memoryGb, + sandboxDiskGb: patch.sandboxDefaults.diskGb, + } + : {}), + ...(patch.defaultPolicy !== undefined + ? { + defaultFreezeAfterSeconds: patch.defaultPolicy.freezeAfterSeconds, + defaultStopAfterSeconds: patch.defaultPolicy.stopAfterSeconds, + defaultArchiveAfterSeconds: patch.defaultPolicy.archiveAfterSeconds, + } + : {}), + ...(patch.s3 !== undefined ? s3Columns(patch.s3) : {}), + ...(patch.sandboxDomain !== undefined + ? { sandboxDomain: patch.sandboxDomain } + : {}), + ...(patch.sandboxDomainAliases !== undefined + ? { sandboxDomainAliases: JSON.stringify(patch.sandboxDomainAliases) } + : {}), + ...(patch.pidsLimit !== undefined ? { pidsLimit: patch.pidsLimit } : {}), + version: sql`${settings.version} + 1`, + updatedAt: now.toISOString(), + }) + .where(eq(settings.id, SETTINGS_ROW_ID)) + .returning() + .get(); + if (!row) { + throw new Error('settings row missing — ensureSettings must run at boot'); + } + return toView(row); +} diff --git a/packages/gateway/src/db/templates.ts b/packages/gateway/src/db/templates.ts new file mode 100644 index 00000000..ee75cded --- /dev/null +++ b/packages/gateway/src/db/templates.ts @@ -0,0 +1,64 @@ +import { eq } from 'drizzle-orm'; +import type { Db } from './db'; +import { type TemplateRow, templates } from './schema'; +import { bumpConfigVersion } from './settings'; + +/** + * Upsert: registering an existing name re-points it at the new image. That + * is the template upgrade front door — build a new image, re-register the + * name, then rebuildSandbox the stock that should move onto it; every + * node hears of the change at its next check-in (the version counts up in + * the same transaction as the write). + * + * updatedAt is the upgrade timestamp: stamped only when the image actually + * changes. A re-register of the same image writes nothing at all — the + * timestamp must not claim an upgrade that did not happen, and the nodes + * are not told about a change that is none. + */ +export function registerTemplate( + db: Db, + input: { name: string; image: string }, +): TemplateRow { + const now = new Date().toISOString(); + const existing = findTemplate(db, input.name); + if (existing?.image === input.image) { + return existing; + } + return db.transaction((tx) => { + if (!existing) { + const row: TemplateRow = { + name: input.name, + image: input.image, + createdAt: now, + updatedAt: now, + }; + tx.insert(templates).values(row).run(); + bumpConfigVersion(tx); + return row; + } + tx.update(templates) + .set({ image: input.image, updatedAt: now }) + .where(eq(templates.name, input.name)) + .run(); + bumpConfigVersion(tx); + return { ...existing, image: input.image, updatedAt: now }; + }); +} + +export function listTemplates(db: Db): TemplateRow[] { + return db.select().from(templates).orderBy(templates.name).all(); +} + +export function findTemplate(db: Db, name: string): TemplateRow | undefined { + return db.select().from(templates).where(eq(templates.name, name)).get(); +} + +/** Returns true when a row existed and was removed; the version counts up only then. */ +export function removeTemplate(db: Db, name: string): boolean { + if (findTemplate(db, name) === undefined) return false; + db.transaction((tx) => { + tx.delete(templates).where(eq(templates.name, name)).run(); + bumpConfigVersion(tx); + }); + return true; +} diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts index be302d6c..15cf19ab 100644 --- a/packages/gateway/src/main.ts +++ b/packages/gateway/src/main.ts @@ -8,6 +8,7 @@ import { buildGatewayApp } from './app'; import { NameCache } from './cache'; import { loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; +import { ensureSettings } from './db/settings'; import { Finder } from './find'; import { Fleet } from './fleet'; import { httpAskNode } from './lookup'; @@ -55,6 +56,8 @@ if (config.DORMICE_GATEWAY_DB_PATH !== ':memory:') { // Migrate on every boot; a fresh install needs no separate setup step. const db = openDb(config.DORMICE_GATEWAY_DB_PATH); migrateDb(db, fileURLToPath(new URL('../drizzle', import.meta.url))); +// The fleet's settings row, seeded from the env exactly once (db/settings.ts). +ensureSettings(db, config); // The fleet from the nodes table (a node that is down is still a node); // the cache and the readings fill in as nodes report and callers ask. diff --git a/packages/server/package.json b/packages/server/package.json index f05001ba..f08a0414 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -29,6 +29,10 @@ "./shutdown": { "types": "./dist/shutdown.d.ts", "default": "./dist/shutdown.js" + }, + "./s3-store": { + "types": "./dist/archive/s3-store.d.ts", + "default": "./dist/archive/s3-store.js" } }, "files": [ diff --git a/packages/server/src/policy.ts b/packages/server/src/policy.ts index 36666fb5..2ec4a1b0 100644 --- a/packages/server/src/policy.ts +++ b/packages/server/src/policy.ts @@ -4,14 +4,10 @@ import { lifecyclePolicySchema, } from '@dormice/shared'; -/** - * The default distance from stopped to archived, applied only when the - * daemon actually has an archiver (S3 configured) — the shared default - * stays null because a promise nobody can honor is a standing lie. Since - * runtime settings landed this is only the first-boot SEED of the ledger's - * defaultPolicy.archiveAfterSeconds; the ledger value is what acquires read. - */ -export const ARCHIVE_DEFAULT_SECONDS = 7 * 24 * 60 * 60; +// The first-boot seed of defaultPolicy.archiveAfterSeconds lives with the +// shared policy schema now that the gateway seeds it too; re-exported so +// the daemon's callers keep one import path. +export { ARCHIVE_DEFAULT_SECONDS } from '@dormice/shared'; /** An archive-asking policy on a daemon that has no archive store. */ export class ArchiveDisabledError extends Error { diff --git a/packages/server/tsup.config.ts b/packages/server/tsup.config.ts index 49d14560..2dadd3e7 100644 --- a/packages/server/tsup.config.ts +++ b/packages/server/tsup.config.ts @@ -36,6 +36,7 @@ export default defineConfig({ 'src/keyed-queue.ts', 'src/db/lock.ts', 'src/shutdown.ts', + 'src/archive/s3-store.ts', ], format: ['esm'], dts: true, diff --git a/packages/shared/src/policy.ts b/packages/shared/src/policy.ts index 67be1a29..8371bb28 100644 --- a/packages/shared/src/policy.ts +++ b/packages/shared/src/policy.ts @@ -73,3 +73,13 @@ export const DEFAULT_LIFECYCLE_POLICY: LifecyclePolicy = { stopAfterSeconds: 3 * 24 * 60 * 60, archiveAfterSeconds: null, }; + +/** + * The default distance from stopped to archived, applied only where an + * archive store is actually configured — the shared default above stays + * null because a promise nobody can honor is a standing lie. The first-boot + * SEED of the fleet's defaultPolicy.archiveAfterSeconds (the gateway's + * settings table, and before it the daemon's); the settings value is what + * acquires read. + */ +export const ARCHIVE_DEFAULT_SECONDS = 7 * 24 * 60 * 60; From ee195dbbf24f2d6c54e2399ca2a65ecffefeb145 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 15:49:06 +0800 Subject: [PATCH 31/89] The gateway gets three gates, mints the fleet's API keys and hosts the console MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design records #9, #20, #24. The gateway's door is now three: the nodes' gate (the fleet token and nothing else — a check-in is a machine reporting), the sandbox gate (the fleet token, any live key the gateway minted, or the console session — everything that addresses a sandbox, on the native and the E2B face alike) and the admin gate (the fleet token or the console session only — everything that configures the fleet; a live key gets the honest 403 naming the rule). listNodes and removeNode move behind the admin gate. The API key verbs, the console account, the session cookie, the login throttle and the static console arrive from the daemon as files, with their tests: one console for the fleet, one account, set up with the fleet token. The console's terminal needs a per-sandbox envd token that only the sandbox's node can mint (the secret is the node's); the verb is now /envdToken on the node's API scope, and the gateway's /envdToken finds the node by id and asks it under the fleet token. The console and the e2e suite call the new path. --- e2e/src/console.test.ts | 4 +- .../src/features/sandboxes/envd-client.ts | 2 +- packages/console/src/lib/api.ts | 2 +- packages/console/vite.config.ts | 2 +- packages/gateway/package.json | 2 + packages/gateway/src/app.test.ts | 28 +- packages/gateway/src/app.ts | 91 +++- packages/gateway/src/auth.ts | 238 +++++++++++ packages/gateway/src/http-error.ts | 11 + packages/gateway/src/login-throttle.ts | 76 ++++ packages/gateway/src/main.ts | 13 + packages/gateway/src/routes/api-keys.test.ts | 327 +++++++++++++++ packages/gateway/src/routes/api-keys.ts | 156 +++++++ packages/gateway/src/routes/console.test.ts | 397 ++++++++++++++++++ packages/gateway/src/routes/console.ts | 231 ++++++++++ packages/gateway/src/routes/e2b.ts | 13 +- packages/gateway/src/routes/envd-token.ts | 68 +++ packages/gateway/src/routes/native.ts | 13 +- packages/gateway/src/routes/nodes.ts | 27 +- packages/gateway/src/testing.ts | 57 ++- packages/server/src/app.ts | 4 +- packages/server/src/routes/console.test.ts | 4 +- packages/server/src/routes/console.ts | 36 +- packages/server/src/routes/envd-token.ts | 41 ++ packages/shared/src/envd-token.ts | 25 ++ packages/shared/src/index.ts | 1 + pnpm-lock.yaml | 6 + 27 files changed, 1799 insertions(+), 76 deletions(-) create mode 100644 packages/gateway/src/auth.ts create mode 100644 packages/gateway/src/http-error.ts create mode 100644 packages/gateway/src/login-throttle.ts create mode 100644 packages/gateway/src/routes/api-keys.test.ts create mode 100644 packages/gateway/src/routes/api-keys.ts create mode 100644 packages/gateway/src/routes/console.test.ts create mode 100644 packages/gateway/src/routes/console.ts create mode 100644 packages/gateway/src/routes/envd-token.ts create mode 100644 packages/server/src/routes/envd-token.ts create mode 100644 packages/shared/src/envd-token.ts diff --git a/e2e/src/console.test.ts b/e2e/src/console.test.ts index d30c187d..bec9c855 100644 --- a/e2e/src/console.test.ts +++ b/e2e/src/console.test.ts @@ -160,7 +160,7 @@ describe('web console over a real daemon', () => { describe('browser-side signed download URLs (the Office preview foundation)', () => { // The console's preview pane recomputes the file signature in the browser - // (envd-client.ts signedDownloadUrl) from the token /console/envdToken + // (envd-client.ts signedDownloadUrl) from the token /envdToken // hands it. This pins the whole chain end-to-end — console minting, the // formula REWRITTEN here rather than imported (a black box pins the // formula itself, not a shared implementation's self-consistency), and @@ -199,7 +199,7 @@ describe('browser-side signed download URLs (the Office preview foundation)', () }); // Mint the token exactly the way the browser does: cookie + console header. - const minted = await fetch(`${endpoint()}/console/envdToken`, { + const minted = await fetch(`${endpoint()}/envdToken`, { method: 'POST', headers: { cookie: session, diff --git a/packages/console/src/features/sandboxes/envd-client.ts b/packages/console/src/features/sandboxes/envd-client.ts index a568fe93..e8dd2f85 100644 --- a/packages/console/src/features/sandboxes/envd-client.ts +++ b/packages/console/src/features/sandboxes/envd-client.ts @@ -3,7 +3,7 @@ * unary Filesystem/Process RPCs and the plain-HTTP file faces — the exact * wire the official e2b SDK speaks (see envd-pty.ts for why: a console-only * endpoint would be a second truth). Everything authenticates with the - * per-sandbox envd access token minted via /console/envdToken. + * per-sandbox envd access token minted via /envdToken. * * Waking: the unary filesystem verbs wake a frozen sandbox (using files IS * using the sandbox); Process/List is read-only and never wakes. Callers diff --git a/packages/console/src/lib/api.ts b/packages/console/src/lib/api.ts index 8ef558e0..de955395 100644 --- a/packages/console/src/lib/api.ts +++ b/packages/console/src/lib/api.ts @@ -283,4 +283,4 @@ export const updatePolicy = (name: string, policy: LifecyclePolicyOverride) => // The terminal's key: trades the session cookie for one sandbox's envd // access token, so the browser can speak to the envd surface directly. export const mintEnvdToken = (sandboxId: string) => - rpc<{ envdAccessToken: string }>('/console/envdToken', { sandboxId }); + rpc<{ envdAccessToken: string }>('/envdToken', { sandboxId }); diff --git a/packages/console/vite.config.ts b/packages/console/vite.config.ts index cc380302..4aa7e32b 100644 --- a/packages/console/vite.config.ts +++ b/packages/console/vite.config.ts @@ -31,7 +31,7 @@ export default defineConfig({ // knob. proxy: { '/console/auth': 'http://127.0.0.1:3676', - '/console/envdToken': 'http://127.0.0.1:3676', + '/envdToken': 'http://127.0.0.1:3676', '/listSandboxes': 'http://127.0.0.1:3676', '/destroySandbox': 'http://127.0.0.1:3676', '/acquireSandbox': 'http://127.0.0.1:3676', diff --git a/packages/gateway/package.json b/packages/gateway/package.json index d4928d73..73511745 100644 --- a/packages/gateway/package.json +++ b/packages/gateway/package.json @@ -17,6 +17,8 @@ "dependencies": { "@dormice/server": "workspace:*", "@dormice/shared": "workspace:*", + "@fastify/cookie": "^11.0.2", + "@fastify/static": "^9.1.3", "better-sqlite3": "^12.11.1", "drizzle-orm": "^0.45.2", "fastify": "^5.10.0", diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index fd6b2715..230a0cca 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -9,6 +9,7 @@ import { buildGatewayApp } from './app'; import { NameCache } from './cache'; import { loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; +import { ensureSettings } from './db/settings'; import { Finder } from './find'; import { Fleet } from './fleet'; import { httpAskNode } from './lookup'; @@ -122,6 +123,12 @@ class FakeNode { const name = body.name as string | undefined; const found = name === undefined ? undefined : this.sandboxes.get(name); switch (path) { + case '/envdToken': { + // The daemon's HMAC stands in as a string only this node would mint. + return json(200, { + envdAccessToken: `envd-${this.id}-${String(body.sandboxId)}`, + }); + } case '/lookupSandbox': { const sandbox = 'id' in body ? this.byId(body.id as string) : found; return json( @@ -232,6 +239,7 @@ async function gateway( DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '100', ...env, }); + ensureSettings(db, config); const fleet = new Fleet(db, opts.startedAt); const cache = new NameCache(); const finder = new Finder(fleet, cache, httpAskNode(TOKEN), { @@ -240,6 +248,7 @@ async function gateway( const logs = opts.logs; const app = buildGatewayApp({ config, + db, fleet, finder, locks: new KeyedQueue(), @@ -849,6 +858,23 @@ describe('acquire: placing and finding', () => { }); describe('using, destroying, and the cache', () => { + it('envdToken is minted by the node that runs the sandbox, found by id, under the fleet token; an unknown id is a 404', async () => { + const h = await gateway(['b', 'c']); + const created = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'x' })); + const minted = await rpc(h, '/envdToken', { sandboxId: created.id }); + expect(minted.status).toBe(200); + expect(minted.body).toEqual({ + envdAccessToken: `envd-${created.nodeId}-${created.id}`, + }); + const home = h.nodes.find((n) => n.id === created.nodeId); + expect(home?.hits.find((hit) => hit.path === '/envdToken')?.auth).toBe( + `Bearer ${TOKEN}`, + ); + const nobody = await rpc(h, '/envdToken', { sandboxId: randomUUID() }); + expect(nobody.status).toBe(404); + expect(message(nobody)).toMatch(/is on no node/); + }); + it("files round-trip; a node's 404 for a missing file passes through and the sandbox stays cached", async () => { const h = await gateway(['a']); const a = h.nodes[0] as FakeNode; @@ -911,7 +937,7 @@ describe('using, destroying, and the cache', () => { const listed = await rpc(h, '/listSandboxes'); expect(listed.status).toBe(501); expect(message(listed)).toContain('call the node directly'); - expect((await rpc(h, '/createApiKey', { name: 'k' })).status).toBe(501); + expect((await rpc(h, '/registerTemplate', { name: 'k' })).status).toBe(501); expect((await rpc(h, '/acquireSandbx', { name: 'x' })).status).toBe(404); expect((await rpc(h, '/execCommand', { command: 'x' })).status).toBe(400); expect(h.nodes[0]?.hits).toEqual([]); diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts index 906e8e77..51c42fc4 100644 --- a/packages/gateway/src/app.ts +++ b/packages/gateway/src/app.ts @@ -1,6 +1,6 @@ import http from 'node:http'; -import { tokensEqual } from '@dormice/server/auth'; import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import fastifyCookie from '@fastify/cookie'; import fastify, { type FastifyError, type FastifyServerFactory } from 'fastify'; import { serializerCompiler, @@ -9,20 +9,29 @@ import { } from 'fastify-type-provider-zod'; import { type Logger, pino } from 'pino'; import { z } from 'zod'; +import { requireAdminAuth, requireApiAuth, tokensEqual } from './auth'; import { classify, isOriginForm } from './classify'; import type { Config } from './config'; +import { getConsoleAccount } from './db/account'; +import { isLiveApiKey, verifyApiKeyToken } from './db/api-keys'; +import type { Db } from './db/db'; import { renderError } from './errors'; import type { Finder } from './find'; import type { Fleet } from './fleet'; import type { PlacementKnobs } from './placement'; import { createRawFaces } from './raw'; +import { apiKeyRoutes } from './routes/api-keys'; +import { consoleRoutes } from './routes/console'; import { e2bControlRoutes } from './routes/e2b'; +import { envdTokenRoutes } from './routes/envd-token'; import { nativeRoutes } from './routes/native'; -import { nodeRoutes } from './routes/nodes'; +import { checkInRoutes, nodeRoutes } from './routes/nodes'; import { type BuildInfo, readBuildInfo } from './version'; export interface GatewayAppDeps { config: Config; + /** The gateway's own tables (db/schema.ts); ensureSettings has run on it. */ + db: Db; fleet: Fleet; finder: Finder; /** One queue for the whole gateway: the create and destroy verbs of both faces share per-name slots. */ @@ -31,6 +40,12 @@ export interface GatewayAppDeps { logger?: Logger | boolean; /** The build identity /healthz reports; null when built outside a checkout. */ build?: BuildInfo | null; + /** + * Where the built web console lives; main.ts resolves the monorepo + * layout, tests inject a fixture. Absent means /console answers an + * honest 404. + */ + consoleDistDir?: string; } /** Placement's knobs, read once from the config. */ @@ -47,14 +62,27 @@ export function placementKnobs(config: Config): PlacementKnobs { * validator and serializer, one error dialect, /healthz open), without * the daemon's body. Building is separate from listening so tests inject * requests without a port; the raw faces need real sockets. + * + * Three gates, three answers to "who may call" (design records #9, #20, + * #22): the nodes' own gate (the fleet token and nothing else — a node + * checking in is a machine, not a person or a client); the sandbox gate + * (the fleet token, any live key the gateway minted, or the console's + * session — everything that addresses a sandbox); and the admin gate (the + * fleet token or the console session only — everything that configures + * the fleet: keys, settings, templates, domains, nodes). A live key at the + * admin gate gets an honest 403 naming the rule. Toward the nodes the + * gateway always speaks the fleet token (forward.ts): the node trusts the + * door whole. */ export function buildGatewayApp({ config, + db, fleet, finder, locks, logger = true, build = readBuildInfo(), + consoleDistDir, }: GatewayAppDeps) { const loggerInstance = typeof logger === 'boolean' ? pino({ enabled: logger }) : logger; @@ -134,24 +162,68 @@ export function buildGatewayApp({ async () => ({ status: 'ok' as const, build }), ); - // One credential opens the door: the fleet's token, presented as a - // Bearer by callers and by nodes checking in alike. Keys minted by the - // gateway — many, expiring, revocable — arrive with its key table. + // Cookie parsing app-wide: the two gates read the console's session + // cookie, the /console surface mints and clears it. + app.register(fastifyCookie); + + // The one adjudication of "does this bare credential open the sandbox + // door": the fleet token (constant-time compare — the bootstrap + // credential, always valid) or any live key the gateway minted (sha256 + // indexed lookup, judged per request so a mint or revoke takes effect on + // the very next call). Both faces — the native Bearer header and the + // E2B X-API-KEY hook — feed this same closure: one truth, two dialects. + const isCredential = (bare: string): boolean => + tokensEqual(bare, token) || verifyApiKeyToken(db, bare) !== null; + const isFleetToken = (bare: string): boolean => tokensEqual(bare, token); + // Read per request: setup can replace the account (and void its + // sessions) while the gateway runs. + const sessionSecret = () => getConsoleAccount(db)?.sessionSecret ?? null; + const apiAuth = requireApiAuth(isCredential, sessionSecret); + const adminAuth = requireAdminAuth( + isFleetToken, + (bare) => isLiveApiKey(db, bare), + sessionSecret, + ); + const knobs = placementKnobs(config); - app.register(async (api) => { - api.addHook('onRequest', async (request, reply) => { + + // The nodes' gate: the fleet token alone. A key or a session is a + // caller's credential, and a check-in is not a call — it is a machine + // reporting; a node presenting anything else is misconfigured. + app.register(async (nodesFace) => { + nodesFace.addHook('onRequest', async (request, reply) => { const header = request.headers.authorization; const bare = header?.startsWith('Bearer ') ? header.slice(7) : null; - if (bare === null || !tokensEqual(bare, token)) { + if (bare === null || !isFleetToken(bare)) { await reply.code(401).send({ message: 'missing or invalid API token' }); } }); - await api.register(nodeRoutes, { fleet, cache: finder.cache }); + await nodesFace.register(checkInRoutes, { fleet }); + }); + + // The sandbox gate: everything that addresses a sandbox. + app.register(async (api) => { + api.addHook('onRequest', apiAuth); + await api.register(envdTokenRoutes, { finder, token }); // Its own sub-scope: the byte-preserving body parser it installs must // not reach the gateway's own verbs, which keep Fastify's JSON parsing. await api.register(nativeRoutes, { fleet, finder, locks, knobs, token }); }); + // The admin gate: everything that configures the fleet. + app.register(async (admin) => { + admin.addHook('onRequest', adminAuth); + await admin.register(apiKeyRoutes, { db }); + await admin.register(nodeRoutes, { fleet, cache: finder.cache }); + }); + + // The web console: account + session endpoints (open — setup and login + // carry the credentials themselves) and the static SPA. Its API calls go + // through the gates above. + app.register(async (scope) => { + await scope.register(consoleRoutes, { config, db, consoleDistDir }); + }); + // The E2B control plane, its own auth and dialect, like the daemon's. app.register(e2bControlRoutes, { fleet, @@ -159,6 +231,7 @@ export function buildGatewayApp({ locks, knobs, token, + isCredential, prefix: '/e2b/api', }); diff --git a/packages/gateway/src/auth.ts b/packages/gateway/src/auth.ts new file mode 100644 index 00000000..0c8ecbff --- /dev/null +++ b/packages/gateway/src/auth.ts @@ -0,0 +1,238 @@ +import { + createHash, + createHmac, + randomBytes, + type ScryptOptions, + scrypt, + timingSafeEqual, +} from 'node:crypto'; +import type { FastifyRequest, onRequestAsyncHookHandler } from 'fastify'; + +// Hand-rolled instead of util.promisify: promisify picks the overload +// without the options argument, and the cost parameters live there. +function scryptAsync( + password: string, + salt: Buffer, + keylen: number, + options: ScryptOptions, +): Promise { + return new Promise((resolve, reject) => { + scrypt(password, salt, keylen, options, (err, key) => + err ? reject(err) : resolve(key), + ); + }); +} + +const sha256 = (value: string) => createHash('sha256').update(value).digest(); + +/** Constant-time string comparison; both sides hashed so lengths never leak. */ +export function tokensEqual(presented: string, expected: string): boolean { + return timingSafeEqual(sha256(presented), sha256(expected)); +} + +/** + * Password hashing for the console account: scrypt (in node:crypto, zero + * dependencies — the whole reason it wins over bcrypt/argon2 here). The + * parameters ride inside the stored string, so they can change later + * without invalidating old hashes. + * + * N=2^14, r=8, p=1 (~16 MiB, tens of ms): the standard interactive-login + * cost. The online-guessing defense is the login throttle; this cost is + * for the offline case, a stolen ledger file. + */ +const SCRYPT_N = 16384; +const SCRYPT_R = 8; +const SCRYPT_P = 1; +const SCRYPT_KEYLEN = 32; + +export async function hashPassword(password: string): Promise { + const salt = randomBytes(16); + const hash = await scryptAsync(password, salt, SCRYPT_KEYLEN, { + N: SCRYPT_N, + r: SCRYPT_R, + p: SCRYPT_P, + }); + return [ + 'scrypt', + SCRYPT_N, + SCRYPT_R, + SCRYPT_P, + salt.toString('base64'), + hash.toString('base64'), + ].join('$'); +} + +export async function verifyPassword( + password: string, + stored: string, +): Promise { + const [scheme, n, r, p, saltB64, hashB64] = stored.split('$'); + if (scheme !== 'scrypt' || !n || !r || !p || !saltB64 || !hashB64) { + return false; + } + const expected = Buffer.from(hashB64, 'base64'); + const actual = await scryptAsync( + password, + Buffer.from(saltB64, 'base64'), + expected.length, + { N: Number(n), r: Number(r), p: Number(p) }, + ); + return actual.length === expected.length && timingSafeEqual(actual, expected); +} + +/** The session-cookie HMAC key: random, stored on the account row. */ +export function mintSessionSecret(): string { + return randomBytes(32).toString('hex'); +} + +/** + * The web console's session cookie. Stateless on purpose: the daemon is + * crash-only, and an in-memory session table would log every operator out + * on each restart. The value carries its own expiry and an HMAC over it — + * the same pattern as the envd access token — so a restart changes nothing. + * + * The HMAC key is the account's sessionSecret, not the API token: the two + * credentials rotate independently. Re-running setup (password change or + * reset) regenerates the secret and voids every session — the semantics a + * password change should have — while rotating the API token leaves the + * console signed in. + */ +export const SESSION_COOKIE = 'dormice_session'; +export const SESSION_TTL_SECONDS = 7 * 24 * 60 * 60; + +/** + * Cookie-authenticated requests must also carry this header. A cross-origin + * page cannot send a custom header without a CORS preflight, and the daemon + * answers no preflights — this closes the hole SameSite leaves open + * (SameSite ignores the port, so another local web app counts as same-site). + */ +export const CONSOLE_HEADER = 'x-dormice-console'; + +function sessionHmac(secret: string, expiresAtSeconds: number): string { + return createHmac('sha256', secret) + .update(`console-session:${expiresAtSeconds}`) + .digest('hex'); +} + +export function mintSession(secret: string, nowMs = Date.now()): string { + const expiresAt = Math.floor(nowMs / 1000) + SESSION_TTL_SECONDS; + return `${expiresAt}.${sessionHmac(secret, expiresAt)}`; +} + +export function verifySession( + secret: string, + value: string, + nowMs = Date.now(), +): boolean { + const dot = value.indexOf('.'); + if (dot < 0) return false; + const expiresAt = Number(value.slice(0, dot)); + // The expiry is plaintext in the cookie — nothing secret to compare in + // constant time. The HMAC comparison below is the constant-time one. + if (!Number.isInteger(expiresAt) || expiresAt * 1000 <= nowMs) return false; + return tokensEqual(value.slice(dot + 1), sessionHmac(secret, expiresAt)); +} + +/** + * The console-session leg shared verbatim by both auth hooks: cookie + * present, an account exists (secret non-null), the CSRF header rode along + * (see CONSOLE_HEADER), and the HMAC verifies. The session secret is + * fetched per request, not captured at startup: setup can replace the + * account (and its secret) while the daemon runs, and the arbiter must + * judge against the current one. Null means no account exists yet — no + * cookie can be valid. + */ +function sessionCookieValid( + request: FastifyRequest, + getSessionSecret: () => string | null, +): boolean { + // The jar exists where the app registered @fastify/cookie (app.ts does). + // Typed structurally so this module needs no plugin type augmentation. + const jar = (request as { cookies?: Record }) + .cookies; + const cookie = jar?.[SESSION_COOKIE]; + const secret = getSessionSecret(); + return Boolean( + cookie && + secret !== null && + request.headers[CONSOLE_HEADER] !== undefined && + verifySession(secret, cookie), + ); +} + +/** + * The single arbiter of who may call the API (/healthz stays open — + * liveness probes have no secrets). Two credentials open the same door: + * a Bearer credential (SDK, CLI, curl — the env token or any live API + * key, adjudicated by isCredential) and the web console's session + * cookie (which additionally requires the console header, see above). A + * second route surface with its own auth would be a second truth. + * + * isCredential judges bare tokens, so both faces (this Bearer header and + * the E2B X-API-KEY hook) feed it the same canonical form — one closure, + * one truth, two dialects. The 'Bearer ' prefix is public framing, not a + * secret, so stripping it needs no constant time; the secret comparisons + * live inside isCredential. + */ +export function requireApiAuth( + isCredential: (bareToken: string) => boolean, + getSessionSecret: () => string | null, +): onRequestAsyncHookHandler { + return async (request, reply) => { + const header = request.headers.authorization; + const bare = header?.startsWith('Bearer ') ? header.slice(7) : null; + if (bare !== null && isCredential(bare)) { + return; + } + if (sessionCookieValid(request, getSessionSecret)) { + return; + } + await reply.code(401).send({ message: 'missing or invalid API token' }); + }; +} + +/** + * The admin gate for everything that configures the fleet — the apiKey + * management verbs, the settings, templates, ingress and node verbs + * (design record #9): only the env token (Bearer) or a console session + * may pass. A key that is otherwise valid gets an honest 403 instead of a + * silent 401 — key-manages-key would let one leaked credential mint itself + * an unrevoked successor and revoke every legitimate peer, and a leaked + * automation key must not be able to raise the very limits that contain + * it or re-point the templates every node boots from; the refusal names + * the rule. The console-setup door (routes/console.ts) rests on the same + * doctrine: a machine credential must not escalate into managing the + * fleet. + * + * Leg order matters twice. The isLiveApiKey lookup runs only after both + * accepting legs failed, so a console session with a stray key header + * still passes, and the ledger is consulted only for requests already + * being refused. And isLiveApiKey must be a pure read that never stamps + * lastUsedAt — the request is being refused, not honored. A disabled or + * expired key is no longer a valid credential and falls through to the + * same 401 as garbage: a 403 for it would leak that the row exists. + */ +export function requireAdminAuth( + isEnvToken: (bareToken: string) => boolean, + isLiveApiKey: (bareToken: string) => boolean, + getSessionSecret: () => string | null, +): onRequestAsyncHookHandler { + return async (request, reply) => { + const header = request.headers.authorization; + const bare = header?.startsWith('Bearer ') ? header.slice(7) : null; + if (bare !== null && isEnvToken(bare)) { + return; + } + if (sessionCookieValid(request, getSessionSecret)) { + return; + } + if (bare !== null && isLiveApiKey(bare)) { + await reply.code(403).send({ + message: + 'API keys cannot manage API keys, settings, templates, domains or nodes — use DORMICE_API_TOKEN or the console', + }); + return; + } + await reply.code(401).send({ message: 'missing or invalid API token' }); + }; +} diff --git a/packages/gateway/src/http-error.ts b/packages/gateway/src/http-error.ts new file mode 100644 index 00000000..0adfff47 --- /dev/null +++ b/packages/gateway/src/http-error.ts @@ -0,0 +1,11 @@ +/** + * An error that carries its HTTP status. Handlers throw it; the app's + * global error handler turns it into the protocol's `{ message }` body + * with this status code. + */ +export function httpError( + statusCode: number, + message: string, +): Error & { statusCode: number } { + return Object.assign(new Error(message), { statusCode }); +} diff --git a/packages/gateway/src/login-throttle.ts b/packages/gateway/src/login-throttle.ts new file mode 100644 index 00000000..ad060cef --- /dev/null +++ b/packages/gateway/src/login-throttle.ts @@ -0,0 +1,76 @@ +/** + * Failure backoff for the console's credential endpoints. Passwords are + * human-chosen (low entropy, unlike the 128-bit API token), so online + * guessing became a real surface the moment they arrived — this is the + * counterpart the password design must ship with. + * + * In-memory on purpose: crash-only means a restart honestly forgets the + * counters, and persisting attacker state in the ledger would be a second + * database to babysit for no real gain. Keyed by client IP; behind the + * usual single reverse proxy every client shares 127.0.0.1, collapsing this + * into a global throttle — acceptable and arguably right for a single + * account (it is the account being guessed at, not the caller). + */ + +/** Free attempts before delays start: typos are not attacks. */ +const FREE_FAILURES = 5; +/** Delay doubles per failure past the free ones, capped here. */ +const MAX_DELAY_SECONDS = 300; +/** Counters idle this long are forgotten (also the sweep horizon). */ +const FORGET_AFTER_MS = 60 * 60 * 1000; + +interface Entry { + failures: number; + /** Epoch ms before which further attempts are refused. */ + blockedUntil: number; + lastFailureAt: number; +} + +export class LoginThrottle { + private entries = new Map(); + + /** Seconds the caller must still wait, or 0 when an attempt is allowed. */ + retryAfterSeconds(key: string, nowMs = Date.now()): number { + const entry = this.entries.get(key); + if (!entry) return 0; + if (nowMs - entry.lastFailureAt > FORGET_AFTER_MS) { + this.entries.delete(key); + return 0; + } + return Math.max(0, Math.ceil((entry.blockedUntil - nowMs) / 1000)); + } + + recordFailure(key: string, nowMs = Date.now()): void { + this.sweep(nowMs); + const entry = this.entries.get(key) ?? { + failures: 0, + blockedUntil: 0, + lastFailureAt: 0, + }; + entry.failures += 1; + entry.lastFailureAt = nowMs; + const past = entry.failures - FREE_FAILURES; + if (past > 0) { + const delay = Math.min(2 ** (past - 1), MAX_DELAY_SECONDS); + entry.blockedUntil = nowMs + delay * 1000; + } + this.entries.set(key, entry); + } + + clear(key: string): void { + this.entries.delete(key); + } + + /** + * Drops idle counters on the write path — no timer to manage, and the map + * stays bounded by "distinct keys failing within the last hour", which a + * loopback-bound daemon can always afford. + */ + private sweep(nowMs: number): void { + for (const [key, entry] of this.entries) { + if (nowMs - entry.lastFailureAt > FORGET_AFTER_MS) { + this.entries.delete(key); + } + } + } +} diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts index 15cf19ab..e590f4f9 100644 --- a/packages/gateway/src/main.ts +++ b/packages/gateway/src/main.ts @@ -1,3 +1,4 @@ +import { existsSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { KeyedQueue } from '@dormice/server/keyed-queue'; import { acquireSingleWriterLock } from '@dormice/server/lock'; @@ -80,13 +81,25 @@ log.info( : 'dormice-gateway build: no version identity (built outside a git checkout)', ); +// The built web console, by the monorepo layout: dist/main.js sits two +// levels under packages/gateway, the console's dist beside it. Absent +// (a deploy without the console built), /console answers an honest 404. +const consoleDistDir = fileURLToPath( + new URL('../../console/dist', import.meta.url), +); +if (!existsSync(consoleDistDir)) { + log.warn(`web console not found at ${consoleDistDir} — /console disabled`); +} + const app = buildGatewayApp({ config, + db, fleet, finder, locks, logger: log, build, + consoleDistDir: existsSync(consoleDistDir) ? consoleDistDir : undefined, }); // Same red line as the daemon: loopback only, host not configurable — the diff --git a/packages/gateway/src/routes/api-keys.test.ts b/packages/gateway/src/routes/api-keys.test.ts new file mode 100644 index 00000000..3884d2b5 --- /dev/null +++ b/packages/gateway/src/routes/api-keys.test.ts @@ -0,0 +1,327 @@ +import { describe, expect, it } from 'vitest'; +import { CONSOLE_HEADER, SESSION_COOKIE } from '../auth'; +import { TEST_TOKEN, testGateway } from '../testing'; + +type TestApp = ReturnType['app']; + +const authed = { authorization: `Bearer ${TEST_TOKEN}` }; + +function rpc(app: TestApp, url: string, payload: Record = {}) { + return app.inject({ method: 'POST', url, headers: authed, payload }); +} + +/** Mint through the wire and hand back everything a test needs. */ +async function mint(app: TestApp, name: string, expiresAt?: string) { + const res = await rpc(app, '/createApiKey', { + name, + ...(expiresAt ? { expiresAt } : {}), + }); + expect(res.statusCode).toBe(200); + const body = res.json(); + return { + id: body.apiKey.id as string, + token: body.token as string, + apiKey: body.apiKey, + }; +} + +/** + * Whether a credential opens the sandbox gate. The verb behind it that + * needs no node is one of the fleet-wide verbs the gateway does not route + * yet: its honest 501 is "you are through the door"; a 401 is not. (The + * next cut merges those verbs and this probe moves to a real answer.) + */ +const useKey = (app: TestApp, token: string, url = '/listSandboxes') => + app.inject({ + method: 'POST', + url, + headers: { authorization: `Bearer ${token}` }, + payload: {}, + }); +const OPENED = 501; + +describe('API keys on the gateway', () => { + it('mints a 64-hex token, shown once and never stored in the view', async () => { + const { app } = testGateway(); + const res = await rpc(app, '/createApiKey', { name: 'ci' }); + expect(res.statusCode).toBe(200); + const body = res.json(); + expect(body.token).toMatch(/^[0-9a-f]{64}$/); + expect(body.apiKey).toMatchObject({ + name: 'ci', + prefix: body.token.slice(0, 8), + lastUsedAt: null, + expiresAt: null, + disabledAt: null, + revokedAt: null, + }); + // The view carries no secret — not the token, not its hash. + expect(JSON.stringify(body.apiKey)).not.toContain(body.token); + expect(Object.keys(body.apiKey)).not.toContain('keyHash'); + }); + + it('a minted key opens the sandbox gate on both faces; revoking closes it on the next request', async () => { + const { app } = testGateway(); + const { id, token } = await mint(app, 'ci'); + + expect((await useKey(app, token)).statusCode).toBe(OPENED); + // The E2B face judges the same credential, under its own convention. + const e2b = await app.inject({ + method: 'GET', + url: '/e2b/api/v2/sandboxes', + headers: { 'x-api-key': `e2b_${token}` }, + }); + expect(e2b.statusCode).toBe(501); + + expect((await rpc(app, '/revokeApiKey', { id })).json()).toEqual({ + revoked: true, + }); + expect((await useKey(app, token)).statusCode).toBe(401); + expect( + ( + await app.inject({ + method: 'GET', + url: '/e2b/api/v2/sandboxes', + headers: { 'x-api-key': `e2b_${token}` }, + }) + ).statusCode, + ).toBe(401); + + // The fleet token is the bootstrap credential: revocation never touches it. + expect((await useKey(app, TEST_TOKEN)).statusCode).toBe(OPENED); + }); + + it('refuses a second active key under the same name with a 409, and frees the name after revoke', async () => { + const { app } = testGateway(); + const { id } = await mint(app, 'ci'); + const dup = await rpc(app, '/createApiKey', { name: 'ci' }); + expect(dup.statusCode).toBe(409); + expect(dup.json().message).toMatch(/'ci' already exists/); + + await rpc(app, '/revokeApiKey', { id }); + expect((await rpc(app, '/createApiKey', { name: 'ci' })).statusCode).toBe( + 200, + ); + }); + + it('revoke is idempotent: an unknown or already-revoked id answers { revoked: false }', async () => { + const { app } = testGateway(); + expect((await rpc(app, '/revokeApiKey', { id: 'ghost' })).json()).toEqual({ + revoked: false, + }); + const { id } = await mint(app, 'ci'); + await rpc(app, '/revokeApiKey', { id }); + expect((await rpc(app, '/revokeApiKey', { id })).json()).toEqual({ + revoked: false, + }); + }); + + it('lists every key ever minted, revoked rows included, newest first', async () => { + const { app } = testGateway(); + const { id } = await mint(app, 'old'); + await rpc(app, '/revokeApiKey', { id }); + await mint(app, 'new'); + + const keys = (await rpc(app, '/listApiKeys')).json().apiKeys; + expect(keys).toHaveLength(2); + expect(keys[0].name).toBe('new'); + expect(keys[0].revokedAt).toBeNull(); + expect(keys[1].name).toBe('old'); + expect(keys[1].revokedAt).not.toBeNull(); + }); + + it('stamps lastUsedAt on first use and throttles the write to 60s granularity', async () => { + const { app } = testGateway(); + const { token } = await mint(app, 'ci'); + + await useKey(app, token); + const first = (await rpc(app, '/listApiKeys')).json().apiKeys[0]; + expect(first.lastUsedAt).not.toBeNull(); + + // A second use inside the 60s window must not move the stamp. + await useKey(app, token); + const second = (await rpc(app, '/listApiKeys')).json().apiKeys[0]; + expect(second.lastUsedAt).toBe(first.lastUsedAt); + }); + + it('disable parks the key reversibly: 401 while disabled, open again after enable', async () => { + const { app } = testGateway(); + const { id, token } = await mint(app, 'ci'); + expect((await useKey(app, token)).statusCode).toBe(OPENED); + + const disabled = ( + await rpc(app, '/updateApiKey', { id, disabled: true }) + ).json().apiKey; + expect(disabled.disabledAt).not.toBeNull(); + expect((await useKey(app, token)).statusCode).toBe(401); + + // Disabling twice is idempotent: the original stamp stays. + const again = ( + await rpc(app, '/updateApiKey', { id, disabled: true }) + ).json().apiKey; + expect(again.disabledAt).toBe(disabled.disabledAt); + + const enabled = ( + await rpc(app, '/updateApiKey', { id, disabled: false }) + ).json().apiKey; + expect(enabled.disabledAt).toBeNull(); + expect((await useKey(app, token)).statusCode).toBe(OPENED); + }); + + it('expiry closes the door: a past expiresAt is 401, clearing it reopens', async () => { + const { app } = testGateway(); + const past = new Date(Date.now() - 1000).toISOString(); + const { id, token } = await mint(app, 'ttl', past); + expect((await useKey(app, token)).statusCode).toBe(401); + + const cleared = ( + await rpc(app, '/updateApiKey', { id, expiresAt: null }) + ).json().apiKey; + expect(cleared.expiresAt).toBeNull(); + expect((await useKey(app, token)).statusCode).toBe(OPENED); + + const future = new Date(Date.now() + 3600_000).toISOString(); + await rpc(app, '/updateApiKey', { id, expiresAt: future }); + expect((await useKey(app, token)).statusCode).toBe(OPENED); + }); + + it('normalizes expiresAt on write: wire precision variants land as toISOString()', async () => { + const { app } = testGateway(); + const { apiKey } = await mint(app, 'ttl', '2030-01-01T00:00:00Z'); + expect(apiKey.expiresAt).toBe('2030-01-01T00:00:00.000Z'); + }); + + it('updateApiKey renames, refuses collisions honestly, and leaves history alone', async () => { + const { app } = testGateway(); + const { id } = await mint(app, 'ci'); + const other = await mint(app, 'laptop'); + + const renamed = ( + await rpc(app, '/updateApiKey', { id, name: 'ci-2026' }) + ).json().apiKey; + expect(renamed.name).toBe('ci-2026'); + + // Onto a live name: refused like create. + const clash = await rpc(app, '/updateApiKey', { id, name: 'laptop' }); + expect(clash.statusCode).toBe(409); + + // Onto a revoked name: revoke freed it. + await rpc(app, '/revokeApiKey', { id: other.id }); + expect( + (await rpc(app, '/updateApiKey', { id, name: 'laptop' })).statusCode, + ).toBe(200); + + // Unknown id is a 404; a revoked row is history, not editable. + expect( + (await rpc(app, '/updateApiKey', { id: 'ghost', name: 'x' })).statusCode, + ).toBe(404); + const edited = await rpc(app, '/updateApiKey', { + id: other.id, + name: 'zombie', + }); + expect(edited.statusCode).toBe(409); + expect(edited.json().message).toMatch(/rotation history/); + }); + + it('a no-op patch changes nothing', async () => { + const { app } = testGateway(); + const { id } = await mint(app, 'ci'); + const before = (await rpc(app, '/listApiKeys')).json().apiKeys; + const res = await rpc(app, '/updateApiKey', { + id, + name: 'ci', + disabled: false, + }); + expect(res.statusCode).toBe(200); + expect((await rpc(app, '/listApiKeys')).json().apiKeys).toEqual(before); + }); + + it('carries expiresAt from mint into the list', async () => { + const { app } = testGateway(); + const future = new Date(Date.now() + 86_400_000).toISOString(); + await mint(app, 'ttl', future); + const keys = (await rpc(app, '/listApiKeys')).json().apiKeys; + expect(keys[0].expiresAt).toBe(future); + }); + + it('admin-only: a live key gets an honest 403 on every management and node verb, without a lastUsedAt fingerprint', async () => { + const { app } = testGateway(); + const { id, token } = await mint(app, 'ci'); + const asKey = { authorization: `Bearer ${token}` }; + + const attempts = [ + ['/createApiKey', { name: 'evil' }], + ['/listApiKeys', {}], + ['/updateApiKey', { id, disabled: true }], + ['/revokeApiKey', { id }], + ['/listNodes', {}], + ['/removeNode', { id: 'node-x' }], + ] as const; + for (const [url, payload] of attempts) { + const res = await app.inject({ + method: 'POST', + url, + headers: asKey, + payload, + }); + expect(res.statusCode).toBe(403); + expect(res.json().message).toMatch(/cannot manage API keys/); + } + + // The refusals honored nothing: no lastUsedAt fingerprint, key untouched. + const row = (await rpc(app, '/listApiKeys')).json().apiKeys[0]; + expect(row.lastUsedAt).toBeNull(); + expect(row.disabledAt).toBeNull(); + expect(row.revokedAt).toBeNull(); + + // Garbage stays garbage: 401, not 403. + expect( + ( + await app.inject({ + method: 'POST', + url: '/createApiKey', + headers: { authorization: 'Bearer not-a-key' }, + payload: { name: 'x' }, + }) + ).statusCode, + ).toBe(401); + }); + + it('a key never passes the nodes gate: a check-in under a minted key is refused', async () => { + const { app } = testGateway(); + const { token } = await mint(app, 'ci'); + const res = await app.inject({ + method: 'POST', + url: '/checkIn', + headers: { authorization: `Bearer ${token}` }, + payload: {}, + }); + expect(res.statusCode).toBe(401); + }); + + it('admin-only: a console session opens the management verbs', async () => { + const { app } = testGateway(); + const setup = await app.inject({ + method: 'POST', + url: '/console/auth/setup', + payload: { + token: TEST_TOKEN, + username: 'operator', + password: 'horse pass', + }, + }); + expect(setup.statusCode).toBe(200); + const cookie = setup.cookies.find((c) => c.name === SESSION_COOKIE); + expect(cookie).toBeDefined(); + + const res = await app.inject({ + method: 'POST', + url: '/createApiKey', + headers: { [CONSOLE_HEADER]: '1' }, + cookies: { [SESSION_COOKIE]: (cookie as { value: string }).value }, + payload: { name: 'from-console' }, + }); + expect(res.statusCode).toBe(200); + expect((await useKey(app, res.json().token)).statusCode).toBe(OPENED); + }); +}); diff --git a/packages/gateway/src/routes/api-keys.ts b/packages/gateway/src/routes/api-keys.ts new file mode 100644 index 00000000..b2a14eed --- /dev/null +++ b/packages/gateway/src/routes/api-keys.ts @@ -0,0 +1,156 @@ +import { + type ApiKey, + createApiKeyRequestSchema, + createApiKeyResponseSchema, + listApiKeysResponseSchema, + revokeApiKeyRequestSchema, + revokeApiKeyResponseSchema, + updateApiKeyRequestSchema, + updateApiKeyResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { + createApiKey, + findActiveApiKeyByName, + findApiKeyById, + listApiKeys, + revokeApiKey, + updateApiKey, +} from '../db/api-keys'; +import type { Db } from '../db/db'; +import type { ApiKeyRow } from '../db/schema'; +import { httpError } from '../http-error'; + +export interface ApiKeyRoutesOptions { + db: Db; +} + +/** The wire view: everything but the hash — no secret ever leaves the row. */ +function view(row: ApiKeyRow): ApiKey { + return { + id: row.id, + name: row.name, + prefix: row.prefix, + createdAt: row.createdAt, + lastUsedAt: row.lastUsedAt, + expiresAt: row.expiresAt, + disabledAt: row.disabledAt, + revokedAt: row.revokedAt, + }; +} + +/** + * API key management: mint, list, edit, revoke — the fleet's client + * credentials, minted here at the one door (design record #20; nodes + * accept only the fleet token). Pure table verbs. Admin-only: + * buildGatewayApp registers this plugin behind requireAdminAuth (fleet + * token or console session; a live key gets an honest 403), because a + * credential must not manage the credential ledger it lives in. + * Verification itself lives in db/api-keys.ts and is consulted by the + * app's isCredential closure, not here. + */ +export const apiKeyRoutes: FastifyPluginAsyncZod = async ( + app, + { db }, +) => { + app.post( + '/createApiKey', + { + schema: { + body: createApiKeyRequestSchema, + response: { 200: createApiKeyResponseSchema }, + }, + }, + async (request) => { + const { name, expiresAt } = request.body; + // Two live credentials answering to one name is a rotation mistake, + // not a goal — refused by name, like removeTemplate's 409. The + // partial unique index backstops this check as a schema fact; no + // await sits between check and insert, so they cannot race. + if (findActiveApiKeyByName(db, name)) { + throw httpError( + 409, + `an active API key named '${name}' already exists — revoke it first or pick another name`, + ); + } + const { row, token } = createApiKey(db, name, expiresAt); + // The token itself never reaches the log. + request.log.info( + { apiKey: row.id, name, prefix: row.prefix, expiresAt: row.expiresAt }, + 'API key minted', + ); + return { apiKey: view(row), token }; + }, + ); + + app.post( + '/listApiKeys', + { + schema: { + response: { 200: listApiKeysResponseSchema }, + }, + }, + async () => ({ apiKeys: listApiKeys(db).map(view) }), + ); + + app.post( + '/updateApiKey', + { + schema: { + body: updateApiKeyRequestSchema, + response: { 200: updateApiKeyResponseSchema }, + }, + }, + async (request) => { + const { id, ...patch } = request.body; + // Adjudication happens here, in order, with no await between the + // checks and the write (better-sqlite3 is sync — they cannot race). + const row = findApiKeyById(db, id); + if (!row) { + throw httpError(404, `no API key with id '${id}'`); + } + if (row.revokedAt !== null) { + throw httpError( + 409, + `API key "${row.name}" is revoked — revoked rows are rotation history and cannot be changed`, + ); + } + if (patch.name !== undefined && patch.name !== row.name) { + // Same courtesy as create: the name must not collide with any + // non-revoked key. findActiveApiKeyByName cannot return this row + // itself — the names differ. + if (findActiveApiKeyByName(db, patch.name)) { + throw httpError( + 409, + `an active API key named '${patch.name}' already exists — revoke it first or pick another name`, + ); + } + } + const updated = updateApiKey(db, row, patch); + if (updated !== row) { + request.log.info( + { apiKey: row.id, name: updated.name, patch }, + 'API key updated', + ); + } + return { apiKey: view(updated) }; + }, + ); + + app.post( + '/revokeApiKey', + { + schema: { + body: revokeApiKeyRequestSchema, + response: { 200: revokeApiKeyResponseSchema }, + }, + }, + async (request) => { + const revoked = revokeApiKey(db, request.body.id); + if (revoked) { + request.log.info({ apiKey: request.body.id }, 'API key revoked'); + } + return { revoked }; + }, + ); +}; diff --git a/packages/gateway/src/routes/console.test.ts b/packages/gateway/src/routes/console.test.ts new file mode 100644 index 00000000..bca3b73c --- /dev/null +++ b/packages/gateway/src/routes/console.test.ts @@ -0,0 +1,397 @@ +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { + CONSOLE_HEADER, + hashPassword, + mintSession, + mintSessionSecret, + SESSION_COOKIE, + SESSION_TTL_SECONDS, + verifyPassword, + verifySession, +} from '../auth'; +import { TEST_TOKEN, testGateway } from '../testing'; + +const USERNAME = 'operator'; +const PASSWORD = 'correct horse battery'; + +type TestApp = ReturnType['app']; + +function testApp(consoleDistDir?: string): TestApp { + return testGateway({}, { consoleDistDir }).app; +} + +/** A minimal built console: an index.html and one hashed asset. */ +function fixtureDist(): string { + const dir = mkdtempSync(join(tmpdir(), 'dormice-consoledist-')); + writeFileSync(join(dir, 'index.html'), 'dormice console'); + mkdirSync(join(dir, 'assets')); + writeFileSync(join(dir, 'assets', 'app-abc123.js'), 'console.log("ui")'); + return dir; +} + +async function setup( + app: TestApp, + { token = TEST_TOKEN, username = USERNAME, password = PASSWORD } = {}, +) { + return app.inject({ + method: 'POST', + url: '/console/auth/setup', + payload: { token, username, password }, + }); +} + +async function login( + app: TestApp, + { username = USERNAME, password = PASSWORD } = {}, +) { + return app.inject({ + method: 'POST', + url: '/console/auth/login', + payload: { username, password }, + }); +} + +/** The Set-Cookie value for the session cookie, parsed by fastify's helper. */ +function sessionCookie(res: { cookies: Array> }) { + const cookie = res.cookies.find((c) => c.name === SESSION_COOKIE); + expect(cookie).toBeDefined(); + return cookie as { value: string } & Record; +} + +/** A verb behind the admin gate, the console's everyday food. */ +async function listNodes( + app: TestApp, + cookieValue: string, + headers: Record = { [CONSOLE_HEADER]: '1' }, +) { + return app.inject({ + method: 'POST', + url: '/listNodes', + cookies: { [SESSION_COOKIE]: cookieValue }, + headers, + payload: {}, + }); +} + +describe('password hashing', () => { + it('round-trips and rejects a wrong password', async () => { + const stored = await hashPassword(PASSWORD); + expect(stored.startsWith('scrypt$')).toBe(true); + expect(await verifyPassword(PASSWORD, stored)).toBe(true); + expect(await verifyPassword('not the password', stored)).toBe(false); + }); + + it('salts: two hashes of the same password differ', async () => { + expect(await hashPassword(PASSWORD)).not.toBe(await hashPassword(PASSWORD)); + }); + + it('rejects garbage stored values instead of throwing', async () => { + expect(await verifyPassword(PASSWORD, '')).toBe(false); + expect(await verifyPassword(PASSWORD, 'bcrypt$whatever')).toBe(false); + }); +}); + +describe('session mint/verify', () => { + const SECRET = mintSessionSecret(); + + it('round-trips a fresh session', () => { + expect(verifySession(SECRET, mintSession(SECRET))).toBe(true); + }); + + it('rejects an expired session', () => { + const past = Date.now() - (SESSION_TTL_SECONDS + 10) * 1000; + expect(verifySession(SECRET, mintSession(SECRET, past))).toBe(false); + }); + + it('rejects a tampered expiry: the HMAC covers it', () => { + const value = mintSession(SECRET); + const [exp, mac] = value.split('.'); + const later = `${Number(exp) + 3600}.${mac}`; + expect(verifySession(SECRET, later)).toBe(false); + }); + + it('rejects garbage and sessions minted under another secret', () => { + expect(verifySession(SECRET, 'not-a-session')).toBe(false); + expect(verifySession(SECRET, '')).toBe(false); + expect(verifySession(SECRET, mintSession(mintSessionSecret()))).toBe(false); + }); +}); + +describe('POST /console/auth/status', () => { + it('reports whether setup has happened', async () => { + const app = testApp(); + const before = await app.inject({ + method: 'POST', + url: '/console/auth/status', + payload: {}, + }); + expect(before.json()).toEqual({ accountExists: false }); + await setup(app); + const after = await app.inject({ + method: 'POST', + url: '/console/auth/status', + payload: {}, + }); + expect(after.json()).toEqual({ accountExists: true }); + }); +}); + +describe('POST /console/auth/setup', () => { + it('rejects a wrong token without creating anything', async () => { + const app = testApp(); + const res = await setup(app, { token: 'wrong-token-wrong-token-wrong-tk' }); + expect(res.statusCode).toBe(401); + expect(res.cookies).toHaveLength(0); + expect((await login(app)).statusCode).toBe(409); + }); + + it('creates the account and signs the caller in', async () => { + const app = testApp(); + const res = await setup(app); + expect(res.statusCode).toBe(200); + const cookie = sessionCookie(res); + expect(cookie.httpOnly).toBe(true); + expect(cookie.sameSite).toBe('Strict'); + expect(cookie.path).toBe('/'); + expect(cookie.maxAge).toBe(SESSION_TTL_SECONDS); + }); + + it('refuses a short password', async () => { + const res = await setup(testApp(), { password: 'short' }); + expect(res.statusCode).toBe(400); + }); + + it('re-setup overwrites the account and voids old sessions', async () => { + const app = testApp(); + const first = sessionCookie(await setup(app)); + // The recovery path: the token alone resets username and password. + const res = await setup(app, { + username: 'renamed', + password: 'brand-new-pass', + }); + expect(res.statusCode).toBe(200); + expect((await listNodes(app, first.value)).statusCode).toBe(401); + expect( + (await login(app, { username: 'renamed', password: 'brand-new-pass' })) + .statusCode, + ).toBe(200); + expect((await login(app)).statusCode).toBe(401); + }); +}); + +describe('POST /console/auth/login', () => { + it('answers 409 before setup — an honest pointer, not a guess counted', async () => { + const res = await login(testApp()); + expect(res.statusCode).toBe(409); + expect(res.json().message).toContain('setup'); + }); + + it('rejects wrong credentials without setting a cookie', async () => { + const app = testApp(); + await setup(app); + const wrongPass = await login(app, { password: 'wrong password' }); + expect(wrongPass.statusCode).toBe(401); + expect(wrongPass.cookies).toHaveLength(0); + const wrongUser = await login(app, { username: 'someone-else' }); + expect(wrongUser.statusCode).toBe(401); + }); + + it('signs in with the right credentials', async () => { + const app = testApp(); + await setup(app); + const res = await login(app); + expect(res.statusCode).toBe(200); + const cookie = sessionCookie(res); + expect(cookie.httpOnly).toBe(true); + expect(cookie.maxAge).toBe(SESSION_TTL_SECONDS); + }); +}); + +describe('login throttle over the wire', () => { + it('backs off after repeated failures — even the right credential waits', async () => { + const app = testApp(); + await setup(app); + for (let i = 0; i < 8; i++) { + const res = await login(app, { password: 'wrong password' }); + expect([401, 429]).toContain(res.statusCode); + } + const blocked = await login(app); + expect(blocked.statusCode).toBe(429); + expect(blocked.json().message).toContain('retry in'); + // Setup shares the same counters: guessing tokens is the same game. + expect((await setup(app)).statusCode).toBe(429); + }); + + it('a success clears the slate', async () => { + const app = testApp(); + await setup(app); + for (let i = 0; i < 4; i++) { + await login(app, { password: 'wrong password' }); + } + expect((await login(app)).statusCode).toBe(200); + expect((await login(app, { password: 'wrong password' })).statusCode).toBe( + 401, + ); + }); +}); + +describe('cookie-authenticated access to the gates', () => { + it('a fresh session cookie opens the admin gate', async () => { + const app = testApp(); + await setup(app); + const cookie = sessionCookie(await login(app)); + const res = await listNodes(app, cookie.value); + expect(res.statusCode).toBe(200); + expect(res.json()).toEqual({ nodes: [] }); + }); + + it('and the sandbox gate: the console header rides along, the verb judges the rest', async () => { + const app = testApp(); + await setup(app); + const cookie = sessionCookie(await login(app)); + const res = await app.inject({ + method: 'POST', + url: '/envdToken', + cookies: { [SESSION_COOKIE]: cookie.value }, + headers: { [CONSOLE_HEADER]: '1' }, + payload: { sandboxId: 'sb-nowhere' }, + }); + // Through the gate; no node holds the id, so the verb's own 404. + expect(res.statusCode).toBe(404); + expect(res.json().message).toContain('is on no node'); + const bare = await app.inject({ + method: 'POST', + url: '/envdToken', + cookies: { [SESSION_COOKIE]: cookie.value }, + payload: { sandboxId: 'sb-nowhere' }, + }); + expect(bare.statusCode).toBe(401); + }); + + it('the cookie alone is not enough: the console header is required', async () => { + const app = testApp(); + await setup(app); + const cookie = sessionCookie(await login(app)); + const res = await listNodes(app, cookie.value, {}); + expect(res.statusCode).toBe(401); + }); + + it('rejects a tampered cookie', async () => { + const app = testApp(); + await setup(app); + const cookie = sessionCookie(await login(app)); + const res = await listNodes(app, `${cookie.value}ff`); + expect(res.statusCode).toBe(401); + }); + + it('rejects any cookie while no account exists', async () => { + // A cookie minted under some secret proves nothing when the table has + // no account (e.g. the database was recreated). + const res = await listNodes(testApp(), mintSession(mintSessionSecret())); + expect(res.statusCode).toBe(401); + }); + + it('does not open the E2B surface: that has its own auth', async () => { + const app = testApp(); + await setup(app); + const cookie = sessionCookie(await login(app)); + const res = await app.inject({ + method: 'GET', + url: '/e2b/api/v2/sandboxes', + cookies: { [SESSION_COOKIE]: cookie.value }, + headers: { [CONSOLE_HEADER]: '1' }, + }); + expect(res.statusCode).toBe(401); + }); + + it('nor the nodes gate: a check-in is a machine reporting, not a session', async () => { + const app = testApp(); + await setup(app); + const cookie = sessionCookie(await login(app)); + const res = await app.inject({ + method: 'POST', + url: '/checkIn', + cookies: { [SESSION_COOKIE]: cookie.value }, + headers: { [CONSOLE_HEADER]: '1' }, + payload: {}, + }); + expect(res.statusCode).toBe(401); + }); +}); + +describe('POST /console/auth/logout', () => { + it('clears the session cookie', async () => { + const res = await testApp().inject({ + method: 'POST', + url: '/console/auth/logout', + }); + expect(res.statusCode).toBe(200); + const cookie = sessionCookie(res); + expect(cookie.value).toBe(''); + }); +}); + +describe('GET / — the bare-origin redirect', () => { + it('sends a browser (html Accept) to /console/', async () => { + const res = await testApp(fixtureDist()).inject({ + method: 'GET', + url: '/', + headers: { accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8' }, + }); + expect(res.statusCode).toBe(302); + expect(res.headers.location).toBe('/console/'); + }); + + it('keeps the honest 404 for non-browser clients', async () => { + // curl's default is Accept: */* — no html, no redirect. + const res = await testApp(fixtureDist()).inject({ + method: 'GET', + url: '/', + headers: { accept: '*/*' }, + }); + expect(res.statusCode).toBe(404); + expect(res.json().message).toContain('not found'); + }); + + it('redirects even when the console is not built — the 404 there points at pnpm build', async () => { + const res = await testApp().inject({ + method: 'GET', + url: '/', + headers: { accept: 'text/html' }, + }); + expect(res.statusCode).toBe(302); + }); +}); + +describe('static console at /console', () => { + it('serves index.html and assets from the injected dist', async () => { + const app = testApp(fixtureDist()); + const index = await app.inject({ method: 'GET', url: '/console/' }); + expect(index.statusCode).toBe(200); + expect(index.body).toContain('dormice console'); + const asset = await app.inject({ + method: 'GET', + url: '/console/assets/app-abc123.js', + }); + expect(asset.statusCode).toBe(200); + }); + + it('falls back to index.html for client-side routes (SPA)', async () => { + const app = testApp(fixtureDist()); + const res = await app.inject({ + method: 'GET', + url: '/console/sandboxes/deep', + }); + expect(res.statusCode).toBe(200); + expect(res.body).toContain('dormice console'); + }); + + it('answers an honest 404 when the console is not built', async () => { + const res = await testApp().inject({ method: 'GET', url: '/console' }); + expect(res.statusCode).toBe(404); + expect(res.json().message).toContain('pnpm build'); + }); +}); diff --git a/packages/gateway/src/routes/console.ts b/packages/gateway/src/routes/console.ts new file mode 100644 index 00000000..5c0f49f7 --- /dev/null +++ b/packages/gateway/src/routes/console.ts @@ -0,0 +1,231 @@ +import fastifyStatic from '@fastify/static'; +import type { FastifyReply } from 'fastify'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { z } from 'zod'; +import { + hashPassword, + mintSession, + mintSessionSecret, + SESSION_COOKIE, + SESSION_TTL_SECONDS, + tokensEqual, + verifyPassword, +} from '../auth'; +import type { Config } from '../config'; +import { getConsoleAccount, setConsoleAccount } from '../db/account'; +import type { Db } from '../db/db'; +import { LoginThrottle } from '../login-throttle'; + +export interface ConsoleRoutesOptions { + config: Config; + db: Db; + /** + * Where the built web console lives (packages/console/dist). Injected so + * tests point it at a fixture and embedders can omit it; absent means + * /console answers an honest 404 instead of guessing at paths. + */ + consoleDistDir?: string; +} + +// No Secure flag: the gateway speaks plain http on 127.0.0.1 by design, and +// behind a TLS reverse proxy the browser-facing side is the proxy's job. +const COOKIE_OPTIONS = { + httpOnly: true, + sameSite: 'strict', + path: '/', +} as const; + +const messageResponse = z.object({ message: z.string() }); + +/** + * The web console's own surface: account + session endpoints and the static + * SPA — on the gateway, the fleet's one door, since design record #24: + * one console, one account, whatever the number of nodes. Everything else + * the console does goes through the gateway's verbs with the session + * cookie — the same verbs, the same truth as the SDK and CLI. + * + * The credential model: the fleet token is the root of trust (machine + * credential, lives in the gateway's env), the account is the human + * convenience derived from it. Setup requires the token and overwrites the + * account — first-run initialization, password change and forgot-password + * are all that one verb, so there is no registration race (an open "first + * visitor becomes admin" door on a public URL) and no recovery flow. + */ +export const consoleRoutes: FastifyPluginAsyncZod< + ConsoleRoutesOptions +> = async (app, { config, db, consoleDistDir }) => { + // Per-app, not module-global: each daemon (and each test app) gets its + // own counters. Shared by login and setup — both are credential guesses. + const throttle = new LoginThrottle(); + + const setSessionCookie = (reply: FastifyReply, sessionSecret: string) => { + reply.setCookie(SESSION_COOKIE, mintSession(sessionSecret), { + ...COOKIE_OPTIONS, + // The cookie lives exactly as long as the HMAC inside it is valid. + maxAge: SESSION_TTL_SECONDS, + }); + }; + + // Open by design: it answers only "is setup still pending", which the + // login page needs before any credential exists. An attacker learns + // nothing usable — completing setup requires the API token either way. + app.post( + '/console/auth/status', + { + schema: { + response: { 200: z.object({ accountExists: z.boolean() }) }, + }, + }, + async () => ({ accountExists: getConsoleAccount(db) !== undefined }), + ); + + app.post( + '/console/auth/setup', + { + schema: { + body: z.object({ + token: z.string().min(1), + username: z.string().trim().min(1).max(64), + // Length is the only strength rule: composition rules push people + // toward Password1! and help nobody. + password: z.string().min(8).max(128), + }), + response: { + 200: z.object({ loggedIn: z.literal(true) }), + 401: messageResponse, + 429: messageResponse, + }, + }, + }, + async (request, reply) => { + const wait = throttle.retryAfterSeconds(request.ip); + if (wait > 0) { + return reply.code(429).send({ + message: `too many failed attempts — retry in ${wait}s`, + }); + } + // Deliberately the fleet token only, never a minted API key: this verb + // resets the human account, and a leaked machine credential must not + // escalate into a console takeover. The token's root of trust is + // filesystem access to the gateway's env file — exactly what a + // recovery path should require. + if (!tokensEqual(request.body.token, config.DORMICE_API_TOKEN)) { + throttle.recordFailure(request.ip); + return reply.code(401).send({ message: 'invalid API token' }); + } + throttle.clear(request.ip); + const account = setConsoleAccount(db, { + username: request.body.username, + passwordHash: await hashPassword(request.body.password), + // A fresh secret voids every existing session — the semantics a + // password (re)set should have. + sessionSecret: mintSessionSecret(), + }); + setSessionCookie(reply, account.sessionSecret); + return { loggedIn: true as const }; + }, + ); + + app.post( + '/console/auth/login', + { + schema: { + // min(1) only: the password policy is enforced where passwords are + // set; login must accept whatever was stored. + body: z.object({ + username: z.string().min(1), + password: z.string().min(1), + }), + response: { + 200: z.object({ loggedIn: z.literal(true) }), + 401: messageResponse, + 409: messageResponse, + 429: messageResponse, + }, + }, + }, + async (request, reply) => { + const wait = throttle.retryAfterSeconds(request.ip); + if (wait > 0) { + return reply.code(429).send({ + message: `too many failed attempts — retry in ${wait}s`, + }); + } + const account = getConsoleAccount(db); + if (!account) { + // Not a failed guess (nothing exists to guess at), so no throttle + // hit — an honest pointer to setup instead. + return reply.code(409).send({ + message: 'no account exists yet — complete setup with the API token', + }); + } + // Evaluate both factors unconditionally so a wrong username costs the + // same time as a wrong password. + const usernameOk = tokensEqual(request.body.username, account.username); + const passwordOk = await verifyPassword( + request.body.password, + account.passwordHash, + ); + if (!usernameOk || !passwordOk) { + throttle.recordFailure(request.ip); + return reply + .code(401) + .send({ message: 'invalid username or password' }); + } + throttle.clear(request.ip); + setSessionCookie(reply, account.sessionSecret); + return { loggedIn: true as const }; + }, + ); + + app.post( + '/console/auth/logout', + { + schema: { + response: { 200: z.object({ loggedIn: z.literal(false) }) }, + }, + }, + async (_request, reply) => { + reply.clearCookie(SESSION_COOKIE, COOKIE_OPTIONS); + return { loggedIn: false as const }; + }, + ); + + // The bare-origin convenience: a browser landing on / is a human looking + // for the console — send them there (even unbuilt, /console's "run pnpm + // build" 404 beats "route not found"). Machines never GET / with an html + // Accept, so they keep the honest 404 from the app-wide arbiter. + app.get('/', async (request, reply) => { + if (request.headers.accept?.includes('text/html')) { + return reply.redirect('/console/'); + } + return reply.callNotFound(); + }); + + if (consoleDistDir) { + await app.register( + async (scope) => { + await scope.register(fastifyStatic, { root: consoleDistDir }); + // SPA fallback: the router owns paths under /console, so any GET + // that matches no file is a client-side route — serve the app and + // let it resolve. Everything else keeps the honest 404. + scope.setNotFoundHandler((request, reply) => { + if (request.method === 'GET') { + return reply.sendFile('index.html'); + } + reply.code(404).send({ + message: `route ${request.method} ${request.url} not found`, + }); + }); + }, + { prefix: '/console' }, + ); + } else { + app.get('/console', async (_request, reply) => + reply.code(404).send({ + message: + 'web console not available: packages/console/dist was not found at startup — run `pnpm build` first', + }), + ); + } +}; diff --git a/packages/gateway/src/routes/e2b.ts b/packages/gateway/src/routes/e2b.ts index 4b5ad57e..83a7640c 100644 --- a/packages/gateway/src/routes/e2b.ts +++ b/packages/gateway/src/routes/e2b.ts @@ -1,4 +1,3 @@ -import { tokensEqual } from '@dormice/server/auth'; import type { KeyedQueue } from '@dormice/server/keyed-queue'; import { sandboxNameSchema } from '@dormice/shared'; import type { FastifyError, FastifyReply, FastifyRequest } from 'fastify'; @@ -24,7 +23,10 @@ export interface E2bRoutesOptions { finder: Finder; locks: KeyedQueue; knobs: PlacementKnobs; + /** The fleet token, presented to the nodes when forwarding. */ token: string; + /** The app's one adjudication of a bare credential (fleet token or a live minted key). */ + isCredential: (bareToken: string) => boolean; } /** @@ -37,7 +39,7 @@ export interface E2bRoutesOptions { */ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( app, - { fleet, finder, locks, knobs, token }, + { fleet, finder, locks, knobs, token, isCredential }, ) => { app.addContentTypeParser( 'application/json', @@ -56,13 +58,14 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( }); }); - // The daemon's own X-API-KEY convention (`e2b_`), over the - // fleet's one token. + // The daemon's own X-API-KEY convention (`e2b_`; the prefix + // is the SDK's, not a secret): the fleet token or any live key the + // gateway minted, judged by the same closure as the native Bearer face. app.addHook('onRequest', async (request, reply) => { const presented = request.headers['x-api-key']; const key = Array.isArray(presented) ? presented[0] : presented; const bare = key?.startsWith('e2b_') ? key.slice(4) : key; - if (bare === undefined || !tokensEqual(bare, token)) { + if (bare === undefined || !isCredential(bare)) { await reply.code(401).send({ code: 401, message: 'invalid API key' }); } }); diff --git a/packages/gateway/src/routes/envd-token.ts b/packages/gateway/src/routes/envd-token.ts new file mode 100644 index 00000000..e1fef49f --- /dev/null +++ b/packages/gateway/src/routes/envd-token.ts @@ -0,0 +1,68 @@ +import { + envdTokenRequestSchema, + envdTokenResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { relay } from '../errors'; +import type { Finder } from '../find'; +import { forwardCapture, replay } from '../forward'; +import { httpError } from '../http-error'; +import { refuse, verdict } from './verdict'; + +export interface EnvdTokenRoutesOptions { + finder: Finder; + token: string; +} + +/** + * envdToken through the gateway: the console (or any caller through the + * door) names a sandbox by id; the gateway finds the node that runs it + * and asks that node to mint, under the fleet token, and hands the answer + * back verbatim. The token is an HMAC under the node's own signing secret + * — nothing the gateway holds could mint it, which is exactly right: the + * node that runs the sandbox is the one that judges its envd traffic. + * Found by id, never by name: a terminal is opened on a sandbox the + * caller already sees. + */ +export const envdTokenRoutes: FastifyPluginAsyncZod< + EnvdTokenRoutesOptions +> = async (app, { finder, token }) => { + app.post( + '/envdToken', + { + schema: { + body: envdTokenRequestSchema, + response: { 200: envdTokenResponseSchema }, + }, + }, + async (request, reply) => { + const { sandboxId } = request.body; + const judged = verdict( + await finder.byId(sandboxId), + `sandbox "${sandboxId}"`, + ); + if (judged.kind === 'refuse') return refuse(reply, judged); + if (judged.kind === 'none') { + throw httpError(404, `sandbox "${sandboxId}" is on no node`); + } + const { node } = judged; + reply.hijack(); + await relay( + reply.raw, + 'native', + request.log, + async () => { + // The body was parsed by Fastify (a JSON verb on the gateway's + // own face); the node gets the same words re-serialized. + const answer = await forwardCapture(request.raw, reply.raw, { + target: { endpoint: node.endpoint, token }, + credential: 'bearer', + body: Buffer.from(JSON.stringify(request.body)), + }); + if (answer !== null) replay(reply.raw, answer); + }, + (error) => ({ status: 502, message: `${error.message} — retry` }), + ); + }, + ); +}; diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index 088fbc6f..0b41723c 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -47,10 +47,11 @@ export const NAMED_VERBS = [ /** * The verbs that address the daemon, not a sandbox: fleet lists, host - * readings, templates, settings, ingress, upgrade, keys. Asking every - * node and merging, or answering from the gateway's own tables, comes - * with the configuration authority; until then each answers an honest - * 501 naming the alternative, instead of a misleading answer from + * readings, templates, settings, ingress, upgrade. The ones the gateway + * answers from its own tables leave this list as they arrive (the API + * keys did with the configuration authority); asking every node and + * merging the rest comes in a later cut. Until then each answers an + * honest 501 naming the alternative, instead of a misleading answer from * whichever node the gateway happened to pick. */ export const UNNAMED_VERBS = [ @@ -70,10 +71,6 @@ export const UNNAMED_VERBS = [ 'listTemplates', 'removeTemplate', 'updateSettings', - 'createApiKey', - 'listApiKeys', - 'updateApiKey', - 'revokeApiKey', ] as const; export interface NativeRoutesOptions { diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index 76401a88..0df5ba6f 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -10,6 +10,10 @@ import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import type { NameCache } from '../cache'; import { downReason, type Fleet, STARTUP_GRACE_MS } from '../fleet'; +export interface CheckInRoutesOptions { + fleet: Fleet; +} + export interface NodeRoutesOptions { fleet: Fleet; cache: NameCache; @@ -21,15 +25,12 @@ function refusal(statusCode: number, message: string): Error { } /** - * The gateway's own verbs about its nodes: the check-in the nodes send - * (RULES/协议.md「网关」), and what an operator reads and does about them. - * Everything listNodes answers is what the gateway already holds, so - * answering costs no node anything. + * The check-in the nodes send (RULES/协议.md「网关」) — behind the nodes' + * own gate in app.ts: the fleet token and nothing else. */ -export const nodeRoutes: FastifyPluginAsyncZod = async ( - app, - { fleet, cache }, -) => { +export const checkInRoutes: FastifyPluginAsyncZod< + CheckInRoutesOptions +> = async (app, { fleet }) => { /** * Per node, the ids it was last reported to share an endpoint with * (sorted, joined) — so the warning below is said when the situation @@ -103,7 +104,17 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( return {}; }, ); +}; +/** + * What an operator reads and does about the nodes — behind the admin gate. + * Everything listNodes answers is what the gateway already holds, so + * answering costs no node anything. + */ +export const nodeRoutes: FastifyPluginAsyncZod = async ( + app, + { fleet, cache }, +) => { app.post( '/listNodes', { diff --git a/packages/gateway/src/testing.ts b/packages/gateway/src/testing.ts index 083a1be6..70586bcd 100644 --- a/packages/gateway/src/testing.ts +++ b/packages/gateway/src/testing.ts @@ -1,10 +1,25 @@ +import { fileURLToPath } from 'node:url'; +import { KeyedQueue } from '@dormice/server/keyed-queue'; import type { CheckInRequest, NodeReading } from '@dormice/shared'; +import { buildGatewayApp } from './app'; +import { NameCache } from './cache'; +import { loadConfig } from './config'; +import { migrateDb, openDb } from './db/db'; +import { ensureSettings } from './db/settings'; +import { Finder } from './find'; +import { Fleet } from './fleet'; +import { type AskNode, httpAskNode } from './lookup'; /** * Test scaffolding shared by the gateway's suites: a node's reading and - * check-in with a few knobs turned. Not shipped — nothing under src/ but - * main.ts is bundled (tsup.config.ts). + * check-in with a few knobs turned, and a gateway app over an in-memory + * database for the suites about the gateway's own tables and gates. Not + * shipped — nothing under src/ but main.ts is bundled (tsup.config.ts). */ +export const TEST_TOKEN = 'fleet-token-fleet-token-fleet-token-fleet'; + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); + export function reading( over: { cpu?: number | null; @@ -58,3 +73,41 @@ export function checkInOf( reading: reading(over), }; } + +/** + * A gateway app over a fresh in-memory database, for app.inject(): the + * suites about keys, the console and the settings verbs need no node and + * no socket. Through loadConfig on purpose: defaults are adjudicated once, + * in the schema. + */ +export function testGateway( + env: Record = {}, + opts: { consoleDistDir?: string; ask?: AskNode } = {}, +) { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const config = loadConfig({ + DORMICE_API_TOKEN: TEST_TOKEN, + DORMICE_GATEWAY_DB_PATH: ':memory:', + ...env, + }); + ensureSettings(db, config); + const fleet = new Fleet(db); + const finder = new Finder( + fleet, + new NameCache(), + opts.ask ?? httpAskNode(TEST_TOKEN), + { warn: () => {} }, + ); + const app = buildGatewayApp({ + config, + db, + fleet, + finder, + locks: new KeyedQueue(), + logger: false, + build: null, + consoleDistDir: opts.consoleDistDir, + }); + return { app, db, fleet, config }; +} diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index f1ba8da0..2b8783e6 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -27,6 +27,7 @@ import type { KeyedQueue } from './keyed-queue'; import { apiKeyRoutes } from './routes/api-keys'; import { configRoutes } from './routes/config'; import { consoleRoutes } from './routes/console'; +import { envdTokenRoutes } from './routes/envd-token'; import { hostRoutes } from './routes/host'; import { ingressRoutes } from './routes/ingress'; import { sandboxRoutes } from './routes/sandboxes'; @@ -282,6 +283,7 @@ export function buildApp({ swap, }); await api.register(upgradeRoutes, { updater }); + await api.register(envdTokenRoutes, { envdSigningSecret }); }); // The apiKey management verbs and updateSettings sit behind the stricter @@ -308,9 +310,7 @@ export function buildApp({ await scope.register(consoleRoutes, { config, db, - apiAuth, consoleDistDir, - envdSigningSecret, }); }); diff --git a/packages/server/src/routes/console.test.ts b/packages/server/src/routes/console.test.ts index c15fc764..6b829a19 100644 --- a/packages/server/src/routes/console.test.ts +++ b/packages/server/src/routes/console.test.ts @@ -304,7 +304,7 @@ describe('cookie-authenticated API access', () => { }); }); -describe('POST /console/envdToken', () => { +describe('POST /envdToken', () => { async function mint( app: TestApp, cookieValue: string, @@ -312,7 +312,7 @@ describe('POST /console/envdToken', () => { ) { return app.inject({ method: 'POST', - url: '/console/envdToken', + url: '/envdToken', cookies: { [SESSION_COOKIE]: cookieValue }, headers, payload: { sandboxId: 'sb-terminal' }, diff --git a/packages/server/src/routes/console.ts b/packages/server/src/routes/console.ts index ffbdb915..90a3c574 100644 --- a/packages/server/src/routes/console.ts +++ b/packages/server/src/routes/console.ts @@ -1,5 +1,5 @@ import fastifyStatic from '@fastify/static'; -import type { FastifyReply, onRequestAsyncHookHandler } from 'fastify'; +import type { FastifyReply } from 'fastify'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { z } from 'zod'; import { @@ -14,25 +14,17 @@ import { import type { Config } from '../config'; import { getConsoleAccount, setConsoleAccount } from '../db/account'; import type { Db } from '../db/db'; -import { mintEnvdToken } from '../e2b/protocol'; import { LoginThrottle } from '../login-throttle'; export interface ConsoleRoutesOptions { config: Config; db: Db; - /** The API-wide auth arbiter, built once in app.ts — one truth. */ - apiAuth: onRequestAsyncHookHandler; /** * Where the built web console lives (packages/console/dist). Injected so * tests point it at a fixture and embedders can omit it; absent means * /console answers an honest 404 instead of guessing at paths. */ consoleDistDir?: string; - /** - * HMAC key for the envd tokens this surface mints — the ledger's signing - * secret, never the API token (they rotate independently). - */ - envdSigningSecret: string; } // No Secure flag: the daemon speaks plain http on 127.0.0.1 by design, and @@ -59,7 +51,7 @@ const messageResponse = z.object({ message: z.string() }); */ export const consoleRoutes: FastifyPluginAsyncZod< ConsoleRoutesOptions -> = async (app, { config, db, apiAuth, consoleDistDir, envdSigningSecret }) => { +> = async (app, { config, db, consoleDistDir }) => { // Per-app, not module-global: each daemon (and each test app) gets its // own counters. Shared by login and setup — both are credential guesses. const throttle = new LoginThrottle(); @@ -197,30 +189,6 @@ export const consoleRoutes: FastifyPluginAsyncZod< }, ); - // The console's terminal speaks to the envd surface directly — the same - // wire the e2b SDK uses — but envd auth is the per-sandbox HMAC keyed by - // the daemon's signing secret, which never leaves the daemon and the - // browser deliberately never holds. This trades the session for exactly - // one sandbox's token; the API-wide arbiter guards it, so the cookie path - // also needs the console header. Minting is stateless on purpose (like - // the secret itself): a made-up sandboxId yields a token that opens - // nothing. - app.post( - '/console/envdToken', - { - onRequest: apiAuth, - schema: { - body: z.object({ sandboxId: z.string().min(1) }), - response: { - 200: z.object({ envdAccessToken: z.string() }), - }, - }, - }, - async (request) => ({ - envdAccessToken: mintEnvdToken(envdSigningSecret, request.body.sandboxId), - }), - ); - // The bare-origin convenience: a browser landing on / is a human looking // for the console — send them there (even unbuilt, /console's "run pnpm // build" 404 beats "route not found"). Machines never GET / with an html diff --git a/packages/server/src/routes/envd-token.ts b/packages/server/src/routes/envd-token.ts new file mode 100644 index 00000000..3f584dfa --- /dev/null +++ b/packages/server/src/routes/envd-token.ts @@ -0,0 +1,41 @@ +import { + envdTokenRequestSchema, + envdTokenResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { mintEnvdToken } from '../e2b/protocol'; + +export interface EnvdTokenRoutesOptions { + /** + * HMAC key for the envd tokens this verb mints — the ledger's signing + * secret, never the API token (they rotate independently). + */ + envdSigningSecret: string; +} + +/** + * The console's terminal speaks to the envd surface directly — the same + * wire the e2b SDK uses — but envd auth is the per-sandbox HMAC keyed by + * this node's signing secret, which never leaves the node and the browser + * deliberately never holds. This verb trades an API credential for exactly + * one sandbox's token; the API-wide arbiter guards it like every other + * verb. Minting is stateless on purpose (like the secret itself): a + * made-up sandboxId yields a token that opens nothing. The gateway asks + * the sandbox's node this question on the console's behalf. + */ +export const envdTokenRoutes: FastifyPluginAsyncZod< + EnvdTokenRoutesOptions +> = async (app, { envdSigningSecret }) => { + app.post( + '/envdToken', + { + schema: { + body: envdTokenRequestSchema, + response: { 200: envdTokenResponseSchema }, + }, + }, + async (request) => ({ + envdAccessToken: mintEnvdToken(envdSigningSecret, request.body.sandboxId), + }), + ); +}; diff --git a/packages/shared/src/envd-token.ts b/packages/shared/src/envd-token.ts new file mode 100644 index 00000000..43b31101 --- /dev/null +++ b/packages/shared/src/envd-token.ts @@ -0,0 +1,25 @@ +import { z } from 'zod'; + +/** + * envdToken(sandboxId) — mints the per-sandbox envd access token the + * in-sandbox API (E2B's envd surface: terminal, files, processes) accepts, + * for a caller who is already through the API's front door. The console + * uses it to open a terminal: the browser holds a session, never the + * signing secret, and trades the one for exactly one sandbox's token. + * + * Minting is stateless: the token is an HMAC over the sandbox id under + * the node's signing secret, so a made-up id yields a token that opens + * nothing. Only the node that runs the sandbox can mint it (the secret + * never leaves its ledger); the gateway finds that node by id and asks. + */ +export const envdTokenRequestSchema = z.object({ + sandboxId: z.string().min(1), +}); + +export type EnvdTokenRequest = z.infer; + +export const envdTokenResponseSchema = z.object({ + envdAccessToken: z.string(), +}); + +export type EnvdTokenResponse = z.infer; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 296c5f5b..b2fed36c 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -2,6 +2,7 @@ export * from './acquire'; export * from './api-keys'; export * from './config'; export * from './destroy'; +export * from './envd-token'; export * from './exec'; export * from './files'; export * from './gateway'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c520a471..421a0853 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -188,6 +188,12 @@ importers: '@dormice/shared': specifier: workspace:* version: link:../shared + '@fastify/cookie': + specifier: ^11.0.2 + version: 11.0.2 + '@fastify/static': + specifier: ^9.1.3 + version: 9.1.3 better-sqlite3: specifier: ^12.11.1 version: 12.11.1 From b8e3614741de710d3190ae8f1d5eaecff395d910 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 16:03:08 +0800 Subject: [PATCH 32/89] Managed swap leaves the fleet-wide settings: it is one machine's knob A 29 GB test box and a 243 GB production box want different swap targets, so the knob cannot be a fleet-wide setting the gateway hands to every node alike. It leaves runtimeSettings, updateSettings and getConfig's swap block; on the gateway it is a column of the node's row, set per node (the verb lands with the settings verbs), and the node's own settings column stays as its copy, read by the boot reconcile. The console's swap dialog and row go with the wire (the node page brings the knob back per machine in the next cut), and the settings route's post-write work is the pids sweep alone. --- e2e/src/native.test.ts | 24 ---- packages/console/messages/de/settings.json | 10 -- packages/console/messages/en/settings.json | 10 -- packages/console/messages/es/settings.json | 10 -- packages/console/messages/fr/settings.json | 10 -- packages/console/messages/ja/settings.json | 10 -- packages/console/messages/ko/settings.json | 10 -- packages/console/messages/pt-BR/settings.json | 10 -- packages/console/messages/ru/settings.json | 10 -- packages/console/messages/zh-CN/settings.json | 10 -- packages/console/messages/zh-TW/settings.json | 10 -- .../components/RuntimeSettingsCard.tsx | 104 +----------------- .../settings/hooks/useUpdateSettings.ts | 2 +- packages/server/src/app.ts | 16 +-- packages/server/src/db/settings.ts | 11 +- packages/server/src/main.ts | 15 ++- packages/server/src/routes/config.ts | 10 +- packages/server/src/routes/settings.test.ts | 95 +--------------- packages/server/src/routes/settings.ts | 58 ++-------- packages/shared/src/config.ts | 21 +--- packages/shared/src/settings.ts | 33 ++---- website/content/docs/console.mdx | 15 +-- website/content/docs/http-api.mdx | 2 +- 23 files changed, 60 insertions(+), 446 deletions(-) diff --git a/e2e/src/native.test.ts b/e2e/src/native.test.ts index 3ecc559c..c044b5fc 100644 --- a/e2e/src/native.test.ts +++ b/e2e/src/native.test.ts @@ -787,30 +787,6 @@ describe('the observability verbs over a real daemon', () => { await client().updateSettings({ pidsLimit: before.pidsLimit }); }); - it('the swap knob follows getConfig: refused where unmanageable, accepted where real', async () => { - // The suite runs in two worlds — fake mode (CI, dev Macs), where - // managed swap is deterministically unavailable, and docker mode on a - // real Linux host, where it is real. Either way getConfig's - // `supported` is the adjudication and updateSettings must agree. - const { swap } = await client().getConfig(); - if (!swap.supported) { - // Unmanageable host: setting a target is a 400, not a silently - // stored dead value. - await expect( - client().updateSettings({ swapGb: 8 }), - ).rejects.toMatchObject({ - name: 'DormiceApiError', - status: 400, - message: expect.stringMatching(/Linux host with the docker executor/), - }); - return; - } - // Capable host: the knob is accepted. Target 0 = "manage none" — a - // recorded no-op, so the shared exam machine gains no swapfile. - const { settings } = await client().updateSettings({ swapGb: 0 }); - expect(settings.swapGb).toBe(0); - }); - it('getSandboxMetrics samples a live sandbox and 404s after destroy', async () => { await client().acquireSandbox('obs-metrics-key'); const sample = await client().getSandboxMetrics('obs-metrics-key'); diff --git a/packages/console/messages/de/settings.json b/packages/console/messages/de/settings.json index 5b65f88f..f72c808a 100644 --- a/packages/console/messages/de/settings.json +++ b/packages/console/messages/de/settings.json @@ -50,13 +50,8 @@ "settings_row_defaults": "Standardquoten neuer Sandboxes", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB Speicher · {disk} GiB Datenträger", "settings_row_policy": "Standard-Lebenszyklus-Richtlinie", - "settings_row_swap": "Zusätzlicher Swap", "settings_row_pids": "pids-Limit je Sandbox", "settings_row_pids_value": "{n} (Prozess- und Thread-Budget der Sandbox auf dem Host; bei Erreichen endet die ganze Sandbox)", - "settings_swap_unsupported": "Auf diesem Host nicht unterstützt (braucht Linux-Host + docker-Executor)", - "settings_swap_line_shrink": "Ziel {target} GiB · eingehängt {active} GiB — Verkleinern greift beim nächsten Neustart des Hosts", - "settings_swap_line_grow": "Ziel {target} GiB · eingehängt {active} GiB — Vergrößern unvollständig, siehe Daemon-Logs", - "settings_swap_line_ok": "{target} GiB (System-Swap nicht mitgezählt)", "settings_defaults_dialog_title": "Standardquoten neuer Sandboxes anpassen", "settings_defaults_dialog_desc": "CPU/Speicher greifen, wenn das nächste Mal ein Container geboren wird (auch bei Bestands-Sandboxes nach einem Kaltstart); die Datenträgergröße wird bei der Geburt des Datenträgers festgelegt (Ersterstellung und Archiv-Wiederherstellung) — der Datenträger ist die Sandbox selbst und wird nie an Ort und Stelle umdimensioniert. Vor dem Verkleinern prüfen, dass der Wert nicht unter dem echten Inhalt archivierter Sandboxes liegt.", "settings_defaults_saved": "Standardquoten neuer Sandboxes aktualisiert", @@ -65,11 +60,6 @@ "settings_defaults_memory_desc": "Ein OOM in der Sandbox beendet die ganze Sandbox (der Abgleich holt sie zurück) — lassen Sie dem Agent etwas Luft.", "settings_defaults_disk_label": "Datenträger (GiB)", "settings_defaults_disk_desc": "Nominelle Quote: Sparse-Images zahlen nur für echte Schreibvorgänge; Überbuchung überwachen Sie am Pegel des Datenlaufwerks.", - "settings_swap_dialog_title": "Zusätzlichen Swap anpassen", - "settings_swap_dialog_desc": "Die Swap-Kapazität entspricht grob der Menge an Sandbox-Speicher, die gleichzeitig überwintern kann (Einfrieren drückt Speicher in den Swap). Vergrößern wirkt sofort; Verkleinern wartet auf den nächsten Neustart des Hosts — genutzter Swap wird nie zwangsweise entfernt, das würde den Speicher aller überwinternden Sandboxes zurück ins RAM ziehen.", - "settings_swap_saved": "Swap-Ziel geändert auf {value} GiB", - "settings_swap_label": "Von Dormice verwalteter Swap gesamt (GiB)", - "settings_swap_field_desc": "Zusätzlich zum System-Swap; 0 = keiner. Die Swap-Datei belegt echten Platz auf dem Datenlaufwerk (nicht sparse) — werfen Sie vor dem Vergrößern einen Blick auf den Pegel des Datenlaufwerks auf der Übersichtsseite.", "settings_pids_dialog_title": "pids-Limit der Sandbox anpassen", "settings_pids_dialog_desc": "Unter gVisor ist dies keine in der Sandbox sichtbare Prozessanzahl, sondern ihr Prozess- und Thread-Budget auf dem Host. Wird es erreicht, bekommt nichts in der Sandbox einen Fehler: Die ganze Sandbox endet sofort (Exit-Code 2, kein OOM). Gilt ab dem Speichern: Neue Container starten mit dem neuen Wert, laufende Sandboxes werden sofort an Ort und Stelle darauf gebracht (ein cgroup-Schreibvorgang auf dem Host, den ihre Prozesse nicht bemerken), eingefrorene oder gestoppte Sandboxes übernehmen ihn beim nächsten Aufwachen. Kein Neuaufbau, kein Kaltstart.", "settings_pids_saved": "pids-Limit der Sandbox geändert auf {value}", diff --git a/packages/console/messages/en/settings.json b/packages/console/messages/en/settings.json index 1eab2d21..1105ca49 100644 --- a/packages/console/messages/en/settings.json +++ b/packages/console/messages/en/settings.json @@ -50,13 +50,8 @@ "settings_row_defaults": "New sandbox default quotas", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB memory · {disk} GiB disk", "settings_row_policy": "Default lifecycle policy", - "settings_row_swap": "Additional swap space", "settings_row_pids": "Sandbox pids cap", "settings_row_pids_value": "{n} (the sandbox's host-side process + thread budget; hitting it exits the whole sandbox)", - "settings_swap_unsupported": "Not supported on this host (requires a Linux host + docker executor)", - "settings_swap_line_shrink": "Target {target} GiB · mounted {active} GiB — shrinking takes effect on the next host reboot", - "settings_swap_line_grow": "Target {target} GiB · mounted {active} GiB — growth incomplete, see daemon logs", - "settings_swap_line_ok": "{target} GiB (system swap not included)", "settings_defaults_dialog_title": "Adjust default quotas for new sandboxes", "settings_defaults_dialog_desc": "CPU/memory take effect the next time a container is born (including existing sandboxes cold-started after a stop); disk size is fixed when the disk is born (first creation and archive restore) — the disk is the sandbox itself and is never resized in place, so before lowering it make sure it is not smaller than an archived sandbox's real content.", "settings_defaults_saved": "New sandbox default quotas updated", @@ -65,11 +60,6 @@ "settings_defaults_memory_desc": "An in-sandbox OOM exits the whole sandbox (reconciliation recovers it), so leave the agent some headroom.", "settings_defaults_disk_label": "Disk (GiB)", "settings_defaults_disk_desc": "Nominal quota: sparse images only pay for real writes; oversell is watched via the data disk level.", - "settings_swap_dialog_title": "Adjust additional swap space", - "settings_swap_dialog_desc": "Swap capacity roughly equals how much sandbox memory can hibernate at once (freezing squeezes memory into swap). Growing takes effect immediately; shrinking waits for the next host reboot — in-use swap is never force-removed, as that would pull all hibernating sandbox memory back into RAM.", - "settings_swap_saved": "Additional swap target changed to {value} GiB", - "settings_swap_label": "Total Dormice-managed swap (GiB)", - "settings_swap_field_desc": "Added on top of system swap; 0 = none. The swap file really occupies data disk space (it is not sparse), so check the data disk level on the overview page before growing it.", "settings_pids_dialog_title": "Adjust sandbox pids cap", "settings_pids_dialog_desc": "Under gVisor this is not a process count the sandbox can see; it is the sandbox's host-side process + thread budget. Hitting it gives nothing inside an error: the whole sandbox exits at once (exit code 2, not an OOM). Takes effect on save: new containers are born with the new value, running sandboxes are brought to it in place right away (a host-side cgroup write their processes never notice), and frozen or stopped sandboxes adopt it at their next wake. No rebuild, no cold start.", "settings_pids_saved": "Sandbox pids cap changed to {value}", diff --git a/packages/console/messages/es/settings.json b/packages/console/messages/es/settings.json index 39137890..f4c982c1 100644 --- a/packages/console/messages/es/settings.json +++ b/packages/console/messages/es/settings.json @@ -50,13 +50,8 @@ "settings_row_defaults": "Cuotas predeterminadas de los sandboxes nuevos", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB de memoria · {disk} GiB de disco", "settings_row_policy": "Política de ciclo de vida predeterminada", - "settings_row_swap": "Espacio de swap adicional", "settings_row_pids": "Límite pids por sandbox", "settings_row_pids_value": "{n} (presupuesto de procesos y hilos del sandbox en el host; al alcanzarlo, todo el sandbox termina)", - "settings_swap_unsupported": "Este host no lo admite (requiere un host Linux + ejecutor docker)", - "settings_swap_line_shrink": "Objetivo {target} GiB · montado {active} GiB — la reducción se aplica al próximo reinicio del host", - "settings_swap_line_grow": "Objetivo {target} GiB · montado {active} GiB — la ampliación quedó incompleta, revisa los registros del daemon", - "settings_swap_line_ok": "{target} GiB (sin contar el swap del sistema)", "settings_defaults_dialog_title": "Ajustar las cuotas predeterminadas de los sandboxes nuevos", "settings_defaults_dialog_desc": "La CPU y la memoria se aplican en el próximo nacimiento de un contenedor (incluidos los sandboxes existentes que arrancan en frío tras una parada); el tamaño del disco queda fijado al nacer el disco (primera creación y restauración de archivado) — el disco es el sandbox en sí y nunca se redimensiona en el sitio, así que antes de bajarlo asegúrate de que no quede por debajo del contenido real de un sandbox archivado.", "settings_defaults_saved": "Cuotas predeterminadas de los sandboxes nuevos actualizadas", @@ -65,11 +60,6 @@ "settings_defaults_memory_desc": "Un OOM dentro del sandbox hace salir todo el sandbox (la reconciliación lo recupera), así que deja margen al agente.", "settings_defaults_disk_label": "Disco (GiB)", "settings_defaults_disk_desc": "Cuota nominal: las imágenes dispersas solo pagan por lo que se escribe de verdad; la sobreventa se vigila con el nivel del disco de datos.", - "settings_swap_dialog_title": "Ajustar el espacio de swap adicional", - "settings_swap_dialog_desc": "La capacidad de swap equivale más o menos a cuánta memoria de sandboxes puede hibernar a la vez (al congelar, la memoria se comprime hacia swap). Ampliarlo se aplica de inmediato; reducirlo espera al próximo reinicio del host — el swap en uso nunca se desmonta a la fuerza, porque eso devolvería a la RAM toda la memoria de los sandboxes en hibernación.", - "settings_swap_saved": "El objetivo de swap adicional cambió a {value} GiB", - "settings_swap_label": "Swap total gestionado por Dormice (GiB)", - "settings_swap_field_desc": "Se añade por encima del swap del sistema; 0 = ninguno. El archivo de swap ocupa espacio real del disco de datos (no es disperso), así que revisa el nivel del disco de datos en el panel antes de ampliarlo.", "settings_pids_dialog_title": "Ajustar el límite pids del sandbox", "settings_pids_dialog_desc": "Bajo gVisor no es un número de procesos visible dentro del sandbox, sino su presupuesto de procesos e hilos en el host. Al alcanzarlo nada dentro recibe un error: todo el sandbox termina de inmediato (código de salida 2, no OOM). Se aplica al guardar: los contenedores nuevos nacen con el nuevo valor, los sandboxes en ejecución se llevan a él en su sitio de inmediato (una escritura de cgroup en el host que sus procesos no notan) y los sandboxes congelados o detenidos lo adoptan en su siguiente despertar. Sin reconstrucción ni arranque en frío.", "settings_pids_saved": "Límite pids del sandbox cambiado a {value}", diff --git a/packages/console/messages/fr/settings.json b/packages/console/messages/fr/settings.json index ae306160..97ae53fe 100644 --- a/packages/console/messages/fr/settings.json +++ b/packages/console/messages/fr/settings.json @@ -50,13 +50,8 @@ "settings_row_defaults": "Quotas par défaut des nouvelles sandbox", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB de mémoire · {disk} GiB de disque", "settings_row_policy": "Politique de cycle de vie par défaut", - "settings_row_swap": "Espace swap supplémentaire", "settings_row_pids": "Limite pids par sandbox", "settings_row_pids_value": "{n} (budget de processus et threads de la sandbox côté hôte ; l'atteindre termine toute la sandbox)", - "settings_swap_unsupported": "Non pris en charge sur cet hôte (requiert un hôte Linux + l'exécuteur docker)", - "settings_swap_line_shrink": "Cible {target} GiB · monté {active} GiB — la réduction prend effet au prochain redémarrage de l'hôte", - "settings_swap_line_grow": "Cible {target} GiB · monté {active} GiB — extension incomplète, voir les journaux du daemon", - "settings_swap_line_ok": "{target} GiB (hors swap système)", "settings_defaults_dialog_title": "Ajuster les quotas par défaut des nouvelles sandbox", "settings_defaults_dialog_desc": "CPU et mémoire prennent effet à la prochaine naissance d'un conteneur (y compris les sandbox existantes redémarrées à froid après un arrêt) ; la taille du disque est figée à la naissance du disque (première création et restauration d'archive) — le disque est la sandbox elle-même et n'est jamais redimensionné en place ; avant de la réduire, vérifiez qu'elle n'est pas inférieure au contenu réel d'une sandbox archivée.", "settings_defaults_saved": "Quotas par défaut des nouvelles sandbox mis à jour", @@ -65,11 +60,6 @@ "settings_defaults_memory_desc": "Un OOM dans la sandbox fait sortir toute la sandbox (la réconciliation la récupère) ; laissez de la marge à l'agent.", "settings_defaults_disk_label": "Disque (GiB)", "settings_defaults_disk_desc": "Quota nominal : les images sparse ne paient que les écritures réelles ; la survente se surveille via le niveau du disque de données.", - "settings_swap_dialog_title": "Ajuster l'espace swap supplémentaire", - "settings_swap_dialog_desc": "La capacité de swap correspond à peu près à la quantité de mémoire de sandbox pouvant hiberner en même temps (le gel pousse la mémoire dans le swap). L'augmentation prend effet immédiatement ; la réduction attend le prochain redémarrage de l'hôte — un swap en cours d'utilisation n'est jamais démonté de force, cela ramènerait toute la mémoire des sandbox hibernées en RAM.", - "settings_swap_saved": "Cible de swap supplémentaire changée à {value} GiB", - "settings_swap_label": "Swap total géré par Dormice (GiB)", - "settings_swap_field_desc": "S'ajoute au swap système ; 0 = aucun. Le fichier de swap occupe réellement le disque de données (il n'est pas sparse) : vérifiez le niveau du disque de données sur le tableau de bord avant de l'augmenter.", "settings_pids_dialog_title": "Ajuster la limite pids de la sandbox", "settings_pids_dialog_desc": "Sous gVisor ce n'est pas un nombre de processus visible dans la sandbox, mais son budget de processus et threads côté hôte. L'atteindre ne renvoie aucune erreur à l'intérieur : toute la sandbox se termine aussitôt (code de sortie 2, pas un OOM). Prend effet à l'enregistrement : les nouveaux conteneurs naissent avec la nouvelle valeur, les sandboxes en cours d'exécution y sont amenées sur place immédiatement (une écriture cgroup côté hôte que leurs processus ne remarquent pas), et les sandboxes gelées ou arrêtées l'adoptent à leur prochain réveil. Sans reconstruction ni démarrage à froid.", "settings_pids_saved": "Limite pids de la sandbox changée à {value}", diff --git a/packages/console/messages/ja/settings.json b/packages/console/messages/ja/settings.json index 78d1b957..5040d668 100644 --- a/packages/console/messages/ja/settings.json +++ b/packages/console/messages/ja/settings.json @@ -50,13 +50,8 @@ "settings_row_defaults": "新規サンドボックスの既定クォータ", "settings_row_defaults_value": "{cpus} CPU · メモリ {memory} GiB · ディスク {disk} GiB", "settings_row_policy": "既定のライフサイクルポリシー", - "settings_row_swap": "追加 swap 領域", "settings_row_pids": "サンドボックスの pids 上限", "settings_row_pids_value": "{n}(サンドボックスのホスト側プロセス+スレッド予算。上限到達で全体が終了)", - "settings_swap_unsupported": "このホストでは非対応です(Linux ホスト + docker エグゼキューターが必要)", - "settings_swap_line_shrink": "目標 {target} GiB · 現在のマウント {active} GiB — 縮小は次回のホスト再起動時に反映されます", - "settings_swap_line_grow": "目標 {target} GiB · 現在のマウント {active} GiB — 拡張が未完了です。詳細は daemon のログを参照してください", - "settings_swap_line_ok": "{target} GiB(システム標準の swap は別枠)", "settings_defaults_dialog_title": "新規サンドボックスの既定クォータの調整", "settings_defaults_dialog_desc": "CPU とメモリは次にコンテナが生成されるときに反映されます(停止後にコールドスタートする既存サンドボックスを含む)。ディスクはディスク作成時に確定します(初回作成とアーカイブからの復元)— ディスクはサンドボックスの本体であり、その場でのサイズ変更は決して行われないため、縮小する前にアーカイブ済みサンドボックスの実際の内容量を下回らないか確認してください。", "settings_defaults_saved": "新規サンドボックスの既定クォータを更新しました", @@ -65,11 +60,6 @@ "settings_defaults_memory_desc": "サンドボックス内で上限を超えて OOM になるとサンドボックス全体が終了します(整合処理で復旧)。エージェントに少し余裕を残してください。", "settings_defaults_disk_label": "ディスク(GiB)", "settings_defaults_disk_desc": "名目上のクォータです:スパースイメージは実際に書き込まれた分だけ消費し、オーバーコミットはデータディスク残量の観察で管理します。", - "settings_swap_dialog_title": "追加 swap 領域の調整", - "settings_swap_dialog_desc": "swap の容量 ≈ 同時に冬眠できるサンドボックスメモリの量です(凍結はメモリを swap に押し出します)。拡張は即時反映、縮小は次回のホスト再起動待ちです — 使用中の swap を強制的に外すことはありません。外すと冬眠中のサンドボックスのメモリがすべて物理メモリに引き戻されてしまいます。", - "settings_swap_saved": "追加 swap の目標値を {value} GiB に変更しました", - "settings_swap_label": "Dormice が管理する swap の総量(GiB)", - "settings_swap_field_desc": "システム標準の swap に加えて追加します。0 = 追加しない。swap ファイルはデータディスクを実際に消費するため(スパースではありません)、拡張前に概要ページのデータディスク残量を確認してください。", "settings_pids_dialog_title": "サンドボックスの pids 上限を調整", "settings_pids_dialog_desc": "gVisor ではこれはサンドボックス内から見えるプロセス数ではなく、ホスト側でのプロセス+スレッド予算です。上限に達しても内部にはエラーが届かず、サンドボックス全体が即時に終了します(終了コード 2、OOM ではありません)。保存すると直ちに反映されます:新しいコンテナは新しい値で生まれ、実行中のサンドボックスはその場で即座に追従し(ホスト側の cgroup 書き込みで、内部のプロセスは気付きません)、凍結中・停止中のサンドボックスは次の復帰時に追従します。再構築もコールドスタートも不要です。", "settings_pids_saved": "サンドボックスの pids 上限を {value} に変更しました", diff --git a/packages/console/messages/ko/settings.json b/packages/console/messages/ko/settings.json index 16aeeba4..56af2b6a 100644 --- a/packages/console/messages/ko/settings.json +++ b/packages/console/messages/ko/settings.json @@ -50,13 +50,8 @@ "settings_row_defaults": "새 샌드박스 기본 할당량", "settings_row_defaults_value": "{cpus} CPU · 메모리 {memory} GiB · 디스크 {disk} GiB", "settings_row_policy": "기본 수명 주기 정책", - "settings_row_swap": "추가 swap 공간", "settings_row_pids": "샌드박스 pids 상한", "settings_row_pids_value": "{n}(샌드박스의 호스트 측 프로세스+스레드 예산, 상한 도달 시 전체 종료)", - "settings_swap_unsupported": "이 호스트에서는 지원되지 않음(Linux 호스트 + docker 실행기 필요)", - "settings_swap_line_shrink": "목표 {target} GiB · 현재 마운트 {active} GiB — 축소는 다음 호스트 재부팅 시 적용", - "settings_swap_line_grow": "목표 {target} GiB · 현재 마운트 {active} GiB — 확장 미완료, daemon 로그 참조", - "settings_swap_line_ok": "{target} GiB(시스템 자체 swap 별도)", "settings_defaults_dialog_title": "새 샌드박스의 기본 할당량 조정", "settings_defaults_dialog_desc": "CPU/메모리는 다음에 컨테이너가 태어날 때 적용됩니다(중지 후 콜드 스타트되는 기존 샌드박스 포함). 디스크는 디스크가 태어날 때 확정됩니다(최초 생성과 아카이브 복원) — 디스크는 샌드박스의 본체라서 절대 제자리에서 크기를 바꾸지 않으니, 줄이기 전에 아카이브된 샌드박스의 실제 내용보다 작아지지 않는지 확인하세요.", "settings_defaults_saved": "새 샌드박스 기본 할당량이 업데이트되었습니다", @@ -65,11 +60,6 @@ "settings_defaults_memory_desc": "샌드박스 안에서 한도를 넘는 OOM은 샌드박스 전체를 종료시킵니다(정합 복구로 되살림). 에이전트에게 여유를 남겨 두세요.", "settings_defaults_disk_label": "디스크(GiB)", "settings_defaults_disk_desc": "명목 할당량: 희소 이미지는 실제로 쓴 만큼만 차지하며, 초과 판매는 데이터 디스크 수위로 관찰합니다.", - "settings_swap_dialog_title": "추가 swap 공간 조정", - "settings_swap_dialog_desc": "swap 용량 ≈ 동시에 겨울잠 잘 수 있는 샌드박스 메모리 총량(동결은 메모리를 swap으로 밀어 넣습니다). 늘리기는 즉시 적용되고, 줄이기는 다음 호스트 재부팅을 기다립니다 — 사용 중인 swap은 절대 강제로 떼어내지 않습니다. 그러면 겨울잠 중인 샌드박스의 메모리가 전부 물리 메모리로 끌려 나오기 때문입니다.", - "settings_swap_saved": "추가 swap 목표가 {value} GiB로 변경되었습니다", - "settings_swap_label": "Dormice가 관리하는 swap 총량(GiB)", - "settings_swap_field_desc": "시스템 자체 swap에 더해 추가합니다. 0 = 추가 안 함. swap 파일은 데이터 디스크 공간을 실제로 차지하므로(희소 아님), 늘리기 전에 개요 페이지의 데이터 디스크 수위를 확인하세요.", "settings_pids_dialog_title": "샌드박스 pids 상한 조정", "settings_pids_dialog_desc": "gVisor에서 이 값은 샌드박스 안에서 보이는 프로세스 수가 아니라 호스트 측 프로세스+스레드 예산입니다. 상한에 닿아도 내부에는 어떤 오류도 전달되지 않고 샌드박스 전체가 즉시 종료됩니다(종료 코드 2, OOM 아님). 저장 즉시 적용됩니다: 새 컨테이너는 새 값으로 생성되고, 실행 중인 샌드박스는 그 자리에서 바로 따라가며(호스트 측 cgroup 쓰기라 내부 프로세스는 알아차리지 못함), 동결·중지된 샌드박스는 다음 깨어남 때 따라갑니다. 재구축도 콜드 스타트도 없습니다.", "settings_pids_saved": "샌드박스 pids 상한이 {value}로 변경되었습니다", diff --git a/packages/console/messages/pt-BR/settings.json b/packages/console/messages/pt-BR/settings.json index c2c39cf5..f377bf6a 100644 --- a/packages/console/messages/pt-BR/settings.json +++ b/packages/console/messages/pt-BR/settings.json @@ -50,13 +50,8 @@ "settings_row_defaults": "Cotas padrão de novos sandboxes", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB de memória · {disk} GiB de disco", "settings_row_policy": "Política padrão de ciclo de vida", - "settings_row_swap": "Espaço de swap adicional", "settings_row_pids": "Limite pids por sandbox", "settings_row_pids_value": "{n} (orçamento de processos e threads do sandbox no host; ao atingi-lo, todo o sandbox encerra)", - "settings_swap_unsupported": "Sem suporte neste host (requer host Linux + executor docker)", - "settings_swap_line_shrink": "Alvo {target} GiB · montado {active} GiB — a redução entra em vigor no próximo reinício do host", - "settings_swap_line_grow": "Alvo {target} GiB · montado {active} GiB — crescimento incompleto, veja os logs do daemon", - "settings_swap_line_ok": "{target} GiB (swap do sistema não incluído)", "settings_defaults_dialog_title": "Ajustar cotas padrão de novos sandboxes", "settings_defaults_dialog_desc": "CPU/memória entram em vigor no próximo nascimento de contêiner (inclusive sandboxes existentes que fazem cold start após parar); o tamanho do disco é fixado quando o disco nasce (primeira criação e restauração de arquivamento) — o disco é o próprio sandbox e nunca é redimensionado no lugar; antes de reduzir, garanta que não fique menor que o conteúdo real de um sandbox arquivado.", "settings_defaults_saved": "Cotas padrão de novos sandboxes atualizadas", @@ -65,11 +60,6 @@ "settings_defaults_memory_desc": "Um OOM dentro do sandbox encerra o sandbox inteiro (a reconciliação o recupera), então deixe alguma folga para o agente.", "settings_defaults_disk_label": "Disco (GiB)", "settings_defaults_disk_desc": "Cota nominal: imagens esparsas só pagam por gravações reais; o excesso de venda é vigiado pelo nível do disco de dados.", - "settings_swap_dialog_title": "Ajustar espaço de swap adicional", - "settings_swap_dialog_desc": "A capacidade de swap equivale, grosso modo, a quanta memória de sandbox pode hibernar ao mesmo tempo (congelar empurra a memória para o swap). Aumentar entra em vigor imediatamente; reduzir espera o próximo reinício do host — swap em uso nunca é removido à força, pois isso puxaria toda a memória dos sandboxes hibernando de volta para a RAM.", - "settings_swap_saved": "Alvo de swap adicional alterado para {value} GiB", - "settings_swap_label": "Total de swap gerenciado pelo Dormice (GiB)", - "settings_swap_field_desc": "Somado ao swap do sistema; 0 = nenhum. O arquivo de swap ocupa espaço real no disco de dados (não é esparso), então confira o nível do disco de dados no painel antes de aumentar.", "settings_pids_dialog_title": "Ajustar o limite pids do sandbox", "settings_pids_dialog_desc": "No gVisor isto não é uma contagem de processos visível dentro do sandbox, e sim seu orçamento de processos e threads no host. Ao atingi-lo nada dentro recebe erro: todo o sandbox encerra na hora (código de saída 2, não OOM). Vale ao salvar: contêineres novos nascem com o novo valor, sandboxes em execução são levados a ele no lugar na mesma hora (uma escrita de cgroup no host que seus processos nem percebem) e sandboxes congelados ou parados o adotam no próximo despertar. Sem reconstrução e sem cold start.", "settings_pids_saved": "Limite pids do sandbox alterado para {value}", diff --git a/packages/console/messages/ru/settings.json b/packages/console/messages/ru/settings.json index 0bcf1e76..1d4c79c3 100644 --- a/packages/console/messages/ru/settings.json +++ b/packages/console/messages/ru/settings.json @@ -50,13 +50,8 @@ "settings_row_defaults": "Квоты новых песочниц по умолчанию", "settings_row_defaults_value": "{cpus} CPU · {memory} ГиБ памяти · {disk} ГиБ диска", "settings_row_policy": "Политика жизненного цикла по умолчанию", - "settings_row_swap": "Дополнительный swap", "settings_row_pids": "Лимит pids на песочницу", "settings_row_pids_value": "{n} (бюджет процессов и потоков песочницы на хосте; при достижении вся песочница завершается)", - "settings_swap_unsupported": "На этом хосте не поддерживается (нужен Linux-хост и исполнитель docker)", - "settings_swap_line_shrink": "Цель {target} ГиБ · подключено {active} ГиБ — уменьшение вступит в силу после следующей перезагрузки хоста", - "settings_swap_line_grow": "Цель {target} ГиБ · подключено {active} ГиБ — увеличение не завершено, подробности в логах daemon", - "settings_swap_line_ok": "{target} ГиБ (системный swap не считается)", "settings_defaults_dialog_title": "Квоты новых песочниц по умолчанию", "settings_defaults_dialog_desc": "CPU и память применяются при следующем рождении контейнера (включая холодный старт существующих песочниц после остановки); размер диска фиксируется при рождении диска (первое создание и восстановление из архива) — диск и есть песочница, на месте он никогда не меняется, поэтому перед уменьшением убедитесь, что новый размер не меньше реального содержимого архивированных песочниц.", "settings_defaults_saved": "Квоты новых песочниц по умолчанию обновлены", @@ -65,11 +60,6 @@ "settings_defaults_memory_desc": "OOM внутри песочницы завершает её целиком (сверка восстановит), оставьте агенту запас.", "settings_defaults_disk_label": "Диск (ГиБ)", "settings_defaults_disk_desc": "Номинальная квота: разреженный образ платит только за реальные записи; за переподпиской следите по уровню диска данных.", - "settings_swap_dialog_title": "Дополнительный swap", - "settings_swap_dialog_desc": "Объём swap ≈ сколько памяти песочниц может одновременно впасть в спячку (заморозка вытесняет память в swap). Увеличение действует сразу; уменьшение ждёт следующей перезагрузки хоста — используемый swap никогда не отключается силой, иначе память всех спящих песочниц разом вернулась бы в RAM.", - "settings_swap_saved": "Целевой объём дополнительного swap изменён на {value} ГиБ", - "settings_swap_label": "Общий swap под управлением Dormice (ГиБ)", - "settings_swap_field_desc": "Добавляется поверх системного swap; 0 = не добавлять. Файл swap реально занимает место на диске данных (он не разреженный), перед увеличением взгляните на уровень диска данных на дашборде.", "settings_pids_dialog_title": "Изменить лимит pids песочницы", "settings_pids_dialog_desc": "В gVisor это не число процессов, видимое внутри песочницы, а её бюджет процессов и потоков на стороне хоста. При достижении ничего внутри не получает ошибку: вся песочница завершается сразу (код выхода 2, не OOM). Действует сразу после сохранения: новые контейнеры рождаются с новым значением, работающие песочницы тут же переводятся на него на месте (запись в cgroup на стороне хоста, которую их процессы не замечают), а замороженные или остановленные подхватывают его при следующем пробуждении. Без пересборки и холодного старта.", "settings_pids_saved": "Лимит pids песочницы изменён на {value}", diff --git a/packages/console/messages/zh-CN/settings.json b/packages/console/messages/zh-CN/settings.json index 73910de9..73be9c5c 100644 --- a/packages/console/messages/zh-CN/settings.json +++ b/packages/console/messages/zh-CN/settings.json @@ -50,13 +50,8 @@ "settings_row_defaults": "新沙箱默认配额", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB 内存 · {disk} GiB 磁盘", "settings_row_policy": "默认生命周期策略", - "settings_row_swap": "追加 swap 空间", "settings_row_pids": "沙箱 pids 上限", "settings_row_pids_value": "{n}(整箱在宿主侧的进程+线程预算,撞顶即整箱退出)", - "settings_swap_unsupported": "本宿主不支持(需要 Linux 宿主 + docker 执行器)", - "settings_swap_line_shrink": "目标 {target} GiB · 当前挂载 {active} GiB — 缩容在下次重启宿主时生效", - "settings_swap_line_grow": "目标 {target} GiB · 当前挂载 {active} GiB — 增容未完成,详见 daemon 日志", - "settings_swap_line_ok": "{target} GiB(系统自带 swap 另计)", "settings_defaults_dialog_title": "调整新沙箱的默认配额", "settings_defaults_dialog_desc": "CPU/内存在下一次容器出生时生效(含停止后冷启动的存量沙箱);磁盘在磁盘出生时定型(首次创建与归档恢复) — 磁盘是沙箱的本体,永不原地改尺寸,调小前注意别小于归档沙箱的真实内容。", "settings_defaults_saved": "新沙箱默认配额已更新", @@ -65,11 +60,6 @@ "settings_defaults_memory_desc": "沙箱内超限 OOM 会让整箱退出(对账救回),给 agent 留点余量。", "settings_defaults_disk_label": "磁盘(GiB)", "settings_defaults_disk_desc": "名义配额:稀疏镜像只为真实写入付费,超卖靠数据盘水位观察。", - "settings_swap_dialog_title": "调整追加 swap 空间", - "settings_swap_dialog_desc": "swap 容量 ≈ 能同时冬眠多少沙箱内存(冻结把内存挤进 swap)。调大立即生效;调小要等下次重启宿主 — 正在用的 swap 绝不强拆,那会把冬眠沙箱的内存全拽回物理内存。", - "settings_swap_saved": "追加 swap 目标已改为 {value} GiB", - "settings_swap_label": "Dormice 管理的 swap 总量(GiB)", - "settings_swap_field_desc": "在系统自带 swap 之外追加,0 = 不追加。swap 文件真实占据数据盘空间(不是稀疏的),调大前看一眼总览页的数据盘水位。", "settings_pids_dialog_title": "调整沙箱 pids 上限", "settings_pids_dialog_desc": "gVisor 下这不是沙箱内能看见的进程数,而是整箱在宿主侧的进程+线程预算:撞顶时沙箱内收不到任何错误,整箱瞬间退出(退出码 2,非 OOM)。保存即生效:新建容器按新值出生,运行中的沙箱当场就地跟上(宿主侧 cgroup 写入,箱内进程无感),冻结/停止的沙箱在下一次唤醒时跟上;全程不重建、不冷启动。", "settings_pids_saved": "沙箱 pids 上限已改为 {value}", diff --git a/packages/console/messages/zh-TW/settings.json b/packages/console/messages/zh-TW/settings.json index dd4d3250..7ba5a4f5 100644 --- a/packages/console/messages/zh-TW/settings.json +++ b/packages/console/messages/zh-TW/settings.json @@ -50,13 +50,8 @@ "settings_row_defaults": "新沙箱預設配額", "settings_row_defaults_value": "{cpus} CPU · {memory} GiB 記憶體 · {disk} GiB 磁碟", "settings_row_policy": "預設生命週期策略", - "settings_row_swap": "追加 swap 空間", "settings_row_pids": "沙箱 pids 上限", "settings_row_pids_value": "{n}(整箱在宿主側的程序+執行緒預算,撞頂即整箱退出)", - "settings_swap_unsupported": "本主機不支援(需要 Linux 主機 + docker 執行器)", - "settings_swap_line_shrink": "目標 {target} GiB · 目前掛載 {active} GiB — 縮容在下次重啟主機時生效", - "settings_swap_line_grow": "目標 {target} GiB · 目前掛載 {active} GiB — 增容未完成,詳見 daemon 日誌", - "settings_swap_line_ok": "{target} GiB(系統內建 swap 另計)", "settings_defaults_dialog_title": "調整新沙箱的預設配額", "settings_defaults_dialog_desc": "CPU/記憶體在下一次容器出生時生效(含停止後冷啟動的既有沙箱);磁碟在磁碟出生時定型(首次建立與封存還原)— 磁碟是沙箱的本體,永不原地改尺寸,調小前注意別小於已封存沙箱的真實內容。", "settings_defaults_saved": "新沙箱預設配額已更新", @@ -65,11 +60,6 @@ "settings_defaults_memory_desc": "沙箱內超限 OOM 會讓整箱退出(對帳救回),給 agent 留點餘裕。", "settings_defaults_disk_label": "磁碟(GiB)", "settings_defaults_disk_desc": "名目配額:稀疏映像檔只為真實寫入付費,超賣靠資料磁碟水位觀察。", - "settings_swap_dialog_title": "調整追加 swap 空間", - "settings_swap_dialog_desc": "swap 容量 ≈ 能同時冬眠多少沙箱記憶體(凍結把記憶體擠入 swap)。調大立即生效;調小要等下次重啟主機 — 正在用的 swap 絕不強拆,那會把冬眠沙箱的記憶體全拽回實體記憶體。", - "settings_swap_saved": "追加 swap 目標已改為 {value} GiB", - "settings_swap_label": "Dormice 管理的 swap 總量(GiB)", - "settings_swap_field_desc": "在系統內建 swap 之外追加,0 = 不追加。swap 檔案真實佔據資料磁碟空間(不是稀疏的),調大前看一眼總覽頁的資料磁碟水位。", "settings_pids_dialog_title": "調整沙箱 pids 上限", "settings_pids_dialog_desc": "gVisor 下這不是沙箱內能看見的程序數,而是整箱在宿主側的程序+執行緒預算:撞頂時沙箱內收不到任何錯誤,整箱瞬間退出(退出碼 2,非 OOM)。儲存即生效:新建容器按新值出生,執行中的沙箱當場就地跟上(宿主側 cgroup 寫入,箱內程序無感),凍結/停止的沙箱在下一次喚醒時跟上;全程不重建、不冷啟動。", "settings_pids_saved": "沙箱 pids 上限已改為 {value}", diff --git a/packages/console/src/features/settings/components/RuntimeSettingsCard.tsx b/packages/console/src/features/settings/components/RuntimeSettingsCard.tsx index 27f18ddf..cab7f401 100644 --- a/packages/console/src/features/settings/components/RuntimeSettingsCard.tsx +++ b/packages/console/src/features/settings/components/RuntimeSettingsCard.tsx @@ -34,12 +34,11 @@ import { useUpdateSettings } from '../hooks/useUpdateSettings'; * 一个弹窗,给哪组就整组替换(updatePolicy 的规矩:界面上看到什么就写 * 下什么)。改的是"之后"不是"已经":容量上限管下一次创建,默认配额管 * 下一次出生的磁盘/容器,默认策略管下一次 acquire 创建的沙箱 — 存量 - * 沙箱一根汗毛都不动,这句话在每个弹窗里都说清。两个例外:swap 改的是 - * 宿主不是沙箱,增容立即、缩容等重启(swapLine 负责把这个时间差摆在 - * 明面上);pids 上限反而会触达存量沙箱 — 保存时就地扫一遍运行中的壳 - * (docker update,箱内无感),冻结/停止的在下一次唤醒跟上,都不重建。 - * 归档存储与沙箱域名不在这张卡:前者是独立的 - * 归档卡(六字段撑不进一行的形制),后者语义归域名页。 + * 沙箱一根汗毛都不动,这句话在每个弹窗里都说清。一个例外:pids 上限会 + * 触达存量沙箱 — 保存时就地扫一遍运行中的壳(docker update,箱内无感), + * 冻结/停止的在下一次唤醒跟上,都不重建。归档存储与沙箱域名不在这张卡: + * 前者是独立的归档卡(六字段撑不进一行的形制),后者语义归域名页。追加 + * swap 是每台机器自己的旋钮,2026-09-14 随集群刀 2 搬去节点页(刀 3)。 */ function EditRow({ @@ -259,88 +258,6 @@ function PidsLimitDialog({ settings }: { settings: RuntimeSettings }) { ); } -function SwapDialog({ settings }: { settings: RuntimeSettings }) { - const [open, setOpen] = useState(false); - const [value, setValue] = useState(''); - const { pending, error, setError, submit } = useUpdateSettings(() => - setOpen(false), - ); - - const valid = - value.trim() !== '' && - Number.isInteger(Number(value)) && - Number(value) >= 0; - - return ( - { - setOpen(next); - if (next) { - setValue(String(settings.swapGb)); - setError(null); - } - }} - > - - - - {m.settings_swap_dialog_title()} - {m.settings_swap_dialog_desc()} - -
{ - event.preventDefault(); - void submit( - { swapGb: Number(value) }, - m.settings_swap_saved({ value: Number(value) }), - ); - }} - > - - - - {m.settings_swap_label()} - - setValue(event.target.value)} - /> - - {m.settings_swap_field_desc()} - - - {error && {error}} - - - - -
- - - ); -} - -/** - * swap 行的真话:目标与现实一致时一句话完事;缩容等重启、增容没跑完时 - * 把两个数都摆出来 — 只报目标会在这两种时刻撒谎。 - */ -function swapLine(targetGb: number, activeGb: number): string { - if (activeGb > targetGb) { - return m.settings_swap_line_shrink({ target: targetGb, active: activeGb }); - } - if (activeGb < targetGb) { - return m.settings_swap_line_grow({ target: targetGb, active: activeGb }); - } - return m.settings_swap_line_ok({ target: targetGb }); -} - function DefaultPolicyDialog({ settings, archiveEnabled, @@ -540,17 +457,6 @@ export function RuntimeSettingsCard({ data }: { data: GetConfigResponse }) { value={m.settings_row_pids_value({ n: settings.pidsLimit })} dialog={} /> - : undefined - } - />
); diff --git a/packages/console/src/features/settings/hooks/useUpdateSettings.ts b/packages/console/src/features/settings/hooks/useUpdateSettings.ts index fc25a2cf..d41a230a 100644 --- a/packages/console/src/features/settings/hooks/useUpdateSettings.ts +++ b/packages/console/src/features/settings/hooks/useUpdateSettings.ts @@ -6,7 +6,7 @@ import { queryClient } from '@/lib/queryClient'; /** * updateSettings 的提交半件,运营旋钮卡与归档卡共用:pending/error 状态 - * + 成功 toast 关窗。失败也刷新 config — swap 的 500 语义是"目标已存但 + * + 成功 toast 关窗。失败也刷新 config — pids 上限的 500 语义是"目标已存但 * 应用失败",账本真的变了,行里必须立刻说真话;设置页读 config,总览的 * 容量卡走 getHostMetrics 自己的轮询。 */ diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 2b8783e6..b0c2f63e 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -35,7 +35,6 @@ import { settingsRoutes } from './routes/settings'; import { templateRoutes } from './routes/templates'; import { upgradeRoutes } from './routes/upgrade'; import { createSandboxProxy } from './sandbox-proxy'; -import type { SwapControl } from './swap'; import { Updater } from './updater'; import { readBuildInfo } from './version'; @@ -78,12 +77,6 @@ export interface AppDeps { * getIngress answers { managed: false } and setIngress refuses. */ ingress?: Ingress; - /** - * The managed-swap surface, present exactly when the daemon can manage - * swap — main.ts builds one on Linux with the docker executor. Absent, - * getConfig reports { supported: false } and a swapGb patch is refused. - */ - swap?: SwapControl; /** * Test seam over updateSettings' S3 round-trip probe (routes/settings.ts) * — a unit test forges S3's answers instead of needing a live store. @@ -125,7 +118,6 @@ export function buildApp({ consoleDistDir, archiver, ingress, - swap, probeS3, sources = configSources(), updater = new Updater({ @@ -276,12 +268,7 @@ export function buildApp({ await api.register(templateRoutes, { db }); await api.register(hostRoutes, { config, db, executor }); await api.register(ingressRoutes, { ingress }); - await api.register(configRoutes, { - config, - db, - sources, - swap, - }); + await api.register(configRoutes, { config, db, sources }); await api.register(upgradeRoutes, { updater }); await api.register(envdTokenRoutes, { envdSigningSecret }); }); @@ -298,7 +285,6 @@ export function buildApp({ db, executor, locks, - swap, ...(probeS3 ? { probeS3 } : {}), }); }); diff --git a/packages/server/src/db/settings.ts b/packages/server/src/db/settings.ts index 65e7e93d..99c323cd 100644 --- a/packages/server/src/db/settings.ts +++ b/packages/server/src/db/settings.ts @@ -151,7 +151,6 @@ function toView(row: RuntimeSettingsRow): RuntimeSettings { stopAfterSeconds: row.defaultStopAfterSeconds, archiveAfterSeconds: row.defaultArchiveAfterSeconds, }, - swapGb: row.swapGb, s3: row.s3Endpoint === OFF ? null @@ -172,6 +171,15 @@ function toView(row: RuntimeSettingsRow): RuntimeSettings { }; } +/** + * The node's managed-swap target — a knob of this machine, not of the + * fleet, so it left the settings wire (shared/settings.ts) and is read by + * the one consumer that acts on it, the boot reconcile in main.ts. + */ +export function readSwapTarget(db: Db): number { + return readRow(db).swapGb; +} + /** * The knobs in force, read fresh at each use site — a better-sqlite3 point * read costs microseconds, and reading live is what makes a console edit @@ -253,7 +261,6 @@ export function writeRuntimeSettings( defaultArchiveAfterSeconds: patch.defaultPolicy.archiveAfterSeconds, } : {}), - ...(patch.swapGb !== undefined ? { swapGb: patch.swapGb } : {}), ...(patch.s3 !== undefined ? s3Columns(patch.s3) : {}), ...(patch.sandboxDomain !== undefined ? { sandboxDomain: patch.sandboxDomain ?? OFF } diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index edc0c972..5f290ea5 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -10,7 +10,11 @@ import { type Config, loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; import { listSandboxes } from './db/ledger'; import { acquireSingleWriterLock } from './db/lock'; -import { ensureRuntimeSettings, readRuntimeSettings } from './db/settings'; +import { + ensureRuntimeSettings, + readRuntimeSettings, + readSwapTarget, +} from './db/settings'; import { WatcherTable } from './e2b/watcher-table'; import { DockerExecutor } from './executor/docker'; import type { Executor } from './executor/executor'; @@ -184,8 +188,10 @@ if (config.DORMICE_INGRESS_FILE) { // is a test double — e2e boots real daemons with it, and those must never // touch the host's swap). The boot reconcile is what makes shrink-by- // reboot converge and puts grown blocks back after a restart; its failure -// is loud but not fatal — swap is capacity, not correctness, and -// getConfig's swap.activeGb reports the shortfall honestly. +// is loud but not fatal — swap is capacity, not correctness. The target is +// this node's row of the fleet configuration (the gateway's +// updateNodeSettings), applied here at boot and, once the node pulls its +// configuration, whenever the bundle moves it. let swap: SwapManager | undefined; if (config.DORMICE_EXECUTOR === 'docker' && process.platform === 'linux') { swap = new SwapManager({ @@ -193,7 +199,7 @@ if (config.DORMICE_EXECUTOR === 'docker' && process.platform === 'linux') { log: (msg) => log.info(msg), }); try { - await swap.reconcile(readRuntimeSettings(db).swapGb); + await swap.reconcile(readSwapTarget(db)); } catch (error) { log.error(error, 'boot swap reconcile failed'); } @@ -253,7 +259,6 @@ const app = buildApp({ consoleDistDir: existsSync(consoleDistDir) ? consoleDistDir : undefined, archiver, ingress, - swap, updater, watchers, }); diff --git a/packages/server/src/routes/config.ts b/packages/server/src/routes/config.ts index d6ccbc8b..256d5be8 100644 --- a/packages/server/src/routes/config.ts +++ b/packages/server/src/routes/config.ts @@ -3,14 +3,11 @@ import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { CONFIG_KEYS, type Config, type ConfigSources } from '../config'; import type { Db } from '../db/db'; import { readRuntimeSettings } from '../db/settings'; -import type { SwapControl } from '../swap'; export interface ConfigRoutesOptions { config: Config; db: Db; sources: ConfigSources; - /** The managed-swap surface; absent = this host cannot manage swap. */ - swap?: SwapControl; } /** @@ -26,7 +23,7 @@ export interface ConfigRoutesOptions { */ export const configRoutes: FastifyPluginAsyncZod = async ( app, - { config, db, sources, swap }, + { config, db, sources }, ) => { app.post( '/getConfig', @@ -59,11 +56,6 @@ export const configRoutes: FastifyPluginAsyncZod = async ( ? settings.defaultPolicy.archiveAfterSeconds : null, }, - // Read live so a pending shrink (target < mounted, waiting for a - // host reboot) or a failed grow is visible, not papered over. - swap: swap - ? { supported: true, activeGb: (await swap.status()).activeGb } - : { supported: false, activeGb: 0 }, settings, }; }, diff --git a/packages/server/src/routes/settings.test.ts b/packages/server/src/routes/settings.test.ts index 00677733..71fd5ce9 100644 --- a/packages/server/src/routes/settings.test.ts +++ b/packages/server/src/routes/settings.test.ts @@ -16,7 +16,6 @@ import { createSandbox, overwriteState } from '../db/ledger'; import { readRuntimeSettings } from '../db/settings'; import { FakeExecutor } from '../executor/fake'; import { KeyedQueue } from '../keyed-queue'; -import type { SwapControl, SwapStatus } from '../swap'; const MIGRATIONS = fileURLToPath(new URL('../../drizzle', import.meta.url)); const TOKEN = 'test-token-test-token-test-token'; @@ -49,7 +48,6 @@ function freshDb() { function appOn( db: ReturnType, env: Record = {}, - swap?: SwapControl, probeS3: (s3: S3Settings) => Promise = () => Promise.resolve(), executor: FakeExecutor = new FakeExecutor(), ) { @@ -65,28 +63,12 @@ function appOn( executor, locks: new KeyedQueue(), logger: false, - swap, // Forged by default: most tests here are about the settings machinery, // not S3's availability. Probe-behavior tests inject their own. probeS3, }); } -/** A ledger of reconcile calls standing in for the real block juggler. */ -function fakeSwap(activeGb = 0): SwapControl & { reconciled: number[] } { - const status = (): Promise => - Promise.resolve({ activeGb, blocks: [] }); - const control = { - reconciled: [] as number[], - status, - reconcile(targetGb: number) { - control.reconciled.push(targetGb); - return status(); - }, - }; - return control; -} - type App = ReturnType; function rpc( @@ -127,8 +109,6 @@ describe('runtime settings: seeding', () => { sandboxDefaults: { cpus: 1, memoryGb: 2, diskGb: 20 }, // No S3 seed in this env, so the seeded default never archives. defaultPolicy: { ...DEFAULT_LIFECYCLE_POLICY, archiveAfterSeconds: null }, - // Managed swap has no env seed — it is born from the console. - swapGb: 0, s3: null, sandboxDomain: null, // Aliases have no env seed either — console-era editing only. @@ -432,49 +412,6 @@ describe('updateSettings', () => { expect(body.archive).toEqual({ enabled: false, defaultSeconds: null }); }); - it('refuses a swap target where the daemon cannot manage swap', async () => { - // No SwapControl injected — the Mac-dev / fake-executor daemon shape. - const app = appOn(freshDb()); - const res = await rpc(app, '/updateSettings', { swapGb: 32 }); - expect(res.statusCode).toBe(400); - expect(res.json().message).toMatch(/Linux host with the docker executor/); - // And getConfig says so up front, so the console never offers the knob. - const body = getConfigResponseSchema.parse( - (await rpc(app, '/getConfig')).json(), - ); - expect(body.swap).toEqual({ supported: false, activeGb: 0 }); - }); - - it('saves a swap target and reconciles it immediately', async () => { - const swap = fakeSwap(32); - const app = appOn(freshDb(), {}, swap); - const res = await rpc(app, '/updateSettings', { swapGb: 32 }); - expect(res.statusCode).toBe(200); - expect(updateSettingsResponseSchema.parse(res.json()).settings.swapGb).toBe( - 32, - ); - expect(swap.reconciled).toEqual([32]); - // A patch without swapGb must not re-trigger the block juggler. - await rpc(app, '/updateSettings', { pidsLimit: 512 }); - expect(swap.reconciled).toEqual([32]); - const body = getConfigResponseSchema.parse( - (await rpc(app, '/getConfig')).json(), - ); - expect(body.swap).toEqual({ supported: true, activeGb: 32 }); - }); - - it('keeps the saved target and answers 500 when applying it fails', async () => { - // ENOSPC mid-grow: the target must survive (boot and the next edit - // retry it) and the error must name what happened. - const swap = fakeSwap(); - swap.reconcile = () => Promise.reject(new Error('fallocate: ENOSPC')); - const app = appOn(freshDb(), {}, swap); - const res = await rpc(app, '/updateSettings', { swapGb: 512 }); - expect(res.statusCode).toBe(500); - expect(res.json().message).toMatch(/target saved.*ENOSPC/); - expect((await settingsOf(app)).swapGb).toBe(512); - }); - it('pidsLimit: live for the executor, floored, never unlimited, recorded', async () => { const app = appOn(freshDb(), { DORMICE_SANDBOX_PIDS_LIMIT: '512' }); expect((await settingsOf(app)).pidsLimit).toBe(512); @@ -501,7 +438,7 @@ describe('updateSettings', () => { undefined, () => readRuntimeSettings(db).pidsLimit, ); - const app = appOn(db, {}, undefined, undefined, executor); + const app = appOn(db, {}, undefined, executor); const busy = (await rpc(app, '/acquireSandbox', { name: 'busy' })).json() .sandbox.id as string; const idle = (await rpc(app, '/acquireSandbox', { name: 'idle' })).json() @@ -531,7 +468,7 @@ describe('updateSettings', () => { undefined, () => readRuntimeSettings(db).pidsLimit, ); - const app = appOn(db, {}, undefined, undefined, executor); + const app = appOn(db, {}, undefined, executor); await rpc(app, '/acquireSandbox', { name: 'stubborn' }); vi.spyOn(executor, 'convergePidsLimit').mockRejectedValue( new Error('runsc refused: no such luck'), @@ -546,32 +483,6 @@ describe('updateSettings', () => { expect((await settingsOf(app)).pidsLimit).toBe(2048); }); - it('one patch, two host realities: a failed swap grow does not spare the pids sweep, and both verdicts come back', async () => { - const db = freshDb(); - const executor = new FakeExecutor( - undefined, - () => readRuntimeSettings(db).pidsLimit, - ); - const swap = fakeSwap(); - swap.reconcile = () => Promise.reject(new Error('fallocate: ENOSPC')); - const app = appOn(db, {}, swap, undefined, executor); - const busy = (await rpc(app, '/acquireSandbox', { name: 'busy' })).json() - .sandbox.id as string; - - const res = await rpc(app, '/updateSettings', { - swapGb: 512, - pidsLimit: 2048, - }); - expect(res.statusCode).toBe(500); - // The swap verdict first, and only the swap's: the sweep ran and every - // running shell followed, so it has nothing to add. - expect(res.json().message).toMatch(/^swap target saved.*ENOSPC$/); - expect(executor.pidsLimitOf(busy)).toBe(2048); - const saved = await settingsOf(app); - expect(saved.swapGb).toBe(512); - expect(saved.pidsLimit).toBe(2048); - }); - it('is admin-only: an API key gets an honest 403', async () => { const app = appOn(freshDb()); const minted = await rpc(app, '/createApiKey', { name: 'robot' }); @@ -677,7 +588,7 @@ describe('updateSettings: the S3 archive store', () => { }); it("an S3-refused probe (4xx) answers 400 with S3's own words", async () => { - const app = appOn(freshDb(), {}, undefined, () => + const app = appOn(freshDb(), {}, () => Promise.reject(new S3ProbeError('AccessDenied: key rejected', 403)), ); const res = await rpc(app, '/updateSettings', { s3: S3_PATCH }); diff --git a/packages/server/src/routes/settings.ts b/packages/server/src/routes/settings.ts index 2ae2f44f..9bd9ecc7 100644 --- a/packages/server/src/routes/settings.ts +++ b/packages/server/src/routes/settings.ts @@ -16,7 +16,6 @@ import { import type { Executor } from '../executor/executor'; import type { KeyedQueue } from '../keyed-queue'; import { sweepPidsLimit } from '../pids-sweep'; -import type { SwapControl } from '../swap'; export interface SettingsRoutesOptions { db: Db; @@ -27,13 +26,6 @@ export interface SettingsRoutesOptions { */ executor: Executor; locks: KeyedQueue; - /** - * The managed-swap surface, present exactly when the daemon can manage - * swap (Linux host, docker executor — main.ts's adjudication). Absent, - * a swapGb patch is refused: an unconfigurable knob must refuse, not - * silently store a target nothing will ever reconcile. - */ - swap?: SwapControl; /** Test seam over the S3 round-trip probe; production uses the real one. */ probeS3?: (s3: S3Settings) => Promise; } @@ -47,15 +39,14 @@ export interface SettingsRoutesOptions { * A ledger write with immediate effect: the consumers read live (the * executor's births, resolvePolicy's defaults, the archiver's store, the * sandbox proxy's domain, the executor's pids cap at each birth and - * wake), so nothing here restarts or wakes a sandbox. Two knobs have a - * reality on the host that the write alone does not move, and each is - * reconciled right after it: managed swap (a swapfile) and the pids cap - * on the shells running right now (a cgroup write their processes never - * notice). + * wake), so nothing here restarts or wakes a sandbox. One knob has a + * reality on the host that the write alone does not move, and it is + * reconciled right after: the pids cap on the shells running right now (a + * cgroup write their processes never notice). */ export const settingsRoutes: FastifyPluginAsyncZod< SettingsRoutesOptions -> = async (app, { db, executor, locks, swap, probeS3 = defaultProbeS3 }) => { +> = async (app, { db, executor, locks, probeS3 = defaultProbeS3 }) => { app.post( '/updateSettings', { @@ -87,12 +78,6 @@ export const settingsRoutes: FastifyPluginAsyncZod< 'invalid default policy: archiving requires an S3 archive store — configure one in the console settings first', }); } - if (patch.swapGb !== undefined && swap === undefined) { - return reply.code(400).send({ - message: - 'managing swap requires a Linux host with the docker executor', - }); - } // The alias-list guard, judged against the post-patch state — domain // and aliases may arrive in one patch, which is exactly how the // console swaps the canonical domain atomically. Violations are @@ -199,7 +184,6 @@ export const settingsRoutes: FastifyPluginAsyncZod< `defaultPolicy=${patch.defaultPolicy.freezeAfterSeconds}s/${patch.defaultPolicy.stopAfterSeconds ?? 'never'}/${patch.defaultPolicy.archiveAfterSeconds ?? 'never'}`, ] : []), - ...(patch.swapGb !== undefined ? [`swapGb=${patch.swapGb}`] : []), // Endpoint and bucket only — the keys never reach the log, the // same "value never crosses" rule as the wire's. ...(patch.s3 !== undefined @@ -225,29 +209,6 @@ export const settingsRoutes: FastifyPluginAsyncZod< : []), ].join(', ')}`, ); - // Reconcile the host after the write — each knob with a reality out - // there on its own, neither's failure sparing the other: the ledger - // holds both targets now, and a swapfile that would not grow says - // nothing about the shells that are waiting for their cap. The - // verdicts are collected and answered together; one patch carrying - // both knobs gets both. - const unfollowed: string[] = []; - // Swap: growing mounts new blocks now, shrinking defers itself (the - // planner never touches an active block). A failed grow — ENOSPC, - // most likely — leaves the target saved on purpose: the boot - // reconcile and the next edit retry it, and getConfig's swap.activeGb - // reports the divergence honestly. - if (patch.swapGb !== undefined && swap !== undefined) { - try { - await swap.reconcile(patch.swapGb); - } catch (error) { - unfollowed.push( - `swap target saved (${patch.swapGb} GiB) but applying it failed: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } - } // Pids cap: the write already reached every future birth and wake; // the sweep brings the shells running right now along — an operator // raising the cap during an incident is looking at exactly those. A @@ -257,14 +218,11 @@ export const settingsRoutes: FastifyPluginAsyncZod< const sweep = await sweepPidsLimit(db, executor, locks); app.log.info(sweep, 'pids cap sweep after updateSettings'); if (sweep.failures.length > 0) { - unfollowed.push( - `pids cap saved (${patch.pidsLimit}) but ${sweep.failures.length} of ${sweep.considered} active sandboxes kept their old cap until their next wake — ${sweep.failures[0]}`, - ); + return reply.code(500).send({ + message: `pids cap saved (${patch.pidsLimit}) but ${sweep.failures.length} of ${sweep.considered} active sandboxes kept their old cap until their next wake — ${sweep.failures[0]}`, + }); } } - if (unfollowed.length > 0) { - return reply.code(500).send({ message: unfollowed.join('; ') }); - } return { settings }; }, ); diff --git a/packages/shared/src/config.ts b/packages/shared/src/config.ts index c0b35511..de8d9348 100644 --- a/packages/shared/src/config.ts +++ b/packages/shared/src/config.ts @@ -2,11 +2,10 @@ import { z } from 'zod'; import { runtimeSettingsSchema } from './settings'; /** - * getConfig() — the daemon's effective configuration: every env knob with - * the value actually in force and where it came from, plus the runtime - * settings that live in the ledger (see settings.ts — for those knobs the - * env entries below are first-boot seeds, and `settings` is what is in - * force). The same discipline as doctor: report effective values, never + * getConfig() — the effective configuration: every env knob with the value + * actually in force and where it came from, plus the runtime settings that + * live in the table (see settings.ts — for those knobs the env entries + * below are first-boot seeds, and `settings` is what is in force). The same discipline as doctor: report effective values, never * parrot a config file. Secrets are reported as present-or-absent only — * their value never crosses the wire, whoever asks. */ @@ -43,18 +42,6 @@ export const getConfigResponseSchema = z.object({ */ defaultSeconds: z.number().int().nullable(), }), - /** - * The daemon-managed swap surface (see settings.ts `swapGb` — the - * target). `supported` is false where the daemon cannot manage swap - * (non-Linux, or the fake executor); `activeGb` is how much managed - * swap is actually mounted right now — it lags the target after a - * shrink (which waits for a host reboot) and after a failed grow, and - * clients surface that divergence instead of pretending. - */ - swap: z.object({ - supported: z.boolean(), - activeGb: z.number().nonnegative(), - }), /** The ledger-resident operator knobs actually in force — see settings.ts. */ settings: runtimeSettingsSchema, }); diff --git a/packages/shared/src/settings.ts b/packages/shared/src/settings.ts index c43f339d..17494725 100644 --- a/packages/shared/src/settings.ts +++ b/packages/shared/src/settings.ts @@ -2,12 +2,16 @@ import { z } from 'zod'; import { lifecyclePolicySchema } from './policy'; /** - * Runtime settings — the operator knobs that live in the ledger, not the - * environment. The dividing line (2026-07-19): a knob belongs here exactly - * when changing it is an operations decision that must not require shell - * access and a restart (what new sandboxes get); it stays an env - * variable when changing it makes a different daemon (port, token, - * executor, data dir). + * Runtime settings — the fleet-wide operator knobs, the ones that live in + * a table, not the environment. The dividing line (2026-07-19): a knob + * belongs here exactly when changing it is an operations decision that + * must not require shell access and a restart (what new sandboxes get); + * it stays an env variable when changing it makes a different process + * (port, token, executor, data dir). Since the configuration authority + * moved to the gateway (2026-09-14) the table is the gateway's and every + * node keeps a copy; a knob that is one machine's rather than the fleet's + * — the managed swap target — is a node's row instead (gateway.ts + * updateNodeSettings). * * The sandbox domain and the S3 archive store used to sit on the env side * of that line; overturned 2026-07-26: the archive backend and the port- @@ -102,19 +106,6 @@ export const runtimeSettingsSchema = z.object({ sandboxDefaults: sandboxResourceDefaultsSchema, /** What acquire() gives a sandbox that asks for nothing. Existing sandboxes keep theirs. */ defaultPolicy: lifecyclePolicySchema, - /** - * Total daemon-managed swap, GiB, held as swapfiles on the data dir — - * ON TOP of whatever swap the host already has (the install-time - * swapfile stays fstab's business; the two never fight). Swap capacity - * is roughly "how much sandbox memory can hibernate at once" — freezing - * squeezes a sandbox's memory into swap. Growing takes effect - * immediately; shrinking is deferred to the next host reboot, because - * swapoff would drag every frozen sandbox's memory back into RAM - * (getConfig's `swap.activeGb` reports what is actually mounted). - * 0 = manage none. Ignored on hosts that cannot swap (see getConfig's - * `swap.supported`), where updateSettings refuses to set it. - */ - swapGb: z.number().int().nonnegative(), /** * The S3 archive store in force; null = archiving is off and sandboxes * park at stopped forever. The read shape — keys withheld (see @@ -167,7 +158,6 @@ export const updateSettingsRequestSchema = z .object({ sandboxDefaults: sandboxResourceDefaultsSchema.optional(), defaultPolicy: lifecyclePolicySchema.optional(), - swapGb: z.number().int().nonnegative().optional(), /** Write shape (all six fields, secret included); null clears the store and turns archiving off. */ s3: s3ArchiveSettingsSchema.nullable().optional(), /** A bare hostname; null turns the sandbox proxy and domain fields off. */ @@ -202,13 +192,12 @@ export const updateSettingsRequestSchema = z patch.pidsLimit !== undefined || patch.sandboxDefaults !== undefined || patch.defaultPolicy !== undefined || - patch.swapGb !== undefined || patch.s3 !== undefined || patch.sandboxDomain !== undefined || patch.sandboxDomainAliases !== undefined, { message: - 'updateSettings needs at least one of sandboxDefaults, defaultPolicy, swapGb, s3, sandboxDomain, sandboxDomainAliases, pidsLimit', + 'updateSettings needs at least one of sandboxDefaults, defaultPolicy, s3, sandboxDomain, sandboxDomainAliases, pidsLimit', }, ); diff --git a/website/content/docs/console.mdx b/website/content/docs/console.mdx index 71bb5424..9d8a9d29 100644 --- a/website/content/docs/console.mdx +++ b/website/content/docs/console.mdx @@ -117,15 +117,12 @@ Three more pages round out the operator view: for zero-downtime domain switches; a copyable wildcard DNS record guide, applied live, no dependency on the managed reverse proxy. - **Settings** — two halves. The operational knobs (default resources - for new sandboxes, default lifecycle policy, - extra managed [swap](/docs/core-concepts#swap-and-freezing), and the - S3 [archive store](/docs/archiving)) live in - the daemon's database and are edited right here, taking effect - immediately — no restart, no sandbox touched. The swap knob is the one - with an asymmetry: growing mounts new swap in seconds, shrinking waits - for the next host reboot — active swap is never unmounted, because - that would drag every frozen sandbox's memory back into RAM. The - archive store card probes the bucket with a real write-read-delete + for new sandboxes, default lifecycle policy, and the S3 + [archive store](/docs/archiving)) live in the platform's database and + are edited right here, taking effect immediately — no restart, no + sandbox touched. The extra managed + [swap](/docs/core-concepts#swap-and-freezing) a machine keeps is that + machine's own setting, edited per node. The archive store card probes the bucket with a real write-read-delete round trip before saving, never shows the keys back, and refuses to move or clear the store while sandboxes are archived in it. Everything else is the effective environment configuration, read-only, with diff --git a/website/content/docs/http-api.mdx b/website/content/docs/http-api.mdx index 6c6ef862..e2a6c413 100644 --- a/website/content/docs/http-api.mdx +++ b/website/content/docs/http-api.mdx @@ -55,7 +55,7 @@ The [E2B compatibility surface](/docs/e2b-sdks) is a separate wire under | `POST /listSandboxMetrics` | every measurable sandbox's sample in one answer | — | | `POST /listSandboxImages` | each sandbox's born image vs its template's current one | — | | `POST /getConfig` | effective configuration: env knobs (read-only; secrets reported present-or-absent, value never sent) plus the live runtime `settings` | — | -| `POST /updateSettings` | rewrite the runtime settings (new-sandbox defaults, default policy, managed swap, the S3 archive store, the sandbox domain, the per-sandbox pids cap) — immediate effect, no restart; each provided group replaces that group whole. Managed swap (`swapGb`) grows immediately; shrinking waits for the next host reboot (an active swapfile is never unmounted — that would drag every frozen sandbox's memory back into RAM). The `s3` group takes all six fields every time (keys included — they are never read back) and is probed with a real write-read-delete round trip before anything is saved; `s3: null` turns archiving off. `sandboxDomain` takes a bare hostname or `null`; `sandboxDomainAliases` takes the full list of inbound-only alias hostnames (`[]` clears) — swapping the canonical domain is one call carrying both fields, see [Exposing ports](/docs/ports). `pidsLimit` (floor 256, never unlimited) reaches new containers at birth, running sandboxes in place before the call returns, and frozen or stopped ones at their next wake — no rebuild | 400 empty patch, a `pidsLimit` below 256, a default policy that asks to archive with no store in force, `swapGb` on a host that cannot manage swap, a failed probe (S3 answered 4xx), clearing/moving the store while sandboxes are archived in it (named by count), or an alias list that contradicts the post-patch state (duplicates, the canonical domain listed as an alias, or aliases left standing with no canonical domain); 500 when a value was saved but the host would not follow — `swapGb`'s grow failed, or a running sandbox refused the new `pidsLimit` (named; it follows at its next wake); a patch carrying both knobs gets both verdicts in one message; 502 when the probed store is unreachable | +| `POST /updateSettings` | rewrite the runtime settings (new-sandbox defaults, default policy, the S3 archive store, the sandbox domain, the per-sandbox pids cap) — immediate effect, no restart; each provided group replaces that group whole. The `s3` group takes all six fields every time (keys included — they are never read back) and is probed with a real write-read-delete round trip before anything is saved; `s3: null` turns archiving off. `sandboxDomain` takes a bare hostname or `null`; `sandboxDomainAliases` takes the full list of inbound-only alias hostnames (`[]` clears) — swapping the canonical domain is one call carrying both fields, see [Exposing ports](/docs/ports). `pidsLimit` (floor 256, never unlimited) reaches new containers at birth, running sandboxes in place before the call returns, and frozen or stopped ones at their next wake — no rebuild | 400 empty patch, a `pidsLimit` below 256, a default policy that asks to archive with no store in force, a failed probe (S3 answered 4xx), clearing/moving the store while sandboxes are archived in it (named by count), or an alias list that contradicts the post-patch state (duplicates, the canonical domain listed as an alias, or aliases left standing with no canonical domain); 500 when a value was saved but the host would not follow — a running sandbox refused the new `pidsLimit` (named; it follows at its next wake); 502 when the probed store is unreachable | | `POST /getIngress` | domains bound on the daemon's managed reverse proxy, with live DNS and certificate probes | — | | `POST /setIngress` | rewrite the managed proxy config to exactly this domain list (empty list unbinds all) | 400 when the daemon manages no proxy | From edf03dea8b93080944b2547395ee442a8d90dec5 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 16:21:36 +0800 Subject: [PATCH 33/89] The gateway answers the configuration verbs: settings, templates, the front door, a node's swap target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getConfig and updateSettings now write the gateway's settings table (version counted up on every write, the nodes compare at check-in); the one guard that needs the fleet — clearing or moving the S3 archive store — counts archived and restoring sandboxes from every node's last check-in reading, and refuses with 503 Retry-After naming a node that has not reported since the gateway started rather than guess. registerTemplate/listTemplates live on the gateway's table; removeTemplate asks every node the new read-only daemon verb templateUsers (two seconds, in parallel, on the gateway's account) and refuses while any node names a sandbox (409, by node) or is silent (503, Retry-After). getIngress/setIngress own the gateway machine's Caddy file when DORMICE_INGRESS_FILE is set (the daemon's Ingress, upstream = the gateway port). updateNodeSettings sets one node's managed-swap target on its row; listNodes reports swapGb and configVersion per node. All behind the admin gate; the unnamed-verb 501 list shrinks accordingly. testGateway derives its config sources through configSources(rawEnv): the parsed config drops unset optional knobs, and deriving sources from its keys left those getConfig entries without a source (a 500 the new suite caught). --- packages/gateway/package.json | 1 + packages/gateway/src/app.test.ts | 2 +- packages/gateway/src/app.ts | 46 +- packages/gateway/src/fleet.ts | 26 + packages/gateway/src/ingress.test.ts | 277 +++++++++++ packages/gateway/src/ingress.ts | 242 +++++++++ packages/gateway/src/lookup.ts | 53 +- packages/gateway/src/main.ts | 21 + packages/gateway/src/probe.ts | 62 +++ packages/gateway/src/routes/ingress.ts | 83 ++++ packages/gateway/src/routes/native.ts | 21 +- packages/gateway/src/routes/nodes.ts | 67 ++- packages/gateway/src/routes/settings.test.ts | 469 ++++++++++++++++++ packages/gateway/src/routes/settings.ts | 274 ++++++++++ packages/gateway/src/routes/templates.test.ts | 158 ++++++ packages/gateway/src/routes/templates.ts | 129 +++++ packages/gateway/src/testing.ts | 35 +- packages/server/src/app.ts | 2 + packages/server/src/routes/template-users.ts | 35 ++ packages/shared/src/gateway.ts | 33 ++ packages/shared/src/templates.ts | 21 + pnpm-lock.yaml | 3 + 22 files changed, 2011 insertions(+), 49 deletions(-) create mode 100644 packages/gateway/src/ingress.test.ts create mode 100644 packages/gateway/src/ingress.ts create mode 100644 packages/gateway/src/probe.ts create mode 100644 packages/gateway/src/routes/ingress.ts create mode 100644 packages/gateway/src/routes/settings.test.ts create mode 100644 packages/gateway/src/routes/settings.ts create mode 100644 packages/gateway/src/routes/templates.test.ts create mode 100644 packages/gateway/src/routes/templates.ts create mode 100644 packages/server/src/routes/template-users.ts diff --git a/packages/gateway/package.json b/packages/gateway/package.json index 73511745..a11d53a5 100644 --- a/packages/gateway/package.json +++ b/packages/gateway/package.json @@ -21,6 +21,7 @@ "@fastify/static": "^9.1.3", "better-sqlite3": "^12.11.1", "drizzle-orm": "^0.45.2", + "execa": "^9.6.1", "fastify": "^5.10.0", "fastify-type-provider-zod": "^7.0.0", "pino": "^10.3.1", diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 230a0cca..3ee7f1e6 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -937,7 +937,7 @@ describe('using, destroying, and the cache', () => { const listed = await rpc(h, '/listSandboxes'); expect(listed.status).toBe(501); expect(message(listed)).toContain('call the node directly'); - expect((await rpc(h, '/registerTemplate', { name: 'k' })).status).toBe(501); + expect((await rpc(h, '/getHostMetrics')).status).toBe(501); expect((await rpc(h, '/acquireSandbx', { name: 'x' })).status).toBe(404); expect((await rpc(h, '/execCommand', { command: 'x' })).status).toBe(400); expect(h.nodes[0]?.hits).toEqual([]); diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts index 51c42fc4..39956994 100644 --- a/packages/gateway/src/app.ts +++ b/packages/gateway/src/app.ts @@ -11,21 +11,26 @@ import { type Logger, pino } from 'pino'; import { z } from 'zod'; import { requireAdminAuth, requireApiAuth, tokensEqual } from './auth'; import { classify, isOriginForm } from './classify'; -import type { Config } from './config'; +import { type Config, type ConfigSources, configSources } from './config'; import { getConsoleAccount } from './db/account'; import { isLiveApiKey, verifyApiKeyToken } from './db/api-keys'; import type { Db } from './db/db'; import { renderError } from './errors'; import type { Finder } from './find'; import type { Fleet } from './fleet'; +import type { Ingress } from './ingress'; +import { type AskVerb, httpAsk } from './lookup'; import type { PlacementKnobs } from './placement'; import { createRawFaces } from './raw'; import { apiKeyRoutes } from './routes/api-keys'; import { consoleRoutes } from './routes/console'; import { e2bControlRoutes } from './routes/e2b'; import { envdTokenRoutes } from './routes/envd-token'; +import { ingressRoutes } from './routes/ingress'; import { nativeRoutes } from './routes/native'; import { checkInRoutes, nodeRoutes } from './routes/nodes'; +import { settingsRoutes } from './routes/settings'; +import { templateRoutes } from './routes/templates'; import { type BuildInfo, readBuildInfo } from './version'; export interface GatewayAppDeps { @@ -46,8 +51,30 @@ export interface GatewayAppDeps { * honest 404. */ consoleDistDir?: string; + /** + * The managed reverse-proxy front door, present exactly when + * DORMICE_INGRESS_FILE is set. Absent, getIngress answers + * { managed: false } and setIngress refuses. + */ + ingress?: Ingress; + /** Test seam over updateSettings' S3 round-trip probe; production probes for real. */ + probeS3?: SettingsProbe; + /** + * How the gateway asks a node a verb on its own account (removeTemplate's + * templateUsers). Defaults to HTTP under the fleet token; tests script it. + */ + ask?: AskVerb; + /** + * Which knobs came from the environment versus defaults, for getConfig. + * Defaults to reading process.env; tests inject a fixed map. + */ + sources?: ConfigSources; } +type SettingsProbe = NonNullable< + Parameters[1]['probeS3'] +>; + /** Placement's knobs, read once from the config. */ export function placementKnobs(config: Config): PlacementKnobs { return { @@ -83,6 +110,10 @@ export function buildGatewayApp({ logger = true, build = readBuildInfo(), consoleDistDir, + ingress, + probeS3, + ask, + sources = configSources(), }: GatewayAppDeps) { const loggerInstance = typeof logger === 'boolean' ? pino({ enabled: logger }) : logger; @@ -215,6 +246,19 @@ export function buildGatewayApp({ admin.addHook('onRequest', adminAuth); await admin.register(apiKeyRoutes, { db }); await admin.register(nodeRoutes, { fleet, cache: finder.cache }); + await admin.register(settingsRoutes, { + config, + db, + fleet, + sources, + ...(probeS3 ? { probeS3 } : {}), + }); + await admin.register(templateRoutes, { + db, + fleet, + ask: ask ?? httpAsk(token), + }); + await admin.register(ingressRoutes, { ingress }); }); // The web console: account + session endpoints (open — setup and login diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index cb5a93ab..8a05d841 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -2,6 +2,7 @@ import type { BuildInfo, CheckInRequest, NodeReading } from '@dormice/shared'; import { eq } from 'drizzle-orm'; import type { Db } from './db/db'; import { nodes } from './db/schema'; +import { bumpConfigVersion } from './db/settings'; /** * A node as the gateway knows it: the persistent row (id, endpoint, @@ -18,6 +19,10 @@ export interface NodeState { readonly id: string; endpoint: string; readonly addedAt: string; + /** The node's own setting, from its row: managed swap on its data disk, GiB. */ + swapGb: number; + /** The configuration version the node last reported running; null before it has said. */ + configVersion: number | null; lastCheckInAt: Date | null; intervalSeconds: number | null; build: BuildInfo | null; @@ -85,6 +90,8 @@ export class Fleet { id: row.id, endpoint: row.endpoint, addedAt: row.addedAt, + swapGb: row.swapGb, + configVersion: null, lastCheckInAt: null, intervalSeconds: null, build: null, @@ -136,6 +143,8 @@ export class Fleet { id: report.nodeId, endpoint: report.endpoint, addedAt, + swapGb: 0, + configVersion: null, lastCheckInAt: null, intervalSeconds: null, build: null, @@ -176,6 +185,23 @@ export class Fleet { return { node, joined, movedFrom }; } + /** + * The one per-node setting, written to the row and counted as a + * configuration change in the same transaction, so the node that pulls + * the next bundle gets the new target under the new version and never + * one without the other. Answers false for an unknown id. + */ + setSwapGb(id: string, swapGb: number): boolean { + const node = this.members.get(id); + if (node === undefined) return false; + this.db.transaction((tx) => { + tx.update(nodes).set({ swapGb }).where(eq(nodes.id, id)).run(); + bumpConfigVersion(tx); + }); + node.swapGb = swapGb; + return true; + } + /** The operator's word that the node is gone for good (routes/nodes.ts refuses it for a node still checking in); one removed while briefly silent re-adds itself at its next check-in. */ remove(id: string): boolean { const existed = this.members.delete(id); diff --git a/packages/gateway/src/ingress.test.ts b/packages/gateway/src/ingress.test.ts new file mode 100644 index 00000000..e6482d83 --- /dev/null +++ b/packages/gateway/src/ingress.test.ts @@ -0,0 +1,277 @@ +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { getIngressResponseSchema } from '@dormice/shared'; +import { describe, expect, it } from 'vitest'; +import { Ingress } from './ingress'; +import { TEST_TOKEN, testGateway } from './testing'; + +// The managed front door: the config file is the single source of truth, +// so most of what matters is what ends up in the file — and that a failed +// reload never leaves file and running proxy telling different stories. + +const authed = { authorization: `Bearer ${TEST_TOKEN}` }; + +function tmpFile(): string { + return path.join( + mkdtempSync(path.join(tmpdir(), 'dormice-ingress-')), + 'Caddyfile', + ); +} + +function testIngress( + overrides: Partial[0]> = {}, +) { + const filePath = overrides.filePath ?? tmpFile(); + const reloads: string[] = []; + const ingress = new Ingress({ + filePath, + upstreamPort: 3677, + runCommand: async (command) => { + reloads.push(command); + return { ok: true, stderr: '' }; + }, + resolveDomain: async () => ({ dnsAddresses: ['1.2.3.4'], dnsError: null }), + probeTls: async () => ({ tlsOk: true, tlsError: null }), + ...overrides, + }); + return { ingress, filePath, reloads }; +} + +describe('Ingress file round-trip', () => { + it('binds domains: marker, one site each, :80 catch-all pointing at the gateway — and reads them back in order', async () => { + const { ingress, filePath, reloads } = testIngress(); + await ingress.setDomains(['console.example.com', 'api.example.com']); + const content = readFileSync(filePath, 'utf8'); + expect(content.startsWith('# Managed by Dormice')).toBe(true); + expect(content).toContain('console.example.com {'); + expect(content).toContain('api.example.com {'); + // The no-lockout guarantee: IP access survives every bind. + expect(content).toContain(':80 {'); + expect(content).toContain('reverse_proxy 127.0.0.1:3677'); + expect(content).toContain('flush_interval -1'); + expect(ingress.domains()).toEqual([ + 'console.example.com', + 'api.example.com', + ]); + expect(reloads).toHaveLength(1); + }); + + it('lowercases and dedups: hostnames are case-insensitive, the file decides once', async () => { + const { ingress } = testIngress(); + await ingress.setDomains([ + 'Console.Example.COM', + 'console.example.com', + 'api.example.com', + ]); + expect(ingress.domains()).toEqual([ + 'console.example.com', + 'api.example.com', + ]); + }); + + it('clears back to IP-only, and reports empty before any file exists', async () => { + const { ingress, filePath } = testIngress(); + expect(ingress.domains()).toEqual([]); + await ingress.setDomains(['console.example.com']); + await ingress.setDomains([]); + const content = readFileSync(filePath, 'utf8'); + expect(content).not.toContain('example.com'); + expect(content).toContain(':80 {'); + expect(ingress.domains()).toEqual([]); + }); + + it('defaults the reload command to caddy reload against its own file', async () => { + const filePath = tmpFile(); + const reloads: string[] = []; + const ingress = new Ingress({ + filePath, + upstreamPort: 3677, + runCommand: async (command) => { + reloads.push(command); + return { ok: true, stderr: '' }; + }, + }); + await ingress.setDomains([]); + expect(reloads).toEqual([ + `caddy reload --config ${filePath} --adapter caddyfile`, + ]); + }); + + it('serializes concurrent binds: the last write wins, reloads never interleave', async () => { + const filePath = tmpFile(); + const order: string[] = []; + const ingress = new Ingress({ + filePath, + upstreamPort: 3677, + runCommand: async () => { + order.push( + `reload:${readFileSync(filePath, 'utf8').includes('a.example.com') ? 'a' : 'b'}`, + ); + await new Promise((resolve) => setTimeout(resolve, 10)); + return { ok: true, stderr: '' }; + }, + }); + await Promise.all([ + ingress.setDomains(['a.example.com']), + ingress.setDomains(['b.example.com']), + ]); + expect(order).toEqual(['reload:a', 'reload:b']); + expect(ingress.domains()).toEqual(['b.example.com']); + }); +}); + +describe('Ingress refusals and rollback', () => { + it('refuses to overwrite a file it did not write', async () => { + const filePath = tmpFile(); + writeFileSync(filePath, 'example.org {\n\trespond "mine"\n}\n'); + const { ingress } = testIngress({ filePath }); + await expect(ingress.setDomains(['console.example.com'])).rejects.toThrow( + /not written by Dormice/, + ); + expect(readFileSync(filePath, 'utf8')).toContain('respond "mine"'); + expect(ingress.domains()).toEqual([]); + }); + + it('restores the previous file when the reload fails', async () => { + const { ingress, filePath } = testIngress(); + await ingress.setDomains(['good.example.com']); + const { ingress: failing } = testIngress({ + filePath, + runCommand: async () => ({ ok: false, stderr: 'adapting config: no' }), + }); + await expect(failing.setDomains(['bad.example.com'])).rejects.toThrow( + /adapting config: no/, + ); + expect(readFileSync(filePath, 'utf8')).toContain('good.example.com'); + }); + + it('removes the file it just created when the first-ever reload fails', async () => { + const { ingress, filePath } = testIngress({ + runCommand: async () => ({ ok: false, stderr: 'caddy not running' }), + }); + await expect(ingress.setDomains(['x.example.com'])).rejects.toThrow( + /caddy not running/, + ); + expect(existsSync(filePath)).toBe(false); + }); +}); + +describe('ingress routes on the gateway', () => { + type App = ReturnType['app']; + const rpc = ( + app: App, + url: string, + payload: Record = {}, + headers: Record = authed, + ) => app.inject({ method: 'POST', url, headers, payload }); + + it('without a managed ingress: getIngress is honest, setIngress refuses naming the gateway env', async () => { + const { app } = testGateway(); + const get = await rpc(app, '/getIngress'); + expect(get.statusCode).toBe(200); + expect(getIngressResponseSchema.parse(get.json())).toEqual({ + managed: false, + domains: [], + }); + const set = await rpc(app, '/setIngress', { + domains: ['a.example.com'], + }); + expect(set.statusCode).toBe(400); + expect(set.json().message).toContain('DORMICE_INGRESS_FILE'); + expect(set.json().message).toContain('gateway'); + }); + + it('binds two, probes each, drops one, clears', async () => { + const { ingress } = testIngress(); + const { app } = testGateway({}, { ingress }); + + const set = await rpc(app, '/setIngress', { + domains: ['console.example.com', 'api.example.com'], + }); + expect(set.statusCode).toBe(200); + expect(set.json()).toEqual({ + domains: ['console.example.com', 'api.example.com'], + }); + + const get = await rpc(app, '/getIngress'); + const status = getIngressResponseSchema.parse(get.json()); + const probe = { + dnsAddresses: ['1.2.3.4'], + dnsError: null, + tlsOk: true, + tlsError: null, + }; + expect(status).toEqual({ + managed: true, + domains: [ + { domain: 'console.example.com', probe }, + { domain: 'api.example.com', probe }, + ], + }); + + const dropped = await rpc(app, '/setIngress', { + domains: ['api.example.com'], + }); + expect(dropped.json()).toEqual({ domains: ['api.example.com'] }); + + const cleared = await rpc(app, '/setIngress', { domains: [] }); + expect(cleared.statusCode).toBe(200); + expect(cleared.json()).toEqual({ domains: [] }); + }); + + it('rejects a domain with a scheme at the schema gate', async () => { + const { ingress } = testIngress(); + const { app } = testGateway({}, { ingress }); + const res = await rpc(app, '/setIngress', { + domains: ['https://console.example.com'], + }); + expect(res.statusCode).toBe(400); + expect(res.json().message).toContain('bare hostname'); + }); + + it('maps a foreign file to 409 and a failed reload to 500, with the reason', async () => { + const filePath = tmpFile(); + writeFileSync(filePath, 'someone-elses-site {\n}\n'); + const foreign = testGateway( + {}, + { ingress: testIngress({ filePath }).ingress }, + ); + const refused = await rpc(foreign.app, '/setIngress', { + domains: ['a.example.com'], + }); + expect(refused.statusCode).toBe(409); + expect(refused.json().message).toContain('not written by Dormice'); + + const broken = testGateway( + {}, + { + ingress: testIngress({ + runCommand: async () => ({ ok: false, stderr: 'connection refused' }), + }).ingress, + }, + ); + const failed = await rpc(broken.app, '/setIngress', { + domains: ['a.example.com'], + }); + expect(failed.statusCode).toBe(500); + expect(failed.json().message).toContain('connection refused'); + }); + + it('is admin-only (design record #9): a minted key can neither read nor rewrite the front door', async () => { + const { ingress } = testIngress(); + const { app } = testGateway({}, { ingress }); + const minted = await rpc(app, '/createApiKey', { name: 'robot' }); + const asKey = { authorization: `Bearer ${minted.json().token}` }; + const set = await rpc( + app, + '/setIngress', + { domains: ['evil.example.com'] }, + asKey, + ); + expect(set.statusCode).toBe(403); + expect(set.json().message).toMatch(/cannot manage API keys/); + expect((await rpc(app, '/getIngress', {}, asKey)).statusCode).toBe(403); + expect(ingress.domains()).toEqual([]); + }); +}); diff --git a/packages/gateway/src/ingress.ts b/packages/gateway/src/ingress.ts new file mode 100644 index 00000000..139cfeca --- /dev/null +++ b/packages/gateway/src/ingress.ts @@ -0,0 +1,242 @@ +import { Resolver } from 'node:dns/promises'; +import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; +import tls from 'node:tls'; +import type { GetIngressResponse, IngressProbe } from '@dormice/shared'; +import { execa } from 'execa'; + +/** + * The fleet's front door: a Caddy config file the gateway owns, rewritten + * on setIngress and reloaded into the running Caddy. The file is the + * single source of truth — no table column, no boot-time reconcile: an + * operator can read it, Caddy runs it, and getIngress parses it back. TLS + * is entirely Caddy's job (ACME issuance and renewal); this class only + * decides what the file says. The one Caddy with a public face in a fleet + * sits in front of the gateway (design record #24): the API domain and + * the sandbox wildcard both land here, the nodes' own Caddys face the + * intranet only. + * + * The generated shape is one site block per bound domain (Caddy obtains a + * certificate per domain and auto-redirects http:// to https) plus + * a plain :80 catch-all proxying by IP. The catch-all is the no-lockout + * guarantee — a bind that never converges (typo'd domain, missing DNS + * record) leaves IP access untouched. Caddy inserts its automatic HTTPS + * redirects after host-matched routes but before user catch-alls, so the + * domain sites and the catch-all coexist. + */ + +/** + * Ownership marker, line one of every file this class writes. A file the + * knob points at that lacks it was written by someone else — refused, never + * overwritten. install.sh writes the same marker (kept in sync by hand). + */ +const MARKER = '# Managed by Dormice — setIngress rewrites this file.'; + +/** The knob points at a file this daemon does not own. */ +export class UnmanagedIngressFileError extends Error {} + +export interface IngressOptions { + /** The Caddy config file the gateway owns (DORMICE_INGRESS_FILE). */ + filePath: string; + /** Where the proxy forwards to: the gateway's own loopback port. */ + upstreamPort: number; + /** + * Shell command that makes the running proxy read the new config. + * Defaults to `caddy reload` against the standard Caddyfile — right when + * the gateway owns the whole file; an operator whose own Caddyfile imports + * a fragment overrides this to reload the outer file instead. + */ + reloadCommand?: string; + /** Test seam; production shells out through execa. */ + runCommand?: (command: string) => Promise<{ ok: boolean; stderr: string }>; + /** Test seams for the two probes; production asks DNS and 127.0.0.1:443. */ + resolveDomain?: ( + domain: string, + ) => Promise>; + probeTls?: ( + domain: string, + ) => Promise>; +} + +export class Ingress { + private readonly filePath: string; + private readonly upstreamPort: number; + private readonly reloadCommand: string; + private readonly runCommand: NonNullable; + private readonly resolveDomain: NonNullable; + private readonly probeTls: NonNullable; + /** Serializes writes: two concurrent binds must not interleave write+reload. */ + private queue: Promise = Promise.resolve(); + + constructor(options: IngressOptions) { + this.filePath = options.filePath; + this.upstreamPort = options.upstreamPort; + this.reloadCommand = + options.reloadCommand ?? + `caddy reload --config ${options.filePath} --adapter caddyfile`; + this.runCommand = options.runCommand ?? runShellCommand; + this.resolveDomain = options.resolveDomain ?? resolveDomainDns; + this.probeTls = options.probeTls ?? probeLocalTls; + } + + /** + * The currently bound domains, read back from the file in the order they + * are served. Empty when nothing is bound, the file does not exist yet, + * or the file is not ours — a foreign file's site addresses are not the + * gateway's to report. + */ + domains(): string[] { + if (!existsSync(this.filePath)) return []; + const content = readFileSync(this.filePath, 'utf8'); + if (!content.includes('Managed by Dormice')) return []; + const found: string[] = []; + for (const line of content.split('\n')) { + // Site addresses sit at column 0 in the generated shape; indented + // lines are directives (`\treverse_proxy … {`), never sites. The + // leading-`:` exclusion keeps the :80 catch-all out of the answer. + const site = /^([^\s#:{][^\s{]*)\s*\{/.exec(line); + if (site?.[1]) found.push(site[1]); + } + return found; + } + + /** + * Rewrites the file to serve exactly the given set (empty = back to + * IP-only) and reloads the proxy. Hostnames are case-insensitive, so the + * set is lowercased and deduped here — the one place that decides what + * the file says. On a failed reload the previous file is restored — + * `caddy reload` rejects a bad config without applying it, so file and + * running proxy stay consistent — and the failure is thrown with Caddy's + * own words. + */ + setDomains(domains: string[]): Promise { + const wanted = [...new Set(domains.map((domain) => domain.toLowerCase()))]; + const run = this.queue.then(() => this.apply(wanted)); + this.queue = run.catch(() => {}); + return run; + } + + /** Everything getIngress reports: the file's word plus live probes. */ + async status(): Promise { + const statuses = await Promise.all( + this.domains().map(async (domain) => { + const [dns, cert] = await Promise.all([ + this.resolveDomain(domain), + this.probeTls(domain), + ]); + return { domain, probe: { ...dns, ...cert } }; + }), + ); + return { managed: true, domains: statuses }; + } + + private async apply(domains: string[]): Promise { + const previous = existsSync(this.filePath) + ? readFileSync(this.filePath, 'utf8') + : null; + if ( + previous !== null && + previous.trim() !== '' && + !previous.includes('Managed by Dormice') + ) { + throw new UnmanagedIngressFileError( + `${this.filePath} was not written by Dormice — refusing to overwrite it; ` + + 'move your configuration elsewhere, or point DORMICE_INGRESS_FILE at a file the gateway may own', + ); + } + writeFileSync(this.filePath, this.render(domains)); + const reload = await this.runCommand(this.reloadCommand); + if (!reload.ok) { + if (previous === null) unlinkSync(this.filePath); + else writeFileSync(this.filePath, previous); + throw new Error( + `reloading the proxy failed (${this.reloadCommand}): ${reload.stderr.trim()} — the previous configuration was restored`, + ); + } + } + + private render(domains: string[]): string { + // flush_interval -1 streams byte-by-byte: buffering would dam the + // console terminal and E2B's streaming exec (measured through Caddy). + const site = (address: string) => + `${address} {\n\treverse_proxy 127.0.0.1:${this.upstreamPort} {\n\t\tflush_interval -1\n\t}\n}\n`; + return [`${MARKER}\n`, ...domains.map(site), site(':80')].join('\n'); + } +} + +async function runShellCommand( + command: string, +): Promise<{ ok: boolean; stderr: string }> { + try { + await execa(command, { shell: true, timeout: 30_000 }); + return { ok: true, stderr: '' }; + } catch (error) { + const stderr = + error instanceof Error + ? ('stderr' in error && String(error.stderr)) || error.message + : String(error); + return { ok: false, stderr }; + } +} + +/** + * What the domain resolves to right now. "No record" (the state before the + * operator's A record lands or propagates) is an empty list, not an error; + * dnsError is reserved for the resolver itself failing. + */ +async function resolveDomainDns( + domain: string, +): Promise> { + const resolver = new Resolver({ timeout: 3_000, tries: 1 }); + const lookup = async (kind: 'resolve4' | 'resolve6') => { + try { + return await resolver[kind](domain); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'ENOTFOUND' || code === 'ENODATA') return []; + throw error; + } + }; + try { + const [v4, v6] = await Promise.all([ + lookup('resolve4'), + lookup('resolve6'), + ]); + return { dnsAddresses: [...v4, ...v6], dnsError: null }; + } catch (error) { + return { + dnsAddresses: [], + dnsError: error instanceof Error ? error.message : String(error), + }; + } +} + +/** + * Does the local proxy serve a valid trusted certificate for the domain? + * A handshake against 127.0.0.1:443 with the domain as SNI — deliberately + * not a fetch of the public URL: cloud NAT usually cannot hairpin a host's + * own public IP, so the honest local fact is "certificate issued and + * served", and public reachability stays the security group's question. + */ +function probeLocalTls( + domain: string, +): Promise> { + return new Promise((resolve) => { + const socket = tls.connect({ + host: '127.0.0.1', + port: 443, + servername: domain, + rejectUnauthorized: true, + }); + const done = (result: Pick) => { + socket.destroy(); + resolve(result); + }; + socket.setTimeout(3_000, () => + done({ tlsOk: false, tlsError: 'timed out connecting to 127.0.0.1:443' }), + ); + socket.once('secureConnect', () => done({ tlsOk: true, tlsError: null })); + socket.once('error', (error) => + done({ tlsOk: false, tlsError: error.message }), + ); + }); +} diff --git a/packages/gateway/src/lookup.ts b/packages/gateway/src/lookup.ts index ad471898..bed6f169 100644 --- a/packages/gateway/src/lookup.ts +++ b/packages/gateway/src/lookup.ts @@ -3,11 +3,13 @@ import { lookupSandboxResponseSchema, type SandboxState, } from '@dormice/shared'; +import type { z } from 'zod'; /** - * Asking one node "do you hold this sandbox?" — the daemon's lookupSandbox - * verb, the one question the gateway puts to a node on its own account - * (everything else it sends is a caller's request, forwarded raw). Two + * Asking one node a question on the gateway's own account — the daemon's + * lookupSandbox ("do you hold this sandbox?") and templateUsers ("which + * of yours still use this template?"), the two read-only verbs the + * gateway sends that are not a caller's request forwarded raw. Two * seconds, not more: a node that cannot answer a ledger read in two * seconds is a node in trouble, and the caller is waiting on the whole * round. There is no second, slower deadline — slow is down (design @@ -33,6 +35,19 @@ export type AskNode = ( query: LookupQuery, ) => Promise; +/** A node's answer to any verb asked on the gateway's account: parsed, or silence with the transport's word. */ +export type Asked = + | { kind: 'answer'; value: T } + | { kind: 'silent'; why: string }; + +/** Asks one node one verb, validated by the schema of its answer. */ +export type AskVerb = ( + node: AskedNode, + verb: string, + body: unknown, + schema: z.ZodType, +) => Promise>; + /** * The transport's word for a failure, for a log line or a 503's sentence: * a system code where there is one (ECONNREFUSED, UND_ERR_CONNECT_TIMEOUT), @@ -64,16 +79,16 @@ export function causeOf(error: unknown): string { * quirk comes with fetch: it refuses the Fetch standard's "bad ports" * (9, 22, 25, 6000 …) without dialling — no node's front lives on one. */ -export function httpAskNode(token: string): AskNode { - return async (node, query) => { +export function httpAsk(token: string): AskVerb { + return async (node, verb, body, schema) => { try { - const res = await fetch(`${node.endpoint}/lookupSandbox`, { + const res = await fetch(`${node.endpoint}/${verb}`, { method: 'POST', headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json', }, - body: JSON.stringify(query), + body: JSON.stringify(body), signal: AbortSignal.timeout(LOOKUP_TIMEOUT_MS), redirect: 'manual', }); @@ -81,15 +96,29 @@ export function httpAskNode(token: string): AskNode { const text = await res.text(); return { kind: 'silent', - why: `lookupSandbox answered ${res.status}: ${text.slice(0, 200)}`, + why: `${verb} answered ${res.status}: ${text.slice(0, 200)}`, }; } - const answer = lookupSandboxResponseSchema.parse(await res.json()); - return answer.found - ? { kind: 'found', ...answer.sandbox } - : { kind: 'absent' }; + return { kind: 'answer', value: schema.parse(await res.json()) }; } catch (error) { return { kind: 'silent', why: causeOf(error) }; } }; } + +/** lookupSandbox over httpAsk, in the words find.ts reads. */ +export function httpAskNode(token: string): AskNode { + const ask = httpAsk(token); + return async (node, query) => { + const asked = await ask( + node, + 'lookupSandbox', + query, + lookupSandboxResponseSchema, + ); + if (asked.kind === 'silent') return asked; + return asked.value.found + ? { kind: 'found', ...asked.value.sandbox } + : { kind: 'absent' }; + }; +} diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts index e590f4f9..da982859 100644 --- a/packages/gateway/src/main.ts +++ b/packages/gateway/src/main.ts @@ -12,6 +12,7 @@ import { migrateDb, openDb } from './db/db'; import { ensureSettings } from './db/settings'; import { Finder } from './find'; import { Fleet } from './fleet'; +import { Ingress } from './ingress'; import { httpAskNode } from './lookup'; import { readBuildInfo } from './version'; @@ -81,6 +82,25 @@ log.info( : 'dormice-gateway build: no version identity (built outside a git checkout)', ); +// The managed front door, present exactly when the knob names a file +// (the archiver precedent). The upstream is this gateway: the fleet's one +// public Caddy sits in front of the one door. +const ingress = + config.DORMICE_INGRESS_FILE === undefined + ? undefined + : new Ingress({ + filePath: config.DORMICE_INGRESS_FILE, + upstreamPort: config.DORMICE_GATEWAY_PORT, + ...(config.DORMICE_INGRESS_RELOAD_CMD + ? { reloadCommand: config.DORMICE_INGRESS_RELOAD_CMD } + : {}), + }); +if (ingress) { + log.info( + `managed ingress: ${config.DORMICE_INGRESS_FILE} (domains bound from the console reach this gateway)`, + ); +} + // The built web console, by the monorepo layout: dist/main.js sits two // levels under packages/gateway, the console's dist beside it. Absent // (a deploy without the console built), /console answers an honest 404. @@ -100,6 +120,7 @@ const app = buildGatewayApp({ logger: log, build, consoleDistDir: existsSync(consoleDistDir) ? consoleDistDir : undefined, + ingress, }); // Same red line as the daemon: loopback only, host not configurable — the diff --git a/packages/gateway/src/probe.ts b/packages/gateway/src/probe.ts new file mode 100644 index 00000000..85ff37ce --- /dev/null +++ b/packages/gateway/src/probe.ts @@ -0,0 +1,62 @@ +import { randomUUID } from 'node:crypto'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { type S3Settings, S3Store } from '@dormice/server/s3-store'; + +/** Thrown by probeS3 with the S3 error's own words and its HTTP status (when S3 answered at all). */ +export class S3ProbeError extends Error { + constructor( + message: string, + readonly httpStatusCode: number | undefined, + ) { + super(message); + } +} + +/** + * A put+get+delete round trip against the candidate store, run BEFORE the + * settings are written — a probe failure must leave the table untouched: + * S3 credentials are static facts, and wrong ones saved would turn every + * node's next archive tick into error noise. The gateway probes itself, + * as the authority that is about to write the store every node will use + * (a node could probe on its behalf, but the one who writes the fact is + * the one who checks it). Runs through the daemon's own S3Store (real + * file streams, the same WHEN_REQUIRED checksum posture), so what the + * probe proves is exactly what archiving on a node will do. + */ +export async function probeS3(s3: S3Settings): Promise { + const store = new S3Store(s3); + const dir = await mkdtemp(path.join(tmpdir(), 'dormice-s3-probe-')); + const key = `dormice-probe-${randomUUID()}`; + const body = 'dormice archive-store probe'; + try { + const up = path.join(dir, 'up'); + const down = path.join(dir, 'down'); + await writeFile(up, body); + try { + await store.put(key, up); + await store.get(key, down); + } catch (error) { + throw toProbeError(error); + } + if ((await readFile(down, 'utf8')) !== body) { + throw new S3ProbeError( + 'the probe object came back with different content — the store is not a faithful S3', + undefined, + ); + } + } finally { + await store.delete(key).catch(() => {}); + await rm(dir, { recursive: true, force: true }); + } +} + +function toProbeError(error: unknown): S3ProbeError { + if (error instanceof Error) { + const status = (error as { $metadata?: { httpStatusCode?: number } }) + .$metadata?.httpStatusCode; + return new S3ProbeError(`${error.name}: ${error.message}`, status); + } + return new S3ProbeError(String(error), undefined); +} diff --git a/packages/gateway/src/routes/ingress.ts b/packages/gateway/src/routes/ingress.ts new file mode 100644 index 00000000..5d1e1f83 --- /dev/null +++ b/packages/gateway/src/routes/ingress.ts @@ -0,0 +1,83 @@ +import { + getIngressResponseSchema, + setIngressRequestSchema, + setIngressResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { httpError } from '../http-error'; +import { type Ingress, UnmanagedIngressFileError } from '../ingress'; + +export interface IngressRoutesOptions { + /** Present exactly when DORMICE_INGRESS_FILE is set (the archiver precedent). */ + ingress?: Ingress; +} + +/** + * The fleet's front door, over the wire: bind a domain from the console + * (or SDK) instead of hand-editing a Caddyfile over SSH. Without a managed + * ingress the read is honest ({ managed: false }) and the write refuses + * with the reason — never a silent no-op. Behind the admin gate (design + * record #9): an automation key must not be able to rewrite the fleet's + * Caddyfile. + */ +export const ingressRoutes: FastifyPluginAsyncZod< + IngressRoutesOptions +> = async (app, { ingress }) => { + app.post( + '/getIngress', + { + schema: { + response: { 200: getIngressResponseSchema }, + }, + }, + async () => (ingress ? ingress.status() : { managed: false, domains: [] }), + ); + + app.post( + '/setIngress', + { + schema: { + body: setIngressRequestSchema, + response: { 200: setIngressResponseSchema }, + }, + }, + async (request) => { + if (!ingress) { + throw httpError( + 400, + 'this gateway manages no reverse proxy — set DORMICE_INGRESS_FILE in its env (install.sh sets up Caddy and points it at /etc/caddy/Caddyfile), or configure your proxy directly', + ); + } + const previous = ingress.domains(); + try { + await ingress.setDomains(request.body.domains); + } catch (error) { + if (error instanceof UnmanagedIngressFileError) { + throw httpError(409, error.message); + } + throw httpError( + 500, + error instanceof Error ? error.message : String(error), + ); + } + // The file (not the request) is the truth to report and to log: + // read back what setDomains actually wrote (lowercased, deduped). + const domains = ingress.domains(); + const changes = [ + ...domains + .filter((domain) => !previous.includes(domain)) + .map((domain) => `bound ${domain}`), + ...previous + .filter((domain) => !domains.includes(domain)) + .map((domain) => `unbound ${domain}`), + ]; + request.log.info( + { changes, domains }, + `ingress updated: ${changes.join(', ') || 'domains unchanged'} — now serving ${ + domains.length ? domains.join(', ') : 'plain-HTTP IP access only' + }`, + ); + return { domains }; + }, + ); +}; diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index 0b41723c..93855773 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -46,13 +46,13 @@ export const NAMED_VERBS = [ ] as const; /** - * The verbs that address the daemon, not a sandbox: fleet lists, host - * readings, templates, settings, ingress, upgrade. The ones the gateway - * answers from its own tables leave this list as they arrive (the API - * keys did with the configuration authority); asking every node and - * merging the rest comes in a later cut. Until then each answers an - * honest 501 naming the alternative, instead of a misleading answer from - * whichever node the gateway happened to pick. + * The verbs that address the daemon, not a sandbox, and that the gateway + * cannot answer from its own tables: fleet lists, host readings, the + * upgrade. Asking every node and merging comes in a later cut; until then + * each answers an honest 501 naming the alternative, instead of a + * misleading answer from whichever node the gateway happened to pick. + * (Keys, settings, templates and ingress left this list with the + * configuration authority.) */ export const UNNAMED_VERBS = [ 'listSandboxes', @@ -61,16 +61,9 @@ export const UNNAMED_VERBS = [ 'getFleetTimeline', 'getHostMetrics', 'getHostMetricsHistory', - 'getConfig', 'checkUpgrade', 'applyUpgrade', 'getUpgradeStatus', - 'getIngress', - 'setIngress', - 'registerTemplate', - 'listTemplates', - 'removeTemplate', - 'updateSettings', ] as const; export interface NativeRoutesOptions { diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index 0df5ba6f..60950961 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -3,12 +3,20 @@ import { checkInResponseSchema, listNodesRequestSchema, listNodesResponseSchema, + type NodeView, removeNodeRequestSchema, removeNodeResponseSchema, + updateNodeSettingsRequestSchema, + updateNodeSettingsResponseSchema, } from '@dormice/shared'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import type { NameCache } from '../cache'; -import { downReason, type Fleet, STARTUP_GRACE_MS } from '../fleet'; +import { + downReason, + type Fleet, + type NodeState, + STARTUP_GRACE_MS, +} from '../fleet'; export interface CheckInRoutesOptions { fleet: Fleet; @@ -125,19 +133,33 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( }, async () => { const now = new Date(); - return { - nodes: fleet.all().map((node) => ({ - id: node.id, - endpoint: node.endpoint, - addedAt: node.addedAt, - lastCheckInAt: node.lastCheckInAt?.toISOString() ?? null, - intervalSeconds: node.intervalSeconds, - reachable: downReason(node, now) === null, - build: node.build, - reading: node.reading, - placedSinceCheckIn: node.placedSinceCheckIn, - })), - }; + return { nodes: fleet.all().map((node) => view(node, now)) }; + }, + ); + + app.post( + '/updateNodeSettings', + { + schema: { + body: updateNodeSettingsRequestSchema, + response: { 200: updateNodeSettingsResponseSchema }, + }, + }, + async (request) => { + const { id, swapGb } = request.body; + if (!fleet.setSwapGb(id, swapGb)) { + throw refusal( + 404, + `no node with id '${id}' — listNodes shows which exist`, + ); + } + const node = fleet.get(id); + if (node === undefined) throw refusal(404, `no node with id '${id}'`); + request.log.info( + { nodeId: id, swapGb }, + 'node swap target set; the node applies it at its next check-in', + ); + return { node: view(node, new Date()) }; }, ); @@ -194,3 +216,20 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( }, ); }; + +/** What listNodes and updateNodeSettings answer for one node: the row, the last check-in, the gateway's own counters. */ +function view(node: NodeState, now: Date): NodeView { + return { + id: node.id, + endpoint: node.endpoint, + addedAt: node.addedAt, + swapGb: node.swapGb, + configVersion: node.configVersion, + lastCheckInAt: node.lastCheckInAt?.toISOString() ?? null, + intervalSeconds: node.intervalSeconds, + reachable: downReason(node, now) === null, + build: node.build, + reading: node.reading, + placedSinceCheckIn: node.placedSinceCheckIn, + }; +} diff --git a/packages/gateway/src/routes/settings.test.ts b/packages/gateway/src/routes/settings.test.ts new file mode 100644 index 00000000..2bed6ef9 --- /dev/null +++ b/packages/gateway/src/routes/settings.test.ts @@ -0,0 +1,469 @@ +import { + ARCHIVE_DEFAULT_SECONDS, + DEFAULT_LIFECYCLE_POLICY, + getConfigResponseSchema, + updateSettingsResponseSchema, +} from '@dormice/shared'; +import { describe, expect, it } from 'vitest'; +import { CONFIG_KEYS } from '../config'; +import { readConfigVersion } from '../db/settings'; +import { S3ProbeError } from '../probe'; +import { checkInOf, TEST_TOKEN, testGateway } from '../testing'; + +type Harness = ReturnType; +type App = Harness['app']; + +const authed = { authorization: `Bearer ${TEST_TOKEN}` }; + +/** An env S3 seed — the four core variables, as a spreadable set. */ +const S3_ENV = { + DORMICE_S3_ENDPOINT: 'http://127.0.0.1:9000', + DORMICE_S3_BUCKET: 'seed-bucket', + DORMICE_S3_ACCESS_KEY_ID: 'seed-key', + DORMICE_S3_SECRET_ACCESS_KEY: 'seed-secret-never-on-the-wire', +}; + +/** The same store as an updateSettings write-shape patch. */ +const S3_PATCH = { + endpoint: 'http://127.0.0.1:9000', + bucket: 'patched-bucket', + accessKeyId: 'patch-key', + secretAccessKey: 'patch-secret-never-on-the-wire', + region: 'us-east-1', + forcePathStyle: true, +}; + +function rpc( + app: App, + url: string, + payload: Record = {}, + headers: Record = authed, +) { + return app.inject({ method: 'POST', url, headers, payload }); +} + +async function settingsOf(app: App) { + const res = await rpc(app, '/getConfig'); + expect(res.statusCode).toBe(200); + return getConfigResponseSchema.parse(res.json()).settings; +} + +/** A node the fleet knows, with the census its last check-in reported; one address per id. */ +function reporting( + h: Harness, + id: string, + over: Parameters[2] = {}, +) { + const host = + 1 + ([...id].reduce((sum, ch) => sum + ch.charCodeAt(0), 0) % 250); + const outcome = h.fleet.checkIn( + checkInOf(id, `http://10.0.0.${host}:80`, over), + ); + if ('refused' in outcome) throw new Error(outcome.refused); + return outcome.node; +} + +describe('getConfig on the gateway', () => { + it('reports every gateway knob with value and source, and the settings in force', async () => { + const { app } = testGateway({ DORMICE_SANDBOX_DISK_GB: '7' }); + const res = await rpc(app, '/getConfig'); + expect(res.statusCode).toBe(200); + const body = getConfigResponseSchema.parse(res.json()); + const byKey = new Map(body.entries.map((e) => [e.key, e])); + // Complete: one entry per knob the gateway's config schema knows. + expect(body.entries).toHaveLength(Object.keys(CONFIG_KEYS).length); + expect(byKey.get('DORMICE_SANDBOX_DISK_GB')).toMatchObject({ + value: '7', + source: 'env', + }); + expect(byKey.get('DORMICE_GATEWAY_PORT')).toMatchObject({ + value: '3677', + source: 'default', + }); + // Optional and unset: honestly null, not invented. + expect(byKey.get('DORMICE_SANDBOX_DOMAIN')).toMatchObject({ value: null }); + expect(body.settings.sandboxDefaults.diskGb).toBe(7); + expect(body.archive).toEqual({ enabled: false, defaultSeconds: null }); + }); + + it('withholds secrets, reporting only their presence', async () => { + const { app } = testGateway(S3_ENV); + const body = getConfigResponseSchema.parse( + (await rpc(app, '/getConfig')).json(), + ); + const token = body.entries.find((e) => e.key === 'DORMICE_API_TOKEN'); + expect(token).toMatchObject({ value: null, redacted: true }); + const key = body.entries.find((e) => e.key === 'DORMICE_S3_ACCESS_KEY_ID'); + expect(key).toMatchObject({ value: null, redacted: true }); + // The raw secrets appear nowhere in the whole response. + const text = JSON.stringify(body); + expect(text).not.toContain(TEST_TOKEN); + expect(text).not.toContain('seed-key'); + expect(text).not.toContain('seed-secret'); + // An S3 seed turns archiving on with the one-week default. + expect(body.archive).toEqual({ + enabled: true, + defaultSeconds: ARCHIVE_DEFAULT_SECONDS, + }); + }); + + it('is admin-only: a minted key reads 403, the console session reads it', async () => { + const { app } = testGateway(); + const minted = await rpc(app, '/createApiKey', { name: 'robot' }); + const keyToken = minted.json().token as string; + const refused = await rpc( + app, + '/getConfig', + {}, + { authorization: `Bearer ${keyToken}` }, + ); + expect(refused.statusCode).toBe(403); + expect(refused.json().message).toMatch(/cannot manage API keys/); + }); +}); + +describe('updateSettings on the gateway', () => { + it('sets the pids cap live, floors it, and never accepts unlimited; every write counts the version up', async () => { + const { app, db } = testGateway(); + expect((await settingsOf(app)).pidsLimit).toBe(4096); + expect(readConfigVersion(db)).toBe(1); + + const raised = await rpc(app, '/updateSettings', { pidsLimit: 8192 }); + expect(raised.statusCode).toBe(200); + expect( + updateSettingsResponseSchema.parse(raised.json()).settings.pidsLimit, + ).toBe(8192); + expect((await settingsOf(app)).pidsLimit).toBe(8192); + expect(readConfigVersion(db)).toBe(2); + + const tooLow = await rpc(app, '/updateSettings', { pidsLimit: 255 }); + expect(tooLow.statusCode).toBe(400); + expect(tooLow.json().message).toMatch(/pidsLimit.*at least 256/); + expect((await settingsOf(app)).pidsLimit).toBe(8192); + // A refused write is not a change the nodes must hear about. + expect(readConfigVersion(db)).toBe(2); + expect( + (await rpc(app, '/updateSettings', { pidsLimit: 256 })).statusCode, + ).toBe(200); + expect( + (await rpc(app, '/updateSettings', { pidsLimit: 0 })).statusCode, + ).toBe(400); + }); + + it('replaces provided groups whole and leaves the rest untouched', async () => { + const { app } = testGateway({ DORMICE_SANDBOX_MEMORY_GB: '4' }); + await rpc(app, '/updateSettings', { pidsLimit: 512 }); + const settings = await settingsOf(app); + expect(settings.pidsLimit).toBe(512); + expect(settings.sandboxDefaults.memoryGb).toBe(4); + expect(settings.updatedAt).not.toBeNull(); + }); + + it('an empty patch is a caller confusion, 400', async () => { + const { app } = testGateway(); + expect((await rpc(app, '/updateSettings', {})).statusCode).toBe(400); + }); + + it('a new default policy is stored for the nodes to hand to their next acquire', async () => { + const { app } = testGateway(); + const res = await rpc(app, '/updateSettings', { + defaultPolicy: { + freezeAfterSeconds: 42, + stopAfterSeconds: null, + archiveAfterSeconds: null, + }, + }); + expect(res.statusCode).toBe(200); + expect((await settingsOf(app)).defaultPolicy).toEqual({ + freezeAfterSeconds: 42, + stopAfterSeconds: null, + archiveAfterSeconds: null, + }); + }); + + it('refuses an archiving default when no S3 store is configured, judged after the patch', async () => { + const { app } = testGateway(); + const res = await rpc(app, '/updateSettings', { + defaultPolicy: { + freezeAfterSeconds: 600, + stopAfterSeconds: 3600, + archiveAfterSeconds: 7200, + }, + }); + expect(res.statusCode).toBe(400); + expect(res.json().message).toMatch(/archiving requires an S3 archive/); + // But arriving together with the store that honors it is legal. + const together = await rpc(app, '/updateSettings', { + s3: S3_PATCH, + defaultPolicy: { + freezeAfterSeconds: 600, + stopAfterSeconds: 3600, + archiveAfterSeconds: 7200, + }, + }); + expect(together.statusCode).toBe(200); + // And { s3: null, archiving default } in one patch is refused. + const contradiction = await rpc(app, '/updateSettings', { + s3: null, + defaultPolicy: { + freezeAfterSeconds: 600, + stopAfterSeconds: 3600, + archiveAfterSeconds: 7200, + }, + }); + expect(contradiction.statusCode).toBe(400); + }); + + it('is admin-only: a minted key gets an honest 403 and the settings stand', async () => { + const { app } = testGateway(); + const minted = await rpc(app, '/createApiKey', { name: 'robot' }); + expect(minted.statusCode).toBe(200); + const keyToken = minted.json().token as string; + const refused = await rpc( + app, + '/updateSettings', + { pidsLimit: 999 }, + { authorization: `Bearer ${keyToken}` }, + ); + expect(refused.statusCode).toBe(403); + expect(refused.json().message).toMatch(/cannot manage API keys/); + expect((await settingsOf(app)).pidsLimit).toBe(4096); + }); +}); + +describe('updateSettings: the S3 archive store', () => { + it('a passing probe writes the store and answers the view shape, keys never echoed', async () => { + const probed: string[] = []; + const { app } = testGateway( + {}, + { + probeS3: async (s3) => { + probed.push(s3.bucket); + }, + }, + ); + const res = await rpc(app, '/updateSettings', { s3: S3_PATCH }); + expect(res.statusCode).toBe(200); + expect(probed).toEqual(['patched-bucket']); + expect(updateSettingsResponseSchema.parse(res.json()).settings.s3).toEqual({ + endpoint: 'http://127.0.0.1:9000', + bucket: 'patched-bucket', + region: 'us-east-1', + forcePathStyle: true, + }); + expect(res.body).not.toContain('patch-secret-never-on-the-wire'); + expect(res.body).not.toContain('patch-key'); + // The adjudication flipped live: archiving is now available. + const body = getConfigResponseSchema.parse( + (await rpc(app, '/getConfig')).json(), + ); + expect(body.archive.enabled).toBe(true); + }); + + it('an unreachable store answers 502 and the table stays untouched', async () => { + const { app } = testGateway( + {}, + { + probeS3: () => Promise.reject(new Error('connect ECONNREFUSED')), + }, + ); + const res = await rpc(app, '/updateSettings', { s3: S3_PATCH }); + expect(res.statusCode).toBe(502); + expect(res.json().message).toMatch(/nothing was saved/); + expect((await settingsOf(app)).s3).toBeNull(); + }); + + it("an S3-refused probe (4xx) answers 400 with S3's own words", async () => { + const { app } = testGateway( + {}, + { + probeS3: () => + Promise.reject(new S3ProbeError('AccessDenied: key rejected', 403)), + }, + ); + const res = await rpc(app, '/updateSettings', { s3: S3_PATCH }); + expect(res.statusCode).toBe(400); + expect(res.json().message).toMatch(/AccessDenied: key rejected/); + expect((await settingsOf(app)).s3).toBeNull(); + }); + + it('refuses to clear or move the store while any node reports sandboxes archived in it, by fleet count', async () => { + const h = testGateway(S3_ENV); + reporting(h, 'b', { archived: 2 }); + reporting(h, 'c', { restoring: 1 }); + + const cleared = await rpc(h.app, '/updateSettings', { s3: null }); + expect(cleared.statusCode).toBe(400); + expect(cleared.json().message).toMatch(/3 sandboxes are archived/); + expect(cleared.json().message).toMatch(/across the fleet/); + + const moved = await rpc(h.app, '/updateSettings', { + s3: { ...S3_PATCH, endpoint: S3_ENV.DORMICE_S3_ENDPOINT }, + }); + expect(moved.statusCode).toBe(400); + expect(moved.json().message).toMatch(/moving it to another/); + + // Same endpoint+bucket, new credentials: nothing moves, allowed. + const rotated = await rpc(h.app, '/updateSettings', { + s3: { + ...S3_PATCH, + endpoint: S3_ENV.DORMICE_S3_ENDPOINT, + bucket: S3_ENV.DORMICE_S3_BUCKET, + }, + }); + expect(rotated.statusCode).toBe(200); + expect((await settingsOf(h.app)).s3?.bucket).toBe('seed-bucket'); + }); + + it('a node that has not reported since the gateway started makes the count unknown: 503 with Retry-After, not a guess', async () => { + const h = testGateway(S3_ENV); + reporting(h, 'b', { archived: 0 }); + // A node known from its row (a previous gateway life) that has not + // checked in yet: its disks cannot be counted. + const silent = reporting(h, 'c'); + silent.reading = null; + silent.lastCheckInAt = null; + const res = await rpc(h.app, '/updateSettings', { s3: null }); + expect(res.statusCode).toBe(503); + expect(res.headers['retry-after']).toBe('15'); + expect(res.json().message).toMatch(/node c has not checked in/); + expect((await settingsOf(h.app)).s3?.bucket).toBe('seed-bucket'); + // Once it has reported (nothing archived there), the clear goes through. + reporting(h, 'c'); + expect((await rpc(h.app, '/updateSettings', { s3: null })).statusCode).toBe( + 200, + ); + expect((await settingsOf(h.app)).s3).toBeNull(); + }); + + it('enabling from off is allowed even with archived rows reported — the drift repair path', async () => { + const h = testGateway(); + reporting(h, 'b', { archived: 5 }); + const res = await rpc(h.app, '/updateSettings', { s3: S3_PATCH }); + expect(res.statusCode).toBe(200); + expect((await settingsOf(h.app)).s3?.bucket).toBe('patched-bucket'); + }); + + it('with no node at all the count is zero and the store may be cleared', async () => { + const { app } = testGateway(S3_ENV); + expect((await rpc(app, '/updateSettings', { s3: null })).statusCode).toBe( + 200, + ); + }); +}); + +describe('updateSettings: the sandbox domain', () => { + it('sets, reports and clears the domain, with immediate effect on getConfig', async () => { + const { app } = testGateway(); + const set = await rpc(app, '/updateSettings', { + sandboxDomain: 'sbx.example.com', + }); + expect(set.statusCode).toBe(200); + expect( + updateSettingsResponseSchema.parse(set.json()).settings.sandboxDomain, + ).toBe('sbx.example.com'); + expect((await settingsOf(app)).sandboxDomain).toBe('sbx.example.com'); + + const cleared = await rpc(app, '/updateSettings', { sandboxDomain: null }); + expect(cleared.statusCode).toBe(200); + expect((await settingsOf(app)).sandboxDomain).toBeNull(); + }); + + it('refuses anything but a bare hostname', async () => { + const { app } = testGateway(); + for (const bad of [ + 'https://sbx.example.com', + 'sbx.example.com:8080', + '.sbx.example.com', + 'single-label', + ]) { + const res = await rpc(app, '/updateSettings', { sandboxDomain: bad }); + expect(res.statusCode, bad).toBe(400); + const alias = await rpc(app, '/updateSettings', { + sandboxDomain: 'sbx.example.com', + sandboxDomainAliases: [bad], + }); + expect(alias.statusCode, bad).toBe(400); + } + }); + + it('sets, reports and clears aliases', async () => { + const { app } = testGateway({ DORMICE_SANDBOX_DOMAIN: 'sbx.example.com' }); + const set = await rpc(app, '/updateSettings', { + sandboxDomainAliases: ['a.example.com', 'b.example.com'], + }); + expect(set.statusCode).toBe(200); + expect((await settingsOf(app)).sandboxDomainAliases).toEqual([ + 'a.example.com', + 'b.example.com', + ]); + const cleared = await rpc(app, '/updateSettings', { + sandboxDomainAliases: [], + }); + expect(cleared.statusCode).toBe(200); + expect((await settingsOf(app)).sandboxDomainAliases).toEqual([]); + }); + + it('refuses alias lists that contradict the post-patch state, honestly', async () => { + const { app } = testGateway({ DORMICE_SANDBOX_DOMAIN: 'sbx.example.com' }); + + const dup = await rpc(app, '/updateSettings', { + sandboxDomainAliases: ['a.example.com', 'A.example.com'], + }); + expect(dup.statusCode).toBe(400); + expect(dup.json().message).toContain('more than once'); + + const overlap = await rpc(app, '/updateSettings', { + sandboxDomainAliases: ['SBX.example.com'], + }); + expect(overlap.statusCode).toBe(400); + expect(overlap.json().message).toContain('already the sandbox domain'); + + const orphanApp = testGateway().app; + const orphan = await rpc(orphanApp, '/updateSettings', { + sandboxDomainAliases: ['a.example.com'], + }); + expect(orphan.statusCode).toBe(400); + expect(orphan.json().message).toContain('set sandboxDomain first'); + + expect( + ( + await rpc(app, '/updateSettings', { + sandboxDomainAliases: ['a.example.com'], + }) + ).statusCode, + ).toBe(200); + const dangling = await rpc(app, '/updateSettings', { sandboxDomain: null }); + expect(dangling.statusCode).toBe(400); + expect(dangling.json().message).toContain('sandboxDomainAliases'); + + // The atomic swap, expressed whole, passes... + const swap = await rpc(app, '/updateSettings', { + sandboxDomain: 'a.example.com', + sandboxDomainAliases: ['sbx.example.com'], + }); + expect(swap.statusCode).toBe(200); + const swapped = updateSettingsResponseSchema.parse(swap.json()).settings; + expect(swapped.sandboxDomain).toBe('a.example.com'); + expect(swapped.sandboxDomainAliases).toEqual(['sbx.example.com']); + // ...and the full clear. + const clear = await rpc(app, '/updateSettings', { + sandboxDomain: null, + sandboxDomainAliases: [], + }); + expect(clear.statusCode).toBe(200); + }); +}); + +describe('the default policy seed', () => { + it('follows the shared default with archiving off, and the one-week default with a store', async () => { + expect((await settingsOf(testGateway().app)).defaultPolicy).toEqual({ + ...DEFAULT_LIFECYCLE_POLICY, + archiveAfterSeconds: null, + }); + expect( + (await settingsOf(testGateway(S3_ENV).app)).defaultPolicy + .archiveAfterSeconds, + ).toBe(ARCHIVE_DEFAULT_SECONDS); + }); +}); diff --git a/packages/gateway/src/routes/settings.ts b/packages/gateway/src/routes/settings.ts new file mode 100644 index 00000000..c650af40 --- /dev/null +++ b/packages/gateway/src/routes/settings.ts @@ -0,0 +1,274 @@ +import type { S3Settings } from '@dormice/server/s3-store'; +import { + getConfigResponseSchema, + updateSettingsRequestSchema, + updateSettingsResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { z } from 'zod'; +import { CONFIG_KEYS, type Config, type ConfigSources } from '../config'; +import type { Db } from '../db/db'; +import { readS3Settings, readSettings, writeSettings } from '../db/settings'; +import type { Fleet } from '../fleet'; +import { probeS3 as defaultProbeS3, S3ProbeError } from '../probe'; + +export interface SettingsRoutesOptions { + config: Config; + db: Db; + fleet: Fleet; + sources: ConfigSources; + /** Test seam over the S3 round-trip probe; production uses the real one. */ + probeS3?: (s3: S3Settings) => Promise; +} + +/** + * The fleet's settings, read and written at the one door (design record + * #22). getConfig answers the gateway's env knobs (read-only; secrets + * present-or-absent) plus the settings table in force; updateSettings is + * a table write with a version bump — every node hears of it at its next + * check-in and applies it there (the pids sweep over running shells, the + * archiver's store, the proxy's domain are the node's to reconcile, so + * nothing here restarts or wakes a sandbox). Behind the admin gate: a + * leaked automation key must not be able to move the limits that contain + * it. + * + * The one guard that needs the fleet: moving or clearing the archive store + * strands every disk archived in it, and those disks live on the nodes' + * ledgers. The gateway holds no sandbox state, but every node's last + * check-in carries its census by state, so the count of archived and + * restoring sandboxes across the fleet is at hand — for every node that + * has reported since this gateway started. One that has not is a node + * whose disks cannot be counted, and the write refuses (503, retry after + * its next check-in) rather than guess. + */ +export const settingsRoutes: FastifyPluginAsyncZod< + SettingsRoutesOptions +> = async (app, { config, db, fleet, sources, probeS3 = defaultProbeS3 }) => { + app.post( + '/getConfig', + { + schema: { + response: { 200: getConfigResponseSchema }, + }, + }, + async () => { + const settings = readSettings(db); + const enabled = settings.s3 !== null; + return { + entries: (Object.keys(CONFIG_KEYS) as Array).map( + (key) => { + const { sensitive } = CONFIG_KEYS[key]; + const value = config[key]; + return { + key, + value: sensitive || value === undefined ? null : String(value), + source: sources[key], + ...(sensitive && value !== undefined ? { redacted: true } : {}), + }; + }, + ), + archive: { + enabled, + defaultSeconds: enabled + ? settings.defaultPolicy.archiveAfterSeconds + : null, + }, + settings, + }; + }, + ); + + app.post( + '/updateSettings', + { + schema: { + body: updateSettingsRequestSchema, + response: { + 200: updateSettingsResponseSchema, + 400: z.object({ message: z.string() }), + 502: z.object({ message: z.string() }), + 503: z.object({ message: z.string() }), + }, + }, + }, + async (request, reply) => { + const patch = request.body; + // The updatePolicy doctrine, judged against the post-patch state (an + // s3 group and an archiving default may arrive in one patch): a + // default that promises archiving on a fleet with no store would be + // a standing lie in every acquire. + const current = readSettings(db); + const s3After = + patch.s3 !== undefined ? patch.s3 !== null : current.s3 !== null; + if ( + !s3After && + patch.defaultPolicy !== undefined && + patch.defaultPolicy.archiveAfterSeconds !== null + ) { + return reply.code(400).send({ + message: + 'invalid default policy: archiving requires an S3 archive store — configure one in the console settings first', + }); + } + // The alias-list guard, judged against the post-patch state — domain + // and aliases may arrive in one patch, which is exactly how the + // console swaps the canonical domain atomically. Violations are + // refused, never silently rewritten (the echoed settings must be + // what was written). Before the s3 block on purpose: pure in-memory + // checks don't queue behind a network probe. + if ( + patch.sandboxDomain !== undefined || + patch.sandboxDomainAliases !== undefined + ) { + const domainAfter = + patch.sandboxDomain !== undefined + ? patch.sandboxDomain + : current.sandboxDomain; + const aliasesAfter = + patch.sandboxDomainAliases ?? current.sandboxDomainAliases; + const lower = aliasesAfter.map((alias) => alias.toLowerCase()); + const dup = aliasesAfter.find( + (alias, i) => lower.indexOf(alias.toLowerCase()) !== i, + ); + if (dup !== undefined) { + return reply.code(400).send({ + message: `sandboxDomainAliases lists ${dup} more than once — hostnames are case-insensitive, send each alias exactly once`, + }); + } + if (domainAfter !== null && lower.includes(domainAfter.toLowerCase())) { + return reply.code(400).send({ + message: `${domainAfter} is already the sandbox domain — sandboxDomainAliases only takes the extra hostnames`, + }); + } + if (domainAfter === null && aliasesAfter.length > 0) { + return reply.code(400).send({ + message: + patch.sandboxDomain === null + ? `clearing sandboxDomain would leave ${aliasesAfter.length} alias${aliasesAfter.length === 1 ? '' : 'es'} pointing at nothing — clear sandboxDomainAliases (send []) in the same request` + : 'sandboxDomainAliases needs a sandbox domain in force — set sandboxDomain first', + }); + } + } + if (patch.s3 !== undefined) { + // The moving-store guard: archived disks live in the current + // endpoint+bucket, and pointing elsewhere (or clearing) would + // strand them. Enabling from off is always allowed — when drift + // left archived rows behind with no store, pointing back at the + // original bucket is the one repair path. Credential/region/ + // path-style changes move nothing and pass freely. + const store = readS3Settings(db); + const moving = + patch.s3 === null || + (store !== null && + (patch.s3.endpoint !== store.endpoint || + patch.s3.bucket !== store.bucket)); + if (store !== null && moving) { + const held = archivedAcrossFleet(fleet); + if ('unknown' in held) { + reply.header('retry-after', '15'); + return reply.code(503).send({ + message: `${held.unknown.map((id) => `node ${id}`).join(', ')} ${held.unknown.length === 1 ? 'has' : 'have'} not checked in since the gateway started, so the sandboxes archived in the current store cannot be counted — retry after ${held.unknown.length === 1 ? 'its' : 'their'} next check-in, or remove ${held.unknown.length === 1 ? 'it' : 'them'} if gone for good`, + }); + } + if (held.count > 0) { + return reply.code(400).send({ + message: `${held.count} sandbox${held.count === 1 ? ' is' : 'es are'} archived or restoring in the current store across the fleet — restore or destroy them before ${ + patch.s3 === null + ? 'clearing the archive store' + : 'moving it to another endpoint or bucket' + }`, + }); + } + } + if (patch.s3 !== null) { + // Probe BEFORE the write — a failure leaves the table untouched + // (probe.ts has why). + try { + await probeS3(patch.s3); + } catch (error) { + const probeFailure = + error instanceof S3ProbeError + ? error + : new S3ProbeError( + error instanceof Error ? error.message : String(error), + undefined, + ); + const status = + probeFailure.httpStatusCode !== undefined && + probeFailure.httpStatusCode >= 400 && + probeFailure.httpStatusCode < 500 + ? (400 as const) + : (502 as const); + return reply.code(status).send({ + message: `the S3 store did not pass a write-read-delete probe, nothing was saved — ${probeFailure.message}`, + }); + } + } + } + const settings = writeSettings(db, patch, new Date()); + request.log.info( + { settings }, + `fleet settings updated: ${[ + ...(patch.sandboxDefaults !== undefined + ? [ + `sandboxDefaults=${patch.sandboxDefaults.cpus}cpu/${patch.sandboxDefaults.memoryGb}GiB/${patch.sandboxDefaults.diskGb}GiB`, + ] + : []), + ...(patch.defaultPolicy !== undefined + ? [ + `defaultPolicy=${patch.defaultPolicy.freezeAfterSeconds}s/${patch.defaultPolicy.stopAfterSeconds ?? 'never'}/${patch.defaultPolicy.archiveAfterSeconds ?? 'never'}`, + ] + : []), + // Endpoint and bucket only — the keys never reach the log, the + // same "value never crosses" rule as the wire's. + ...(patch.s3 !== undefined + ? [ + patch.s3 === null + ? 's3=cleared' + : `s3=${patch.s3.endpoint}/${patch.s3.bucket}`, + ] + : []), + ...(patch.sandboxDomain !== undefined + ? [`sandboxDomain=${patch.sandboxDomain ?? 'cleared'}`] + : []), + ...(patch.sandboxDomainAliases !== undefined + ? [ + patch.sandboxDomainAliases.length === 0 + ? 'sandboxDomainAliases=cleared' + : `sandboxDomainAliases=${patch.sandboxDomainAliases.join('/')}`, + ] + : []), + ...(patch.pidsLimit !== undefined + ? [`pidsLimit=${patch.pidsLimit}`] + : []), + ].join(', ')}; the nodes apply it at their next check-in`, + ); + return { settings }; + }, + ); +}; + +/** + * How many sandboxes across the fleet are archived or restoring, by the + * nodes' last readings — or the ids of the nodes whose count is unknown + * because they have not checked in since this gateway started. A node + * that is merely late is still counted by its last reading: a sandbox + * archives over minutes, not the seconds a check-in can be late by, and + * the guard errs toward refusing anyway (a non-zero count refuses). + */ +function archivedAcrossFleet( + fleet: Fleet, +): { count: number } | { unknown: string[] } { + const unknown: string[] = []; + let count = 0; + for (const node of fleet.all()) { + if (node.reading === null) { + unknown.push(node.id); + continue; + } + count += + node.reading.sandboxes.byState.archived + + node.reading.sandboxes.byState.restoring; + } + return unknown.length > 0 ? { unknown: unknown.sort() } : { count }; +} diff --git a/packages/gateway/src/routes/templates.test.ts b/packages/gateway/src/routes/templates.test.ts new file mode 100644 index 00000000..22ecd97f --- /dev/null +++ b/packages/gateway/src/routes/templates.test.ts @@ -0,0 +1,158 @@ +import { describe, expect, it } from 'vitest'; +import { readConfigVersion } from '../db/settings'; +import type { AskVerb } from '../lookup'; +import { checkInOf, TEST_TOKEN, testGateway } from '../testing'; + +const authed = { authorization: `Bearer ${TEST_TOKEN}` }; + +/** + * A fleet whose nodes answer templateUsers from a script: which names + * each node reports for a template, or silence. The route is about the + * decision, not the transport (lookup.ts httpAsk is the transport, and + * app.test.ts exercises it over sockets). + */ +function templatesGateway(users: Record) { + const askVerb: AskVerb = async (node, verb, _body, schema) => { + if (verb !== 'templateUsers') throw new Error(`unexpected verb ${verb}`); + const answer = users[node.id]; + if (answer === undefined || answer === 'silent') { + return { kind: 'silent', why: 'ECONNREFUSED' }; + } + return { kind: 'answer', value: schema.parse({ sandboxNames: answer }) }; + }; + const { app, db, fleet } = testGateway({}, { askVerb }); + let host = 1; + for (const id of Object.keys(users)) { + const outcome = fleet.checkIn(checkInOf(id, `http://10.0.0.${host++}:80`)); + if ('refused' in outcome) throw new Error(outcome.refused); + } + return { app, db }; +} + +type App = ReturnType['app']; + +function rpc( + app: App, + url: string, + payload: Record = {}, + headers: Record = authed, +) { + return app.inject({ method: 'POST', url, headers, payload }); +} + +describe('templates on the gateway', () => { + it('registers, lists, and counts the version up for the nodes; registering is admin-only', async () => { + const { app, db } = templatesGateway({}); + const anon = await app.inject({ + method: 'POST', + url: '/registerTemplate', + payload: { name: 'py', image: 'img-a' }, + }); + expect(anon.statusCode).toBe(401); + + const res = await rpc(app, '/registerTemplate', { + name: 'py', + image: 'img-a', + }); + expect(res.statusCode).toBe(200); + expect(res.json().template).toMatchObject({ name: 'py', image: 'img-a' }); + expect(readConfigVersion(db)).toBe(2); + const listed = await rpc(app, '/listTemplates'); + expect(listed.json().templates).toMatchObject([ + { name: 'py', image: 'img-a' }, + ]); + + const minted = await rpc(app, '/createApiKey', { name: 'robot' }); + const asKey = { authorization: `Bearer ${minted.json().token}` }; + expect( + (await rpc(app, '/registerTemplate', { name: 'x', image: 'i' }, asKey)) + .statusCode, + ).toBe(403); + expect((await rpc(app, '/listTemplates', {}, asKey)).statusCode).toBe(403); + }); + + it('re-registering re-points the name and keeps its birth date — the upgrade verb; the same image is a no-op the nodes do not hear about', async () => { + const { app, db } = templatesGateway({}); + const first = ( + await rpc(app, '/registerTemplate', { name: 'py', image: 'img-a' }) + ).json().template; + expect(first.updatedAt).toBe(first.createdAt); + const version = readConfigVersion(db); + await new Promise((resolve) => setTimeout(resolve, 5)); + const same = ( + await rpc(app, '/registerTemplate', { name: 'py', image: 'img-a' }) + ).json().template; + expect(same.updatedAt).toBe(first.updatedAt); + expect(readConfigVersion(db)).toBe(version); + const second = ( + await rpc(app, '/registerTemplate', { name: 'py', image: 'img-b' }) + ).json().template; + expect(second.image).toBe('img-b'); + expect(second.createdAt).toBe(first.createdAt); + expect(Date.parse(second.updatedAt)).toBeGreaterThan( + Date.parse(first.updatedAt), + ); + expect(readConfigVersion(db)).toBe(version + 1); + expect((await rpc(app, '/listTemplates')).json().templates).toHaveLength(1); + }); + + it("rejects a malformed name, and 'base' as reserved", async () => { + const { app } = templatesGateway({}); + const bad = await rpc(app, '/registerTemplate', { + name: '-bad', + image: 'img', + }); + expect(bad.statusCode).toBe(400); + const base = await rpc(app, '/registerTemplate', { + name: 'base', + image: 'img', + }); + expect(base.statusCode).toBe(400); + expect(base.json().message).toMatch(/'base' is reserved/); + }); + + it('removal asks every node; a name in use anywhere is a 409 naming the sandboxes by node', async () => { + const { app, db } = templatesGateway({ b: ['alice', 'bob'], c: [] }); + await rpc(app, '/registerTemplate', { name: 'py', image: 'img-a' }); + const version = readConfigVersion(db); + const refused = await rpc(app, '/removeTemplate', { name: 'py' }); + expect(refused.statusCode).toBe(409); + expect(refused.json().message).toContain('alice, bob on node b'); + expect(refused.json().message).not.toContain('node c'); + expect((await rpc(app, '/listTemplates')).json().templates).toHaveLength(1); + expect(readConfigVersion(db)).toBe(version); + }); + + it('a silent node holds the removal: 503 with Retry-After naming it', async () => { + const { app } = templatesGateway({ b: [], c: 'silent' }); + await rpc(app, '/registerTemplate', { name: 'py', image: 'img-a' }); + const res = await rpc(app, '/removeTemplate', { name: 'py' }); + expect(res.statusCode).toBe(503); + expect(res.headers['retry-after']).toBe('15'); + expect(res.json().message).toMatch(/node c \(ECONNREFUSED\)/); + expect((await rpc(app, '/listTemplates')).json().templates).toHaveLength(1); + }); + + it('with every node answering "unused" the template goes, the version counts up, and a second removal is an honest false', async () => { + const { app, db } = templatesGateway({ b: [], c: [] }); + await rpc(app, '/registerTemplate', { name: 'py', image: 'img-a' }); + const version = readConfigVersion(db); + expect((await rpc(app, '/removeTemplate', { name: 'py' })).json()).toEqual({ + removed: true, + }); + expect(readConfigVersion(db)).toBe(version + 1); + expect((await rpc(app, '/listTemplates')).json().templates).toEqual([]); + expect((await rpc(app, '/removeTemplate', { name: 'py' })).json()).toEqual({ + removed: false, + }); + expect(readConfigVersion(db)).toBe(version + 1); + }); + + it('with no node at all a removal needs nobody', async () => { + const { app } = templatesGateway({}); + await rpc(app, '/registerTemplate', { name: 'py', image: 'img-a' }); + expect((await rpc(app, '/removeTemplate', { name: 'py' })).json()).toEqual({ + removed: true, + }); + }); +}); diff --git a/packages/gateway/src/routes/templates.ts b/packages/gateway/src/routes/templates.ts new file mode 100644 index 00000000..62903f9e --- /dev/null +++ b/packages/gateway/src/routes/templates.ts @@ -0,0 +1,129 @@ +import { + listTemplatesResponseSchema, + registerTemplateRequestSchema, + registerTemplateResponseSchema, + removeTemplateRequestSchema, + removeTemplateResponseSchema, + templateUsersResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { z } from 'zod'; +import type { Db } from '../db/db'; +import { + listTemplates, + registerTemplate, + removeTemplate, +} from '../db/templates'; +import type { Fleet } from '../fleet'; +import type { AskVerb } from '../lookup'; +import { RETRY_AFTER_SECONDS } from '../raw'; + +export interface TemplateRoutesOptions { + db: Db; + fleet: Fleet; + /** Asks one node one verb on the gateway's account (lookup.ts httpAsk). */ + ask: AskVerb; +} + +/** + * Templates, fleet-wide: a name for an image, registered once here and + * carried to every node in the configuration bundle (design record #29 — + * a template sandbox's cold wake resolves name → image on its node, with + * or without a gateway present). Registration is configuration, not a + * check: the image is not looked for anywhere — it may legitimately + * arrive later, and a node whose Docker lacks it fails the create with + * Docker's own honest error, as the daemon always has. + * + * Removal is the one verb that needs the nodes: a template deleted under + * a sandbox would wake it onto a dangling name, and the sandboxes live on + * the nodes' ledgers. So every node is asked (templateUsers, two seconds, + * in parallel — the lookup's discipline) and the removal is refused while + * any node names a sandbox (409, the names by node) or any node is silent + * (503, Retry-After: a silent node may hold users nobody can see). + */ +export const templateRoutes: FastifyPluginAsyncZod< + TemplateRoutesOptions +> = async (app, { db, fleet, ask }) => { + app.post( + '/registerTemplate', + { + schema: { + body: registerTemplateRequestSchema, + response: { 200: registerTemplateResponseSchema }, + }, + }, + async (request) => { + const template = registerTemplate(db, request.body); + request.log.info( + { template: template.name, image: template.image }, + 'template registered; the nodes learn it at their next check-in', + ); + return { template }; + }, + ); + + app.post( + '/listTemplates', + { + schema: { + response: { 200: listTemplatesResponseSchema }, + }, + }, + async () => ({ templates: listTemplates(db) }), + ); + + app.post( + '/removeTemplate', + { + schema: { + body: removeTemplateRequestSchema, + response: { + 200: removeTemplateResponseSchema, + 409: z.object({ message: z.string() }), + 503: z.object({ message: z.string() }), + }, + }, + }, + async (request, reply) => { + const { name } = request.body; + const answers = await Promise.all( + fleet.all().map(async (node) => ({ + node, + asked: await ask( + node, + 'templateUsers', + { name }, + templateUsersResponseSchema, + ), + })), + ); + const silent = answers.flatMap(({ node, asked }) => + asked.kind === 'silent' ? [`node ${node.id} (${asked.why})`] : [], + ); + if (silent.length > 0) { + reply.header('retry-after', String(RETRY_AFTER_SECONDS)); + return reply.code(503).send({ + message: `cannot remove template '${name}' while a node has not answered whether its sandboxes use it: ${silent.join(', ')} — retry after Retry-After, or remove the node if it is gone for good`, + }); + } + const users = answers.flatMap(({ node, asked }) => + asked.kind === 'answer' && asked.value.sandboxNames.length > 0 + ? [`${asked.value.sandboxNames.join(', ')} on node ${node.id}`] + : [], + ); + if (users.length > 0) { + return reply.code(409).send({ + message: `template '${name}' is used by sandboxes: ${users.join('; ')} — destroy them or move them to another template first`, + }); + } + const removed = removeTemplate(db, name); + if (removed) { + request.log.info( + { template: name }, + 'template removed; the nodes drop it at their next check-in', + ); + } + return { removed }; + }, + ); +}; diff --git a/packages/gateway/src/testing.ts b/packages/gateway/src/testing.ts index 70586bcd..99e6560e 100644 --- a/packages/gateway/src/testing.ts +++ b/packages/gateway/src/testing.ts @@ -3,12 +3,13 @@ import { KeyedQueue } from '@dormice/server/keyed-queue'; import type { CheckInRequest, NodeReading } from '@dormice/shared'; import { buildGatewayApp } from './app'; import { NameCache } from './cache'; -import { loadConfig } from './config'; +import { configSources, loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; import { ensureSettings } from './db/settings'; import { Finder } from './find'; import { Fleet } from './fleet'; -import { type AskNode, httpAskNode } from './lookup'; +import type { Ingress } from './ingress'; +import { type AskNode, type AskVerb, httpAskNode } from './lookup'; /** * Test scaffolding shared by the gateway's suites: a node's reading and @@ -26,12 +27,16 @@ export function reading( cores?: number; active?: number; frozen?: number; + archived?: number; + restoring?: number; memAvail?: number; diskAvail?: number | null; } = {}, ): NodeReading { const frozen = over.frozen ?? 0; const active = over.active ?? 10; + const archived = over.archived ?? 0; + const restoring = over.restoring ?? 0; return { host: { cpuCount: over.cores ?? 8, @@ -50,8 +55,8 @@ export function reading( availableBytes: over.diskAvail ?? 5e11, }, sandboxes: { - total: active + frozen, - byState: { active, frozen, stopped: 0, archived: 0, restoring: 0 }, + total: active + frozen + archived + restoring, + byState: { active, frozen, stopped: 0, archived, restoring }, }, }; } @@ -82,15 +87,24 @@ export function checkInOf( */ export function testGateway( env: Record = {}, - opts: { consoleDistDir?: string; ask?: AskNode } = {}, + opts: { + consoleDistDir?: string; + ask?: AskNode; + /** Scripted answers to the verbs the gateway asks nodes itself (templateUsers). */ + askVerb?: AskVerb; + ingress?: Ingress; + /** Forged by default: the suites here are about the settings machinery, not S3's availability. */ + probeS3?: NonNullable[0]['probeS3']>; + } = {}, ) { const db = openDb(':memory:'); migrateDb(db, MIGRATIONS); - const config = loadConfig({ + const rawEnv = { DORMICE_API_TOKEN: TEST_TOKEN, DORMICE_GATEWAY_DB_PATH: ':memory:', ...env, - }); + }; + const config = loadConfig(rawEnv); ensureSettings(db, config); const fleet = new Fleet(db); const finder = new Finder( @@ -108,6 +122,13 @@ export function testGateway( logger: false, build: null, consoleDistDir: opts.consoleDistDir, + ingress: opts.ingress, + ask: opts.askVerb, + probeS3: opts.probeS3 ?? (() => Promise.resolve()), + // Off the same raw env the config was parsed from, through the real + // function: the parsed object drops unset optional knobs, so deriving + // sources from its keys would leave those entries without a source. + sources: configSources(rawEnv), }); return { app, db, fleet, config }; } diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index b0c2f63e..01bb3621 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -32,6 +32,7 @@ import { hostRoutes } from './routes/host'; import { ingressRoutes } from './routes/ingress'; import { sandboxRoutes } from './routes/sandboxes'; import { settingsRoutes } from './routes/settings'; +import { templateUsersRoutes } from './routes/template-users'; import { templateRoutes } from './routes/templates'; import { upgradeRoutes } from './routes/upgrade'; import { createSandboxProxy } from './sandbox-proxy'; @@ -266,6 +267,7 @@ export function buildApp({ archiver, }); await api.register(templateRoutes, { db }); + await api.register(templateUsersRoutes, { db }); await api.register(hostRoutes, { config, db, executor }); await api.register(ingressRoutes, { ingress }); await api.register(configRoutes, { config, db, sources }); diff --git a/packages/server/src/routes/template-users.ts b/packages/server/src/routes/template-users.ts new file mode 100644 index 00000000..f0567ec1 --- /dev/null +++ b/packages/server/src/routes/template-users.ts @@ -0,0 +1,35 @@ +import { + templateUsersRequestSchema, + templateUsersResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import type { Db } from '../db/db'; +import { sandboxNamesUsingTemplate } from '../db/templates'; + +export interface TemplateUsersRoutesOptions { + db: Db; +} + +/** + * The gateway's second question on its own account (the first is + * lookupSandbox): which sandboxes here still use a template? Asked of + * every node before the gateway removes one, so no sandbox anywhere + * wakes onto a dangling name. A ledger read, nothing more — no slot, no + * wake, no touch. + */ +export const templateUsersRoutes: FastifyPluginAsyncZod< + TemplateUsersRoutesOptions +> = async (app, { db }) => { + app.post( + '/templateUsers', + { + schema: { + body: templateUsersRequestSchema, + response: { 200: templateUsersResponseSchema }, + }, + }, + async (request) => ({ + sandboxNames: sandboxNamesUsingTemplate(db, request.body.name), + }), + ); +}; diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts index 7022acfc..201de5a4 100644 --- a/packages/shared/src/gateway.ts +++ b/packages/shared/src/gateway.ts @@ -118,6 +118,10 @@ export const nodeViewSchema = z.object({ endpoint: z.string(), /** ISO 8601 UTC — the first check-in. */ addedAt: z.iso.datetime(), + /** The node's own setting: managed swap on its data disk, GiB (updateNodeSettings). */ + swapGb: z.number().int().nonnegative(), + /** The configuration version the node last reported it runs; null until it has said (or before it pulls one). */ + configVersion: z.number().int().nullable(), /** ISO 8601 UTC — null only right after a gateway start, before the node's next check-in. */ lastCheckInAt: z.iso.datetime().nullable(), intervalSeconds: z.number().int().positive().nullable(), @@ -159,3 +163,32 @@ export const removeNodeResponseSchema = z.object({ }); export type RemoveNodeResponse = z.infer; + +/** + * updateNodeSettings — the one per-node knob: how much swap the node's + * daemon manages on its own data disk, on top of the host's own. A + * machine's setting, not the fleet's (a 29 GB box and a 243 GB box want + * different numbers), so it lives on the node's row and reaches that node + * alone with the next configuration bundle. Growing takes effect at the + * node within one check-in; shrinking waits for that host's next reboot — + * an active swapfile is never unmounted (server/swap.ts has the rule). + * Refused (400) for a node whose last reading says its daemon cannot + * manage swap (a non-Linux host, the fake executor): a target nothing + * will ever reconcile must refuse, not be stored. 404 for an unknown id. + */ +export const updateNodeSettingsRequestSchema = z.object({ + id: z.string().min(1), + swapGb: z.number().int().nonnegative(), +}); + +export type UpdateNodeSettingsRequest = z.infer< + typeof updateNodeSettingsRequestSchema +>; + +export const updateNodeSettingsResponseSchema = z.object({ + node: nodeViewSchema, +}); + +export type UpdateNodeSettingsResponse = z.infer< + typeof updateNodeSettingsResponseSchema +>; diff --git a/packages/shared/src/templates.ts b/packages/shared/src/templates.ts index bafc0d16..4349124b 100644 --- a/packages/shared/src/templates.ts +++ b/packages/shared/src/templates.ts @@ -92,3 +92,24 @@ export const removeTemplateResponseSchema = z.object({ export type RemoveTemplateResponse = z.infer< typeof removeTemplateResponseSchema >; + +/** + * templateUsers({ name }) — "which of your sandboxes still use this + * template?", the question the gateway puts to every node before it + * removes a template (design record #29: templates are the gateway's, the + * sandboxes that reference them are the nodes'). Read-only. A node + * answers with the names, the gateway refuses the removal while any node + * names one, and a node that does not answer holds the removal too: a + * template deleted under a sandbox would wake it onto a dangling name. + */ +export const templateUsersRequestSchema = z.object({ + name: templateNameSchema, +}); + +export type TemplateUsersRequest = z.infer; + +export const templateUsersResponseSchema = z.object({ + sandboxNames: z.array(z.string()), +}); + +export type TemplateUsersResponse = z.infer; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 421a0853..5e3734aa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -200,6 +200,9 @@ importers: drizzle-orm: specifier: ^0.45.2 version: 0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.11.1)(kysely@0.28.17) + execa: + specifier: ^9.6.1 + version: 9.6.1 fastify: specifier: ^5.10.0 version: 5.10.0 From c536b30b391adcb845373f17f97605cc9c75761b Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 17:23:44 +0800 Subject: [PATCH 34/89] The node takes its configuration from the gateway: the check-in is the pull, the ledger holds a copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every daemon is a node of a gateway now (design record #22: a single machine is a fleet of one; DORMICE_GATEWAY_ENDPOINT defaults to the gateway beside it). The check-in carries the version of the configuration copy the node runs, and the gateway answers its current version — with the whole bundle (fleet settings with the store's keys, the node's own swap target, every template) whenever the two differ. No pull verb, no record of who was told what: the node states what it runs, the gateway compares. The node writes the bundle whole into runtime_settings (now versioned, migration 0025) and the templates table in one transaction, then moves the two knobs with a reality on the host: the pids cap over running shells and the managed swap target. A node without a copy fetches one before it listens — after the startup guard, so a daemon that will refuse to start never registers with the fleet — and reports `configVersion: null` meanwhile, which placement reads as "not open yet". A node holding a copy serves on it while its gateway is away. Gone from the node: the settings, config, template, API-key, ingress and console routes, the login throttle, the S3 probe, the api_keys and console_account code (their tables stay for the one-time import in cut 4), @fastify/cookie and @fastify/static. Its env loses the thirteen fleet knobs; a machine upgraded with them still in its env file gets one boot line naming them. The node judges one credential, the fleet token. New: POST /envdToken and /templateUsers are its answers to the gateway; nodes report `managedSwap` so updateNodeSettings refuses a target a daemon cannot honor. Tests follow the move: the daemon's suites configure the node through `configureNode` (a bundle applied the way a check-in would) instead of env seeds; the SDK's and CLI's suites embed a gateway beside the daemon through the gateway's new library surface and drive the fleet verbs there; the e2e setup boots node A behind its own gateway in every mode (the fleet exam keeps its own gateway and nodes), and the console, settings, ingress, template and key suites point at the door, waiting for the node's copy to settle where the node must act. --- e2e/src/cli.test.ts | 90 +- e2e/src/console.test.ts | 45 +- e2e/src/e2b.test.ts | 28 +- e2e/src/gateway.test.ts | 178 ++-- e2e/src/helpers.ts | 98 +++ e2e/src/ingress.test.ts | 22 +- e2e/src/native.test.ts | 121 ++- e2e/src/settings-hot.test.ts | 73 +- e2e/src/setup/daemon.ts | 277 ++++--- packages/cli/package.json | 1 + packages/cli/src/commands.test.ts | 84 +- packages/cli/src/doctor.ts | 25 +- packages/gateway/package.json | 6 + packages/gateway/src/app.test.ts | 4 + packages/gateway/src/app.ts | 2 +- packages/gateway/src/db/node-config.ts | 31 + packages/gateway/src/fleet.ts | 1 + packages/gateway/src/index.ts | 16 + packages/gateway/src/placement.ts | 15 +- packages/gateway/src/routes/nodes.test.ts | 173 ++++ packages/gateway/src/routes/nodes.ts | 52 +- packages/gateway/src/routes/settings.ts | 8 +- packages/gateway/src/testing.ts | 27 +- packages/gateway/tsup.config.ts | 11 +- packages/sdk/package.json | 1 + packages/sdk/src/client.test.ts | 174 ++-- packages/server/drizzle/0025_config-copy.sql | 2 + .../server/drizzle/meta/0025_snapshot.json | 780 ++++++++++++++++++ packages/server/drizzle/meta/_journal.json | 7 + packages/server/package.json | 2 - packages/server/src/app.test.ts | 552 +++---------- packages/server/src/app.ts | 137 +-- packages/server/src/archive/probe.ts | 62 -- packages/server/src/check-in.test.ts | 110 ++- packages/server/src/check-in.ts | 64 +- packages/server/src/config.test.ts | 94 +-- packages/server/src/config.ts | 259 ++---- packages/server/src/db/account.ts | 51 -- packages/server/src/db/api-keys.ts | 210 ----- packages/server/src/db/schema.ts | 118 ++- packages/server/src/db/settings.ts | 342 +++----- packages/server/src/db/templates.ts | 59 +- packages/server/src/e2b/compat.test.ts | 153 +--- packages/server/src/index.ts | 11 + packages/server/src/ingress.test.ts | 279 ------- packages/server/src/ingress.ts | 239 ------ packages/server/src/login-throttle.test.ts | 53 -- packages/server/src/login-throttle.ts | 76 -- packages/server/src/main.ts | 241 +++--- packages/server/src/metrics-sampler.test.ts | 20 +- packages/server/src/node-config.test.ts | 288 +++++++ packages/server/src/node-config.ts | 91 ++ packages/server/src/routes/api-keys.ts | 154 ---- packages/server/src/routes/config.ts | 63 -- packages/server/src/routes/console.test.ts | 436 ---------- packages/server/src/routes/console.ts | 229 ----- packages/server/src/routes/envd-token.test.ts | 83 ++ packages/server/src/routes/ingress.ts | 81 -- .../server/src/routes/observability.test.ts | 132 +-- packages/server/src/routes/settings.test.ts | 761 ----------------- packages/server/src/routes/settings.ts | 229 ----- packages/server/src/routes/spec.test.ts | 20 +- packages/server/src/routes/templates.ts | 81 -- packages/server/src/sandbox-proxy.test.ts | 52 +- packages/server/src/sandbox-proxy.ts | 4 +- packages/server/src/testing.ts | 151 ++++ packages/shared/src/config.ts | 8 + packages/shared/src/gateway.ts | 69 +- pnpm-lock.yaml | 12 +- 69 files changed, 3390 insertions(+), 5038 deletions(-) create mode 100644 e2e/src/helpers.ts create mode 100644 packages/gateway/src/db/node-config.ts create mode 100644 packages/gateway/src/index.ts create mode 100644 packages/gateway/src/routes/nodes.test.ts create mode 100644 packages/server/drizzle/0025_config-copy.sql create mode 100644 packages/server/drizzle/meta/0025_snapshot.json delete mode 100644 packages/server/src/archive/probe.ts delete mode 100644 packages/server/src/db/account.ts delete mode 100644 packages/server/src/db/api-keys.ts delete mode 100644 packages/server/src/ingress.test.ts delete mode 100644 packages/server/src/ingress.ts delete mode 100644 packages/server/src/login-throttle.test.ts delete mode 100644 packages/server/src/login-throttle.ts create mode 100644 packages/server/src/node-config.test.ts create mode 100644 packages/server/src/node-config.ts delete mode 100644 packages/server/src/routes/api-keys.ts delete mode 100644 packages/server/src/routes/config.ts delete mode 100644 packages/server/src/routes/console.test.ts delete mode 100644 packages/server/src/routes/console.ts create mode 100644 packages/server/src/routes/envd-token.test.ts delete mode 100644 packages/server/src/routes/ingress.ts delete mode 100644 packages/server/src/routes/settings.test.ts delete mode 100644 packages/server/src/routes/settings.ts delete mode 100644 packages/server/src/routes/templates.ts create mode 100644 packages/server/src/testing.ts diff --git a/e2e/src/cli.test.ts b/e2e/src/cli.test.ts index ccd781b1..61befa80 100644 --- a/e2e/src/cli.test.ts +++ b/e2e/src/cli.test.ts @@ -30,6 +30,21 @@ function cli(...args: string[]) { }); } +/** + * The same binary pointed at the fleet's door — node A's own gateway: the + * template and apikey command groups speak to it (design record #22), the + * way an operator's shell would with DORMICE_ENDPOINT set to the gateway. + */ +function door(...args: string[]) { + return run('node', [CLI, ...args], { + env: { + ...process.env, + DORMICE_ENDPOINT: inject('dormiceGatewayEndpoint'), + DORMICE_API_TOKEN: inject('dormiceToken'), + }, + }); +} + describe('dor CLI against a real daemon', () => { it('sandbox ls shows a sandbox acquired through the SDK', async () => { const sdk = new Dormice({ @@ -128,22 +143,22 @@ describe('dor CLI against a real daemon', () => { expect(pulled.stdout).toBe('through the CLI\n'); }); - it('template add, ls and rm run the registration life through the real binary', async () => { - const added = await cli('template', 'add', 'cli-tpl', 'img:cli'); + it('template add, ls and rm run the registration life through the real binary, at the door', async () => { + const added = await door('template', 'add', 'cli-tpl', 'img:cli'); expect(added.stdout).toContain('Registered template "cli-tpl" -> img:cli.'); - const listed = await cli('template', 'ls'); + const listed = await door('template', 'ls'); expect(listed.stdout).toMatch(/NAME\s{2,}IMAGE\s{2,}CREATED/); expect(listed.stdout).toMatch(/cli-tpl\s{2,}img:cli/); - const removed = await cli('template', 'rm', 'cli-tpl'); + const removed = await door('template', 'rm', 'cli-tpl'); expect(removed.stdout).toContain('Removed template "cli-tpl".'); - const again = await cli('template', 'rm', 'cli-tpl'); + const again = await door('template', 'rm', 'cli-tpl'); expect(again.stdout).toContain('nothing to remove'); }); - it('apikey create, ls and revoke run the rotation life through the real binary', async () => { - const created = await cli('apikey', 'create', 'cli-key'); + it('apikey create, ls and revoke run the rotation life through the real binary, at the door', async () => { + const created = await door('apikey', 'create', 'cli-key'); const lines = created.stdout.trim().split('\n'); expect(lines[0]).toMatch( /^Created API key "cli-key" \(prefix [0-9a-f]{8}\)\./, @@ -153,55 +168,36 @@ describe('dor CLI against a real daemon', () => { expect(lines[2]).toContain('never be shown again'); // The minted key IS a DORMICE_API_TOKEN — same variable, new value: - // exactly what rotation looks like from a client's shell. - const keyed = await run('node', [CLI, 'sandbox', 'ls'], { - env: { - ...process.env, - DORMICE_ENDPOINT: inject('dormiceEndpoint'), - DORMICE_API_TOKEN: token, - }, - }); - expect(keyed.stdout).toBeDefined(); + // exactly what rotation looks like from a client's shell, pointed at + // the door (a node knows only the fleet token). The probe is a + // destroy of a name nobody holds: a named verb the door answers once + // the key is through, `nothing to destroy` when it is. + const keyed = (extra: string[] = []) => + run('node', [CLI, 'sandbox', 'destroy', 'cli-key-probe', ...extra], { + env: { + ...process.env, + DORMICE_ENDPOINT: inject('dormiceGatewayEndpoint'), + DORMICE_API_TOKEN: token, + }, + }); + expect((await keyed()).stdout).toContain('nothing to destroy'); - const listed = await cli('apikey', 'ls'); + const listed = await door('apikey', 'ls'); expect(listed.stdout).toMatch(/NAME\s{2,}PREFIX\s{2,}CREATED/); expect(listed.stdout).toMatch(/cli-key\s{2,}[0-9a-f]{8}.*active/); // Disable parks the credential (next request dies), enable revives it. - const disabled = await cli('apikey', 'disable', 'cli-key'); + const disabled = await door('apikey', 'disable', 'cli-key'); expect(disabled.stdout).toContain('Disabled API key "cli-key"'); - await expect( - run('node', [CLI, 'sandbox', 'ls'], { - env: { - ...process.env, - DORMICE_ENDPOINT: inject('dormiceEndpoint'), - DORMICE_API_TOKEN: token, - }, - }), - ).rejects.toMatchObject({ code: 1 }); - const enabled = await cli('apikey', 'enable', 'cli-key'); + await expect(keyed()).rejects.toMatchObject({ code: 1 }); + const enabled = await door('apikey', 'enable', 'cli-key'); expect(enabled.stdout).toContain('Enabled API key "cli-key"'); - const revived = await run('node', [CLI, 'sandbox', 'ls'], { - env: { - ...process.env, - DORMICE_ENDPOINT: inject('dormiceEndpoint'), - DORMICE_API_TOKEN: token, - }, - }); - expect(revived.stdout).toBeDefined(); + expect((await keyed()).stdout).toContain('nothing to destroy'); - const revoked = await cli('apikey', 'revoke', 'cli-key'); + const revoked = await door('apikey', 'revoke', 'cli-key'); expect(revoked.stdout).toContain('Revoked API key "cli-key"'); - await expect( - run('node', [CLI, 'sandbox', 'ls'], { - env: { - ...process.env, - DORMICE_ENDPOINT: inject('dormiceEndpoint'), - DORMICE_API_TOKEN: token, - }, - }), - ).rejects.toMatchObject({ code: 1 }); - const again = await cli('apikey', 'revoke', 'cli-key'); + await expect(keyed()).rejects.toMatchObject({ code: 1 }); + const again = await door('apikey', 'revoke', 'cli-key'); expect(again.stdout).toContain('nothing to revoke'); }); diff --git a/e2e/src/console.test.ts b/e2e/src/console.test.ts index bec9c855..16222104 100644 --- a/e2e/src/console.test.ts +++ b/e2e/src/console.test.ts @@ -1,16 +1,18 @@ import { describe, expect, inject, it } from 'vitest'; -// The web console, black-box: plain fetch against the built daemon, the -// same requests a browser would make. The daemon serves packages/console/dist -// (pnpm build ran before this suite), so this also proves the monorepo -// path hop in main.ts survives the dist layout. +// The web console, black-box: plain fetch against the built gateway — the +// console lives at the fleet's door since the configuration moved there +// (design record #22) — the same requests a browser would make. The +// gateway serves packages/console/dist (pnpm build ran before this +// suite), so this also proves the monorepo path hop in its main.ts +// survives the dist layout. // // The tests run in file order on purpose: they walk the account's real // story — no account, setup with the token, password logins, reset — and // only this file touches the account, so the other suites (Bearer-only) // never race it. -const endpoint = () => inject('dormiceEndpoint'); +const endpoint = () => inject('dormiceGatewayEndpoint'); const USERNAME = 'operator'; const PASSWORD = 'e2e console password'; @@ -35,8 +37,9 @@ function cookieOf(res: Response): string { return cookie; } -async function listSandboxes(cookie: string, withHeader = true) { - return fetch(`${endpoint()}/listSandboxes`, { +/** A session's way through the gates: listNodes sits behind the admin gate, which a console session opens. */ +async function listNodes(cookie: string, withHeader = true) { + return fetch(`${endpoint()}/listNodes`, { method: 'POST', headers: { 'content-type': 'application/json', @@ -92,7 +95,7 @@ describe('web console over a real daemon', () => { password: PASSWORD, }); expect(res.status).toBe(200); - const list = await listSandboxes(cookieOf(res)); + const list = await listNodes(cookieOf(res)); expect(list.status).toBe(200); const status = await post('/console/auth/status', {}); expect(await status.json()).toEqual({ accountExists: true }); @@ -107,16 +110,16 @@ describe('web console over a real daemon', () => { expect(res.headers.getSetCookie()).toHaveLength(0); }); - it('login yields a session cookie that opens the native API', async () => { + it('login yields a session cookie that opens the gates: node A is listed', async () => { const res = await post('/console/auth/login', { username: USERNAME, password: PASSWORD, }); expect(res.status).toBe(200); - const list = await listSandboxes(cookieOf(res)); + const list = await listNodes(cookieOf(res)); expect(list.status).toBe(200); - const body = (await list.json()) as { sandboxes: unknown[] }; - expect(Array.isArray(body.sandboxes)).toBe(true); + const body = (await list.json()) as { nodes: Array<{ id: string }> }; + expect(body.nodes.map((n) => n.id)).toEqual(['node-1']); }); it('the cookie without the console header stays locked out', async () => { @@ -124,7 +127,7 @@ describe('web console over a real daemon', () => { username: USERNAME, password: PASSWORD, }); - const list = await listSandboxes(cookieOf(res), false); + const list = await listNodes(cookieOf(res), false); expect(list.status).toBe(401); }); @@ -143,7 +146,7 @@ describe('web console over a real daemon', () => { expect(reset.status).toBe(200); // The forgot-password semantics, observed on the wire: the old session // and the old password are both dead, the new pair works. - expect((await listSandboxes(before)).status).toBe(401); + expect((await listNodes(before)).status).toBe(401); const oldLogin = await post('/console/auth/login', { username: USERNAME, password: PASSWORD, @@ -154,17 +157,19 @@ describe('web console over a real daemon', () => { password: 'a brand new password', }); expect(newLogin.status).toBe(200); - expect((await listSandboxes(cookieOf(newLogin))).status).toBe(200); + expect((await listNodes(cookieOf(newLogin))).status).toBe(200); }); }); describe('browser-side signed download URLs (the Office preview foundation)', () => { // The console's preview pane recomputes the file signature in the browser // (envd-client.ts signedDownloadUrl) from the token /envdToken - // hands it. This pins the whole chain end-to-end — console minting, the - // formula REWRITTEN here rather than imported (a black box pins the - // formula itself, not a shared implementation's self-consistency), and - // the root /files door. + // hands it. This pins the whole chain end-to-end — console minting at + // the gateway (which asks the sandbox's node), the formula REWRITTEN + // here rather than imported (a black box pins the formula itself, not a + // shared implementation's self-consistency), and the root /files door, + // which is the node's today: the gateway's sandbox-domain face is the + // next step of the move (RULES/协议.md「网关」). it('a console-minted token signs a working /files URL with the browser formula', async () => { // Continue the account story: re-setup with the token so this describe // owns known credentials regardless of what ran before it. @@ -223,7 +228,7 @@ describe('browser-side signed download URLs (the Office preview foundation)', () ); const signature = `v1_${btoa(String.fromCharCode(...new Uint8Array(digest))).replace(/=+$/, '')}`; const url = (extra = '') => - `${endpoint()}/files?path=pixel.png${extra}&signature=${encodeURIComponent(signature)}&signature_expiration=${exp}`; + `${inject('dormiceEndpoint')}/files?path=pixel.png${extra}&signature=${encodeURIComponent(signature)}&signature_expiration=${exp}`; const res = await fetch(url()); expect(res.status).toBe(200); diff --git a/e2e/src/e2b.test.ts b/e2e/src/e2b.test.ts index f19e0e2a..4fdc84fe 100644 --- a/e2e/src/e2b.test.ts +++ b/e2e/src/e2b.test.ts @@ -2,6 +2,7 @@ import http from 'node:http'; import { Dormice } from '@dormice/sdk'; import { CommandExitError, Sandbox } from 'e2b'; import { describe, expect, inject, it } from 'vitest'; +import { door, settled } from './helpers'; // The compatibility promise, verified with the promise's own artifact: the // OFFICIAL e2b package, pointed at the daemon by exactly two URLs (plus its @@ -72,18 +73,26 @@ describe('official e2b SDK against the daemon', () => { } }); - it('a ledger API key drives the official SDK exactly like the env token', async () => { - // Minted over the native face; the E2B face accepts it as e2b_ — - // pure hex by construction, so even the Python SDK's client-side - // e2b_[0-9a-f]+ validation would let it through. + it('a minted API key drives the official SDK through the door exactly like the fleet token', async () => { + // Minted at the gateway over the native face; the gateway's E2B face + // accepts it as e2b_ — pure hex by construction, so even the + // Python SDK's client-side e2b_[0-9a-f]+ validation would let it + // through. A node knows only the fleet token, so the keyed SDK is + // pointed at the door: the E2B control plane there places on node A + // and forwards by id. const dormice = new Dormice({ - endpoint: inject('dormiceEndpoint'), + endpoint: door(), token: inject('dormiceToken'), }); const { apiKey, token } = await dormice.createApiKey('e2b-face'); + const atDoor = { + apiUrl: `${door()}/e2b/api`, + sandboxUrl: `${door()}/e2b/envd`, + }; try { const sbx = await Sandbox.create({ ...connection(), + ...atDoor, apiKey: `e2b_${token}`, }); try { @@ -97,7 +106,7 @@ describe('official e2b SDK against the daemon', () => { } // Revoked: the same key is refused at the control-plane door. await expect( - Sandbox.create({ ...connection(), apiKey: `e2b_${token}` }), + Sandbox.create({ ...connection(), ...atDoor, apiKey: `e2b_${token}` }), ).rejects.toThrow(/invalid API key/); }); @@ -718,13 +727,15 @@ describe('official e2b SDK against the daemon', () => { // consumes the name as its templateID — aliases are the same wire. // The image must exist in docker mode; the base image serves both. const dormice = new Dormice({ - endpoint: inject('dormiceEndpoint'), + endpoint: door(), token: inject('dormiceToken'), }); await dormice.registerTemplate( 'e2e-tpl', process.env.DORMICE_BASE_IMAGE ?? 'img:e2e-tpl', ); + // Registered at the door; node A holds it after its next check-in. + await settled(); const sbx = await Sandbox.create('e2e-tpl', connection()); try { const info = await sbx.getInfo(); @@ -939,10 +950,11 @@ describe.runIf(process.env.DORMICE_EXECUTOR === 'docker')( const image = process.env.DORMICE_BASE_IMAGE; if (!image) throw new Error('docker e2e requires DORMICE_BASE_IMAGE'); const dormice = new Dormice({ - endpoint: inject('dormiceEndpoint'), + endpoint: door(), token: inject('dormiceToken'), }); await dormice.registerTemplate('e2e-real-tpl', image); + await settled(); const sbx = await Sandbox.create('e2e-real-tpl', connection()); try { expect((await sbx.getInfo()).templateId).toBe('e2e-real-tpl'); diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts index 31c2f481..f4448ac0 100644 --- a/e2e/src/gateway.test.ts +++ b/e2e/src/gateway.test.ts @@ -7,19 +7,25 @@ import { join } from 'node:path'; import { Dormice } from '@dormice/sdk'; import { Sandbox } from 'e2b'; import { describe, expect, inject, it } from 'vitest'; +import { + configSettled, + listNodes as listFleetNodes, + rpc as post, + until, +} from './helpers'; -// The gateway exam: two real daemons behind a real gateway, all three -// booted the production way and driven only over the wire — the SDK, the +// The fleet exam: two real daemons behind a real gateway, all three booted +// the production way and driven only over the wire — the SDK, the // official e2b package and plain fetch. Direct calls to a node exist only // to stage what the gateway must then find (a sandbox built behind its // back, a name on two nodes, a destroy it did not see). Skipped in docker -// mode, where the setup boots node A alone. -const skip = inject('dormiceGatewayEndpoint') === null; +// mode, where the setup boots node A (and its own gateway) alone. +const skip = inject('dormiceFleetNodes') === null; -const gateway = () => inject('dormiceGatewayEndpoint') as string; -const token = () => inject('dormiceGatewayToken') as string; +const gateway = () => inject('dormiceFleetGateway') as string; +const token = () => inject('dormiceFleetToken') as string; const nodes = () => - inject('dormiceGatewayNodes') as Array<{ id: string; endpoint: string }>; + inject('dormiceFleetNodes') as Array<{ id: string; endpoint: string }>; const viaGateway = () => new Dormice({ endpoint: gateway(), token: token() }); function direct(id: string) { const node = nodes().find((n) => n.id === id); @@ -28,56 +34,14 @@ function direct(id: string) { } const other = (id: string) => (id === 'node-b' ? 'node-c' : 'node-b'); -/** Polls until the probe answers something — nodes check in on their own clock, not ours. */ -async function until( - probe: () => Promise, - timeoutMs = 10_000, -): Promise { - const deadline = Date.now() + timeoutMs; - for (;;) { - const value = await probe(); - if (value !== undefined) return value; - if (Date.now() > deadline) throw new Error('condition never became true'); - await new Promise((resolve) => setTimeout(resolve, 100)); - } -} - -async function rpc( +const rpc = ( path: string, payload: unknown = {}, bearer = token(), endpoint = gateway(), -): Promise<{ status: number; body: unknown; headers: Headers }> { - const res = await fetch(`${endpoint}${path}`, { - method: 'POST', - headers: { - authorization: `Bearer ${bearer}`, - 'content-type': 'application/json', - }, - body: JSON.stringify(payload), - }); - const text = await res.text(); - return { - status: res.status, - body: text ? JSON.parse(text) : null, - headers: res.headers, - }; -} +) => post(endpoint, path, payload, bearer); -interface ListedNode { - id: string; - endpoint: string; - reachable: boolean; - lastCheckInAt: string | null; - build: { commit: string } | null; - reading: { sandboxes: { byState: { active: number } } } | null; - placedSinceCheckIn: number; -} -async function listNodes(): Promise { - const { status, body } = await rpc('/listNodes'); - expect(status).toBe(200); - return (body as { nodes: ListedNode[] }).nodes; -} +const listNodes = () => listFleetNodes(gateway(), token()); const status = (error: unknown) => (error as { status?: number }).status; const message = (r: { body: unknown }) => @@ -94,7 +58,7 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { expect((await rpc('/listNodes', {}, 'x'.repeat(64))).status).toBe(401); }); - it('both nodes checked in: reachable, with a reading and the build they run', async () => { + it('both nodes checked in: reachable, with a reading, the build they run and the configuration version they took from this gateway', async () => { const listed = await until(async () => { const seen = await listNodes(); return seen.length === 2 && seen.every((n) => n.reachable) @@ -102,12 +66,111 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { : undefined; }); expect(listed.map((n) => n.id).sort()).toEqual(['node-b', 'node-c']); + const { body } = await rpc('/getConfig'); + const version = (body as { configVersion: number }).configVersion; for (const node of listed) { expect(node.reading?.sandboxes.byState.active).toBeGreaterThanOrEqual(0); expect(node.build?.commit).toMatch(/^[0-9a-f]{7,}$/); expect(node.endpoint).toBe( nodes().find((n) => n.id === node.id)?.endpoint, ); + // A node boots on the gateway's bundle and reports its version back. + expect(node.configVersion).toBe(version); + } + }); + + it("a settings write at the gateway reaches both nodes within their check-in: the copy's defaults shape the next acquire", async () => { + const before = (await rpc('/getConfig')).body as { + configVersion: number; + settings: { + sandboxDefaults: { cpus: number; memoryGb: number; diskGb: number }; + }; + }; + const { status } = await rpc('/updateSettings', { + sandboxDefaults: { ...before.settings.sandboxDefaults, cpus: 3 }, + }); + expect(status).toBe(200); + try { + const version = await configSettled(gateway(), token()); + expect(version).toBe(before.configVersion + 1); + // Asked of the node directly: its own copy answers, no gateway in the path. + const created = await direct('node-b').acquireSandbox('gw-config'); + try { + expect(created.sandbox.spec.cpus).toBe(3); + } finally { + await direct('node-b').destroySandbox('gw-config'); + } + } finally { + await rpc('/updateSettings', { + sandboxDefaults: before.settings.sandboxDefaults, + }); + await configSettled(gateway(), token()); + } + }); + + it('a template registered at the gateway is usable on every node; removal asks the nodes and is refused while one holds a sandbox on it', async () => { + await viaGateway().registerTemplate('gw-tpl', 'img:gw-tpl'); + await configSettled(gateway(), token()); + const staged = await direct('node-b').acquireSandbox('gw-tpl-user', { + template: 'gw-tpl', + }); + try { + expect(staged.sandbox.template).toBe('gw-tpl'); + await expect(viaGateway().removeTemplate('gw-tpl')).rejects.toMatchObject( + { + status: 409, + message: expect.stringMatching(/gw-tpl-user on node node-b/), + }, + ); + } finally { + await direct('node-b').destroySandbox('gw-tpl-user'); + } + expect(await viaGateway().removeTemplate('gw-tpl')).toEqual({ + removed: true, + }); + await configSettled(gateway(), token()); + // Gone from the nodes' copies too: an acquire on it is the node's own 400. + await expect( + direct('node-c').acquireSandbox('gw-tpl-late', { template: 'gw-tpl' }), + ).rejects.toMatchObject({ status: 400 }); + }); + + it('envdToken through the gateway is minted by the sandbox’s node and opens its envd surface through the gateway', async () => { + const created = await viaGateway().acquireSandbox('gw-envd'); + try { + const minted = await rpc('/envdToken', { sandboxId: created.sandbox.id }); + expect(minted.status).toBe(200); + const { envdAccessToken } = minted.body as { envdAccessToken: string }; + const stat = await fetch( + `${gateway()}/e2b/envd/filesystem.Filesystem/Stat`, + { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'e2b-sandbox-id': created.sandbox.id, + 'x-access-token': envdAccessToken, + }, + body: JSON.stringify({ path: '/home/user' }), + }, + ); + expect(stat.status).toBe(200); + // Per sandbox: the same token opens no other. + const stranger = await fetch( + `${gateway()}/e2b/envd/filesystem.Filesystem/Stat`, + { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'e2b-sandbox-id': randomUUID(), + 'x-access-token': envdAccessToken, + }, + body: JSON.stringify({ path: '/home/user' }), + }, + ); + expect(stat.status).toBe(200); + expect(stranger.status).not.toBe(200); + } finally { + await viaGateway().destroySandbox('gw-envd'); } }); @@ -420,8 +483,15 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { token: token(), }); try { + // A booting node checks in before it listens (it takes its first + // configuration bundle from that check-in): reachable with + // configVersion null is "joined, not open yet"; the version it + // reports at its first check-in after listen is the cue that its + // port is open — placement waits for the same cue. await until(async () => - (await listNodes()).some((n) => n.id === 'node-d' && n.reachable) + (await listNodes()).some( + (n) => n.id === 'node-d' && n.reachable && n.configVersion !== null, + ) ? true : undefined, ).catch((error) => { diff --git a/e2e/src/helpers.ts b/e2e/src/helpers.ts new file mode 100644 index 00000000..42168071 --- /dev/null +++ b/e2e/src/helpers.ts @@ -0,0 +1,98 @@ +import { inject } from 'vitest'; + +/** + * What every suite that talks to a gateway needs, black-box: a POST in the + * native dialect, a poll, and the one question the configuration move + * (2026-09-14) makes every write ask — has my change reached the node yet? + * The gateway answers it itself: getConfig carries the fleet configuration + * version, listNodes carries the version each node last said it runs. + */ + +/** Polls until the probe answers something — nodes check in on their own clock, not ours. */ +export async function until( + probe: () => Promise, + timeoutMs = 10_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + const value = await probe(); + if (value !== undefined) return value; + if (Date.now() > deadline) throw new Error('condition never became true'); + await new Promise((resolve) => setTimeout(resolve, 100)); + } +} + +export async function rpc( + endpoint: string, + path: string, + payload: unknown = {}, + bearer: string, +): Promise<{ status: number; body: unknown; headers: Headers }> { + const res = await fetch(`${endpoint}${path}`, { + method: 'POST', + headers: { + authorization: `Bearer ${bearer}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(payload), + }); + const text = await res.text(); + return { + status: res.status, + body: text ? JSON.parse(text) : null, + headers: res.headers, + }; +} + +export interface ListedNode { + id: string; + endpoint: string; + reachable: boolean; + configVersion: number | null; + swapGb: number; + lastCheckInAt: string | null; + build: { commit: string } | null; + reading: { + sandboxes: { + byState: { active: number; archived: number; restoring: number }; + }; + } | null; + placedSinceCheckIn: number; +} + +export async function listNodes( + gateway: string, + token: string, +): Promise { + const { status, body } = await rpc(gateway, '/listNodes', {}, token); + if (status !== 200) throw new Error(`listNodes answered ${status}`); + return (body as { nodes: ListedNode[] }).nodes; +} + +/** + * Waits until every node of the gateway runs the gateway's current + * configuration version — a write at the gateway has then reached the + * nodes' copies (the exam's nodes check in every second). + */ +export async function configSettled( + gateway: string, + token: string, +): Promise { + return until(async () => { + const { status, body } = await rpc(gateway, '/getConfig', {}, token); + if (status !== 200) throw new Error(`getConfig answered ${status}`); + const version = (body as { configVersion: number }).configVersion; + const nodes = await listNodes(gateway, token); + return nodes.length > 0 && nodes.every((n) => n.configVersion === version) + ? version + : undefined; + }); +} + +/** Node A's own gateway (a fleet of one) — where the fleet's configuration verbs answer in every mode. */ +export const door = () => inject('dormiceGatewayEndpoint'); + +/** A write at node A's gateway, then the wait for node A to run it. */ +export async function settled(): Promise { + return configSettled(door(), inject('dormiceToken')); +} diff --git a/e2e/src/ingress.test.ts b/e2e/src/ingress.test.ts index 7f0fb538..b289230c 100644 --- a/e2e/src/ingress.test.ts +++ b/e2e/src/ingress.test.ts @@ -2,27 +2,27 @@ import { readFileSync, writeFileSync } from 'node:fs'; import { Dormice } from '@dormice/sdk'; import { describe, expect, inject, it } from 'vitest'; -// Web domain binding, black-box: the daemon owns a Caddy config file (the -// exam points DORMICE_INGRESS_FILE into its temp dir, reload is a no-op — -// Caddy's half is real-machine acceptance). The file is an operator-visible -// artifact, so reading and tampering with it from the test is fair play: -// that is exactly what an operator can do. +// Web domain binding, black-box: the gateway owns a Caddy config file (the +// exam points its DORMICE_INGRESS_FILE into the temp dir, reload is a no-op +// — Caddy's half is real-machine acceptance). The file is an operator- +// visible artifact, so reading and tampering with it from the test is fair +// play: that is exactly what an operator can do. // -// Tests in this file share the daemon's single ingress state, so they run +// Tests in this file share the gateway's single ingress state, so they run // as one ordered story instead of independent keys. function client() { return new Dormice({ - endpoint: inject('dormiceEndpoint'), + endpoint: inject('dormiceGatewayEndpoint'), token: inject('dormiceToken'), }); } -describe('ingress domain binding over a real daemon', () => { +describe('ingress domain binding over a real gateway', () => { it('walks bind → add → status → drop → clear, with the file telling the same story', async () => { const file = inject('dormiceIngressFile'); - // Managed but unbound: the daemon has a file knob, nothing written yet. + // Managed but unbound: the gateway has a file knob, nothing written yet. const before = await client().getIngress(); expect(before).toEqual({ managed: true, domains: [] }); @@ -41,6 +41,10 @@ describe('ingress domain binding over a real daemon', () => { expect(content).toContain('console.dormice-e2e.test {'); expect(content).toContain('api.dormice-e2e.test {'); expect(content).toContain(':80 {'); // the no-lockout catch-all + // The catch-all and every site point at the gateway, the fleet's door. + expect(content).toContain( + `reverse_proxy 127.0.0.1:${new URL(inject('dormiceGatewayEndpoint')).port}`, + ); // Status carries live probes per domain; on record-less domains they // are honest reds/empties, never invented greens. diff --git a/e2e/src/native.test.ts b/e2e/src/native.test.ts index c044b5fc..b74de9c7 100644 --- a/e2e/src/native.test.ts +++ b/e2e/src/native.test.ts @@ -1,6 +1,7 @@ import { spawn } from 'node:child_process'; import { DEFAULT_LIFECYCLE_POLICY, Dormice } from '@dormice/sdk'; import { describe, expect, inject, it } from 'vitest'; +import { door, settled } from './helpers'; // One daemon serves the whole run, so every test uses its own sandbox name to // stay independent of the others. @@ -8,6 +9,22 @@ function client(token = inject('dormiceToken')) { return new Dormice({ endpoint: inject('dormiceEndpoint'), token }); } +/** + * The fleet's door — node A's own gateway, a fleet of one. Templates, + * keys and settings are registered there (design record #22); the node + * takes them from its next check-in, so a write here is followed by + * settled() before the node is asked to act on it. + */ +function viaDoor(token = inject('dormiceToken')) { + return new Dormice({ endpoint: door(), token }); +} + +/** registerTemplate at the door, then the wait for node A to hold it. */ +async function registerTemplate(name: string, image: string) { + await viaDoor().registerTemplate(name, image); + await settled(); +} + function sleep(seconds: number) { return new Promise((resolve) => setTimeout(resolve, seconds * 1000)); } @@ -489,7 +506,7 @@ describe('native API over a real daemon', () => { // config; the fake plays any name. The daemon's own base image serves // both: a real boot there, an arbitrary string here. const image = process.env.DORMICE_BASE_IMAGE ?? 'img:native'; - await client().registerTemplate('native-tpl', image); + await registerTemplate('native-tpl', image); const created = await client().acquireSandbox('tpl-key', { template: 'native-tpl', }); @@ -499,44 +516,47 @@ describe('native API over a real daemon', () => { ); expect(listed?.template).toBe('native-tpl'); - await expect(client().removeTemplate('native-tpl')).rejects.toMatchObject({ + // Removal is the gateway's verb, and it asks node A first. + await expect(viaDoor().removeTemplate('native-tpl')).rejects.toMatchObject({ name: 'DormiceApiError', status: 409, - message: expect.stringMatching(/tpl-key/), + message: expect.stringMatching(/tpl-key on node node-1/), }); // The migration story: re-home the sandbox onto a successor template, // and the old name — no longer referenced — becomes removable without // destroying anything. An unknown target is refused, never stored. - await client().registerTemplate('native-tpl-v2', image); + await registerTemplate('native-tpl-v2', image); await expect( client().updateTemplate('tpl-key', 'ghost-tpl'), ).rejects.toMatchObject({ status: 400 }); const moved = await client().updateTemplate('tpl-key', 'native-tpl-v2'); expect(moved.sandbox.template).toBe('native-tpl-v2'); - expect(await client().removeTemplate('native-tpl')).toEqual({ + expect(await viaDoor().removeTemplate('native-tpl')).toEqual({ removed: true, }); await client().destroySandbox('tpl-key'); - expect(await client().removeTemplate('native-tpl-v2')).toEqual({ + expect(await viaDoor().removeTemplate('native-tpl-v2')).toEqual({ removed: true, }); }); - it('API keys: the rolling-rotation story — mint, work, park, revoke, the env token survives', async () => { - const { apiKey, token } = await client().createApiKey('rotation'); + it('API keys: the rolling-rotation story at the door — mint, work, park, revoke, the fleet token survives; a node knows only the fleet token', async () => { + const { apiKey, token } = await viaDoor().createApiKey('rotation'); expect(token).toMatch(/^[0-9a-f]{64}$/); expect(apiKey.prefix).toBe(token.slice(0, 8)); - // A fresh client on the minted key does real work — rotation means the - // new credential is live before the old one dies. - const keyed = client(token); + // A fresh client on the minted key does real work through the door — + // placed on node A under the fleet token; rotation means the new + // credential is live before the old one dies. + const keyed = viaDoor(token); const created = await keyed.acquireSandbox('rotation-key'); expect(created.sandbox.name).toBe('rotation-key'); + expect(created.sandbox.nodeId).toBe('node-1'); await keyed.destroySandbox('rotation-key'); - const listed = await client().listApiKeys(); + const listed = await viaDoor().listApiKeys(); const mine = listed.find((k) => k.name === 'rotation'); expect(mine?.lastUsedAt).not.toBeNull(); @@ -551,20 +571,32 @@ describe('native API over a real daemon', () => { status: 403, }); + // The node judges one credential, the fleet token: a minted key is a + // stranger there, however live it is at the door. + await expect(client(token).listSandboxes()).rejects.toMatchObject({ + status: 401, + }); + // Disable is the reversible hold: 401 while parked, alive again after. - await client().updateApiKey(apiKey.id, { disabled: true }); - await expect(keyed.listSandboxes()).rejects.toMatchObject({ status: 401 }); - await client().updateApiKey(apiKey.id, { disabled: false }); - expect(await keyed.listSandboxes()).toBeDefined(); + // The probe is a destroy of a name nobody holds — a named verb the + // door answers itself once the key is through. + await viaDoor().updateApiKey(apiKey.id, { disabled: true }); + await expect(keyed.destroySandbox('rotation-probe')).rejects.toMatchObject({ + status: 401, + }); + await viaDoor().updateApiKey(apiKey.id, { disabled: false }); + expect(await keyed.destroySandbox('rotation-probe')).toEqual({ + destroyed: false, + }); - expect(await client().revokeApiKey(apiKey.id)).toEqual({ revoked: true }); - await expect(keyed.listSandboxes()).rejects.toMatchObject({ + expect(await viaDoor().revokeApiKey(apiKey.id)).toEqual({ revoked: true }); + await expect(keyed.destroySandbox('rotation-probe')).rejects.toMatchObject({ name: 'DormiceApiError', status: 401, }); - // The env token is the bootstrap credential: revocation never touches it. + // The fleet token is the bootstrap credential: revocation never touches it. expect(await client().listSandboxes()).toBeDefined(); - expect(await client().revokeApiKey(apiKey.id)).toEqual({ + expect(await viaDoor().revokeApiKey(apiKey.id)).toEqual({ revoked: false, }); }); @@ -575,7 +607,7 @@ describe('native API over a real daemon', () => { // at an unbuilt image on purpose: registration is config and observation // never boots anything, so no engine ever has to pull it. const image = process.env.DORMICE_BASE_IMAGE ?? 'img:lineage-v1'; - await client().registerTemplate('lineage-tpl', image); + await registerTemplate('lineage-tpl', image); const created = await client().acquireSandbox('lineage-key', { template: 'lineage-tpl', }); @@ -594,7 +626,7 @@ describe('native API over a real daemon', () => { }); // Re-registering moves nextImage; the live shell honestly stays behind. - await client().registerTemplate('lineage-tpl', 'img:lineage-v2'); + await registerTemplate('lineage-tpl', 'img:lineage-v2'); expect(await mine()).toMatchObject({ image, nextImage: 'img:lineage-v2', @@ -603,7 +635,7 @@ describe('native API over a real daemon', () => { // Point the name back and rebuild: no shell means no image and nothing // to upgrade; the wake boots the template's current image, in sync. - await client().registerTemplate('lineage-tpl', image); + await registerTemplate('lineage-tpl', image); await client().rebuildSandbox('lineage-key'); expect(await mine()).toMatchObject({ image: null, @@ -618,7 +650,7 @@ describe('native API over a real daemon', () => { }); await client().destroySandbox('lineage-key'); - expect(await client().removeTemplate('lineage-tpl')).toEqual({ + expect(await viaDoor().removeTemplate('lineage-tpl')).toEqual({ removed: true, }); }); @@ -631,7 +663,7 @@ describe('native API over a real daemon', () => { it.skipIf(process.env.DORMICE_EXECUTOR === 'docker')( 'a template upgrade reaches a frozen sandbox on its next touch: shell swapped, data kept, audited', async () => { - await client().registerTemplate('swap-tpl', 'img:swap-v1'); + await registerTemplate('swap-tpl', 'img:swap-v1'); await client().acquireSandbox('swap-key', { template: 'swap-tpl', policy: { @@ -659,7 +691,7 @@ describe('native API over a real daemon', () => { } // Operator re-points the template; the frozen shell is now stale. - await client().registerTemplate('swap-tpl', 'img:swap-v2'); + await registerTemplate('swap-tpl', 'img:swap-v2'); // The touch is an ordinary use verb — every wake entry (native, envd, // port proxy) funnels into the same wakeSandbox, and the read itself @@ -676,7 +708,7 @@ describe('native API over a real daemon', () => { ).toMatchObject({ image: 'img:swap-v2', upgradable: false }); await client().destroySandbox('swap-key'); - await client().removeTemplate('swap-tpl'); + await viaDoor().removeTemplate('swap-tpl'); }, ); @@ -685,7 +717,7 @@ describe('native API over a real daemon', () => { it.runIf(process.env.DORMICE_EXECUTOR === 'docker')( 'a template whose image is missing fails create with a named, honest error', async () => { - await client().registerTemplate('hollow-tpl', 'img:does-not-exist'); + await registerTemplate('hollow-tpl', 'img:does-not-exist'); await expect( client().acquireSandbox('hollow-key', { template: 'hollow-tpl' }), ).rejects.toMatchObject({ @@ -699,7 +731,7 @@ describe('native API over a real daemon', () => { expect(await client().destroySandbox('hollow-key')).toEqual({ destroyed: false, }); - expect(await client().removeTemplate('hollow-tpl')).toEqual({ + expect(await viaDoor().removeTemplate('hollow-tpl')).toEqual({ removed: true, }); }, @@ -759,32 +791,37 @@ describe('native API over a real daemon', () => { }); describe('the observability verbs over a real daemon', () => { - it('getConfig reports effective knobs and never leaks the token', async () => { - const config = await client().getConfig(); + it('getConfig at the door reports effective knobs and never leaks the token', async () => { + const config = await viaDoor().getConfig(); const token = config.entries.find((e) => e.key === 'DORMICE_API_TOKEN'); expect(token).toMatchObject({ value: null, redacted: true }); // Black-box secrecy: the real token appears nowhere in the response. expect(JSON.stringify(config)).not.toContain(inject('dormiceToken')); - // The exam daemon runs with miniS3 configured, so the daemon's own + // The exam's gateway is seeded with miniS3, so the fleet's own // adjudication says archiving is live, with the one-week default. expect(config.archive).toEqual({ enabled: true, defaultSeconds: 7 * 24 * 60 * 60, }); + expect(config.configVersion).toBeGreaterThanOrEqual(1); + // The node answers no configuration verb of its own anymore. + await expect(client().getConfig()).rejects.toMatchObject({ status: 404 }); }); - it('updateSettings moves a ledger knob with immediate effect', async () => { - const before = (await client().getConfig()).settings; - const { settings } = await client().updateSettings({ - pidsLimit: before.pidsLimit + 1, + it('updateSettings at the door moves a fleet knob; the version counts up and node A runs it within a check-in', async () => { + const before = await viaDoor().getConfig(); + const { settings } = await viaDoor().updateSettings({ + pidsLimit: before.settings.pidsLimit + 1, }); - expect(settings.pidsLimit).toBe(before.pidsLimit + 1); + expect(settings.pidsLimit).toBe(before.settings.pidsLimit + 1); expect(settings.updatedAt).not.toBeNull(); - expect((await client().getConfig()).settings.pidsLimit).toBe( - before.pidsLimit + 1, - ); - // Restore: the exam daemon is shared by every suite in this run. - await client().updateSettings({ pidsLimit: before.pidsLimit }); + const after = await viaDoor().getConfig(); + expect(after.settings.pidsLimit).toBe(before.settings.pidsLimit + 1); + expect(after.configVersion).toBe(before.configVersion + 1); + expect(await settled()).toBe(after.configVersion); + // Restore: the exam's gateway is shared by every suite in this run. + await viaDoor().updateSettings({ pidsLimit: before.settings.pidsLimit }); + await settled(); }); it('getSandboxMetrics samples a live sandbox and 404s after destroy', async () => { diff --git a/e2e/src/settings-hot.test.ts b/e2e/src/settings-hot.test.ts index 3226944b..527cc6b5 100644 --- a/e2e/src/settings-hot.test.ts +++ b/e2e/src/settings-hot.test.ts @@ -1,17 +1,26 @@ import http from 'node:http'; import { Dormice } from '@dormice/sdk'; import { describe, expect, inject, it } from 'vitest'; +import { door, listNodes, settled, until } from './helpers'; -// The runtime-settings hot path for the two knobs that moved into the -// ledger on 2026-07-26: the S3 archive store and the sandbox domain. The -// exam daemon is shared by every suite in this run, so each test here -// restores what it changed in a finally — and the S3 tests deliberately -// never move the shared store (rotating credentials against the same -// bucket, probing an unreachable endpoint, and exercising the refusal -// paths are all observation-safe; the full archive cycle over the shared -// store is archive.test.ts's exam). +// The fleet-settings hot path for the two knobs that moved into the ledger +// on 2026-07-26 — the S3 archive store and the sandbox domain — as they +// travel since 2026-09-14: written at the gateway, carried to node A with +// its next check-in (settled() waits for that), acted on there. The exam's +// gateway and daemon are shared by every suite in this run, so each test +// here restores what it changed in a finally — and the S3 tests +// deliberately never move the shared store (rotating credentials against +// the same bucket, probing an unreachable endpoint, and exercising the +// refusal paths are all observation-safe; the full archive cycle over the +// shared store is archive.test.ts's exam). +/** The door: where settings are read and written. */ function client() { + return new Dormice({ endpoint: door(), token: inject('dormiceToken') }); +} + +/** Node A, for what it does with the settings it was handed. */ +function node() { return new Dormice({ endpoint: inject('dormiceEndpoint'), token: inject('dormiceToken'), @@ -49,16 +58,16 @@ describe('the S3 archive store as a live ledger setting', () => { const dormice = client(); const config = await dormice.getConfig(); const s3 = config.settings.s3; - // The exam daemon boots with the miniS3 env seed — the ledger view - // carries the four non-secret fields and nothing else. + // The exam's gateway boots with the miniS3 env seed — the settings + // view carries the four non-secret fields and nothing else. expect(s3).toMatchObject({ bucket: 'e2e-archive', forcePathStyle: true }); // Black-box secrecy: neither key appears anywhere in the response. expect(JSON.stringify(config)).not.toContain('e2e-secret'); // Same endpoint and bucket, re-typed keys: a rotation moves nothing, - // so it passes the guard, and the daemon probes it against the real + // so it passes the guard, and the gateway probes it against the real // miniS3 over the wire before saving. - if (s3 === null) throw new Error('exam daemon lost its S3 seed'); + if (s3 === null) throw new Error('exam gateway lost its S3 seed'); const { settings } = await dormice.updateSettings({ s3: { endpoint: s3.endpoint, @@ -71,6 +80,7 @@ describe('the S3 archive store as a live ledger setting', () => { }); expect(settings.s3?.bucket).toBe('e2e-archive'); expect(JSON.stringify(settings)).not.toContain('e2e-secret'); + await settled(); }); it('refuses an unreachable store with S3’s own words and saves nothing', async () => { @@ -91,15 +101,15 @@ describe('the S3 archive store as a live ledger setting', () => { status: 502, message: expect.stringMatching(/nothing was saved/), }); - // The ledger did not move. + // The table did not move. expect((await dormice.getConfig()).settings.s3).toEqual(before); }); - it('refuses to clear or move the store while a sandbox is archived', async () => { + it('refuses to clear or move the store while a sandbox is archived on a node — counted from the nodes’ check-ins', async () => { const dormice = client(); // Park one of our own sandboxes in the archive so the guard has // something to protect, whatever the other suites are doing. - await dormice.acquireSandbox('settings-hot-held', { + await node().acquireSandbox('settings-hot-held', { policy: { freezeAfterSeconds: 1, stopAfterSeconds: 2, @@ -109,7 +119,7 @@ describe('the S3 archive store as a live ledger setting', () => { try { const deadline = Date.now() + 15_000; for (;;) { - const mine = (await dormice.listSandboxes()).find( + const mine = (await node().listSandboxes()).find( (s) => s.name === 'settings-hot-held', ); if (mine?.state === 'archived') break; @@ -118,13 +128,23 @@ describe('the S3 archive store as a live ledger setting', () => { } await sleep(0.25); } + // The gateway holds no sandbox state: it knows of the archived disk + // from node A's next reading. Asking before that could clear the + // store for real, so the reading is waited for first. + await until(async () => + (await listNodes(door(), inject('dormiceToken'))).some( + (n) => (n.reading?.sandboxes.byState.archived ?? 0) > 0, + ) + ? true + : undefined, + ); await expect(dormice.updateSettings({ s3: null })).rejects.toMatchObject({ status: 400, message: expect.stringMatching(/archived or restoring/), }); const current = (await dormice.getConfig()).settings.s3; - if (current === null) throw new Error('exam daemon lost its S3 store'); + if (current === null) throw new Error('exam gateway lost its S3 store'); await expect( dormice.updateSettings({ s3: { @@ -145,20 +165,20 @@ describe('the S3 archive store as a live ledger setting', () => { 'e2e-archive', ); } finally { - await client().destroySandbox('settings-hot-held'); + await node().destroySandbox('settings-hot-held'); } }); }); describe('the sandbox domain as a live ledger setting', () => { - it('a domain edit engages and disengages the proxy without a restart', async () => { + it('a domain edit at the door engages and disengages node A’s proxy within a check-in, no restart', async () => { const dormice = client(); const seeded = (await dormice.getConfig()).settings.sandboxDomain; expect(seeded).toBe('sbx.dormice.test'); // A sandbox created before the switch: the proxy resolves by id, so // the same sandbox answers under whatever domain is in force. - const { sandbox } = await dormice.acquireSandbox('settings-hot-domain'); + const { sandbox } = await node().acquireSandbox('settings-hot-domain'); try { // The switch window is kept to three loopback round trips — the exam // daemon is shared, and other suites build hosts on the seed domain. @@ -171,6 +191,7 @@ describe('the sandbox domain as a live ledger setting', () => { const proxied = (status: number) => [200, 502].includes(status); await dormice.updateSettings({ sandboxDomain: 'alt.dormice.test' }); try { + await settled(); const viaAlt = await throughProxy(altHost, '/hot?x=1'); expect(viaAlt.status).toSatisfy(proxied); // The seed domain is out of force: its hosts are plain Fastify @@ -178,6 +199,7 @@ describe('the sandbox domain as a live ledger setting', () => { expect((await throughProxy(seededHost, '/hot')).status).toBe(404); } finally { await dormice.updateSettings({ sandboxDomain: seeded }); + await settled(); } // Restored: the seed domain proxies again, the alt one is gone. expect((await throughProxy(seededHost, '/hot')).status).toSatisfy( @@ -185,19 +207,19 @@ describe('the sandbox domain as a live ledger setting', () => { ); expect((await throughProxy(altHost, '/hot')).status).toBe(404); } finally { - await client().destroySandbox('settings-hot-domain'); + await node().destroySandbox('settings-hot-domain'); } }); it('aliases route inbound alongside the canonical domain, and the swap is atomic', async () => { const dormice = client(); const seeded = (await dormice.getConfig()).settings.sandboxDomain; - if (seeded === null) throw new Error('exam daemon lost its domain seed'); + if (seeded === null) throw new Error('exam gateway lost its domain seed'); expect((await dormice.getConfig()).settings.sandboxDomainAliases).toEqual( [], ); - const { sandbox } = await dormice.acquireSandbox('settings-hot-alias'); + const { sandbox } = await node().acquireSandbox('settings-hot-alias'); const aliasHost = `8000-${sandbox.id}.alias.dormice.test`; const seededHost = `8000-${sandbox.id}.${seeded}`; const proxied = (status: number) => [200, 502].includes(status); @@ -207,6 +229,7 @@ describe('the sandbox domain as a live ledger setting', () => { await dormice.updateSettings({ sandboxDomainAliases: ['alias.dormice.test'], }); + await settled(); expect((await throughProxy(aliasHost, '/hot')).status).toSatisfy(proxied); expect((await throughProxy(seededHost, '/hot')).status).toSatisfy( proxied, @@ -220,6 +243,7 @@ describe('the sandbox domain as a live ledger setting', () => { }); expect(settings.sandboxDomain).toBe('alias.dormice.test'); expect(settings.sandboxDomainAliases).toEqual([seeded]); + await settled(); expect((await throughProxy(aliasHost, '/hot')).status).toSatisfy(proxied); expect((await throughProxy(seededHost, '/hot')).status).toSatisfy( proxied, @@ -232,7 +256,8 @@ describe('the sandbox domain as a live ledger setting', () => { sandboxDomain: seeded, sandboxDomainAliases: [], }); - await client().destroySandbox('settings-hot-alias'); + await settled(); + await node().destroySandbox('settings-hot-alias'); } }); }); diff --git a/e2e/src/setup/daemon.ts b/e2e/src/setup/daemon.ts index 41b11a4c..07ce330d 100644 --- a/e2e/src/setup/daemon.ts +++ b/e2e/src/setup/daemon.ts @@ -15,32 +15,41 @@ export interface FleetNodeHandle { declare module 'vitest' { export interface ProvidedContext { + /** Node A, the daemon every sandbox suite talks to directly. */ dormiceEndpoint: string; + /** The one token of the exam: node A's, its gateway's. */ dormiceToken: string; - /** The Caddy config file the exam daemon owns — an operator-visible artifact. */ + /** + * Node A's own gateway — a fleet of one, the shape every install has + * (design record #22). The fleet's configuration verbs (keys, + * settings, templates, domains) and the console answer here, in + * every mode. + */ + dormiceGatewayEndpoint: string; + /** The Caddy config file node A's gateway owns — an operator-visible artifact. */ dormiceIngressFile: string; /** The built daemon entry, for tests that boot a daemon of their own. */ dormiceDaemonMain: string; + /** The built gateway entry, for tests that boot a gateway of their own (its boot refusals). */ + dormiceGatewayMain: string; /** Node A's exact environment — a second daemon on the same ledger must refuse to start. */ dormiceNodeAEnv: Record; - /** The gateway fronting nodes B and C; null in docker mode, where only node A runs. */ - dormiceGatewayEndpoint: string | null; - /** The fleet's one token: the gateway's, and every fronted node's. */ - dormiceGatewayToken: string | null; - /** The fronted nodes, reachable directly — to stage what the gateway must then find. */ - dormiceGatewayNodes: FleetNodeHandle[] | null; /** The exam's S3, for a node a test boots itself. */ dormiceMiniS3Url: string; - /** The built gateway entry, for tests that boot a gateway of their own (its boot refusals). */ - dormiceGatewayMain: string; + /** The fleet exam's gateway, fronting nodes B and C; null in docker mode, where only node A runs. */ + dormiceFleetGateway: string | null; + /** The fleet exam's token: the gateway's, and every fronted node's. */ + dormiceFleetToken: string | null; + /** The fronted nodes, reachable directly — to stage what the gateway must then find. */ + dormiceFleetNodes: FleetNodeHandle[] | null; } } -// The suite is a black box: it boots the daemon exactly the way production -// does (`node dist/main.js` plus environment variables) and talks to it only -// over the wire. Nothing here imports server internals — that is the point; -// this is the safety net that must keep passing while the internals are -// rewritten freely. +// The suite is a black box: it boots the gateway and the daemon exactly the +// way production does (`node dist/main.js` plus environment variables) and +// talks to them only over the wire. Nothing here imports server internals — +// that is the point; this is the safety net that must keep passing while +// the internals are rewritten freely. const MAIN = fileURLToPath( new URL('../../../packages/server/dist/main.js', import.meta.url), ); @@ -48,13 +57,63 @@ const GATEWAY_MAIN = fileURLToPath( new URL('../../../packages/gateway/dist/main.js', import.meta.url), ); +interface GatewaySpec { + port: number; + token: string; + dataDir: string; + miniS3Url: string; + extraEnv?: Record; +} + +/** + * Boots one gateway the production way and waits for /healthz. The + * fleet's operator knobs are its first-boot seeds (the daemon's old + * names): the sandbox domain, the archive store, the managed front door — + * every node takes them from its check-in. + */ +async function bootGateway(spec: GatewaySpec) { + const endpoint = `http://127.0.0.1:${spec.port}`; + const env: Record = { + PATH: process.env.PATH ?? '', + DORMICE_GATEWAY_PORT: String(spec.port), + DORMICE_GATEWAY_DB_PATH: join(spec.dataDir, 'gateway.db'), + DORMICE_API_TOKEN: spec.token, + // A laptop running the suite is not the machine under judgment: the + // CPU gate is opened wide and the disk floor is off. + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '100', + DORMICE_GATEWAY_NODE_MIN_DISK_GB: '0', + // A wildcard sandbox domain so getHost() and the port proxy are + // exercised — no DNS needed, tests spoof the Host header locally. + // Every exam starts on a fresh database, so the seed lands every run, + // and settings-hot.test.ts exercises the live edit on top of it. + DORMICE_SANDBOX_DOMAIN: 'sbx.dormice.test', + // The archiver, pointed at the exam's mini S3 — likewise a first-boot + // seed. Deliberately never inherited from the developer's shell: a + // real DORMICE_S3_* export must never leak an exam's archives into a + // production bucket. + DORMICE_S3_ENDPOINT: spec.miniS3Url, + DORMICE_S3_BUCKET: 'e2e-archive', + DORMICE_S3_ACCESS_KEY_ID: 'e2e-key', + DORMICE_S3_SECRET_ACCESS_KEY: 'e2e-secret', + DORMICE_S3_FORCE_PATH_STYLE: 'true', + ...spec.extraEnv, + }; + const child = spawn('node', [GATEWAY_MAIN], { + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + await waitHealthy(child, endpoint, `gateway at ${spec.port}`); + return { endpoint, token: spec.token, env, kill: () => child.kill() }; +} + interface DaemonSpec { /** DORMICE_NODE_ID; omitted keeps the daemon's default, as node A always has. */ nodeId?: string; port: number; token: string; dataDir: string; - miniS3Url: string; + /** The gateway this node checks in with — and takes its configuration from, before it listens. */ + gateway: string; extraEnv?: Record; } @@ -97,26 +156,11 @@ async function bootDaemon(spec: DaemonSpec) { // Sample every second so history verbs have rows to answer with // inside a test's lifetime. DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS: '1', - // A wildcard sandbox domain so getHost() and the port proxy are - // exercised — no DNS needed, tests spoof the Host header locally. - // A first-boot seed since the setting moved into the ledger: every - // exam starts on a fresh DB, so the seed lands every run, and - // settings-hot.test.ts exercises the live edit on top of it. - DORMICE_SANDBOX_DOMAIN: 'sbx.dormice.test', - // The archiver, pointed at the exam's mini S3 — likewise a - // first-boot seed for the ledger's S3 settings. Deliberately NOT in - // the inherited allowlist: a developer's real DORMICE_S3_* exports - // must never leak an exam's archives into a production bucket. - DORMICE_S3_ENDPOINT: spec.miniS3Url, - DORMICE_S3_BUCKET: 'e2e-archive', - DORMICE_S3_ACCESS_KEY_ID: 'e2e-key', - DORMICE_S3_SECRET_ACCESS_KEY: 'e2e-secret', - DORMICE_S3_FORCE_PATH_STYLE: 'true', - // A managed ingress so the domain-binding verbs run black-box. The - // reload command is a no-op: the exam grades what the daemon writes - // and answers, not Caddy — Caddy's side is real-machine acceptance. - DORMICE_INGRESS_FILE: join(spec.dataDir, 'Caddyfile'), - DORMICE_INGRESS_RELOAD_CMD: 'true', + // The node's gateway, and second-scale check-ins: a configuration + // change at the gateway reaches the node inside a test, and a node's + // readings and absence show inside one. + DORMICE_GATEWAY_ENDPOINT: spec.gateway, + DORMICE_CHECK_IN_INTERVAL_SECONDS: '1', ...spec.extraEnv, }; const child = spawn('node', [MAIN], { @@ -124,13 +168,7 @@ async function bootDaemon(spec: DaemonSpec) { stdio: ['ignore', 'pipe', 'pipe'], }); await waitHealthy(child, endpoint, `daemon ${spec.nodeId ?? 'A'}`); - return { - endpoint, - token: spec.token, - env, - ingressFile: join(spec.dataDir, 'Caddyfile'), - kill: () => child.kill(), - }; + return { endpoint, token: spec.token, env, kill: () => child.kill() }; } async function waitHealthy( @@ -167,17 +205,22 @@ async function waitHealthy( } export default async function setup(project: TestProject) { - if (!existsSync(MAIN)) { - throw new Error( - `daemon build not found at ${MAIN} — run \`pnpm build\` first`, - ); + for (const [what, entry] of [ + ['daemon', MAIN], + ['gateway', GATEWAY_MAIN], + ] as const) { + if (!existsSync(entry)) { + throw new Error( + `${what} build not found at ${entry} — run \`pnpm build\` first`, + ); + } } const dataDir = await mkdtemp(join(tmpdir(), 'dormice-e2e-')); // Random high base port: never collides with a locally running daemon on - // 3676 or a gateway on 3677; the fleet takes the next three. Below - // 32768, where Linux hands out ephemeral ports — a fleet port already - // taken by some outbound connection would be a boot failure for nothing. + // 3676 or a gateway on 3677; the exam takes the next five. Below 32768, + // where Linux hands out ephemeral ports — a port already taken by some + // outbound connection would be a boot failure for nothing. const base = 20000 + Math.floor(Math.random() * 12000); // The exam's own S3 (in-process, test-only): the archive lifecycle runs @@ -186,70 +229,71 @@ export default async function setup(project: TestProject) { // daemon that regressed pit #8 fails here too. const miniS3 = await startMiniS3(); - // Node A: the daemon every existing suite talks to, exactly as before — - // a standalone daemon that checks in with nobody. - const a = await bootDaemon({ - port: base, - token: randomBytes(32).toString('hex'), - dataDir, - miniS3Url: miniS3.url, - }); - project.provide('dormiceEndpoint', a.endpoint); - project.provide('dormiceToken', a.token); - project.provide('dormiceIngressFile', a.ingressFile); - project.provide('dormiceDaemonMain', MAIN); - project.provide('dormiceNodeAEnv', a.env); - project.provide('dormiceMiniS3Url', miniS3.url); - project.provide('dormiceGatewayMain', GATEWAY_MAIN); - - // The gateway exam: two more daemons behind a gateway, sharing A's mini - // S3 bucket the way a real fleet shares one, and sharing one token with - // the gateway the way a real fleet does. Fake mode only — in docker mode - // the startup guard judges containers by label across the whole machine, - // so two daemons on one docker would refuse each other. - const fleet: Array<{ kill: () => void }> = []; + const processes: Array<{ kill: () => void }> = []; const dirs: string[] = [dataDir]; // Whatever came up before a boot failed is killed here: vitest never // calls the teardown of a setup that threw, and a spawned daemon does // not die with its parent. const abandon = async (error: unknown) => { - for (const process of fleet.reverse()) process.kill(); - a.kill(); + for (const process of processes.reverse()) process.kill(); await miniS3.close(); throw error; }; + + // Node A and its gateway — the single-machine install, a fleet of one: + // the gateway first (a daemon takes its configuration from its gateway + // before it listens), then the daemon every existing suite talks to. + const token = randomBytes(32).toString('hex'); + const gatewayA = await bootGateway({ + port: base + 4, + token, + dataDir, + miniS3Url: miniS3.url, + extraEnv: { + // A managed ingress so the domain-binding verbs run black-box. The + // reload command is a no-op: the exam grades what the gateway writes + // and answers, not Caddy — Caddy's side is real-machine acceptance. + DORMICE_INGRESS_FILE: join(dataDir, 'Caddyfile'), + DORMICE_INGRESS_RELOAD_CMD: 'true', + }, + }).catch(abandon); + processes.push(gatewayA); + const a = await bootDaemon({ + port: base, + token, + dataDir, + gateway: gatewayA.endpoint, + }).catch(abandon); + processes.push(a); + project.provide('dormiceEndpoint', a.endpoint); + project.provide('dormiceToken', token); + project.provide('dormiceGatewayEndpoint', gatewayA.endpoint); + project.provide('dormiceIngressFile', join(dataDir, 'Caddyfile')); + project.provide('dormiceDaemonMain', MAIN); + project.provide('dormiceGatewayMain', GATEWAY_MAIN); + project.provide('dormiceNodeAEnv', a.env); + project.provide('dormiceMiniS3Url', miniS3.url); + + // The fleet exam: two more daemons behind a gateway of their own, + // sharing A's mini S3 bucket the way a real fleet shares one, and + // sharing one token with their gateway the way a real fleet does. Fake + // mode only — in docker mode the startup guard judges containers by + // label across the whole machine, so two daemons on one docker would + // refuse each other. if (process.env.DORMICE_EXECUTOR !== 'docker') { - if (!existsSync(GATEWAY_MAIN)) { - await abandon( - new Error( - `gateway build not found at ${GATEWAY_MAIN} — run \`pnpm build\` first`, - ), - ); - } - const gatewayPort = base + 3; - const gatewayEndpoint = `http://127.0.0.1:${gatewayPort}`; const fleetToken = randomBytes(32).toString('hex'); - // The gateway first, so the nodes' first check-in lands; a gateway - // that comes up after its nodes learns them at their next check-in - // anyway, but the exam should not start on that slack. - const gateway = spawn('node', [GATEWAY_MAIN], { - env: { - PATH: process.env.PATH ?? '', - DORMICE_GATEWAY_PORT: String(gatewayPort), - DORMICE_GATEWAY_DB_PATH: join(dataDir, 'gateway.db'), - DORMICE_API_TOKEN: fleetToken, - // A tiny active limit so the gate can be reached with a handful of - // sandboxes; the CPU gate is opened wide — a laptop running the - // suite is not the machine under judgment; the disk floor is off - // for the same reason. - DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2', - DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '100', - DORMICE_GATEWAY_NODE_MIN_DISK_GB: '0', - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - fleet.push({ kill: () => gateway.kill() }); - await waitHealthy(gateway, gatewayEndpoint, 'gateway').catch(abandon); + const fleetDir = await mkdtemp(join(tmpdir(), 'dormice-e2e-fleet-')); + dirs.push(fleetDir); + const fleetGateway = await bootGateway({ + port: base + 3, + token: fleetToken, + dataDir: fleetDir, + miniS3Url: miniS3.url, + // A tiny active limit so the gate can be reached with a handful of + // sandboxes. + extraEnv: { DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '2' }, + }).catch(abandon); + processes.push(fleetGateway); const nodes: FleetNodeHandle[] = []; for (const [index, id] of (['node-b', 'node-c'] as const).entries()) { const nodeDir = await mkdtemp(join(tmpdir(), `dormice-e2e-${id}-`)); @@ -259,30 +303,23 @@ export default async function setup(project: TestProject) { port: base + 1 + index, token: fleetToken, dataDir: nodeDir, - miniS3Url: miniS3.url, - extraEnv: { - DORMICE_GATEWAY_ENDPOINT: gatewayEndpoint, - // Second-scale check-ins so a node's readings and its absence - // show inside a test. - DORMICE_CHECK_IN_INTERVAL_SECONDS: '1', - }, + gateway: fleetGateway.endpoint, }).catch(abandon); - fleet.push(node); + processes.push(node); nodes.push({ id, endpoint: node.endpoint }); } - project.provide('dormiceGatewayEndpoint', gatewayEndpoint); - project.provide('dormiceGatewayToken', fleetToken); - project.provide('dormiceGatewayNodes', nodes); + project.provide('dormiceFleetGateway', fleetGateway.endpoint); + project.provide('dormiceFleetToken', fleetToken); + project.provide('dormiceFleetNodes', nodes); } else { - project.provide('dormiceGatewayEndpoint', null); - project.provide('dormiceGatewayToken', null); - project.provide('dormiceGatewayNodes', null); + project.provide('dormiceFleetGateway', null); + project.provide('dormiceFleetToken', null); + project.provide('dormiceFleetNodes', null); } return async () => { - // The nodes first (they stop checking in), then the gateway. - for (const process of fleet.reverse()) process.kill(); - a.kill(); + // The nodes first (they stop checking in), then the gateways. + for (const process of processes.reverse()) process.kill(); await miniS3.close(); for (const dir of dirs) await rm(dir, { recursive: true, force: true }); }; diff --git a/packages/cli/package.json b/packages/cli/package.json index 0ebdb526..722142bd 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -43,6 +43,7 @@ "commander": "^14.0.0" }, "devDependencies": { + "@dormice/gateway": "workspace:*", "@dormice/server": "workspace:*" } } diff --git a/packages/cli/src/commands.test.ts b/packages/cli/src/commands.test.ts index 319590d9..c77f9fac 100644 --- a/packages/cli/src/commands.test.ts +++ b/packages/cli/src/commands.test.ts @@ -1,7 +1,9 @@ import { fileURLToPath } from 'node:url'; +import { testGateway } from '@dormice/gateway'; import { Dormice } from '@dormice/sdk'; import { buildApp, + configureNode, FakeExecutor, KeyedQueue, loadConfig, @@ -40,6 +42,12 @@ const MIGRATIONS = fileURLToPath( let app: ReturnType; let client: Dormice; let endpoint: string; +// The fleet's door, embedded like the daemon: the apikey and template +// commands speak to it (design record #22). No node checks in with it — +// these commands need none. +let gatewayApp: ReturnType['app']; +let gatewayClient: Dormice; +let gatewayEndpoint: string; beforeAll(async () => { const db = openDb(':memory:'); @@ -51,6 +59,8 @@ beforeAll(async () => { DORMICE_NODE_ID: 'node-test', DORMICE_API_TOKEN: TOKEN, }); + // The configuration copy a check-in would have applied. + configureNode(db); app = buildApp({ config, db, @@ -67,10 +77,20 @@ beforeAll(async () => { } endpoint = `http://127.0.0.1:${address.port}`; client = new Dormice({ endpoint, token: TOKEN }); + + gatewayApp = testGateway({ DORMICE_API_TOKEN: TOKEN }).app; + await gatewayApp.listen({ host: '127.0.0.1', port: 0 }); + const gatewayAddress = gatewayApp.server.address(); + if (typeof gatewayAddress !== 'object' || gatewayAddress === null) { + throw new Error('expected a TCP address'); + } + gatewayEndpoint = `http://127.0.0.1:${gatewayAddress.port}`; + gatewayClient = new Dormice({ endpoint: gatewayEndpoint, token: TOKEN }); }); afterAll(async () => { await app.close(); + await gatewayApp.close(); }); describe('clientFromEnv', () => { @@ -226,93 +246,95 @@ describe('parseLabels', () => { describe('apikey commands over real HTTP', () => { it('create, ls and revoke walk the rotation life end to end', async () => { - expect(await apikeyLs(client)).toBe('No API keys.'); + expect(await apikeyLs(gatewayClient)).toBe('No API keys.'); - const created = await apikeyCreate(client, 'ci'); + const created = await apikeyCreate(gatewayClient, 'ci'); const lines = created.split('\n'); expect(lines[0]).toMatch(/^Created API key "ci" \(prefix [0-9a-f]{8}\)\.$/); expect(lines[1]).toMatch(/^[0-9a-f]{64}$/); expect(lines[2]).toBe('Store it now — it will never be shown again.'); - const output = await apikeyLs(client); + const output = await apikeyLs(gatewayClient); expect(output.split('\n')[0]).toMatch( /^NAME\s{2,}PREFIX\s{2,}CREATED\s{2,}LAST USED\s{2,}EXPIRES\s{2,}STATUS$/, ); expect(output).toMatch(/ci\s{2,}[0-9a-f]{8}.*never\s{2,}never\s{2,}active/); - expect(await apikeyRevoke(client, 'ci')).toBe( + expect(await apikeyRevoke(gatewayClient, 'ci')).toBe( 'Revoked API key "ci" — it stops working immediately.', ); - expect(await apikeyRevoke(client, 'ci')).toBe( + expect(await apikeyRevoke(gatewayClient, 'ci')).toBe( 'No active API key named "ci" — nothing to revoke.', ); - expect(await apikeyLs(client)).toMatch(/ci\s{2,}.*revoked/); + expect(await apikeyLs(gatewayClient)).toMatch(/ci\s{2,}.*revoked/); }); it('disable parks a key by name; enable resumes it; a disabled key still revokes by name', async () => { - await apikeyCreate(client, 'park-me'); + await apikeyCreate(gatewayClient, 'park-me'); - expect(await apikeyDisable(client, 'park-me')).toBe( + expect(await apikeyDisable(gatewayClient, 'park-me')).toBe( 'Disabled API key "park-me" — it stops working until re-enabled.', ); - expect(await apikeyLs(client)).toMatch(/park-me\s{2,}.*disabled/); + expect(await apikeyLs(gatewayClient)).toMatch(/park-me\s{2,}.*disabled/); - expect(await apikeyEnable(client, 'park-me')).toBe( + expect(await apikeyEnable(gatewayClient, 'park-me')).toBe( 'Enabled API key "park-me".', ); - expect(await apikeyLs(client)).toMatch(/park-me\s{2,}.*active/); + expect(await apikeyLs(gatewayClient)).toMatch(/park-me\s{2,}.*active/); // Disabled keys keep their name — revoke must still reach them by it. - await apikeyDisable(client, 'park-me'); - expect(await apikeyRevoke(client, 'park-me')).toBe( + await apikeyDisable(gatewayClient, 'park-me'); + expect(await apikeyRevoke(gatewayClient, 'park-me')).toBe( 'Revoked API key "park-me" — it stops working immediately.', ); - await expect(apikeyDisable(client, 'park-me')).rejects.toThrow( + await expect(apikeyDisable(gatewayClient, 'park-me')).rejects.toThrow( /no API key named "park-me"/, ); }); it('--expires mints a TTL key through end-of-day and refuses garbage dates', async () => { - const created = await apikeyCreate(client, 'ttl', '2030-06-15'); + const created = await apikeyCreate(gatewayClient, 'ttl', '2030-06-15'); expect(created.split('\n')[0]).toMatch( /^Created API key "ttl" \(prefix [0-9a-f]{8}, expires 2030-06-1[56]T.*\)\.$/, ); - expect(await apikeyLs(client)).toMatch(/ttl\s{2,}.*active/); - - await expect(apikeyCreate(client, 'bad', 'next tuesday')).rejects.toThrow( - /--expires must be a date like 2026-12-31/, - ); - await expect(apikeyCreate(client, 'bad', '2030-02-31')).rejects.toThrow( - /--expires/, - ); + expect(await apikeyLs(gatewayClient)).toMatch(/ttl\s{2,}.*active/); + + await expect( + apikeyCreate(gatewayClient, 'bad', 'next tuesday'), + ).rejects.toThrow(/--expires must be a date like 2026-12-31/); + await expect( + apikeyCreate(gatewayClient, 'bad', '2030-02-31'), + ).rejects.toThrow(/--expires/); }); it("a minted key is refused on the management verbs with the server's honest 403", async () => { - const created = await apikeyCreate(client, 'not-admin'); + const created = await apikeyCreate(gatewayClient, 'not-admin'); const token = created.split('\n')[1] ?? ''; expect(token).toMatch(/^[0-9a-f]{64}$/); - const keyed = new Dormice({ endpoint, token }); + const keyed = new Dormice({ endpoint: gatewayEndpoint, token }); await expect(apikeyLs(keyed)).rejects.toThrow( - /cannot manage API keys or settings — use DORMICE_API_TOKEN or the console/, + /cannot manage API keys, settings, templates, domains or nodes — use DORMICE_API_TOKEN or the console/, ); }); }); describe('template commands over real HTTP', () => { it('add, ls and rm walk the registration life end to end', async () => { - expect(await templateLs(client)).toBe('No templates.'); + expect(await templateLs(gatewayClient)).toBe('No templates.'); - expect(await templateAdd(client, 'py311', 'img:py311')).toBe( + expect(await templateAdd(gatewayClient, 'py311', 'img:py311')).toBe( 'Registered template "py311" -> img:py311.', ); - const output = await templateLs(client); + const output = await templateLs(gatewayClient); expect(output.split('\n')[0]).toMatch( /^NAME\s{2,}IMAGE\s{2,}CREATED\s{2,}UPDATED$/, ); expect(output).toMatch(/py311\s{2,}img:py311/); - expect(await templateRm(client, 'py311')).toBe('Removed template "py311".'); - expect(await templateRm(client, 'py311')).toBe( + expect(await templateRm(gatewayClient, 'py311')).toBe( + 'Removed template "py311".', + ); + expect(await templateRm(gatewayClient, 'py311')).toBe( 'No template named "py311" — nothing to remove.', ); }); diff --git a/packages/cli/src/doctor.ts b/packages/cli/src/doctor.ts index fff96f3d..1633ec1a 100644 --- a/packages/cli/src/doctor.ts +++ b/packages/cli/src/doctor.ts @@ -604,10 +604,13 @@ const CHECKS: DoctorCheck[] = [ id: 's3-config', title: 'S3 archive configuration', run: async (ctx) => { - // Since the settings moved into the ledger these variables are - // first-boot seeds; doctor stays an offline preflight (env, files, - // commands — never the daemon), so it reports on the seed and points - // at the console for the value in force. + // The S3 variables are the gateway's first-boot seeds now (the + // fleet's settings live in the gateway's table, edited from the + // console; a node takes them from its check-in). Doctor stays an + // offline preflight (env, files, commands — never a process), so it + // reports on the seed in the environment it was given — the + // gateway's env file on the machine that runs the gateway — and + // points at the console for the value in force. const wanted = [ 'DORMICE_S3_ENDPOINT', 'DORMICE_S3_BUCKET', @@ -617,19 +620,19 @@ const CHECKS: DoctorCheck[] = [ const missing = wanted.filter((name) => !ctx.env[name]); if (missing.length === wanted.length) { return skip( - 'DORMICE_S3_* not set — no first-boot seed; archiving can be switched on from the console settings at any time', + "DORMICE_S3_* not set in this environment — no first-boot seed for the gateway; archiving can be switched on from the console settings at any time (a node never reads these: they are the gateway's)", ); } if (missing.length > 0) { // The daemon's config schema refuses this too; naming it here saves // one failed boot. return fail( - `partial S3 set: ${missing.join(', ')} missing — the daemon refuses a half-configured seed`, - 'set all four DORMICE_S3_* variables (endpoint, bucket, key id, secret) or none', + `partial S3 set: ${missing.join(', ')} missing — the gateway refuses a half-configured seed`, + 'set all four DORMICE_S3_* variables (endpoint, bucket, key id, secret) in the gateway env, or none', ); } return pass( - 'all four DORMICE_S3_* variables set — a first-boot seed; once the daemon has booted, the ledger (console settings) rules', + "all four DORMICE_S3_* variables set — the gateway's first-boot seed; once it has started, its settings table (console settings) rules", ); }, }, @@ -668,7 +671,7 @@ const CHECKS: DoctorCheck[] = [ const file = ctx.env.DORMICE_INGRESS_FILE; if (!file) { return skip( - 'DORMICE_INGRESS_FILE not set — the daemon manages no reverse proxy; bind domains by editing your proxy config directly', + "DORMICE_INGRESS_FILE not set in this environment — no managed reverse proxy; it is the gateway's variable (the gateway rewrites the file on setIngress), bind domains by editing your proxy config directly", ); } const caddy = await ctx.run('caddy', ['version']); @@ -681,7 +684,7 @@ const CHECKS: DoctorCheck[] = [ const active = await ctx.run('systemctl', ['is-active', 'caddy']); if (!active.ok) { return fail( - 'the caddy service is not active — nothing proxies the outside world to the daemon', + 'the caddy service is not active — nothing proxies the outside world to the gateway', 'systemctl start caddy (and `journalctl -u caddy` for why it stopped)', ); } @@ -693,7 +696,7 @@ const CHECKS: DoctorCheck[] = [ if (!content.includes('Managed by Dormice')) { return warn( `${file} was not written by Dormice — setIngress refuses to overwrite it, so web domain binding is effectively off`, - 'move your config elsewhere and re-run install.sh, or point DORMICE_INGRESS_FILE at a file the daemon may own', + 'move your config elsewhere and re-run install.sh, or point DORMICE_INGRESS_FILE at a file the gateway may own', ); } // The generated shape puts site addresses at column 0; every diff --git a/packages/gateway/package.json b/packages/gateway/package.json index a11d53a5..c461c26b 100644 --- a/packages/gateway/package.json +++ b/packages/gateway/package.json @@ -5,6 +5,12 @@ "description": "Dormice gateway: the fleet's one door — places new sandboxes across nodes, finds existing ones by asking, forwards everything else", "license": "Apache-2.0", "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, "files": [ "dist", "drizzle" diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 3ee7f1e6..db4ba12c 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -129,6 +129,10 @@ class FakeNode { envdAccessToken: `envd-${this.id}-${String(body.sandboxId)}`, }); } + case '/templateUsers': { + // This double records no template per sandbox: nothing here uses one. + return json(200, { sandboxNames: [] }); + } case '/lookupSandbox': { const sandbox = 'id' in body ? this.byId(body.id as string) : found; return json( diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts index 39956994..17cbc15c 100644 --- a/packages/gateway/src/app.ts +++ b/packages/gateway/src/app.ts @@ -229,7 +229,7 @@ export function buildGatewayApp({ await reply.code(401).send({ message: 'missing or invalid API token' }); } }); - await nodesFace.register(checkInRoutes, { fleet }); + await nodesFace.register(checkInRoutes, { fleet, db }); }); // The sandbox gate: everything that addresses a sandbox. diff --git a/packages/gateway/src/db/node-config.ts b/packages/gateway/src/db/node-config.ts new file mode 100644 index 00000000..ad0ac7d2 --- /dev/null +++ b/packages/gateway/src/db/node-config.ts @@ -0,0 +1,31 @@ +import type { NodeConfigBundle } from '@dormice/shared'; +import type { Db } from './db'; +import { readConfigVersion, readS3Settings, readSettings } from './settings'; +import { listTemplates } from './templates'; + +/** + * The bundle a node applies (shared nodeConfigBundleSchema): the settings + * row with the store's keys, the node's own row, every template — read + * back to back on one synchronous connection, so a node never receives + * one version's number with another version's content (nothing runs + * between two better-sqlite3 statements in the same tick). + */ +export function readNodeConfig( + db: Db, + node: { swapGb: number }, +): NodeConfigBundle { + const settings = readSettings(db); + return { + version: readConfigVersion(db), + settings: { + sandboxDefaults: settings.sandboxDefaults, + defaultPolicy: settings.defaultPolicy, + s3: readS3Settings(db), + sandboxDomain: settings.sandboxDomain, + sandboxDomainAliases: settings.sandboxDomainAliases, + pidsLimit: settings.pidsLimit, + }, + node: { swapGb: node.swapGb }, + templates: listTemplates(db), + }; +} diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 8a05d841..15f278ab 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -180,6 +180,7 @@ export class Fleet { node.intervalSeconds = report.intervalSeconds; node.build = report.build; node.reading = report.reading; + node.configVersion = report.configVersion; node.placedSinceCheckIn = 0; node.placedIds.clear(); return { node, joined, movedFrom }; diff --git a/packages/gateway/src/index.ts b/packages/gateway/src/index.ts new file mode 100644 index 00000000..0c275dbe --- /dev/null +++ b/packages/gateway/src/index.ts @@ -0,0 +1,16 @@ +/** + * Library surface of the gateway: importing this has no side effects. + * Used by the suites that embed a gateway on an ephemeral port the way + * they embed a daemon (the SDK's and the CLI's — the verbs they exercise + * for keys, settings and templates answer here now). Booting the real + * gateway lives in main.ts. + */ +export { buildGatewayApp, type GatewayAppDeps } from './app'; +export { NameCache } from './cache'; +export { type Config, loadConfig } from './config'; +export { type Db, migrateDb, openDb } from './db/db'; +export { ensureSettings } from './db/settings'; +export { Finder } from './find'; +export { Fleet } from './fleet'; +export { type AskNode, type AskVerb, httpAsk, httpAskNode } from './lookup'; +export { checkInOf, reading, TEST_TOKEN, testGateway } from './testing'; diff --git a/packages/gateway/src/placement.ts b/packages/gateway/src/placement.ts index 456e26ec..326e4dd3 100644 --- a/packages/gateway/src/placement.ts +++ b/packages/gateway/src/placement.ts @@ -59,7 +59,10 @@ export interface Placement { * node with room per core and little memory left is still chosen, and * memory pressure is judged where it is felt, by the node's own admission * (design record #27, after the cluster), not guessed from a - * fifteen-second-old figure the picks do not move. + * fifteen-second-old figure the picks do not move. And a node whose last + * check-in reported no configuration copy is refused: a daemon booting + * without one fetches it before it listens, and until its next check-in + * says otherwise the gateway must assume the port is still shut. */ export function pick( nodes: readonly NodeState[], @@ -86,6 +89,16 @@ export function pick( refuse('has not reported a reading'); continue; } + // A node that reported no configuration copy has no defaults to build + // a sandbox from — and is not listening yet: the daemon fetches its + // first bundle before it opens its port (server/main.ts), and a create + // sent there would be refused at the socket. + if (node.configVersion === null) { + refuse( + 'holds no configuration copy yet — its first bundle rides on its next check-in', + ); + continue; + } const cpuUsedPct = reading.host.cpuUsedPct; if (cpuUsedPct !== null && cpuUsedPct > knobs.cpuLimitPct) { refuse( diff --git a/packages/gateway/src/routes/nodes.test.ts b/packages/gateway/src/routes/nodes.test.ts new file mode 100644 index 00000000..a8b0ccd2 --- /dev/null +++ b/packages/gateway/src/routes/nodes.test.ts @@ -0,0 +1,173 @@ +import { + checkInResponseSchema, + listNodesResponseSchema, +} from '@dormice/shared'; +import { describe, expect, it } from 'vitest'; +import { readConfigVersion } from '../db/settings'; +import { checkInOf, TEST_TOKEN, testGateway } from '../testing'; + +// The check-in as the pull, and the one per-node knob — over app.inject(): +// what a node posts, what it gets back, what listNodes then shows. + +const authed = { authorization: `Bearer ${TEST_TOKEN}` }; + +type App = ReturnType['app']; + +function rpc( + app: App, + url: string, + payload: object = {}, + headers: Record = authed, +) { + return app.inject({ method: 'POST', url, headers, payload }); +} + +const S3_ENV = { + DORMICE_S3_ENDPOINT: 'http://127.0.0.1:9000', + DORMICE_S3_BUCKET: 'seed-bucket', + DORMICE_S3_ACCESS_KEY_ID: 'seed-key', + DORMICE_S3_SECRET_ACCESS_KEY: 'seed-secret-for-the-nodes-only', +}; + +async function checkIn( + app: App, + id: string, + over: Parameters[2] = {}, +) { + const res = await rpc( + app, + '/checkIn', + checkInOf(id, 'http://10.0.0.7:80', over), + ); + expect(res.statusCode).toBe(200); + return checkInResponseSchema.parse(res.json()); +} + +async function nodes(app: App) { + return listNodesResponseSchema.parse((await rpc(app, '/listNodes')).json()) + .nodes; +} + +describe('the check-in as the configuration pull', () => { + it('a node with no copy gets the whole bundle: settings with the store keys, its swap target, the templates, under the current version', async () => { + const { app, db } = testGateway({ + ...S3_ENV, + DORMICE_SANDBOX_DOMAIN: 'sbx.example.com', + DORMICE_SANDBOX_PIDS_LIMIT: '512', + }); + await rpc(app, '/registerTemplate', { name: 'py', image: 'img-a' }); + expect(readConfigVersion(db)).toBe(2); + + const answer = await checkIn(app, 'b', { configVersion: null }); + expect(answer.configVersion).toBe(2); + const config = answer.config; + if (config === undefined) + throw new Error('no bundle for a node without a copy'); + expect(config.version).toBe(2); + expect(config.settings).toEqual({ + sandboxDefaults: { cpus: 1, memoryGb: 2, diskGb: 10 }, + defaultPolicy: expect.objectContaining({ archiveAfterSeconds: 604800 }), + // Keys included: the node presents them to S3 itself. + s3: { + endpoint: 'http://127.0.0.1:9000', + bucket: 'seed-bucket', + accessKeyId: 'seed-key', + secretAccessKey: 'seed-secret-for-the-nodes-only', + region: 'us-east-1', + forcePathStyle: false, + }, + sandboxDomain: 'sbx.example.com', + sandboxDomainAliases: [], + pidsLimit: 512, + }); + expect(config.node).toEqual({ swapGb: 0 }); + expect(config.templates).toMatchObject([{ name: 'py', image: 'img-a' }]); + }); + + it('a node reporting the current version gets the version alone; listNodes shows what each node said it runs', async () => { + const { app } = testGateway(); + const same = await checkIn(app, 'b', { configVersion: 1 }); + expect(same).toEqual({ configVersion: 1 }); + await checkIn(app, 'c', { configVersion: null }); + const listed = await nodes(app); + expect(listed.map((n) => [n.id, n.configVersion]).sort()).toEqual([ + ['b', 1], + ['c', null], + ]); + }); + + it('every write counts the version up and the next check-in of a node on the old version carries the change', async () => { + const { app } = testGateway(); + expect(await checkIn(app, 'b', { configVersion: 1 })).toEqual({ + configVersion: 1, + }); + expect( + (await rpc(app, '/updateSettings', { pidsLimit: 8192 })).statusCode, + ).toBe(200); + const after = await checkIn(app, 'b', { configVersion: 1 }); + expect(after.configVersion).toBe(2); + expect(after.config?.settings.pidsLimit).toBe(8192); + // Caught up: nothing rides along anymore. + expect(await checkIn(app, 'b', { configVersion: 2 })).toEqual({ + configVersion: 2, + }); + // A version the gateway never issued (a restored gateway database) is + // still "not mine": the bundle comes. + expect( + (await checkIn(app, 'b', { configVersion: 9 })).config, + ).toBeDefined(); + }); +}); + +describe('updateNodeSettings', () => { + it('404 for an unknown node, 503 before the node has reported, 400 for a daemon that cannot manage swap', async () => { + const { app, fleet } = testGateway(); + const unknown = await rpc(app, '/updateNodeSettings', { + id: 'ghost', + swapGb: 8, + }); + expect(unknown.statusCode).toBe(404); + + // Known from a row, silent since this gateway started: capability unknown. + fleet.checkIn(checkInOf('b', 'http://10.0.0.7:80')); + const silent = fleet.get('b'); + if (!silent) throw new Error('no node b'); + silent.reading = null; + const early = await rpc(app, '/updateNodeSettings', { id: 'b', swapGb: 8 }); + expect(early.statusCode).toBe(503); + expect(early.headers['retry-after']).toBe('15'); + expect(early.json().message).toMatch( + /has not checked in since the gateway started/, + ); + + fleet.checkIn(checkInOf('b', 'http://10.0.0.7:80', { managedSwap: null })); + const unable = await rpc(app, '/updateNodeSettings', { + id: 'b', + swapGb: 8, + }); + expect(unable.statusCode).toBe(400); + expect(unable.json().message).toMatch(/cannot manage swap/); + expect((await nodes(app)).find((n) => n.id === 'b')?.swapGb).toBe(0); + }); + + it('stores the target on the node row, counts the version up, and the node hears of it at its next check-in — the other node does not', async () => { + const { app, db } = testGateway(); + await checkIn(app, 'b', { configVersion: 1, managedSwap: { activeGb: 0 } }); + await checkIn(app, 'c', { configVersion: 1 }); + const res = await rpc(app, '/updateNodeSettings', { id: 'b', swapGb: 16 }); + expect(res.statusCode).toBe(200); + expect(res.json().node).toMatchObject({ id: 'b', swapGb: 16 }); + expect(readConfigVersion(db)).toBe(2); + + const b = await checkIn(app, 'b', { configVersion: 1 }); + expect(b.config?.node).toEqual({ swapGb: 16 }); + const c = await checkIn(app, 'c', { configVersion: 1 }); + expect(c.config?.node).toEqual({ swapGb: 0 }); + // A shrink is stored as written: the node's reconcile is what defers it. + expect( + (await rpc(app, '/updateNodeSettings', { id: 'b', swapGb: 0 })).json() + .node.swapGb, + ).toBe(0); + expect(readConfigVersion(db)).toBe(3); + }); +}); diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index 60950961..e6d14bd4 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -11,15 +11,20 @@ import { } from '@dormice/shared'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import type { NameCache } from '../cache'; +import type { Db } from '../db/db'; +import { readNodeConfig } from '../db/node-config'; +import { readConfigVersion } from '../db/settings'; import { downReason, type Fleet, type NodeState, STARTUP_GRACE_MS, } from '../fleet'; +import { RETRY_AFTER_SECONDS } from '../raw'; export interface CheckInRoutesOptions { fleet: Fleet; + db: Db; } export interface NodeRoutesOptions { @@ -34,11 +39,15 @@ function refusal(statusCode: number, message: string): Error { /** * The check-in the nodes send (RULES/协议.md「网关」) — behind the nodes' - * own gate in app.ts: the fleet token and nothing else. + * own gate in app.ts: the fleet token and nothing else. The answer is the + * configuration version, and the whole bundle when the node's differs + * (design record #22, shared nodeConfigBundleSchema): the check-in is the + * pull. No record is kept of who was told what — the node states what it + * runs at every check-in, and the comparison is the whole protocol. */ export const checkInRoutes: FastifyPluginAsyncZod< CheckInRoutesOptions -> = async (app, { fleet }) => { +> = async (app, { fleet, db }) => { /** * Per node, the ids it was last reported to share an endpoint with * (sorted, joined) — so the warning below is said when the situation @@ -109,7 +118,17 @@ export const checkInRoutes: FastifyPluginAsyncZod< 'the node no longer shares its endpoint with another', ); } - return {}; + const version = readConfigVersion(db); + if (request.body.configVersion === version) { + return { configVersion: version }; + } + request.log.info( + { nodeId: node.id, runs: request.body.configVersion, current: version }, + request.body.configVersion === null + ? 'a node with no configuration copy checked in; the bundle rides on this answer' + : 'a node runs another configuration version; the bundle rides on this answer', + ); + return { configVersion: version, config: readNodeConfig(db, node) }; }, ); }; @@ -145,16 +164,35 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( response: { 200: updateNodeSettingsResponseSchema }, }, }, - async (request) => { + async (request, reply) => { const { id, swapGb } = request.body; - if (!fleet.setSwapGb(id, swapGb)) { + const node = fleet.get(id); + if (node === undefined) { throw refusal( 404, `no node with id '${id}' — listNodes shows which exist`, ); } - const node = fleet.get(id); - if (node === undefined) throw refusal(404, `no node with id '${id}'`); + // Whether this node's daemon can manage swap at all is the node's + // word, carried in its reading (shared nodeReadingSchema managedSwap): + // a target for a daemon that cannot honor it would sit in the row + // forever, applied by nothing and shown by listNodes as if it were + // real. Unknown (the node has not reported since this gateway + // started) is unknown, not a guess either way. + if (node.reading === null) { + reply.header('retry-after', String(RETRY_AFTER_SECONDS)); + throw refusal( + 503, + `node ${id} has not checked in since the gateway started, so whether its daemon manages swap is unknown — retry after its next check-in`, + ); + } + if (node.reading.managedSwap === null) { + throw refusal( + 400, + `node ${id} cannot manage swap: its daemon reports no managed-swap capability (a Linux host running the docker executor has it) — a target there would never be applied, so none is stored`, + ); + } + fleet.setSwapGb(id, swapGb); request.log.info( { nodeId: id, swapGb }, 'node swap target set; the node applies it at its next check-in', diff --git a/packages/gateway/src/routes/settings.ts b/packages/gateway/src/routes/settings.ts index c650af40..d10bd869 100644 --- a/packages/gateway/src/routes/settings.ts +++ b/packages/gateway/src/routes/settings.ts @@ -8,7 +8,12 @@ import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { z } from 'zod'; import { CONFIG_KEYS, type Config, type ConfigSources } from '../config'; import type { Db } from '../db/db'; -import { readS3Settings, readSettings, writeSettings } from '../db/settings'; +import { + readConfigVersion, + readS3Settings, + readSettings, + writeSettings, +} from '../db/settings'; import type { Fleet } from '../fleet'; import { probeS3 as defaultProbeS3, S3ProbeError } from '../probe'; @@ -74,6 +79,7 @@ export const settingsRoutes: FastifyPluginAsyncZod< : null, }, settings, + configVersion: readConfigVersion(db), }; }, ); diff --git a/packages/gateway/src/testing.ts b/packages/gateway/src/testing.ts index 99e6560e..2d663faa 100644 --- a/packages/gateway/src/testing.ts +++ b/packages/gateway/src/testing.ts @@ -12,10 +12,12 @@ import type { Ingress } from './ingress'; import { type AskNode, type AskVerb, httpAskNode } from './lookup'; /** - * Test scaffolding shared by the gateway's suites: a node's reading and - * check-in with a few knobs turned, and a gateway app over an in-memory - * database for the suites about the gateway's own tables and gates. Not - * shipped — nothing under src/ but main.ts is bundled (tsup.config.ts). + * Test scaffolding shared by the gateway's suites — and, through index.ts, + * by the SDK's and the CLI's, which embed a gateway on an ephemeral port + * for the verbs that answer here: a node's reading and check-in with a few + * knobs turned, and a gateway app over an in-memory database for the + * suites about the gateway's own tables and gates. Never part of a + * running gateway: main.ts imports none of it. */ export const TEST_TOKEN = 'fleet-token-fleet-token-fleet-token-fleet'; @@ -31,6 +33,8 @@ export function reading( restoring?: number; memAvail?: number; diskAvail?: number | null; + /** Null = a daemon that cannot manage swap (the fake executor's word). */ + managedSwap?: { activeGb: number } | null; } = {}, ): NodeReading { const frozen = over.frozen ?? 0; @@ -58,18 +62,26 @@ export function reading( total: active + frozen + archived + restoring, byState: { active, frozen, stopped: 0, archived, restoring }, }, + managedSwap: + over.managedSwap === undefined ? { activeGb: 0 } : over.managedSwap, }; } export function checkInOf( nodeId: string, endpoint: string, - over: Parameters[0] & { intervalSeconds?: number } = {}, + over: Parameters[0] & { + intervalSeconds?: number; + configVersion?: number | null; + } = {}, ): CheckInRequest { return { nodeId, endpoint, intervalSeconds: over.intervalSeconds ?? 15, + // A configured node by default: placement refuses one without a copy, + // and most suites are about nodes that run one. + configVersion: over.configVersion === undefined ? 1 : over.configVersion, build: { commit: 'abc1234', title: 'a commit', @@ -107,10 +119,13 @@ export function testGateway( const config = loadConfig(rawEnv); ensureSettings(db, config); const fleet = new Fleet(db); + // Under the fleet token the config carries: a suite that embeds a real + // node beside this gateway (the SDK's) gives both the same token, and + // the lookups must present it, not the scaffolding's default. const finder = new Finder( fleet, new NameCache(), - opts.ask ?? httpAskNode(TEST_TOKEN), + opts.ask ?? httpAskNode(config.DORMICE_API_TOKEN), { warn: () => {} }, ); const app = buildGatewayApp({ diff --git a/packages/gateway/tsup.config.ts b/packages/gateway/tsup.config.ts index d607d033..1ab151a5 100644 --- a/packages/gateway/tsup.config.ts +++ b/packages/gateway/tsup.config.ts @@ -22,11 +22,14 @@ function git(args: string): string { const commitTime = git('log -1 --format=%cI'); export default defineConfig({ - // A service, not a library: main.ts is the one entry and nothing imports - // the package (e2e boots dist/main.js as a process), so no index and no - // declarations. - entry: ['src/main.ts'], + // A service first: main.ts is the entry e2e boots as a process. The + // index is the library surface the SDK's and the CLI's suites embed a + // gateway through (the verbs they test for keys, settings and templates + // answer at the gateway), so it ships with declarations like the + // daemon's. + entry: ['src/main.ts', 'src/index.ts'], format: ['esm'], + dts: true, clean: true, env: { DORMICE_BUILD_COMMIT: git('rev-parse --short HEAD'), diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 74450b94..2f5c32d5 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -47,6 +47,7 @@ "undici": "^8.7.0" }, "devDependencies": { + "@dormice/gateway": "workspace:*", "@dormice/server": "workspace:*" } } diff --git a/packages/sdk/src/client.test.ts b/packages/sdk/src/client.test.ts index 73d4d07a..d5799280 100644 --- a/packages/sdk/src/client.test.ts +++ b/packages/sdk/src/client.test.ts @@ -1,13 +1,16 @@ import { createServer } from 'node:http'; import { fileURLToPath } from 'node:url'; +import { checkInOf, testGateway } from '@dormice/gateway'; import { buildApp, + configureNode, type Db, FakeExecutor, KeyedQueue, loadConfig, migrateDb, openDb, + registerTestTemplate, scanOnce, } from '@dormice/server'; import { DEFAULT_LIFECYCLE_POLICY } from '@dormice/shared'; @@ -33,6 +36,23 @@ let endpoint: string; let db: Db; let executor: FakeExecutor; let locks: KeyedQueue; +// The fleet's door, embedded like the daemon: the verbs that configure the +// fleet — keys, settings, templates — answer there (design record #22), +// and the SDK speaks the same wire to either. The embedded daemon is its +// one node, joined by a check-in the harness plays (joinNode): the +// gateway then places on it and asks it over real HTTP, under the one +// token both share. +let gateway: ReturnType; +let gatewayClient: Dormice; +let gatewayEndpoint: string; + +/** The daemon reports for duty — fresh, so the fleet's "down" clock never starts on a long suite. */ +function joinNode() { + const outcome = gateway.fleet.checkIn( + checkInOf('node-test', endpoint, { active: 0 }), + ); + if ('refused' in outcome) throw new Error(outcome.refused); +} beforeAll(async () => { db = openDb(':memory:'); @@ -46,6 +66,8 @@ beforeAll(async () => { DORMICE_NODE_ID: 'node-test', DORMICE_API_TOKEN: TOKEN, }); + // The configuration copy a check-in would have applied. + configureNode(db); app = buildApp({ config, db, executor, locks, logger: false }); // Port 0: the OS hands out a free ephemeral port, so tests never collide // with a locally running daemon. @@ -56,10 +78,20 @@ beforeAll(async () => { } endpoint = `http://127.0.0.1:${address.port}`; client = new Dormice({ endpoint, token: TOKEN }); + + gateway = testGateway({ DORMICE_API_TOKEN: TOKEN }); + await gateway.app.listen({ host: '127.0.0.1', port: 0 }); + const gatewayAddress = gateway.app.server.address(); + if (typeof gatewayAddress !== 'object' || gatewayAddress === null) { + throw new Error('expected a TCP address'); + } + gatewayEndpoint = `http://127.0.0.1:${gatewayAddress.port}`; + gatewayClient = new Dormice({ endpoint: gatewayEndpoint, token: TOKEN }); }); afterAll(async () => { await app.close(); + await gateway.app.close(); }); describe('Dormice.acquireSandbox over real HTTP', () => { @@ -324,28 +356,37 @@ describe('Dormice.acquireSandbox over real HTTP', () => { }); describe('templates over real HTTP', () => { - it('registers, lists, applies at acquire, and removes through the full life', async () => { - await client.registerTemplate('tpl-sdk', 'img-sdk'); - expect(await client.listTemplates()).toMatchObject([ + it('registers, lists and removes at the gateway; the node it asks over the wire makes removal a 409 naming the sandbox', async () => { + joinNode(); + await gatewayClient.registerTemplate('tpl-sdk', 'img-sdk'); + expect(await gatewayClient.listTemplates()).toMatchObject([ { name: 'tpl-sdk', image: 'img-sdk' }, ]); + // The bundle's arrival, played on the node's copy; a sandbox built + // from the template lives on the node. + registerTestTemplate(db, 'tpl-sdk', 'img-sdk'); const res = await client.acquireSandbox('tpl-user', { template: 'tpl-sdk', }); expect(res.sandbox.template).toBe('tpl-sdk'); expect(await executor.imageOf(res.sandbox.id)).toBe('img-sdk'); - // In use: removal is refused, naming the key that holds it. - await expect(client.removeTemplate('tpl-sdk')).rejects.toMatchObject({ - name: 'DormiceApiError', - status: 409, - message: expect.stringMatching(/tpl-user/), - }); - + // The gateway asks its node templateUsers over real HTTP, then refuses. + await expect(gatewayClient.removeTemplate('tpl-sdk')).rejects.toMatchObject( + { + name: 'DormiceApiError', + status: 409, + message: expect.stringMatching(/tpl-user on node node-test/), + }, + ); await client.destroySandbox('tpl-user'); - expect(await client.removeTemplate('tpl-sdk')).toEqual({ removed: true }); - expect(await client.removeTemplate('tpl-sdk')).toEqual({ removed: false }); + expect(await gatewayClient.removeTemplate('tpl-sdk')).toEqual({ + removed: true, + }); + expect(await gatewayClient.removeTemplate('tpl-sdk')).toEqual({ + removed: false, + }); }); it("surfaces the server's 400 for an unknown template", async () => { @@ -359,54 +400,68 @@ describe('templates over real HTTP', () => { }); describe('API keys over real HTTP', () => { + /** A minted key does real work through the gateway: placed on the node under the fleet token, destroyed the same way. */ + const opens = async (keyed: Dormice) => { + const created = await keyed.acquireSandbox('keyed-user'); + expect(created.sandbox.nodeId).toBe('node-test'); + await keyed.destroySandbox('keyed-user'); + }; + it('mints a key a fresh client can use, revokes it, and the door closes', async () => { - const { apiKey, token } = await client.createApiKey('sdk-rotation'); + joinNode(); + const { apiKey, token } = await gatewayClient.createApiKey('sdk-rotation'); expect(token).toMatch(/^[0-9a-f]{64}$/); expect(apiKey.prefix).toBe(token.slice(0, 8)); // The rotation story: a new client on the minted key does real work. - const keyed = new Dormice({ endpoint, token }); - await keyed.acquireSandbox('keyed-user'); - await keyed.destroySandbox('keyed-user'); + const keyed = new Dormice({ endpoint: gatewayEndpoint, token }); + await opens(keyed); - const listed = await client.listApiKeys(); + const listed = await gatewayClient.listApiKeys(); const mine = listed.find((k) => k.name === 'sdk-rotation'); expect(mine?.lastUsedAt).not.toBeNull(); - expect(await client.revokeApiKey(apiKey.id)).toEqual({ + expect(await gatewayClient.revokeApiKey(apiKey.id)).toEqual({ revoked: true, }); - await expect(keyed.listSandboxes()).rejects.toMatchObject({ status: 401 }); - expect(await client.revokeApiKey(apiKey.id)).toEqual({ + await expect(keyed.acquireSandbox('keyed-user')).rejects.toMatchObject({ + status: 401, + }); + expect(await gatewayClient.revokeApiKey(apiKey.id)).toEqual({ revoked: false, }); }); it("surfaces the server's 409 for a duplicate active name", async () => { - const { apiKey } = await client.createApiKey('sdk-dup'); - await expect(client.createApiKey('sdk-dup')).rejects.toMatchObject({ + const { apiKey } = await gatewayClient.createApiKey('sdk-dup'); + await expect(gatewayClient.createApiKey('sdk-dup')).rejects.toMatchObject({ status: 409, message: expect.stringMatching(/sdk-dup/), }); - await client.revokeApiKey(apiKey.id); + await gatewayClient.revokeApiKey(apiKey.id); }); it('updateApiKey renames, parks and expires a key in place', async () => { + joinNode(); const future = new Date(Date.now() + 3600_000).toISOString(); - const { apiKey, token } = await client.createApiKey('sdk-edit', { + const { apiKey, token } = await gatewayClient.createApiKey('sdk-edit', { expiresAt: future, }); expect(apiKey.expiresAt).toBe(future); - const keyed = new Dormice({ endpoint, token }); - await keyed.listSandboxes(); + const keyed = new Dormice({ endpoint: gatewayEndpoint, token }); + await opens(keyed); // Park it: the credential dies on the next request, reversibly. - const parked = await client.updateApiKey(apiKey.id, { disabled: true }); + const parked = await gatewayClient.updateApiKey(apiKey.id, { + disabled: true, + }); expect(parked.apiKey.disabledAt).not.toBeNull(); - await expect(keyed.listSandboxes()).rejects.toMatchObject({ status: 401 }); + await expect(keyed.acquireSandbox('keyed-user')).rejects.toMatchObject({ + status: 401, + }); - const resumed = await client.updateApiKey(apiKey.id, { + const resumed = await gatewayClient.updateApiKey(apiKey.id, { disabled: false, name: 'sdk-edit-2', expiresAt: null, @@ -416,59 +471,68 @@ describe('API keys over real HTTP', () => { disabledAt: null, expiresAt: null, }); - await keyed.listSandboxes(); + await opens(keyed); - await client.revokeApiKey(apiKey.id); + await gatewayClient.revokeApiKey(apiKey.id); }); - it('a ledger key gets the honest 403 on the management verbs', async () => { - const { apiKey, token } = await client.createApiKey('sdk-not-admin'); - const keyed = new Dormice({ endpoint, token }); + it('a minted key gets the honest 403 on the management verbs', async () => { + const { apiKey, token } = await gatewayClient.createApiKey('sdk-not-admin'); + const keyed = new Dormice({ endpoint: gatewayEndpoint, token }); await expect(keyed.listApiKeys()).rejects.toMatchObject({ status: 403, message: expect.stringMatching(/cannot manage API keys/), }); - await client.revokeApiKey(apiKey.id); + await gatewayClient.revokeApiKey(apiKey.id); + }); + + it('a node knows only the fleet token: a minted key is a 401 there', async () => { + const { apiKey, token } = await gatewayClient.createApiKey('sdk-at-node'); + const keyed = new Dormice({ endpoint, token }); + await expect(keyed.listSandboxes()).rejects.toMatchObject({ status: 401 }); + await gatewayClient.revokeApiKey(apiKey.id); }); }); -describe('runtime settings over real HTTP', () => { - it('updates a knob and reads it back through getConfig', async () => { - const before = (await client.getConfig()).settings; - const { settings } = await client.updateSettings({ - pidsLimit: before.pidsLimit + 1, +describe('fleet settings over real HTTP', () => { + it('updates a knob and reads it back through getConfig, the version counting up', async () => { + const before = await gatewayClient.getConfig(); + const { settings } = await gatewayClient.updateSettings({ + pidsLimit: before.settings.pidsLimit + 1, }); - expect(settings.pidsLimit).toBe(before.pidsLimit + 1); + expect(settings.pidsLimit).toBe(before.settings.pidsLimit + 1); expect(settings.updatedAt).not.toBeNull(); - expect((await client.getConfig()).settings.pidsLimit).toBe( - before.pidsLimit + 1, - ); - // Restore: other suites share this daemon's ledger. - await client.updateSettings({ pidsLimit: before.pidsLimit }); + const after = await gatewayClient.getConfig(); + expect(after.settings.pidsLimit).toBe(before.settings.pidsLimit + 1); + expect(after.configVersion).toBe(before.configVersion + 1); + // Restore: other suites share this gateway's tables. + await gatewayClient.updateSettings({ + pidsLimit: before.settings.pidsLimit, + }); }); it('is admin-only, like the apiKey verbs', async () => { - const { apiKey, token } = await client.createApiKey('sdk-settings'); - const keyed = new Dormice({ endpoint, token }); + const { apiKey, token } = await gatewayClient.createApiKey('sdk-settings'); + const keyed = new Dormice({ endpoint: gatewayEndpoint, token }); await expect( keyed.updateSettings({ pidsLimit: 12345 }), ).rejects.toMatchObject({ status: 403, - message: expect.stringMatching(/cannot manage API keys or settings/), + message: expect.stringMatching(/cannot manage API keys/), }); - await client.revokeApiKey(apiKey.id); + await gatewayClient.revokeApiKey(apiKey.id); }); }); describe('the observability verbs over real HTTP', () => { it('getConfig reports the knobs and withholds the token', async () => { - // Source attribution is asserted server-side with injected sources; - // this test app reads the real process.env, which proves nothing here. - const config = await client.getConfig(); + // Source attribution is asserted gateway-side with injected sources; + // here the gateway's env is the harness's, which proves nothing more. + const config = await gatewayClient.getConfig(); const token = config.entries.find((e) => e.key === 'DORMICE_API_TOKEN'); expect(token).toMatchObject({ value: null, redacted: true }); - const port = config.entries.find((e) => e.key === 'DORMICE_PORT'); - expect(port).toMatchObject({ value: '3676' }); + const port = config.entries.find((e) => e.key === 'DORMICE_GATEWAY_PORT'); + expect(port).toMatchObject({ value: '3677' }); expect(config.archive.enabled).toBe(false); }); diff --git a/packages/server/drizzle/0025_config-copy.sql b/packages/server/drizzle/0025_config-copy.sql new file mode 100644 index 00000000..80070b69 --- /dev/null +++ b/packages/server/drizzle/0025_config-copy.sql @@ -0,0 +1,2 @@ +ALTER TABLE `runtime_settings` ADD `config_version` integer;--> statement-breakpoint +ALTER TABLE `runtime_settings` ADD `config_applied_at` text; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0025_snapshot.json b/packages/server/drizzle/meta/0025_snapshot.json new file mode 100644 index 00000000..4adf7281 --- /dev/null +++ b/packages/server/drizzle/meta/0025_snapshot.json @@ -0,0 +1,780 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "2769cadc-8aa9-42b5-b4c5-0b60d0138113", + "prevId": "e20f0c46-e1c7-4734-ace9-9dc7f88d40a6", + "tables": { + "api_keys": { + "name": "api_keys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + "key_hash" + ], + "isUnique": true + }, + "api_keys_active_name_idx": { + "name": "api_keys_active_name_idx", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"api_keys\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "console_account": { + "name": "console_account", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_secret": { + "name": "session_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "daemon_secrets": { + "name": "daemon_secrets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "envd_signing_secret": { + "name": "envd_signing_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "fleet_snapshots": { + "name": "fleet_snapshots", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "active": { + "name": "active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "frozen": { + "name": "frozen", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stopped": { + "name": "stopped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "restoring": { + "name": "restoring", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "host_metrics_samples": { + "name": "host_metrics_samples", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "cpu_used_pct": { + "name": "cpu_used_pct", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_available_bytes": { + "name": "mem_available_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_total_bytes": { + "name": "swap_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_used_bytes": { + "name": "swap_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_total_bytes": { + "name": "disk_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_used_bytes": { + "name": "disk_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_available_bytes": { + "name": "disk_available_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "runtime_settings": { + "name": "runtime_settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_version": { + "name": "config_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config_applied_at": { + "name": "config_applied_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_cpus": { + "name": "sandbox_cpus", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_memory_gb": { + "name": "sandbox_memory_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_disk_gb": { + "name": "sandbox_disk_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_freeze_after_seconds": { + "name": "default_freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_stop_after_seconds": { + "name": "default_stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_archive_after_seconds": { + "name": "default_archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_gb": { + "name": "swap_gb", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "s3_endpoint": { + "name": "s3_endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_access_key_id": { + "name": "s3_access_key_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_secret_access_key": { + "name": "s3_secret_access_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_region": { + "name": "s3_region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_force_path_style": { + "name": "s3_force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain": { + "name": "sandbox_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain_aliases": { + "name": "sandbox_domain_aliases", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pids_limit": { + "name": "pids_limit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sandbox_metrics_samples": { + "name": "sandbox_metrics_samples", + "columns": { + "sandbox_id": { + "name": "sandbox_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "at": { + "name": "at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cpu_count": { + "name": "cpu_count", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cpu_used_pct": { + "name": "cpu_used_pct", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_used_bytes": { + "name": "mem_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_cache_bytes": { + "name": "mem_cache_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_used_bytes": { + "name": "swap_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_total_bytes": { + "name": "swap_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_used_bytes": { + "name": "disk_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "disk_total_bytes": { + "name": "disk_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "sandbox_metrics_samples_sandbox_at_idx": { + "name": "sandbox_metrics_samples_sandbox_at_idx", + "columns": [ + "sandbox_id", + "at" + ], + "isUnique": false + }, + "sandbox_metrics_samples_at_idx": { + "name": "sandbox_metrics_samples_at_idx", + "columns": [ + "at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sandboxes": { + "name": "sandboxes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "freeze_after_seconds": { + "name": "freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stop_after_seconds": { + "name": "stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "archive_after_seconds": { + "name": "archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpus": { + "name": "cpus", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "memory_gb": { + "name": "memory_gb", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_gb": { + "name": "disk_gb", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_active_at": { + "name": "last_active_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_exit_at": { + "name": "last_exit_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_exit_code": { + "name": "last_exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_exit_cause": { + "name": "last_exit_cause", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "envs": { + "name": "envs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "on_deadline": { + "name": "on_deadline", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "paused_by_user": { + "name": "paused_by_user", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "sandboxes_name_unique": { + "name": "sandboxes_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index 8b0882e6..938835ce 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -176,6 +176,13 @@ "when": 1789370887387, "tag": "0024_drop-max-sandboxes", "breakpoints": true + }, + { + "idx": 25, + "version": "6", + "when": 1789375849689, + "tag": "0025_config-copy", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/server/package.json b/packages/server/package.json index f08a0414..bca3d704 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -48,9 +48,7 @@ "@aws-sdk/client-s3": "^3.1083.0", "@aws-sdk/lib-storage": "^3.1083.0", "@dormice/shared": "workspace:*", - "@fastify/cookie": "^11.0.2", "@fastify/multipart": "^10.0.0", - "@fastify/static": "^9.1.3", "better-sqlite3": "^12.11.1", "dockerode": "^5.0.1", "drizzle-orm": "^0.45.2", diff --git a/packages/server/src/app.test.ts b/packages/server/src/app.test.ts index 906876cc..c6179dc3 100644 --- a/packages/server/src/app.test.ts +++ b/packages/server/src/app.test.ts @@ -12,7 +12,6 @@ import { buildApp } from './app'; import { Archiver } from './archive/archiver'; import { MemStore } from './archive/mem-store'; import { objectKey } from './archive/store'; -import { CONSOLE_HEADER, SESSION_COOKIE } from './auth'; import { loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; import { findById, transition } from './db/ledger'; @@ -21,12 +20,19 @@ import { KeyedQueue } from './keyed-queue'; import { ARCHIVE_DEFAULT_SECONDS } from './policy'; import { reconcile } from './reconciler'; import { scanOnce } from './scanner'; +import { + configureNode, + registerTestTemplate, + TEST_S3, + type TestConfig, +} from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); const TOKEN = 'test-token-test-token-test-token'; function testApp( executor: FakeExecutor = new FakeExecutor(), + configured: TestConfig = {}, env: Record = {}, ) { const db = openDb(':memory:'); @@ -39,6 +45,11 @@ function testApp( DORMICE_API_TOKEN: TOKEN, ...env, }); + // The configuration copy a check-in would have applied: the node reads + // every knob from it, so a test that wants a domain or a store + // configures the node the way the gateway would. `env` is for the + // node's own identity (its data dir, its base image), nothing else. + configureNode(db, configured); const locks = new KeyedQueue(); const app = buildApp({ config, db, executor, locks, logger: false }); return { app, db, executor, locks }; @@ -1134,9 +1145,12 @@ describe('POST /updateMetadata', () => { describe('POST /updateTemplate', () => { it('re-homes the sandbox and does not refresh the idle clock', async () => { - const { app } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py-a', image: 'img-a' }); - await rpc(app, '/registerTemplate', { name: 'py-b', image: 'img-b' }); + const { app } = testApp(new FakeExecutor(), { + templates: [ + { name: 'py-a', image: 'img-a' }, + { name: 'py-b', image: 'img-b' }, + ], + }); const created = ( await acquire(app, { name: 'alice', template: 'py-a' }) ).json(); @@ -1152,9 +1166,12 @@ describe('POST /updateTemplate', () => { }); it('a frozen sandbox stays frozen; the next wake swaps the shell onto the new template, data intact', async () => { - const { app, db, executor, locks } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py-a', image: 'img-a' }); - await rpc(app, '/registerTemplate', { name: 'py-b', image: 'img-b' }); + const { app, db, executor, locks } = testApp(new FakeExecutor(), { + templates: [ + { name: 'py-a', image: 'img-a' }, + { name: 'py-b', image: 'img-b' }, + ], + }); const created = ( await acquire(app, { name: 'alice', template: 'py-a' }) ).json().sandbox; @@ -1201,8 +1218,9 @@ describe('POST /updateTemplate', () => { }); it('null detaches back to the base image', async () => { - const { app } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py-a', image: 'img-a' }); + const { app } = testApp(new FakeExecutor(), { + templates: [{ name: 'py-a', image: 'img-a' }], + }); await acquire(app, { name: 'alice', template: 'py-a' }); const res = await rpc(app, '/updateTemplate', { @@ -1211,11 +1229,11 @@ describe('POST /updateTemplate', () => { }); expect(res.statusCode).toBe(200); expect(res.json().sandbox.template).toBeNull(); - // With no rows referencing it, the old template can now be removed — - // the migration story this verb exists for. - expect( - (await rpc(app, '/removeTemplate', { name: 'py-a' })).json(), - ).toEqual({ removed: true }); + // With no rows referencing it, the gateway's removal guard — which + // asks this node — finds nobody: the migration story this verb exists for. + expect((await rpc(app, '/templateUsers', { name: 'py-a' })).json()).toEqual( + { sandboxNames: [] }, + ); }); it('rejects an unknown template with 400 and an unknown key with 404', async () => { @@ -1238,8 +1256,9 @@ describe('POST /updateTemplate', () => { }); it('treats a same-template update as the goal state', async () => { - const { app } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py-a', image: 'img-a' }); + const { app } = testApp(new FakeExecutor(), { + templates: [{ name: 'py-a', image: 'img-a' }], + }); await acquire(app, { name: 'alice', template: 'py-a' }); const res = await rpc(app, '/updateTemplate', { name: 'alice', @@ -1294,10 +1313,10 @@ describe('POST /destroySandbox', () => { describe('the archiver through the app', () => { /** - * testApp plus a MemStore-backed archiver — the S3-configured daemon. - * The env S3 seed is what flips the ledger's live adjudication - * (archiveEnabled); the MemStore stands in for the S3 those settings - * describe, so the routes' answer and the archiver's plumbing agree. + * testApp plus a MemStore-backed archiver — the S3-configured node. The + * copy's S3 store is what flips the live adjudication (archiveEnabled); + * the MemStore stands in for the S3 those settings describe, so the + * routes' answer and the archiver's plumbing agree. */ function archiverTestApp(executor: FakeExecutor = new FakeExecutor()) { const db = openDb(':memory:'); @@ -1306,11 +1325,8 @@ describe('the archiver through the app', () => { DORMICE_DB_PATH: ':memory:', DORMICE_NODE_ID: 'node-test', DORMICE_API_TOKEN: TOKEN, - DORMICE_S3_ENDPOINT: 'http://127.0.0.1:9000', - DORMICE_S3_BUCKET: 'exam', - DORMICE_S3_ACCESS_KEY_ID: 'exam-key', - DORMICE_S3_SECRET_ACCESS_KEY: 'exam-secret', }); + configureNode(db, { s3: TEST_S3 }); const locks = new KeyedQueue(); const store = new MemStore(); const archiver = new Archiver({ @@ -1518,73 +1534,11 @@ describe('the archiver through the app', () => { }); }); -describe('templates', () => { - it('registers, lists and requires auth like every native verb', async () => { - const { app } = testApp(); - const anon = await app.inject({ - method: 'POST', - url: '/registerTemplate', - payload: { name: 'py', image: 'img-a' }, - }); - expect(anon.statusCode).toBe(401); - - const res = await rpc(app, '/registerTemplate', { - name: 'py', - image: 'img-a', - }); - expect(res.statusCode).toBe(200); - expect(res.json().template).toMatchObject({ name: 'py', image: 'img-a' }); - - const listed = await rpc(app, '/listTemplates'); - expect(listed.json().templates).toMatchObject([ - { name: 'py', image: 'img-a' }, - ]); - }); - - it('re-registering re-points the name and keeps its birth date — the upgrade verb', async () => { - const { app } = testApp(); - const first = ( - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-a' }) - ).json().template; - // Born un-upgraded: the upgrade timestamp starts at the birth date. - expect(first.updatedAt).toBe(first.createdAt); - // Millisecond timestamps need real time to pass to tell apart. - await new Promise((resolve) => setTimeout(resolve, 5)); - // Same image again: idempotent, and updatedAt must not claim an upgrade. - const same = ( - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-a' }) - ).json().template; - expect(same.updatedAt).toBe(first.updatedAt); - const second = ( - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-b' }) - ).json().template; - expect(second.image).toBe('img-b'); - expect(second.createdAt).toBe(first.createdAt); - // A real image change stamps the upgrade time. - expect(Date.parse(second.updatedAt)).toBeGreaterThan( - Date.parse(first.updatedAt), - ); - expect((await rpc(app, '/listTemplates')).json().templates).toHaveLength(1); - }); - - it("rejects a malformed name, and 'base' as reserved", async () => { - const { app } = testApp(); - const bad = await rpc(app, '/registerTemplate', { - name: '-bad', - image: 'img', - }); - expect(bad.statusCode).toBe(400); - const base = await rpc(app, '/registerTemplate', { - name: 'base', - image: 'img', - }); - expect(base.statusCode).toBe(400); - expect(base.json().message).toMatch(/'base' is reserved/); - }); - +describe('templates on the node: the copy at work', () => { it('acquire with a template creates the sandbox from its image and records the name', async () => { - const { app, executor } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-a' }); + const { app, executor } = testApp(new FakeExecutor(), { + templates: [{ name: 'py', image: 'img-a' }], + }); const res = await acquire(app, { name: 'alice', template: 'py' }); expect(res.statusCode).toBe(200); const sandbox = res.json().sandbox; @@ -1611,8 +1565,9 @@ describe('templates', () => { }); it('a valid template on an existing key is not applied — creation-time only', async () => { - const { app } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-a' }); + const { app } = testApp(new FakeExecutor(), { + templates: [{ name: 'py', image: 'img-a' }], + }); const created = (await acquire(app, { name: 'alice' })).json().sandbox; expect(created.template).toBeNull(); const again = (await acquire(app, { name: 'alice', template: 'py' })).json() @@ -1621,39 +1576,56 @@ describe('templates', () => { expect(again.template).toBeNull(); }); - it('refuses to remove a template while sandboxes use it, naming the keys', async () => { - const { app } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-a' }); + it("templateUsers names the sandboxes still on a template — the gateway's removal guard asks this", async () => { + const { app } = testApp(new FakeExecutor(), { + templates: [{ name: 'py', image: 'img-a' }], + }); await acquire(app, { name: 'alice', template: 'py' }); - - const refused = await rpc(app, '/removeTemplate', { name: 'py' }); - expect(refused.statusCode).toBe(409); - expect(refused.json().message).toBe( - "template 'py' is used by 1 sandbox(es): alice — destroy them first", - ); - + await acquire(app, { name: 'bob', template: 'py' }); + await acquire(app, { name: 'carol' }); + + const users = await rpc(app, '/templateUsers', { name: 'py' }); + expect(users.statusCode).toBe(200); + expect(users.json().sandboxNames.sort()).toEqual(['alice', 'bob']); + // A name nobody uses, and a name that is no template at all: both an + // honest empty list — the question is about this ledger's rows. await rpc(app, '/destroySandbox', { name: 'alice' }); - expect((await rpc(app, '/removeTemplate', { name: 'py' })).json()).toEqual({ - removed: true, - }); - // Idempotent on an unknown name, like destroySandbox. - expect((await rpc(app, '/removeTemplate', { name: 'py' })).json()).toEqual({ - removed: false, + await rpc(app, '/destroySandbox', { name: 'bob' }); + expect((await rpc(app, '/templateUsers', { name: 'py' })).json()).toEqual({ + sandboxNames: [], }); + expect( + (await rpc(app, '/templateUsers', { name: 'ghost' })).json(), + ).toEqual({ sandboxNames: [] }); + // Like every native verb, behind the token; and a malformed name is a 400. + expect( + ( + await app.inject({ + method: 'POST', + url: '/templateUsers', + payload: { name: 'py' }, + }) + ).statusCode, + ).toBe(401); + expect( + (await rpc(app, '/templateUsers', { name: '-bad' })).statusCode, + ).toBe(400); }); it('re-point then rebuild moves the sandbox onto the new image — the immediate front door', async () => { - const { app, executor } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v1' }); + const { app, db, executor } = testApp(new FakeExecutor(), { + templates: [{ name: 'py', image: 'img-v1' }], + }); const created = ( await acquire(app, { name: 'alice', template: 'py' }) ).json().sandbox; expect(await executor.imageOf(created.id)).toBe('img-v1'); - // Operator builds a new image and re-points the name; a running shell - // is never touched behind the sandbox's back — the stock moves on an - // explicit rebuild (here) or on the next cold wake (tests below). - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v2' }); + // Operator builds a new image and re-points the name at the gateway; + // the next bundle brings it here. A running shell is never touched + // behind the sandbox's back — the stock moves on an explicit rebuild + // (here) or on the next cold wake (tests below). + registerTestTemplate(db, 'py', 'img-v2'); expect(await executor.imageOf(created.id)).toBe('img-v1'); await rpc(app, '/rebuildSandbox', { name: 'alice' }); @@ -1666,8 +1638,9 @@ describe('templates', () => { describe('cold wakes converge onto the current image', () => { it('frozen + stale: the wake swaps the shell, keeps the data, and records the swap', async () => { - const { app, db, executor, locks } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v1' }); + const { app, db, executor, locks } = testApp(new FakeExecutor(), { + templates: [{ name: 'py', image: 'img-v1' }], + }); const created = ( await acquire(app, { name: 'alice', template: 'py' }) ).json().sandbox; @@ -1692,7 +1665,7 @@ describe('cold wakes converge onto the current image', () => { expect(sweep.failures).toEqual([]); expect(executor.stateOf(created.id)).toBe('paused'); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v2' }); + registerTestTemplate(db, 'py', 'img-v2'); const woken = (await acquire(app, { name: 'alice' })).json().sandbox; expect(woken.id).toBe(created.id); expect(woken.state).toBe('active'); @@ -1710,8 +1683,9 @@ describe('cold wakes converge onto the current image', () => { }); it('frozen + fresh: a plain unpause, no shell removed', async () => { - const { app, db, executor, locks } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v1' }); + const { app, db, executor, locks } = testApp(new FakeExecutor(), { + templates: [{ name: 'py', image: 'img-v1' }], + }); const created = ( await acquire(app, { name: 'alice', template: 'py' }) ).json().sandbox; @@ -1729,8 +1703,9 @@ describe('cold wakes converge onto the current image', () => { }); it('stopped + stale: the same convergence — stop kept the old shell, the wake replaces it', async () => { - const { app, db, executor, locks } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v1' }); + const { app, db, executor, locks } = testApp(new FakeExecutor(), { + templates: [{ name: 'py', image: 'img-v1' }], + }); const created = ( await acquire(app, { name: 'alice', @@ -1744,7 +1719,7 @@ describe('cold wakes converge onto the current image', () => { expect(executor.stateOf(created.id)).toBe('stopped'); expect(await executor.imageOf(created.id)).toBe('img-v1'); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v2' }); + registerTestTemplate(db, 'py', 'img-v2'); const woken = (await acquire(app, { name: 'alice' })).json().sandbox; expect(woken.state).toBe('active'); expect(await executor.imageOf(created.id)).toBe('img-v2'); @@ -1768,8 +1743,9 @@ describe('cold wakes converge onto the current image', () => { }); it('a vanished shell is not judged stale — the start builds from the current image by itself', async () => { - const { app, db, executor, locks } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v1' }); + const { app, db, executor, locks } = testApp(new FakeExecutor(), { + templates: [{ name: 'py', image: 'img-v1' }], + }); const created = ( await acquire(app, { name: 'alice', @@ -1781,7 +1757,7 @@ describe('cold wakes converge onto the current image', () => { await scanOnce(db, executor, locks, after(created.lastActiveAt, 120)); executor.vanishContainer(created.id); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v2' }); + registerTestTemplate(db, 'py', 'img-v2'); const woken = (await acquire(app, { name: 'alice' })).json().sandbox; expect(woken.state).toBe('active'); // Converged all the same, but through start's own rebuild — no shell @@ -1803,9 +1779,13 @@ describe('POST /getHostMetrics', () => { it('answers a schema-valid snapshot with honest host readings', async () => { // tmpdir() exists on every platform, so the data-disk reading is real. - const { app } = testApp(new FakeExecutor(), { - DORMICE_DATA_DIR: tmpdir(), - }); + const { app } = testApp( + new FakeExecutor(), + {}, + { + DORMICE_DATA_DIR: tmpdir(), + }, + ); const res = await rpc(app, '/getHostMetrics'); expect(res.statusCode).toBe(200); const body = hostMetricsResponseSchema.parse(res.json()); @@ -1826,9 +1806,13 @@ describe('POST /getHostMetrics', () => { }); it('reports a missing data dir as null — absent, not invented', async () => { - const { app } = testApp(new FakeExecutor(), { - DORMICE_DATA_DIR: '/no/such/dormice-data', - }); + const { app } = testApp( + new FakeExecutor(), + {}, + { + DORMICE_DATA_DIR: '/no/such/dormice-data', + }, + ); const res = await rpc(app, '/getHostMetrics'); expect(res.statusCode).toBe(200); expect(res.json().dataDisk).toBeNull(); @@ -1867,312 +1851,6 @@ describe('POST /getHostMetrics', () => { }); }); -describe('API keys', () => { - /** Mint through the wire and hand back everything a test needs. */ - async function mint( - app: ReturnType['app'], - name: string, - expiresAt?: string, - ) { - const res = await rpc(app, '/createApiKey', { - name, - ...(expiresAt ? { expiresAt } : {}), - }); - expect(res.statusCode).toBe(200); - const body = res.json(); - return { - id: body.apiKey.id as string, - token: body.token as string, - apiKey: body.apiKey, - }; - } - - const useKey = ( - app: ReturnType['app'], - token: string, - url = '/listSandboxes', - ) => - app.inject({ - method: 'POST', - url, - headers: { authorization: `Bearer ${token}` }, - payload: {}, - }); - - it('mints a 64-hex token, shown once and never stored in the view', async () => { - const { app } = testApp(); - const res = await rpc(app, '/createApiKey', { name: 'ci' }); - expect(res.statusCode).toBe(200); - const body = res.json(); - expect(body.token).toMatch(/^[0-9a-f]{64}$/); - expect(body.apiKey).toMatchObject({ - name: 'ci', - prefix: body.token.slice(0, 8), - lastUsedAt: null, - expiresAt: null, - disabledAt: null, - revokedAt: null, - }); - // The view carries no secret — not the token, not its hash. - expect(JSON.stringify(body.apiKey)).not.toContain(body.token); - expect(Object.keys(body.apiKey)).not.toContain('keyHash'); - }); - - it('a minted key opens the Bearer door; revoking closes it on the next request', async () => { - const { app } = testApp(); - const { id, token } = await mint(app, 'ci'); - - expect((await useKey(app, token)).statusCode).toBe(200); - - expect((await rpc(app, '/revokeApiKey', { id })).json()).toEqual({ - revoked: true, - }); - expect((await useKey(app, token)).statusCode).toBe(401); - - // The env token is the bootstrap credential: revocation never touches it. - expect((await rpc(app, '/listSandboxes')).statusCode).toBe(200); - }); - - it('refuses a second active key under the same name with a 409, and frees the name after revoke', async () => { - const { app } = testApp(); - const { id } = await mint(app, 'ci'); - const dup = await rpc(app, '/createApiKey', { name: 'ci' }); - expect(dup.statusCode).toBe(409); - expect(dup.json().message).toMatch(/'ci' already exists/); - - await rpc(app, '/revokeApiKey', { id }); - expect((await rpc(app, '/createApiKey', { name: 'ci' })).statusCode).toBe( - 200, - ); - }); - - it('revoke is idempotent: an unknown or already-revoked id answers { revoked: false }', async () => { - const { app } = testApp(); - expect((await rpc(app, '/revokeApiKey', { id: 'ghost' })).json()).toEqual({ - revoked: false, - }); - const { id } = await mint(app, 'ci'); - await rpc(app, '/revokeApiKey', { id }); - expect((await rpc(app, '/revokeApiKey', { id })).json()).toEqual({ - revoked: false, - }); - }); - - it('lists every key ever minted, revoked rows included, newest first', async () => { - const { app } = testApp(); - const { id } = await mint(app, 'old'); - await rpc(app, '/revokeApiKey', { id }); - await mint(app, 'new'); - - const keys = (await rpc(app, '/listApiKeys')).json().apiKeys; - expect(keys).toHaveLength(2); - expect(keys[0].name).toBe('new'); - expect(keys[0].revokedAt).toBeNull(); - expect(keys[1].name).toBe('old'); - expect(keys[1].revokedAt).not.toBeNull(); - }); - - it('stamps lastUsedAt on first use and throttles the write to 60s granularity', async () => { - const { app } = testApp(); - const { token } = await mint(app, 'ci'); - - await useKey(app, token); - const first = (await rpc(app, '/listApiKeys')).json().apiKeys[0]; - expect(first.lastUsedAt).not.toBeNull(); - - // A second use inside the 60s window must not move the stamp. - await useKey(app, token); - const second = (await rpc(app, '/listApiKeys')).json().apiKeys[0]; - expect(second.lastUsedAt).toBe(first.lastUsedAt); - }); - - it('a console session passes the admin gate too: a key minted from the console', async () => { - const { app } = testApp(); - const setup = await app.inject({ - method: 'POST', - url: '/console/auth/setup', - payload: { token: TOKEN, username: 'operator', password: 'horse pass' }, - }); - const cookie = setup.cookies.find((c) => c.name === SESSION_COOKIE); - const minted = await app.inject({ - method: 'POST', - url: '/createApiKey', - headers: { [CONSOLE_HEADER]: '1' }, - cookies: { [SESSION_COOKIE]: (cookie as { value: string }).value }, - payload: { name: 'by-console' }, - }); - expect(minted.statusCode).toBe(200); - expect(minted.json().apiKey.name).toBe('by-console'); - expect((await useKey(app, minted.json().token)).statusCode).toBe(200); - }); - - it('disable parks the key reversibly: 401 while disabled, 200 again after enable', async () => { - const { app } = testApp(); - const { id, token } = await mint(app, 'ci'); - expect((await useKey(app, token)).statusCode).toBe(200); - - const disabled = ( - await rpc(app, '/updateApiKey', { id, disabled: true }) - ).json().apiKey; - expect(disabled.disabledAt).not.toBeNull(); - expect((await useKey(app, token)).statusCode).toBe(401); - - // Disabling twice is idempotent: the original stamp stays, no new event. - const again = ( - await rpc(app, '/updateApiKey', { id, disabled: true }) - ).json().apiKey; - expect(again.disabledAt).toBe(disabled.disabledAt); - - const enabled = ( - await rpc(app, '/updateApiKey', { id, disabled: false }) - ).json().apiKey; - expect(enabled.disabledAt).toBeNull(); - expect((await useKey(app, token)).statusCode).toBe(200); - }); - - it('expiry closes the door: a past expiresAt is 401, clearing it reopens', async () => { - const { app } = testApp(); - const past = new Date(Date.now() - 1000).toISOString(); - const { id, token } = await mint(app, 'ttl', past); - expect((await useKey(app, token)).statusCode).toBe(401); - - const cleared = ( - await rpc(app, '/updateApiKey', { id, expiresAt: null }) - ).json().apiKey; - expect(cleared.expiresAt).toBeNull(); - expect((await useKey(app, token)).statusCode).toBe(200); - - const future = new Date(Date.now() + 3600_000).toISOString(); - await rpc(app, '/updateApiKey', { id, expiresAt: future }); - expect((await useKey(app, token)).statusCode).toBe(200); - }); - - it('normalizes expiresAt on write: wire precision variants land as toISOString()', async () => { - const { app } = testApp(); - // No-millis wire form would sort AFTER a with-millis "now" while being - // chronologically earlier — the ledger must store the canonical shape. - const { apiKey } = await mint(app, 'ttl', '2030-01-01T00:00:00Z'); - expect(apiKey.expiresAt).toBe('2030-01-01T00:00:00.000Z'); - }); - - it('updateApiKey renames, refuses collisions honestly, and leaves history alone', async () => { - const { app } = testApp(); - const { id } = await mint(app, 'ci'); - const other = await mint(app, 'laptop'); - - const renamed = ( - await rpc(app, '/updateApiKey', { id, name: 'ci-2026' }) - ).json().apiKey; - expect(renamed.name).toBe('ci-2026'); - - // Onto a live name: refused like create. - const clash = await rpc(app, '/updateApiKey', { id, name: 'laptop' }); - expect(clash.statusCode).toBe(409); - - // Onto a revoked name: revoke freed it. - await rpc(app, '/revokeApiKey', { id: other.id }); - expect( - (await rpc(app, '/updateApiKey', { id, name: 'laptop' })).statusCode, - ).toBe(200); - - // Unknown id is a 404; a revoked row is history, not editable. - expect( - (await rpc(app, '/updateApiKey', { id: 'ghost', name: 'x' })).statusCode, - ).toBe(404); - const edited = await rpc(app, '/updateApiKey', { - id: other.id, - name: 'zombie', - }); - expect(edited.statusCode).toBe(409); - expect(edited.json().message).toMatch(/rotation history/); - }); - - it('a no-op patch changes nothing', async () => { - const { app } = testApp(); - const { id } = await mint(app, 'ci'); - const before = (await rpc(app, '/listApiKeys')).json().apiKeys; - - const res = await rpc(app, '/updateApiKey', { - id, - name: 'ci', - disabled: false, - }); - expect(res.statusCode).toBe(200); - expect(res.json().apiKey.name).toBe('ci'); - expect((await rpc(app, '/listApiKeys')).json().apiKeys).toEqual(before); - }); - - it('carries expiresAt from mint into the list', async () => { - const { app } = testApp(); - const future = new Date(Date.now() + 86_400_000).toISOString(); - await mint(app, 'ttl', future); - const keys = (await rpc(app, '/listApiKeys')).json().apiKeys; - expect(keys[0].expiresAt).toBe(future); - }); - - it('admin-only: a live key gets an honest 403 on every management verb, without a lastUsedAt fingerprint', async () => { - const { app } = testApp(); - const { id, token } = await mint(app, 'ci'); - const asKey = { authorization: `Bearer ${token}` }; - - const attempts = [ - ['/createApiKey', { name: 'evil' }], - ['/listApiKeys', {}], - ['/updateApiKey', { id, disabled: true }], - ['/revokeApiKey', { id }], - ] as const; - for (const [url, payload] of attempts) { - const res = await app.inject({ - method: 'POST', - url, - headers: asKey, - payload, - }); - expect(res.statusCode).toBe(403); - expect(res.json().message).toMatch(/cannot manage API keys/); - } - - // The refusals honored nothing: no lastUsedAt fingerprint, key untouched. - const row = (await rpc(app, '/listApiKeys')).json().apiKeys[0]; - expect(row.lastUsedAt).toBeNull(); - expect(row.disabledAt).toBeNull(); - expect(row.revokedAt).toBeNull(); - - // Garbage stays garbage: 401, not 403. - expect( - ( - await app.inject({ - method: 'POST', - url: '/createApiKey', - headers: { authorization: 'Bearer not-a-key' }, - payload: { name: 'x' }, - }) - ).statusCode, - ).toBe(401); - }); - - it('admin-only: a console session opens the management verbs', async () => { - const { app } = testApp(); - const setup = await app.inject({ - method: 'POST', - url: '/console/auth/setup', - payload: { token: TOKEN, username: 'operator', password: 'horse pass' }, - }); - expect(setup.statusCode).toBe(200); - const cookie = setup.cookies.find((c) => c.name === SESSION_COOKIE); - expect(cookie).toBeDefined(); - - const res = await app.inject({ - method: 'POST', - url: '/createApiKey', - headers: { [CONSOLE_HEADER]: '1' }, - cookies: { [SESSION_COOKIE]: (cookie as { value: string }).value }, - payload: { name: 'from-console' }, - }); - expect(res.statusCode).toBe(200); - }); -}); - describe('POST /lookupSandbox', () => { it('answers by name and by id with the state, without waking or touching the idle clock', async () => { const { app, db } = testApp(); diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 01bb3621..498e6be4 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -1,6 +1,5 @@ import http from 'node:http'; import nodePath from 'node:path'; -import fastifyCookie from '@fastify/cookie'; import fastify, { type FastifyError, type FastifyServerFactory } from 'fastify'; import { serializerCompiler, @@ -10,30 +9,19 @@ import { import { type Logger, pino } from 'pino'; import { z } from 'zod'; import type { Archiver } from './archive/archiver'; -import type { S3Settings } from './archive/s3-store'; -import { requireAdminAuth, requireApiAuth, tokensEqual } from './auth'; -import { type Config, type ConfigSources, configSources } from './config'; -import { getConsoleAccount } from './db/account'; -import { isLiveApiKey, verifyApiKeyToken } from './db/api-keys'; +import { requireApiAuth, tokensEqual } from './auth'; +import type { Config } from './config'; import type { Db } from './db/db'; import { getOrCreateSigningSecret } from './db/secrets'; -import { ensureRuntimeSettings } from './db/settings'; import { registerE2bCompat } from './e2b'; import { ProcessTable } from './e2b/process-table'; import { WatcherTable } from './e2b/watcher-table'; import type { Executor } from './executor/executor'; -import type { Ingress } from './ingress'; import type { KeyedQueue } from './keyed-queue'; -import { apiKeyRoutes } from './routes/api-keys'; -import { configRoutes } from './routes/config'; -import { consoleRoutes } from './routes/console'; import { envdTokenRoutes } from './routes/envd-token'; import { hostRoutes } from './routes/host'; -import { ingressRoutes } from './routes/ingress'; import { sandboxRoutes } from './routes/sandboxes'; -import { settingsRoutes } from './routes/settings'; import { templateUsersRoutes } from './routes/template-users'; -import { templateRoutes } from './routes/templates'; import { upgradeRoutes } from './routes/upgrade'; import { createSandboxProxy } from './sandbox-proxy'; import { Updater } from './updater'; @@ -41,6 +29,7 @@ import { readBuildInfo } from './version'; export interface AppDeps { config: Config; + /** The ledger, holding a configuration copy (db/settings.ts applyNodeConfig) — main.ts waits for one before building the app; tests apply one. */ db: Db; executor: Executor; /** @@ -57,39 +46,15 @@ export interface AppDeps { logger?: boolean | Logger; /** Tests may inspect the one daemon-wide watcher registry. */ watchers?: WatcherTable; - /** - * Where the built web console lives; main.ts resolves the monorepo - * layout, tests inject a fixture. Absent means /console answers an - * honest 404. - */ - consoleDistDir?: string; /** * The archive/restore engine. Whether archiving is AVAILABLE is not its - * presence but its enabled() — a live read of the ledger's S3 settings, - * so the console can switch archiving on and off without a restart. + * presence but its enabled() — a live read of the copy's S3 settings, + * so a bundle that turns archiving on applies without a restart. * Optional purely as a test convenience: many app tests exercise no * archive path, and to them an absent archiver equals a disabled one * (both make archiveEnabled(db) the sole adjudicator refuse). */ archiver?: Archiver; - /** - * The managed reverse-proxy front door, present exactly when - * DORMICE_INGRESS_FILE is set (same rule as the archiver). Absent, - * getIngress answers { managed: false } and setIngress refuses. - */ - ingress?: Ingress; - /** - * Test seam over updateSettings' S3 round-trip probe (routes/settings.ts) - * — a unit test forges S3's answers instead of needing a live store. - * Production omits it and probes for real. - */ - probeS3?: (s3: S3Settings) => Promise; - /** - * Which knobs came from the environment versus defaults, for getConfig. - * Defaults to reading process.env — right for the daemon; tests that - * assert on sources inject a fixed map instead of trusting the shell. - */ - sources?: ConfigSources; /** * The daemon's own upgrade window. main.ts injects one that knows the * checkout the daemon runs from; the default knows no checkout, so @@ -106,6 +71,14 @@ export interface AppDeps { * exports), so request validation, TypeScript types and — later — OpenAPI * docs all derive from a single definition. * + * The node's face (design record #22, 2026-09-14): the sandbox verbs, the + * host's observation verbs, its own upgrade, and the two read-only + * questions its gateway asks on its own account (lookupSandbox, + * templateUsers) — behind one credential, the fleet token. Everything + * that configures the fleet (settings, templates, API keys, domains, the + * console and its sessions) is the gateway's; a key a caller presents is + * judged there, and toward this node the gateway speaks the fleet token. + * * Building the app is separate from listening so tests can inject requests * without opening a port. */ @@ -116,11 +89,7 @@ export function buildApp({ locks, logger = true, watchers = new WatcherTable(), - consoleDistDir, archiver, - ingress, - probeS3, - sources = configSources(), updater = new Updater({ repoDir: null, build: readBuildInfo(), @@ -128,12 +97,6 @@ export function buildApp({ executor: config.DORMICE_EXECUTOR, }), }: AppDeps) { - // Idempotent get-or-seed: main.ts already ran it (the executor reads - // settings before buildApp), tests build the app directly and need it here. - // "Is archiving available" is no longer adjudicated here — it lives in the - // ledger settings and every consumer reads it live (db/settings.ts - // archiveEnabled), so a console edit applies without a restart. - ensureRuntimeSettings(db, config); // Always a pino instance (booleans are normalized into one): two fastify() // call shapes would give the instance two different types. const loggerInstance = @@ -145,8 +108,8 @@ export function buildApp({ // The sandbox port proxy sits in front of routing — it triages by Host // header, so it must see the request before Fastify's router 404s a // wildcard host's arbitrary path. Mounted unconditionally: the domain is - // a live ledger setting now, so the proxy must already be in the path - // when the operator sets one — with no domain in force, matches() is + // a live setting of the copy, so the proxy must already be in the path + // when a bundle sets one — with no domain in force, matches() is // constantly false and the upgrade hook destroys non-matching sockets // exactly as stock Fastify (which never handles upgrades) would. // app.inject() bypasses the factory, so the proxy is exercised over real @@ -218,40 +181,18 @@ export function buildApp({ async () => ({ status: 'ok' as const }), ); - // Cookie parsing app-wide: the auth arbiter reads the console's session - // cookie on the native routes, the /console surface mints and clears it. - app.register(fastifyCookie); - // The one adjudication of "does this bare credential open the door": - // the env token (constant-time compare — the bootstrap credential, - // always valid) or any active ledger API key (sha256 indexed lookup, - // judged per request so a mint or revoke takes effect on the very next - // call). Both faces — the native Bearer header and the E2B X-API-KEY - // hook — feed this same closure: one truth, two dialects. + // the fleet token, constant-time compared — the only credential a node + // knows. Minted API keys are the gateway's to judge; it forwards under + // this token. Both faces — the native Bearer header and the E2B + // X-API-KEY hook — feed this same closure: one truth, two dialects. No + // session leg: the console lives at the gateway, so no cookie is ever + // valid here. const isCredential = (bare: string): boolean => - tokensEqual(bare, config.DORMICE_API_TOKEN) || - verifyApiKeyToken(db, bare) !== null; + tokensEqual(bare, config.DORMICE_API_TOKEN); + const apiAuth = requireApiAuth(isCredential, () => null); - // Built once, used by every guarded surface. The secret getter reads the - // ledger per request because setup can replace the account (and void its - // sessions) while the daemon runs — a captured value would keep dead - // sessions alive until restart. - const apiAuth = requireApiAuth( - isCredential, - () => getConsoleAccount(db)?.sessionSecret ?? null, - ); - - // The apiKey verbs are admin-only: a credential must not be able to - // manage the credential ledger it lives in (key-manages-key is a - // self-replication ladder for a leaked key). Env token or console - // session only; a live ledger key gets an honest 403, not a silent 401. - const adminAuth = requireAdminAuth( - (bare) => tokensEqual(bare, config.DORMICE_API_TOKEN), - (bare) => isLiveApiKey(db, bare), - () => getConsoleAccount(db)?.sessionSecret ?? null, - ); - - // The envd/signed-URL derivation base. Captured once — unlike the session + // The envd/signed-URL derivation base. Captured once — unlike a session // secret there is no verb that rotates it (see db/secrets.ts) — and NOT // the API token: the two credentials must rotate independently. const envdSigningSecret = getOrCreateSigningSecret(db); @@ -266,42 +207,12 @@ export function buildApp({ watchers, archiver, }); - await api.register(templateRoutes, { db }); await api.register(templateUsersRoutes, { db }); await api.register(hostRoutes, { config, db, executor }); - await api.register(ingressRoutes, { ingress }); - await api.register(configRoutes, { config, db, sources }); await api.register(upgradeRoutes, { updater }); await api.register(envdTokenRoutes, { envdSigningSecret }); }); - // The apiKey management verbs and updateSettings sit behind the stricter - // admin gate — their own scope, because a Fastify hook guards a whole - // scope and these verbs share a different answer to "who may call": - // credentials must not manage credentials, and a leaked automation key - // must not be able to raise the very limits that contain it. - app.register(async (admin) => { - admin.addHook('onRequest', adminAuth); - await admin.register(apiKeyRoutes, { db }); - await admin.register(settingsRoutes, { - db, - executor, - locks, - ...(probeS3 ? { probeS3 } : {}), - }); - }); - - // The web console: account + session endpoints (open — setup and login - // carry the credentials themselves) and the static SPA. Its API calls go - // through the routes above. - app.register(async (scope) => { - await scope.register(consoleRoutes, { - config, - db, - consoleDistDir, - }); - }); - // The E2B compatibility surface lives beside the native API with its own // auth (X-API-KEY / X-Access-Token) and its own error dialect. app.register(async (compat) => { diff --git a/packages/server/src/archive/probe.ts b/packages/server/src/archive/probe.ts deleted file mode 100644 index a8f84b43..00000000 --- a/packages/server/src/archive/probe.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; -import type { S3Settings } from './s3-store'; -import { S3Store } from './s3-store'; - -/** Thrown by probeS3 with the S3 error's own words and its HTTP status (when S3 answered at all). */ -export class S3ProbeError extends Error { - constructor( - message: string, - readonly httpStatusCode: number | undefined, - ) { - super(message); - } -} - -/** - * A put+get+delete round trip against the candidate store, run BEFORE the - * settings are written — a probe failure must leave the ledger untouched. - * Deliberately the opposite of swap's save-then-reconcile: a swap target - * that failed to apply is still the right target (capacity problems heal), - * but S3 credentials are static facts — wrong ones saved would just turn - * every later scan tick into error noise. Runs through S3Store itself - * (real file streams, the same WHEN_REQUIRED checksum posture), so what - * the probe proves is exactly what archiving will do. - */ -export async function probeS3(s3: S3Settings): Promise { - const store = new S3Store(s3); - const dir = await mkdtemp(path.join(tmpdir(), 'dormice-s3-probe-')); - const key = `dormice-probe-${randomUUID()}`; - const body = 'dormice archive-store probe'; - try { - const up = path.join(dir, 'up'); - const down = path.join(dir, 'down'); - await writeFile(up, body); - try { - await store.put(key, up); - await store.get(key, down); - } catch (error) { - throw toProbeError(error); - } - if ((await readFile(down, 'utf8')) !== body) { - throw new S3ProbeError( - 'the probe object came back with different content — the store is not a faithful S3', - undefined, - ); - } - } finally { - await store.delete(key).catch(() => {}); - await rm(dir, { recursive: true, force: true }); - } -} - -function toProbeError(error: unknown): S3ProbeError { - if (error instanceof Error) { - const status = (error as { $metadata?: { httpStatusCode?: number } }) - .$metadata?.httpStatusCode; - return new S3ProbeError(`${error.name}: ${error.message}`, status); - } - return new S3ProbeError(String(error), undefined); -} diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index fc00c959..e28b7610 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -1,12 +1,14 @@ import http from 'node:http'; import type { AddressInfo } from 'node:net'; import { fileURLToPath } from 'node:url'; -import { checkInRequestSchema } from '@dormice/shared'; +import { checkInRequestSchema, type NodeConfigBundle } from '@dormice/shared'; import { afterEach, describe, expect, it } from 'vitest'; import { CheckIn, type CheckInOptions, readNodeReading } from './check-in'; import { migrateDb, openDb } from './db/db'; import { createSandbox } from './db/ledger'; +import { applyNodeConfig, readConfigVersion } from './db/settings'; import { CpuSampler } from './host-metrics'; +import { testBundle } from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); const TOKEN = 'shared-token-shared-token-shared-token'; @@ -70,15 +72,27 @@ function logSpy() { }; } +/** A gateway answer: the current version, and the bundle when the node's differs. */ +function answering(version: number, bundle?: NodeConfigBundle) { + return { + status: 200, + body: JSON.stringify({ + configVersion: version, + ...(bundle === undefined ? {} : { config: bundle }), + }), + }; +} + function options( gatewayEndpoint: string, log: CheckInOptions['log'], over: Partial = {}, -): CheckInOptions { +): CheckInOptions & { db: ReturnType } { const db = openDb(':memory:'); migrateDb(db, MIGRATIONS); const cpu = new CpuSampler(); return { + db, gateway: gatewayEndpoint, token: TOKEN, nodeId: 'node-7', @@ -90,6 +104,10 @@ function options( committedAt: '2026-09-14T00:00:00.000Z', }, readReading: () => readNodeReading(db, cpu, '/nonexistent-data-dir'), + // The daemon's wiring in miniature: the copy is the ledger's, applied + // by the pure write (node-config.ts's hooks are its own suite). + configVersion: () => readConfigVersion(db), + applyConfig: async (bundle) => applyNodeConfig(db, bundle), log, ...over, }; @@ -97,7 +115,7 @@ function options( describe('CheckIn', () => { it('posts a check-in the gateway can parse: shared token, id, endpoint, interval, build, reading', async () => { - const gw = await gateway(() => ({ status: 200, body: '{}' })); + const gw = await gateway(() => answering(1)); const { log, warns } = logSpy(); await new CheckIn(options(gw.endpoint, log)).once(); expect(warns).toEqual([]); @@ -117,6 +135,19 @@ describe('CheckIn', () => { total: 0, byState: { active: 0, frozen: 0, stopped: 0, archived: 0, restoring: 0 }, }); + // No swap manager here: honestly "cannot manage swap", and no copy yet. + expect(body.reading.managedSwap).toBeNull(); + expect(body.configVersion).toBeNull(); + }); + + it('the reading carries the managed swap when the daemon has one', async () => { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const reading = await readNodeReading(db, new CpuSampler(), '/tmp', { + status: async () => ({ activeGb: 16, blocks: [] }), + reconcile: async () => ({ activeGb: 16, blocks: [] }), + }); + expect(reading.managedSwap).toEqual({ activeGb: 16 }); }); it('the reading counts the ledger by state', async () => { @@ -145,7 +176,9 @@ describe('CheckIn', () => { it('logs a failing gateway once, and its recovery once — not every tick', async () => { let status = 500; - const gw = await gateway(() => ({ status, body: '{"message":"boom"}' })); + const gw = await gateway(() => + status === 200 ? answering(1) : { status, body: '{"message":"boom"}' }, + ); const { log, warns, infos } = logSpy(); const checkIn = new CheckIn(options(gw.endpoint, log)); await checkIn.once(); @@ -234,8 +267,75 @@ describe('CheckIn', () => { ); }); + it('a bundle in the answer is applied and the next check-in reports its version; a matching version gets no bundle', async () => { + const bundle = testBundle( + { sandboxDomain: 'sbx.example.com', pidsLimit: 512 }, + 7, + ); + let sent = 0; + const gw = await gateway(() => { + sent += 1; + return sent === 1 ? answering(7, bundle) : answering(7); + }); + const { log, warns, infos } = logSpy(); + const opts = options(gw.endpoint, log); + const checkIn = new CheckIn(opts); + await checkIn.once(); + expect(warns).toEqual([]); + expect(readConfigVersion(opts.db)).toBe(7); + await checkIn.once(); + expect(checkInRequestSchema.parse(gw.seen[1]?.body).configVersion).toBe(7); + expect(infos).toEqual([]); + }); + + it("a bundle that cannot be applied is this tick's failure, and the version stays so the gateway sends it again", async () => { + const bundle = testBundle({}, 3); + const gw = await gateway(() => answering(3, bundle)); + const { log, warns, details } = logSpy(); + const opts = options(gw.endpoint, log, { + applyConfig: async () => { + throw new Error('disk full'); + }, + }); + const checkIn = new CheckIn(opts); + await checkIn.once(); + expect(warns).toHaveLength(1); + expect((details[0] as { error: string }).error).toMatch( + /configuration v3 from the gateway could not be applied: disk full/, + ); + expect(readConfigVersion(opts.db)).toBeNull(); + // The next check-in still says "no copy" — the gateway's cue to resend. + await checkIn.once(); + expect( + checkInRequestSchema.parse(gw.seen[1]?.body).configVersion, + ).toBeNull(); + }); + + it('untilConfigured() asks until a bundle lands, on the interval, and returns at once when a copy exists', async () => { + let sent = 0; + const gw = await gateway(() => { + sent += 1; + // The gateway is down for the first two asks, then answers with the bundle. + return sent < 3 + ? { status: 503, body: '{"message":"starting"}' } + : answering(2, testBundle({}, 2)); + }); + const { log } = logSpy(); + const opts = options(gw.endpoint, log); + const checkIn = new CheckIn(opts); + const started = Date.now(); + await checkIn.untilConfigured(); + expect(readConfigVersion(opts.db)).toBe(2); + expect(gw.seen).toHaveLength(3); + // Two waits of one interval between the three asks. + expect(Date.now() - started).toBeGreaterThanOrEqual(1_900); + // Holding a copy already: nothing is asked. + await checkIn.untilConfigured(); + expect(gw.seen).toHaveLength(3); + }); + it('ticks on its interval from start() and stops on stop()', async () => { - const gw = await gateway(() => ({ status: 200, body: '{}' })); + const gw = await gateway(() => answering(1)); const { log } = logSpy(); const checkIn = new CheckIn(options(gw.endpoint, log)); checkIn.start(); diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index a208d1a0..e738dd2e 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -2,26 +2,32 @@ import { type BuildInfo, type CheckInRequest, checkInResponseSchema, + type NodeConfigBundle, type NodeReading, } from '@dormice/shared'; import type { Db } from './db/db'; import { countByState, listSandboxes } from './db/ledger'; import { type CpuSampler, readHostReading } from './host-metrics'; +import type { SwapControl } from './swap'; /** - * A node's reading for its check-in: the host half (host-metrics.ts) and - * the ledger's census. The same numbers getHostMetrics answers a caller - * with, minus the daemon-local knobs no gateway places by. + * A node's reading for its check-in: the host half (host-metrics.ts), the + * ledger's census, and what the daemon-managed swap holds — null where + * the daemon manages none (a non-Linux host, the fake executor), which is + * how the gateway knows to refuse a swap target for this node. */ export async function readNodeReading( db: Db, cpu: CpuSampler, dataDir: string, + swap?: SwapControl, ): Promise { const { byState, total } = countByState(listSandboxes(db)); return { ...(await readHostReading(cpu, dataDir)), sandboxes: { total, byState }, + managedSwap: + swap === undefined ? null : { activeGb: (await swap.status()).activeGb }, }; } @@ -41,6 +47,10 @@ export interface CheckInOptions { intervalSeconds: number; build: BuildInfo | null; readReading: () => Promise; + /** The version of the configuration copy this node runs; null while it holds none (db/settings.ts). */ + configVersion: () => number | null; + /** Makes a bundle the gateway answered with real on this node (node-config.ts applyConfig). */ + applyConfig: (bundle: NodeConfigBundle) => Promise; log: CheckInLog; /** Test seam; production uses the platform's fetch. */ fetchImpl?: typeof fetch; @@ -51,10 +61,15 @@ const CHECK_IN_TIMEOUT_MS = 10_000; /** * The node's check-in ticker: every interval, one POST /checkIn to the - * gateway carrying the node's id, where it can be reached, its build and - * a fresh reading (RULES/协议.md「网关」). The gateway learns of a node from - * its first check-in — no registration verb, no nodes file — and reads - * two missed check-ins as down. + * gateway carrying the node's id, where it can be reached, its build, a + * fresh reading and the version of the configuration copy it runs + * (RULES/协议.md「网关」). The gateway learns of a node from its first + * check-in — no registration verb, no nodes file — and reads two missed + * check-ins as down. The answer is the gateway's configuration version, + * and the whole bundle whenever the node's differs: the check-in IS the + * configuration pull (design record #22) — a fresh node, a node that + * missed an edit while the gateway was away, an operator's change a + * second ago, all one mechanism, and nothing for the gateway to remember. * * Chained setTimeout, the daemon's discipline: the next tick is scheduled * when this one is done, so a slow gateway never has ticks pile up. @@ -107,6 +122,7 @@ export class CheckIn { intervalSeconds: opts.intervalSeconds, build: opts.build, reading: await opts.readReading(), + configVersion: opts.configVersion(), }; const res = await (opts.fetchImpl ?? fetch)(`${opts.gateway}/checkIn`, { method: 'POST', @@ -139,11 +155,23 @@ export class CheckIn { : `gateway answered ${res.status} redirecting to ${location} — DORMICE_GATEWAY_ENDPOINT must be the gateway's own address, not a front that redirects`, ); } - checkInResponseSchema.parse(await res.json()); + const answer = checkInResponseSchema.parse(await res.json()); if (this.failing !== null) { opts.log.info(`check-in with gateway ${opts.gateway} answers again`); this.failing = null; } + if (answer.config !== undefined) { + // A bundle that cannot be applied is this tick's failure: the copy + // stays what it was, the next check-in reports the old version, and + // the gateway answers the bundle again — the retry is the protocol. + try { + await opts.applyConfig(answer.config); + } catch (error) { + throw new Error( + `configuration v${answer.config.version} from the gateway could not be applied: ${describe(error)}`, + ); + } + } } catch (error) { const message = describe(error); const failure = message.replace(/\d+/g, '#'); @@ -159,6 +187,26 @@ export class CheckIn { } } + /** + * Blocks until this node holds a configuration copy: a check-in now, + * then one per interval, until a bundle has been applied. For boot + * (main.ts) — a node without configuration has nothing to build a + * sandbox from and does not listen. Never gives up: the gateway is the + * fleet's configuration and there is no other source; each failure is + * logged once by once(), so a gateway down for an hour is one line. The + * check-ins sent here carry `configVersion: null`, which is what keeps + * the gateway from placing on this node before it listens. + */ + async untilConfigured(): Promise { + while (!this.closing && this.opts.configVersion() === null) { + await this.once(); + if (this.opts.configVersion() !== null) return; + await new Promise((resolve) => + setTimeout(resolve, this.opts.intervalSeconds * 1000), + ); + } + } + private schedule(delayMs: number): void { if (this.closing) return; this.timer = setTimeout(async () => { diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index c9a76efd..aa6a5c8d 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { loadConfig, s3Settings } from './config'; +import { ignoredEnvKeys, loadConfig, MOVED_TO_GATEWAY } from './config'; const TOKEN = { DORMICE_API_TOKEN: 'x'.repeat(32) }; @@ -23,7 +23,6 @@ describe('loadConfig executor knobs', () => { }); expect(config.DORMICE_EXECUTOR).toBe('docker'); expect(config.DORMICE_DATA_DIR).toBe('/var/lib/dormice'); - expect(config.DORMICE_SANDBOX_DISK_GB).toBe(10); }); it('rejects a relative DB path in docker mode', () => { @@ -77,85 +76,36 @@ describe('the metrics sampler knobs', () => { }); }); -describe('the pids cap seed', () => { - it('defaults to 4096 and refuses a seed below the wire floor, naming it', () => { - expect(loadConfig(TOKEN).DORMICE_SANDBOX_PIDS_LIMIT).toBe(4096); - expect( - loadConfig({ ...TOKEN, DORMICE_SANDBOX_PIDS_LIMIT: '256' }) - .DORMICE_SANDBOX_PIDS_LIMIT, - ).toBe(256); - // The settings view promises >= 256. A lower seed would be adopted - // into the ledger and leave getConfig unable to serialize its own - // settings (measured: HTTP 500 "Response doesn't match the schema"). - expect(() => - loadConfig({ ...TOKEN, DORMICE_SANDBOX_PIDS_LIMIT: '255' }), - ).toThrow(/DORMICE_SANDBOX_PIDS_LIMIT must be at least 256/); - }); -}); - -describe('the S3 set', () => { - const S3 = { - DORMICE_S3_ENDPOINT: 'http://127.0.0.1:9000', - DORMICE_S3_BUCKET: 'dormice-archive', - DORMICE_S3_ACCESS_KEY_ID: 'minio-user', - DORMICE_S3_SECRET_ACCESS_KEY: 'minio-secret', - }; - - it('parses a full set and adjudicates the archiver as configured', () => { - const config = loadConfig({ ...TOKEN, ...S3 }); - expect(s3Settings(config)).toEqual({ - endpoint: 'http://127.0.0.1:9000', - bucket: 'dormice-archive', - accessKeyId: 'minio-user', - secretAccessKey: 'minio-secret', - region: 'us-east-1', - forcePathStyle: false, +describe('the knobs that moved to the gateway', () => { + it('are not knobs here: the config has no field for them, and the boot line can name the ones an env file still carries', () => { + const config = loadConfig({ + ...TOKEN, + DORMICE_SANDBOX_DISK_GB: '20', + DORMICE_S3_ENDPOINT: 'http://127.0.0.1:9000', }); - }); - - it('adjudicates the archiver as absent when nothing is set', () => { - expect(s3Settings(loadConfig(TOKEN))).toBeNull(); - }); - - it('refuses a partial set, naming exactly the missing variables', () => { - expect(() => - loadConfig({ + expect(Object.keys(config)).not.toContain('DORMICE_SANDBOX_DISK_GB'); + expect(Object.keys(config)).not.toContain('DORMICE_S3_ENDPOINT'); + expect( + ignoredEnvKeys({ ...TOKEN, + DORMICE_SANDBOX_DISK_GB: '20', DORMICE_S3_ENDPOINT: 'http://127.0.0.1:9000', - DORMICE_S3_BUCKET: 'dormice-archive', + DORMICE_PORT: '3676', }), - ).toThrow( - /DORMICE_S3_ACCESS_KEY_ID, DORMICE_S3_SECRET_ACCESS_KEY are missing/, - ); - }); - - it('rejects an endpoint without a scheme', () => { - expect(() => - loadConfig({ ...TOKEN, ...S3, DORMICE_S3_ENDPOINT: 's3.example.com' }), - ).toThrow(/DORMICE_S3_ENDPOINT must be a full http\(s\) URL/); - }); - - it('parses the path-style knob as a real boolean', () => { - // The z.coerce.boolean trap: the string "false" must not become true. - const on = loadConfig({ - ...TOKEN, - ...S3, - DORMICE_S3_FORCE_PATH_STYLE: 'true', - }); - expect(on.DORMICE_S3_FORCE_PATH_STYLE).toBe(true); - const off = loadConfig({ - ...TOKEN, - ...S3, - DORMICE_S3_FORCE_PATH_STYLE: 'false', - }); - expect(off.DORMICE_S3_FORCE_PATH_STYLE).toBe(false); + ).toEqual(['DORMICE_SANDBOX_DISK_GB', 'DORMICE_S3_ENDPOINT']); + expect(ignoredEnvKeys(TOKEN)).toEqual([]); + // Thirteen names, every one an old daemon variable and none of them a knob the node still has. + expect(MOVED_TO_GATEWAY).toHaveLength(13); + for (const key of MOVED_TO_GATEWAY) { + expect(Object.keys(loadConfig(TOKEN))).not.toContain(key); + } }); }); describe('the fleet knobs: gateway, node endpoint, check-in interval', () => { - it('defaults: no gateway (standalone), no node endpoint, 15s check-in', () => { + it('defaults: the gateway beside the daemon (a fleet of one), no node endpoint, 15s check-in', () => { const config = loadConfig(TOKEN); - expect(config.DORMICE_GATEWAY_ENDPOINT).toBeUndefined(); + expect(config.DORMICE_GATEWAY_ENDPOINT).toBe('http://127.0.0.1:3677'); expect(config.DORMICE_NODE_ENDPOINT).toBeUndefined(); expect(config.DORMICE_CHECK_IN_INTERVAL_SECONDS).toBe(15); }); diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index 528a4be6..f40ca14d 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -1,11 +1,6 @@ import { isAbsolute } from 'node:path'; -import { - bareHostnameRegex, - isOriginUrl, - PIDS_LIMIT_MIN, -} from '@dormice/shared'; +import { isOriginUrl } from '@dormice/shared'; import { z } from 'zod'; -import type { S3Settings } from './archive/s3-store'; /** * All configuration comes from environment variables, validated once at @@ -13,6 +8,16 @@ import type { S3Settings } from './archive/s3-store'; * confusing runtime error. Everything has a default except the API token * and, when the docker executor is selected, the base image. * + * What is here is the node's identity and its machine: port, ledger, data + * dir, executor, token, the gateway it belongs to, its tickers. The + * fleet's operator knobs — new-sandbox defaults, the pids cap, the S3 + * store, the sandbox domain, the managed front door — are not: since the + * configuration moved to the gateway (2026-09-14, design record #22) they + * are the gateway's env seeds and its console's knobs, and reach this + * node as a bundle with its check-in (db/settings.ts). Their old variable + * names are still recognised here for one purpose: to say at boot that + * they do nothing (MOVED_TO_GATEWAY below). + * * Variables are prefixed DORMICE_ because the environment is a global * namespace — bare names like PORT collide with whatever else the operator * has exported. @@ -68,118 +73,24 @@ const envSchema = z.object({ DORMICE_BASE_IMAGE: z.string().optional(), /** Sandbox disk images and their mount points live here (docker executor only). */ DORMICE_DATA_DIR: z.string().default('/var/lib/dormice'), - DORMICE_SANDBOX_DISK_GB: z.coerce.number().positive().default(10), - DORMICE_SANDBOX_CPUS: z.coerce.number().positive().default(1), - DORMICE_SANDBOX_MEMORY_GB: z.coerce.number().positive().default(2), - /** - * The pids cgroup cap on each sandbox's container. Under gVisor this is - * NOT "how many processes the sandbox may run": the guest never sees it. - * It caps the sandbox's host-side footprint — the sentry's threads, the - * gofer, and one stub process per guest process (systrap) — and when the - * cap is hit the Go runtime cannot create a thread and the whole sandbox - * dies (exit 2, no OOM flag; measured 2026-09-08). 512 was runc's - * fork-bomb number and killed real 16 GB agent sandboxes running a browser - * plus several node/claude sessions (a production fleet: 13 deaths in 10 days, - * observed peak 470). 4096 is ~8x that peak; the cap still exists so a - * fork bomb takes down its own sandbox and nothing else. A first-boot - * seed since the same day (runtime_settings.pids_limit, edited from the - * console settings page): the incident that earned the new default was - * exactly an operator needing to move this without shell access and a - * restart. Existing containers converge at their next wake (docker - * update, no rebuild). Floored at the wire's PIDS_LIMIT_MIN: the settings - * view promises that floor, so a lower seed adopted into the ledger would - * leave getConfig unable to serialize its own settings (measured: HTTP - * 500 on every call) — refused here, at boot, with the variable named. - */ - DORMICE_SANDBOX_PIDS_LIMIT: z.coerce - .number() - .int() - .min(PIDS_LIMIT_MIN, { - error: `DORMICE_SANDBOX_PIDS_LIMIT must be at least ${PIDS_LIMIT_MIN} — below that a sandbox cannot boot its own runtime`, - }) - .default(4096), DORMICE_RECLAIM_TIMEOUT_SECONDS: z.coerce .number() .int() .positive() .default(45), - /** - * The sandbox wildcard domain behind getHost() — first-boot seed only - * since 2026-07-26: the value in force lives in the ledger - * (runtime_settings.sandbox_domain, edited from the console domains - * page), and once that column holds a value this variable is - * deliberately ignored. With a domain in force, create and connect - * responses carry `domain`, the SDK builds `-.` - * hosts, and requests arriving with such a Host header are proxied into - * that sandbox's port (frozen sandboxes wake on traffic). The operator - * points `*.` DNS plus a TLS-terminating reverse proxy at the - * daemon. A bare hostname: no scheme, no port, no leading or trailing - * dot — the same regex the wire validates against (shared/settings.ts). - */ - DORMICE_SANDBOX_DOMAIN: z - .string() - .regex(bareHostnameRegex, { - error: - 'DORMICE_SANDBOX_DOMAIN must be a bare hostname like sbx.example.com — no scheme, no port, no leading/trailing dots', - }) - .optional(), - /** - * The Caddy config file the daemon owns — the switch for web-based domain - * binding (setIngress rewrites the file, reloads Caddy, Caddy handles the - * certificate). install.sh sets it when it installs Caddy. Unset, the - * daemon never touches any proxy config and setIngress is refused — the - * feature is honestly absent (the SANDBOX_DOMAIN precedent). Absolute: - * a system file must not move with the start directory. - */ - DORMICE_INGRESS_FILE: z - .string() - .refine(isAbsolute, { - error: - 'DORMICE_INGRESS_FILE must be an absolute path, e.g. /etc/caddy/Caddyfile', - }) - .optional(), - /** - * How the daemon tells the running proxy to re-read its config after a - * bind. Defaults to `caddy reload --config ` — - * right when the daemon owns the whole Caddyfile; an operator whose own - * Caddyfile imports a Dormice-owned fragment points this at the outer - * file instead. - */ - DORMICE_INGRESS_RELOAD_CMD: z.string().min(1).optional(), - /** - * The S3-compatible object store behind the archiver (AWS, R2, MinIO, - * OSS in S3-compat mode) — first-boot seeds only since 2026-07-26: the - * store in force lives in the ledger (runtime_settings.s3_*, edited from - * the console settings page), and once those columns hold a value these - * variables are deliberately ignored. The four core variables still come - * as a set (a half-configured seed refuses to boot, same as ever); with - * none of them, the seed is "archiving off" — the console can turn it on - * at any time. Endpoint is a full URL including scheme (MinIO speaks - * http, the clouds https). - */ - DORMICE_S3_ENDPOINT: z - .url({ - protocol: /^https?$/, - error: - 'DORMICE_S3_ENDPOINT must be a full http(s) URL, e.g. https://s3.example.com or http://127.0.0.1:9000', - }) - .optional(), - DORMICE_S3_BUCKET: z.string().min(1).optional(), - DORMICE_S3_ACCESS_KEY_ID: z.string().min(1).optional(), - DORMICE_S3_SECRET_ACCESS_KEY: z.string().min(1).optional(), - DORMICE_S3_REGION: z.string().default('us-east-1'), - /** Path-style addressing: MinIO needs true; the clouds route by subdomain. */ - DORMICE_S3_FORCE_PATH_STYLE: z.stringbool().default(false), /** * The gateway this daemon is a node of — its intranet address, e.g. - * http://10.0.0.5:3677. Set, the daemon checks in with it every + * http://10.0.0.5:3677. The daemon checks in with it every * DORMICE_CHECK_IN_INTERVAL_SECONDS (check-in.ts): its readings, its - * build, and where it can be reached. That check-in is the gateway's - * only source of "which nodes exist and how full are they" — no - * registration, no nodes file. Unset, the daemon is the whole platform - * by itself, as it always was, and checks in with nobody. The token it - * presents is DORMICE_API_TOKEN: gateway and nodes share one, and the - * gateway speaks to every node with the same one. + * build, where it can be reached, and which configuration version it + * runs — and takes the fleet's configuration from the answer. That + * check-in is the gateway's only source of "which nodes exist and how + * full are they" — no registration, no nodes file — and the node's only + * source of its settings and templates. Every daemon is a node of a + * gateway (design record #22: a single machine is a fleet of one); the + * default is the gateway install.sh puts beside the daemon. The token + * it presents is DORMICE_API_TOKEN: gateway and nodes share one, and + * the gateway speaks to every node with the same one. */ DORMICE_GATEWAY_ENDPOINT: z .url({ @@ -188,7 +99,7 @@ const envSchema = z.object({ 'DORMICE_GATEWAY_ENDPOINT must be a full http(s) URL, e.g. http://10.0.0.5:3677', }) .transform((url) => url.replace(/\/+$/, '')) - .optional(), + .default('http://127.0.0.1:3677'), /** * Where the gateway reaches this node — the address it forwards to. * Default: this daemon's own loopback address, right when gateway and @@ -289,7 +200,6 @@ const checkedSchema = envSchema // explicit value is the operator's word and is taken as written. .refine( (cfg) => - cfg.DORMICE_GATEWAY_ENDPOINT === undefined || isLoopbackUrl(cfg.DORMICE_GATEWAY_ENDPOINT) !== false || cfg.DORMICE_NODE_ENDPOINT !== undefined, { @@ -306,7 +216,6 @@ const checkedSchema = envSchema // elsewhere. Refused here, at boot, where the operator is looking. .refine( (cfg) => - cfg.DORMICE_GATEWAY_ENDPOINT === undefined || isLoopbackUrl(cfg.DORMICE_GATEWAY_ENDPOINT) !== false || cfg.DORMICE_NODE_ID !== 'node-1', { @@ -314,26 +223,7 @@ const checkedSchema = envSchema 'DORMICE_NODE_ID is required when DORMICE_GATEWAY_ENDPOINT is not loopback: the gateway tells nodes apart by it, and node-1 (the default) is what every other unconfigured node says — the second to check in is refused as a twin. Give this node a name of its own, e.g. its hostname', path: ['DORMICE_NODE_ID'], }, - ) - // All-or-none: a half-configured store would make the archiver's - // existence ambiguous, and ambiguity here decides real policy defaults. - .superRefine((cfg, ctx) => { - const wanted = [ - 'DORMICE_S3_ENDPOINT', - 'DORMICE_S3_BUCKET', - 'DORMICE_S3_ACCESS_KEY_ID', - 'DORMICE_S3_SECRET_ACCESS_KEY', - ] as const; - const missing = wanted.filter((name) => cfg[name] === undefined); - const first = missing[0]; - if (first !== undefined && missing.length < wanted.length) { - ctx.addIssue({ - code: 'custom', - message: `the DORMICE_S3_* variables come as a set: ${missing.join(', ')} ${missing.length === 1 ? 'is' : 'are'} missing — set all four to enable the archiver, or none to disable it`, - path: [first], - }); - } - }); + ); export type Config = z.infer; @@ -342,84 +232,31 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config { } /** - * Every knob the daemon has, in display order, with its secrecy flag — the - * single adjudication of "what getConfig reports". A Record over keyof - * Config so the compiler refuses a new env variable until it is listed - * here too: a knob that exists but is invisible would be a silent lie. + * The fleet's operator knobs, by their old daemon names — variables this + * process reads nothing from since the configuration moved to the gateway + * (2026-09-14). They live on the gateway's side now: as its env seeds at + * first start (same names), then in its settings table, edited from the + * console. A node that still carries them in its env file is a machine + * upgraded across the move; main.ts says so once at boot, naming them, + * instead of silently doing something else than the operator wrote. */ -export const CONFIG_KEYS: Record = { - DORMICE_PORT: { sensitive: false }, - DORMICE_DB_PATH: { sensitive: false }, - DORMICE_NODE_ID: { sensitive: false }, - DORMICE_API_TOKEN: { sensitive: true }, - DORMICE_EXECUTOR: { sensitive: false }, - DORMICE_BASE_IMAGE: { sensitive: false }, - DORMICE_DATA_DIR: { sensitive: false }, - DORMICE_SCAN_INTERVAL_SECONDS: { sensitive: false }, - DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS: { sensitive: false }, - DORMICE_METRICS_RETENTION_HOURS: { sensitive: false }, - DORMICE_SANDBOX_DISK_GB: { sensitive: false }, - DORMICE_SANDBOX_CPUS: { sensitive: false }, - DORMICE_SANDBOX_MEMORY_GB: { sensitive: false }, - DORMICE_SANDBOX_PIDS_LIMIT: { sensitive: false }, - DORMICE_RECLAIM_TIMEOUT_SECONDS: { sensitive: false }, - DORMICE_SANDBOX_DOMAIN: { sensitive: false }, - DORMICE_INGRESS_FILE: { sensitive: false }, - DORMICE_INGRESS_RELOAD_CMD: { sensitive: false }, - DORMICE_S3_ENDPOINT: { sensitive: false }, - DORMICE_S3_BUCKET: { sensitive: false }, - DORMICE_S3_ACCESS_KEY_ID: { sensitive: true }, - DORMICE_S3_SECRET_ACCESS_KEY: { sensitive: true }, - DORMICE_S3_REGION: { sensitive: false }, - DORMICE_S3_FORCE_PATH_STYLE: { sensitive: false }, - DORMICE_GATEWAY_ENDPOINT: { sensitive: false }, - DORMICE_NODE_ENDPOINT: { sensitive: false }, - DORMICE_CHECK_IN_INTERVAL_SECONDS: { sensitive: false }, -}; +export const MOVED_TO_GATEWAY = [ + 'DORMICE_SANDBOX_DISK_GB', + 'DORMICE_SANDBOX_CPUS', + 'DORMICE_SANDBOX_MEMORY_GB', + 'DORMICE_SANDBOX_PIDS_LIMIT', + 'DORMICE_SANDBOX_DOMAIN', + 'DORMICE_INGRESS_FILE', + 'DORMICE_INGRESS_RELOAD_CMD', + 'DORMICE_S3_ENDPOINT', + 'DORMICE_S3_BUCKET', + 'DORMICE_S3_ACCESS_KEY_ID', + 'DORMICE_S3_SECRET_ACCESS_KEY', + 'DORMICE_S3_REGION', + 'DORMICE_S3_FORCE_PATH_STYLE', +] as const; -export type ConfigSources = Record; - -/** - * Which knobs the operator set explicitly versus which fell back to - * defaults. Read off the raw environment at load time — the parsed config - * cannot tell the two apart once defaults are applied. - */ -export function configSources( - env: NodeJS.ProcessEnv = process.env, -): ConfigSources { - return Object.fromEntries( - (Object.keys(CONFIG_KEYS) as Array).map((key) => [ - key, - env[key] !== undefined ? 'env' : 'default', - ]), - ) as ConfigSources; -} - -/** - * The one adjudicator of the S3 first-boot seed: null unless the whole - * DORMICE_S3_* set is present (a partial set never gets past the schema). - * Since 2026-07-26 this decides only what ensureRuntimeSettings seeds a - * virgin ledger with — the store in force is the ledger's - * (db/settings.ts readS3Settings), and everything that used to hang off - * this answer (whether the Archiver has a store, whether new sandboxes - * default to archiving, whether archive-asking policies are accepted) - * reads the ledger live. - */ -export function s3Settings(config: Config): S3Settings | null { - if ( - config.DORMICE_S3_ENDPOINT === undefined || - config.DORMICE_S3_BUCKET === undefined || - config.DORMICE_S3_ACCESS_KEY_ID === undefined || - config.DORMICE_S3_SECRET_ACCESS_KEY === undefined - ) { - return null; - } - return { - endpoint: config.DORMICE_S3_ENDPOINT, - bucket: config.DORMICE_S3_BUCKET, - accessKeyId: config.DORMICE_S3_ACCESS_KEY_ID, - secretAccessKey: config.DORMICE_S3_SECRET_ACCESS_KEY, - region: config.DORMICE_S3_REGION, - forcePathStyle: config.DORMICE_S3_FORCE_PATH_STYLE, - }; +/** Which of MOVED_TO_GATEWAY the environment still sets, for the boot line. */ +export function ignoredEnvKeys(env: NodeJS.ProcessEnv = process.env): string[] { + return MOVED_TO_GATEWAY.filter((key) => env[key] !== undefined); } diff --git a/packages/server/src/db/account.ts b/packages/server/src/db/account.ts deleted file mode 100644 index bdabf813..00000000 --- a/packages/server/src/db/account.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { eq } from 'drizzle-orm'; -import type { Db } from './db'; -import { type ConsoleAccountRow, consoleAccount } from './schema'; - -/** - * The console's single human account (see the schema comment for why it is - * a singleton). The fixed id makes setup an upsert: presenting the API - * token overwrites whatever is there — account creation, password change - * and forgot-password are all the same verb. - */ -const ACCOUNT_ID = 1; - -export function getConsoleAccount(db: Db): ConsoleAccountRow | undefined { - return db - .select() - .from(consoleAccount) - .where(eq(consoleAccount.id, ACCOUNT_ID)) - .get(); -} - -export function setConsoleAccount( - db: Db, - input: { username: string; passwordHash: string; sessionSecret: string }, -): ConsoleAccountRow { - const now = new Date().toISOString(); - const row: ConsoleAccountRow = { - id: ACCOUNT_ID, - username: input.username, - passwordHash: input.passwordHash, - sessionSecret: input.sessionSecret, - createdAt: now, - updatedAt: now, - }; - db.insert(consoleAccount) - .values(row) - .onConflictDoUpdate({ - target: consoleAccount.id, - set: { - username: input.username, - passwordHash: input.passwordHash, - sessionSecret: input.sessionSecret, - updatedAt: now, - }, - }) - .run(); - const stored = getConsoleAccount(db); - if (!stored) { - throw new Error('console account vanished mid-setup'); - } - return stored; -} diff --git a/packages/server/src/db/api-keys.ts b/packages/server/src/db/api-keys.ts deleted file mode 100644 index 0ddc1d7a..00000000 --- a/packages/server/src/db/api-keys.ts +++ /dev/null @@ -1,210 +0,0 @@ -import { createHash, randomBytes, randomUUID } from 'node:crypto'; -import { and, desc, eq, gt, isNull, lt, or, sql } from 'drizzle-orm'; -import type { Db } from './db'; -import { type ApiKeyRow, apiKeys } from './schema'; - -/** - * lastUsedAt write granularity. A hot-polling client authenticates many - * times a second; writing the ledger on every hit would turn the WAL into - * an EKG for zero information. One write a minute answers the only question - * the column exists for — "is this key still alive, roughly since when". - */ -const LAST_USED_GRANULARITY_MS = 60_000; - -/** sha256 hex of the bare key material — the only form the ledger stores. */ -export function hashApiKeyToken(token: string): string { - return createHash('sha256').update(token).digest('hex'); -} - -/** - * The one write-side shape for timestamps that later feed string - * comparisons. Wire ISO input has variable precision ("…00Z" sorts after - * "…00.500Z" while being chronologically earlier), so every expiresAt is - * normalized to exact toISOString() output before it touches the ledger — - * after that, plain string compare is chronologically sound (the same - * argument the lastUsedAt throttle rests on). - */ -function normalizeIso(value: string): string { - return new Date(value).toISOString(); -} - -/** - * Mints a key and returns the material exactly once — the row keeps only - * the hash, so this return value is the caller's single chance to see it. - * Pure hex, no brand prefix: the official Python E2B SDK validates - * `e2b_[0-9a-f]+` client-side, so anything non-hex could never be used on - * the X-API-KEY face at all. - * - * The caller (route) has already refused a duplicate active name with a - * 409; the partial unique index backstops that check as a schema fact. - */ -export function createApiKey( - db: Db, - name: string, - expiresAt: string | undefined, -): { row: ApiKeyRow; token: string } { - const token = randomBytes(32).toString('hex'); - const row: ApiKeyRow = { - id: randomUUID(), - name, - keyHash: hashApiKeyToken(token), - prefix: token.slice(0, 8), - createdAt: new Date().toISOString(), - lastUsedAt: null, - expiresAt: expiresAt ? normalizeIso(expiresAt) : null, - disabledAt: null, - revokedAt: null, - }; - db.insert(apiKeys).values(row).run(); - return { row, token }; -} - -export function findActiveApiKeyByName( - db: Db, - name: string, -): ApiKeyRow | undefined { - return db - .select() - .from(apiKeys) - .where(and(eq(apiKeys.name, name), isNull(apiKeys.revokedAt))) - .get(); -} - -export function findApiKeyById(db: Db, id: string): ApiKeyRow | undefined { - return db.select().from(apiKeys).where(eq(apiKeys.id, id)).get(); -} - -/** - * Every key ever minted, revoked ones included — the rotation history. - * createdAt has millisecond granularity, so two keys minted back-to-back - * can tie; rowid breaks the tie by insertion order, and it is trustworthy - * here because api_keys rows are never deleted (revoke is soft), so rowids - * are never reused. - */ -export function listApiKeys(db: Db): ApiKeyRow[] { - return db - .select() - .from(apiKeys) - .orderBy(desc(apiKeys.createdAt), desc(sql`rowid`)) - .all(); -} - -/** - * Soft-revokes the key with this id. Returns false when it does not exist - * or is already revoked — the desired end state was already true. The row - * survives as history; the name is immediately free for a new key. - */ -export function revokeApiKey(db: Db, id: string): boolean { - const row = findApiKeyById(db, id); - if (!row || row.revokedAt !== null) { - return false; - } - db.update(apiKeys) - .set({ revokedAt: new Date().toISOString() }) - .where(eq(apiKeys.id, id)) - .run(); - return true; -} - -/** - * Edits a non-revoked key in place. The route has already adjudicated the - * 404 (unknown id), the 409s (revoked row, name collision) — this function - * only computes the changed-field set against the row it was handed and - * writes once. A field equal to its current value is not a change (the - * updatePolicy idiom: a no-op patch is the goal state, not an error), so - * disabling an already-disabled key keeps its original disabledAt. The - * returned row carries what changed; the route logs it. - */ -export function updateApiKey( - db: Db, - row: ApiKeyRow, - patch: { name?: string; expiresAt?: string | null; disabled?: boolean }, -): ApiKeyRow { - const changes: Partial = {}; - if (patch.name !== undefined && patch.name !== row.name) { - changes.name = patch.name; - } - if (patch.expiresAt !== undefined) { - const next = - patch.expiresAt === null ? null : normalizeIso(patch.expiresAt); - if (next !== row.expiresAt) { - changes.expiresAt = next; - } - } - if (patch.disabled === true && row.disabledAt === null) { - changes.disabledAt = new Date().toISOString(); - } else if (patch.disabled === false && row.disabledAt !== null) { - changes.disabledAt = null; - } - - if (Object.keys(changes).length === 0) { - return row; - } - db.update(apiKeys).set(changes).where(eq(apiKeys.id, row.id)).run(); - return { ...row, ...changes }; -} - -/** - * The single liveness adjudication: revoked, disabled and expired all close - * the door, in one WHERE. Pure read — it never stamps lastUsedAt — so the - * admin gate can consult it for its honest 403 without a refused request - * leaving "recently used" fingerprints. The expiry compare is a plain - * string > against toISOString(now), sound because expiresAt is normalized - * on write (see normalizeIso). - */ -export function findLiveApiKeyByHash( - db: Db, - hash: string, -): { id: string; lastUsedAt: string | null } | undefined { - return db - .select({ id: apiKeys.id, lastUsedAt: apiKeys.lastUsedAt }) - .from(apiKeys) - .where( - and( - eq(apiKeys.keyHash, hash), - isNull(apiKeys.revokedAt), - isNull(apiKeys.disabledAt), - or( - isNull(apiKeys.expiresAt), - gt(apiKeys.expiresAt, new Date().toISOString()), - ), - ), - ) - .get(); -} - -/** findLiveApiKeyByHash for callers holding the bare token — hashing stays in this module. */ -export function isLiveApiKey(db: Db, bareToken: string): boolean { - return findLiveApiKeyByHash(db, hashApiKeyToken(bareToken)) !== undefined; -} - -/** - * The ledger leg of credential verification: does this bare token match a - * live key? An indexed exact-match lookup on sha256(token) — not a - * timing-safe scan, deliberately: the comparison can at worst leak bytes of - * sha256(key), which preimage resistance makes worthless to an attacker - * (the argument GitHub token storage rests on). - * - * A hit answers the key's id and stamps lastUsedAt — only a hit: - * verification is the one moment a credential was actually honored. Throttled to LAST_USED_GRANULARITY_MS so a polling client does - * not write the ledger per request. ISO strings compare lexicographically - * as timestamps, so the cutoff is a plain string <. - */ -export function verifyApiKeyToken(db: Db, bareToken: string): string | null { - const row = findLiveApiKeyByHash(db, hashApiKeyToken(bareToken)); - if (!row) { - return null; - } - const now = Date.now(); - const cutoff = new Date(now - LAST_USED_GRANULARITY_MS).toISOString(); - db.update(apiKeys) - .set({ lastUsedAt: new Date(now).toISOString() }) - .where( - and( - eq(apiKeys.id, row.id), - or(isNull(apiKeys.lastUsedAt), lt(apiKeys.lastUsedAt, cutoff)), - ), - ) - .run(); - return row.id; -} diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index a9b5c107..e4833ab5 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -88,10 +88,16 @@ export const sandboxes = sqliteTable('sandboxes', { export type SandboxRow = typeof sandboxes.$inferSelect; /** - * Registered templates: a name for a Docker image that lives on this host. - * The host's Docker daemon is the image store; this table only records which - * name points where. Sandboxes reference templates by name (column above), - * so re-pointing a name upgrades every future shell built for it. + * Templates: a name for a Docker image that lives on this host. The host's + * Docker daemon is the image store; this table only records which name + * points where. Sandboxes reference templates by name (column above), so + * re-pointing a name upgrades every future shell built for it. + * + * Since the configuration moved to the gateway (2026-09-14) this table is + * the node's copy: registered and removed there, written here whole with + * every configuration bundle (db/settings.ts applyNodeConfig) and read at + * every birth and wake — so a node whose gateway is away still resolves + * its names. */ export const templates = sqliteTable('templates', { name: text('name').primaryKey(), @@ -210,15 +216,13 @@ export const hostMetricsSamples = sqliteTable('host_metrics_samples', { export type HostMetricsSampleRow = typeof hostMetricsSamples.$inferSelect; /** - * The console's one human account (the fixed id makes "at most one row" a - * schema fact, not a convention). The API token stays the root of trust: - * presenting it (re)creates this row — that IS the forgot-password path — - * while day-to-day console logins are username + password. - * - * sessionSecret is the HMAC key for session cookies. It lives here and not - * in the token so the two credentials rotate independently: re-running - * setup regenerates it (every session out — correct for a password reset), - * rotating the API token leaves console sessions alone. + * The console's one human account — the gateway's table since 2026-09-14 + * (the console is served there; packages/gateway/src/db/schema.ts has the + * living definition). Kept in the node's ledger, unread and unwritten, + * until the one-time import of a single machine's old tables into the + * gateway (cut 4) has run; it is dropped then, not before — a migration + * that dropped it now would delete the operator's account ahead of the + * step that moves it. */ export const consoleAccount = sqliteTable('console_account', { id: integer('id').primaryKey(), @@ -253,21 +257,30 @@ export const daemonSecrets = sqliteTable('daemon_secrets', { export type DaemonSecretsRow = typeof daemonSecrets.$inferSelect; /** - * Runtime settings — the operator knobs that must be changeable from the - * console without shell access and a restart (shared/settings.ts draws the - * line). One row, fixed id — the console_account singleton pattern. Born at - * first boot, seeded from the env variables of the same names; from then on - * the ledger is the single truth and env edits to those knobs are ignored - * (two live sources for one knob is a standing ambiguity). The discipline - * is per knob, not per row: a column added by a later migration gets its - * one env consultation on the first boot that sees it NULL (see - * ensureRuntimeSettings' adopt step). + * Runtime settings — this node's copy of the fleet configuration (design + * record #22, 2026-09-14): the gateway holds the settings and every node + * keeps a copy, so a node whose gateway is away still knows what a new + * sandbox gets and where its archives live. One row, fixed id — the + * console_account singleton pattern — written whole by applyNodeConfig + * (db/settings.ts) with every bundle the check-in brings, never edited + * here: there is no verb on the node that changes a knob. + * + * config_version says which bundle the row is. NULL means the row is not + * a copy at all: the daemon's own settings from before the move (seeded + * from the env, edited from the console), kept for the one-time import + * into the gateway (cut 4) and never read by the node again — the node + * reports "no copy" and takes the gateway's bundle at its first check-in, + * which overwrites every column. * * Typed columns, not a JSON blob: the schema IS the vocabulary, and a knob * that exists but is invisible to migrations would drift silently. */ export const runtimeSettings = sqliteTable('runtime_settings', { id: integer('id').primaryKey(), + /** The bundle's version (shared nodeConfigBundleSchema); NULL = not a copy, see above. */ + configVersion: integer('config_version'), + /** ISO 8601 UTC — when this copy was applied; the boot log's "how old is what I run". */ + configAppliedAt: text('config_applied_at'), sandboxCpus: real('sandbox_cpus').notNull(), sandboxMemoryGb: real('sandbox_memory_gb').notNull(), sandboxDiskGb: real('sandbox_disk_gb').notNull(), @@ -276,21 +289,20 @@ export const runtimeSettings = sqliteTable('runtime_settings', { defaultStopAfterSeconds: integer('default_stop_after_seconds'), /** NULL = never archive — forced when the daemon has no archiver. */ defaultArchiveAfterSeconds: integer('default_archive_after_seconds'), - /** Daemon-managed swap target, GiB (see swap.ts); 0 = manage none. */ + /** This node's managed-swap target, GiB (swap.ts) — its own row at the gateway (updateNodeSettings); 0 = manage none. */ swapGb: integer('swap_gb').notNull().default(0), /** - * The S3 archive store and the sandbox domain (added 2026-07-26). These - * columns are per-knob three-state: NULL = the column is younger than - * this row and has never been adjudicated (alive only between the - * migration and the next boot's ensureRuntimeSettings, which adopts the - * env value once); '' on the two decider columns (s3Endpoint, - * sandboxDomain) = explicitly off. The sentinel never leaves - * db/settings.ts — toView maps '' back to the wire's null. + * The S3 archive store and the sandbox domain. In a copy the two + * decider columns (s3Endpoint, sandboxDomain) are two-state: a value, + * or NULL = off. Rows from before the move used '' for off and NULL for + * "never adjudicated"; a copy never writes '' and the readers never see + * one — they refuse a row without config_version (db/settings.ts). * * s3SecretAccessKey is stored plaintext, like envdSigningSecret and * sessionSecret above: a credential the daemon must present verbatim to - * S3 cannot be hashed. It never crosses the wire (shared/settings.ts - * s3ArchiveViewSchema withholds both keys). + * S3 cannot be hashed. It never crosses the node's wire (shared + * s3ArchiveViewSchema withholds both keys); it arrives with the bundle + * over the gateway→node wire under the fleet token. */ s3Endpoint: text('s3_endpoint'), s3Bucket: text('s3_bucket'), @@ -299,46 +311,28 @@ export const runtimeSettings = sqliteTable('runtime_settings', { s3Region: text('s3_region'), s3ForcePathStyle: integer('s3_force_path_style', { mode: 'boolean' }), sandboxDomain: text('sandbox_domain'), - /** - * Inbound-only alias domains (added 2026-08-31), a JSON string array. - * Same NULL-until-adopted three-state as above, but no '' sentinel — - * '[]' already says "no aliases", and the adopt step always writes '[]' - * without consulting the env: the alias list is console-era operations - * editing, not a first-boot identity, so it deliberately has no env - * variable. - */ + /** Inbound-only alias domains, a JSON string array ('[]' = none). Nullable only for rows from before the move. */ sandboxDomainAliases: text('sandbox_domain_aliases'), /** - * The pids cgroup cap on every sandbox container (added 2026-09-08). - * Same NULL-until-adopted three-state; the adopt step consults - * DORMICE_SANDBOX_PIDS_LIMIT once, so an upgraded daemon keeps the value - * its env has been running with. Never '' — there is no "off": a cap - * always exists (shared/settings.ts enforces the floor). + * The pids cgroup cap on every sandbox container. Never "off": a cap + * always exists (shared/settings.ts enforces the floor). Nullable only + * for rows from before the move. */ pidsLimit: integer('pids_limit'), - /** Null until the first updateSettings: "still exactly the seed" is information. */ + /** The old single-machine row's last edit — the gateway's timestamp now; kept for the cut-4 import, never written by the node. */ updatedAt: text('updated_at'), }); export type RuntimeSettingsRow = typeof runtimeSettings.$inferSelect; /** - * Ledger-minted API keys: full-power peers of DORMICE_API_TOKEN that exist - * so credentials can rotate without an env edit and a restart. The env - * token itself never lives here — it stays the bootstrap/recovery - * credential, checked from config. - * - * keyHash is sha256 of the key material, not scrypt: a key is 256 random - * bits, not a human password, so offline brute force is moot and a slow - * KDF would only tax every authenticated request. Verification is an - * indexed exact-match lookup on the hash; the timing of that comparison - * can at worst leak sha256(key) bytes, which preimage resistance makes - * worthless (the same argument GitHub tokens rest on). - * - * Revocation is soft (revokedAt) — the row stays as rotation history and - * keeps lastUsedAt readable after the credential dies. "At most one ACTIVE - * key per name" is a schema fact via the partial unique index below (the - * console_account fixed-id philosophy); a revoked name is free for reuse. + * API keys — the gateway's table since 2026-09-14: keys are minted and + * judged at the fleet's one door (packages/gateway/src/db/schema.ts has + * the living definition), and toward a node the gateway speaks the fleet + * token alone. Kept in the node's ledger, unread and unwritten, until the + * cut-4 import into the gateway has run — the same reasoning as + * console_account above: a key an operator's automation still holds must + * move, not vanish. */ export const apiKeys = sqliteTable( 'api_keys', diff --git a/packages/server/src/db/settings.ts b/packages/server/src/db/settings.ts index 99c323cd..2459a542 100644 --- a/packages/server/src/db/settings.ts +++ b/packages/server/src/db/settings.ts @@ -1,145 +1,139 @@ -import { - DEFAULT_LIFECYCLE_POLICY, - type RuntimeSettings, - type UpdateSettingsRequest, -} from '@dormice/shared'; -import { and, eq, isNull } from 'drizzle-orm'; +import type { NodeConfigBundle, RuntimeSettings } from '@dormice/shared'; +import { eq } from 'drizzle-orm'; import type { S3Settings } from '../archive/s3-store'; -import { type Config, s3Settings } from '../config'; -import { ARCHIVE_DEFAULT_SECONDS } from '../policy'; import type { Db } from './db'; -import { type RuntimeSettingsRow, runtimeSettings } from './schema'; +import { type RuntimeSettingsRow, runtimeSettings, templates } from './schema'; /** The console_account fixed-id pattern: "at most one row" as a schema fact. */ const SETTINGS_ROW_ID = 1; /** - * The per-knob three-state (see schema.ts): NULL = never adjudicated, '' = - * explicitly off. '' is a storage sentinel and never leaves this file. + * The fleet settings as this node runs them: the wire's read shape (keys + * withheld — readS3Settings has them, for the one consumer that presents + * them to S3) minus the gateway's edit timestamp, which is the gateway's + * to answer. */ -const OFF = ''; +export type NodeSettings = Omit; /** - * Get-or-seed, run at every boot before anything reads a knob, in two - * idempotent steps: - * - * 1. Insert-or-nothing — a fresh install seeds every column from the env - * variables (and the shared zero-config defaults). The archive default - * is adjudicated right here: an S3 seed present means new sandboxes - * archive after a week, absent means never — the same semantics the - * boot-time archiver adjudication used to produce. - * 2. Adopt-if-virgin — a daemon upgraded onto a schema with new columns - * finds them NULL on its existing row; each such column gets its one - * env consultation now (the knob's ledger life begins at its first - * value). The adopt step never touches defaultArchiveAfterSeconds: an - * existing row's default policy is the operator's property, and one - * group's seed must not rewrite another group (the update doctrine). - * - * After the first boot both steps match zero rows. "The env is ignored - * once the ledger speaks" is per knob: a console clear writes '' (not - * NULL), so a restart never resurrects the env value. + * Thrown by every reader while the node holds no configuration copy. A + * wiring guard, not a runtime state: main.ts blocks before listen until + * the first bundle has been applied, so no request can observe it — + * reaching it means something read a knob before boot finished. */ -export function ensureRuntimeSettings(db: Db, config: Config): void { - const s3Seed = s3Settings(config); - db.insert(runtimeSettings) - .values({ - id: SETTINGS_ROW_ID, - sandboxCpus: config.DORMICE_SANDBOX_CPUS, - sandboxMemoryGb: config.DORMICE_SANDBOX_MEMORY_GB, - sandboxDiskGb: config.DORMICE_SANDBOX_DISK_GB, - defaultFreezeAfterSeconds: DEFAULT_LIFECYCLE_POLICY.freezeAfterSeconds, - defaultStopAfterSeconds: DEFAULT_LIFECYCLE_POLICY.stopAfterSeconds, - defaultArchiveAfterSeconds: s3Seed ? ARCHIVE_DEFAULT_SECONDS : null, - // No env seed: managed swap is born from the console, not the env — - // install.sh's base swapfile already covers "a host needs swap". - swapGb: 0, - ...s3Columns(s3Seed), - sandboxDomain: config.DORMICE_SANDBOX_DOMAIN ?? OFF, - // Aliases are console-era operations editing with deliberately no - // env variable — every install starts with none. - sandboxDomainAliases: '[]', - pidsLimit: config.DORMICE_SANDBOX_PIDS_LIMIT, - updatedAt: null, - }) - .onConflictDoNothing() - .run(); - // Adopt from the env, not a constant: an upgraded daemon has been - // running its fleet at whatever its env says, and the ledger's first - // value must be that — not a default that silently moves the cap. - db.update(runtimeSettings) - .set({ pidsLimit: config.DORMICE_SANDBOX_PIDS_LIMIT }) - .where( - and( - eq(runtimeSettings.id, SETTINGS_ROW_ID), - isNull(runtimeSettings.pidsLimit), - ), - ) - .run(); - db.update(runtimeSettings) - .set(s3Columns(s3Seed)) - .where( - and( - eq(runtimeSettings.id, SETTINGS_ROW_ID), - isNull(runtimeSettings.s3Endpoint), - ), - ) - .run(); - db.update(runtimeSettings) - .set({ sandboxDomain: config.DORMICE_SANDBOX_DOMAIN ?? OFF }) - .where( - and( - eq(runtimeSettings.id, SETTINGS_ROW_ID), - isNull(runtimeSettings.sandboxDomain), - ), - ) - .run(); - // Its own adopt, not a rider on sandboxDomain's: an upgraded row has - // that column decided while this one is still NULL. No env to consult. - db.update(runtimeSettings) - .set({ sandboxDomainAliases: '[]' }) - .where( - and( - eq(runtimeSettings.id, SETTINGS_ROW_ID), - isNull(runtimeSettings.sandboxDomainAliases), - ), - ) - .run(); +export class NoConfigError extends Error { + constructor() { + super( + 'this node holds no configuration copy yet — it is applied at the first check-in with the gateway, before the daemon listens', + ); + this.name = 'NoConfigError'; + } } -/** The six S3 columns as one unit: a store, or the '' decider + NULL rest. */ -function s3Columns(s3: S3Settings | null) { - return s3 - ? { - s3Endpoint: s3.endpoint, - s3Bucket: s3.bucket, - s3AccessKeyId: s3.accessKeyId, - s3SecretAccessKey: s3.secretAccessKey, - s3Region: s3.region, - s3ForcePathStyle: s3.forcePathStyle, - } - : { - s3Endpoint: OFF, - s3Bucket: null, - s3AccessKeyId: null, - s3SecretAccessKey: null, - s3Region: null, - s3ForcePathStyle: null, - }; +/** + * The version of the copy this node runs, or null when it holds none: no + * row, or a row from before the configuration moved to the gateway + * (schema.ts runtimeSettings has the story). The check-in reports it, and + * the gateway answers the whole bundle whenever it differs from its own. + */ +export function readConfigVersion(db: Db): number | null { + return ( + db + .select({ version: runtimeSettings.configVersion }) + .from(runtimeSettings) + .where(eq(runtimeSettings.id, SETTINGS_ROW_ID)) + .get()?.version ?? null + ); } -function virginError(column: string): Error { - return new Error( - `runtime settings column ${column} was never adjudicated — ensureRuntimeSettings must run at boot`, - ); +/** When the copy this node runs was applied (ISO 8601), for the boot log. */ +export function readConfigAppliedAt(db: Db): string | null { + return readRow(db).configAppliedAt; } -function toView(row: RuntimeSettingsRow): RuntimeSettings { - if (row.s3Endpoint === null) throw virginError('s3_endpoint'); - if (row.sandboxDomain === null) throw virginError('sandbox_domain'); - if (row.sandboxDomainAliases === null) { - throw virginError('sandbox_domain_aliases'); - } - if (row.pidsLimit === null) throw virginError('pids_limit'); +/** + * Writes a bundle whole: the settings row (every column — an old + * single-machine row is overwritten, not merged) and the templates table + * (delete-all, insert-all) in one transaction, so no reader ever sees the + * new version with the old content, or a settings row from one version + * beside templates from another. The pure write; the two knobs with a + * reality on the host that a write does not move (the pids cap on running + * shells, the managed swap) are node-config.ts's to reconcile afterwards. + */ +export function applyNodeConfig( + db: Db, + bundle: NodeConfigBundle, + now = new Date(), +): void { + const { settings, node } = bundle; + const row = { + id: SETTINGS_ROW_ID, + configVersion: bundle.version, + configAppliedAt: now.toISOString(), + sandboxCpus: settings.sandboxDefaults.cpus, + sandboxMemoryGb: settings.sandboxDefaults.memoryGb, + sandboxDiskGb: settings.sandboxDefaults.diskGb, + defaultFreezeAfterSeconds: settings.defaultPolicy.freezeAfterSeconds, + defaultStopAfterSeconds: settings.defaultPolicy.stopAfterSeconds, + defaultArchiveAfterSeconds: settings.defaultPolicy.archiveAfterSeconds, + swapGb: node.swapGb, + s3Endpoint: settings.s3?.endpoint ?? null, + s3Bucket: settings.s3?.bucket ?? null, + s3AccessKeyId: settings.s3?.accessKeyId ?? null, + s3SecretAccessKey: settings.s3?.secretAccessKey ?? null, + s3Region: settings.s3?.region ?? null, + s3ForcePathStyle: settings.s3?.forcePathStyle ?? null, + sandboxDomain: settings.sandboxDomain, + sandboxDomainAliases: JSON.stringify(settings.sandboxDomainAliases), + pidsLimit: settings.pidsLimit, + }; + const { id: _id, ...set } = row; + db.transaction((tx) => { + tx.insert(runtimeSettings) + .values(row) + .onConflictDoUpdate({ target: runtimeSettings.id, set }) + .run(); + tx.delete(templates).run(); + if (bundle.templates.length > 0) { + tx.insert(templates).values(bundle.templates).run(); + } + }); +} + +/** + * The copy read back whole, keys included — what the node runs, in the + * bundle's own shape. For the boot log and for tests that edit a copy in + * place; the request handlers read the narrower views below. + */ +export function readNodeConfig(db: Db): NodeConfigBundle { + const row = readRow(db); + const view = toView(row); + return { + version: row.configVersion as number, + settings: { + sandboxDefaults: view.sandboxDefaults, + defaultPolicy: view.defaultPolicy, + s3: readS3Settings(db), + sandboxDomain: view.sandboxDomain, + sandboxDomainAliases: view.sandboxDomainAliases, + pidsLimit: view.pidsLimit, + }, + node: { swapGb: row.swapGb }, + templates: db.select().from(templates).orderBy(templates.name).all(), + }; +} + +function readRow(db: Db): RuntimeSettingsRow { + const row = db + .select() + .from(runtimeSettings) + .where(eq(runtimeSettings.id, SETTINGS_ROW_ID)) + .get(); + if (!row || row.configVersion === null) throw new NoConfigError(); + return row; +} + +function toView(row: RuntimeSettingsRow): NodeSettings { return { sandboxDefaults: { cpus: row.sandboxCpus, @@ -152,29 +146,28 @@ function toView(row: RuntimeSettingsRow): RuntimeSettings { archiveAfterSeconds: row.defaultArchiveAfterSeconds, }, s3: - row.s3Endpoint === OFF + row.s3Endpoint === null ? null : { endpoint: row.s3Endpoint, - // biome-ignore-start lint/style/noNonNullAssertion: the six columns write as one unit (s3Columns) + // biome-ignore-start lint/style/noNonNullAssertion: a copy writes every column (applyNodeConfig), and readRow refuses anything that is not a copy bucket: row.s3Bucket!, region: row.s3Region!, forcePathStyle: row.s3ForcePathStyle!, - // biome-ignore-end lint/style/noNonNullAssertion: the six columns write as one unit (s3Columns) }, - sandboxDomain: row.sandboxDomain === OFF ? null : row.sandboxDomain, + sandboxDomain: row.sandboxDomain, // The one writer JSON.stringifies an array; a corrupt value should // throw right here, not read as "no aliases". - sandboxDomainAliases: JSON.parse(row.sandboxDomainAliases) as string[], - pidsLimit: row.pidsLimit, - updatedAt: row.updatedAt, + sandboxDomainAliases: JSON.parse(row.sandboxDomainAliases!) as string[], + pidsLimit: row.pidsLimit!, + // biome-ignore-end lint/style/noNonNullAssertion: a copy writes every column (applyNodeConfig), and readRow refuses anything that is not a copy }; } /** - * The node's managed-swap target — a knob of this machine, not of the - * fleet, so it left the settings wire (shared/settings.ts) and is read by - * the one consumer that acts on it, the boot reconcile in main.ts. + * The node's managed-swap target — its own row at the gateway + * (updateNodeSettings), applied by the boot reconcile in main.ts and by + * node-config.ts when a bundle moves it. */ export function readSwapTarget(db: Db): number { return readRow(db).swapGb; @@ -182,12 +175,10 @@ export function readSwapTarget(db: Db): number { /** * The knobs in force, read fresh at each use site — a better-sqlite3 point - * read costs microseconds, and reading live is what makes a console edit - * apply to the very next acquire without a restart. Throws when the row is - * missing: that means ensureRuntimeSettings never ran, a wiring bug worth a - * loud death, not a silent fallback to env. + * read costs microseconds, and reading live is what makes a bundle applied + * a moment ago reach the very next acquire without a restart. */ -export function readRuntimeSettings(db: Db): RuntimeSettings { +export function readRuntimeSettings(db: Db): NodeSettings { return toView(readRow(db)); } @@ -199,17 +190,16 @@ export function readRuntimeSettings(db: Db): RuntimeSettings { */ export function readS3Settings(db: Db): S3Settings | null { const row = readRow(db); - if (row.s3Endpoint === null) throw virginError('s3_endpoint'); - if (row.s3Endpoint === OFF) return null; + if (row.s3Endpoint === null) return null; return { endpoint: row.s3Endpoint, - // biome-ignore-start lint/style/noNonNullAssertion: the six columns write as one unit (s3Columns) + // biome-ignore-start lint/style/noNonNullAssertion: the six columns write as one unit (applyNodeConfig) bucket: row.s3Bucket!, accessKeyId: row.s3AccessKeyId!, secretAccessKey: row.s3SecretAccessKey!, region: row.s3Region!, forcePathStyle: row.s3ForcePathStyle!, - // biome-ignore-end lint/style/noNonNullAssertion: the six columns write as one unit (s3Columns) + // biome-ignore-end lint/style/noNonNullAssertion: the six columns write as one unit (applyNodeConfig) }; } @@ -217,67 +207,3 @@ export function readS3Settings(db: Db): S3Settings | null { export function archiveEnabled(db: Db): boolean { return readS3Settings(db) !== null; } - -function readRow(db: Db): RuntimeSettingsRow { - const row = db - .select() - .from(runtimeSettings) - .where(eq(runtimeSettings.id, SETTINGS_ROW_ID)) - .get(); - if (!row) { - throw new Error( - 'runtime settings row missing — ensureRuntimeSettings must run at boot', - ); - } - return row; -} - -/** - * Applies an updateSettings patch: each provided group replaces that group - * whole, absent groups keep their stored values (shared/settings.ts is the - * arbiter of that contract). Validation — the archive-without-archiver - * refusal, the moving-store guard, the S3 probe — happened at the route; - * this is the pure write. - */ -export function writeRuntimeSettings( - db: Db, - patch: UpdateSettingsRequest, - now: Date, -): RuntimeSettings { - const row = db - .update(runtimeSettings) - .set({ - ...(patch.sandboxDefaults !== undefined - ? { - sandboxCpus: patch.sandboxDefaults.cpus, - sandboxMemoryGb: patch.sandboxDefaults.memoryGb, - sandboxDiskGb: patch.sandboxDefaults.diskGb, - } - : {}), - ...(patch.defaultPolicy !== undefined - ? { - defaultFreezeAfterSeconds: patch.defaultPolicy.freezeAfterSeconds, - defaultStopAfterSeconds: patch.defaultPolicy.stopAfterSeconds, - defaultArchiveAfterSeconds: patch.defaultPolicy.archiveAfterSeconds, - } - : {}), - ...(patch.s3 !== undefined ? s3Columns(patch.s3) : {}), - ...(patch.sandboxDomain !== undefined - ? { sandboxDomain: patch.sandboxDomain ?? OFF } - : {}), - ...(patch.sandboxDomainAliases !== undefined - ? { sandboxDomainAliases: JSON.stringify(patch.sandboxDomainAliases) } - : {}), - ...(patch.pidsLimit !== undefined ? { pidsLimit: patch.pidsLimit } : {}), - updatedAt: now.toISOString(), - }) - .where(eq(runtimeSettings.id, SETTINGS_ROW_ID)) - .returning() - .get(); - if (!row) { - throw new Error( - 'runtime settings row missing — ensureRuntimeSettings must run at boot', - ); - } - return toView(row); -} diff --git a/packages/server/src/db/templates.ts b/packages/server/src/db/templates.ts index 4a224648..06c43dcb 100644 --- a/packages/server/src/db/templates.ts +++ b/packages/server/src/db/templates.ts @@ -3,59 +3,21 @@ import type { Db } from './db'; import { sandboxes, type TemplateRow, templates } from './schema'; /** - * Upsert: registering an existing name re-points it at the new image. That - * is the template upgrade front door — build a new image, re-register the - * name, then rebuildSandbox the stock that should move onto it. - * - * updatedAt is the upgrade timestamp: stamped only when the image actually - * changes. A re-register of the same image writes nothing at all — the - * timestamp must not claim an upgrade that didn't happen. Read-then-write - * needs no lock: better-sqlite3 is synchronous, there is no await between. + * Readers over the node's copy of the templates table (schema.ts). The + * writers live at the gateway — registerTemplate re-points a name, the + * template upgrade front door; removeTemplate asks every node first — and + * the copy is replaced whole with each bundle (db/settings.ts + * applyNodeConfig). Nothing on the node edits a template. */ -export function registerTemplate( - db: Db, - input: { name: string; image: string }, -): TemplateRow { - const now = new Date().toISOString(); - const existing = findTemplate(db, input.name); - if (!existing) { - const row: TemplateRow = { - name: input.name, - image: input.image, - createdAt: now, - updatedAt: now, - }; - db.insert(templates).values(row).run(); - return row; - } - if (existing.image === input.image) { - return existing; - } - db.update(templates) - .set({ image: input.image, updatedAt: now }) - .where(eq(templates.name, input.name)) - .run(); - return { ...existing, image: input.image, updatedAt: now }; -} - -export function listTemplates(db: Db): TemplateRow[] { - return db.select().from(templates).all(); -} export function findTemplate(db: Db, name: string): TemplateRow | undefined { return db.select().from(templates).where(eq(templates.name, name)).get(); } -/** Returns true when a row existed and was removed. */ -export function removeTemplate(db: Db, name: string): boolean { - const existed = findTemplate(db, name) !== undefined; - db.delete(templates).where(eq(templates.name, name)).run(); - return existed; -} - /** - * Names of sandboxes still created from this template — removal is - * refused while this is non-empty, so wakes never resolve a dangling name. + * Names of sandboxes still created from this template — the node's answer + * to the gateway's templateUsers question: removal is refused at the + * gateway while any node names one, so wakes never resolve a dangling name. */ export function sandboxNamesUsingTemplate(db: Db, name: string): string[] { return db @@ -71,8 +33,9 @@ export function sandboxNamesUsingTemplate(db: Db, name: string): string[] { * next shell boots. Null means the base image — expressed as undefined so * the executor falls back to its own configured default. A registered name * resolves to the template's *current* image; a missing row means the - * removal guard was bypassed (ledger drift), which is worth an honest crash, - * not a silent fallback to the wrong image. + * removal guard was bypassed (a template removed while this node was out + * of the fleet), which is worth an honest crash, not a silent fallback to + * the wrong image. */ export function resolveImage( db: Db, diff --git a/packages/server/src/e2b/compat.test.ts b/packages/server/src/e2b/compat.test.ts index 41a0dce6..46b84efc 100644 --- a/packages/server/src/e2b/compat.test.ts +++ b/packages/server/src/e2b/compat.test.ts @@ -19,6 +19,12 @@ import { KeyedQueue } from '../keyed-queue'; import { freezeSandbox, stopSandbox } from '../lifecycle'; import { sampleOnce } from '../metrics-sampler'; import { scanOnce } from '../scanner'; +import { + configureNode, + registerTestTemplate, + TEST_S3, + type TestConfig, +} from '../testing'; import { mintEnvdToken } from './protocol'; import { WatcherTable } from './watcher-table'; @@ -36,6 +42,7 @@ function tickOpts() { function testApp( executor: FakeExecutor = new FakeExecutor(), + configured: TestConfig = {}, env: Record = {}, ) { const db = openDb(':memory:'); @@ -46,6 +53,9 @@ function testApp( DORMICE_API_TOKEN: TOKEN, ...env, }); + // The configuration copy a check-in would have applied (testing.ts); + // `env` is for the node's own identity (its base image), nothing else. + configureNode(db, configured); const locks = new KeyedQueue(); const watchers = new WatcherTable(); const app = buildApp({ @@ -86,14 +96,9 @@ async function createSandbox( return res.json(); } -async function registerTemplate(t: TestApp, name: string, image: string) { - const res = await t.app.inject({ - method: 'POST', - url: '/registerTemplate', - headers: { authorization: `Bearer ${TOKEN}` }, - payload: { name, image }, - }); - expect(res.statusCode).toBe(200); +/** A template as the gateway's registerTemplate then the next bundle would leave it here. */ +function registerTemplate(t: TestApp, name: string, image: string) { + registerTestTemplate(t.db, name, image); } // The envd token derives from the app's ledger-stored signing secret, not @@ -241,106 +246,6 @@ describe('E2B control plane', () => { expect(bare.statusCode).toBe(201); }); - it('a ledger API key opens the X-API-KEY door too, until revoked', async () => { - const t = testApp(); - // Minted over the native face — the same credential truth serves both. - const minted = await t.app.inject({ - method: 'POST', - url: '/createApiKey', - headers: { authorization: `Bearer ${TOKEN}` }, - payload: { name: 'e2b-client' }, - }); - const { token, apiKey } = minted.json(); - - const prefixed = await t.app.inject({ - method: 'POST', - url: '/e2b/api/sandboxes', - headers: { 'x-api-key': `e2b_${token}` }, - payload: {}, - }); - expect(prefixed.statusCode).toBe(201); - - await t.app.inject({ - method: 'POST', - url: '/revokeApiKey', - headers: { authorization: `Bearer ${TOKEN}` }, - payload: { id: apiKey.id }, - }); - const revoked = await t.app.inject({ - method: 'POST', - url: '/e2b/api/sandboxes', - headers: { 'x-api-key': `e2b_${token}` }, - payload: {}, - }); - expect(revoked.statusCode).toBe(401); - expect(revoked.json()).toEqual({ code: 401, message: 'invalid API key' }); - }); - - it('disabled and expired ledger keys are rejected on the X-API-KEY face too', async () => { - const t = testApp(); - const native = { authorization: `Bearer ${TOKEN}` }; - const useE2b = (token: string) => - t.app.inject({ - method: 'POST', - url: '/e2b/api/sandboxes', - headers: { 'x-api-key': `e2b_${token}` }, - payload: {}, - }); - - const parked = ( - await t.app.inject({ - method: 'POST', - url: '/createApiKey', - headers: native, - payload: { name: 'parked' }, - }) - ).json(); - await t.app.inject({ - method: 'POST', - url: '/updateApiKey', - headers: native, - payload: { id: parked.apiKey.id, disabled: true }, - }); - const disabled = await useE2b(parked.token); - expect(disabled.statusCode).toBe(401); - expect(disabled.json()).toEqual({ code: 401, message: 'invalid API key' }); - - const stale = ( - await t.app.inject({ - method: 'POST', - url: '/createApiKey', - headers: native, - payload: { - name: 'stale', - expiresAt: new Date(Date.now() - 1000).toISOString(), - }, - }) - ).json(); - const expired = await useE2b(stale.token); - expect(expired.statusCode).toBe(401); - expect(expired.json()).toEqual({ code: 401, message: 'invalid API key' }); - }); - - it('E2B-created sandboxes attribute to the key that asked', async () => { - const t = testApp(); - const minted = ( - await t.app.inject({ - method: 'POST', - url: '/createApiKey', - headers: { authorization: `Bearer ${TOKEN}` }, - payload: { name: 'e2b-agent' }, - }) - ).json(); - - const created = await t.app.inject({ - method: 'POST', - url: '/e2b/api/sandboxes', - headers: { 'x-api-key': `e2b_${minted.token}` }, - payload: {}, - }); - expect(created.statusCode).toBe(201); - }); - it('creates a fresh sandbox per call — E2B semantics, no key given', async () => { const t = testApp(); const first = await createSandbox(t); @@ -2381,7 +2286,7 @@ describe('signed file URLs at the daemon root', () => { describe('browser-direct signed files: the 49983 subdomain form and CORS', () => { const DOMAIN = 'sbx.dormice.test'; const subdomainApp = () => - testApp(new FakeExecutor(), { DORMICE_SANDBOX_DOMAIN: DOMAIN }); + testApp(new FakeExecutor(), { sandboxDomain: DOMAIN }); it('answers the preflight open and cacheable — it carries no credentials to judge', async () => { const t = testApp(); @@ -2481,13 +2386,8 @@ describe('browser-direct signed files: the 49983 subdomain form and CORS', () => it('an alias domain pins the sandbox at full strength, and never leaks outbound', async () => { const t = subdomainApp(); const ALIAS = 'alias.dormice.test'; - const set = await t.app.inject({ - method: 'POST', - url: '/updateSettings', - headers: { authorization: `Bearer ${TOKEN}` }, - payload: { sandboxDomainAliases: [ALIAS] }, - }); - expect(set.statusCode).toBe(200); + // An alias added at the gateway arrives with the next bundle. + configureNode(t.db, { sandboxDomainAliases: [ALIAS] }); // Aliases are inbound-only: create responses keep the canonical domain. const a = await createSandbox(t); @@ -2691,9 +2591,13 @@ describe('E2B templates', () => { }); it("'base', the configured base image name, and absence all mean the base image", async () => { - const t = testApp(new FakeExecutor(), { - DORMICE_BASE_IMAGE: 'dormice-base:test', - }); + const t = testApp( + new FakeExecutor(), + {}, + { + DORMICE_BASE_IMAGE: 'dormice-base:test', + }, + ); for (const payload of [ {}, { templateID: 'base' }, @@ -2751,9 +2655,9 @@ describe('E2B templates', () => { describe('E2B surface vs the archiver', () => { /** - * testApp plus a MemStore-backed archiver — the S3-configured daemon. - * The env S3 seed flips the ledger's live adjudication (archiveEnabled); - * the MemStore stands in for the S3 those settings describe. + * testApp plus a MemStore-backed archiver — the S3-configured node. The + * copy's S3 store flips the live adjudication (archiveEnabled); the + * MemStore stands in for the S3 those settings describe. */ function archiverTestApp(executor: FakeExecutor = new FakeExecutor()) { const db = openDb(':memory:'); @@ -2762,11 +2666,8 @@ describe('E2B surface vs the archiver', () => { DORMICE_DB_PATH: ':memory:', DORMICE_NODE_ID: 'node-test', DORMICE_API_TOKEN: TOKEN, - DORMICE_S3_ENDPOINT: 'http://127.0.0.1:9000', - DORMICE_S3_BUCKET: 'exam', - DORMICE_S3_ACCESS_KEY_ID: 'exam-key', - DORMICE_S3_SECRET_ACCESS_KEY: 'exam-secret', }); + configureNode(db, { s3: TEST_S3 }); const locks = new KeyedQueue(); const watchers = new WatcherTable(); const store = new MemStore(); diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index f7f7e809..b5109ba0 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -6,6 +6,7 @@ export { type AppDeps, buildApp } from './app'; export { type Config, loadConfig } from './config'; export { type Db, migrateDb, openDb } from './db/db'; +export { applyNodeConfig, readConfigVersion } from './db/settings'; export { DockerExecutor, type DockerExecutorOptions, @@ -16,3 +17,13 @@ export { KeyedQueue, SKIPPED } from './keyed-queue'; export { type SampleResult, sampleOnce } from './metrics-sampler'; export { type ReconcileResult, reconcile } from './reconciler'; export { type ScanResult, scanOnce } from './scanner'; +// Test scaffolding for the suites that embed the daemon (sdk, cli): a +// configuration bundle applied the way a check-in would — the daemon reads +// every knob from its copy, so a test configures the node, not an env. +export { + configureNode, + registerTestTemplate, + TEST_S3, + type TestConfig, + testBundle, +} from './testing'; diff --git a/packages/server/src/ingress.test.ts b/packages/server/src/ingress.test.ts deleted file mode 100644 index daa2ebed..00000000 --- a/packages/server/src/ingress.test.ts +++ /dev/null @@ -1,279 +0,0 @@ -import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { getIngressResponseSchema } from '@dormice/shared'; -import { describe, expect, it } from 'vitest'; -import { buildApp } from './app'; -import { loadConfig } from './config'; -import { migrateDb, openDb } from './db/db'; -import { FakeExecutor } from './executor/fake'; -import { Ingress } from './ingress'; -import { KeyedQueue } from './keyed-queue'; - -// The managed front door: the config file is the single source of truth, -// so most of what matters is what ends up in the file — and that a failed -// reload never leaves file and running proxy telling different stories. - -const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); -const TOKEN = 'test-token-test-token-test-token'; -const authed = { authorization: `Bearer ${TOKEN}` }; - -function tmpFile(): string { - return path.join( - mkdtempSync(path.join(tmpdir(), 'dormice-ingress-')), - 'Caddyfile', - ); -} - -function testIngress( - overrides: Partial[0]> = {}, -) { - const filePath = overrides.filePath ?? tmpFile(); - const reloads: string[] = []; - const ingress = new Ingress({ - filePath, - upstreamPort: 3676, - runCommand: async (command) => { - reloads.push(command); - return { ok: true, stderr: '' }; - }, - resolveDomain: async () => ({ dnsAddresses: ['1.2.3.4'], dnsError: null }), - probeTls: async () => ({ tlsOk: true, tlsError: null }), - ...overrides, - }); - return { ingress, filePath, reloads }; -} - -describe('Ingress file round-trip', () => { - it('binds domains: marker, one site each, :80 catch-all — and reads them back in order', async () => { - const { ingress, filePath, reloads } = testIngress(); - await ingress.setDomains(['console.example.com', 'api.example.com']); - const content = readFileSync(filePath, 'utf8'); - expect(content.startsWith('# Managed by Dormice')).toBe(true); - expect(content).toContain('console.example.com {'); - expect(content).toContain('api.example.com {'); - // The no-lockout guarantee: IP access survives every bind. - expect(content).toContain(':80 {'); - expect(content).toContain('reverse_proxy 127.0.0.1:3676'); - expect(content).toContain('flush_interval -1'); - expect(ingress.domains()).toEqual([ - 'console.example.com', - 'api.example.com', - ]); - expect(reloads).toHaveLength(1); - }); - - it('lowercases and dedups: hostnames are case-insensitive, the file decides once', async () => { - const { ingress } = testIngress(); - await ingress.setDomains([ - 'Console.Example.COM', - 'console.example.com', - 'api.example.com', - ]); - expect(ingress.domains()).toEqual([ - 'console.example.com', - 'api.example.com', - ]); - }); - - it('clears back to IP-only, and reports empty before any file exists', async () => { - const { ingress, filePath } = testIngress(); - expect(ingress.domains()).toEqual([]); - await ingress.setDomains(['console.example.com']); - await ingress.setDomains([]); - const content = readFileSync(filePath, 'utf8'); - expect(content).not.toContain('example.com'); - expect(content).toContain(':80 {'); - expect(ingress.domains()).toEqual([]); - }); - - it('defaults the reload command to caddy reload against its own file', async () => { - const filePath = tmpFile(); - const reloads: string[] = []; - const ingress = new Ingress({ - filePath, - upstreamPort: 3676, - runCommand: async (command) => { - reloads.push(command); - return { ok: true, stderr: '' }; - }, - }); - await ingress.setDomains([]); - expect(reloads).toEqual([ - `caddy reload --config ${filePath} --adapter caddyfile`, - ]); - }); - - it('serializes concurrent binds: the last write wins, reloads never interleave', async () => { - const filePath = tmpFile(); - const order: string[] = []; - const ingress = new Ingress({ - filePath, - upstreamPort: 3676, - runCommand: async () => { - order.push( - `reload:${readFileSync(filePath, 'utf8').includes('a.example.com') ? 'a' : 'b'}`, - ); - await new Promise((resolve) => setTimeout(resolve, 10)); - return { ok: true, stderr: '' }; - }, - }); - await Promise.all([ - ingress.setDomains(['a.example.com']), - ingress.setDomains(['b.example.com']), - ]); - // Each reload observed the file its own write produced — the second - // write waited for the first reload instead of racing it. - expect(order).toEqual(['reload:a', 'reload:b']); - expect(ingress.domains()).toEqual(['b.example.com']); - }); -}); - -describe('Ingress refusals and rollback', () => { - it('refuses to overwrite a file it did not write', async () => { - const filePath = tmpFile(); - writeFileSync(filePath, 'example.org {\n\trespond "mine"\n}\n'); - const { ingress } = testIngress({ filePath }); - await expect(ingress.setDomains(['console.example.com'])).rejects.toThrow( - /not written by Dormice/, - ); - // The foreign file is untouched, and not ours to report domains from. - expect(readFileSync(filePath, 'utf8')).toContain('respond "mine"'); - expect(ingress.domains()).toEqual([]); - }); - - it('restores the previous file when the reload fails', async () => { - const { ingress, filePath } = testIngress(); - await ingress.setDomains(['good.example.com']); - const { ingress: failing } = testIngress({ - filePath, - runCommand: async () => ({ ok: false, stderr: 'adapting config: no' }), - }); - await expect(failing.setDomains(['bad.example.com'])).rejects.toThrow( - /adapting config: no/, - ); - expect(readFileSync(filePath, 'utf8')).toContain('good.example.com'); - }); - - it('removes the file it just created when the first-ever reload fails', async () => { - const { ingress, filePath } = testIngress({ - runCommand: async () => ({ ok: false, stderr: 'caddy not running' }), - }); - await expect(ingress.setDomains(['x.example.com'])).rejects.toThrow( - /caddy not running/, - ); - expect(existsSync(filePath)).toBe(false); - }); -}); - -describe('ingress routes', () => { - function testApp(ingress?: Ingress) { - const db = openDb(':memory:'); - migrateDb(db, MIGRATIONS); - const config = loadConfig({ - DORMICE_DB_PATH: ':memory:', - DORMICE_API_TOKEN: TOKEN, - }); - const app = buildApp({ - config, - db, - executor: new FakeExecutor(), - locks: new KeyedQueue(), - logger: false, - ingress, - }); - return app; - } - - const rpc = ( - app: ReturnType, - url: string, - payload: Record = {}, - ) => app.inject({ method: 'POST', url, headers: authed, payload }); - - it('without a managed ingress: getIngress is honest, setIngress refuses', async () => { - const app = testApp(); - const get = await rpc(app, '/getIngress'); - expect(get.statusCode).toBe(200); - expect(getIngressResponseSchema.parse(get.json())).toEqual({ - managed: false, - domains: [], - }); - const set = await rpc(app, '/setIngress', { - domains: ['a.example.com'], - }); - expect(set.statusCode).toBe(400); - expect(set.json().message).toContain('DORMICE_INGRESS_FILE'); - }); - - it('binds two, probes each, drops one, records the diffs, clears', async () => { - const { ingress } = testIngress(); - const app = testApp(ingress); - - const set = await rpc(app, '/setIngress', { - domains: ['console.example.com', 'api.example.com'], - }); - expect(set.statusCode).toBe(200); - expect(set.json()).toEqual({ - domains: ['console.example.com', 'api.example.com'], - }); - - const get = await rpc(app, '/getIngress'); - const status = getIngressResponseSchema.parse(get.json()); - const probe = { - dnsAddresses: ['1.2.3.4'], - dnsError: null, - tlsOk: true, - tlsError: null, - }; - expect(status).toEqual({ - managed: true, - domains: [ - { domain: 'console.example.com', probe }, - { domain: 'api.example.com', probe }, - ], - }); - - const dropped = await rpc(app, '/setIngress', { - domains: ['api.example.com'], - }); - expect(dropped.json()).toEqual({ domains: ['api.example.com'] }); - - const cleared = await rpc(app, '/setIngress', { domains: [] }); - expect(cleared.statusCode).toBe(200); - expect(cleared.json()).toEqual({ domains: [] }); - }); - - it('rejects a domain with a scheme at the schema gate', async () => { - const { ingress } = testIngress(); - const app = testApp(ingress); - const res = await rpc(app, '/setIngress', { - domains: ['https://console.example.com'], - }); - expect(res.statusCode).toBe(400); - expect(res.json().message).toContain('bare hostname'); - }); - - it('maps a foreign file to 409 and a failed reload to 500, with the reason', async () => { - const filePath = tmpFile(); - writeFileSync(filePath, 'someone-elses-site {\n}\n'); - const foreign = testApp(testIngress({ filePath }).ingress); - const refused = await rpc(foreign, '/setIngress', { - domains: ['a.example.com'], - }); - expect(refused.statusCode).toBe(409); - expect(refused.json().message).toContain('not written by Dormice'); - - const broken = testApp( - testIngress({ - runCommand: async () => ({ ok: false, stderr: 'connection refused' }), - }).ingress, - ); - const failed = await rpc(broken, '/setIngress', { - domains: ['a.example.com'], - }); - expect(failed.statusCode).toBe(500); - expect(failed.json().message).toContain('connection refused'); - }); -}); diff --git a/packages/server/src/ingress.ts b/packages/server/src/ingress.ts deleted file mode 100644 index 07345b26..00000000 --- a/packages/server/src/ingress.ts +++ /dev/null @@ -1,239 +0,0 @@ -import { Resolver } from 'node:dns/promises'; -import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; -import tls from 'node:tls'; -import type { GetIngressResponse, IngressProbe } from '@dormice/shared'; -import { execa } from 'execa'; - -/** - * The daemon's own front door: a Caddy config file it owns, rewritten on - * setIngress and reloaded into the running Caddy. The file is the single - * source of truth — no ledger column, no boot-time reconcile: an operator - * can read it, Caddy runs it, and getIngress parses it back. TLS is - * entirely Caddy's job (ACME issuance and renewal); this class only decides - * what the file says. - * - * The generated shape is one site block per bound domain (Caddy obtains a - * certificate per domain and auto-redirects http:// to https) plus - * a plain :80 catch-all proxying by IP. The catch-all is the no-lockout - * guarantee — a bind that never converges (typo'd domain, missing DNS - * record) leaves IP access untouched. Caddy inserts its automatic HTTPS - * redirects after host-matched routes but before user catch-alls, so the - * domain sites and the catch-all coexist. - */ - -/** - * Ownership marker, line one of every file this class writes. A file the - * knob points at that lacks it was written by someone else — refused, never - * overwritten. install.sh writes the same marker (kept in sync by hand). - */ -const MARKER = '# Managed by Dormice — setIngress rewrites this file.'; - -/** The knob points at a file this daemon does not own. */ -export class UnmanagedIngressFileError extends Error {} - -export interface IngressOptions { - /** The Caddy config file the daemon owns (DORMICE_INGRESS_FILE). */ - filePath: string; - /** Where the proxy forwards to: the daemon's own loopback port. */ - upstreamPort: number; - /** - * Shell command that makes the running proxy read the new config. - * Defaults to `caddy reload` against the standard Caddyfile — right when - * the daemon owns the whole file; an operator whose own Caddyfile imports - * a fragment overrides this to reload the outer file instead. - */ - reloadCommand?: string; - /** Test seam; production shells out through execa. */ - runCommand?: (command: string) => Promise<{ ok: boolean; stderr: string }>; - /** Test seams for the two probes; production asks DNS and 127.0.0.1:443. */ - resolveDomain?: ( - domain: string, - ) => Promise>; - probeTls?: ( - domain: string, - ) => Promise>; -} - -export class Ingress { - private readonly filePath: string; - private readonly upstreamPort: number; - private readonly reloadCommand: string; - private readonly runCommand: NonNullable; - private readonly resolveDomain: NonNullable; - private readonly probeTls: NonNullable; - /** Serializes writes: two concurrent binds must not interleave write+reload. */ - private queue: Promise = Promise.resolve(); - - constructor(options: IngressOptions) { - this.filePath = options.filePath; - this.upstreamPort = options.upstreamPort; - this.reloadCommand = - options.reloadCommand ?? - `caddy reload --config ${options.filePath} --adapter caddyfile`; - this.runCommand = options.runCommand ?? runShellCommand; - this.resolveDomain = options.resolveDomain ?? resolveDomainDns; - this.probeTls = options.probeTls ?? probeLocalTls; - } - - /** - * The currently bound domains, read back from the file in the order they - * are served. Empty when nothing is bound, the file does not exist yet, - * or the file is not ours — a foreign file's site addresses are not this - * daemon's to report. - */ - domains(): string[] { - if (!existsSync(this.filePath)) return []; - const content = readFileSync(this.filePath, 'utf8'); - if (!content.includes('Managed by Dormice')) return []; - const found: string[] = []; - for (const line of content.split('\n')) { - // Site addresses sit at column 0 in the generated shape; indented - // lines are directives (`\treverse_proxy … {`), never sites. The - // leading-`:` exclusion keeps the :80 catch-all out of the answer. - const site = /^([^\s#:{][^\s{]*)\s*\{/.exec(line); - if (site?.[1]) found.push(site[1]); - } - return found; - } - - /** - * Rewrites the file to serve exactly the given set (empty = back to - * IP-only) and reloads the proxy. Hostnames are case-insensitive, so the - * set is lowercased and deduped here — the one place that decides what - * the file says. On a failed reload the previous file is restored — - * `caddy reload` rejects a bad config without applying it, so file and - * running proxy stay consistent — and the failure is thrown with Caddy's - * own words. - */ - setDomains(domains: string[]): Promise { - const wanted = [...new Set(domains.map((domain) => domain.toLowerCase()))]; - const run = this.queue.then(() => this.apply(wanted)); - this.queue = run.catch(() => {}); - return run; - } - - /** Everything getIngress reports: the file's word plus live probes. */ - async status(): Promise { - const statuses = await Promise.all( - this.domains().map(async (domain) => { - const [dns, cert] = await Promise.all([ - this.resolveDomain(domain), - this.probeTls(domain), - ]); - return { domain, probe: { ...dns, ...cert } }; - }), - ); - return { managed: true, domains: statuses }; - } - - private async apply(domains: string[]): Promise { - const previous = existsSync(this.filePath) - ? readFileSync(this.filePath, 'utf8') - : null; - if ( - previous !== null && - previous.trim() !== '' && - !previous.includes('Managed by Dormice') - ) { - throw new UnmanagedIngressFileError( - `${this.filePath} was not written by Dormice — refusing to overwrite it; ` + - 'move your configuration elsewhere, or point DORMICE_INGRESS_FILE at a file the daemon may own', - ); - } - writeFileSync(this.filePath, this.render(domains)); - const reload = await this.runCommand(this.reloadCommand); - if (!reload.ok) { - if (previous === null) unlinkSync(this.filePath); - else writeFileSync(this.filePath, previous); - throw new Error( - `reloading the proxy failed (${this.reloadCommand}): ${reload.stderr.trim()} — the previous configuration was restored`, - ); - } - } - - private render(domains: string[]): string { - // flush_interval -1 streams byte-by-byte: buffering would dam the - // console terminal and E2B's streaming exec (measured through Caddy). - const site = (address: string) => - `${address} {\n\treverse_proxy 127.0.0.1:${this.upstreamPort} {\n\t\tflush_interval -1\n\t}\n}\n`; - return [`${MARKER}\n`, ...domains.map(site), site(':80')].join('\n'); - } -} - -async function runShellCommand( - command: string, -): Promise<{ ok: boolean; stderr: string }> { - try { - await execa(command, { shell: true, timeout: 30_000 }); - return { ok: true, stderr: '' }; - } catch (error) { - const stderr = - error instanceof Error - ? ('stderr' in error && String(error.stderr)) || error.message - : String(error); - return { ok: false, stderr }; - } -} - -/** - * What the domain resolves to right now. "No record" (the state before the - * operator's A record lands or propagates) is an empty list, not an error; - * dnsError is reserved for the resolver itself failing. - */ -async function resolveDomainDns( - domain: string, -): Promise> { - const resolver = new Resolver({ timeout: 3_000, tries: 1 }); - const lookup = async (kind: 'resolve4' | 'resolve6') => { - try { - return await resolver[kind](domain); - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code === 'ENOTFOUND' || code === 'ENODATA') return []; - throw error; - } - }; - try { - const [v4, v6] = await Promise.all([ - lookup('resolve4'), - lookup('resolve6'), - ]); - return { dnsAddresses: [...v4, ...v6], dnsError: null }; - } catch (error) { - return { - dnsAddresses: [], - dnsError: error instanceof Error ? error.message : String(error), - }; - } -} - -/** - * Does the local proxy serve a valid trusted certificate for the domain? - * A handshake against 127.0.0.1:443 with the domain as SNI — deliberately - * not a fetch of the public URL: cloud NAT usually cannot hairpin a host's - * own public IP, so the honest local fact is "certificate issued and - * served", and public reachability stays the security group's question. - */ -function probeLocalTls( - domain: string, -): Promise> { - return new Promise((resolve) => { - const socket = tls.connect({ - host: '127.0.0.1', - port: 443, - servername: domain, - rejectUnauthorized: true, - }); - const done = (result: Pick) => { - socket.destroy(); - resolve(result); - }; - socket.setTimeout(3_000, () => - done({ tlsOk: false, tlsError: 'timed out connecting to 127.0.0.1:443' }), - ); - socket.once('secureConnect', () => done({ tlsOk: true, tlsError: null })); - socket.once('error', (error) => - done({ tlsOk: false, tlsError: error.message }), - ); - }); -} diff --git a/packages/server/src/login-throttle.test.ts b/packages/server/src/login-throttle.test.ts deleted file mode 100644 index 35ae2491..00000000 --- a/packages/server/src/login-throttle.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { LoginThrottle } from './login-throttle'; - -const T0 = 1_700_000_000_000; - -describe('LoginThrottle', () => { - it('allows the free failures without any delay', () => { - const throttle = new LoginThrottle(); - for (let i = 0; i < 5; i++) { - expect(throttle.retryAfterSeconds('ip', T0)).toBe(0); - throttle.recordFailure('ip', T0); - } - expect(throttle.retryAfterSeconds('ip', T0)).toBe(0); - }); - - it('backs off exponentially past the free failures, capped', () => { - const throttle = new LoginThrottle(); - for (let i = 0; i < 5; i++) throttle.recordFailure('ip', T0); - throttle.recordFailure('ip', T0); // 6th: 1s - expect(throttle.retryAfterSeconds('ip', T0)).toBe(1); - throttle.recordFailure('ip', T0); // 7th: 2s - expect(throttle.retryAfterSeconds('ip', T0)).toBe(2); - for (let i = 0; i < 20; i++) throttle.recordFailure('ip', T0); - // Deep in: capped at 5 minutes, never unbounded. - expect(throttle.retryAfterSeconds('ip', T0)).toBe(300); - }); - - it('the delay drains with time', () => { - const throttle = new LoginThrottle(); - for (let i = 0; i < 7; i++) throttle.recordFailure('ip', T0); - expect(throttle.retryAfterSeconds('ip', T0 + 500)).toBe(2); - expect(throttle.retryAfterSeconds('ip', T0 + 2_000)).toBe(0); - }); - - it('success clears the counter; keys are independent', () => { - const throttle = new LoginThrottle(); - for (let i = 0; i < 8; i++) throttle.recordFailure('a', T0); - expect(throttle.retryAfterSeconds('a', T0)).toBeGreaterThan(0); - expect(throttle.retryAfterSeconds('b', T0)).toBe(0); - throttle.clear('a'); - expect(throttle.retryAfterSeconds('a', T0)).toBe(0); - }); - - it('forgets idle counters after an hour', () => { - const throttle = new LoginThrottle(); - for (let i = 0; i < 8; i++) throttle.recordFailure('ip', T0); - const later = T0 + 61 * 60 * 1000; - expect(throttle.retryAfterSeconds('ip', later)).toBe(0); - // And the slate is genuinely clean: the next failure is a free one. - throttle.recordFailure('ip', later); - expect(throttle.retryAfterSeconds('ip', later)).toBe(0); - }); -}); diff --git a/packages/server/src/login-throttle.ts b/packages/server/src/login-throttle.ts deleted file mode 100644 index ad060cef..00000000 --- a/packages/server/src/login-throttle.ts +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Failure backoff for the console's credential endpoints. Passwords are - * human-chosen (low entropy, unlike the 128-bit API token), so online - * guessing became a real surface the moment they arrived — this is the - * counterpart the password design must ship with. - * - * In-memory on purpose: crash-only means a restart honestly forgets the - * counters, and persisting attacker state in the ledger would be a second - * database to babysit for no real gain. Keyed by client IP; behind the - * usual single reverse proxy every client shares 127.0.0.1, collapsing this - * into a global throttle — acceptable and arguably right for a single - * account (it is the account being guessed at, not the caller). - */ - -/** Free attempts before delays start: typos are not attacks. */ -const FREE_FAILURES = 5; -/** Delay doubles per failure past the free ones, capped here. */ -const MAX_DELAY_SECONDS = 300; -/** Counters idle this long are forgotten (also the sweep horizon). */ -const FORGET_AFTER_MS = 60 * 60 * 1000; - -interface Entry { - failures: number; - /** Epoch ms before which further attempts are refused. */ - blockedUntil: number; - lastFailureAt: number; -} - -export class LoginThrottle { - private entries = new Map(); - - /** Seconds the caller must still wait, or 0 when an attempt is allowed. */ - retryAfterSeconds(key: string, nowMs = Date.now()): number { - const entry = this.entries.get(key); - if (!entry) return 0; - if (nowMs - entry.lastFailureAt > FORGET_AFTER_MS) { - this.entries.delete(key); - return 0; - } - return Math.max(0, Math.ceil((entry.blockedUntil - nowMs) / 1000)); - } - - recordFailure(key: string, nowMs = Date.now()): void { - this.sweep(nowMs); - const entry = this.entries.get(key) ?? { - failures: 0, - blockedUntil: 0, - lastFailureAt: 0, - }; - entry.failures += 1; - entry.lastFailureAt = nowMs; - const past = entry.failures - FREE_FAILURES; - if (past > 0) { - const delay = Math.min(2 ** (past - 1), MAX_DELAY_SECONDS); - entry.blockedUntil = nowMs + delay * 1000; - } - this.entries.set(key, entry); - } - - clear(key: string): void { - this.entries.delete(key); - } - - /** - * Drops idle counters on the write path — no timer to manage, and the map - * stays bounded by "distinct keys failing within the last hour", which a - * loopback-bound daemon can always afford. - */ - private sweep(nowMs: number): void { - for (const [key, entry] of this.entries) { - if (nowMs - entry.lastFailureAt > FORGET_AFTER_MS) { - this.entries.delete(key); - } - } - } -} diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index 5f290ea5..cbbd4f52 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -6,12 +6,14 @@ import { buildApp } from './app'; import { Archiver } from './archive/archiver'; import { LedgerArchiveStore } from './archive/ledger-store'; import { CheckIn, readNodeReading } from './check-in'; -import { type Config, loadConfig } from './config'; +import { type Config, ignoredEnvKeys, loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; import { listSandboxes } from './db/ledger'; import { acquireSingleWriterLock } from './db/lock'; import { - ensureRuntimeSettings, + readConfigAppliedAt, + readConfigVersion, + readNodeConfig, readRuntimeSettings, readSwapTarget, } from './db/settings'; @@ -21,9 +23,9 @@ import type { Executor } from './executor/executor'; import { FakeExecutor } from './executor/fake'; import { gib, HostDiskGrower } from './host-disk'; import { CpuSampler } from './host-metrics'; -import { Ingress } from './ingress'; import { KeyedQueue } from './keyed-queue'; import { sampleOnce } from './metrics-sampler'; +import { applyConfig } from './node-config'; import { sweepPidsLimit } from './pids-sweep'; import { reconcile } from './reconciler'; import { scanOnce } from './scanner'; @@ -46,6 +48,18 @@ function fatal(message: string): never { const config = loadConfig(); +// The fleet's operator knobs left the node's environment for the gateway +// (config.ts MOVED_TO_GATEWAY). An env file that still carries them is a +// machine upgraded across the move: say so once, by name, rather than +// silently run on other values than the operator wrote. +const ignored = ignoredEnvKeys(); +if (ignored.length > 0) { + log.warn( + { ignored }, + `${ignored.length} environment variable${ignored.length === 1 ? '' : 's'} moved to the gateway and ${ignored.length === 1 ? 'does' : 'do'} nothing on a node: ${ignored.join(', ')} — the fleet's settings are the gateway's (its env seeds them once; the console edits them); remove ${ignored.length === 1 ? 'it' : 'them'} from this node's env file`, + ); +} + // One ledger, one daemon — enforced, not assumed. A second instance would // run its own destructive reconcile against sandboxes this one is still // operating, well before it ever loses the race for the port. The handle @@ -67,12 +81,6 @@ if (config.DORMICE_DB_PATH !== ':memory:') { const db = openDb(config.DORMICE_DB_PATH); migrateDb(db, fileURLToPath(new URL('../drizzle', import.meta.url))); -// Seed the runtime settings before anything can read a knob (the executor -// reads them at every disk/container birth). The archive adjudication — -// "env S3 seed present means new sandboxes archive after a week" — lives -// inside ensure now, next to the seeding it belongs to. -ensureRuntimeSettings(db, config); - function buildExecutor(cfg: Config, log: (msg: string) => void): Executor { // Live from the ledger: a console edit reaches the next birth directly. // Both executors get the same view — the fake on constants would diverge @@ -142,9 +150,9 @@ const locks = new KeyedQueue(); const watchers = new WatcherTable(); // One Archiver for the daemon's whole life — its restore tracker is daemon -// memory and must survive settings edits; whether archiving is available -// is the store provider's live answer from the ledger, not a boot fact. -// Temp transfers stage next to the disks (same filesystem — they are +// memory and must survive configuration changes; whether archiving is +// available is the store provider's live answer from the copy, not a boot +// fact. Temp transfers stage next to the disks (same filesystem — they are // disk-sized, and /tmp may be RAM). const archiver = new Archiver({ db, @@ -155,81 +163,57 @@ const archiver = new Archiver({ log: (msg) => log.info(msg), watchers, }); -if (archiver.enabled()) { - await archiver.init(); - const s3View = readRuntimeSettings(db).s3; - log.info(`archiver enabled: bucket ${s3View?.bucket} at ${s3View?.endpoint}`); -} else { - log.info( - 'archiver disabled: no S3 store in the ledger settings (configure one in the console)', - ); -} - -// The managed front door exists exactly when its file knob is set (the -// archiver's rule). The file itself is the source of truth for the bound -// domains — nothing to reconcile at boot, Caddy is already running it. -let ingress: Ingress | undefined; -if (config.DORMICE_INGRESS_FILE) { - ingress = new Ingress({ - filePath: config.DORMICE_INGRESS_FILE, - upstreamPort: config.DORMICE_PORT, - reloadCommand: config.DORMICE_INGRESS_RELOAD_CMD, - }); - const domains = ingress.domains(); - log.info( - `ingress managed at ${config.DORMICE_INGRESS_FILE}: ${domains.length ? domains.join(', ') : 'no domain bound (IP access only)'}`, - ); -} else { - log.info('ingress not managed: DORMICE_INGRESS_FILE not configured'); -} // Managed swap exists exactly where the daemon can honor it: a Linux host // (swapon is the kernel's) running the docker executor (the fake executor // is a test double — e2e boots real daemons with it, and those must never -// touch the host's swap). The boot reconcile is what makes shrink-by- -// reboot converge and puts grown blocks back after a restart; its failure -// is loud but not fatal — swap is capacity, not correctness. The target is -// this node's row of the fleet configuration (the gateway's -// updateNodeSettings), applied here at boot and, once the node pulls its -// configuration, whenever the bundle moves it. +// touch the host's swap). Built here, reconciled below once the target is +// known: the target is this node's row of the fleet configuration (the +// gateway's updateNodeSettings), and the reading the check-in carries +// says whether this daemon manages swap at all — null here is how the +// gateway knows to refuse a target for this node. let swap: SwapManager | undefined; if (config.DORMICE_EXECUTOR === 'docker' && process.platform === 'linux') { swap = new SwapManager({ dir: path.join(config.DORMICE_DATA_DIR, 'swap'), log: (msg) => log.info(msg), }); - try { - await swap.reconcile(readSwapTarget(db)); - } catch (error) { - log.error(error, 'boot swap reconcile failed'); - } } else { log.info('managed swap unavailable: requires Linux + the docker executor'); } -// Same eligibility as managed swap, same reasoning: the data-disk auto-grow -// touches the host, so only a real deployment (Linux + docker executor) -// gets one — e2e daemons on the fake executor must never run resize2fs on -// a developer's machine. Within that gate host-disk.ts judges the layout -// itself and declines anything it cannot fully reason about. -const diskGrower = - config.DORMICE_EXECUTOR === 'docker' && process.platform === 'linux' - ? new HostDiskGrower({ - dataDir: config.DORMICE_DATA_DIR, - log: (msg) => log.info(msg), - }) - : undefined; +// The build identity, for the check-in and the upgrade window below. +const build = readBuildInfo(); -// The web console ships beside the server in the monorepo; this file sits -// one level under packages/server both as src/main.ts and as dist/main.js, -// so the relative hop to packages/console/dist is the same either way. A -// missing dist is loud but not fatal: the API works without the console. -const consoleDistDir = fileURLToPath( - new URL('../../console/dist', import.meta.url), -); -if (!existsSync(consoleDistDir)) { - log.warn(`web console not found at ${consoleDistDir} — /console disabled`); -} +// This node's check-in with its gateway (check-in.ts): its readings, its +// build, where it can be reached, and which configuration version it +// runs — and the fleet's configuration comes back with the answer. Every +// daemon is a node of a gateway (design record #22: a single machine is a +// fleet of one, the gateway on 127.0.0.1:3677 by default). The CpuSampler +// is its own — a delta spans "since this instance's last sample", and the +// route's and the metrics ticker's windows must not be stolen +// (host-metrics.ts). Not primed: the first check-in then reports +// cpuUsedPct null — "no interval yet" — which placement lets through as +// unknown; a sample a few milliseconds before it would make that first +// reading a percentage over the sliver in between, near 0 or near 100 by +// luck (found by review, 2026-09-14). +const nodeEndpoint = + config.DORMICE_NODE_ENDPOINT ?? `http://127.0.0.1:${config.DORMICE_PORT}`; +const checkInCpu = new CpuSampler(); +const checkIn = new CheckIn({ + gateway: config.DORMICE_GATEWAY_ENDPOINT, + token: config.DORMICE_API_TOKEN, + nodeId: config.DORMICE_NODE_ID, + endpoint: nodeEndpoint, + intervalSeconds: config.DORMICE_CHECK_IN_INTERVAL_SECONDS, + build, + readReading: () => + readNodeReading(db, checkInCpu, config.DORMICE_DATA_DIR, swap), + configVersion: () => readConfigVersion(db), + applyConfig: (bundle) => + applyConfig(bundle, { db, executor, locks, swap, log, beat }), + log, +}); // The daemon's own upgrade window compares the commit baked into this // build against the checkout it runs from — main.js sits at @@ -237,7 +221,6 @@ if (!existsSync(consoleDistDir)) { // so three hops up is the repo root either way. No checkout (a dist // copied elsewhere) means checking is honestly unavailable, not guessed. const repoRoot = fileURLToPath(new URL('../../..', import.meta.url)); -const build = readBuildInfo(); const updater = new Updater({ repoDir: existsSync(path.join(repoRoot, '.git')) ? repoRoot : null, build, @@ -256,9 +239,7 @@ const app = buildApp({ executor, locks, logger: log, - consoleDistDir: existsSync(consoleDistDir) ? consoleDistDir : undefined, archiver, - ingress, updater, watchers, }); @@ -277,6 +258,72 @@ if (refusal !== null) { fatal(refusal); } +// A node without a configuration copy has nothing to build a sandbox from +// — no defaults, no templates, no archive store — so it does not listen +// until it holds one: one check-in now, then one per interval, until the +// gateway answers with the bundle. After the startup guard on purpose: a +// daemon that will refuse to start must not first register itself with +// the fleet. A first install waits for its gateway here (install.sh +// starts the gateway first); a machine upgraded across the move +// (2026-09-14) has its old single-machine settings in the row but no +// copy, and takes the gateway's bundle the same way. The check-ins sent +// meanwhile report "no configuration", which keeps the gateway from +// placing sandboxes here before the port is open (gateway placement.ts): +// listNodes shows such a node with configVersion null until its first +// check-in after listen. A node that already holds a copy runs it, stale +// or not, and takes the current one at that check-in — a node whose +// gateway is away still serves; that is what the copy is for. +if (readConfigVersion(db) === null) { + log.info( + `no configuration copy in the ledger — asking gateway ${config.DORMICE_GATEWAY_ENDPOINT} before anything else (retrying every ${config.DORMICE_CHECK_IN_INTERVAL_SECONDS}s until it answers)`, + ); + await checkIn.untilConfigured(); +} +{ + const copy = readNodeConfig(db); + log.info( + { + version: copy.version, + appliedAt: readConfigAppliedAt(db), + templates: copy.templates.length, + pidsLimit: copy.settings.pidsLimit, + swapGb: copy.node.swapGb, + sandboxDomain: copy.settings.sandboxDomain, + }, + copy.settings.s3 === null + ? `running configuration v${copy.version}; archiver disabled: no S3 store in the fleet settings (configure one in the console)` + : `running configuration v${copy.version}; archiver enabled: bucket ${copy.settings.s3.bucket} at ${copy.settings.s3.endpoint}`, + ); +} +if (archiver.enabled()) { + await archiver.init(); +} + +// The boot swap reconcile is what makes shrink-by-reboot converge and puts +// grown blocks back after a restart; its failure is loud but not fatal — +// swap is capacity, not correctness. Later moves of the target arrive +// with a bundle (node-config.ts). +if (swap !== undefined) { + try { + await swap.reconcile(readSwapTarget(db)); + } catch (error) { + log.error(error, 'boot swap reconcile failed'); + } +} + +// Same eligibility as managed swap, same reasoning: the data-disk auto-grow +// touches the host, so only a real deployment (Linux + docker executor) +// gets one — e2e daemons on the fake executor must never run resize2fs on +// a developer's machine. Within that gate host-disk.ts judges the layout +// itself and declines anything it cannot fully reason about. +const diskGrower = + config.DORMICE_EXECUTOR === 'docker' && process.platform === 'linux' + ? new HostDiskGrower({ + dataDir: config.DORMICE_DATA_DIR, + log: (msg) => log.info(msg), + }) + : undefined; + // Repair ledger/reality drift left by a crash — before serving traffic, so // every request runs against a ledger that reflects what actually exists. // A failure here is fatal on purpose: a daemon that cannot read reality @@ -317,40 +364,14 @@ app.log.info( // the daemon to the outside world is a reverse proxy's job. await app.listen({ host: '127.0.0.1', port: config.DORMICE_PORT }); -// A node of a fleet reports to its gateway; a daemon on its own reports to -// nobody. Started after listen on purpose: the check-in names where the -// gateway may forward to, and that door must be open before the gateway -// hears of it. Its CpuSampler is its own — a delta spans "since this -// instance's last sample", and the route's and the metrics ticker's -// windows must not be stolen (host-metrics.ts). -let checkIn: CheckIn | undefined; -if (config.DORMICE_GATEWAY_ENDPOINT !== undefined) { - const nodeEndpoint = - config.DORMICE_NODE_ENDPOINT ?? `http://127.0.0.1:${config.DORMICE_PORT}`; - // Not primed: the first check-in then reports cpuUsedPct null — "no - // interval yet" — which placement lets through as unknown. A sample a - // few milliseconds before it would make that first reading a percentage - // over the sliver in between, near 0 or near 100 by luck, and a freshly - // restarted node could sit out its first interval on a number that - // meant nothing (found by review, 2026-09-14). - const checkInCpu = new CpuSampler(); - checkIn = new CheckIn({ - gateway: config.DORMICE_GATEWAY_ENDPOINT, - token: config.DORMICE_API_TOKEN, - nodeId: config.DORMICE_NODE_ID, - endpoint: nodeEndpoint, - intervalSeconds: config.DORMICE_CHECK_IN_INTERVAL_SECONDS, - build, - readReading: () => readNodeReading(db, checkInCpu, config.DORMICE_DATA_DIR), - log, - }); - checkIn.start(); - log.info( - `node ${config.DORMICE_NODE_ID} checks in with gateway ${config.DORMICE_GATEWAY_ENDPOINT} every ${config.DORMICE_CHECK_IN_INTERVAL_SECONDS}s, reachable at ${nodeEndpoint}`, - ); -} else { - log.info('no gateway: standalone daemon (DORMICE_GATEWAY_ENDPOINT unset)'); -} +// The check-in ticker starts after listen on purpose: a check-in names +// where the gateway may forward to and, from now on, reports a +// configuration copy — placement's cue that this node is open for +// business — so that door must be open before the gateway hears it. +checkIn.start(); +log.info( + `node ${config.DORMICE_NODE_ID} checks in with gateway ${config.DORMICE_GATEWAY_ENDPOINT} every ${config.DORMICE_CHECK_IN_INTERVAL_SECONDS}s, reachable at ${nodeEndpoint}`, +); // systemd stops the daemon with SIGTERM. Shutdown is bounded on purpose // (shutdown.ts has the measurements): close the app — preClose ends the @@ -379,7 +400,7 @@ const close = async (signal: NodeJS.Signals) => { process.removeListener('SIGINT', onSigint); clearTimeout(heartbeatTimer); clearTimeout(metricsTimer); - checkIn?.stop(); + checkIn.stop(); watchdog.stop(); app.log.info( `${signal} received — shutting down (grace ${SHUTDOWN_GRACE_MS}ms)`, diff --git a/packages/server/src/metrics-sampler.test.ts b/packages/server/src/metrics-sampler.test.ts index d72a3682..67d477db 100644 --- a/packages/server/src/metrics-sampler.test.ts +++ b/packages/server/src/metrics-sampler.test.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url'; import { count } from 'drizzle-orm'; import { describe, expect, it } from 'vitest'; import { buildApp } from './app'; -import { CONFIG_KEYS, type ConfigSources, loadConfig } from './config'; +import { loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; import { FLEET_SNAPSHOT_KEEP_DAYS, insertMetricsTick } from './db/metrics'; import { @@ -15,6 +15,7 @@ import { CpuSampler, type HostSample } from './host-metrics'; import { KeyedQueue } from './keyed-queue'; import { freezeSandbox, stopSandbox } from './lifecycle'; import { sampleOnce } from './metrics-sampler'; +import { configureNode } from './testing'; // The sampler, unit-level: one tick's writes, the measurable-states gate, // the vanished-container skip, retention pruning and the destroy cascade. @@ -47,13 +48,6 @@ const HOST: HostSample = { diskAvailableBytes: null, }; -function fixedSources(): ConfigSources { - const all = Object.fromEntries( - Object.keys(CONFIG_KEYS).map((key) => [key, 'default']), - ) as ConfigSources; - return { ...all, DORMICE_API_TOKEN: 'env' }; -} - function harness() { const db = openDb(':memory:'); migrateDb(db, MIGRATIONS); @@ -61,16 +55,10 @@ function harness() { DORMICE_DB_PATH: ':memory:', DORMICE_API_TOKEN: TOKEN, }); + configureNode(db); const executor = new FakeExecutor(); const locks = new KeyedQueue(); - const app = buildApp({ - config, - db, - executor, - locks, - logger: false, - sources: fixedSources(), - }); + const app = buildApp({ config, db, executor, locks, logger: false }); return { app, db, executor, locks }; } diff --git a/packages/server/src/node-config.test.ts b/packages/server/src/node-config.test.ts new file mode 100644 index 00000000..bed72fb3 --- /dev/null +++ b/packages/server/src/node-config.test.ts @@ -0,0 +1,288 @@ +import { randomUUID } from 'node:crypto'; +import { fileURLToPath } from 'node:url'; +import { DEFAULT_LIFECYCLE_POLICY } from '@dormice/shared'; +import { describe, expect, it, vi } from 'vitest'; +import { type Db, migrateDb, openDb } from './db/db'; +import { createSandbox, overwriteState } from './db/ledger'; +import { + applyNodeConfig, + archiveEnabled, + NoConfigError, + readConfigVersion, + readNodeConfig, + readRuntimeSettings, + readS3Settings, + readSwapTarget, +} from './db/settings'; +import { findTemplate } from './db/templates'; +import { FakeExecutor } from './executor/fake'; +import { KeyedQueue } from './keyed-queue'; +import { applyConfig } from './node-config'; +import type { SwapControl, SwapStatus } from './swap'; +import { TEST_S3, testBundle } from './testing'; + +// The configuration copy: the pure write and its readers (db/settings.ts), +// and the applier that makes a bundle real on the host (node-config.ts). + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); + +function ledger(): Db { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + return db; +} + +function logSpy() { + const lines: Array<{ level: string; msg: string; obj: unknown }> = []; + const log = { + info: (obj: unknown, msg: string) => + lines.push({ level: 'info', msg, obj }), + warn: (obj: unknown, msg: string) => + lines.push({ level: 'warn', msg, obj }), + error: (obj: unknown, msg: string) => + lines.push({ level: 'error', msg, obj }), + }; + return { log, lines }; +} + +/** A swap manager that only records the targets it was asked to reconcile to. */ +function fakeSwap(): SwapControl & { targets: number[] } { + const targets: number[] = []; + const status: SwapStatus = { activeGb: 0, blocks: [] }; + return { + targets, + status: async () => status, + reconcile: async (targetGb) => { + targets.push(targetGb); + return status; + }, + }; +} + +describe('the copy in the ledger', () => { + it('holds no copy until a bundle is applied: readers refuse, the version is null', () => { + const db = ledger(); + expect(readConfigVersion(db)).toBeNull(); + expect(() => readRuntimeSettings(db)).toThrow(NoConfigError); + expect(() => readS3Settings(db)).toThrow(/no configuration copy yet/); + expect(() => readSwapTarget(db)).toThrow(NoConfigError); + }); + + it('a bundle is written whole and read back the same, keys withheld from the view and kept for the store', () => { + const db = ledger(); + const bundle = testBundle( + { + cpus: 2, + memoryGb: 4, + diskGb: 20, + s3: TEST_S3, + sandboxDomain: 'sbx.example.com', + sandboxDomainAliases: ['alt.example.com'], + pidsLimit: 512, + swapGb: 16, + // In name order, as the copy reads them back. + templates: [ + { name: 'node', image: 'img-node' }, + { name: 'py', image: 'img-py' }, + ], + }, + 5, + ); + applyNodeConfig(db, bundle); + expect(readConfigVersion(db)).toBe(5); + expect(readNodeConfig(db)).toEqual(bundle); + const view = readRuntimeSettings(db); + expect(view.s3).toEqual({ + endpoint: TEST_S3.endpoint, + bucket: TEST_S3.bucket, + region: TEST_S3.region, + forcePathStyle: TEST_S3.forcePathStyle, + }); + expect(JSON.stringify(view)).not.toContain('exam-secret'); + expect(readS3Settings(db)).toEqual(TEST_S3); + expect(archiveEnabled(db)).toBe(true); + expect(readSwapTarget(db)).toBe(16); + expect(findTemplate(db, 'py')?.image).toBe('img-py'); + }); + + it('the next bundle replaces everything: a store cleared, a template gone, a knob moved', () => { + const db = ledger(); + applyNodeConfig( + db, + testBundle( + { s3: TEST_S3, templates: [{ name: 'py', image: 'img-py' }] }, + 1, + ), + ); + applyNodeConfig( + db, + testBundle({ templates: [{ name: 'node', image: 'img-node' }] }, 2), + ); + expect(readConfigVersion(db)).toBe(2); + expect(readRuntimeSettings(db).s3).toBeNull(); + expect(archiveEnabled(db)).toBe(false); + expect(readRuntimeSettings(db).defaultPolicy).toEqual({ + ...DEFAULT_LIFECYCLE_POLICY, + archiveAfterSeconds: null, + }); + expect(findTemplate(db, 'py')).toBeUndefined(); + expect(findTemplate(db, 'node')?.image).toBe('img-node'); + }); + + it('an old single-machine row (no version) is not a copy: readers refuse until the first bundle overwrites it', () => { + const db = ledger(); + // What migration 0025 leaves behind on an upgraded machine: the row + // with the daemon's own settings and config_version NULL. + applyNodeConfig(db, testBundle({ pidsLimit: 999 }, 1)); + db.$client.exec( + "UPDATE runtime_settings SET config_version = NULL, s3_endpoint = '' WHERE id = 1", + ); + expect(readConfigVersion(db)).toBeNull(); + expect(() => readRuntimeSettings(db)).toThrow(NoConfigError); + applyNodeConfig(db, testBundle({ pidsLimit: 512 }, 4)); + expect(readConfigVersion(db)).toBe(4); + expect(readRuntimeSettings(db).pidsLimit).toBe(512); + expect(readRuntimeSettings(db).s3).toBeNull(); + }); +}); + +describe('applyConfig: the bundle made real on the host', () => { + function node(cap = 4096) { + const db = ledger(); + const executor = new FakeExecutor( + undefined, + () => readRuntimeSettings(db).pidsLimit, + ); + const locks = new KeyedQueue(); + const swap = fakeSwap(); + const { log, lines } = logSpy(); + applyNodeConfig(db, testBundle({ pidsLimit: cap }, 1)); + return { db, executor, locks, swap, log, lines }; + } + + async function running(db: Db, executor: FakeExecutor, name: string) { + const id = randomUUID(); + await executor.create(id); + return createSandbox(db, { + id, + name, + nodeId: 'node-test', + policy: DEFAULT_LIFECYCLE_POLICY, + }); + } + + it('the first copy is written and nothing else runs: boot does its own sweep and swap reconcile', async () => { + const db = ledger(); + const executor = new FakeExecutor(); + const swap = fakeSwap(); + const { log, lines } = logSpy(); + await applyConfig(testBundle({ pidsLimit: 512, swapGb: 8 }, 1), { + db, + executor, + locks: new KeyedQueue(), + swap, + log, + }); + expect(readConfigVersion(db)).toBe(1); + expect(swap.targets).toEqual([]); + expect(lines.map((l) => l.msg)).toEqual([ + 'first configuration copy applied from the gateway', + ]); + }); + + it('a moved pids cap sweeps the running shells in place; frozen ones follow at their wake', async () => { + const { db, executor, locks, swap, log } = node(4096); + const busy = await running(db, executor, 'busy'); + const idle = await running(db, executor, 'idle'); + await executor.freeze(idle.id); + overwriteState(db, idle.id, 'frozen'); + + await applyConfig(testBundle({ pidsLimit: 2048 }, 2), { + db, + executor, + locks, + swap, + log, + }); + expect(readRuntimeSettings(db).pidsLimit).toBe(2048); + expect(executor.pidsLimitOf(busy.id)).toBe(2048); + expect(executor.stateOf(busy.id)).toBe('running'); + expect(executor.pidsLimitOf(idle.id)).toBe(4096); + await executor.unfreeze(idle.id); + expect(executor.pidsLimitOf(idle.id)).toBe(2048); + // The swap target did not move: no reconcile. + expect(swap.targets).toEqual([]); + }); + + it('a shell the runtime refuses is logged by name; the copy is applied all the same', async () => { + const { db, executor, locks, swap, log, lines } = node(4096); + await running(db, executor, 'stubborn'); + vi.spyOn(executor, 'convergePidsLimit').mockRejectedValue( + new Error('runsc refused'), + ); + await applyConfig(testBundle({ pidsLimit: 2048 }, 2), { + db, + executor, + locks, + swap, + log, + }); + expect(readConfigVersion(db)).toBe(2); + const warned = lines.find((l) => l.level === 'warn'); + expect(warned?.msg).toMatch(/pids cap moved to 2048; 1 running shell/); + expect((warned?.obj as { failures: string[] }).failures).toEqual([ + 'stubborn: runsc refused', + ]); + }); + + it('a moved swap target reconciles the managed swap; a failure is logged, never thrown', async () => { + const { db, executor, locks, swap, log, lines } = node(); + await applyConfig(testBundle({ swapGb: 16 }, 2), { + db, + executor, + locks, + swap, + log, + }); + expect(swap.targets).toEqual([16]); + expect(readSwapTarget(db)).toBe(16); + // Same target again in a bundle that changed something else: no reconcile. + await applyConfig( + testBundle({ swapGb: 16, sandboxDomain: 'sbx.example.com' }, 3), + { db, executor, locks, swap, log }, + ); + expect(swap.targets).toEqual([16]); + + const failing: SwapControl = { + status: async () => ({ activeGb: 0, blocks: [] }), + reconcile: async () => { + throw new Error('fallocate: No space left on device'); + }, + }; + await expect( + applyConfig(testBundle({ swapGb: 32 }, 4), { + db, + executor, + locks, + swap: failing, + log, + }), + ).resolves.toBeUndefined(); + expect(readSwapTarget(db)).toBe(32); + expect(lines.at(-1)?.level).toBe('error'); + expect(lines.at(-1)?.msg).toMatch( + /swap reconcile after a configuration change failed/, + ); + }); + + it('without a swap manager the target is stored and nothing is reconciled', async () => { + const { db, executor, locks, log } = node(); + await applyConfig(testBundle({ swapGb: 16 }, 2), { + db, + executor, + locks, + log, + }); + expect(readSwapTarget(db)).toBe(16); + }); +}); diff --git a/packages/server/src/node-config.ts b/packages/server/src/node-config.ts new file mode 100644 index 00000000..c5cbe9d2 --- /dev/null +++ b/packages/server/src/node-config.ts @@ -0,0 +1,91 @@ +import type { NodeConfigBundle } from '@dormice/shared'; +import type { Db } from './db/db'; +import { + applyNodeConfig, + readConfigVersion, + readRuntimeSettings, + readSwapTarget, +} from './db/settings'; +import type { Executor } from './executor/executor'; +import type { KeyedQueue } from './keyed-queue'; +import { sweepPidsLimit } from './pids-sweep'; +import type { SwapControl } from './swap'; + +export interface ConfigApplierDeps { + db: Db; + executor: Executor; + locks: KeyedQueue; + /** Managed swap, where the host has it (main.ts decides); absent, the target is stored and nothing else. */ + swap?: SwapControl; + log: { + info(obj: unknown, msg: string): void; + warn(obj: unknown, msg: string): void; + error(obj: unknown, msg: string): void; + }; + /** The heartbeat watchdog's ear, for the pids sweep. */ + beat?: () => void; +} + +/** + * A bundle from the gateway, made real on this node. The ledger copy + * first (db/settings.ts applyNodeConfig — from that instant every birth, + * wake, scan tick and proxy request reads the new values), then the two + * knobs with a reality on the host that a write alone does not move: the + * pids cap on the shells running right now (a cgroup write their + * processes never notice; pids-sweep.ts) and the managed swap target + * (grow now, shrink at the next reboot; swap.ts). Each runs only when its + * value moved — a bundle that changed a template, the archive store or + * the domain touches no shell — and never after the first copy: at boot, + * main.ts's own sweep and swap reconcile follow. Their failures are + * logged, never thrown: the copy is applied and the version reported + * either way; a shell the runtime refuses converges at its next wake, a + * swap block that failed to mount is retried at the next boot — capacity, + * not correctness. + */ +export async function applyConfig( + bundle: NodeConfigBundle, + deps: ConfigApplierDeps, +): Promise { + const { db, executor, locks, swap, log, beat } = deps; + const before = + readConfigVersion(db) === null + ? null + : { + pidsLimit: readRuntimeSettings(db).pidsLimit, + swapGb: readSwapTarget(db), + }; + applyNodeConfig(db, bundle); + log.info( + { + version: bundle.version, + from: before === null ? null : undefined, + templates: bundle.templates.length, + pidsLimit: bundle.settings.pidsLimit, + swapGb: bundle.node.swapGb, + archive: bundle.settings.s3 === null ? 'off' : bundle.settings.s3.bucket, + sandboxDomain: bundle.settings.sandboxDomain, + }, + before === null + ? 'first configuration copy applied from the gateway' + : 'configuration applied from the gateway', + ); + if (before === null) return; + if (bundle.settings.pidsLimit !== before.pidsLimit) { + const sweep = await sweepPidsLimit(db, executor, locks, beat); + if (sweep.failures.length > 0) { + log.warn( + sweep, + `pids cap moved to ${bundle.settings.pidsLimit}; ${sweep.failures.length} running shell(s) kept the old cap until their next wake`, + ); + } else { + log.info(sweep, `pids cap moved to ${bundle.settings.pidsLimit}`); + } + } + if (swap !== undefined && bundle.node.swapGb !== before.swapGb) { + try { + await swap.reconcile(bundle.node.swapGb); + } catch (error) { + log.error(error, 'swap reconcile after a configuration change failed'); + } + } +} diff --git a/packages/server/src/routes/api-keys.ts b/packages/server/src/routes/api-keys.ts deleted file mode 100644 index c086e528..00000000 --- a/packages/server/src/routes/api-keys.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { - type ApiKey, - createApiKeyRequestSchema, - createApiKeyResponseSchema, - listApiKeysResponseSchema, - revokeApiKeyRequestSchema, - revokeApiKeyResponseSchema, - updateApiKeyRequestSchema, - updateApiKeyResponseSchema, -} from '@dormice/shared'; -import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; -import { - createApiKey, - findActiveApiKeyByName, - findApiKeyById, - listApiKeys, - revokeApiKey, - updateApiKey, -} from '../db/api-keys'; -import type { Db } from '../db/db'; -import type { ApiKeyRow } from '../db/schema'; -import { httpError } from '../http-error'; - -export interface ApiKeyRoutesOptions { - db: Db; -} - -/** The wire view: everything but the hash — no secret ever leaves the row. */ -function view(row: ApiKeyRow): ApiKey { - return { - id: row.id, - name: row.name, - prefix: row.prefix, - createdAt: row.createdAt, - lastUsedAt: row.lastUsedAt, - expiresAt: row.expiresAt, - disabledAt: row.disabledAt, - revokedAt: row.revokedAt, - }; -} - -/** - * API key management: mint, list, edit, revoke. Pure ledger verbs — no - * executor, no locks. Admin-only: buildApp registers this plugin behind - * requireAdminAuth (env token or console session; a live key gets an - * honest 403), because a credential must not manage the credential ledger - * it lives in. Verification itself lives in db/api-keys.ts and is - * consulted by buildApp's isCredential closure, not here. - */ -export const apiKeyRoutes: FastifyPluginAsyncZod = async ( - app, - { db }, -) => { - app.post( - '/createApiKey', - { - schema: { - body: createApiKeyRequestSchema, - response: { 200: createApiKeyResponseSchema }, - }, - }, - async (request) => { - const { name, expiresAt } = request.body; - // Two live credentials answering to one name is a rotation mistake, - // not a goal — refused by name, like removeTemplate's 409. The - // partial unique index backstops this check as a schema fact; no - // await sits between check and insert, so they cannot race. - if (findActiveApiKeyByName(db, name)) { - throw httpError( - 409, - `an active API key named '${name}' already exists — revoke it first or pick another name`, - ); - } - const { row, token } = createApiKey(db, name, expiresAt); - // The token itself never reaches the log. - request.log.info( - { apiKey: row.id, name, prefix: row.prefix, expiresAt: row.expiresAt }, - 'API key minted', - ); - return { apiKey: view(row), token }; - }, - ); - - app.post( - '/listApiKeys', - { - schema: { - response: { 200: listApiKeysResponseSchema }, - }, - }, - async () => ({ apiKeys: listApiKeys(db).map(view) }), - ); - - app.post( - '/updateApiKey', - { - schema: { - body: updateApiKeyRequestSchema, - response: { 200: updateApiKeyResponseSchema }, - }, - }, - async (request) => { - const { id, ...patch } = request.body; - // Adjudication happens here, in order, with no await between the - // checks and the write (better-sqlite3 is sync — they cannot race). - const row = findApiKeyById(db, id); - if (!row) { - throw httpError(404, `no API key with id '${id}'`); - } - if (row.revokedAt !== null) { - throw httpError( - 409, - `API key "${row.name}" is revoked — revoked rows are rotation history and cannot be changed`, - ); - } - if (patch.name !== undefined && patch.name !== row.name) { - // Same courtesy as create: the name must not collide with any - // non-revoked key. findActiveApiKeyByName cannot return this row - // itself — the names differ. - if (findActiveApiKeyByName(db, patch.name)) { - throw httpError( - 409, - `an active API key named '${patch.name}' already exists — revoke it first or pick another name`, - ); - } - } - const updated = updateApiKey(db, row, patch); - if (updated !== row) { - request.log.info( - { apiKey: row.id, name: updated.name, patch }, - 'API key updated', - ); - } - return { apiKey: view(updated) }; - }, - ); - - app.post( - '/revokeApiKey', - { - schema: { - body: revokeApiKeyRequestSchema, - response: { 200: revokeApiKeyResponseSchema }, - }, - }, - async (request) => { - const revoked = revokeApiKey(db, request.body.id); - if (revoked) { - request.log.info({ apiKey: request.body.id }, 'API key revoked'); - } - return { revoked }; - }, - ); -}; diff --git a/packages/server/src/routes/config.ts b/packages/server/src/routes/config.ts deleted file mode 100644 index 256d5be8..00000000 --- a/packages/server/src/routes/config.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { getConfigResponseSchema } from '@dormice/shared'; -import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; -import { CONFIG_KEYS, type Config, type ConfigSources } from '../config'; -import type { Db } from '../db/db'; -import { readRuntimeSettings } from '../db/settings'; - -export interface ConfigRoutesOptions { - config: Config; - db: Db; - sources: ConfigSources; -} - -/** - * The daemon's effective configuration: the env knobs (read-only — editing - * those stays on the host, /etc/dormice/env plus a restart, because a - * daemon that rewrites its own environment is a different security - * decision entirely) plus the ledger-resident runtime settings, which DO - * have a write verb (updateSettings, admin scope). For the knobs that - * moved into the ledger — capacity, sandbox defaults, the default policy, - * the S3 archive store, the sandbox domain — the env entries below are - * first-boot seeds; `settings` is what is in force. Secrets never cross - * the wire: set-or-unset is all anyone learns. - */ -export const configRoutes: FastifyPluginAsyncZod = async ( - app, - { config, db, sources }, -) => { - app.post( - '/getConfig', - { - schema: { - response: { 200: getConfigResponseSchema }, - }, - }, - async () => { - const settings = readRuntimeSettings(db); - // "Is archiving available" is the ledger's live answer — the same - // adjudication every consumer reads (db/settings.ts archiveEnabled). - const enabled = settings.s3 !== null; - return { - entries: (Object.keys(CONFIG_KEYS) as Array).map( - (key) => { - const { sensitive } = CONFIG_KEYS[key]; - const value = config[key]; - return { - key, - value: sensitive || value === undefined ? null : String(value), - source: sources[key], - ...(sensitive && value !== undefined ? { redacted: true } : {}), - }; - }, - ), - archive: { - enabled, - defaultSeconds: enabled - ? settings.defaultPolicy.archiveAfterSeconds - : null, - }, - settings, - }; - }, - ); -}; diff --git a/packages/server/src/routes/console.test.ts b/packages/server/src/routes/console.test.ts deleted file mode 100644 index 6b829a19..00000000 --- a/packages/server/src/routes/console.test.ts +++ /dev/null @@ -1,436 +0,0 @@ -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { describe, expect, it } from 'vitest'; -import { buildApp } from '../app'; -import { - CONSOLE_HEADER, - hashPassword, - mintSession, - mintSessionSecret, - SESSION_COOKIE, - SESSION_TTL_SECONDS, - verifyPassword, - verifySession, -} from '../auth'; -import { loadConfig } from '../config'; -import { migrateDb, openDb } from '../db/db'; -import { FakeExecutor } from '../executor/fake'; -import { KeyedQueue } from '../keyed-queue'; - -const MIGRATIONS = fileURLToPath(new URL('../../drizzle', import.meta.url)); -const TOKEN = 'test-token-test-token-test-token'; -const USERNAME = 'operator'; -const PASSWORD = 'correct horse battery'; - -function testApp(consoleDistDir?: string) { - const db = openDb(':memory:'); - migrateDb(db, MIGRATIONS); - const config = loadConfig({ - DORMICE_DB_PATH: ':memory:', - DORMICE_API_TOKEN: TOKEN, - }); - return buildApp({ - config, - db, - executor: new FakeExecutor(), - locks: new KeyedQueue(), - logger: false, - consoleDistDir, - }); -} - -type TestApp = ReturnType; - -/** A minimal built console: an index.html and one hashed asset. */ -function fixtureDist(): string { - const dir = mkdtempSync(join(tmpdir(), 'dormice-consoledist-')); - writeFileSync(join(dir, 'index.html'), 'dormice console'); - mkdirSync(join(dir, 'assets')); - writeFileSync(join(dir, 'assets', 'app-abc123.js'), 'console.log("ui")'); - return dir; -} - -async function setup( - app: TestApp, - { token = TOKEN, username = USERNAME, password = PASSWORD } = {}, -) { - return app.inject({ - method: 'POST', - url: '/console/auth/setup', - payload: { token, username, password }, - }); -} - -async function login( - app: TestApp, - { username = USERNAME, password = PASSWORD } = {}, -) { - return app.inject({ - method: 'POST', - url: '/console/auth/login', - payload: { username, password }, - }); -} - -/** The Set-Cookie value for the session cookie, parsed by fastify's helper. */ -function sessionCookie(res: { cookies: Array> }) { - const cookie = res.cookies.find((c) => c.name === SESSION_COOKIE); - expect(cookie).toBeDefined(); - return cookie as { value: string } & Record; -} - -describe('password hashing', () => { - it('round-trips and rejects a wrong password', async () => { - const stored = await hashPassword(PASSWORD); - expect(stored.startsWith('scrypt$')).toBe(true); - expect(await verifyPassword(PASSWORD, stored)).toBe(true); - expect(await verifyPassword('not the password', stored)).toBe(false); - }); - - it('salts: two hashes of the same password differ', async () => { - expect(await hashPassword(PASSWORD)).not.toBe(await hashPassword(PASSWORD)); - }); - - it('rejects garbage stored values instead of throwing', async () => { - expect(await verifyPassword(PASSWORD, '')).toBe(false); - expect(await verifyPassword(PASSWORD, 'bcrypt$whatever')).toBe(false); - }); -}); - -describe('session mint/verify', () => { - const SECRET = mintSessionSecret(); - - it('round-trips a fresh session', () => { - expect(verifySession(SECRET, mintSession(SECRET))).toBe(true); - }); - - it('rejects an expired session', () => { - const past = Date.now() - (SESSION_TTL_SECONDS + 10) * 1000; - expect(verifySession(SECRET, mintSession(SECRET, past))).toBe(false); - }); - - it('rejects a tampered expiry: the HMAC covers it', () => { - const value = mintSession(SECRET); - const [exp, mac] = value.split('.'); - const later = `${Number(exp) + 3600}.${mac}`; - expect(verifySession(SECRET, later)).toBe(false); - }); - - it('rejects garbage and sessions minted under another secret', () => { - expect(verifySession(SECRET, 'not-a-session')).toBe(false); - expect(verifySession(SECRET, '')).toBe(false); - expect(verifySession(SECRET, mintSession(mintSessionSecret()))).toBe(false); - }); -}); - -describe('POST /console/auth/status', () => { - it('reports whether setup has happened', async () => { - const app = testApp(); - const before = await app.inject({ - method: 'POST', - url: '/console/auth/status', - payload: {}, - }); - expect(before.json()).toEqual({ accountExists: false }); - await setup(app); - const after = await app.inject({ - method: 'POST', - url: '/console/auth/status', - payload: {}, - }); - expect(after.json()).toEqual({ accountExists: true }); - }); -}); - -describe('POST /console/auth/setup', () => { - it('rejects a wrong token without creating anything', async () => { - const app = testApp(); - const res = await setup(app, { token: 'wrong-token-wrong-token-wrong-tk' }); - expect(res.statusCode).toBe(401); - expect(res.cookies).toHaveLength(0); - expect((await login(app)).statusCode).toBe(409); - }); - - it('creates the account and signs the caller in', async () => { - const app = testApp(); - const res = await setup(app); - expect(res.statusCode).toBe(200); - const cookie = sessionCookie(res); - expect(cookie.httpOnly).toBe(true); - expect(cookie.sameSite).toBe('Strict'); - expect(cookie.path).toBe('/'); - expect(cookie.maxAge).toBe(SESSION_TTL_SECONDS); - }); - - it('refuses a short password', async () => { - const res = await setup(testApp(), { password: 'short' }); - expect(res.statusCode).toBe(400); - }); - - it('re-setup overwrites the account and voids old sessions', async () => { - const app = testApp(); - const first = sessionCookie(await setup(app)); - // The recovery path: the token alone resets username and password. - const res = await setup(app, { - username: 'renamed', - password: 'brand-new-pass', - }); - expect(res.statusCode).toBe(200); - expect((await list(app, first.value)).statusCode).toBe(401); - expect( - (await login(app, { username: 'renamed', password: 'brand-new-pass' })) - .statusCode, - ).toBe(200); - expect((await login(app)).statusCode).toBe(401); - }); -}); - -async function list( - app: TestApp, - cookieValue: string, - headers: Record = { [CONSOLE_HEADER]: '1' }, -) { - return app.inject({ - method: 'POST', - url: '/listSandboxes', - cookies: { [SESSION_COOKIE]: cookieValue }, - headers, - payload: {}, - }); -} - -describe('POST /console/auth/login', () => { - it('answers 409 before setup — an honest pointer, not a guess counted', async () => { - const res = await login(testApp()); - expect(res.statusCode).toBe(409); - expect(res.json().message).toContain('setup'); - }); - - it('rejects wrong credentials without setting a cookie', async () => { - const app = testApp(); - await setup(app); - const wrongPass = await login(app, { password: 'wrong password' }); - expect(wrongPass.statusCode).toBe(401); - expect(wrongPass.cookies).toHaveLength(0); - const wrongUser = await login(app, { username: 'someone-else' }); - expect(wrongUser.statusCode).toBe(401); - }); - - it('signs in with the right credentials', async () => { - const app = testApp(); - await setup(app); - const res = await login(app); - expect(res.statusCode).toBe(200); - const cookie = sessionCookie(res); - expect(cookie.httpOnly).toBe(true); - expect(cookie.maxAge).toBe(SESSION_TTL_SECONDS); - }); -}); - -describe('login throttle over the wire', () => { - it('backs off after repeated failures — even the right credential waits', async () => { - const app = testApp(); - await setup(app); - for (let i = 0; i < 8; i++) { - const res = await login(app, { password: 'wrong password' }); - expect([401, 429]).toContain(res.statusCode); - } - const blocked = await login(app); - expect(blocked.statusCode).toBe(429); - expect(blocked.json().message).toContain('retry in'); - // Setup shares the same counters: guessing tokens is the same game. - expect((await setup(app)).statusCode).toBe(429); - }); - - it('a success clears the slate', async () => { - const app = testApp(); - await setup(app); - for (let i = 0; i < 4; i++) { - await login(app, { password: 'wrong password' }); - } - expect((await login(app)).statusCode).toBe(200); - expect((await login(app, { password: 'wrong password' })).statusCode).toBe( - 401, - ); - }); -}); - -describe('cookie-authenticated API access', () => { - it('a fresh session cookie opens the native API', async () => { - const app = testApp(); - await setup(app); - const cookie = sessionCookie(await login(app)); - const res = await list(app, cookie.value); - expect(res.statusCode).toBe(200); - expect(res.json()).toEqual({ sandboxes: [] }); - }); - - it('the cookie alone is not enough: the console header is required', async () => { - const app = testApp(); - await setup(app); - const cookie = sessionCookie(await login(app)); - const res = await list(app, cookie.value, {}); - expect(res.statusCode).toBe(401); - }); - - it('rejects a tampered cookie', async () => { - const app = testApp(); - await setup(app); - const cookie = sessionCookie(await login(app)); - const res = await list(app, `${cookie.value}ff`); - expect(res.statusCode).toBe(401); - }); - - it('rejects any cookie while no account exists', async () => { - // A cookie minted under some secret proves nothing when the ledger has - // no account (e.g. the ledger was recreated). - const res = await list(testApp(), mintSession(mintSessionSecret())); - expect(res.statusCode).toBe(401); - }); - - it('does not open the E2B surface: that has its own auth', async () => { - const app = testApp(); - await setup(app); - const cookie = sessionCookie(await login(app)); - const res = await app.inject({ - method: 'GET', - url: '/e2b/api/v2/sandboxes', - cookies: { [SESSION_COOKIE]: cookie.value }, - headers: { [CONSOLE_HEADER]: '1' }, - }); - expect(res.statusCode).toBe(401); - }); -}); - -describe('POST /envdToken', () => { - async function mint( - app: TestApp, - cookieValue: string, - headers: Record = { [CONSOLE_HEADER]: '1' }, - ) { - return app.inject({ - method: 'POST', - url: '/envdToken', - cookies: { [SESSION_COOKIE]: cookieValue }, - headers, - payload: { sandboxId: 'sb-terminal' }, - }); - } - - it('a session cookie mints the exact token the envd surface accepts', async () => { - const app = testApp(); - await setup(app); - const cookie = sessionCookie(await login(app)); - const res = await mint(app, cookie.value); - expect(res.statusCode).toBe(200); - const { envdAccessToken } = res.json() as { envdAccessToken: string }; - // The envd surface itself is the judge — the token derives from the - // ledger's signing secret, which nothing outside the daemon (this test - // included) can recompute. Auth passing shows as anything-but-401. - const probe = (sandboxId: string) => - app.inject({ - method: 'POST', - url: '/e2b/envd/filesystem.Filesystem/Stat', - headers: { - 'e2b-sandbox-id': sandboxId, - 'x-access-token': envdAccessToken, - }, - payload: { path: '/home/user' }, - }); - expect((await probe('sb-terminal')).statusCode).not.toBe(401); - // Per-sandbox: the same token opens no other sandbox. - expect((await probe('sb-other')).statusCode).toBe(401); - }); - - it('goes through the API-wide arbiter: no console header, no token', async () => { - const app = testApp(); - await setup(app); - const cookie = sessionCookie(await login(app)); - const res = await mint(app, cookie.value, {}); - expect(res.statusCode).toBe(401); - }); - - it('rejects a tampered cookie', async () => { - const app = testApp(); - await setup(app); - const cookie = sessionCookie(await login(app)); - const res = await mint(app, `${cookie.value}ff`); - expect(res.statusCode).toBe(401); - }); -}); - -describe('POST /console/auth/logout', () => { - it('clears the session cookie', async () => { - const res = await testApp().inject({ - method: 'POST', - url: '/console/auth/logout', - }); - expect(res.statusCode).toBe(200); - const cookie = sessionCookie(res); - expect(cookie.value).toBe(''); - }); -}); - -describe('GET / — the bare-origin redirect', () => { - it('sends a browser (html Accept) to /console/', async () => { - const res = await testApp(fixtureDist()).inject({ - method: 'GET', - url: '/', - headers: { accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8' }, - }); - expect(res.statusCode).toBe(302); - expect(res.headers.location).toBe('/console/'); - }); - - it('keeps the honest 404 for non-browser clients', async () => { - // curl's default is Accept: */* — no html, no redirect. - const res = await testApp(fixtureDist()).inject({ - method: 'GET', - url: '/', - headers: { accept: '*/*' }, - }); - expect(res.statusCode).toBe(404); - expect(res.json().message).toContain('not found'); - }); - - it('redirects even when the console is not built — the 404 there points at pnpm build', async () => { - const res = await testApp().inject({ - method: 'GET', - url: '/', - headers: { accept: 'text/html' }, - }); - expect(res.statusCode).toBe(302); - }); -}); - -describe('static console at /console', () => { - it('serves index.html and assets from the injected dist', async () => { - const app = testApp(fixtureDist()); - const index = await app.inject({ method: 'GET', url: '/console/' }); - expect(index.statusCode).toBe(200); - expect(index.body).toContain('dormice console'); - const asset = await app.inject({ - method: 'GET', - url: '/console/assets/app-abc123.js', - }); - expect(asset.statusCode).toBe(200); - }); - - it('falls back to index.html for client-side routes (SPA)', async () => { - const app = testApp(fixtureDist()); - const res = await app.inject({ - method: 'GET', - url: '/console/sandboxes/deep', - }); - expect(res.statusCode).toBe(200); - expect(res.body).toContain('dormice console'); - }); - - it('answers an honest 404 when the console is not built', async () => { - const res = await testApp().inject({ method: 'GET', url: '/console' }); - expect(res.statusCode).toBe(404); - expect(res.json().message).toContain('pnpm build'); - }); -}); diff --git a/packages/server/src/routes/console.ts b/packages/server/src/routes/console.ts deleted file mode 100644 index 90a3c574..00000000 --- a/packages/server/src/routes/console.ts +++ /dev/null @@ -1,229 +0,0 @@ -import fastifyStatic from '@fastify/static'; -import type { FastifyReply } from 'fastify'; -import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; -import { z } from 'zod'; -import { - hashPassword, - mintSession, - mintSessionSecret, - SESSION_COOKIE, - SESSION_TTL_SECONDS, - tokensEqual, - verifyPassword, -} from '../auth'; -import type { Config } from '../config'; -import { getConsoleAccount, setConsoleAccount } from '../db/account'; -import type { Db } from '../db/db'; -import { LoginThrottle } from '../login-throttle'; - -export interface ConsoleRoutesOptions { - config: Config; - db: Db; - /** - * Where the built web console lives (packages/console/dist). Injected so - * tests point it at a fixture and embedders can omit it; absent means - * /console answers an honest 404 instead of guessing at paths. - */ - consoleDistDir?: string; -} - -// No Secure flag: the daemon speaks plain http on 127.0.0.1 by design, and -// behind a TLS reverse proxy the browser-facing side is the proxy's job. -const COOKIE_OPTIONS = { - httpOnly: true, - sameSite: 'strict', - path: '/', -} as const; - -const messageResponse = z.object({ message: z.string() }); - -/** - * The web console's own surface: account + session endpoints and the static - * SPA. Everything else the console does goes through the native RPC routes - * with the session cookie — same routes, same truth as the SDK and CLI. - * - * The credential model: the API token is the root of trust (machine - * credential, lives in server env), the account is the human convenience - * derived from it. Setup requires the token and overwrites the account — - * first-run initialization, password change and forgot-password are all - * that one verb, so there is no registration race (an open "first visitor - * becomes admin" door on a public URL) and no recovery flow to build. - */ -export const consoleRoutes: FastifyPluginAsyncZod< - ConsoleRoutesOptions -> = async (app, { config, db, consoleDistDir }) => { - // Per-app, not module-global: each daemon (and each test app) gets its - // own counters. Shared by login and setup — both are credential guesses. - const throttle = new LoginThrottle(); - - const setSessionCookie = (reply: FastifyReply, sessionSecret: string) => { - reply.setCookie(SESSION_COOKIE, mintSession(sessionSecret), { - ...COOKIE_OPTIONS, - // The cookie lives exactly as long as the HMAC inside it is valid. - maxAge: SESSION_TTL_SECONDS, - }); - }; - - // Open by design: it answers only "is setup still pending", which the - // login page needs before any credential exists. An attacker learns - // nothing usable — completing setup requires the API token either way. - app.post( - '/console/auth/status', - { - schema: { - response: { 200: z.object({ accountExists: z.boolean() }) }, - }, - }, - async () => ({ accountExists: getConsoleAccount(db) !== undefined }), - ); - - app.post( - '/console/auth/setup', - { - schema: { - body: z.object({ - token: z.string().min(1), - username: z.string().trim().min(1).max(64), - // Length is the only strength rule: composition rules push people - // toward Password1! and help nobody. - password: z.string().min(8).max(128), - }), - response: { - 200: z.object({ loggedIn: z.literal(true) }), - 401: messageResponse, - 429: messageResponse, - }, - }, - }, - async (request, reply) => { - const wait = throttle.retryAfterSeconds(request.ip); - if (wait > 0) { - return reply.code(429).send({ - message: `too many failed attempts — retry in ${wait}s`, - }); - } - // Deliberately the env token only, never a ledger API key: this verb - // resets the human account, and a leaked machine credential must not - // escalate into a console takeover. The token's root of trust is - // filesystem access to /etc/dormice/env — exactly what a recovery - // path should require. - if (!tokensEqual(request.body.token, config.DORMICE_API_TOKEN)) { - throttle.recordFailure(request.ip); - return reply.code(401).send({ message: 'invalid API token' }); - } - throttle.clear(request.ip); - const account = setConsoleAccount(db, { - username: request.body.username, - passwordHash: await hashPassword(request.body.password), - // A fresh secret voids every existing session — the semantics a - // password (re)set should have. - sessionSecret: mintSessionSecret(), - }); - setSessionCookie(reply, account.sessionSecret); - return { loggedIn: true as const }; - }, - ); - - app.post( - '/console/auth/login', - { - schema: { - // min(1) only: the password policy is enforced where passwords are - // set; login must accept whatever was stored. - body: z.object({ - username: z.string().min(1), - password: z.string().min(1), - }), - response: { - 200: z.object({ loggedIn: z.literal(true) }), - 401: messageResponse, - 409: messageResponse, - 429: messageResponse, - }, - }, - }, - async (request, reply) => { - const wait = throttle.retryAfterSeconds(request.ip); - if (wait > 0) { - return reply.code(429).send({ - message: `too many failed attempts — retry in ${wait}s`, - }); - } - const account = getConsoleAccount(db); - if (!account) { - // Not a failed guess (nothing exists to guess at), so no throttle - // hit — an honest pointer to setup instead. - return reply.code(409).send({ - message: 'no account exists yet — complete setup with the API token', - }); - } - // Evaluate both factors unconditionally so a wrong username costs the - // same time as a wrong password. - const usernameOk = tokensEqual(request.body.username, account.username); - const passwordOk = await verifyPassword( - request.body.password, - account.passwordHash, - ); - if (!usernameOk || !passwordOk) { - throttle.recordFailure(request.ip); - return reply - .code(401) - .send({ message: 'invalid username or password' }); - } - throttle.clear(request.ip); - setSessionCookie(reply, account.sessionSecret); - return { loggedIn: true as const }; - }, - ); - - app.post( - '/console/auth/logout', - { - schema: { - response: { 200: z.object({ loggedIn: z.literal(false) }) }, - }, - }, - async (_request, reply) => { - reply.clearCookie(SESSION_COOKIE, COOKIE_OPTIONS); - return { loggedIn: false as const }; - }, - ); - - // The bare-origin convenience: a browser landing on / is a human looking - // for the console — send them there (even unbuilt, /console's "run pnpm - // build" 404 beats "route not found"). Machines never GET / with an html - // Accept, so they keep the honest 404 from the app-wide arbiter. - app.get('/', async (request, reply) => { - if (request.headers.accept?.includes('text/html')) { - return reply.redirect('/console/'); - } - return reply.callNotFound(); - }); - - if (consoleDistDir) { - await app.register( - async (scope) => { - await scope.register(fastifyStatic, { root: consoleDistDir }); - // SPA fallback: the router owns paths under /console, so any GET - // that matches no file is a client-side route — serve the app and - // let it resolve. Everything else keeps the honest 404. - scope.setNotFoundHandler((request, reply) => { - if (request.method === 'GET') { - return reply.sendFile('index.html'); - } - reply.code(404).send({ - message: `route ${request.method} ${request.url} not found`, - }); - }); - }, - { prefix: '/console' }, - ); - } else { - app.get('/console', async (_request, reply) => - reply.code(404).send({ - message: - 'web console not available: packages/console/dist was not found at startup — run `pnpm build` first', - }), - ); - } -}; diff --git a/packages/server/src/routes/envd-token.test.ts b/packages/server/src/routes/envd-token.test.ts new file mode 100644 index 00000000..a5b0b79a --- /dev/null +++ b/packages/server/src/routes/envd-token.test.ts @@ -0,0 +1,83 @@ +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { buildApp } from '../app'; +import { loadConfig } from '../config'; +import { migrateDb, openDb } from '../db/db'; +import { FakeExecutor } from '../executor/fake'; +import { KeyedQueue } from '../keyed-queue'; +import { configureNode } from '../testing'; + +const MIGRATIONS = fileURLToPath(new URL('../../drizzle', import.meta.url)); +const TOKEN = 'test-token-test-token-test-token'; + +function testApp() { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const config = loadConfig({ + DORMICE_DB_PATH: ':memory:', + DORMICE_API_TOKEN: TOKEN, + }); + configureNode(db); + return buildApp({ + config, + db, + executor: new FakeExecutor(), + locks: new KeyedQueue(), + logger: false, + }); +} + +describe('POST /envdToken', () => { + it('the fleet token mints the exact token the envd surface accepts, per sandbox', async () => { + const app = testApp(); + const res = await app.inject({ + method: 'POST', + url: '/envdToken', + headers: { authorization: `Bearer ${TOKEN}` }, + payload: { sandboxId: 'sb-terminal' }, + }); + expect(res.statusCode).toBe(200); + const { envdAccessToken } = res.json() as { envdAccessToken: string }; + // The envd surface itself is the judge — the token derives from the + // ledger's signing secret, which nothing outside the daemon (this test + // included) can recompute. Auth passing shows as anything-but-401. + const probe = (sandboxId: string) => + app.inject({ + method: 'POST', + url: '/e2b/envd/filesystem.Filesystem/Stat', + headers: { + 'e2b-sandbox-id': sandboxId, + 'x-access-token': envdAccessToken, + }, + payload: { path: '/home/user' }, + }); + expect((await probe('sb-terminal')).statusCode).not.toBe(401); + // Per-sandbox: the same token opens no other sandbox. + expect((await probe('sb-other')).statusCode).toBe(401); + }); + + it('sits behind the API-wide arbiter: no token, no mint — and the console session is not a credential here', async () => { + const app = testApp(); + expect( + ( + await app.inject({ + method: 'POST', + url: '/envdToken', + payload: { sandboxId: 'sb-terminal' }, + }) + ).statusCode, + ).toBe(401); + // A cookie is the gateway's business; on a node it opens nothing. + expect( + ( + await app.inject({ + method: 'POST', + url: '/envdToken', + headers: { 'x-dormice-console': '1' }, + cookies: { dormice_session: '99999999999.deadbeef' }, + payload: { sandboxId: 'sb-terminal' }, + }) + ).statusCode, + ).toBe(401); + }); +}); diff --git a/packages/server/src/routes/ingress.ts b/packages/server/src/routes/ingress.ts deleted file mode 100644 index 6dcf06aa..00000000 --- a/packages/server/src/routes/ingress.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { - getIngressResponseSchema, - setIngressRequestSchema, - setIngressResponseSchema, -} from '@dormice/shared'; -import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; -import { httpError } from '../http-error'; -import { type Ingress, UnmanagedIngressFileError } from '../ingress'; - -export interface IngressRoutesOptions { - /** Present exactly when DORMICE_INGRESS_FILE is set (the archiver precedent). */ - ingress?: Ingress; -} - -/** - * The daemon's front door, over the wire: bind a domain from the console - * (or SDK) instead of hand-editing a Caddyfile over SSH. Without a managed - * ingress the read is honest ({ managed: false }) and the write refuses - * with the reason — never a silent no-op. - */ -export const ingressRoutes: FastifyPluginAsyncZod< - IngressRoutesOptions -> = async (app, { ingress }) => { - app.post( - '/getIngress', - { - schema: { - response: { 200: getIngressResponseSchema }, - }, - }, - async () => (ingress ? ingress.status() : { managed: false, domains: [] }), - ); - - app.post( - '/setIngress', - { - schema: { - body: setIngressRequestSchema, - response: { 200: setIngressResponseSchema }, - }, - }, - async (request) => { - if (!ingress) { - throw httpError( - 400, - 'this daemon manages no reverse proxy — set DORMICE_INGRESS_FILE (install.sh sets up Caddy and points it at /etc/caddy/Caddyfile), or configure your proxy directly', - ); - } - const previous = ingress.domains(); - try { - await ingress.setDomains(request.body.domains); - } catch (error) { - if (error instanceof UnmanagedIngressFileError) { - throw httpError(409, error.message); - } - throw httpError( - 500, - error instanceof Error ? error.message : String(error), - ); - } - // The file (not the request) is the truth to report and to log: - // read back what setDomains actually wrote (lowercased, deduped). - const domains = ingress.domains(); - const changes = [ - ...domains - .filter((domain) => !previous.includes(domain)) - .map((domain) => `bound ${domain}`), - ...previous - .filter((domain) => !domains.includes(domain)) - .map((domain) => `unbound ${domain}`), - ]; - request.log.info( - { changes, domains }, - `ingress updated: ${changes.join(', ') || 'domains unchanged'} — now serving ${ - domains.length ? domains.join(', ') : 'plain-HTTP IP access only' - }`, - ); - return { domains }; - }, - ); -}; diff --git a/packages/server/src/routes/observability.test.ts b/packages/server/src/routes/observability.test.ts index 6c1d1e33..6a45d7d2 100644 --- a/packages/server/src/routes/observability.test.ts +++ b/packages/server/src/routes/observability.test.ts @@ -1,10 +1,5 @@ -import { mkdtempSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { - type ConfigEntry, - getConfigResponseSchema, getFleetTimelineResponseSchema, getHostMetricsHistoryResponseSchema, getSandboxMetricsHistoryResponseSchema, @@ -14,9 +9,7 @@ import { } from '@dormice/shared'; import { describe, expect, it } from 'vitest'; import { buildApp } from '../app'; -import { Archiver } from '../archive/archiver'; -import { MemStore } from '../archive/mem-store'; -import { CONFIG_KEYS, type ConfigSources, loadConfig } from '../config'; +import { loadConfig } from '../config'; import { migrateDb, openDb } from '../db/db'; import { insertMetricsTick, MAX_POINTS } from '../db/metrics'; import { FAKE_BASE_IMAGE, FakeExecutor } from '../executor/fake'; @@ -24,41 +17,28 @@ import { CpuSampler, type HostSample } from '../host-metrics'; import { KeyedQueue } from '../keyed-queue'; import { freezeSandbox, stopSandbox } from '../lifecycle'; import { sampleOnce } from '../metrics-sampler'; -import { ARCHIVE_DEFAULT_SECONDS } from '../policy'; +import { configureNode, registerTestTemplate } from '../testing'; -// The observability verbs, app-level: getConfig, getSandboxMetrics and -// the history windows — the console's food, so the tests eat exactly what -// a browser would. +// The observability verbs, app-level: getSandboxMetrics, the history +// windows and the image lineage — the console's food, so the tests eat +// exactly what a browser would. const MIGRATIONS = fileURLToPath(new URL('../../drizzle', import.meta.url)); const TOKEN = 'test-token-test-token-test-token'; const authed = { authorization: `Bearer ${TOKEN}` }; -/** All-defaults source map; tests override the keys they assert on. */ -function fixedSources(overrides: Partial = {}): ConfigSources { - const all = Object.fromEntries( - Object.keys(CONFIG_KEYS).map((key) => [key, 'default']), - ) as ConfigSources; - return { ...all, ...overrides, DORMICE_API_TOKEN: 'env' }; -} - -function testApp(env: Record = {}) { +function testApp() { const db = openDb(':memory:'); migrateDb(db, MIGRATIONS); const config = loadConfig({ DORMICE_DB_PATH: ':memory:', DORMICE_NODE_ID: 'node-test', DORMICE_API_TOKEN: TOKEN, - ...env, }); + configureNode(db); const executor = new FakeExecutor(); const locks = new KeyedQueue(); - const sources = fixedSources( - Object.fromEntries( - Object.keys(env).map((key) => [key, 'env']), - ) as Partial, - ); - const app = buildApp({ config, db, executor, locks, logger: false, sources }); + const app = buildApp({ config, db, executor, locks, logger: false }); return { app, db, executor, locks }; } @@ -96,89 +76,6 @@ function hostReading(cpuUsedPct: number | null): HostSample { }; } -describe('getConfig', () => { - it('reports every knob with value and source, and validates', async () => { - const { app } = testApp({ DORMICE_SANDBOX_DISK_GB: '7' }); - const res = await rpc(app, '/getConfig'); - expect(res.statusCode).toBe(200); - const body = getConfigResponseSchema.parse(res.json()); - - const byKey = new Map(body.entries.map((e: ConfigEntry) => [e.key, e])); - // Complete: one entry per knob the config schema knows. - expect(body.entries).toHaveLength(Object.keys(CONFIG_KEYS).length); - expect(byKey.get('DORMICE_SANDBOX_DISK_GB')).toMatchObject({ - value: '7', - source: 'env', - }); - expect(byKey.get('DORMICE_PORT')).toMatchObject({ - value: '3676', - source: 'default', - }); - // Optional and unset: honestly null, not invented. - expect(byKey.get('DORMICE_SANDBOX_DOMAIN')).toMatchObject({ value: null }); - }); - - it('withholds secrets, reporting only their presence', async () => { - const { app } = testApp(); - const body = getConfigResponseSchema.parse( - (await rpc(app, '/getConfig')).json(), - ); - const token = body.entries.find( - (e: ConfigEntry) => e.key === 'DORMICE_API_TOKEN', - ); - expect(token).toMatchObject({ value: null, redacted: true }); - // The raw token must appear nowhere in the whole response. - expect(JSON.stringify(body)).not.toContain(TOKEN); - }); - - it('adjudicates archive availability: off without an archiver', async () => { - const { app } = testApp(); - const body = getConfigResponseSchema.parse( - (await rpc(app, '/getConfig')).json(), - ); - expect(body.archive).toEqual({ enabled: false, defaultSeconds: null }); - }); - - it('reports the archive default when an S3 store is configured', async () => { - // The adjudication is the ledger's, seeded here from the env S3 set. - const db = openDb(':memory:'); - migrateDb(db, MIGRATIONS); - const executor = new FakeExecutor(); - const locks = new KeyedQueue(); - const config = loadConfig({ - DORMICE_DB_PATH: ':memory:', - DORMICE_API_TOKEN: TOKEN, - DORMICE_S3_ENDPOINT: 'http://127.0.0.1:9000', - DORMICE_S3_BUCKET: 'exam', - DORMICE_S3_ACCESS_KEY_ID: 'exam-key', - DORMICE_S3_SECRET_ACCESS_KEY: 'exam-secret', - }); - const archiver = new Archiver({ - db, - executor, - locks, - store: new MemStore(), - tmpDir: mkdtempSync(path.join(tmpdir(), 'dormice-obs-')), - }); - const app = buildApp({ - config, - db, - executor, - locks, - logger: false, - sources: fixedSources(), - archiver, - }); - const body = getConfigResponseSchema.parse( - (await rpc(app, '/getConfig')).json(), - ); - expect(body.archive).toEqual({ - enabled: true, - defaultSeconds: ARCHIVE_DEFAULT_SECONDS, - }); - }); -}); - describe('getSandboxMetrics', () => { it('answers a single sample for a running sandbox', async () => { const { app } = testApp(); @@ -619,8 +516,8 @@ describe('listSandboxImages', () => { } it('walks a template upgrade: in sync, left behind, rebuilt, in sync again', async () => { - const { app } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v1' }); + const { app, db } = testApp(); + registerTestTemplate(db, 'py', 'img-v1'); const created = ( await rpc(app, '/acquireSandbox', { name: 'alice', template: 'py' }) ).json().sandbox; @@ -636,8 +533,9 @@ describe('listSandboxImages', () => { }, ]); - // Re-registering moves nextImage; the live shell honestly stays behind. - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v2' }); + // Re-pointing the template (the gateway's registerTemplate, arriving + // with the next bundle) moves nextImage; the live shell stays behind. + registerTestTemplate(db, 'py', 'img-v2'); expect(await images(app)).toMatchObject([ { image: 'img-v1', nextImage: 'img-v2', upgradable: true }, ]); @@ -666,13 +564,13 @@ describe('listSandboxImages', () => { it('answers every row: a stopped shell keeps its old image, honestly upgradable', async () => { const { app, db, executor } = testApp(); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v1' }); + registerTestTemplate(db, 'py', 'img-v1'); const created = ( await rpc(app, '/acquireSandbox', { name: 'cold', template: 'py' }) ).json().sandbox; await freezeSandbox(db, executor, created.id); await stopSandbox(db, executor, created.id); - await rpc(app, '/registerTemplate', { name: 'py', image: 'img-v2' }); + registerTestTemplate(db, 'py', 'img-v2'); // The exited container is still the shell: waking it would boot the old // image, so the row is honestly reported as upgradable. diff --git a/packages/server/src/routes/settings.test.ts b/packages/server/src/routes/settings.test.ts deleted file mode 100644 index 71fd5ce9..00000000 --- a/packages/server/src/routes/settings.test.ts +++ /dev/null @@ -1,761 +0,0 @@ -import { fileURLToPath } from 'node:url'; -import { - DEFAULT_LIFECYCLE_POLICY, - getConfigResponseSchema, - updateSettingsResponseSchema, -} from '@dormice/shared'; -import { sql } from 'drizzle-orm'; -import { afterAll, describe, expect, it, vi } from 'vitest'; -import { buildApp } from '../app'; -import { type MiniS3, startMiniS3 } from '../archive/mini-s3'; -import { S3ProbeError } from '../archive/probe'; -import type { S3Settings } from '../archive/s3-store'; -import { loadConfig } from '../config'; -import { migrateDb, openDb } from '../db/db'; -import { createSandbox, overwriteState } from '../db/ledger'; -import { readRuntimeSettings } from '../db/settings'; -import { FakeExecutor } from '../executor/fake'; -import { KeyedQueue } from '../keyed-queue'; - -const MIGRATIONS = fileURLToPath(new URL('../../drizzle', import.meta.url)); -const TOKEN = 'test-token-test-token-test-token'; -const authed = { authorization: `Bearer ${TOKEN}` }; - -/** An env S3 seed — the four core variables, as a spreadable set. */ -const S3_ENV = { - DORMICE_S3_ENDPOINT: 'http://127.0.0.1:9000', - DORMICE_S3_BUCKET: 'seed-bucket', - DORMICE_S3_ACCESS_KEY_ID: 'seed-key', - DORMICE_S3_SECRET_ACCESS_KEY: 'seed-secret-never-on-the-wire', -}; - -/** The same store as an updateSettings write-shape patch. */ -const S3_PATCH = { - endpoint: 'http://127.0.0.1:9000', - bucket: 'patched-bucket', - accessKeyId: 'patch-key', - secretAccessKey: 'patch-secret-never-on-the-wire', - region: 'us-east-1', - forcePathStyle: true, -}; - -function freshDb() { - const db = openDb(':memory:'); - migrateDb(db, MIGRATIONS); - return db; -} - -function appOn( - db: ReturnType, - env: Record = {}, - probeS3: (s3: S3Settings) => Promise = () => Promise.resolve(), - executor: FakeExecutor = new FakeExecutor(), -) { - const config = loadConfig({ - DORMICE_DB_PATH: ':memory:', - DORMICE_NODE_ID: 'node-test', - DORMICE_API_TOKEN: TOKEN, - ...env, - }); - return buildApp({ - config, - db, - executor, - locks: new KeyedQueue(), - logger: false, - // Forged by default: most tests here are about the settings machinery, - // not S3's availability. Probe-behavior tests inject their own. - probeS3, - }); -} - -type App = ReturnType; - -function rpc( - app: App, - url: string, - payload: Record = {}, - headers: Record = authed, -) { - return app.inject({ method: 'POST', url, headers, payload }); -} - -async function settingsOf(app: App) { - const res = await rpc(app, '/getConfig'); - expect(res.statusCode).toBe(200); - return getConfigResponseSchema.parse(res.json()).settings; -} - -/** Parks an archived row in the ledger — the moving-store guard's trigger. */ -function seedArchivedRow(db: ReturnType, name: string) { - const row = createSandbox(db, { - id: crypto.randomUUID(), - name, - nodeId: 'node-test', - policy: { - freezeAfterSeconds: 300, - stopAfterSeconds: 3600, - archiveAfterSeconds: 7200, - }, - }); - overwriteState(db, row.id, 'archived'); - return row; -} - -describe('runtime settings: seeding', () => { - it('seeds from the env at first boot, defaults where the env is silent', async () => { - const app = appOn(freshDb(), { DORMICE_SANDBOX_DISK_GB: '20' }); - expect(await settingsOf(app)).toEqual({ - sandboxDefaults: { cpus: 1, memoryGb: 2, diskGb: 20 }, - // No S3 seed in this env, so the seeded default never archives. - defaultPolicy: { ...DEFAULT_LIFECYCLE_POLICY, archiveAfterSeconds: null }, - s3: null, - sandboxDomain: null, - // Aliases have no env seed either — console-era editing only. - sandboxDomainAliases: [], - // Seeded from DORMICE_SANDBOX_PIDS_LIMIT (default 4096). - pidsLimit: 4096, - updatedAt: null, - }); - }); - - it('an env S3 seed lands in the ledger, keys withheld, archive default on', async () => { - const app = appOn(freshDb(), { - ...S3_ENV, - DORMICE_S3_FORCE_PATH_STYLE: 'true', - DORMICE_SANDBOX_DOMAIN: 'sbx.example.com', - }); - const res = await rpc(app, '/getConfig'); - const body = getConfigResponseSchema.parse(res.json()); - expect(body.settings.s3).toEqual({ - endpoint: 'http://127.0.0.1:9000', - bucket: 'seed-bucket', - region: 'us-east-1', - forcePathStyle: true, - }); - expect(body.settings.sandboxDomain).toBe('sbx.example.com'); - expect(body.archive.enabled).toBe(true); - // The seed's archive adjudication: an S3 seed means a 7-day default. - expect(body.settings.defaultPolicy.archiveAfterSeconds).toBe( - 7 * 24 * 60 * 60, - ); - // Neither key ever crosses the wire, in any spelling. - const raw = res.body; - expect(raw).not.toContain('seed-secret-never-on-the-wire'); - expect(raw).not.toContain('seed-key'); - }); - - it('the ledger wins over a later env edit — seeds are read once', async () => { - const db = freshDb(); - appOn(db, { DORMICE_SANDBOX_DISK_GB: '5' }); - // Same ledger, "restarted" with a different env: the row already - // exists, so the new env value is deliberately ignored... - const rebooted = appOn(db, { DORMICE_SANDBOX_DISK_GB: '9' }); - expect((await settingsOf(rebooted)).sandboxDefaults.diskGb).toBe(5); - // ...while getConfig still reports what the env says, as an entry. - const body = getConfigResponseSchema.parse( - (await rpc(rebooted, '/getConfig')).json(), - ); - expect( - body.entries.find((e) => e.key === 'DORMICE_SANDBOX_DISK_GB')?.value, - ).toBe('9'); - }); - - it('adopts env values once for columns younger than the row, then never again', async () => { - // First life: a row born before the s3/domain columns existed — - // simulated by nulling them back out (exactly what the migration - // leaves on an upgraded daemon's existing row). - const db = freshDb(); - appOn(db); - db.run( - sql`UPDATE runtime_settings SET s3_endpoint = NULL, s3_bucket = NULL, s3_access_key_id = NULL, s3_secret_access_key = NULL, s3_region = NULL, s3_force_path_style = NULL, sandbox_domain = NULL, sandbox_domain_aliases = NULL, pids_limit = NULL`, - ); - - // The upgraded daemon's first boot: virgin columns adopt the env. - const upgraded = appOn(db, { - ...S3_ENV, - DORMICE_SANDBOX_DOMAIN: 'sbx.example.com', - DORMICE_SANDBOX_PIDS_LIMIT: '2048', - }); - const adopted = await settingsOf(upgraded); - expect(adopted.s3?.bucket).toBe('seed-bucket'); - expect(adopted.sandboxDomain).toBe('sbx.example.com'); - // The alias column adopts too — always to none, no env to consult. - expect(adopted.sandboxDomainAliases).toEqual([]); - // The pids cap adopts the value the fleet has been running under — the - // env's, not the new default that would silently move it. - expect(adopted.pidsLimit).toBe(2048); - // Adoption never rewrites the standing default policy: this row - // pre-existed with "never archive", and another group's seed must not - // change it. - expect(adopted.defaultPolicy.archiveAfterSeconds).toBeNull(); - - // A later boot with a different env: the columns have spoken, the env - // is done. - const later = appOn(db, { - ...S3_ENV, - DORMICE_S3_BUCKET: 'other-bucket', - DORMICE_SANDBOX_DOMAIN: 'other.example.com', - DORMICE_SANDBOX_PIDS_LIMIT: '3000', - }); - const kept = await settingsOf(later); - expect(kept.s3?.bucket).toBe('seed-bucket'); - expect(kept.sandboxDomain).toBe('sbx.example.com'); - expect(kept.pidsLimit).toBe(2048); - }); - - it('a console clear survives a restart with the env seed still set', async () => { - const db = freshDb(); - const first = appOn(db, { - ...S3_ENV, - DORMICE_SANDBOX_DOMAIN: 'sbx.example.com', - }); - expect( - (await rpc(first, '/updateSettings', { s3: null, sandboxDomain: null })) - .statusCode, - ).toBe(200); - - // "Restart" with the same env: cleared is a decision, not a virgin - // column — the env must not resurrect either knob. - const rebooted = appOn(db, { - ...S3_ENV, - DORMICE_SANDBOX_DOMAIN: 'sbx.example.com', - }); - const settings = await settingsOf(rebooted); - expect(settings.s3).toBeNull(); - expect(settings.sandboxDomain).toBeNull(); - expect(settings.sandboxDomainAliases).toEqual([]); - }); -}); - -describe('updateSettings', () => { - it('sets the pids cap live, floors it, and never accepts unlimited', async () => { - const app = appOn(freshDb()); - expect((await settingsOf(app)).pidsLimit).toBe(4096); - - const raised = await rpc(app, '/updateSettings', { pidsLimit: 8192 }); - expect(raised.statusCode).toBe(200); - expect( - updateSettingsResponseSchema.parse(raised.json()).settings.pidsLimit, - ).toBe(8192); - expect((await settingsOf(app)).pidsLimit).toBe(8192); - - // The floor is the wire's, not the console's: below it a sandbox - // cannot boot its own runtime, so the daemon refuses, named. - const tooLow = await rpc(app, '/updateSettings', { pidsLimit: 255 }); - expect(tooLow.statusCode).toBe(400); - expect(tooLow.json().message).toMatch(/pidsLimit.*at least 256/); - expect((await settingsOf(app)).pidsLimit).toBe(8192); - // Exactly the floor passes; there is no "unlimited" spelling at all. - expect( - (await rpc(app, '/updateSettings', { pidsLimit: 256 })).statusCode, - ).toBe(200); - expect( - (await rpc(app, '/updateSettings', { pidsLimit: 0 })).statusCode, - ).toBe(400); - }); - it('sets the pids cap live and floors it', async () => { - const app = appOn(freshDb()); - const set = await rpc(app, '/updateSettings', { pidsLimit: 8192 }); - expect(set.statusCode).toBe(200); - expect( - updateSettingsResponseSchema.parse(set.json()).settings.pidsLimit, - ).toBe(8192); - expect((await settingsOf(app)).pidsLimit).toBe(8192); - - // Below the floor a sandbox cannot boot its own runtime — refused, and - // the ledger keeps the value in force. "Unlimited" has no spelling. - const low = await rpc(app, '/updateSettings', { pidsLimit: 255 }); - expect(low.statusCode).toBe(400); - expect(low.json().message).toMatch(/at least 256/); - expect((await settingsOf(app)).pidsLimit).toBe(8192); - }); - it('pidsLimit: adopted from the env on an upgraded row, then editable live with a floor', async () => { - // An upgraded daemon: the row predates the pids_limit column, and its - // env has been running the fleet at 512 — the ledger's first value - // must be that, not a default that silently moves the cap. - const db = freshDb(); - appOn(db); - db.run(sql`UPDATE runtime_settings SET pids_limit = NULL`); - const upgraded = appOn(db, { DORMICE_SANDBOX_PIDS_LIMIT: '512' }); - expect((await settingsOf(upgraded)).pidsLimit).toBe(512); - - // Below the floor is refused by the schema, ledger untouched. - const tooLow = await rpc(upgraded, '/updateSettings', { pidsLimit: 255 }); - expect(tooLow.statusCode).toBe(400); - expect(tooLow.json().message).toMatch(/at least 256/); - expect((await settingsOf(upgraded)).pidsLimit).toBe(512); - - const raised = await rpc(upgraded, '/updateSettings', { pidsLimit: 4096 }); - expect(raised.statusCode).toBe(200); - expect( - updateSettingsResponseSchema.parse(raised.json()).settings.pidsLimit, - ).toBe(4096); - - // The ledger has spoken: a later env edit is ignored. - const later = appOn(db, { DORMICE_SANDBOX_PIDS_LIMIT: '999' }); - expect((await settingsOf(later)).pidsLimit).toBe(4096); - }); - it('a new default policy applies to the next acquire, not existing sandboxes', async () => { - const app = appOn(freshDb()); - const before = await rpc(app, '/acquireSandbox', { name: 'old' }); - expect(before.json().sandbox.policy.freezeAfterSeconds).toBe( - DEFAULT_LIFECYCLE_POLICY.freezeAfterSeconds, - ); - - await rpc(app, '/updateSettings', { - defaultPolicy: { - freezeAfterSeconds: 42, - stopAfterSeconds: null, - archiveAfterSeconds: null, - }, - }); - - const created = await rpc(app, '/acquireSandbox', { name: 'new' }); - expect(created.json().sandbox.policy).toMatchObject({ - freezeAfterSeconds: 42, - stopAfterSeconds: null, - }); - // Existing sandboxes keep the policy they were born with. - const woken = await rpc(app, '/acquireSandbox', { name: 'old' }); - expect(woken.json().sandbox.policy.freezeAfterSeconds).toBe( - DEFAULT_LIFECYCLE_POLICY.freezeAfterSeconds, - ); - }); - - it('replaces provided groups whole and leaves the rest untouched', async () => { - const app = appOn(freshDb(), { DORMICE_SANDBOX_MEMORY_GB: '4' }); - await rpc(app, '/updateSettings', { pidsLimit: 512 }); - const settings = await settingsOf(app); - expect(settings.pidsLimit).toBe(512); - expect(settings.sandboxDefaults.memoryGb).toBe(4); - expect(settings.updatedAt).not.toBeNull(); - }); - - it('refuses an archiving default when no S3 store is configured', async () => { - const app = appOn(freshDb()); - const res = await rpc(app, '/updateSettings', { - defaultPolicy: { - freezeAfterSeconds: 600, - stopAfterSeconds: 3600, - archiveAfterSeconds: 7200, - }, - }); - expect(res.statusCode).toBe(400); - expect(res.json().message).toMatch(/archiving requires an S3 archive/); - // But arriving together with the store that honors it is legal. - const together = await rpc(app, '/updateSettings', { - s3: S3_PATCH, - defaultPolicy: { - freezeAfterSeconds: 600, - stopAfterSeconds: 3600, - archiveAfterSeconds: 7200, - }, - }); - expect(together.statusCode).toBe(200); - // The reverse combination promises what the same patch takes away. - const contradictory = await rpc(app, '/updateSettings', { - s3: null, - defaultPolicy: { - freezeAfterSeconds: 600, - stopAfterSeconds: 3600, - archiveAfterSeconds: 7200, - }, - }); - expect(contradictory.statusCode).toBe(400); - }); - - it('refuses an empty patch and a disordered default policy', async () => { - const app = appOn(freshDb()); - expect((await rpc(app, '/updateSettings', {})).statusCode).toBe(400); - const disordered = await rpc(app, '/updateSettings', { - defaultPolicy: { - freezeAfterSeconds: 100, - stopAfterSeconds: 50, - archiveAfterSeconds: null, - }, - }); - expect(disordered.statusCode).toBe(400); - }); - - it('masks a drifted archive default: the store cleared after it was set', async () => { - // First life: a store exists (env seed), the operator sets an - // archiving default — legal, accepted. - const app = appOn(freshDb(), S3_ENV); - const set = await rpc(app, '/updateSettings', { - defaultPolicy: { - freezeAfterSeconds: 600, - stopAfterSeconds: 3600, - archiveAfterSeconds: 7200, - }, - }); - expect(set.statusCode).toBe(200); - - // The store is cleared (no archived rows — the guard allows it). The - // stored threshold survives (and would resurface with a new store), - // but a new acquire must not be promised an archive the daemon cannot - // perform. - expect((await rpc(app, '/updateSettings', { s3: null })).statusCode).toBe( - 200, - ); - const acquired = await rpc(app, '/acquireSandbox', { name: 'drift' }); - expect(acquired.statusCode).toBe(200); - expect(acquired.json().sandbox.policy.archiveAfterSeconds).toBeNull(); - // The ledger itself still remembers the operator's choice. - expect((await settingsOf(app)).defaultPolicy.archiveAfterSeconds).toBe( - 7200, - ); - // And getConfig's adjudication flipped live, no restart involved. - const body = getConfigResponseSchema.parse( - (await rpc(app, '/getConfig')).json(), - ); - expect(body.archive).toEqual({ enabled: false, defaultSeconds: null }); - }); - - it('pidsLimit: live for the executor, floored, never unlimited, recorded', async () => { - const app = appOn(freshDb(), { DORMICE_SANDBOX_PIDS_LIMIT: '512' }); - expect((await settingsOf(app)).pidsLimit).toBe(512); - - const raised = await rpc(app, '/updateSettings', { pidsLimit: 4096 }); - expect(raised.statusCode).toBe(200); - expect( - updateSettingsResponseSchema.parse(raised.json()).settings.pidsLimit, - ).toBe(4096); - expect((await settingsOf(app)).pidsLimit).toBe(4096); - - // The floor is the wire's, not the console's: below it a sandbox - // cannot boot its own runtime, so "stricter" would mean "dead". - const tooLow = await rpc(app, '/updateSettings', { pidsLimit: 255 }); - expect(tooLow.statusCode).toBe(400); - expect(tooLow.json().message).toMatch(/at least 256/); - expect((await settingsOf(app)).pidsLimit).toBe(4096); - }); - - it('pidsLimit: running sandboxes follow the write in place, frozen ones at their wake', async () => { - const db = freshDb(); - // The daemon's wiring: the fake reads the cap live from the ledger. - const executor = new FakeExecutor( - undefined, - () => readRuntimeSettings(db).pidsLimit, - ); - const app = appOn(db, {}, undefined, executor); - const busy = (await rpc(app, '/acquireSandbox', { name: 'busy' })).json() - .sandbox.id as string; - const idle = (await rpc(app, '/acquireSandbox', { name: 'idle' })).json() - .sandbox.id as string; - expect(executor.pidsLimitOf(busy)).toBe(4096); - // Frozen behind the daemon's back the way the scanner would leave it: - // paused shell, frozen row. - await executor.freeze(idle); - overwriteState(db, idle, 'frozen'); - - const raised = await rpc(app, '/updateSettings', { pidsLimit: 2048 }); - expect(raised.statusCode).toBe(200); - // The running shell moved in place — no rebuild, same shell; the - // paused one cannot be updated and waits for its wake. - expect(executor.pidsLimitOf(busy)).toBe(2048); - expect(executor.stateOf(busy)).toBe('running'); - expect(executor.pidsLimitOf(idle)).toBe(4096); - - const woken = await rpc(app, '/acquireSandbox', { name: 'idle' }); - expect(woken.json().created).toBe(false); - expect(executor.pidsLimitOf(idle)).toBe(2048); - }); - - it('pidsLimit: a shell the runtime refuses is named, and the value stays saved', async () => { - const db = freshDb(); - const executor = new FakeExecutor( - undefined, - () => readRuntimeSettings(db).pidsLimit, - ); - const app = appOn(db, {}, undefined, executor); - await rpc(app, '/acquireSandbox', { name: 'stubborn' }); - vi.spyOn(executor, 'convergePidsLimit').mockRejectedValue( - new Error('runsc refused: no such luck'), - ); - - const res = await rpc(app, '/updateSettings', { pidsLimit: 2048 }); - expect(res.statusCode).toBe(500); - expect(res.json().message).toBe( - 'pids cap saved (2048) but 1 of 1 active sandboxes kept their old cap until their next wake — stubborn: runsc refused: no such luck', - ); - // Saved: the next birth and every wake read the new value. - expect((await settingsOf(app)).pidsLimit).toBe(2048); - }); - - it('is admin-only: an API key gets an honest 403', async () => { - const app = appOn(freshDb()); - const minted = await rpc(app, '/createApiKey', { name: 'robot' }); - expect(minted.statusCode).toBe(200); - const keyToken = minted.json().token as string; - - const refused = await rpc( - app, - '/updateSettings', - { pidsLimit: 999 }, - { authorization: `Bearer ${keyToken}` }, - ); - expect(refused.statusCode).toBe(403); - expect(refused.json().message).toMatch( - /cannot manage API keys or settings/, - ); - // And the key still opens normal doors — it is the verb that refused. - expect( - ( - await rpc( - app, - '/listSandboxes', - {}, - { authorization: `Bearer ${keyToken}` }, - ) - ).statusCode, - ).toBe(200); - }); -}); - -describe('updateSettings: the S3 archive store', () => { - // One real store for the probe's happy path — the store contract suite - // already pins S3Store against miniS3; here it proves the probe's - // round trip is the real plumbing, not a stub agreeing with itself. - let miniS3: MiniS3 | undefined; - afterAll(async () => { - await miniS3?.close(); - }); - - it('accepts a store that passes the real probe and reports it keyless', async () => { - miniS3 ??= await startMiniS3(); - const db = freshDb(); - const app = buildApp({ - config: loadConfig({ - DORMICE_DB_PATH: ':memory:', - DORMICE_NODE_ID: 'node-test', - DORMICE_API_TOKEN: TOKEN, - }), - db, - executor: new FakeExecutor(), - locks: new KeyedQueue(), - logger: false, - // No probeS3 injected: the route's real probe runs against miniS3. - }); - const res = await rpc(app, '/updateSettings', { - s3: { - endpoint: miniS3.url, - bucket: 'exam-bucket', - accessKeyId: 'exam-key', - secretAccessKey: 'exam-secret-never-on-the-wire', - region: 'us-east-1', - forcePathStyle: true, - }, - }); - expect(res.statusCode).toBe(200); - const view = updateSettingsResponseSchema.parse(res.json()).settings.s3; - expect(view).toEqual({ - endpoint: miniS3.url, - bucket: 'exam-bucket', - region: 'us-east-1', - forcePathStyle: true, - }); - expect(res.body).not.toContain('exam-secret-never-on-the-wire'); - // The probe cleaned up after itself: no probe object left behind. - expect(miniS3.objects.size).toBe(0); - // The adjudication flipped live: archiving is now available. - const body = getConfigResponseSchema.parse( - (await rpc(app, '/getConfig')).json(), - ); - expect(body.archive.enabled).toBe(true); - }); - - it('an unreachable store answers 502 and the ledger stays untouched', async () => { - const db = freshDb(); - const app = buildApp({ - config: loadConfig({ - DORMICE_DB_PATH: ':memory:', - DORMICE_NODE_ID: 'node-test', - DORMICE_API_TOKEN: TOKEN, - }), - db, - executor: new FakeExecutor(), - locks: new KeyedQueue(), - logger: false, - // Real probe against a port nothing listens on: connection refused. - }); - const res = await rpc(app, '/updateSettings', { - s3: { ...S3_PATCH, endpoint: 'http://127.0.0.1:1' }, - }); - expect(res.statusCode).toBe(502); - expect(res.json().message).toMatch(/nothing was saved/); - expect((await settingsOf(app)).s3).toBeNull(); - }); - - it("an S3-refused probe (4xx) answers 400 with S3's own words", async () => { - const app = appOn(freshDb(), {}, () => - Promise.reject(new S3ProbeError('AccessDenied: key rejected', 403)), - ); - const res = await rpc(app, '/updateSettings', { s3: S3_PATCH }); - expect(res.statusCode).toBe(400); - expect(res.json().message).toMatch(/AccessDenied: key rejected/); - expect((await settingsOf(app)).s3).toBeNull(); - }); - - it('refuses to clear or move the store while sandboxes are archived, by count', async () => { - const db = freshDb(); - const app = appOn(db, S3_ENV); - seedArchivedRow(db, 'held-1'); - seedArchivedRow(db, 'held-2'); - - const cleared = await rpc(app, '/updateSettings', { s3: null }); - expect(cleared.statusCode).toBe(400); - expect(cleared.json().message).toMatch(/2 sandboxes are archived/); - - const moved = await rpc(app, '/updateSettings', { - s3: { ...S3_PATCH, endpoint: S3_ENV.DORMICE_S3_ENDPOINT }, - }); - expect(moved.statusCode).toBe(400); - expect(moved.json().message).toMatch(/moving it to another/); - - // Same endpoint+bucket, new credentials: nothing moves, allowed. - const rotated = await rpc(app, '/updateSettings', { - s3: { - ...S3_PATCH, - endpoint: S3_ENV.DORMICE_S3_ENDPOINT, - bucket: S3_ENV.DORMICE_S3_BUCKET, - }, - }); - expect(rotated.statusCode).toBe(200); - expect((await settingsOf(app)).s3?.bucket).toBe('seed-bucket'); - }); - - it('enabling from off is allowed even with archived rows — the drift repair path', async () => { - const db = freshDb(); - const app = appOn(db); - seedArchivedRow(db, 'stranded'); - const res = await rpc(app, '/updateSettings', { s3: S3_PATCH }); - expect(res.statusCode).toBe(200); - expect((await settingsOf(app)).s3?.bucket).toBe('patched-bucket'); - }); - - it('never echoes the keys: the write answers with the view shape', async () => { - const app = appOn(freshDb()); - const res = await rpc(app, '/updateSettings', { s3: S3_PATCH }); - expect(res.statusCode).toBe(200); - expect(res.body).not.toContain('patch-secret-never-on-the-wire'); - expect(res.body).not.toContain('patch-key'); - }); -}); - -describe('updateSettings: the sandbox domain', () => { - it('sets, reports and clears the domain, with immediate effect on getConfig', async () => { - const app = appOn(freshDb()); - const set = await rpc(app, '/updateSettings', { - sandboxDomain: 'sbx.example.com', - }); - expect(set.statusCode).toBe(200); - expect( - updateSettingsResponseSchema.parse(set.json()).settings.sandboxDomain, - ).toBe('sbx.example.com'); - expect((await settingsOf(app)).sandboxDomain).toBe('sbx.example.com'); - - const cleared = await rpc(app, '/updateSettings', { sandboxDomain: null }); - expect(cleared.statusCode).toBe(200); - expect((await settingsOf(app)).sandboxDomain).toBeNull(); - }); - - it('refuses anything but a bare hostname', async () => { - const app = appOn(freshDb()); - for (const bad of [ - 'https://sbx.example.com', - 'sbx.example.com:8080', - '.sbx.example.com', - 'single-label', - ]) { - const res = await rpc(app, '/updateSettings', { sandboxDomain: bad }); - expect(res.statusCode, bad).toBe(400); - const alias = await rpc(app, '/updateSettings', { - sandboxDomain: 'sbx.example.com', - sandboxDomainAliases: [bad], - }); - expect(alias.statusCode, bad).toBe(400); - } - }); - - it('sets, reports and clears aliases, with immediate effect on getConfig', async () => { - const app = appOn(freshDb(), { - DORMICE_SANDBOX_DOMAIN: 'sbx.example.com', - }); - const set = await rpc(app, '/updateSettings', { - sandboxDomainAliases: ['a.example.com', 'b.example.com'], - }); - expect(set.statusCode).toBe(200); - expect( - updateSettingsResponseSchema.parse(set.json()).settings - .sandboxDomainAliases, - ).toEqual(['a.example.com', 'b.example.com']); - expect((await settingsOf(app)).sandboxDomainAliases).toEqual([ - 'a.example.com', - 'b.example.com', - ]); - - const cleared = await rpc(app, '/updateSettings', { - sandboxDomainAliases: [], - }); - expect(cleared.statusCode).toBe(200); - expect((await settingsOf(app)).sandboxDomainAliases).toEqual([]); - }); - - it('refuses alias lists that contradict the post-patch state, honestly', async () => { - const app = appOn(freshDb(), { - DORMICE_SANDBOX_DOMAIN: 'sbx.example.com', - }); - - // Duplicates within the list — hostnames compare case-insensitively. - const dup = await rpc(app, '/updateSettings', { - sandboxDomainAliases: ['a.example.com', 'A.example.com'], - }); - expect(dup.statusCode).toBe(400); - expect(dup.json().message).toContain('more than once'); - - // The canonical domain listed as its own alias, in any casing. - const overlap = await rpc(app, '/updateSettings', { - sandboxDomainAliases: ['SBX.example.com'], - }); - expect(overlap.statusCode).toBe(400); - expect(overlap.json().message).toContain('already the sandbox domain'); - - // Aliases with no canonical domain in force. - const orphanApp = appOn(freshDb()); - const orphan = await rpc(orphanApp, '/updateSettings', { - sandboxDomainAliases: ['a.example.com'], - }); - expect(orphan.statusCode).toBe(400); - expect(orphan.json().message).toContain('set sandboxDomain first'); - - // Clearing the canonical domain while aliases stand — the refusal - // names the field that must clear alongside. - expect( - ( - await rpc(app, '/updateSettings', { - sandboxDomainAliases: ['a.example.com'], - }) - ).statusCode, - ).toBe(200); - const dangling = await rpc(app, '/updateSettings', { sandboxDomain: null }); - expect(dangling.statusCode).toBe(400); - expect(dangling.json().message).toContain('sandboxDomainAliases'); - - // The same intents, expressed whole, pass: the atomic swap... - const swap = await rpc(app, '/updateSettings', { - sandboxDomain: 'a.example.com', - sandboxDomainAliases: ['sbx.example.com'], - }); - expect(swap.statusCode).toBe(200); - const swapped = updateSettingsResponseSchema.parse(swap.json()).settings; - expect(swapped.sandboxDomain).toBe('a.example.com'); - expect(swapped.sandboxDomainAliases).toEqual(['sbx.example.com']); - // ...and the full clear. - const clear = await rpc(app, '/updateSettings', { - sandboxDomain: null, - sandboxDomainAliases: [], - }); - expect(clear.statusCode).toBe(200); - }); -}); diff --git a/packages/server/src/routes/settings.ts b/packages/server/src/routes/settings.ts deleted file mode 100644 index 9bd9ecc7..00000000 --- a/packages/server/src/routes/settings.ts +++ /dev/null @@ -1,229 +0,0 @@ -import { - updateSettingsRequestSchema, - updateSettingsResponseSchema, -} from '@dormice/shared'; -import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; -import { z } from 'zod'; -import { probeS3 as defaultProbeS3, S3ProbeError } from '../archive/probe'; -import type { S3Settings } from '../archive/s3-store'; -import type { Db } from '../db/db'; -import { countByState, listSandboxes } from '../db/ledger'; -import { - archiveEnabled, - readRuntimeSettings, - writeRuntimeSettings, -} from '../db/settings'; -import type { Executor } from '../executor/executor'; -import type { KeyedQueue } from '../keyed-queue'; -import { sweepPidsLimit } from '../pids-sweep'; - -export interface SettingsRoutesOptions { - db: Db; - /** - * For the pids cap's sweep over running shells after a write — the one - * settings knob with a reality on every running sandbox. Same executor - * and per-sandbox queue as the rest of the daemon. - */ - executor: Executor; - locks: KeyedQueue; - /** Test seam over the S3 round-trip probe; production uses the real one. */ - probeS3?: (s3: S3Settings) => Promise; -} - -/** - * updateSettings — the write half of the runtime settings (the read rides - * on getConfig). Registered in the ADMIN scope: env token or console - * session only, like the apiKey verbs — a leaked automation key must not - * be able to raise the very limits that contain it. - * - * A ledger write with immediate effect: the consumers read live (the - * executor's births, resolvePolicy's defaults, the archiver's store, the - * sandbox proxy's domain, the executor's pids cap at each birth and - * wake), so nothing here restarts or wakes a sandbox. One knob has a - * reality on the host that the write alone does not move, and it is - * reconciled right after: the pids cap on the shells running right now (a - * cgroup write their processes never notice). - */ -export const settingsRoutes: FastifyPluginAsyncZod< - SettingsRoutesOptions -> = async (app, { db, executor, locks, probeS3 = defaultProbeS3 }) => { - app.post( - '/updateSettings', - { - schema: { - body: updateSettingsRequestSchema, - response: { - 200: updateSettingsResponseSchema, - 400: z.object({ message: z.string() }), - 500: z.object({ message: z.string() }), - 502: z.object({ message: z.string() }), - }, - }, - }, - async (request, reply) => { - const patch = request.body; - // The updatePolicy doctrine, judged against the post-patch state (an - // s3 group and an archiving default may arrive in one patch): a - // default that promises archiving on a daemon with no store would be - // a standing lie in every acquire. - const s3After = - patch.s3 !== undefined ? patch.s3 !== null : archiveEnabled(db); - if ( - !s3After && - patch.defaultPolicy !== undefined && - patch.defaultPolicy.archiveAfterSeconds !== null - ) { - return reply.code(400).send({ - message: - 'invalid default policy: archiving requires an S3 archive store — configure one in the console settings first', - }); - } - // The alias-list guard, judged against the post-patch state — domain - // and aliases may arrive in one patch, which is exactly how the - // console swaps the canonical domain atomically. Violations are - // refused, never silently rewritten (the echoed settings must be - // what was written). Before the s3 block on purpose: pure in-memory - // checks don't queue behind a network probe. - if ( - patch.sandboxDomain !== undefined || - patch.sandboxDomainAliases !== undefined - ) { - const current = readRuntimeSettings(db); - const domainAfter = - patch.sandboxDomain !== undefined - ? patch.sandboxDomain - : current.sandboxDomain; - const aliasesAfter = - patch.sandboxDomainAliases ?? current.sandboxDomainAliases; - const lower = aliasesAfter.map((alias) => alias.toLowerCase()); - const dup = aliasesAfter.find( - (alias, i) => lower.indexOf(alias.toLowerCase()) !== i, - ); - if (dup !== undefined) { - return reply.code(400).send({ - message: `sandboxDomainAliases lists ${dup} more than once — hostnames are case-insensitive, send each alias exactly once`, - }); - } - if (domainAfter !== null && lower.includes(domainAfter.toLowerCase())) { - return reply.code(400).send({ - message: `${domainAfter} is already the sandbox domain — sandboxDomainAliases only takes the extra hostnames`, - }); - } - if (domainAfter === null && aliasesAfter.length > 0) { - return reply.code(400).send({ - message: - patch.sandboxDomain === null - ? `clearing sandboxDomain would leave ${aliasesAfter.length} alias${aliasesAfter.length === 1 ? '' : 'es'} pointing at nothing — clear sandboxDomainAliases (send []) in the same request` - : 'sandboxDomainAliases needs a sandbox domain in force — set sandboxDomain first', - }); - } - } - if (patch.s3 !== undefined) { - // The moving-store guard: archived disks live in the current - // endpoint+bucket, and pointing elsewhere (or clearing) would - // strand them. Enabling from off is always allowed — when drift - // left archived rows behind with no store, pointing back at the - // original bucket is the one repair path. Credential/region/ - // path-style changes move nothing and pass freely. - const current = readRuntimeSettings(db).s3; - const moving = - patch.s3 === null || - (current !== null && - (patch.s3.endpoint !== current.endpoint || - patch.s3.bucket !== current.bucket)); - if (current !== null && moving) { - const { byState } = countByState(listSandboxes(db)); - const held = byState.archived + byState.restoring; - if (held > 0) { - return reply.code(400).send({ - message: `${held} sandbox${held === 1 ? ' is' : 'es are'} archived or restoring in the current store — restore or destroy them before ${ - patch.s3 === null - ? 'clearing the archive store' - : 'moving it to another endpoint or bucket' - }`, - }); - } - } - if (patch.s3 !== null) { - // Probe BEFORE the write — a failure leaves the ledger untouched - // (see archive/probe.ts for why this is the opposite of swap's - // save-then-reconcile). - try { - await probeS3(patch.s3); - } catch (error) { - const probeFailure = - error instanceof S3ProbeError - ? error - : new S3ProbeError( - error instanceof Error ? error.message : String(error), - undefined, - ); - const status = - probeFailure.httpStatusCode !== undefined && - probeFailure.httpStatusCode >= 400 && - probeFailure.httpStatusCode < 500 - ? (400 as const) - : (502 as const); - return reply.code(status).send({ - message: `the S3 store did not pass a write-read-delete probe, nothing was saved — ${probeFailure.message}`, - }); - } - } - } - const settings = writeRuntimeSettings(db, patch, new Date()); - request.log.info( - { settings }, - `runtime settings updated: ${[ - ...(patch.sandboxDefaults !== undefined - ? [ - `sandboxDefaults=${patch.sandboxDefaults.cpus}cpu/${patch.sandboxDefaults.memoryGb}GiB/${patch.sandboxDefaults.diskGb}GiB`, - ] - : []), - ...(patch.defaultPolicy !== undefined - ? [ - `defaultPolicy=${patch.defaultPolicy.freezeAfterSeconds}s/${patch.defaultPolicy.stopAfterSeconds ?? 'never'}/${patch.defaultPolicy.archiveAfterSeconds ?? 'never'}`, - ] - : []), - // Endpoint and bucket only — the keys never reach the log, the - // same "value never crosses" rule as the wire's. - ...(patch.s3 !== undefined - ? [ - patch.s3 === null - ? 's3=cleared' - : `s3=${patch.s3.endpoint}/${patch.s3.bucket}`, - ] - : []), - ...(patch.sandboxDomain !== undefined - ? [`sandboxDomain=${patch.sandboxDomain ?? 'cleared'}`] - : []), - // '/' inside the list — ',' is the fragment separator above. - ...(patch.sandboxDomainAliases !== undefined - ? [ - patch.sandboxDomainAliases.length === 0 - ? 'sandboxDomainAliases=cleared' - : `sandboxDomainAliases=${patch.sandboxDomainAliases.join('/')}`, - ] - : []), - ...(patch.pidsLimit !== undefined - ? [`pidsLimit=${patch.pidsLimit}`] - : []), - ].join(', ')}`, - ); - // Pids cap: the write already reached every future birth and wake; - // the sweep brings the shells running right now along — an operator - // raising the cap during an incident is looking at exactly those. A - // shell the runtime refuses keeps its old cap until its next wake, - // and the answer says so by name; the value stays saved either way. - if (patch.pidsLimit !== undefined) { - const sweep = await sweepPidsLimit(db, executor, locks); - app.log.info(sweep, 'pids cap sweep after updateSettings'); - if (sweep.failures.length > 0) { - return reply.code(500).send({ - message: `pids cap saved (${patch.pidsLimit}) but ${sweep.failures.length} of ${sweep.considered} active sandboxes kept their old cap until their next wake — ${sweep.failures[0]}`, - }); - } - } - return { settings }; - }, - ); -}; diff --git a/packages/server/src/routes/spec.test.ts b/packages/server/src/routes/spec.test.ts index de3efead..6a06670e 100644 --- a/packages/server/src/routes/spec.test.ts +++ b/packages/server/src/routes/spec.test.ts @@ -13,6 +13,7 @@ import { readRuntimeSettings } from '../db/settings'; import { FakeExecutor } from '../executor/fake'; import { KeyedQueue } from '../keyed-queue'; import { scanOnce } from '../scanner'; +import { configureNode, TEST_S3, type TestConfig } from '../testing'; const MIGRATIONS = fileURLToPath(new URL('../../drizzle', import.meta.url)); const TOKEN = 'test-token-test-token-test-token'; @@ -26,7 +27,7 @@ const TOKEN = 'test-token-test-token-test-token'; */ function testApp( executor: FakeExecutor = new FakeExecutor(), - env: Record = {}, + configured: TestConfig = {}, ) { const db = openDb(':memory:'); migrateDb(db, MIGRATIONS); @@ -34,8 +35,8 @@ function testApp( DORMICE_DB_PATH: ':memory:', DORMICE_NODE_ID: 'node-test', DORMICE_API_TOKEN: TOKEN, - ...env, }); + configureNode(db, configured); const locks = new KeyedQueue(); const app = buildApp({ config, db, executor, locks, logger: false }); return { app, db, executor, locks }; @@ -49,11 +50,8 @@ function archiverTestApp(executor: FakeExecutor = new FakeExecutor()) { DORMICE_DB_PATH: ':memory:', DORMICE_NODE_ID: 'node-test', DORMICE_API_TOKEN: TOKEN, - DORMICE_S3_ENDPOINT: 'http://127.0.0.1:9000', - DORMICE_S3_BUCKET: 'exam', - DORMICE_S3_ACCESS_KEY_ID: 'exam-key', - DORMICE_S3_SECRET_ACCESS_KEY: 'exam-secret', }); + configureNode(db, { s3: TEST_S3 }); const locks = new KeyedQueue(); const store = new MemStore(); const archiver = new Archiver({ @@ -258,6 +256,7 @@ describe('a global default edit through the cold-wake convergence', () => { memoryGb: sandboxDefaults.memoryGb, }; }); + configureNode(db); const config = loadConfig({ DORMICE_DB_PATH: ':memory:', DORMICE_NODE_ID: 'node-test', @@ -279,11 +278,10 @@ describe('a global default edit through the cold-wake convergence', () => { memoryBytes: 2 * 1024 ** 3, }); - // The fleet-wide knob moves; alice is unpinned (all columns NULL), so - // her next cold wake owes a rebuild onto the new limits. - await rpc(app, '/updateSettings', { - sandboxDefaults: { cpus: 2, memoryGb: 4, diskGb: 10 }, - }); + // The fleet-wide knob moves (at the gateway; the bundle brings it + // here); alice is unpinned (all columns NULL), so her next cold wake + // owes a rebuild onto the new limits. + configureNode(db, { cpus: 2, memoryGb: 4, diskGb: 10 }); const { lastActiveAt } = (await rpc(app, '/listSandboxes')).json() .sandboxes[0]; await scanOnce(db, executor, locks, after(lastActiveAt, 1)); diff --git a/packages/server/src/routes/templates.ts b/packages/server/src/routes/templates.ts deleted file mode 100644 index 1cd335f8..00000000 --- a/packages/server/src/routes/templates.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { - listTemplatesResponseSchema, - registerTemplateRequestSchema, - registerTemplateResponseSchema, - removeTemplateRequestSchema, - removeTemplateResponseSchema, -} from '@dormice/shared'; -import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; -import type { Db } from '../db/db'; -import { - listTemplates, - registerTemplate, - removeTemplate, - sandboxNamesUsingTemplate, -} from '../db/templates'; -import { httpError } from '../http-error'; - -export interface TemplateRoutesOptions { - db: Db; -} - -/** - * Template registration: a name for a Docker image that already lives on - * this host. Pure ledger config — no executor involved: the image is not - * checked for existence (it may legitimately arrive after registration; a - * missing one fails a later create with Docker's own honest error), and - * removal only guards the ledger's referential integrity. - */ -export const templateRoutes: FastifyPluginAsyncZod< - TemplateRoutesOptions -> = async (app, { db }) => { - // Upsert: re-registering a name re-points it at the new image — the - // template upgrade front door. No lock: better-sqlite3 is synchronous, - // there is no await between check and write. - app.post( - '/registerTemplate', - { - schema: { - body: registerTemplateRequestSchema, - response: { 200: registerTemplateResponseSchema }, - }, - }, - async (request) => ({ - template: registerTemplate(db, request.body), - }), - ); - - app.post( - '/listTemplates', - { - schema: { - response: { 200: listTemplatesResponseSchema }, - }, - }, - async () => ({ templates: listTemplates(db) }), - ); - - app.post( - '/removeTemplate', - { - schema: { - body: removeTemplateRequestSchema, - response: { 200: removeTemplateResponseSchema }, - }, - }, - async (request) => { - const { name } = request.body; - // Refused while referenced: a sandbox row pointing at a removed - // template would wake onto a dangling name. Named keys, so the - // operator knows exactly what to destroy. - const users = sandboxNamesUsingTemplate(db, name); - if (users.length > 0) { - throw httpError( - 409, - `template '${name}' is used by ${users.length} sandbox(es): ${users.join(', ')} — destroy them first`, - ); - } - return { removed: removeTemplate(db, name) }; - }, - ); -}; diff --git a/packages/server/src/sandbox-proxy.test.ts b/packages/server/src/sandbox-proxy.test.ts index 0c23c852..21f3c80a 100644 --- a/packages/server/src/sandbox-proxy.test.ts +++ b/packages/server/src/sandbox-proxy.test.ts @@ -13,6 +13,7 @@ import { FakeExecutor } from './executor/fake'; import { KeyedQueue } from './keyed-queue'; import { parseSandboxHost } from './sandbox-proxy'; import { scanOnce } from './scanner'; +import { configureNode } from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); const TOKEN = 'test-token-test-token-test-token'; @@ -33,13 +34,14 @@ describe('sandbox port proxy', () => { async function listeningApp(opts: { domain?: string | null } = {}) { const db = openDb(':memory:'); migrateDb(db, MIGRATIONS); - // domain: null boots with no sandbox domain at all (no env seed). - const domainSeed = opts.domain === undefined ? DOMAIN : opts.domain; + // domain: null boots with no sandbox domain at all in the copy. + configureNode(db, { + sandboxDomain: opts.domain === undefined ? DOMAIN : opts.domain, + }); const config = loadConfig({ DORMICE_DB_PATH: ':memory:', DORMICE_NODE_ID: 'node-test', DORMICE_API_TOKEN: TOKEN, - ...(domainSeed === null ? {} : { DORMICE_SANDBOX_DOMAIN: domainSeed }), }); const executor = new FakeExecutor(); const locks = new KeyedQueue(); @@ -446,18 +448,9 @@ describe('sandbox port proxy', () => { const before = await rawGet(t.port, '/hello', host); expect(before.status).toBe(404); - // The console sets the domain — same daemon, no restart. - const set = await rawRequest(t.port, { - method: 'POST', - path: '/updateSettings', - host: '127.0.0.1', - headers: { - authorization: `Bearer ${TOKEN}`, - 'content-type': 'application/json', - }, - body: JSON.stringify({ sandboxDomain: DOMAIN }), - }); - expect(set.status).toBe(200); + // The console sets the domain at the gateway; the bundle brings it + // here — same daemon, no restart. + configureNode(t.db, { sandboxDomain: DOMAIN }); // The always-mounted proxy now matches, and create responses carry it. const after = await rawGet(t.port, '/hello', host); @@ -466,17 +459,7 @@ describe('sandbox port proxy', () => { await createSandbox(t.port); // Cleared: back to stock-Fastify behavior for the same Host. - const clear = await rawRequest(t.port, { - method: 'POST', - path: '/updateSettings', - host: '127.0.0.1', - headers: { - authorization: `Bearer ${TOKEN}`, - 'content-type': 'application/json', - }, - body: JSON.stringify({ sandboxDomain: null }), - }); - expect(clear.status).toBe(200); + configureNode(t.db, { sandboxDomain: null }); expect((await rawGet(t.port, '/hello', host)).status).toBe(404); }); @@ -486,18 +469,11 @@ describe('sandbox port proxy', () => { const ALIAS = 'alias.dormice.test'; const aliasHost = `8000-${sandboxId}.${ALIAS}`; const canonicalHost = `8000-${sandboxId}.${DOMAIN}`; - const setSettings = async (body: object) => { - const res = await rawRequest(t.port, { - method: 'POST', - path: '/updateSettings', - host: '127.0.0.1', - headers: { - authorization: `Bearer ${TOKEN}`, - 'content-type': 'application/json', - }, - body: JSON.stringify(body), - }); - expect(res.status).toBe(200); + const setSettings = async (body: { + sandboxDomain?: string | null; + sandboxDomainAliases?: string[]; + }) => { + configureNode(t.db, body); }; // Not listed yet: ordinary Fastify traffic. diff --git a/packages/server/src/sandbox-proxy.ts b/packages/server/src/sandbox-proxy.ts index a7eb6fcb..c237e809 100644 --- a/packages/server/src/sandbox-proxy.ts +++ b/packages/server/src/sandbox-proxy.ts @@ -57,7 +57,9 @@ function isEnvdFilesRequest(req: http.IncomingMessage): boolean { * the proxy's per-request getter and the signed-URL host pin both call * this instead of deciding it themselves. */ -export function sandboxDomainsInForce(settings: RuntimeSettings): string[] { +export function sandboxDomainsInForce( + settings: Pick, +): string[] { return settings.sandboxDomain ? [settings.sandboxDomain, ...settings.sandboxDomainAliases] : []; diff --git a/packages/server/src/testing.ts b/packages/server/src/testing.ts new file mode 100644 index 00000000..073bd77c --- /dev/null +++ b/packages/server/src/testing.ts @@ -0,0 +1,151 @@ +import { + DEFAULT_LIFECYCLE_POLICY, + type LifecyclePolicy, + type NodeConfigBundle, + type S3ArchiveSettings, +} from '@dormice/shared'; +import type { Db } from './db/db'; +import { + applyNodeConfig, + readConfigVersion, + readNodeConfig, +} from './db/settings'; +import { ARCHIVE_DEFAULT_SECONDS } from './policy'; + +/** + * Test scaffolding for the suites that embed the daemon (this package's, + * the SDK's, the CLI's): a configuration bundle with a few knobs turned, + * applied the way a check-in would apply it. The daemon reads every knob + * from its copy, so a test that wants a domain, a store or a template + * configures the node instead of seeding an environment — the same + * shape production takes, one write. + */ +export interface TestConfig { + cpus?: number; + memoryGb?: number; + diskGb?: number; + defaultPolicy?: LifecyclePolicy; + /** A store turns the archive default on (one week), as the gateway's seed does. */ + s3?: S3ArchiveSettings | null; + sandboxDomain?: string | null; + sandboxDomainAliases?: string[]; + pidsLimit?: number; + swapGb?: number; + /** Replaces the whole template list; timestamps are stamped now. */ + templates?: Array<{ name: string; image: string }>; +} + +/** The exam's stand-in store: the four fields the ledger echoes back keyless, plus keys that must never reach a wire. */ +export const TEST_S3: S3ArchiveSettings = { + endpoint: 'http://127.0.0.1:9000', + bucket: 'exam', + accessKeyId: 'exam-key', + secretAccessKey: 'exam-secret', + region: 'us-east-1', + forcePathStyle: true, +}; + +export function testBundle( + over: TestConfig = {}, + version = 1, + now = new Date(), +): NodeConfigBundle { + const s3 = over.s3 ?? null; + const stamp = now.toISOString(); + return { + version, + settings: { + sandboxDefaults: { + cpus: over.cpus ?? 1, + memoryGb: over.memoryGb ?? 2, + diskGb: over.diskGb ?? 10, + }, + defaultPolicy: over.defaultPolicy ?? { + ...DEFAULT_LIFECYCLE_POLICY, + archiveAfterSeconds: s3 === null ? null : ARCHIVE_DEFAULT_SECONDS, + }, + s3, + sandboxDomain: over.sandboxDomain ?? null, + sandboxDomainAliases: over.sandboxDomainAliases ?? [], + pidsLimit: over.pidsLimit ?? 4096, + }, + node: { swapGb: over.swapGb ?? 0 }, + templates: (over.templates ?? []).map((t) => ({ + ...t, + createdAt: stamp, + updatedAt: stamp, + })), + }; +} + +/** + * Applies a bundle built from `over` on top of the copy the node already + * holds (or the defaults when it holds none), one version up — a test's + * "the gateway changed X" in one call. Templates given replace the list; + * absent, the current list stays. + */ +export function configureNode(db: Db, over: TestConfig = {}): NodeConfigBundle { + const current = readConfigVersion(db) === null ? null : readNodeConfig(db); + const fresh = testBundle(over, (current?.version ?? 0) + 1); + const bundle: NodeConfigBundle = + current === null + ? fresh + : { + version: fresh.version, + settings: { + sandboxDefaults: + over.cpus !== undefined || + over.memoryGb !== undefined || + over.diskGb !== undefined + ? fresh.settings.sandboxDefaults + : current.settings.sandboxDefaults, + defaultPolicy: + over.defaultPolicy !== undefined || over.s3 !== undefined + ? fresh.settings.defaultPolicy + : current.settings.defaultPolicy, + s3: over.s3 !== undefined ? over.s3 : current.settings.s3, + sandboxDomain: + over.sandboxDomain !== undefined + ? over.sandboxDomain + : current.settings.sandboxDomain, + sandboxDomainAliases: + over.sandboxDomainAliases ?? + current.settings.sandboxDomainAliases, + pidsLimit: over.pidsLimit ?? current.settings.pidsLimit, + }, + node: { swapGb: over.swapGb ?? current.node.swapGb }, + templates: + over.templates !== undefined ? fresh.templates : current.templates, + }; + applyNodeConfig(db, bundle); + return bundle; +} + +/** + * Re-points (or adds) one template in the node's copy, the way the + * gateway's registerTemplate then the next check-in would: the birth date + * stays, the upgrade timestamp moves only when the image changes. + */ +export function registerTestTemplate( + db: Db, + name: string, + image: string, + now = new Date(), +): void { + const current = readConfigVersion(db) === null ? null : readNodeConfig(db); + const stamp = now.toISOString(); + const existing = current?.templates.find((t) => t.name === name); + const templates = [ + ...(current?.templates.filter((t) => t.name !== name) ?? []), + existing === undefined + ? { name, image, createdAt: stamp, updatedAt: stamp } + : existing.image === image + ? existing + : { ...existing, image, updatedAt: stamp }, + ].sort((a, b) => a.name.localeCompare(b.name)); + applyNodeConfig(db, { + ...(current ?? testBundle()), + version: (current?.version ?? 0) + 1, + templates, + }); +} diff --git a/packages/shared/src/config.ts b/packages/shared/src/config.ts index de8d9348..22bf1370 100644 --- a/packages/shared/src/config.ts +++ b/packages/shared/src/config.ts @@ -44,6 +44,14 @@ export const getConfigResponseSchema = z.object({ }), /** The ledger-resident operator knobs actually in force — see settings.ts. */ settings: runtimeSettingsSchema, + /** + * The fleet configuration's version (gateway.ts nodeConfigBundleSchema): + * counted up by every settings, template or node-settings write. Beside + * listNodes' per-node `configVersion` it answers "has my change reached + * every node yet" — a node reporting this number runs exactly this + * configuration. + */ + configVersion: z.number().int().positive(), }); export type GetConfigResponse = z.infer; diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts index 201de5a4..3803e4e9 100644 --- a/packages/shared/src/gateway.ts +++ b/packages/shared/src/gateway.ts @@ -4,6 +4,14 @@ import { hostReadingSchema, sandboxStateCountsSchema, } from './host'; +import { lifecyclePolicySchema } from './policy'; +import { + bareHostnameRegex, + PIDS_LIMIT_MIN, + s3ArchiveSettingsSchema, + sandboxResourceDefaultsSchema, +} from './settings'; +import { templateSchema } from './templates'; /** * The gateway's wire: the verbs between a node and the gateway that fronts @@ -73,6 +81,16 @@ export const nodeReadingSchema = z.object({ total: z.number().int(), byState: sandboxStateCountsSchema, }), + /** + * The daemon-managed swap on this node's data disk (server/swap.ts): + * what is mounted right now. Null where the daemon cannot manage swap + * at all — a non-Linux host, the fake executor — so the gateway refuses + * a target for this node (updateNodeSettings) instead of storing one + * nothing will ever reconcile. + */ + managedSwap: z + .object({ activeGb: z.number().int().nonnegative() }) + .nullable(), }); export type NodeReading = z.infer; @@ -97,12 +115,59 @@ export const checkInRequestSchema = z.object({ intervalSeconds: z.number().int().positive(), build: buildInfoSchema.nullable(), reading: nodeReadingSchema, + /** + * The configuration version this node runs — its copy's + * (nodeConfigBundleSchema); null while it holds no copy. The gateway + * compares it with its own and answers the whole bundle when the two + * differ: the check-in IS the pull, there is no second verb and nothing + * the gateway has to remember about who was told what. + */ + configVersion: z.number().int().nullable(), }); export type CheckInRequest = z.infer; -/** Nothing yet: the configuration version the gateway will answer with arrives with the configuration authority. */ -export const checkInResponseSchema = z.object({}); +/** + * The configuration a node runs, whole (design record #22: the gateway + * holds the fleet's configuration, every node keeps a copy). The fleet + * settings — the archive store WITH its keys, since the node must present + * them to S3 verbatim; this bundle crosses only the gateway→node wire + * under the fleet token and is never an observation answer — this node's + * own row (its managed-swap target) and every template. Whole on purpose, + * not a diff: the node applies it in one transaction and then holds + * exactly what the gateway holds under that version, nothing to merge and + * nothing to miss. It rides on the check-in response whenever the node's + * version differs from the gateway's — a fresh node's null, a node that + * missed an edit while its gateway was away, an operator's change a + * second ago — and the node keeps it: a node whose gateway is down still + * knows its settings and its templates, and serves. + */ +export const nodeConfigBundleSchema = z.object({ + version: z.number().int().positive(), + settings: z.object({ + sandboxDefaults: sandboxResourceDefaultsSchema, + defaultPolicy: lifecyclePolicySchema, + /** Write shape, keys included — see above. Null = archiving off. */ + s3: s3ArchiveSettingsSchema.nullable(), + sandboxDomain: z.string().regex(bareHostnameRegex).nullable(), + sandboxDomainAliases: z.array(z.string().regex(bareHostnameRegex)), + pidsLimit: z.number().int().min(PIDS_LIMIT_MIN), + }), + node: z.object({ + /** This node's managed-swap target, GiB (updateNodeSettings). */ + swapGb: z.number().int().nonnegative(), + }), + templates: z.array(templateSchema), +}); + +export type NodeConfigBundle = z.infer; + +export const checkInResponseSchema = z.object({ + /** The gateway's current configuration version — what the node reports back at its next check-in. */ + configVersion: z.number().int().positive(), + /** Present exactly when the node's reported version differs from the gateway's: the whole bundle to apply. */ + config: nodeConfigBundleSchema.optional(), +}); export type CheckInResponse = z.infer; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5e3734aa..f60346e5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -61,6 +61,9 @@ importers: specifier: ^14.0.0 version: 14.0.3 devDependencies: + '@dormice/gateway': + specifier: workspace:* + version: link:../gateway '@dormice/server': specifier: workspace:* version: link:../server @@ -235,6 +238,9 @@ importers: specifier: ^8.7.0 version: 8.7.0 devDependencies: + '@dormice/gateway': + specifier: workspace:* + version: link:../gateway '@dormice/server': specifier: workspace:* version: link:../server @@ -250,15 +256,9 @@ importers: '@dormice/shared': specifier: workspace:* version: link:../shared - '@fastify/cookie': - specifier: ^11.0.2 - version: 11.0.2 '@fastify/multipart': specifier: ^10.0.0 version: 10.0.0 - '@fastify/static': - specifier: ^9.1.3 - version: 9.1.3 better-sqlite3: specifier: ^12.11.1 version: 12.11.1 From d830de21efe8c00c9258c58cac7c1f5ecd516206 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 17:23:48 +0800 Subject: [PATCH 35/89] The console's dev proxy and settings hints follow the configuration to the gateway MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vite proxies every API path to the gateway on 3677 — the fleet's one door since the move — and the settings page's read-only env table describes the gateway's knobs (its port, database and the three placement gates, plus the fleet seeds) instead of the daemon's; the notes name gateway.env and dormice-gateway, and say a write reaches the nodes at their next check-in. Ten locales, keys aligned. --- packages/console/messages/de/settings.json | 19 ++-- packages/console/messages/en/settings.json | 19 ++-- packages/console/messages/es/settings.json | 19 ++-- packages/console/messages/fr/settings.json | 19 ++-- packages/console/messages/ja/settings.json | 19 ++-- packages/console/messages/ko/settings.json | 19 ++-- packages/console/messages/pt-BR/settings.json | 19 ++-- packages/console/messages/ru/settings.json | 19 ++-- packages/console/messages/zh-CN/settings.json | 19 ++-- packages/console/messages/zh-TW/settings.json | 19 ++-- .../features/settings/pages/SettingsPage.tsx | 32 ++++--- packages/console/vite.config.ts | 88 ++++++++++--------- 12 files changed, 133 insertions(+), 177 deletions(-) diff --git a/packages/console/messages/de/settings.json b/packages/console/messages/de/settings.json index f72c808a..afade226 100644 --- a/packages/console/messages/de/settings.json +++ b/packages/console/messages/de/settings.json @@ -1,22 +1,17 @@ { "$schema": "https://inlang.com/schema/inlang-message-format", - "settings_hint_port": "Daemon-Port; bindet nur an 127.0.0.1 — die Außenseite ist Sache des Reverse-Proxys", - "settings_hint_db_path": "SQLite-Ledger; im docker-Modus ist ein absoluter Pfad Pflicht", - "settings_hint_node_id": "Name dieser Maschine im Ledger (für künftiges Sharding reserviert)", + "settings_hint_gateway_port": "Gateway-Port; bindet nur an 127.0.0.1 — die Außenseite ist Sache des Reverse-Proxys", + "settings_hint_gateway_db_path": "Die eigene SQLite-Datenbank des Gateways: Knoten, Flotteneinstellungen, Vorlagen, API-Schlüssel, Konsolenkonto; muss ein absoluter Pfad sein", "settings_hint_api_token": "Der einzige API-Berechtigungsnachweis; die Konsolenanmeldung tauscht ihn gegen ein httpOnly-Cookie, die Seite kann ihn nie lesen", - "settings_hint_executor": "Executor: docker sind echte Sandboxes, fake ist ein In-Memory-Stub (für Entwicklung/Tests)", - "settings_hint_base_image": "Standard-Image für Sandboxes; im docker-Modus Pflicht", - "settings_hint_data_dir": "Zuhause der Sandbox-Datenträger (disks/*.img); auch temporäre Archivdateien liegen hier", - "settings_hint_scan_interval": "Leerlauf-Scan-Intervall: Jede Runde kühlt Sandboxes an der Schwelle eine Stufe ab", - "settings_hint_metrics_sample_interval": "Messintervall: die Auflösung der Verlaufskurven; Übersichtstrends und Messwerthistorie werden in diesem Takt gespeichert", - "settings_hint_metrics_retention": "Aufbewahrungsdauer der Messwerte je Sandbox; die Flotten-Zustandszählungen werden unabhängig davon immer 30 Tage behalten", + "settings_hint_node_cpu_limit": "Platzierungsschranke: Ein Knoten, dessen letzter Check-in eine Gesamt-CPU über diesem Prozentsatz meldete, nimmt keine neuen Sandboxes an", + "settings_hint_node_active_limit": "Platzierungsschranke: Ein Knoten mit so vielen aktiven Sandboxes (Platzierungen seit dem Check-in eingerechnet) nimmt keine weiteren an; eingefrorene zählen nicht", + "settings_hint_node_min_disk": "Platzierungsschranke: Ein Knoten mit weniger als so vielen GiB frei auf der Datenplatte nimmt keine neuen Sandboxes an — eine volle Datenplatte hält alle Sandboxes des Knotens auf einmal an", "settings_hint_sandbox_disk": "Erststart-Saatwert für die Standard-Datenträgerquote — der wirksame Wert steht oben bei den Betriebsreglern", "settings_hint_sandbox_cpus": "Erststart-Saatwert für die Standard-CPU-Quote — der wirksame Wert steht oben bei den Betriebsreglern", "settings_hint_sandbox_memory": "Erststart-Saatwert für das Standard-Speicherlimit — der wirksame Wert steht oben bei den Betriebsreglern", "settings_hint_sandbox_pids_limit": "Erststart-Seed für das pids-Limit — der gültige Wert steht in der Regler-Karte oben", - "settings_hint_reclaim_timeout": "Maximale Wartezeit beim Einfrieren, um Speicher in den Swap zu drücken", "settings_hint_sandbox_domain": "Erststart-Saatwert für die kanonische Sandbox-Domain — die wirksame Konfiguration (inklusive Aliasse) steht auf der Domains-Seite in der Sandbox-Domain-Karte", - "settings_hint_ingress_file": "Vom Daemon verwaltete Caddy-Konfigurationsdatei; nötig, um auf der Domains-Seite Domains zu binden", + "settings_hint_ingress_file": "Vom Gateway verwaltete Caddy-Konfigurationsdatei; nötig, um auf der Domains-Seite Domains zu binden", "settings_hint_ingress_reload_cmd": "Reload-Befehl nach Änderungen an der Proxy-Konfiguration; Standard ist caddy reload auf der verwalteten Datei selbst", "settings_hint_s3_endpoint": "Erststart-Saatwert für den Archiv-Endpoint — die wirksame Konfiguration steht oben in der Archivspeicher-Karte", "settings_hint_s3_bucket": "Erststart-Saatwert für den Archiv-Bucket — die wirksame Konfiguration steht oben in der Archivspeicher-Karte", @@ -44,7 +39,7 @@ "settings_source_default": "Standardwert", "settings_version_title": "Version", "settings_knobs_title": "Betriebsregler", - "settings_knobs_desc": "Wohnen im Ledger; Änderungen wirken sofort, ohne Neustart — die gleichnamigen Umgebungsvariablen unten sind nur Saatwerte für den ersten Start.", + "settings_knobs_desc": "Liegen in der Einstellungstabelle des Gateways; Änderungen wirken sofort und erreichen jeden Knoten mit seinem nächsten Check-in, ohne Neustart — die gleichnamigen Umgebungsvariablen unten sind nur Saatwerte für den ersten Start des Gateways.", "settings_knobs_last_modified": " Zuletzt geändert: {time}.", "settings_knobs_never_modified": " Nie geändert; noch die Saatwerte.", "settings_row_defaults": "Standardquoten neuer Sandboxes", diff --git a/packages/console/messages/en/settings.json b/packages/console/messages/en/settings.json index 1105ca49..b6e3b95f 100644 --- a/packages/console/messages/en/settings.json +++ b/packages/console/messages/en/settings.json @@ -1,22 +1,17 @@ { "$schema": "https://inlang.com/schema/inlang-message-format", - "settings_hint_port": "Daemon port; binds to 127.0.0.1 only — external exposure is the reverse proxy's job", - "settings_hint_db_path": "SQLite ledger; docker mode requires an absolute path", - "settings_hint_node_id": "This machine's name in the ledger (reserved for future sharding)", + "settings_hint_gateway_port": "Gateway port; binds to 127.0.0.1 only — external exposure is the reverse proxy's job", + "settings_hint_gateway_db_path": "The gateway's own SQLite database: nodes, fleet settings, templates, API keys, the console account; must be an absolute path", "settings_hint_api_token": "The only API credential; console login exchanges it for an httpOnly cookie, so the page can never read it", - "settings_hint_executor": "Executor: docker is real sandboxes, fake is an in-memory stub (for dev/testing)", - "settings_hint_base_image": "Default sandbox image; required in docker mode", - "settings_hint_data_dir": "Home of sandbox disk images (disks/*.img); archive temp files live here too", - "settings_hint_scan_interval": "Idle scan interval: each round cools sandboxes that hit a threshold by one step", - "settings_hint_metrics_sample_interval": "Metrics sampling interval: the resolution of history curves; overview trends and metric history are stored at this cadence", - "settings_hint_metrics_retention": "Retention for per-sandbox metric samples; fleet state counts are always kept 30 days regardless", + "settings_hint_node_cpu_limit": "Placement gate: a node whose last check-in reported whole-machine CPU above this percentage takes no new sandboxes", + "settings_hint_node_active_limit": "Placement gate: a node with this many active sandboxes (placements since its check-in included) takes no more; frozen ones are not counted", + "settings_hint_node_min_disk": "Placement gate: a node whose data disk has less than this many GiB free takes no new sandboxes — a full data disk stops every sandbox on the node at once", "settings_hint_sandbox_disk": "First-boot seed for the default disk quota — the effective value is in the knobs above", "settings_hint_sandbox_cpus": "First-boot seed for the default CPU quota — the effective value is in the knobs above", "settings_hint_sandbox_memory": "First-boot seed for the default memory limit — the effective value is in the knobs above", "settings_hint_sandbox_pids_limit": "First-boot seed for the pids cap — the value in force lives in the knobs card above", - "settings_hint_reclaim_timeout": "Maximum wait for squeezing memory into swap when freezing", "settings_hint_sandbox_domain": "First-boot seed for the canonical sandbox domain — the configuration in force (aliases included) lives on the Domains page", - "settings_hint_ingress_file": "Caddy config file managed by the daemon; required for binding domains on the Domains page", + "settings_hint_ingress_file": "Caddy config file managed by the gateway; required for binding domains on the Domains page", "settings_hint_ingress_reload_cmd": "Reload command after proxy config changes; defaults to caddy reload on the managed file itself", "settings_hint_s3_endpoint": "First-boot seed for the archive endpoint — the store in force lives in the archive card above", "settings_hint_s3_bucket": "First-boot seed for the archive bucket — the store in force lives in the archive card above", @@ -44,7 +39,7 @@ "settings_source_default": "Default", "settings_version_title": "Version", "settings_knobs_title": "Operational knobs", - "settings_knobs_desc": "Stored in the ledger; changes take effect immediately with no restart — the same-named env vars below are only first-boot seed values.", + "settings_knobs_desc": "Stored in the gateway's settings table; changes take effect immediately and reach every node at its next check-in, no restart — the same-named env vars below are only the gateway's first-boot seed values.", "settings_knobs_last_modified": " Last modified {time}.", "settings_knobs_never_modified": " Never changed; still the seed values.", "settings_row_defaults": "New sandbox default quotas", diff --git a/packages/console/messages/es/settings.json b/packages/console/messages/es/settings.json index f4c982c1..44ad5c49 100644 --- a/packages/console/messages/es/settings.json +++ b/packages/console/messages/es/settings.json @@ -1,22 +1,17 @@ { "$schema": "https://inlang.com/schema/inlang-message-format", - "settings_hint_port": "Puerto del daemon; se enlaza solo a 127.0.0.1 — exponerlo hacia fuera es tarea del proxy inverso", - "settings_hint_db_path": "Libro de registro SQLite; en modo docker se exige una ruta absoluta", - "settings_hint_node_id": "Nombre de esta máquina en el libro de registro (reservado para el particionado futuro)", + "settings_hint_gateway_port": "Puerto del gateway; se enlaza solo a 127.0.0.1 — exponerlo hacia fuera es tarea del proxy inverso", + "settings_hint_gateway_db_path": "La base de datos SQLite propia del gateway: nodos, ajustes de la flota, plantillas, claves API, cuenta de la consola; debe ser una ruta absoluta", "settings_hint_api_token": "La única credencial de API; al iniciar sesión la consola la cambia por una cookie httpOnly, así que la página nunca puede leerla", - "settings_hint_executor": "Ejecutor: docker son sandboxes reales, fake es un sustituto en memoria (para desarrollo y pruebas)", - "settings_hint_base_image": "Imagen predeterminada de los sandboxes; obligatoria en modo docker", - "settings_hint_data_dir": "Hogar de las imágenes de disco de los sandboxes (disks/*.img); los archivos temporales de archivado también viven aquí", - "settings_hint_scan_interval": "Periodo del escaneo de inactividad: cada ronda enfría un paso los sandboxes que alcanzan un umbral", - "settings_hint_metrics_sample_interval": "Periodo de muestreo de métricas: la resolución de las curvas históricas; las tendencias del panel y el historial de métricas se guardan con esta cadencia", - "settings_hint_metrics_retention": "Retención de las muestras de métricas por sandbox; los recuentos de estado de la flota se guardan siempre 30 días al margen de esto", + "settings_hint_node_cpu_limit": "Puerta de colocación: un nodo cuyo último registro informó una CPU total por encima de este porcentaje no acepta sandboxes nuevos", + "settings_hint_node_active_limit": "Puerta de colocación: un nodo con este número de sandboxes activos (incluidas las colocaciones desde su registro) no acepta más; los congelados no cuentan", + "settings_hint_node_min_disk": "Puerta de colocación: un nodo con menos de estos GiB libres en el disco de datos no acepta sandboxes nuevos — un disco de datos lleno detiene de golpe todos los sandboxes del nodo", "settings_hint_sandbox_disk": "Valor semilla del primer arranque para la cuota de disco predeterminada — el valor efectivo está en los parámetros de arriba", "settings_hint_sandbox_cpus": "Valor semilla del primer arranque para la cuota de CPU predeterminada — el valor efectivo está en los parámetros de arriba", "settings_hint_sandbox_memory": "Valor semilla del primer arranque para el límite de memoria predeterminado — el valor efectivo está en los parámetros de arriba", "settings_hint_sandbox_pids_limit": "Semilla de primer arranque del límite pids; el valor en vigor está en la tarjeta de ajustes de arriba", - "settings_hint_reclaim_timeout": "Espera máxima al comprimir la memoria hacia swap durante la congelación", "settings_hint_sandbox_domain": "Valor semilla del primer arranque para el dominio canónico de sandbox — la configuración en vigor (alias incluidos) está en la tarjeta de dominio de sandbox de la página Dominios", - "settings_hint_ingress_file": "Archivo de configuración de Caddy gestionado por el daemon; hace falta para vincular dominios en la página Dominios", + "settings_hint_ingress_file": "Archivo de configuración de Caddy gestionado por el gateway; hace falta para vincular dominios en la página Dominios", "settings_hint_ingress_reload_cmd": "Comando de recarga tras cambiar la configuración del proxy; por defecto es caddy reload sobre el propio archivo gestionado", "settings_hint_s3_endpoint": "Valor semilla del primer arranque para el Endpoint de archivado — el almacenamiento en vigor está en la tarjeta de archivado de arriba", "settings_hint_s3_bucket": "Valor semilla del primer arranque para el Bucket de archivado — el almacenamiento en vigor está en la tarjeta de archivado de arriba", @@ -44,7 +39,7 @@ "settings_source_default": "Predeterminado", "settings_version_title": "Versión", "settings_knobs_title": "Parámetros operativos", - "settings_knobs_desc": "Viven en el libro de registro; los cambios se aplican de inmediato sin reiniciar — las variables de entorno del mismo nombre de abajo son solo valores semilla del primer arranque.", + "settings_knobs_desc": "Viven en la tabla de ajustes del gateway; los cambios se aplican de inmediato y llegan a cada nodo en su siguiente registro, sin reiniciar — las variables de entorno del mismo nombre de abajo son solo valores semilla del primer arranque del gateway.", "settings_knobs_last_modified": " Última modificación: {time}.", "settings_knobs_never_modified": " Nunca se cambiaron; siguen los valores semilla.", "settings_row_defaults": "Cuotas predeterminadas de los sandboxes nuevos", diff --git a/packages/console/messages/fr/settings.json b/packages/console/messages/fr/settings.json index 97ae53fe..cbb68e4e 100644 --- a/packages/console/messages/fr/settings.json +++ b/packages/console/messages/fr/settings.json @@ -1,22 +1,17 @@ { "$schema": "https://inlang.com/schema/inlang-message-format", - "settings_hint_port": "Port du daemon ; n'écoute que sur 127.0.0.1 — l'exposition externe est le travail du proxy inverse", - "settings_hint_db_path": "Registre SQLite ; le mode docker exige un chemin absolu", - "settings_hint_node_id": "Le nom de cette machine dans le registre (réservé pour un futur partitionnement)", + "settings_hint_gateway_port": "Port de la passerelle ; n'écoute que sur 127.0.0.1 — l'exposition externe est le travail du proxy inverse", + "settings_hint_gateway_db_path": "La base SQLite propre à la passerelle : nœuds, réglages de la flotte, modèles, clés API, compte de la console ; doit être un chemin absolu", "settings_hint_api_token": "L'unique identifiant d'API ; la connexion à la console l'échange contre un cookie httpOnly, la page ne peut donc jamais le lire", - "settings_hint_executor": "Exécuteur : docker = vraies sandbox, fake = simulacre en mémoire (dev/tests)", - "settings_hint_base_image": "Image par défaut des sandbox ; obligatoire en mode docker", - "settings_hint_data_dir": "Foyer des images disque des sandbox (disks/*.img) ; les fichiers temporaires d'archivage vivent ici aussi", - "settings_hint_scan_interval": "Période du balayage d'inactivité : à chaque tour, les sandbox ayant atteint un seuil descendent d'un cran", - "settings_hint_metrics_sample_interval": "Période d'échantillonnage des métriques : la résolution des courbes d'historique ; tendances du tableau de bord et historiques de métriques sont stockés à cette cadence", - "settings_hint_metrics_retention": "Durée de rétention des échantillons de métriques par sandbox ; les comptages d'états de la flotte sont toujours conservés 30 jours, indépendamment", + "settings_hint_node_cpu_limit": "Porte de placement : un nœud dont le dernier pointage indiquait un CPU machine au-dessus de ce pourcentage ne prend plus de nouvelles sandboxes", + "settings_hint_node_active_limit": "Porte de placement : un nœud avec autant de sandboxes actives (placements depuis son pointage inclus) n'en prend plus ; les gelées ne comptent pas", + "settings_hint_node_min_disk": "Porte de placement : un nœud dont le disque de données a moins de ce nombre de Gio libres ne prend plus de nouvelles sandboxes — un disque de données plein arrête d'un coup toutes les sandboxes du nœud", "settings_hint_sandbox_disk": "Valeur d'amorçage du quota disque par défaut — la valeur effective est dans les réglages ci-dessus", "settings_hint_sandbox_cpus": "Valeur d'amorçage du quota CPU par défaut — la valeur effective est dans les réglages ci-dessus", "settings_hint_sandbox_memory": "Valeur d'amorçage de la limite mémoire par défaut — la valeur effective est dans les réglages ci-dessus", "settings_hint_sandbox_pids_limit": "Graine de premier démarrage de la limite pids ; la valeur en vigueur est dans la carte des réglages ci-dessus", - "settings_hint_reclaim_timeout": "Attente maximale pour pousser la mémoire dans le swap lors du gel", "settings_hint_sandbox_domain": "Valeur d'amorçage du domaine canonique des sandbox — la configuration effective (alias compris) est dans la carte du domaine des sandbox, page Domaines", - "settings_hint_ingress_file": "Fichier de configuration Caddy géré par le daemon ; requis pour lier des domaines depuis la page Domaines", + "settings_hint_ingress_file": "Fichier de configuration Caddy géré par la passerelle ; requis pour lier des domaines depuis la page Domaines", "settings_hint_ingress_reload_cmd": "Commande de rechargement après modification de la configuration du proxy ; par défaut, caddy reload sur le fichier géré lui-même", "settings_hint_s3_endpoint": "Valeur d'amorçage de l'endpoint d'archivage — le stockage effectif est dans la carte d'archivage ci-dessus", "settings_hint_s3_bucket": "Valeur d'amorçage du bucket d'archivage — le stockage effectif est dans la carte d'archivage ci-dessus", @@ -44,7 +39,7 @@ "settings_source_default": "Défaut", "settings_version_title": "Version", "settings_knobs_title": "Réglages d'exploitation", - "settings_knobs_desc": "Stockés dans le registre ; les changements prennent effet immédiatement, sans redémarrage — les variables d'environnement homonymes ci-dessous ne sont que des valeurs d'amorçage du premier démarrage.", + "settings_knobs_desc": "Stockés dans la table de réglages de la passerelle ; les changements prennent effet immédiatement et atteignent chaque nœud à son prochain pointage, sans redémarrage — les variables d'environnement homonymes ci-dessous ne sont que des valeurs d'amorçage du premier démarrage de la passerelle.", "settings_knobs_last_modified": " Dernière modification le {time}.", "settings_knobs_never_modified": " Jamais modifiés ; toujours les valeurs d'amorçage.", "settings_row_defaults": "Quotas par défaut des nouvelles sandbox", diff --git a/packages/console/messages/ja/settings.json b/packages/console/messages/ja/settings.json index 5040d668..5aa6928d 100644 --- a/packages/console/messages/ja/settings.json +++ b/packages/console/messages/ja/settings.json @@ -1,22 +1,17 @@ { "$schema": "https://inlang.com/schema/inlang-message-format", - "settings_hint_port": "daemon のポート。127.0.0.1 のみにバインドし、外部公開はリバースプロキシの役目です", - "settings_hint_db_path": "SQLite の台帳。docker モードでは絶対パスが必須です", - "settings_hint_node_id": "台帳上でのこのマシンの名前(将来のシャーディング用の予約)", + "settings_hint_gateway_port": "ゲートウェイのポート。127.0.0.1 のみにバインドし、外部公開はリバースプロキシの役目です", + "settings_hint_gateway_db_path": "ゲートウェイ自身の SQLite データベース:ノード、フリート設定、テンプレート、API キー、コンソールアカウント。絶対パスが必要です", "settings_hint_api_token": "唯一の API 認証情報。コンソールのログイン時に httpOnly cookie に交換され、ページからは決して読めません", - "settings_hint_executor": "エグゼキューター:docker は本物のサンドボックス、fake はメモリ内のスタブです(開発・テスト用)", - "settings_hint_base_image": "サンドボックスの既定イメージ。docker モードでは必須です", - "settings_hint_data_dir": "サンドボックスのディスクイメージの置き場所(disks/*.img)。アーカイブの一時ファイルもここに置かれます", - "settings_hint_scan_interval": "アイドルスキャンの周期:1 回のスキャンごとに、しきい値に達したサンドボックスを 1 段階冷却します", - "settings_hint_metrics_sample_interval": "メトリクスのサンプリング周期:履歴曲線の解像度で、概要の推移もメトリクス履歴もこの間隔で保存されます", - "settings_hint_metrics_retention": "サンドボックス別メトリクスサンプルの保持期間。フリートの状態カウントは常に 30 日保持され、この値には従いません", + "settings_hint_node_cpu_limit": "配置ゲート:直近のチェックインでマシン全体の CPU がこの割合を超えたノードは新しいサンドボックスを受け付けません", + "settings_hint_node_active_limit": "配置ゲート:アクティブなサンドボックス数(チェックイン後の配置分を含む)がこの数に達したノードは新規を受け付けません。凍結中は数えません", + "settings_hint_node_min_disk": "配置ゲート:データディスクの空きがこの GiB を下回るノードは新規を受け付けません — データディスクが満杯になるとノード上の全サンドボックスが一斉に止まります", "settings_hint_sandbox_disk": "既定ディスククォータの初回起動シード値 — 有効値は上の運用設定にあります", "settings_hint_sandbox_cpus": "既定 CPU クォータの初回起動シード値 — 有効値は上の運用設定にあります", "settings_hint_sandbox_memory": "既定メモリ上限の初回起動シード値 — 有効値は上の運用設定にあります", "settings_hint_sandbox_pids_limit": "pids 上限の初回起動シード値 — 有効な値は上のノブカードにあります", - "settings_hint_reclaim_timeout": "凍結時にメモリを swap へ押し出す際の最大待ち時間", "settings_hint_sandbox_domain": "サンドボックスの正規ドメインの初回起動シード値 — 有効な設定(エイリアス含む)は「ドメイン」ページのサンドボックスドメインカードにあります", - "settings_hint_ingress_file": "daemon が管理する Caddy 設定ファイル。設定すると「ドメイン」ページからウェブでドメインをバインドできます", + "settings_hint_ingress_file": "ゲートウェイが管理する Caddy 設定ファイル。設定すると「ドメイン」ページからウェブでドメインをバインドできます", "settings_hint_ingress_reload_cmd": "プロキシ設定変更後のリロードコマンド。未設定なら既定で管理ファイル自体を caddy reload します", "settings_hint_s3_endpoint": "アーカイブ Endpoint の初回起動シード値 — 有効な設定は上のアーカイブストレージカードにあります", "settings_hint_s3_bucket": "アーカイブ Bucket 名の初回起動シード値 — 有効な設定は上のアーカイブストレージカードにあります", @@ -44,7 +39,7 @@ "settings_source_default": "既定値", "settings_version_title": "バージョン", "settings_knobs_title": "運用設定", - "settings_knobs_desc": "台帳に保存され、変更は再起動なしで即時に反映されます — 下表の同名環境変数は初回起動時のシード値にすぎません。", + "settings_knobs_desc": "ゲートウェイの設定テーブルに保存され、変更は即時に反映され、各ノードには次のチェックインで届きます。再起動は不要です — 下表の同名環境変数はゲートウェイの初回起動時のシード値にすぎません。", "settings_knobs_last_modified": "最終変更:{time}。", "settings_knobs_never_modified": "変更されたことはなく、シード値のままです。", "settings_row_defaults": "新規サンドボックスの既定クォータ", diff --git a/packages/console/messages/ko/settings.json b/packages/console/messages/ko/settings.json index 56af2b6a..3d7b3f0b 100644 --- a/packages/console/messages/ko/settings.json +++ b/packages/console/messages/ko/settings.json @@ -1,22 +1,17 @@ { "$schema": "https://inlang.com/schema/inlang-message-format", - "settings_hint_port": "daemon 포트. 127.0.0.1에만 바인딩하며, 외부 노출은 리버스 프록시의 몫입니다", - "settings_hint_db_path": "SQLite 장부. docker 모드에서는 절대 경로 필수", - "settings_hint_node_id": "장부에 기록되는 이 머신의 이름(향후 샤딩용으로 예약)", + "settings_hint_gateway_port": "게이트웨이 포트. 127.0.0.1에만 바인딩하며, 외부 노출은 리버스 프록시의 몫입니다", + "settings_hint_gateway_db_path": "게이트웨이 자체 SQLite 데이터베이스: 노드, 플릿 설정, 템플릿, API 키, 콘솔 계정. 절대 경로여야 합니다", "settings_hint_api_token": "유일한 API 자격 증명. 콘솔 로그인 시 httpOnly 쿠키로 교환되어 페이지는 절대 읽을 수 없습니다", - "settings_hint_executor": "실행기: docker는 실제 샌드박스, fake는 메모리 내 가짜 실행기(개발/테스트용)", - "settings_hint_base_image": "샌드박스의 기본 이미지. docker 모드에서는 필수", - "settings_hint_data_dir": "샌드박스 디스크 이미지의 집(disks/*.img). 아카이브 임시 파일도 여기에 있습니다", - "settings_hint_scan_interval": "유휴 스캔 주기: 매 라운드마다 임계값에 도달한 샌드박스의 온도를 한 단계 낮춥니다", - "settings_hint_metrics_sample_interval": "메트릭 샘플링 주기: 히스토리 곡선의 해상도이며, 개요 추이와 메트릭 히스토리가 이 주기로 저장됩니다", - "settings_hint_metrics_retention": "샌드박스별 메트릭 샘플의 보관 기간. 함대 상태 카운트는 이와 무관하게 항상 30일 보관", + "settings_hint_node_cpu_limit": "배치 게이트: 마지막 체크인에서 머신 전체 CPU가 이 비율을 넘은 노드는 새 샌드박스를 받지 않습니다", + "settings_hint_node_active_limit": "배치 게이트: 활성 샌드박스 수(체크인 이후 배치된 것 포함)가 이 값에 도달한 노드는 더 받지 않습니다. 동결된 것은 세지 않습니다", + "settings_hint_node_min_disk": "배치 게이트: 데이터 디스크 여유 공간이 이 GiB 미만인 노드는 새 샌드박스를 받지 않습니다 — 데이터 디스크가 가득 차면 노드의 모든 샌드박스가 한꺼번에 멈춥니다", "settings_hint_sandbox_disk": "기본 디스크 할당량의 첫 부팅 시드 값 — 유효 값은 위의 운영 노브에 있습니다", "settings_hint_sandbox_cpus": "기본 CPU 할당량의 첫 부팅 시드 값 — 유효 값은 위의 운영 노브에 있습니다", "settings_hint_sandbox_memory": "기본 메모리 상한의 첫 부팅 시드 값 — 유효 값은 위의 운영 노브에 있습니다", "settings_hint_sandbox_pids_limit": "pids 상한의 최초 부팅 시드 값 — 유효 값은 위 노브 카드에 있습니다", - "settings_hint_reclaim_timeout": "동결 시 메모리를 swap으로 밀어 넣는 최대 대기 시간", "settings_hint_sandbox_domain": "샌드박스 정식 도메인의 첫 부팅 시드 값 — 유효 설정(별칭 포함)은 '도메인' 페이지의 샌드박스 도메인 카드에 있습니다", - "settings_hint_ingress_file": "daemon이 관리하는 Caddy 설정 파일. 설정해야 '도메인' 페이지에서 웹으로 도메인을 바인딩할 수 있습니다", + "settings_hint_ingress_file": "게이트웨이가 관리하는 Caddy 설정 파일. 설정해야 '도메인' 페이지에서 웹으로 도메인을 바인딩할 수 있습니다", "settings_hint_ingress_reload_cmd": "프록시 설정 변경 후 실행할 리로드 명령. 미설정 시 기본값은 관리 파일 자체에 대한 caddy reload", "settings_hint_s3_endpoint": "아카이브 Endpoint의 첫 부팅 시드 값 — 유효 설정은 위의 아카이브 스토리지 카드에 있습니다", "settings_hint_s3_bucket": "아카이브 Bucket 이름의 첫 부팅 시드 값 — 유효 설정은 위의 아카이브 스토리지 카드에 있습니다", @@ -44,7 +39,7 @@ "settings_source_default": "기본값", "settings_version_title": "버전", "settings_knobs_title": "운영 노브", - "settings_knobs_desc": "장부에 저장되어 변경 즉시 적용되며 재시작이 필요 없습니다 — 아래 표의 같은 이름 환경 변수는 첫 부팅의 시드 값일 뿐입니다.", + "settings_knobs_desc": "게이트웨이의 설정 테이블에 저장되어 변경 즉시 적용되고 각 노드에는 다음 체크인 때 전달되며 재시작이 필요 없습니다 — 아래 표의 같은 이름 환경 변수는 게이트웨이 첫 부팅의 시드 값일 뿐입니다.", "settings_knobs_last_modified": " 마지막 수정: {time}.", "settings_knobs_never_modified": " 변경 이력이 없어 아직 시드 값 그대로입니다.", "settings_row_defaults": "새 샌드박스 기본 할당량", diff --git a/packages/console/messages/pt-BR/settings.json b/packages/console/messages/pt-BR/settings.json index f377bf6a..c77d9d6c 100644 --- a/packages/console/messages/pt-BR/settings.json +++ b/packages/console/messages/pt-BR/settings.json @@ -1,22 +1,17 @@ { "$schema": "https://inlang.com/schema/inlang-message-format", - "settings_hint_port": "Porta do daemon; escuta apenas em 127.0.0.1 — expor externamente é trabalho do proxy reverso", - "settings_hint_db_path": "Ledger SQLite; o modo docker exige caminho absoluto", - "settings_hint_node_id": "O nome desta máquina no ledger (reservado para sharding futuro)", + "settings_hint_gateway_port": "Porta do gateway; escuta apenas em 127.0.0.1 — expor externamente é trabalho do proxy reverso", + "settings_hint_gateway_db_path": "O banco SQLite do próprio gateway: nós, configurações da frota, templates, chaves de API, conta do console; deve ser um caminho absoluto", "settings_hint_api_token": "A única credencial de API; o login do console a troca por um cookie httpOnly, então a página nunca consegue lê-la", - "settings_hint_executor": "Executor: docker é sandbox de verdade, fake é um stub em memória (para dev/teste)", - "settings_hint_base_image": "Imagem padrão dos sandboxes; obrigatória no modo docker", - "settings_hint_data_dir": "Casa das imagens de disco dos sandboxes (disks/*.img); os arquivos temporários de arquivamento também moram aqui", - "settings_hint_scan_interval": "Intervalo da varredura de ociosidade: cada rodada esfria em um degrau os sandboxes que atingiram um limite", - "settings_hint_metrics_sample_interval": "Intervalo de amostragem de métricas: a resolução das curvas de histórico; as tendências do painel e o histórico de métricas são gravados nessa cadência", - "settings_hint_metrics_retention": "Retenção das amostras de métricas por sandbox; as contagens de estado da frota são sempre mantidas por 30 dias, independentemente", + "settings_hint_node_cpu_limit": "Portão de alocação: um nó cujo último check-in informou CPU total acima desta porcentagem não recebe novos sandboxes", + "settings_hint_node_active_limit": "Portão de alocação: um nó com esta quantidade de sandboxes ativos (incluindo alocações desde o check-in) não recebe mais; congelados não contam", + "settings_hint_node_min_disk": "Portão de alocação: um nó com menos que estes GiB livres no disco de dados não recebe novos sandboxes — um disco de dados cheio para todos os sandboxes do nó de uma vez", "settings_hint_sandbox_disk": "Semente de primeira inicialização para a cota padrão de disco — o valor efetivo está nos controles acima", "settings_hint_sandbox_cpus": "Semente de primeira inicialização para a cota padrão de CPU — o valor efetivo está nos controles acima", "settings_hint_sandbox_memory": "Semente de primeira inicialização para o limite padrão de memória — o valor efetivo está nos controles acima", "settings_hint_sandbox_pids_limit": "Semente de primeira inicialização do limite pids; o valor em vigor está no cartão de ajustes acima", - "settings_hint_reclaim_timeout": "Espera máxima para empurrar a memória para o swap ao congelar", "settings_hint_sandbox_domain": "Semente de primeira inicialização para o domínio canônico de sandbox — a configuração em vigor (incluindo aliases) está no cartão de domínio de sandbox da página Domínios", - "settings_hint_ingress_file": "Arquivo de configuração do Caddy gerenciado pelo daemon; necessário para vincular domínios na página Domínios", + "settings_hint_ingress_file": "Arquivo de configuração do Caddy gerenciado pelo gateway; necessário para vincular domínios na página Domínios", "settings_hint_ingress_reload_cmd": "Comando de recarga após mudanças na configuração do proxy; o padrão é caddy reload no próprio arquivo gerenciado", "settings_hint_s3_endpoint": "Semente de primeira inicialização para o Endpoint de arquivamento — o armazenamento em vigor está no cartão de arquivamento acima", "settings_hint_s3_bucket": "Semente de primeira inicialização para o Bucket de arquivamento — o armazenamento em vigor está no cartão de arquivamento acima", @@ -44,7 +39,7 @@ "settings_source_default": "Padrão", "settings_version_title": "Versão", "settings_knobs_title": "Controles operacionais", - "settings_knobs_desc": "Moram no ledger; mudanças entram em vigor imediatamente, sem reiniciar — as variáveis de ambiente homônimas abaixo são apenas sementes da primeira inicialização.", + "settings_knobs_desc": "Moram na tabela de configurações do gateway; mudanças entram em vigor imediatamente e chegam a cada nó no próximo check-in, sem reiniciar — as variáveis de ambiente homônimas abaixo são apenas sementes da primeira inicialização do gateway.", "settings_knobs_last_modified": " Última modificação em {time}.", "settings_knobs_never_modified": " Nunca alterados; ainda são os valores-semente.", "settings_row_defaults": "Cotas padrão de novos sandboxes", diff --git a/packages/console/messages/ru/settings.json b/packages/console/messages/ru/settings.json index 1d4c79c3..c46318a4 100644 --- a/packages/console/messages/ru/settings.json +++ b/packages/console/messages/ru/settings.json @@ -1,22 +1,17 @@ { "$schema": "https://inlang.com/schema/inlang-message-format", - "settings_hint_port": "Порт daemon; слушает только 127.0.0.1 — наружу его выставляет обратный прокси", - "settings_hint_db_path": "Реестр SQLite; в режиме docker путь обязан быть абсолютным", - "settings_hint_node_id": "Имя этой машины в реестре (задел под будущее шардирование)", + "settings_hint_gateway_port": "Порт шлюза; слушает только 127.0.0.1 — наружу его выставляет обратный прокси", + "settings_hint_gateway_db_path": "Собственная база SQLite шлюза: узлы, настройки парка, шаблоны, API-ключи, аккаунт консоли; путь должен быть абсолютным", "settings_hint_api_token": "Единственные учётные данные API; при входе в консоль обменивается на httpOnly-cookie, страница его никогда не видит", - "settings_hint_executor": "Исполнитель: docker — настоящие песочницы, fake — заглушка в памяти (для разработки и тестов)", - "settings_hint_base_image": "Образ песочниц по умолчанию; в режиме docker обязателен", - "settings_hint_data_dir": "Дом дисковых образов песочниц (disks/*.img); здесь же временные файлы архивации", - "settings_hint_scan_interval": "Период сканирования простоя: каждый цикл переводит достигшие порога песочницы на ступень холоднее", - "settings_hint_metrics_sample_interval": "Период съёма метрик: разрешение исторических графиков; с этим шагом сохраняются тренды обзора и история метрик", - "settings_hint_metrics_retention": "Срок хранения замеров метрик по песочницам; счётчики состояний парка всегда хранятся 30 дней и от него не зависят", + "settings_hint_node_cpu_limit": "Шлагбаум размещения: узел, у которого в последнем отчёте загрузка CPU выше этого процента, не получает новых песочниц", + "settings_hint_node_active_limit": "Шлагбаум размещения: узел с таким числом активных песочниц (включая размещённые после отчёта) новых не получает; замороженные не считаются", + "settings_hint_node_min_disk": "Шлагбаум размещения: узел, у которого на диске данных свободно меньше этого числа ГиБ, не получает новых песочниц — заполненный диск данных останавливает все песочницы узла разом", "settings_hint_sandbox_disk": "Стартовое значение дисковой квоты по умолчанию — действующее значение в настройках выше", "settings_hint_sandbox_cpus": "Стартовое значение квоты CPU по умолчанию — действующее значение в настройках выше", "settings_hint_sandbox_memory": "Стартовое значение лимита памяти по умолчанию — действующее значение в настройках выше", "settings_hint_sandbox_pids_limit": "Стартовое значение лимита pids при первом запуске; действующее значение в карточке настроек выше", - "settings_hint_reclaim_timeout": "Максимальное ожидание вытеснения памяти в swap при заморозке", "settings_hint_sandbox_domain": "Стартовое значение канонического домена песочниц — действующая конфигурация (включая алиасы) находится в карточке домена песочниц на странице «Домены»", - "settings_hint_ingress_file": "Файл конфигурации Caddy под управлением daemon; без него привязка доменов на странице «Домены» недоступна", + "settings_hint_ingress_file": "Файл конфигурации Caddy под управлением шлюза; без него привязка доменов на странице «Домены» недоступна", "settings_hint_ingress_reload_cmd": "Команда перезагрузки после изменения конфигурации прокси; по умолчанию caddy reload на самом управляемом файле", "settings_hint_s3_endpoint": "Адрес объектного хранилища для архивов; архивация включается, только когда заданы все четыре параметра", "settings_hint_s3_bucket": "Имя бакета для архивов", @@ -44,7 +39,7 @@ "settings_source_default": "По умолчанию", "settings_version_title": "Версия", "settings_knobs_title": "Операционные настройки", - "settings_knobs_desc": "Хранятся в реестре; изменения действуют сразу, без перезапуска — одноимённые переменные окружения ниже лишь задают стартовые значения при первом запуске.", + "settings_knobs_desc": "Хранятся в таблице настроек шлюза; изменения действуют сразу и доходят до каждого узла при его следующем отчёте, без перезапуска — одноимённые переменные окружения ниже лишь задают стартовые значения при первом запуске шлюза.", "settings_knobs_last_modified": " Последнее изменение: {time}.", "settings_knobs_never_modified": " Не менялись; всё ещё стартовые значения.", "settings_row_defaults": "Квоты новых песочниц по умолчанию", diff --git a/packages/console/messages/zh-CN/settings.json b/packages/console/messages/zh-CN/settings.json index 73be9c5c..167e1dfb 100644 --- a/packages/console/messages/zh-CN/settings.json +++ b/packages/console/messages/zh-CN/settings.json @@ -1,22 +1,17 @@ { "$schema": "https://inlang.com/schema/inlang-message-format", - "settings_hint_port": "daemon 端口;只绑 127.0.0.1,对外暴露是反向代理的活", - "settings_hint_db_path": "SQLite 账本;docker 模式强制绝对路径", - "settings_hint_node_id": "这台机器在账本里的名字(为将来分片预留)", + "settings_hint_gateway_port": "网关端口;只绑 127.0.0.1,对外暴露是反向代理的活", + "settings_hint_gateway_db_path": "网关自己的 SQLite 库:节点表、舰队设置、模板、API 密钥、控制台账号;必须是绝对路径", "settings_hint_api_token": "唯一的 API 凭证;控制台登录时换成 httpOnly cookie,页面永远读不到它", - "settings_hint_executor": "执行器:docker 是真沙箱,fake 是内存假执行器(开发/测试用)", - "settings_hint_base_image": "沙箱的默认镜像;docker 模式必填", - "settings_hint_data_dir": "沙箱磁盘镜像的家(disks/*.img);归档临时文件也在这里", - "settings_hint_scan_interval": "空闲扫描周期:每一轮把到阈值的沙箱降一格温度", - "settings_hint_metrics_sample_interval": "指标采样周期:历史曲线的分辨率,总览走势与指标历史都按它落库", - "settings_hint_metrics_retention": "逐沙箱指标样本的保留时长;舰队状态计数恒保 30 天,不随它走", + "settings_hint_node_cpu_limit": "落位闸:节点上一次报到的整机 CPU 高于此百分比就不接新沙箱", + "settings_hint_node_active_limit": "落位闸:节点上活跃沙箱数(含报到后已落位的)达到此数就不接新沙箱;冻结的不算", + "settings_hint_node_min_disk": "落位闸:节点数据盘剩余空间低于此 GiB 就不接新沙箱 — 数据盘写满会让整节点的沙箱一起停摆", "settings_hint_sandbox_disk": "默认磁盘配额的首启种子值 — 生效值在上方运营旋钮里", "settings_hint_sandbox_cpus": "默认 CPU 配额的首启种子值 — 生效值在上方运营旋钮里", "settings_hint_sandbox_memory": "默认内存上限的首启种子值 — 生效值在上方运营旋钮里", "settings_hint_sandbox_pids_limit": "pids 上限的首启种子值 — 生效值在上方运营旋钮里", - "settings_hint_reclaim_timeout": "冻结时挤内存进 swap 的最长等待", "settings_hint_sandbox_domain": "沙箱规范域名的首启种子值 — 生效配置(含别名)在「域名」页的沙箱域名卡里", - "settings_hint_ingress_file": "daemon 接管的 Caddy 配置文件;配了才能在「域名」页网页绑定域名", + "settings_hint_ingress_file": "网关接管的 Caddy 配置文件;配了才能在「域名」页网页绑定域名", "settings_hint_ingress_reload_cmd": "改完代理配置后的重载命令;不配默认 caddy reload 托管文件本身", "settings_hint_s3_endpoint": "归档端点的首启种子值 — 生效配置在上方归档存储卡里", "settings_hint_s3_bucket": "归档桶名的首启种子值 — 生效配置在上方归档存储卡里", @@ -44,7 +39,7 @@ "settings_source_default": "默认值", "settings_version_title": "版本", "settings_knobs_title": "运营旋钮", - "settings_knobs_desc": "住在账本里,改了立即生效,不用重启 — 下表同名环境变量只是首次启动的种子值。", + "settings_knobs_desc": "住在网关的设置表里,改了立即生效、节点在下一次报到时接手,不用重启 — 下表同名环境变量只是网关首次启动的种子值。", "settings_knobs_last_modified": "最后修改于 {time}。", "settings_knobs_never_modified": "从未改过,仍是种子值。", "settings_row_defaults": "新沙箱默认配额", diff --git a/packages/console/messages/zh-TW/settings.json b/packages/console/messages/zh-TW/settings.json index 7ba5a4f5..14ec40c3 100644 --- a/packages/console/messages/zh-TW/settings.json +++ b/packages/console/messages/zh-TW/settings.json @@ -1,22 +1,17 @@ { "$schema": "https://inlang.com/schema/inlang-message-format", - "settings_hint_port": "daemon 連接埠;只綁 127.0.0.1,對外開放是反向代理的活", - "settings_hint_db_path": "SQLite 帳本;docker 模式強制絕對路徑", - "settings_hint_node_id": "這台機器在帳本裡的名字(為將來分片預留)", + "settings_hint_gateway_port": "網關連接埠;只綁 127.0.0.1,對外開放是反向代理的活", + "settings_hint_gateway_db_path": "網關自己的 SQLite 資料庫:節點表、艦隊設定、範本、API 金鑰、控制台帳號;必須是絕對路徑", "settings_hint_api_token": "唯一的 API 憑證;主控台登入時換成 httpOnly cookie,頁面永遠讀不到它", - "settings_hint_executor": "執行器:docker 是真沙箱,fake 是記憶體假執行器(開發/測試用)", - "settings_hint_base_image": "沙箱的預設映像檔;docker 模式必填", - "settings_hint_data_dir": "沙箱磁碟映像檔的家(disks/*.img);封存暫存檔也在這裡", - "settings_hint_scan_interval": "閒置掃描週期:每一輪把到門檻的沙箱降一格溫度", - "settings_hint_metrics_sample_interval": "指標取樣週期:歷史曲線的解析度,總覽走勢與指標歷史都按它寫入資料庫", - "settings_hint_metrics_retention": "逐沙箱指標樣本的保留時長;艦隊狀態計數恆保 30 天,不隨它走", + "settings_hint_node_cpu_limit": "落位閘:節點上一次報到的整機 CPU 高於此百分比就不接新沙箱", + "settings_hint_node_active_limit": "落位閘:節點上活躍沙箱數(含報到後已落位的)達到此數就不接新沙箱;凍結的不算", + "settings_hint_node_min_disk": "落位閘:節點資料碟剩餘空間低於此 GiB 就不接新沙箱 — 資料碟寫滿會讓整節點的沙箱一起停擺", "settings_hint_sandbox_disk": "預設磁碟配額的首次啟動種子值 — 生效值在上方維運旋鈕裡", "settings_hint_sandbox_cpus": "預設 CPU 配額的首次啟動種子值 — 生效值在上方維運旋鈕裡", "settings_hint_sandbox_memory": "預設記憶體上限的首次啟動種子值 — 生效值在上方維運旋鈕裡", "settings_hint_sandbox_pids_limit": "pids 上限的首啟種子值 — 生效值在上方營運旋鈕裡", - "settings_hint_reclaim_timeout": "凍結時把記憶體擠入 swap 的最長等待", "settings_hint_sandbox_domain": "沙箱正規網域的首次啟動種子值 — 生效設定(含別名)在「網域」頁的沙箱網域卡裡", - "settings_hint_ingress_file": "daemon 接管的 Caddy 設定檔;設定了才能在「網域」頁用網頁綁定網域", + "settings_hint_ingress_file": "網關接管的 Caddy 設定檔;設定了才能在「網域」頁用網頁綁定網域", "settings_hint_ingress_reload_cmd": "改完代理設定後的重新載入指令;不設定則預設 caddy reload 託管檔案本身", "settings_hint_s3_endpoint": "封存物件儲存的位址;四件套齊了封存才啟用", "settings_hint_s3_bucket": "封存儲存貯體名稱", @@ -44,7 +39,7 @@ "settings_source_default": "預設值", "settings_version_title": "版本", "settings_knobs_title": "維運旋鈕", - "settings_knobs_desc": "住在帳本裡,改了立即生效,不用重啟 — 下表同名環境變數只是首次啟動的種子值。", + "settings_knobs_desc": "住在網關的設定表裡,改了立即生效、節點在下一次報到時接手,不用重啟 — 下表同名環境變數只是網關首次啟動的種子值。", "settings_knobs_last_modified": "最後修改於 {time}。", "settings_knobs_never_modified": "從未改過,仍是種子值。", "settings_row_defaults": "新沙箱預設配額", diff --git a/packages/console/src/features/settings/pages/SettingsPage.tsx b/packages/console/src/features/settings/pages/SettingsPage.tsx index c5fb8831..dd7317fe 100644 --- a/packages/console/src/features/settings/pages/SettingsPage.tsx +++ b/packages/console/src/features/settings/pages/SettingsPage.tsx @@ -25,25 +25,22 @@ import { useConfig } from '../hooks/useConfig'; /** * 每个旋钮管什么,一句话 — UI 文案,所以住在前端;wire 上只有键、值、 - * 来源(daemon 不说中文)。没列到的键显示为空,不编造。 + * 来源(网关不说中文)。2026-09-14 起 getConfig 是网关的:这里列的是网关 + * 的环境变量 — 它自己的端口/库/落位闸,加舰队设置的首启种子;节点的端 + * 口、执行器、数据盘等不在这张表上(它们是节点机器的事)。没列到的键显示 + * 为空,不编造。 */ const KEY_HINTS: Record string> = { - DORMICE_PORT: m.settings_hint_port, - DORMICE_DB_PATH: m.settings_hint_db_path, - DORMICE_NODE_ID: m.settings_hint_node_id, + DORMICE_GATEWAY_PORT: m.settings_hint_gateway_port, + DORMICE_GATEWAY_DB_PATH: m.settings_hint_gateway_db_path, DORMICE_API_TOKEN: m.settings_hint_api_token, - DORMICE_EXECUTOR: m.settings_hint_executor, - DORMICE_BASE_IMAGE: m.settings_hint_base_image, - DORMICE_DATA_DIR: m.settings_hint_data_dir, - DORMICE_SCAN_INTERVAL_SECONDS: m.settings_hint_scan_interval, - DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS: - m.settings_hint_metrics_sample_interval, - DORMICE_METRICS_RETENTION_HOURS: m.settings_hint_metrics_retention, + DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: m.settings_hint_node_cpu_limit, + DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: m.settings_hint_node_active_limit, + DORMICE_GATEWAY_NODE_MIN_DISK_GB: m.settings_hint_node_min_disk, DORMICE_SANDBOX_DISK_GB: m.settings_hint_sandbox_disk, DORMICE_SANDBOX_CPUS: m.settings_hint_sandbox_cpus, DORMICE_SANDBOX_MEMORY_GB: m.settings_hint_sandbox_memory, DORMICE_SANDBOX_PIDS_LIMIT: m.settings_hint_sandbox_pids_limit, - DORMICE_RECLAIM_TIMEOUT_SECONDS: m.settings_hint_reclaim_timeout, DORMICE_SANDBOX_DOMAIN: m.settings_hint_sandbox_domain, DORMICE_INGRESS_FILE: m.settings_hint_ingress_file, DORMICE_INGRESS_RELOAD_CMD: m.settings_hint_ingress_reload_cmd, @@ -73,9 +70,10 @@ const PAGE_SIZE = 50; * 设置页两段(2026-07-19 用户拍板加运营旋钮):上面是账本里的运营旋钮 * — 容量上限、新沙箱默认配额、默认策略,updateSettings 网页可改、立即 * 生效;下面仍是 env 配置的只读观察窗 — 端口、token、executor 这些 - * "身份与地基"改了就是另一台 daemon,真身留在 /etc/dormice/env,改完 - * 重启生效。daemon 从不写自己的环境文件(那是另一个安全等级的决定), - * 运营旋钮走的是账本:env 同名变量降级为首次启动的种子值。 + * "身份与地基"改了就是另一台网关,真身留在 /etc/dormice/gateway.env,改完 + * 重启生效。网关从不写自己的环境文件(那是另一个安全等级的决定),运营 + * 旋钮走的是网关的设置表,节点在下一次报到时接手:env 同名变量降级为网关 + * 首次启动的种子值(2026-09-14 配置权威搬到网关)。 */ export function SettingsPage() { const { data, isPending, isError, error } = useConfig(); @@ -112,9 +110,9 @@ export function SettingsPage() { {/* 这行不是装饰:两类旋钮的界限与 env 的改法只在这里说。 */}

{m.settings_env_note_1()}{' '} - /etc/dormice/env + /etc/dormice/gateway.env {m.settings_env_note_2()}{' '} - systemctl restart dormice + systemctl restart dormice-gateway {m.settings_env_note_archive()} {data.archive.enabled ? m.settings_archive_enabled({ diff --git a/packages/console/vite.config.ts b/packages/console/vite.config.ts index 4aa7e32b..fed898d4 100644 --- a/packages/console/vite.config.ts +++ b/packages/console/vite.config.ts @@ -26,45 +26,53 @@ export default defineConfig({ alias: { '@': fileURLToPath(new URL('./src', import.meta.url)) }, }, server: { - // Dev mode: Vite owns /console, the daemon (fake executor is fine) owns - // the API — same paths the browser uses in production, so no API base - // knob. - proxy: { - '/console/auth': 'http://127.0.0.1:3676', - '/envdToken': 'http://127.0.0.1:3676', - '/listSandboxes': 'http://127.0.0.1:3676', - '/destroySandbox': 'http://127.0.0.1:3676', - '/acquireSandbox': 'http://127.0.0.1:3676', - '/rebuildSandbox': 'http://127.0.0.1:3676', - '/updatePolicy': 'http://127.0.0.1:3676', - '/updateMetadata': 'http://127.0.0.1:3676', - '/listTemplates': 'http://127.0.0.1:3676', - '/registerTemplate': 'http://127.0.0.1:3676', - '/removeTemplate': 'http://127.0.0.1:3676', - '/listApiKeys': 'http://127.0.0.1:3676', - '/createApiKey': 'http://127.0.0.1:3676', - '/revokeApiKey': 'http://127.0.0.1:3676', - '/updateApiKey': 'http://127.0.0.1:3676', - '/execCommand': 'http://127.0.0.1:3676', - '/writeFile': 'http://127.0.0.1:3676', - '/writeFiles': 'http://127.0.0.1:3676', - '/readFile': 'http://127.0.0.1:3676', - '/readFiles': 'http://127.0.0.1:3676', - '/getHostMetrics': 'http://127.0.0.1:3676', - '/getSandboxMetrics': 'http://127.0.0.1:3676', - '/getSandboxMetricsHistory': 'http://127.0.0.1:3676', - '/listSandboxMetrics': 'http://127.0.0.1:3676', - '/listSandboxImages': 'http://127.0.0.1:3676', - '/getFleetTimeline': 'http://127.0.0.1:3676', - '/getConfig': 'http://127.0.0.1:3676', - '/checkUpgrade': 'http://127.0.0.1:3676', - '/applyUpgrade': 'http://127.0.0.1:3676', - '/getUpgradeStatus': 'http://127.0.0.1:3676', - '/updateSettings': 'http://127.0.0.1:3676', - '/getIngress': 'http://127.0.0.1:3676', - '/setIngress': 'http://127.0.0.1:3676', - // The terminal speaks the envd surface directly, like the e2b SDK. - '/e2b': 'http://127.0.0.1:3676', - }, + // Dev mode: Vite owns /console, the gateway owns the API — the fleet's + // one door (2026-09-14: configuration, keys, templates, the console's + // sessions all answer there, the sandbox verbs are forwarded to the + // node). Same paths the browser uses in production, so no API base + // knob. Run a gateway on 3677 and a daemon (fake executor is fine) + // checking in with it. + proxy: Object.fromEntries( + [ + '/console/auth', + '/envdToken', + '/listSandboxes', + '/destroySandbox', + '/acquireSandbox', + '/rebuildSandbox', + '/updatePolicy', + '/updateMetadata', + '/listTemplates', + '/registerTemplate', + '/removeTemplate', + '/listApiKeys', + '/createApiKey', + '/revokeApiKey', + '/updateApiKey', + '/execCommand', + '/writeFile', + '/writeFiles', + '/readFile', + '/readFiles', + '/getHostMetrics', + '/getSandboxMetrics', + '/getSandboxMetricsHistory', + '/listSandboxMetrics', + '/listSandboxImages', + '/getFleetTimeline', + '/getConfig', + '/checkUpgrade', + '/applyUpgrade', + '/getUpgradeStatus', + '/updateSettings', + '/getIngress', + '/setIngress', + '/listNodes', + '/removeNode', + '/updateNodeSettings', + // The terminal speaks the envd surface directly, like the e2b SDK. + '/e2b', + ].map((path) => [path, 'http://127.0.0.1:3677']), + ), }, }); From cf0da88eb15ff0e599e105e6fe4e79386c2ae27a Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 17:33:24 +0800 Subject: [PATCH 36/89] install.sh installs the gateway beside the daemon: one token in two env files, the gateway first, Caddy at the door MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A daemon takes its first configuration bundle from its gateway before it listens, so a machine needs both processes: install.sh now writes /etc/dormice/gateway.env (the daemon's DORMICE_API_TOKEN verbatim — the fleet has one token — plus the archive-store seed template and, on a machine upgraded across the move, the fleet knobs an operator had in the daemon's env, carried over and commented out where they do nothing now), installs deploy/dormice-gateway.service, restarts the gateway and waits for its /healthz before restarting the daemon, and refuses to continue when the two files carry different tokens. Caddy's catch-all points at the gateway, where the console lives; a Dormice-managed Caddyfile from before the move is re-pointed in place and reloaded. DORMICE_INGRESS_FILE goes into gateway.env. Doctor runs with both env files loaded, so the S3 and ingress checks read the seeds where they are. The daemon unit orders After= the gateway unit, not Requires=: a node on a machine of its own retries a remote gateway. --- deploy/dormice-gateway.service | 18 ++-- deploy/dormice.service | 15 ++- deploy/install.sh | 174 ++++++++++++++++++++++++--------- 3 files changed, 148 insertions(+), 59 deletions(-) diff --git a/deploy/dormice-gateway.service b/deploy/dormice-gateway.service index 45f3eded..5ecd549a 100644 --- a/deploy/dormice-gateway.service +++ b/deploy/dormice-gateway.service @@ -1,11 +1,13 @@ -# Dormice gateway: the fleet's one door in front of one or more daemons. -# Configuration lives in /etc/dormice/gateway.env (full-line comments only -# there — systemd's EnvironmentFile treats an inline comment as part of the -# value). install.sh does not install this unit yet: the two-role install -# (gateway + node on one machine, `--role node` elsewhere) is a later step; -# until then it is copied into place by hand (docs/测试机搭建手册.md). Once it -# is enabled, install.sh re-runs rebuild the gateway with the daemon and -# restart this unit before the daemon's, so both run one commit. +# Dormice gateway: the fleet's one door in front of one or more daemons — +# the fleet's configuration, its API keys and the web console live here, +# the sandbox verbs are forwarded to the node that holds the sandbox. +# Installed by deploy/install.sh beside the daemon (a single machine is a +# fleet of one); configuration lives in /etc/dormice/gateway.env (full-line +# comments only there — systemd's EnvironmentFile treats an inline comment +# as part of the value). install.sh restarts this unit before the daemon's, +# so both run one commit and the daemon's first check-in lands on the new +# gateway. The `--role node` install for a machine without a gateway is a +# later step. [Unit] Description=Dormice gateway (fleet front door) Wants=network-online.target diff --git a/deploy/dormice.service b/deploy/dormice.service index 5dd82255..2a8d1e05 100644 --- a/deploy/dormice.service +++ b/deploy/dormice.service @@ -1,10 +1,15 @@ -# Dormice daemon. Installed by deploy/install.sh; configuration lives in -# /etc/dormice/env (full-line comments only there — systemd's EnvironmentFile -# treats an inline comment as part of the value). +# Dormice daemon — the node that runs the sandboxes. Installed by +# deploy/install.sh; configuration lives in /etc/dormice/env (full-line +# comments only there — systemd's EnvironmentFile treats an inline comment +# as part of the value). It takes the fleet's configuration from its +# gateway at check-in and, holding no copy yet, waits for the gateway +# before it listens — hence After= the gateway unit; not Requires=, since +# a node on a machine of its own has no local gateway and keeps retrying +# the remote one on its own. [Unit] -Description=Dormice daemon (agent sandbox control plane) +Description=Dormice daemon (agent sandbox node) Wants=network-online.target -After=network-online.target docker.service +After=network-online.target docker.service dormice-gateway.service Requires=docker.service [Service] diff --git a/deploy/install.sh b/deploy/install.sh index 8b49fa30..62de2bae 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash # Dormice installer: turns a bare Ubuntu/Debian x86_64 host into a running -# Dormice daemon, then proves it by running `dor doctor`. +# Dormice — a gateway (the fleet's one door: configuration, keys, the web +# console) and a daemon (the node that runs the sandboxes) on one machine, +# a fleet of one — then proves it by running `dor doctor`. # # curl -fsSL https://raw.githubusercontent.com/BitMiracle-AI/Dormice/main/deploy/install.sh | bash # @@ -46,9 +48,12 @@ SHIM_SHA512=87c63197836574b7a2c057d2c0647d2badb679187f0b9175ecf78ac52207cdaa3f10 REPO_URL=https://github.com/BitMiracle-AI/Dormice.git INSTALL_DIR=/opt/dormice ENV_FILE=/etc/dormice/env +GATEWAY_ENV_FILE=/etc/dormice/gateway.env DATA_DIR=/var/lib/dormice +GATEWAY_DATA_DIR=/var/lib/dormice-gateway DAEMON_JSON=/etc/docker/daemon.json PORT=3676 +GATEWAY_PORT=3677 # ---- flags ----------------------------------------------------------------- MIRROR='' @@ -609,11 +614,13 @@ else fi # ---- ingress (Caddy reverse proxy) ------------------------------------------- -# The daemon binds 127.0.0.1 by design; Caddy on :80 is what makes -# http:///console reachable from a browser. The Caddyfile below is -# also what the daemon rewrites when the operator binds a domain in the -# console (setIngress) — Caddy then obtains and renews the TLS certificate -# on its own. Pinned binary with checksum, same posture as gVisor. +# Gateway and daemon bind 127.0.0.1 by design; Caddy on :80 is what makes +# http:///console reachable from a browser. It proxies to the +# GATEWAY — the fleet's one door, which serves the console and forwards +# the sandbox verbs to the daemon. The Caddyfile below is also what the +# gateway rewrites when the operator binds a domain in the console +# (setIngress) — Caddy then obtains and renews the TLS certificate on its +# own. Pinned binary with checksum, same posture as gVisor. log 'ingress (Caddy reverse proxy)' CADDY_VERSION=2.10.0 CADDY_SHA512=626682d623ca04356ab3c9a93a82386cfde6d8243b11f2d0eea9e97ba630c7ada62373401e96b72c6690c98ae8dd004d61fafe477f5249690d5cb251ebbfd2d9 @@ -622,9 +629,10 @@ if command -v caddy >/dev/null; then note "[skip] caddy is installed ($(caddy version | cut -d' ' -f1))" elif ss -ltnH 'sport = :80' 2>/dev/null | grep -q .; then # Another server owns port 80: never fight it. The operator keeps their - # proxy (point it at 127.0.0.1:$PORT); web domain binding stays off. + # proxy (point it at the gateway, 127.0.0.1:$GATEWAY_PORT); web domain + # binding stays off. note "port 80 is already in use and caddy is not installed — skipping the ingress layer" - note "point your own reverse proxy at 127.0.0.1:$PORT; the console's domain binding stays disabled" + note "point your own reverse proxy at 127.0.0.1:$GATEWAY_PORT (the gateway); the console's domain binding stays disabled" else caddy_url="https://github.com/caddyserver/caddy/releases/download/v$CADDY_VERSION/caddy_${CADDY_VERSION}_linux_amd64.tar.gz" [ "$MIRROR" = cn ] && caddy_url="https://ghfast.top/$caddy_url" @@ -636,25 +644,36 @@ else note "installed caddy v$CADDY_VERSION to /usr/local/bin" fi INGRESS_FILE_READY='' +CADDY_REPOINTED='' if command -v caddy >/dev/null; then mkdir -p /etc/caddy if [ ! -f "$CADDYFILE" ]; then - # The marker below is the ownership contract: the daemon refuses to + # The marker below is the ownership contract: the gateway refuses to # rewrite a Caddyfile that lacks it. Kept in sync by hand with - # packages/server/src/ingress.ts. + # packages/gateway/src/ingress.ts. cat >"$CADDYFILE" </dev/null 2>&1 || systemctl restart caddy + note 'reloaded caddy with the re-pointed config' else note '[skip] caddy is running' fi fi # ---- daemon configuration ---------------------------------------------------- +# The daemon's env is the node's identity and its machine: token, executor, +# image, ledger, data dir. The fleet's operator knobs (sandbox defaults, +# the archive store, the sandbox domain, the managed front door) are the +# gateway's since 2026-09-14 — its env seeds them once, the console edits +# them, and the daemon takes them from its check-in. log "daemon configuration ($ENV_FILE)" install -d -m 700 "$DATA_DIR" if [ -f "$ENV_FILE" ]; then @@ -695,57 +722,106 @@ else # No inline comments below: systemd's EnvironmentFile takes the whole line # as the value. Full-line comments are fine. cat >"$ENV_FILE" <"$GATEWAY_ENV_FILE" <>"$GATEWAY_ENV_FILE" + sed -i "s|^$knob=|# moved to $GATEWAY_ENV_FILE (2026-09-14): $knob=|" "$ENV_FILE" + carried="$carried $knob" + fi + done + chmod 600 "$GATEWAY_ENV_FILE" + if [ -n "$carried" ]; then + note "wrote $GATEWAY_ENV_FILE (mode 600) with the fleet token; carried over from $ENV_FILE:$carried" + else + note "wrote $GATEWAY_ENV_FILE (mode 600) with the fleet token" + fi fi # Appended outside the create-once block so an upgrade re-run picks the # knob up too. The knob is what turns on web domain binding in the console. -if [ -n "$INGRESS_FILE_READY" ] && ! grep -q '^DORMICE_INGRESS_FILE=' "$ENV_FILE"; then +if [ -n "$INGRESS_FILE_READY" ] && ! grep -q '^DORMICE_INGRESS_FILE=' "$GATEWAY_ENV_FILE"; then { - echo '# The Caddy config file the daemon owns: enables binding domains (and' + echo '# The Caddy config file the gateway owns: enables binding domains (and' echo '# getting HTTPS) from the console domains page.' echo "DORMICE_INGRESS_FILE=$CADDYFILE" - } >>"$ENV_FILE" - note "added DORMICE_INGRESS_FILE=$CADDYFILE to $ENV_FILE" + } >>"$GATEWAY_ENV_FILE" + note "added DORMICE_INGRESS_FILE=$CADDYFILE to $GATEWAY_ENV_FILE" +fi +if ! grep -q "^DORMICE_API_TOKEN=$API_TOKEN\$" "$GATEWAY_ENV_FILE"; then + die "$GATEWAY_ENV_FILE and $ENV_FILE carry different DORMICE_API_TOKEN values — the fleet has one token; make them the same and re-run" fi -# ---- systemd service --------------------------------------------------------- -log 'systemd service' +# ---- systemd services -------------------------------------------------------- +# Two units, the gateway first: a daemon without a configuration copy takes +# its first bundle from its gateway before it listens, and a re-run just +# built both dists — the two processes of a fleet of one run one commit, +# never two. Restart, not start: both are crash-only by design, so +# restarting them is always safe. +log 'systemd services' +cp "$INSTALL_DIR/deploy/dormice-gateway.service" /etc/systemd/system/dormice-gateway.service cp "$INSTALL_DIR/deploy/dormice.service" /etc/systemd/system/dormice.service systemctl daemon-reload -systemctl enable dormice >/dev/null 2>&1 -# A gateway installed by hand on this machine (deploy/dormice-gateway.service; -# install.sh does not install it yet) was just rebuilt with the daemon and is -# restarted first, so the daemon's first check-in lands on the new one: the two -# processes of a fleet of one run one commit, never two. Running or enabled — -# a unit started by hand and never enabled is running the old dist just the -# same. -if systemctl is-active -q dormice-gateway 2>/dev/null || systemctl is-enabled -q dormice-gateway 2>/dev/null; then - systemctl restart dormice-gateway - note 'restarted dormice-gateway (hand-installed unit, rebuilt with the daemon)' -fi -# Restart, not start: a re-run just built fresh code, and the daemon is -# crash-only by design — restarting it is always safe. +systemctl enable dormice-gateway dormice >/dev/null 2>&1 +systemctl restart dormice-gateway +for _ in $(seq 1 60); do + curl -fsS "http://127.0.0.1:$GATEWAY_PORT/healthz" >/dev/null 2>&1 && break + sleep 0.5 +done +curl -fsS "http://127.0.0.1:$GATEWAY_PORT/healthz" >/dev/null 2>&1 \ + || die "the gateway did not answer /healthz on 127.0.0.1:$GATEWAY_PORT — check: journalctl -u dormice-gateway -n 50" +note "gateway is answering on 127.0.0.1:$GATEWAY_PORT" systemctl restart dormice -note 'enabled and (re)started' +note 'enabled and (re)started both' # ---- verification: the install has not succeeded until doctor says so -------- log 'verification' @@ -760,11 +836,15 @@ for _ in $(seq 1 240); do sleep 0.5 done curl -fsS "http://127.0.0.1:$PORT/healthz" >/dev/null 2>&1 \ - || die "the daemon did not answer /healthz on 127.0.0.1:$PORT — check: journalctl -u dormice -n 50" + || die "the daemon did not answer /healthz on 127.0.0.1:$PORT — check: journalctl -u dormice -n 50 (a daemon with no configuration copy waits for its gateway before it listens)" note "daemon is answering on 127.0.0.1:$PORT" +# Both env files: doctor reads the node's knobs from the daemon's and the +# fleet's seeds (the S3 set, the managed front door) from the gateway's. set -a # shellcheck source=/dev/null . "$ENV_FILE" +# shellcheck source=/dev/null +. "$GATEWAY_ENV_FILE" set +a dor doctor @@ -774,10 +854,12 @@ dor doctor status_write succeeded printf '\nDormice is installed.\n' -printf ' API token: grep ^DORMICE_API_TOKEN %s\n' "$ENV_FILE" -printf ' daemon logs: journalctl -u dormice -f\n' -printf ' CLI: export DORMICE_ENDPOINT=http://127.0.0.1:%s DORMICE_API_TOKEN=; dor sandbox ls\n' "$PORT" -printf ' The daemon listens on 127.0.0.1 only, by design — exposing it is a reverse proxy'"'"'s job.\n' +printf ' API token: grep ^DORMICE_API_TOKEN %s\n' "$ENV_FILE" +printf ' gateway logs: journalctl -u dormice-gateway -f (the door: console, keys, settings, templates)\n' +printf ' daemon logs: journalctl -u dormice -f (the node: sandboxes)\n' +printf ' console: http:///console (Caddy on :80 -> the gateway on 127.0.0.1:%s)\n' "$GATEWAY_PORT" +printf ' CLI: export DORMICE_ENDPOINT=http://127.0.0.1:%s DORMICE_API_TOKEN=; dor sandbox ls\n' "$PORT" +printf ' Both processes listen on 127.0.0.1 only, by design — exposing them is a reverse proxy'"'"'s job.\n' if [ "$(systemctl is-active caddy 2>/dev/null)" = active ]; then printf ' console: http:///console (Caddy on :80 — open your cloud firewall for 80/443,\n' printf ' then bind domains in the domains page for automatic HTTPS)\n' From e41c80585465a9cbbedb39964acbba7a6e642c86 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 17:34:27 +0800 Subject: [PATCH 37/89] The docs describe two doors: the gateway holds the fleet's configuration, the daemon runs the sandboxes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Configuration is split by env file — the gateway's knobs and the fleet seeds in gateway.env, the node's identity in env — with the one token in both. The HTTP API table says which door answers each verb and lists the node verbs; the admin gate names everything that configures the fleet. Installation, console, ports, archiving, doctor and troubleshooting follow the move, and the console page says plainly which of its pages stay dark until the gateway answers the fleet-wide list and observation verbs. README and the skill point the console and the configuration verbs at the gateway. --- README.md | 31 ++--- skills/dormice/SKILL.md | 51 +++++---- website/content/docs/archiving.mdx | 11 +- website/content/docs/configuration.mdx | 137 ++++++++++++++--------- website/content/docs/console.mdx | 19 +++- website/content/docs/doctor.mdx | 32 +++--- website/content/docs/http-api.mdx | 117 +++++++++++-------- website/content/docs/installation.mdx | 40 ++++--- website/content/docs/ports.mdx | 8 +- website/content/docs/troubleshooting.mdx | 13 +-- 10 files changed, 272 insertions(+), 187 deletions(-) diff --git a/README.md b/README.md index 8ca14744..38a0a6c1 100644 --- a/README.md +++ b/README.md @@ -135,28 +135,30 @@ Deliberate deltas from the hosted product: ## Web console -The daemon serves a small web console at `http://127.0.0.1:3676/console` — -sign in with the API token once and it becomes an httpOnly session cookie; -the token itself is never stored anywhere the page can read. The console +The gateway — the fleet's front door, installed beside the daemon — serves +a small web console at `http://127.0.0.1:3677/console`: sign in with the +API token once and it becomes an httpOnly session cookie; the token itself +is never stored anywhere the page can read. The console shows every sandbox with its live lifecycle state (the same `/listSandboxes` the SDK sees), opens a per-sandbox detail view, creates sandboxes (the same idempotent `acquire`, with the lifecycle knobs), releases them, and has a Connect page with copy-paste snippets for every client (E2B SDK, native SDK, CLI) pointed at your own endpoint. -The daemon listens on 127.0.0.1 only, so reaching it from another machine -is a choice you make explicitly, one of two ways: +Both processes listen on 127.0.0.1 only (the gateway on 3677, the daemon +on 3676), so reaching them from another machine is a choice you make +explicitly, one of two ways: - **SSH tunnel** (private, zero setup): - `ssh -L 3676:127.0.0.1:3676 root@host`, then open - `http://127.0.0.1:3676/console`. + `ssh -L 3677:127.0.0.1:3677 root@host`, then open + `http://127.0.0.1:3677/console`. - **Reverse proxy** for the console, the API, and the E2B surface at once — - e.g. Caddy, which also handles TLS certificates automatically once you - give it a domain: + e.g. Caddy pointed at the gateway, which also handles TLS certificates + automatically once you give it a domain: ``` your-domain.example { - reverse_proxy 127.0.0.1:3676 { + reverse_proxy 127.0.0.1:3677 { flush_interval -1 } } @@ -169,8 +171,9 @@ is a choice you make explicitly, one of two ways: ## Cold archive (S3, optional) -Set the four `DORMICE_S3_*` variables and idle sandboxes take the last -step down: a week after stopping (tunable per sandbox via +Configure a store in the console's settings page — or seed the four +`DORMICE_S3_*` variables in the gateway's env (`/etc/dormice/gateway.env`) +— and idle sandboxes take the last step down: a week after stopping (tunable per sandbox via `archiveAfterSeconds`), the disk is packed with `tar` + `zstd`, shipped to any S3-compatible bucket (AWS, Cloudflare R2, MinIO, Alibaba OSS in S3-compat mode), and freed locally. The next `acquireSandbox` answers @@ -242,8 +245,8 @@ pnpm monorepo: | `packages/server` | The daemon: Fastify + SQLite ledger + lifecycle engine | | `packages/sdk` | `@dormice/sdk` — TypeScript client for the native API | | `packages/cli` | `dormice` command-line tool (`dor` for short) | -| `packages/console` | Web console: React SPA, served by the daemon at `/console` | -| `packages/gateway` | The fleet's one door in front of one or more daemons: places new sandboxes, finds existing ones by asking the nodes, forwards everything else | +| `packages/console` | Web console: React SPA, served by the gateway at `/console` | +| `packages/gateway` | The fleet's one door in front of one or more daemons: holds the fleet's settings, templates, API keys and the console; places new sandboxes, finds existing ones by asking the nodes, forwards everything else | | `e2e` | Black-box suite: boots the built daemon, drives it over the wire | | `examples` | Runnable demos: the native SDK, the official `e2b` package, a resident agent | diff --git a/skills/dormice/SKILL.md b/skills/dormice/SKILL.md index 1801fa96..c31c6bea 100644 --- a/skills/dormice/SKILL.md +++ b/skills/dormice/SKILL.md @@ -5,8 +5,10 @@ description: Operate Dormice self-hosted agent sandboxes — acquire a sandbox, # Dormice -Dormice is a self-hosted sandbox platform: one daemon on one machine, and -sandboxes that are **permanent** — idle ones cool down +Dormice is a self-hosted sandbox platform: a gateway (the front door — +console, API keys, fleet settings) and one or more daemons (the nodes that +run the sandboxes; a single machine runs both), and sandboxes that are +**permanent** — idle ones cool down (`active → frozen → stopped → archived`) instead of being destroyed, and any acquire brings them back. Two facts drive every workflow below: @@ -21,17 +23,21 @@ acquire brings them back. Two facts drive every workflow below: ## Connecting -The daemon serves everything on one port and binds to `127.0.0.1:3676` -only. On the server itself use that address directly; from another machine -the operator has either an SSH tunnel -(`ssh -L 3676:127.0.0.1:3676 root@host`, then use `http://127.0.0.1:3676`) -or a reverse-proxy domain (then use `https://their-domain`). Auth is one -API token (created by the installer, hex): ask the user for the endpoint -and token, conventionally held in `DORMICE_ENDPOINT` / `DORMICE_API_TOKEN`. -API keys minted in the console (or with `dor apikey create `) work -everywhere the token does — same variables, revocable per client — except -the apiKey management verbs themselves, which require the env token -(keys cannot manage keys). +Two doors, both bound to `127.0.0.1` only: the gateway on `3677` (the +console, templates, API keys, settings, and every per-sandbox verb, which +it forwards to the node) and the daemon on `3676` (the sandbox verbs plus +the list and observation verbs the gateway does not answer yet). On the +server itself use those addresses directly; from another machine the +operator has either an SSH tunnel (`ssh -L 3677:127.0.0.1:3677 -L +3676:127.0.0.1:3676 root@host`) or a reverse-proxy domain in front of the +gateway (then use `https://their-domain`). Auth is one API token (created +by the installer, hex): ask the user for the endpoint and token, +conventionally held in `DORMICE_ENDPOINT` / `DORMICE_API_TOKEN`. API keys +minted in the console (or with `dor apikey create `) open the +gateway wherever the token does — same variables, revocable per client — +except the verbs that configure the fleet (keys, settings, templates, +domains), which require the token (keys cannot manage keys); a daemon +knows only the token. ## Pick an entry path @@ -119,16 +125,19 @@ an existing sandbox's lifecycle policy in place — no wake, no destroy), `{}` clears — same no-wake manners), `listSandboxes`, `execCommand`, `writeFiles` / `writeFile`, `readFile` / `readFiles`, `rebuildSandbox` (fresh container, `/home/user` kept), `destroySandbox`, -`registerTemplate` / `listTemplates` / `removeTemplate`, -`createApiKey` / `listApiKeys` / `updateApiKey` / `revokeApiKey` -(revocable peers of the API token with optional expiry and a reversible -disable switch; the create response shows the key once, never again; -these four verbs accept only the env token), +`registerTemplate` / `listTemplates` / `removeTemplate` (at the gateway; +nodes learn a template at their next check-in), +`createApiKey` / `listApiKeys` / `updateApiKey` / `revokeApiKey` (at the +gateway: revocable peers of the API token with optional expiry and a +reversible disable switch; the create response shows the key once, never +again; these four verbs accept only the token), `getHostMetrics`, `getSandboxMetrics` / `listSandboxMetrics` (live resource samples; never wake anything), `listSandboxImages` (who still -runs an old template image), -`getConfig` (effective config, secrets redacted), `getIngress` / -`setIngress` (bind domains on the daemon's managed reverse proxy). +runs an old template image) — the list and host verbs at the daemon — +`getConfig` / `updateSettings` (the fleet's settings at the gateway, +secrets redacted; applied by every node at its next check-in), +`getIngress` / `setIngress` (bind domains on the gateway's managed reverse +proxy), `listNodes` (every node and what it last reported). `execCommand` takes `{ name, command, timeoutSeconds?, cwd?, env? }` and returns `{ exitCode, stdout, stderr, ... }` — **a non-zero exit code is a result, diff --git a/website/content/docs/archiving.mdx b/website/content/docs/archiving.mdx index 5795026e..82ddc9d5 100644 --- a/website/content/docs/archiving.mdx +++ b/website/content/docs/archiving.mdx @@ -16,14 +16,15 @@ storage. Configure the store in the console's settings page (the archive store card) — endpoint, bucket, credentials, region, path style. Any S3-compatible store works: AWS, Cloudflare R2, MinIO, Alibaba OSS in -S3-compat mode. Before anything is saved, the daemon writes and reads +S3-compat mode. Before anything is saved, the gateway writes and reads back a probe object against the bucket, so a wrong endpoint or credential fails immediately with S3's own error — nothing half-works -silently. The change applies live; no restart. +silently. The change applies live: every node takes the store from its +next check-in; no restart. -(The four `DORMICE_S3_*` environment variables still exist as -first-boot seeds — [reference](/docs/configuration#archiver-s3) — handy -for provisioning a fresh machine from a config file.) +(The four `DORMICE_S3_*` environment variables still exist as the +gateway's first-boot seeds — [reference](/docs/configuration#archiver-s3) +— handy for provisioning a fresh machine from a config file.) With a store configured, stopped sandboxes archive after 7 more idle days by default. Tune it per sandbox with the policy: diff --git a/website/content/docs/configuration.mdx b/website/content/docs/configuration.mdx index cede20be..cf4e9691 100644 --- a/website/content/docs/configuration.mdx +++ b/website/content/docs/configuration.mdx @@ -1,61 +1,74 @@ --- title: Configuration -description: Every environment variable the daemon reads, with its default and what it does. +description: Every environment variable the gateway and the daemon read, with its default and what it does. --- -All daemon configuration comes from `DORMICE_*` environment variables. -They are validated once at startup: a bad value refuses to boot with a -named error instead of surfacing later as confusing runtime behavior. -Everything has a default except the API token and, in docker mode, the -base image. +A Dormice host runs two processes, each with its own environment file: -On a host set up by `install.sh`, the variables live in -`/etc/dormice/env`. Edit the file and restart the daemon: +- **The gateway** — the fleet's one door: the web console, API keys, + and the fleet's settings (new-sandbox defaults, the archive store, + the sandbox domain, templates) live here. `/etc/dormice/gateway.env`. +- **The daemon** — the node that runs the sandboxes. It takes the + fleet's settings from the gateway at its check-in (every 15 seconds) + and keeps a copy in its own database, so it keeps serving while the + gateway is away. `/etc/dormice/env`. + +A single machine runs both — a fleet of one. All variables are +`DORMICE_*`, validated once at startup: a bad value refuses to boot with +a named error instead of surfacing later as confusing runtime behavior. +Edit a file and restart its process: ```sh -systemctl restart dormice +systemctl restart dormice-gateway # after editing gateway.env +systemctl restart dormice # after editing env ``` To see the values actually in force, use the -[console](/docs/console)'s settings page or the `getConfig` verb — both -report every knob with its effective value and whether it came from the -environment or a default (secrets are reported present-or-absent only). +[console](/docs/console)'s settings page or the gateway's `getConfig` +verb — both report every gateway knob with its effective value and +whether it came from the environment or a default (secrets are reported +present-or-absent only), plus the fleet settings and their version. Unfamiliar terms in the tables below are covered in [Core concepts](/docs/core-concepts). -## Core +## The one token | Variable | Default | What it does | | --- | --- | --- | -| `DORMICE_API_TOKEN` | — **(required)** | Bearer token for the native API, and (with an `e2b_` prefix) the E2B API key. At least 32 characters; generate with `openssl rand -hex 32`. Keep it hex: the official Python e2b SDK validates `e2b_[0-9a-f]+` client-side. This is the bootstrap credential — for day-to-day clients, mint revocable [API keys](/docs/http-api#api-keys) instead and keep this one for recovery. | -| `DORMICE_PORT` | `3676` | The one port everything is served on: native API, E2B surface, web console. Always bound to 127.0.0.1 — there is deliberately no host setting. | -| `DORMICE_DB_PATH` | `data/dormice.db` | Where the SQLite database lives. Must be an **absolute path** in docker mode — a relative path depends on the start directory, and a wrong start directory means an empty database facing real sandboxes. | -| `DORMICE_NODE_ID` | `node-1` | Identifies this machine in the database. Single-machine today; the field keeps the schema shardable later. | +| `DORMICE_API_TOKEN` | — **(required, in both files, the same value)** | The fleet's one token: Bearer token for the native API, and (with an `e2b_` prefix) the E2B API key. Callers present it to the gateway, the daemon checks in with it, the gateway forwards to the daemon under it. At least 32 characters; generate with `openssl rand -hex 32`. Keep it hex: the official Python e2b SDK validates `e2b_[0-9a-f]+` client-side. This is the bootstrap credential — for day-to-day clients, mint revocable [API keys](/docs/http-api#api-keys) at the gateway instead and keep this one for recovery. The daemon itself knows only this token; minted keys are judged at the gateway. | + +## Gateway -## Executor +`/etc/dormice/gateway.env` — the gateway's own knobs, then the +first-boot seeds of the fleet settings. | Variable | Default | What it does | | --- | --- | --- | -| `DORMICE_EXECUTOR` | `fake` | What runs the sandboxes: `fake` is an in-memory executor so a bare checkout works on any machine (development, tests); `docker` runs real gVisor sandboxes and needs a prepared Linux host — see [Installation](/docs/installation). | -| `DORMICE_BASE_IMAGE` | — | The image sandboxes boot from. **Required in docker mode**; `install.sh` builds one and records it here. | -| `DORMICE_DATA_DIR` | `/var/lib/dormice` | Sandbox disk images and their mount points (docker mode). Must be an absolute path in docker mode. | +| `DORMICE_GATEWAY_PORT` | `3677` | The gateway's port: web console, configuration verbs, and every sandbox verb it forwards. Always bound to 127.0.0.1 — there is deliberately no host setting. | +| `DORMICE_GATEWAY_DB_PATH` | `/var/lib/dormice-gateway/gateway.db` | The gateway's SQLite database: nodes, fleet settings, templates, API keys, the console account. Must be an absolute path. | +| `DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT` | `70` | Placement gate: a node whose last check-in reported whole-machine CPU above this takes no new sandboxes. | +| `DORMICE_GATEWAY_NODE_ACTIVE_LIMIT` | `400` | Placement gate: a node with this many active sandboxes (placements since its check-in included) takes no more; frozen ones are not counted. | +| `DORMICE_GATEWAY_NODE_MIN_DISK_GB` | `10` | Placement gate: a node whose data disk has less than this free takes no new sandboxes — a full data disk stops every sandbox on the node at once. | + +### Fleet settings — first-boot seeds -## Per-sandbox resources +The variables in the next three tables are **first-boot seeds**: on the +gateway's first start they are written into its settings table, and +from then on the table is what's in force — edit it live from the +console's settings and domains pages (or the `updateSettings` verb), no +restart needed. Every change counts the fleet configuration's version +up, and each daemon takes the new bundle at its next check-in. Once +seeded, changing these variables in the environment has no effect. + +#### Per-sandbox resources These apply when a container (or disk) is created, so existing sandboxes keep what they were born with. A [rebuild](/docs/persistence#rebuild-a-sandbox) picks up current CPU/memory values — but keeps the disk, so a changed disk size only applies to new sandboxes and restores from archive. The pids cap is the exception: it needs no rebuild — a running -sandbox is brought to a changed value in place the moment it is saved, -and a frozen or stopped one adopts it the next time it wakes. - -The disk, CPU, memory and pids variables are **first-boot seeds**: on the -daemon's first start they are written into its database as runtime -settings, and from then on the database value is what's in force — edit -it live from the console's settings page (or the `updateSettings` verb), -no restart needed. Once seeded, changing these variables in the -environment has no effect. +sandbox is brought to a changed value in place when its node takes the +bundle, and a frozen or stopped one adopts it the next time it wakes. | Variable | Default | What it does | | --- | --- | --- | @@ -64,28 +77,21 @@ environment has no effect. | `DORMICE_SANDBOX_MEMORY_GB` | `2` | Memory cap per sandbox. A sandbox that hits it is killed by gVisor (exit 137) and parks at stopped — files intact, next acquire revives it. | | `DORMICE_SANDBOX_PIDS_LIMIT` | `4096` | Cap on the sandbox's host-side process and thread footprint (the pids cgroup). Under gVisor this is not a count of processes inside the sandbox — the sandbox never sees it, and hitting it kills the whole sandbox at once (exit 2, no OOM). Keeps a fork bomb from taking anything down but its own sandbox; a browser plus several agent sessions in one sandbox needs well over the old default of 512. Floor 256; never unlimited. | -## Lifecycle engine +#### Sandbox network | Variable | Default | What it does | | --- | --- | --- | -| `DORMICE_SCAN_INTERVAL_SECONDS` | `60` | How often the idle scanner runs. Each sweep moves an idle sandbox down at most one state — see [Sandbox lifecycle](/docs/lifecycle). | -| `DORMICE_RECLAIM_TIMEOUT_SECONDS` | `45` | Upper bound on the memory-reclaim step of a freeze. Hitting it is expected on stubborn workloads, not a failure. | - -## Sandbox network - -| Variable | Default | What it does | -| --- | --- | --- | -| `DORMICE_SANDBOX_DOMAIN` | — | First-boot seed for the **canonical** sandbox wildcard domain behind the E2B SDK's `getHost(port)` — see [Exposing ports](/docs/ports). The value in force lives in the daemon's database and is edited live on the console's **Domains** page; once seeded, editing this variable has no effect. Alias domains (inbound-only) have no environment variable at all — they are managed on the same console card. A bare hostname like `sbx.example.com`; you point `*.` DNS plus a TLS-terminating reverse proxy at the daemon. | +| `DORMICE_SANDBOX_DOMAIN` | — | Seed for the **canonical** sandbox wildcard domain behind the E2B SDK's `getHost(port)` — see [Exposing ports](/docs/ports). The value in force is edited live on the console's **Domains** page. Alias domains (inbound-only) have no environment variable at all — they are managed on the same console card. A bare hostname like `sbx.example.com`; you point `*.` DNS plus a TLS-terminating reverse proxy at the daemon. | +| `DORMICE_INGRESS_FILE` | — | The Caddy config file the gateway owns — the switch for binding domains (and getting HTTPS) from the console's Domains page. `install.sh` sets it when it installs Caddy. Unset, the gateway touches no proxy config and `setIngress` is refused. Absolute path. | +| `DORMICE_INGRESS_RELOAD_CMD` | `caddy reload --config ` | How the gateway tells the running proxy to re-read its config after a bind. | -## Archiver (S3) +#### Archiver (S3) -These are **first-boot seeds** too: the store in force lives in the -daemon's database and is configured live from the console's settings -page (the archive store card) — including on a daemon that booted with -none of them set. The four core variables come as a set; a partial set -refuses to boot, naming the missing variables. With a store configured, -stopped sandboxes [archive](/docs/archiving) after 7 more idle days by -default. +The store in force is configured live from the console's settings page +(the archive store card) — including on a fleet that booted with none of +these set. The four core variables come as a set; a partial set refuses +to boot, naming the missing variables. With a store configured, stopped +sandboxes [archive](/docs/archiving) after 7 more idle days by default. | Variable | Default | What it does | | --- | --- | --- | @@ -96,11 +102,40 @@ default. | `DORMICE_S3_REGION` | `us-east-1` | Region, if your store cares. | | `DORMICE_S3_FORCE_PATH_STYLE` | `false` | Path-style addressing: MinIO needs `true`; the clouds route by subdomain. | +## Daemon (node) + +`/etc/dormice/env` — the node's identity and its machine. Nothing here +is a fleet setting. + +| Variable | Default | What it does | +| --- | --- | --- | +| `DORMICE_PORT` | `3676` | The daemon's port: the sandbox verbs, the E2B surface, the sandbox port proxy. Always bound to 127.0.0.1. | +| `DORMICE_DB_PATH` | `data/dormice.db` | Where the node's SQLite database lives (sandboxes, metrics history, its copy of the fleet configuration). Must be an **absolute path** in docker mode — a relative path depends on the start directory, and a wrong start directory means an empty database facing real sandboxes. | +| `DORMICE_NODE_ID` | `node-1` | This node's name — in its sandboxes' rows and to its gateway, which tells nodes apart by it. The default serves a node beside its gateway; a node whose gateway is on another machine must state its own. | +| `DORMICE_GATEWAY_ENDPOINT` | `http://127.0.0.1:3677` | The gateway this daemon is a node of. It checks in there every `DORMICE_CHECK_IN_INTERVAL_SECONDS` — readings, build, where it can be reached, which configuration version it runs — and takes the fleet's configuration from the answer. A daemon with no copy yet waits for its gateway before it listens. | +| `DORMICE_NODE_ENDPOINT` | `http://127.0.0.1:` | Where the gateway reaches this node. Required (an origin: scheme, host, port — no path) when the gateway is on another machine. | +| `DORMICE_CHECK_IN_INTERVAL_SECONDS` | `15` | How often the node checks in; the gateway reads two missed check-ins as down. | + +### Executor + +| Variable | Default | What it does | +| --- | --- | --- | +| `DORMICE_EXECUTOR` | `fake` | What runs the sandboxes: `fake` is an in-memory executor so a bare checkout works on any machine (development, tests); `docker` runs real gVisor sandboxes and needs a prepared Linux host — see [Installation](/docs/installation). | +| `DORMICE_BASE_IMAGE` | — | The image sandboxes boot from. **Required in docker mode**; `install.sh` builds one and records it here. | +| `DORMICE_DATA_DIR` | `/var/lib/dormice` | Sandbox disk images and their mount points (docker mode). Must be an absolute path in docker mode. | + +### Lifecycle engine + +| Variable | Default | What it does | +| --- | --- | --- | +| `DORMICE_SCAN_INTERVAL_SECONDS` | `60` | How often the idle scanner runs. Each sweep moves an idle sandbox down at most one state — see [Sandbox lifecycle](/docs/lifecycle). | +| `DORMICE_RECLAIM_TIMEOUT_SECONDS` | `45` | Upper bound on the memory-reclaim step of a freeze. Hitting it is expected on stubborn workloads, not a failure. | + ## Client-side variables -The `dor` CLI (and nothing else) reads two variables to find the daemon: +The `dor` CLI (and nothing else) reads two variables to find Dormice: | Variable | What it does | | --- | --- | -| `DORMICE_ENDPOINT` | The daemon's base URL, e.g. `http://127.0.0.1:3676`. | -| `DORMICE_API_TOKEN` | The same token the daemon was configured with. | +| `DORMICE_ENDPOINT` | A base URL: the gateway (`http://127.0.0.1:3677`) for the template and API-key commands and everything the gateway forwards; the daemon (`http://127.0.0.1:3676`) for `sandbox ls` and the other list/observation commands the gateway does not answer yet. | +| `DORMICE_API_TOKEN` | The fleet's token, or a key minted at the gateway (keys open the gateway, not the daemon). | diff --git a/website/content/docs/console.mdx b/website/content/docs/console.mdx index 9d8a9d29..cc7303bb 100644 --- a/website/content/docs/console.mdx +++ b/website/content/docs/console.mdx @@ -1,16 +1,23 @@ --- title: Web console -description: The daemon serves its own web UI at /console — watch every sandbox live, open terminals, browse files, and manage templates. +description: The gateway serves the web UI at /console — watch every sandbox live, open terminals, browse files, and manage templates. --- -The daemon hosts a web console at `http://127.0.0.1:3676/console` — same -process, same port, nothing extra to run. Everything it shows comes from -the same API documented in this section, and it follows the same rule: -**watching never wakes a sandbox; using one does, and only behind an -explicit click.** +The gateway hosts a web console at `http://127.0.0.1:3677/console` — +same process, same port, nothing extra to run. Everything it shows comes +from the same API documented in this section, and it follows the same +rule: **watching never wakes a sandbox; using one does, and only behind +an explicit click.** (The console UI is currently Chinese-language.) +**Work in progress:** the console moved to the gateway with the fleet's +configuration. The pages that read the fleet-wide list and observation +verbs — the overview and the sandbox list — stay dark until the gateway +answers those verbs itself, the next step of the move; the settings, +domains, templates, API keys and version pages, and the per-sandbox +workbench, work. + ## Signing in The console has one account with a username and password. On first diff --git a/website/content/docs/doctor.mdx b/website/content/docs/doctor.mdx index ae00c053..4c95d68b 100644 --- a/website/content/docs/doctor.mdx +++ b/website/content/docs/doctor.mdx @@ -91,27 +91,29 @@ know why; the Linux terms in it are covered in - **DORMICE_API_TOKEN** — set and at least 32 characters (the daemon refuses to start without it). -- **S3 archive configuration** — the `DORMICE_S3_*` variables are - first-boot seeds (the store in force lives in the daemon's database, - edited from the console), so three honest states: none set (skipped — - no seed; archiving can still be switched on from the console at any - time), all four set (pass — a seed; once booted, the console - settings rule), or a partial set (fail, naming the missing variables - — the daemon refuses a half-configured seed anyway). +- **S3 archive configuration** — the `DORMICE_S3_*` variables are the + gateway's first-boot seeds (the store in force lives in the gateway's + settings, edited from the console; a node never reads them), so three + honest states: none set (skipped — no seed; archiving can still be + switched on from the console at any time), all four set (pass — a + seed; once the gateway has started, the console settings rule), or a + partial set (fail, naming the missing variables — the gateway refuses + a half-configured seed anyway). `install.sh` runs doctor with both env + files loaded. - **zstd available** — the archiver shells out to `tar -I zstd` on the host at every archive and restore. Checked on every docker host, not just seeded ones: archiving can be switched on from the console at any moment, so a missing binary is a warn without a seed and a fail with one. - **ingress (Caddy)** — skipped when `DORMICE_INGRESS_FILE` is not - set: the daemon manages no reverse proxy, and you bind domains by - editing your proxy config directly. Set, the `caddy` binary must be - installed and the caddy service active — a missing binary means - `setIngress` cannot reload anything, and an inactive service means - nothing proxies the outside world to the daemon. A config file that - doesn't exist yet is a *warn* (the first bind creates it), and so is - one the daemon did not write: `setIngress` refuses to overwrite it, - so web domain binding is effectively off. + set (it is the gateway's variable): no managed reverse proxy, and you + bind domains by editing your proxy config directly. Set, the `caddy` + binary must be installed and the caddy service active — a missing + binary means `setIngress` cannot reload anything, and an inactive + service means nothing proxies the outside world to the gateway. A + config file that doesn't exist yet is a *warn* (the first bind + creates it), and so is one the gateway did not write: `setIngress` + refuses to overwrite it, so web domain binding is effectively off. - **base image available** — present locally; the fix names `images/Dockerfile`, and doctor never pulls. - **docker-mode paths absolute** — a relative `DORMICE_DB_PATH` depends diff --git a/website/content/docs/http-api.mdx b/website/content/docs/http-api.mdx index e2a6c413..328c4fab 100644 --- a/website/content/docs/http-api.mdx +++ b/website/content/docs/http-api.mdx @@ -8,56 +8,69 @@ output entirely in the JSON body. The route name is identical to the SDK method name, so there is nothing the SDK can do that curl cannot: ```sh -curl -X POST http://127.0.0.1:3676/acquireSandbox \ +curl -X POST http://127.0.0.1:3677/acquireSandbox \ -H "Authorization: Bearer $DORMICE_API_TOKEN" \ -H "content-type: application/json" \ -d '{"name": "my-agent"}' ``` +Two doors answer this wire. The **gateway** (`127.0.0.1:3677`) is the +fleet's one door: it answers the configuration verbs itself — templates, +API keys, settings, domains, nodes — and forwards every per-sandbox verb +to the node that holds the sandbox (placing a new name on a node first). +The **daemon** (`127.0.0.1:3676`) is a node: it answers the per-sandbox +verbs and the list and observation verbs directly. Until the gateway +answers the fleet-wide list and observation verbs itself (the next step +of the move), point clients at the daemon for those; the gateway answers +them with an honest `501` naming the node meanwhile. The rows below say +which door. + Two rules cover the whole surface: - **Auth** is `Authorization: Bearer ` on every route except - `GET /healthz`, the open liveness probe (it answers - `{"status":"ok"}`). The token is `DORMICE_API_TOKEN` — or any active + `GET /healthz`, the open liveness probe. The token is + `DORMICE_API_TOKEN` — or, at the gateway, any active [API key](#api-keys), the revocable credentials minted through - `createApiKey` or the console. + `createApiKey` or the console. A node knows only the token. - **Every non-2xx body is `{ "message": "..." }`** — one error shape, - whoever produced the error. 4xx name your mistake; 5xx are the - daemon's. + whoever produced the error. 4xx name your mistake; 5xx are ours. The [E2B compatibility surface](/docs/e2b-sdks) is a separate wire under `/e2b/*` with its own authentication — nothing below applies to it. ## The verbs -| Route | Does | Errors worth knowing | -| --- | --- | --- | -| `POST /acquireSandbox` | create, wake, or restore — idempotent per name | 400 invalid policy/unknown template, 429 at the sandbox limit | -| `POST /updatePolicy` | patch an existing sandbox's [lifecycle policy](/docs/lifecycle#change-the-policy-later) in place; never wakes, never resets the idle clock | 404 unknown name, 400 invalid merged policy | -| `POST /updateMetadata` | replace an existing sandbox's label set wholesale (`{}` clears); never wakes, never resets the idle clock | 404 unknown name | -| `POST /listSandboxes` | every sandbox and its state; never wakes anything | — | -| `POST /execCommand` | run a command, buffered | 404 unknown name, 409 archived/restoring | -| `POST /writeFiles` | batch write, base64 in JSON | 404/409 as above; body over 48 MiB refused | -| `POST /writeFile` | write one file — the single form of `writeFiles` | same as `writeFiles` | -| `POST /readFile` | read one file, base64 out | 404 no such file, 400 not a file, 413 over 16 MiB (names the real size) | -| `POST /readFiles` | batch read, all or nothing, request order | first bad path fails the whole batch; 413 over 48 MiB total | -| `POST /rebuildSandbox` | swap container, keep `/home/user` | 404 unknown name, 409 archived/restoring | -| `POST /destroySandbox` | destroy; idempotent | 409 mid-restore — retry when it finishes | -| `POST /registerTemplate` | upsert name → image | 400 `'base'` is reserved | -| `POST /listTemplates` | every registered template | — | -| `POST /removeTemplate` | unregister; idempotent | 409 while sandboxes still use it (names them) | -| `POST /createApiKey` | mint an [API key](#api-keys), optionally with an `expiresAt`; the token appears in this response once, never again | 409 while a non-revoked key already has this name | -| `POST /listApiKeys` | every key ever minted, revoked ones included, newest first — no secrets | — | -| `POST /updateApiKey` | edit a key by `id`: rename, change/clear `expiresAt`, park/resume via `disabled` — absent fields untouched | 404 unknown id; 409 revoked row or name collision | -| `POST /revokeApiKey` | soft-revoke a key by `id`; idempotent (`revoked: false` when none was) | — | -| `POST /getHostMetrics` | host snapshot; never wakes anything | — | -| `POST /getSandboxMetrics` | one sandbox's live CPU/memory/disk sample; `sample` is `null` when nothing is running | 404 unknown name | -| `POST /listSandboxMetrics` | every measurable sandbox's sample in one answer | — | -| `POST /listSandboxImages` | each sandbox's born image vs its template's current one | — | -| `POST /getConfig` | effective configuration: env knobs (read-only; secrets reported present-or-absent, value never sent) plus the live runtime `settings` | — | -| `POST /updateSettings` | rewrite the runtime settings (new-sandbox defaults, default policy, the S3 archive store, the sandbox domain, the per-sandbox pids cap) — immediate effect, no restart; each provided group replaces that group whole. The `s3` group takes all six fields every time (keys included — they are never read back) and is probed with a real write-read-delete round trip before anything is saved; `s3: null` turns archiving off. `sandboxDomain` takes a bare hostname or `null`; `sandboxDomainAliases` takes the full list of inbound-only alias hostnames (`[]` clears) — swapping the canonical domain is one call carrying both fields, see [Exposing ports](/docs/ports). `pidsLimit` (floor 256, never unlimited) reaches new containers at birth, running sandboxes in place before the call returns, and frozen or stopped ones at their next wake — no rebuild | 400 empty patch, a `pidsLimit` below 256, a default policy that asks to archive with no store in force, a failed probe (S3 answered 4xx), clearing/moving the store while sandboxes are archived in it (named by count), or an alias list that contradicts the post-patch state (duplicates, the canonical domain listed as an alias, or aliases left standing with no canonical domain); 500 when a value was saved but the host would not follow — a running sandbox refused the new `pidsLimit` (named; it follows at its next wake); 502 when the probed store is unreachable | -| `POST /getIngress` | domains bound on the daemon's managed reverse proxy, with live DNS and certificate probes | — | -| `POST /setIngress` | rewrite the managed proxy config to exactly this domain list (empty list unbinds all) | 400 when the daemon manages no proxy | +| Route | Door | Does | Errors worth knowing | +| --- | --- | --- | --- | +| `POST /acquireSandbox` | both | create, wake, or restore — idempotent per name; at the gateway a new name is placed on a node first | 400 invalid policy/unknown template; 503 with `Retry-After` when no node can take a new sandbox (each node and its reason named) | +| `POST /updatePolicy` | both | patch an existing sandbox's [lifecycle policy](/docs/lifecycle#change-the-policy-later) in place; never wakes, never resets the idle clock | 404 unknown name, 400 invalid merged policy | +| `POST /updateMetadata` | both | replace an existing sandbox's label set wholesale (`{}` clears); never wakes, never resets the idle clock | 404 unknown name | +| `POST /listSandboxes` | daemon | every sandbox and its state; never wakes anything | — | +| `POST /execCommand` | both | run a command, buffered | 404 unknown name, 409 archived/restoring | +| `POST /writeFiles` | both | batch write, base64 in JSON | 404/409 as above; body over 48 MiB refused | +| `POST /writeFile` | both | write one file — the single form of `writeFiles` | same as `writeFiles` | +| `POST /readFile` | both | read one file, base64 out | 404 no such file, 400 not a file, 413 over 16 MiB (names the real size) | +| `POST /readFiles` | both | batch read, all or nothing, request order | first bad path fails the whole batch; 413 over 48 MiB total | +| `POST /rebuildSandbox` | both | swap container, keep `/home/user` | 404 unknown name, 409 archived/restoring | +| `POST /destroySandbox` | both | destroy; idempotent | 409 mid-restore — retry when it finishes | +| `POST /registerTemplate` | gateway | upsert name → image; every node learns it at its next check-in | 400 `'base'` is reserved | +| `POST /listTemplates` | gateway | every registered template | — | +| `POST /removeTemplate` | gateway | unregister; idempotent. The gateway asks every node first | 409 while sandboxes on any node still use it (named, by node); 503 with `Retry-After` while a node has not answered | +| `POST /createApiKey` | gateway | mint an [API key](#api-keys), optionally with an `expiresAt`; the token appears in this response once, never again | 409 while a non-revoked key already has this name | +| `POST /listApiKeys` | gateway | every key ever minted, revoked ones included, newest first — no secrets | — | +| `POST /updateApiKey` | gateway | edit a key by `id`: rename, change/clear `expiresAt`, park/resume via `disabled` — absent fields untouched | 404 unknown id; 409 revoked row or name collision | +| `POST /revokeApiKey` | gateway | soft-revoke a key by `id`; idempotent (`revoked: false` when none was) | — | +| `POST /getHostMetrics` | daemon | host snapshot; never wakes anything | — | +| `POST /getSandboxMetrics` | both | one sandbox's live CPU/memory/disk sample; `sample` is `null` when nothing is running | 404 unknown name | +| `POST /listSandboxMetrics` | daemon | every measurable sandbox's sample in one answer | — | +| `POST /listSandboxImages` | daemon | each sandbox's born image vs its template's current one | — | +| `POST /getConfig` | gateway | effective configuration: the gateway's env knobs (read-only; secrets reported present-or-absent, value never sent), the live fleet `settings`, and `configVersion` — the number every node reports back once it runs this configuration | — | +| `POST /updateSettings` | gateway | rewrite the fleet settings (new-sandbox defaults, default policy, the S3 archive store, the sandbox domain, the per-sandbox pids cap) — no restart; the version counts up and every node applies the bundle at its next check-in (within 15 seconds); each provided group replaces that group whole. The `s3` group takes all six fields every time (keys included — they are never read back) and is probed with a real write-read-delete round trip before anything is saved; `s3: null` turns archiving off. `sandboxDomain` takes a bare hostname or `null`; `sandboxDomainAliases` takes the full list of inbound-only alias hostnames (`[]` clears) — swapping the canonical domain is one call carrying both fields, see [Exposing ports](/docs/ports). `pidsLimit` (floor 256, never unlimited) reaches new containers at birth, running sandboxes in place as their node takes the bundle, and frozen or stopped ones at their next wake — no rebuild | 400 empty patch, a `pidsLimit` below 256, a default policy that asks to archive with no store in force, a failed probe (S3 answered 4xx), clearing/moving the store while sandboxes are archived in it (counted across the fleet from the nodes' check-ins), or an alias list that contradicts the post-patch state (duplicates, the canonical domain listed as an alias, or aliases left standing with no canonical domain); 502 when the probed store is unreachable; 503 with `Retry-After` when a node has not checked in since the gateway started, so the archived count is unknown | +| `POST /getIngress` | gateway | domains bound on the gateway's managed reverse proxy, with live DNS and certificate probes | — | +| `POST /setIngress` | gateway | rewrite the managed proxy config to exactly this domain list (empty list unbinds all) | 400 when the gateway manages no proxy | +| `POST /listNodes` | gateway | every node that ever checked in: endpoint, reachable, last check-in, build, reading, the configuration version it runs, its swap target | — | +| `POST /updateNodeSettings` | gateway | set one node's managed-swap target (GiB); the node applies it at its next check-in | 404 unknown node; 400 when its daemon cannot manage swap; 503 before it has checked in | +| `POST /removeNode` | gateway | forget a node that is gone for good | 409 while it still checks in | **Only `acquireSandbox` creates.** The use verbs (`execCommand`, files, `rebuildSandbox`) answer 404 for an unknown name rather than creating a @@ -120,28 +133,34 @@ signal that the policy/template/metadata in the request were ignored. ## API keys -`DORMICE_API_TOKEN` works forever, but rotating it means editing -`/etc/dormice/env` and restarting the daemon. API keys are its -full-power, revocable peers: mint one per client with +`DORMICE_API_TOKEN` works forever, but rotating it means editing both +env files and restarting the gateway and the daemon. API keys are its +full-power, revocable peers at the gateway: mint one per client with `POST /createApiKey {"name": "ci"}` (optionally with `"expiresAt": ""` for a TTL key), hand out the returned 64-hex `token`, and when a key leaks, revoke it — the credential dies on its next request, no config edit, no restart, no other client disturbed. -The token appears in the create response **once**. The daemon stores +The token appears in the create response **once**. The gateway stores only its hash, so nothing can ever show it again — copy it then. Keys -open every door the env token does, the +open every door the fleet token does at the gateway, the [E2B surface](/docs/e2b-sdks) included (`e2b_` works as an E2B API key), with exactly two exceptions: -- **Console account setup/reset** accepts only the env token — a leaked - machine credential must not be able to take over the human account. -- **The four apiKey verbs and `updateSettings`** accept only the env - token or a console session. A key that could manage keys is a - self-replication ladder — one leaked credential minting itself an - unrevoked successor — and a leaked key must not be able to raise the - very limits that contain it, so a valid key gets an honest 403 on - these routes. +- **Console account setup/reset** accepts only the fleet token — a + leaked machine credential must not be able to take over the human + account. +- **Everything that configures the fleet** — the four apiKey verbs, + `updateSettings`, the template verbs, `getIngress`/`setIngress`, and + the node verbs — accepts only the fleet token or a console session. A + key that could manage keys is a self-replication ladder — one leaked + credential minting itself an unrevoked successor — and a leaked key + must not be able to raise the very limits that contain it, so a valid + key gets an honest 403 on these routes. + +Keys are the gateway's: a daemon judges one credential, the fleet +token, and answers a minted key with 401. Toward its nodes the gateway +always speaks the fleet token. Each key has two handles: `name`, the human label (renameable through `updateApiKey`), and `id`, the stable address `updateApiKey` and @@ -155,8 +174,8 @@ granularity), and revoked rows stay in `listApiKeys` forever. Suspect a leak: disable the key to stop the bleeding, then revoke and rotate. Dormice keeps no per-key audit trail — lifecycle moves land in -the daemon's own log (journald on a systemd host), attributed to the -request, not to the credential. +the gateway's and the daemon's own logs (journald on a systemd host), +attributed to the request, not to the credential. ## Commands and files on the wire diff --git a/website/content/docs/installation.mdx b/website/content/docs/installation.mdx index 07ec6ee3..6a10daed 100644 --- a/website/content/docs/installation.mdx +++ b/website/content/docs/installation.mdx @@ -1,6 +1,6 @@ --- title: Installation -description: Install the Dormice daemon on a Linux host with one command. +description: Install Dormice on a Linux host with one command. --- Installing Dormice is one command on a fresh Linux server. The command @@ -8,7 +8,7 @@ downloads an installer script that sets up everything the platform needs; you do not need to know Docker or Linux administration — this page tells you what to type, and explains what happened afterwards. -## Install the daemon +## Install Dormice You need a Linux server ([the host](/docs/core-concepts#the-host)) running Ubuntu or Debian on a regular Intel/AMD CPU (`x86_64`), ideally @@ -28,8 +28,12 @@ The installer sets up everything — runtime), [gVisor](/docs/core-concepts#gvisor) (the extra isolation layer around each sandbox), [swap](/docs/core-concepts#swap-and-freezing) (the disk-as-overflow-memory -that makes freezing work), the network hardening, and the daemon itself -as a systemd service (so it starts on boot and restarts after a crash). +that makes freezing work), the network hardening, and Dormice's two +processes as systemd services (so they start on boot and restart after +a crash): the **gateway** — the front door that serves the web console +and holds the fleet's settings and API keys — and the **daemon** — the +node that runs the sandboxes. On one machine they are a fleet of one; +more machines can join the same gateway later. It finishes by running `dor doctor`, a battery of read-only checks that verifies the install actually works. You can re-run `dor doctor` at any time. @@ -41,9 +45,10 @@ and repairs anything that drifted, and never rotates your API token. The installer generates an [API token](/docs/core-concepts#tokens-sdks-and-the-cli) — a long random -string that acts as the password for programs — and stores it in the -daemon's config file. You need it for every client — SDK, CLI, and -console. Print it with: +string that acts as the password for programs — and stores it in both +config files (`/etc/dormice/env` and `/etc/dormice/gateway.env`, the +same value). You need it for every client — SDK, CLI, and console. +Print it with: ```sh grep ^DORMICE_API_TOKEN /etc/dormice/env @@ -51,8 +56,9 @@ grep ^DORMICE_API_TOKEN /etc/dormice/env ## Connect from another machine -The daemon listens on `127.0.0.1` only — reaching it from another -machine is a choice you make explicitly, one of two ways. +Both processes listen on `127.0.0.1` only — the gateway on `3677`, the +daemon on `3676` — so reaching them from another machine is a choice you +make explicitly, one of two ways. **Option 1: SSH tunnel** (private, zero setup). SSH — the tool you already use to log in to the server — can also carry other traffic @@ -60,17 +66,19 @@ through its encrypted connection, making a port on the server appear on your own machine: ```sh -ssh -L 3676:127.0.0.1:3676 root@your-host +ssh -L 3677:127.0.0.1:3677 -L 3676:127.0.0.1:3676 root@your-host ``` -Keep that command running, and use `http://127.0.0.1:3676` as your -endpoint, as if the daemon were local. +Keep that command running, and use `http://127.0.0.1:3677` (the +gateway: console, keys, settings, and every sandbox verb) and +`http://127.0.0.1:3676` (the daemon, for the list and observation verbs +the gateway does not answer yet) as if they were local. **Option 2: Bind a domain** — serves the API, the E2B surface, and the web console at once, over HTTPS. The installer already placed a reverse proxy ([Caddy](https://caddyserver.com)) at the server's public door — a small program that receives internet requests and forwards them to -the daemon ([what's a reverse +the gateway ([what's a reverse proxy?](/docs/core-concepts#reverse-proxies-domains-and-https)). To give it a domain: @@ -78,14 +86,14 @@ give it a domain: `dormice.example.com`) at the server's public IP — in the dashboard of whoever sold you the domain. 2. Bind the domain on the web console's domains page (or with the - `setIngress` API verb). The daemon rewrites its managed Caddy + `setIngress` API verb). The gateway rewrites its managed Caddy config, and Caddy obtains and renews the HTTPS certificate on its own. The domains page shows each domain's real progress — DNS resolution and certificate status — until it turns green. **Note:** Do not hand-edit the managed Caddy config — the next bind -rewrites it. If you run your own proxy instead, forward to -`127.0.0.1:3676` with buffering disabled (`flush_interval -1` in +rewrites it. If you run your own proxy instead, forward to the gateway, +`127.0.0.1:3677`, with buffering disabled (`flush_interval -1` in Caddy): streamed command output is written frame by frame, and a buffering proxy would hold it all back until the end. Use HTTPS for anything exposed beyond localhost — the API token and the session diff --git a/website/content/docs/ports.mdx b/website/content/docs/ports.mdx index 768ec2ee..7ab68286 100644 --- a/website/content/docs/ports.mdx +++ b/website/content/docs/ports.mdx @@ -24,8 +24,8 @@ separate from the domain you may have bound for the API and console in Pick a domain for sandbox traffic and set it on the console's **Domains** page (the sandbox domain card) — it applies live, no restart. The card shows the exact wildcard DNS record to copy. The -`DORMICE_SANDBOX_DOMAIN` environment variable still works as a -first-boot seed: +`DORMICE_SANDBOX_DOMAIN` environment variable in the gateway's env still +works as a first-boot seed: ```sh DORMICE_SANDBOX_DOMAIN=sbx.example.com @@ -55,7 +55,9 @@ path and operation — never the domain. ## 2. Point DNS and a reverse proxy at the host The daemon itself only binds `127.0.0.1`. The internet-facing half is -one wildcard DNS record plus a reverse proxy that terminates TLS: +one wildcard DNS record plus a reverse proxy that terminates TLS — pointed +at the daemon, which serves the sandbox port proxy (the gateway takes +this face over in a later step): ```text *.sbx.example.com → your host diff --git a/website/content/docs/troubleshooting.mdx b/website/content/docs/troubleshooting.mdx index 72b9572f..4750eba4 100644 --- a/website/content/docs/troubleshooting.mdx +++ b/website/content/docs/troubleshooting.mdx @@ -118,13 +118,12 @@ shows one line: `fork rejected by pids controller`. The sandbox object's the sandbox — immediately, even inside the heartbeat's blind spot (see the entry above). The default is 4096 (it was 512, runc's fork-bomb number — a browser plus a few agent sessions in one sandbox is -enough to pass it). A daemon upgraded from the old default adopts 4096 at -that boot and brings its running sandboxes to it right there; only an -environment that pins `DORMICE_SANDBOX_PIDS_LIMIT` at the old number -keeps it, and the console's settings page (or `updateSettings`'s -`pidsLimit`) is where to raise it — no restart, no rebuild: running -sandboxes are brought to the new value in place as it is saved, frozen or -stopped ones at their next wake. +enough to pass it). The cap is a fleet setting at the gateway +(`DORMICE_SANDBOX_PIDS_LIMIT` in the gateway's env seeds it once), and +the console's settings page (or `updateSettings`'s `pidsLimit`) is where +to raise it — no restart, no rebuild: each node takes the new value at +its next check-in and brings its running sandboxes to it in place, +frozen or stopped ones at their next wake. ## E2B `watchDir` refuses to start From a2c6ddcb3c5957c0446286a17877407ebbaaffdc Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 18:47:58 +0800 Subject: [PATCH 38/89] A node waiting for its first bundle beats the watchdog and says it is not listening; the exam tolerates what other suites change; the CLI is sent to the door MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the configuration move (cut 2), after reading the whole chain, a green local run and eight hand-driven scenarios outside the suite. The CI failure: native's updateSettings exam asserted the shared gateway's version was exactly one more than before, and settings-hot writes to the same gateway in parallel (22 where +1 said 21). It asserts `>` and `>=` now, the discipline the host-metrics exam already kept. Running the suite twice more found two siblings: settings-hot's unreachable-store probe met the moving-store guard's 400 while archive.test.ts had a sandbox archived on the shared node (the guard is judged before the probe, from the node's reading), and e2b's getHost met a domain settings-hot had swapped for a second or two — the swap travels by check-in since the move, so that window grew from milliseconds. Both poll until the shared state is theirs. The heartbeat watchdog starts before boot's awaits, and a node waiting for its first bundle beat nothing: half an hour into waiting for its gateway it was read as stalled and exited, every thirty minutes, for nothing. Each attempt of the wait beats now; the ticker itself never does, for the reason the metrics ticker does not. The check-in's failure sentence said the gateway "places nothing here and forwards no new names" — for a node without a copy the predicament is that it is not listening at all, and it says so. The CLI reference, the quick start, install.sh's closing hint and dor's missing-variable message pointed DORMICE_ENDPOINT at the daemon, where `dor template` and `dor apikey` answer 404 since the move; they point at the gateway, naming `sandbox ls` and `sandbox meta` as the two commands that still go to the daemon until the gateway routes the list verbs. install.sh's closing lines lose a console line that duplicated the conditional one below it and was wrong when Caddy had been skipped. The gateway's settings module carried two `Omit<…, 'swapGb'>` over types that no longer have the field. --- deploy/install.sh | 9 ++--- e2e/src/e2b.test.ts | 26 +++++++++----- e2e/src/native.test.ts | 10 ++++-- e2e/src/settings-hot.test.ts | 42 ++++++++++++++++------- packages/cli/src/commands.ts | 4 +-- packages/gateway/src/db/settings.ts | 15 +++----- packages/server/src/check-in.test.ts | 51 +++++++++++++++++++++++++--- packages/server/src/check-in.ts | 25 +++++++++++++- packages/server/src/main.ts | 4 +++ website/content/docs/cli.mdx | 19 +++++++---- website/content/docs/quickstart.mdx | 29 +++++++++------- 11 files changed, 169 insertions(+), 65 deletions(-) diff --git a/deploy/install.sh b/deploy/install.sh index 62de2bae..00d2c2c9 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -857,10 +857,11 @@ printf '\nDormice is installed.\n' printf ' API token: grep ^DORMICE_API_TOKEN %s\n' "$ENV_FILE" printf ' gateway logs: journalctl -u dormice-gateway -f (the door: console, keys, settings, templates)\n' printf ' daemon logs: journalctl -u dormice -f (the node: sandboxes)\n' -printf ' console: http:///console (Caddy on :80 -> the gateway on 127.0.0.1:%s)\n' "$GATEWAY_PORT" -printf ' CLI: export DORMICE_ENDPOINT=http://127.0.0.1:%s DORMICE_API_TOKEN=; dor sandbox ls\n' "$PORT" +printf ' CLI: export DORMICE_ENDPOINT=http://127.0.0.1:%s DORMICE_API_TOKEN=; dor template ls\n' "$GATEWAY_PORT" +printf ' (the gateway is the door; `dor sandbox ls` and `dor sandbox meta` list, which the gateway\n' +printf ' does not route yet — point DORMICE_ENDPOINT at the daemon, 127.0.0.1:%s, for those)\n' "$PORT" printf ' Both processes listen on 127.0.0.1 only, by design — exposing them is a reverse proxy'"'"'s job.\n' if [ "$(systemctl is-active caddy 2>/dev/null)" = active ]; then - printf ' console: http:///console (Caddy on :80 — open your cloud firewall for 80/443,\n' - printf ' then bind domains in the domains page for automatic HTTPS)\n' + printf ' console: http:///console (Caddy on :80 -> the gateway; open your cloud firewall for\n' + printf ' 80/443, then bind domains in the domains page for automatic HTTPS)\n' fi diff --git a/e2e/src/e2b.test.ts b/e2e/src/e2b.test.ts index 4fdc84fe..f69a5489 100644 --- a/e2e/src/e2b.test.ts +++ b/e2e/src/e2b.test.ts @@ -2,7 +2,7 @@ import http from 'node:http'; import { Dormice } from '@dormice/sdk'; import { CommandExitError, Sandbox } from 'e2b'; import { describe, expect, inject, it } from 'vitest'; -import { door, settled } from './helpers'; +import { door, until as poll, settled } from './helpers'; // The compatibility promise, verified with the promise's own artifact: the // OFFICIAL e2b package, pointed at the daemon by exactly two URLs (plus its @@ -523,14 +523,22 @@ describe('official e2b SDK against the daemon', () => { }); it('getHost builds the wildcard host from the served domain', async () => { - const sbx = await Sandbox.create(connection()); - try { - // Pure client-side string assembly — but from OUR domain field, which - // is the whole point: the daemon told the SDK where sandboxes live. - expect(sbx.getHost(8000)).toBe(`8000-${sbx.sandboxId}.sbx.dormice.test`); - } finally { - await sbx.kill(); - } + // Pure client-side string assembly — but from OUR domain field, which + // is the whole point: the daemon told the SDK where sandboxes live. + // Polled: settings-hot swaps the shared node's domain for a second or + // two at a time (the edit travels by check-in since 2026-09-14), and a + // sandbox created inside that window is told the other domain. The + // seed is the steady state and always comes back. + await poll(async () => { + const sbx = await Sandbox.create(connection()); + try { + return sbx.getHost(8000) === `8000-${sbx.sandboxId}.sbx.dormice.test` + ? true + : undefined; + } finally { + await sbx.kill(); + } + }); }); it('watchDir streams filesystem events for changes made through the SDK', async () => { diff --git a/e2e/src/native.test.ts b/e2e/src/native.test.ts index b74de9c7..f44adbb7 100644 --- a/e2e/src/native.test.ts +++ b/e2e/src/native.test.ts @@ -817,8 +817,14 @@ describe('the observability verbs over a real daemon', () => { expect(settings.updatedAt).not.toBeNull(); const after = await viaDoor().getConfig(); expect(after.settings.pidsLimit).toBe(before.settings.pidsLimit + 1); - expect(after.configVersion).toBe(before.configVersion + 1); - expect(await settled()).toBe(after.configVersion); + // Greater, not exactly one more: the exam's gateway is shared by every + // suite in this run, and another suite's write (settings-hot's domain + // edits) can count the version up between two reads here — CI saw 22 + // where +1 said 21 (2026-09-14). The node's proof keeps its shape: + // settled() returns once node A reports the gateway's current version, + // which is at least the one this write produced. + expect(after.configVersion).toBeGreaterThan(before.configVersion); + expect(await settled()).toBeGreaterThanOrEqual(after.configVersion); // Restore: the exam's gateway is shared by every suite in this run. await viaDoor().updateSettings({ pidsLimit: before.settings.pidsLimit }); await settled(); diff --git a/e2e/src/settings-hot.test.ts b/e2e/src/settings-hot.test.ts index 527cc6b5..be040958 100644 --- a/e2e/src/settings-hot.test.ts +++ b/e2e/src/settings-hot.test.ts @@ -86,18 +86,36 @@ describe('the S3 archive store as a live ledger setting', () => { it('refuses an unreachable store with S3’s own words and saves nothing', async () => { const dormice = client(); const before = (await dormice.getConfig()).settings.s3; - await expect( - dormice.updateSettings({ - s3: { - endpoint: 'http://127.0.0.1:1', - bucket: 'nowhere', - region: 'us-east-1', - forcePathStyle: true, - accessKeyId: 'k', - secretAccessKey: 's', - }, - }), - ).rejects.toMatchObject({ + // Another endpoint is a move, and the moving-store guard is judged + // before the probe: while another suite (archive.test.ts) has a + // sandbox archived on the shared node, the answer is that guard's 400, + // not the probe's 502 — so the attempt repeats until the fleet's + // readings let the probe speak (seen in a local run, 2026-09-14). + const refused = await until(async () => { + try { + await dormice.updateSettings({ + s3: { + endpoint: 'http://127.0.0.1:1', + bucket: 'nowhere', + region: 'us-east-1', + forcePathStyle: true, + accessKeyId: 'k', + secretAccessKey: 's', + }, + }); + throw new Error('an unreachable store was accepted'); + } catch (error) { + const { status, message } = error as { + status?: number; + message: string; + }; + if (status === 400 && /archived or restoring/.test(message)) { + return undefined; + } + return { status, message }; + } + }, 12_000); + expect(refused).toMatchObject({ status: 502, message: expect.stringMatching(/nothing was saved/), }); diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index beeda562..dc2ebd93 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -26,8 +26,8 @@ export function clientFromEnv( .join(' and '); throw new Error( `${missing} must be set, e.g.\n` + - ' export DORMICE_ENDPOINT=http://127.0.0.1:3676\n' + - " export DORMICE_API_TOKEN=", + ' export DORMICE_ENDPOINT=http://127.0.0.1:3677\n' + + ' export DORMICE_API_TOKEN=', ); } return new Dormice({ endpoint, token }); diff --git a/packages/gateway/src/db/settings.ts b/packages/gateway/src/db/settings.ts index 62a088df..0a769e40 100644 --- a/packages/gateway/src/db/settings.ts +++ b/packages/gateway/src/db/settings.ts @@ -13,13 +13,6 @@ import { type SettingsRow, settings } from './schema'; /** The console_account fixed-id pattern: "at most one row" as a schema fact. */ const SETTINGS_ROW_ID = 1; -/** - * The fleet-wide settings as the wire shows them. The daemon's settings - * view once carried the managed-swap target too; on the gateway that knob - * is a node's (nodes.swapGb), so it is absent here. - */ -export type FleetSettings = Omit; - /** * Seeds the settings row from the env at the gateway's first start — * insert-or-nothing, so every later start finds the row and leaves it @@ -76,7 +69,7 @@ function readRow(db: Db): SettingsRow { return row; } -function toView(row: SettingsRow): FleetSettings { +function toView(row: SettingsRow): RuntimeSettings { return { sandboxDefaults: { cpus: row.sandboxCpus, @@ -109,7 +102,7 @@ function toView(row: SettingsRow): FleetSettings { } /** The knobs in force, read fresh at each use — a point read costs microseconds and makes a console edit apply to the very next request. */ -export function readSettings(db: Db): FleetSettings { +export function readSettings(db: Db): RuntimeSettings { return toView(readRow(db)); } @@ -167,9 +160,9 @@ export function bumpConfigVersion(db: Writer): number { */ export function writeSettings( db: Db, - patch: Omit, + patch: UpdateSettingsRequest, now: Date, -): FleetSettings { +): RuntimeSettings { const row = db .update(settings) .set({ diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index e28b7610..4bd932d7 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -311,7 +311,7 @@ describe('CheckIn', () => { ).toBeNull(); }); - it('untilConfigured() asks until a bundle lands, on the interval, and returns at once when a copy exists', async () => { + it('untilConfigured() asks until a bundle lands, on the interval, beating the watchdog per attempt, and returns at once when a copy exists', async () => { let sent = 0; const gw = await gateway(() => { sent += 1; @@ -321,7 +321,12 @@ describe('CheckIn', () => { : answering(2, testBundle({}, 2)); }); const { log } = logSpy(); - const opts = options(gw.endpoint, log); + let beats = 0; + const opts = options(gw.endpoint, log, { + beat: () => { + beats += 1; + }, + }); const checkIn = new CheckIn(opts); const started = Date.now(); await checkIn.untilConfigured(); @@ -329,15 +334,49 @@ describe('CheckIn', () => { expect(gw.seen).toHaveLength(3); // Two waits of one interval between the three asks. expect(Date.now() - started).toBeGreaterThanOrEqual(1_900); - // Holding a copy already: nothing is asked. + // Every attempt, the two refused ones included, beat the watchdog: a + // node waiting for its gateway is alive, not stalled. + expect(beats).toBe(3); + // Holding a copy already: nothing is asked, nothing beats. await checkIn.untilConfigured(); expect(gw.seen).toHaveLength(3); + expect(beats).toBe(3); }); - it('ticks on its interval from start() and stops on stop()', async () => { + it("the first failure's sentence names the cost for where the node stands: not listening without a copy, not placed on with one", async () => { + const gw = await gateway(() => ({ + status: 503, + body: '{"message":"starting"}', + })); + const bare = logSpy(); + await new CheckIn(options(gw.endpoint, bare.log)).once(); + expect(bare.warns).toEqual([ + expect.stringMatching( + /^check-in failed; this node holds no configuration copy and does not listen until the gateway answers with one/, + ), + ]); + const holding = logSpy(); + const opts = options(gw.endpoint, holding.log); + applyNodeConfig(opts.db, testBundle({}, 1)); + await new CheckIn(opts).once(); + expect(holding.warns).toEqual([ + expect.stringMatching( + /^check-in failed; the gateway places nothing here and forwards no new names to this node/, + ), + ]); + }); + + it('ticks on its interval from start() and stops on stop(); the ticker never beats the watchdog', async () => { const gw = await gateway(() => answering(1)); const { log } = logSpy(); - const checkIn = new CheckIn(options(gw.endpoint, log)); + let beats = 0; + const checkIn = new CheckIn( + options(gw.endpoint, log, { + beat: () => { + beats += 1; + }, + }), + ); checkIn.start(); const deadline = Date.now() + 5_000; while (gw.seen.length < 2 && Date.now() < deadline) { @@ -348,5 +387,7 @@ describe('CheckIn', () => { const afterStop = gw.seen.length; await new Promise((resolve) => setTimeout(resolve, 1_200)); expect(gw.seen.length).toBe(afterStop); + // A ticker's liveness must never reassure the watchdog (main.ts). + expect(beats).toBe(0); }); }); diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index e738dd2e..ddbbd702 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -52,6 +52,14 @@ export interface CheckInOptions { /** Makes a bundle the gateway answered with real on this node (node-config.ts applyConfig). */ applyConfig: (bundle: NodeConfigBundle) => Promise; log: CheckInLog; + /** + * The heartbeat watchdog's ear, for untilConfigured() alone: a node + * waiting for its first bundle has no lifecycle work to beat, and each + * attempt — answered or not, bounded by CHECK_IN_TIMEOUT_MS — is the + * wait provably alive. The ticker never beats: a ticker's liveness must + * not reassure the watchdog (main.ts has the 2026-08-13 lesson). + */ + beat?: () => void; /** Test seam; production uses the platform's fetch. */ fetchImpl?: typeof fetch; } @@ -176,10 +184,19 @@ export class CheckIn { const message = describe(error); const failure = message.replace(/\d+/g, '#'); if (failure !== this.failing) { + // What the failure costs depends on where this node stands: one + // holding a copy keeps serving and is merely not placed on; one + // without (untilConfigured, at boot) is not listening at all, and + // "the gateway forwards nothing here" would name the wrong + // predicament (found by review, 2026-09-14). + const cost = + opts.configVersion() === null + ? 'this node holds no configuration copy and does not listen until the gateway answers with one' + : 'the gateway places nothing here and forwards no new names to this node until it answers again'; opts.log.warn( { gateway: opts.gateway, error: message }, this.failing === null - ? 'check-in failed; the gateway places nothing here and forwards no new names to this node until it answers again — retrying every interval' + ? `check-in failed; ${cost} — retrying every interval` : 'check-in still failing, differently — retrying every interval', ); } @@ -200,6 +217,12 @@ export class CheckIn { async untilConfigured(): Promise { while (!this.closing && this.opts.configVersion() === null) { await this.once(); + // Each attempt is the wait provably alive (CheckInOptions.beat): the + // watchdog starts before boot's awaits, and without a beat it read a + // node half an hour into waiting for its gateway as a stalled daemon + // and exited it — every thirty minutes, for nothing (found by + // review, 2026-09-14). + this.opts.beat?.(); if (this.opts.configVersion() !== null) return; await new Promise((resolve) => setTimeout(resolve, this.opts.intervalSeconds * 1000), diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index cbbd4f52..15bf6f01 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -213,6 +213,10 @@ const checkIn = new CheckIn({ applyConfig: (bundle) => applyConfig(bundle, { db, executor, locks, swap, log, beat }), log, + // For the wait before listen alone (check-in.ts untilConfigured): the + // ticker itself never beats, for the reason the metrics ticker does not + // (the watchdog's comment above). + beat, }); // The daemon's own upgrade window compares the commit baked into this diff --git a/website/content/docs/cli.mdx b/website/content/docs/cli.mdx index 298f4bb1..584d7d02 100644 --- a/website/content/docs/cli.mdx +++ b/website/content/docs/cli.mdx @@ -4,15 +4,20 @@ description: Every dor command — sandboxes, templates, and the host doctor — --- The CLI installs as two names for the same binary: `dormice` and `dor` -(the one you actually type). It talks to a daemon over the -[HTTP API](/docs/http-api) and connects with exactly two environment -variables — missing one produces an error naming it: +(the one you actually type). It speaks the [HTTP API](/docs/http-api) +to the gateway — the fleet's one door — and connects with exactly two +environment variables — missing one produces an error naming it: ```sh -export DORMICE_ENDPOINT=http://127.0.0.1:3676 -export DORMICE_API_TOKEN= # on the daemon host: grep ^DORMICE_API_TOKEN /etc/dormice/env +export DORMICE_ENDPOINT=http://127.0.0.1:3677 +export DORMICE_API_TOKEN= # on the host: grep ^DORMICE_API_TOKEN /etc/dormice/env ``` +Two commands list rather than act — `dor sandbox ls` and `dor sandbox +meta` — and the gateway does not route the list verbs yet: for those, +point `DORMICE_ENDPOINT` at the daemon (`http://127.0.0.1:3676`) until +it does. Everything else on this page goes to the gateway. + Errors print as one line on stderr (no stack traces at a shell prompt) and exit 1. @@ -103,11 +108,11 @@ dor apikey revoke # final; frees the name API keys are revocable peers of `DORMICE_API_TOKEN` — same power, but killable without touching the server ([details](/docs/http-api#api-keys)). -`create` prints the 64-hex token exactly once; the daemon keeps only its +`create` prints the 64-hex token exactly once; the gateway keeps only its hash. The minted key goes wherever the token goes, including this CLI's own `DORMICE_API_TOKEN` variable — with one exception: the `apikey` commands themselves require the env token (keys cannot manage keys; a -key gets the daemon's honest 403). `disable` is the reversible half of +key gets the gateway's honest 403). `disable` is the reversible half of `revoke`: the key stops working until re-enabled but keeps its name and history. `revoke` takes effect on the key's next request; revoking a name that has no active key says so honestly — check the spelling before diff --git a/website/content/docs/quickstart.mdx b/website/content/docs/quickstart.mdx index 48174d8d..b63328c7 100644 --- a/website/content/docs/quickstart.mdx +++ b/website/content/docs/quickstart.mdx @@ -5,21 +5,26 @@ description: This guide shows you how to acquire your first sandbox, run command By the end of this page you will have created a sandbox, run a command inside it, written a file into it, and destroyed it — from TypeScript, -from plain `curl`, or from your terminal. It assumes a daemon is -already running (from [Installation](/docs/installation)). +from plain `curl`, or from your terminal. It assumes Dormice is already +installed and running (from [Installation](/docs/installation)). ## 1. Set your environment variables -Clients find the daemon through two environment variables — named -values your shell passes to every program you start. You need the -daemon's address and its [API -token](/docs/core-concepts#tokens-sdks-and-the-cli): +Clients find Dormice through two environment variables — named values +your shell passes to every program you start. You need the gateway's +address — the fleet's one door, `3677` on the host it runs on — and the +[API token](/docs/core-concepts#tokens-sdks-and-the-cli): ```sh -export DORMICE_ENDPOINT=http://127.0.0.1:3676 -export DORMICE_API_TOKEN= # on the daemon host: grep ^DORMICE_API_TOKEN /etc/dormice/env +export DORMICE_ENDPOINT=http://127.0.0.1:3677 +export DORMICE_API_TOKEN= # on the host: grep ^DORMICE_API_TOKEN /etc/dormice/env ``` +The gateway answers everything this page does. Two CLI commands list +rather than act — `dor sandbox ls` and `dor sandbox meta` — and the +gateway does not route the list verbs yet: for those, point +`DORMICE_ENDPOINT` at the daemon (`http://127.0.0.1:3676`) until it does. + ## 2. Get the SDK `@dormice/sdk` is the native TypeScript client. It is not on npm yet — @@ -33,7 +38,7 @@ or skip the SDK entirely and use [curl](#use-curl-instead) or the import { Dormice } from '@dormice/sdk'; const client = new Dormice({ - endpoint: 'http://127.0.0.1:3676', + endpoint: 'http://127.0.0.1:3677', token: process.env.DORMICE_API_TOKEN!, }); @@ -64,9 +69,9 @@ sent in the `Authorization` header — so curl works anywhere the SDK doesn't reach: ```sh -curl -X POST http://127.0.0.1:3676/listSandboxes \ +curl -X POST http://127.0.0.1:3677/acquireSandbox \ -H "Authorization: Bearer $DORMICE_API_TOKEN" \ - -H "content-type: application/json" -d '{}' + -H "content-type: application/json" -d '{"name": "my-agent"}' ``` The full wire surface is in the [HTTP API reference](/docs/http-api). @@ -77,7 +82,7 @@ The `dor` command (long form: `dormice`) covers the same ground from a shell. It connects with the two environment variables from step 1: ```sh -dor sandbox ls # every sandbox, with lifecycle state +dor sandbox ls # every sandbox, with lifecycle state (lists: the daemon's address, step 1) dor sandbox exec my-agent 'uname -r' # run a command, exit code passes through dor sandbox push my-agent ./data.csv # copy a file in… dor sandbox pull my-agent notes.txt # …and out From 5d29c0ca5b4f8dfee1eb787fed81178b469defc1 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 18:52:08 +0800 Subject: [PATCH 39/89] install.sh's closing hint drops the backticks shellcheck reads as an expression inside single quotes --- deploy/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/install.sh b/deploy/install.sh index 00d2c2c9..6c78ee77 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -858,7 +858,7 @@ printf ' API token: grep ^DORMICE_API_TOKEN %s\n' "$ENV_FILE" printf ' gateway logs: journalctl -u dormice-gateway -f (the door: console, keys, settings, templates)\n' printf ' daemon logs: journalctl -u dormice -f (the node: sandboxes)\n' printf ' CLI: export DORMICE_ENDPOINT=http://127.0.0.1:%s DORMICE_API_TOKEN=; dor template ls\n' "$GATEWAY_PORT" -printf ' (the gateway is the door; `dor sandbox ls` and `dor sandbox meta` list, which the gateway\n' +printf ' (the gateway is the door; dor sandbox ls and dor sandbox meta list, which the gateway\n' printf ' does not route yet — point DORMICE_ENDPOINT at the daemon, 127.0.0.1:%s, for those)\n' "$PORT" printf ' Both processes listen on 127.0.0.1 only, by design — exposing them is a reverse proxy'"'"'s job.\n' if [ "$(systemctl is-active caddy 2>/dev/null)" = active ]; then From baaaaad171bf08830c1b54cb1e57fbd79a96de79 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 19:40:47 +0800 Subject: [PATCH 40/89] Every client example knocks at the gateway's door; the list caveat names only the forms that read the list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docs sent the CLI to the gateway on 2026-09-14 but left the SDK, E2B, curl and example-program snippets on the daemon's 3676 — a reader saw one port on the front page and another in the quick start with nothing in between explaining the change. Every verb those snippets use is one the gateway routes (the named sandbox verbs, and the E2B control plane and envd, which e2e proves through the door), so they all move to 3677; the two-doors paragraphs that explain what the daemon still answers stay as they are, and the port-proxy page follows when that face moves. `dor sandbox meta k=v` is updateMetadata, a named verb the gateway forwards; only the label-less form reads the list. The caveat in the CLI page, the quick start and install.sh's closing hint said "meta" whole, and now names the form. Two READMEs named a `dor sandbox release` command that does not exist; the verb is destroy. --- README.md | 6 +++--- deploy/install.sh | 5 +++-- examples/README.md | 17 +++++++++-------- examples/e2b-compat.mjs | 2 +- examples/native-lifecycle.mjs | 2 +- examples/resident-agent.mjs | 2 +- packages/cli/README.md | 12 ++++++++---- packages/sdk/README.md | 2 +- packages/sdk/src/client.ts | 4 ++-- skills/dormice/SKILL.md | 12 ++++++------ website/content/docs/cli.mdx | 10 ++++++---- website/content/docs/e2b-sdks.mdx | 8 ++++---- website/content/docs/index.mdx | 2 +- website/content/docs/quickstart.mdx | 9 +++++---- 14 files changed, 51 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 38a0a6c1..905cf05b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ first release is queued; inside this repo, `pnpm build` produces it.) import { Dormice } from '@dormice/sdk'; const client = new Dormice({ - endpoint: 'http://127.0.0.1:3676', + endpoint: 'http://127.0.0.1:3677', token: process.env.DORMICE_API_TOKEN!, }); @@ -93,8 +93,8 @@ import { Sandbox } from 'e2b'; const sbx = await Sandbox.create({ apiKey: `e2b_${process.env.DORMICE_API_TOKEN}`, - apiUrl: 'http://127.0.0.1:3676/e2b/api', - sandboxUrl: 'http://127.0.0.1:3676/e2b/envd', + apiUrl: 'http://127.0.0.1:3677/e2b/api', + sandboxUrl: 'http://127.0.0.1:3677/e2b/envd', }); ``` diff --git a/deploy/install.sh b/deploy/install.sh index 6c78ee77..c45493ce 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -858,8 +858,9 @@ printf ' API token: grep ^DORMICE_API_TOKEN %s\n' "$ENV_FILE" printf ' gateway logs: journalctl -u dormice-gateway -f (the door: console, keys, settings, templates)\n' printf ' daemon logs: journalctl -u dormice -f (the node: sandboxes)\n' printf ' CLI: export DORMICE_ENDPOINT=http://127.0.0.1:%s DORMICE_API_TOKEN=; dor template ls\n' "$GATEWAY_PORT" -printf ' (the gateway is the door; dor sandbox ls and dor sandbox meta list, which the gateway\n' -printf ' does not route yet — point DORMICE_ENDPOINT at the daemon, 127.0.0.1:%s, for those)\n' "$PORT" +printf ' (the gateway is the door; dor sandbox ls, and dor sandbox meta without labels, read\n' +printf ' the sandbox list, which the gateway does not route yet — point DORMICE_ENDPOINT at the daemon,\n' +printf ' 127.0.0.1:%s, for those)\n' "$PORT" printf ' Both processes listen on 127.0.0.1 only, by design — exposing them is a reverse proxy'"'"'s job.\n' if [ "$(systemctl is-active caddy 2>/dev/null)" = active ]; then printf ' console: http:///console (Caddy on :80 -> the gateway; open your cloud firewall for\n' diff --git a/examples/README.md b/examples/README.md index e1b8e293..2f72b5d5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,6 @@ # Examples -Small, runnable programs against a live Dormice daemon. Each file is +Small, runnable programs against a live Dormice install. Each file is self-contained — read it top to bottom, then run it. | File | What it shows | @@ -11,20 +11,21 @@ self-contained — read it top to bottom, then run it. ## Prerequisites -A running daemon with the docker executor — a host prepared by -[`install.sh`](../deploy/install.sh) is exactly right. The examples speak to -it through the same two environment variables the `dor` CLI uses: +A running Dormice — a host prepared by [`install.sh`](../deploy/install.sh) +is exactly right: the gateway on 3677, a daemon with the docker executor +behind it. The examples speak to the gateway through the same two +environment variables the `dor` CLI uses: ```sh -export DORMICE_ENDPOINT=http://127.0.0.1:3676 # the default; omit when local +export DORMICE_ENDPOINT=http://127.0.0.1:3677 # the default; omit when local export DORMICE_API_TOKEN=... # /etc/dormice/env on the host ``` -The daemon binds to 127.0.0.1 only. To run the examples from your laptop +The gateway binds to 127.0.0.1 only. To run the examples from your laptop against a remote host, open a tunnel first and keep the default endpoint: ```sh -ssh -L 3676:127.0.0.1:3676 root@your-host +ssh -L 3677:127.0.0.1:3677 root@your-host ``` ## Running @@ -44,5 +45,5 @@ In your own project, once the packages are published, the same code runs after `npm install @dormice/sdk e2b`. `resident-agent.mjs` deliberately leaves its sandbox running; when you are -done playing, remove it with `dor sandbox release example-resident-agent` +done playing, remove it with `dor sandbox destroy example-resident-agent` (or `destroySandbox` from the SDK). diff --git a/examples/e2b-compat.mjs b/examples/e2b-compat.mjs index a567ec47..eae23447 100644 --- a/examples/e2b-compat.mjs +++ b/examples/e2b-compat.mjs @@ -5,7 +5,7 @@ // DORMICE_API_TOKEN=... node examples/e2b-compat.mjs import { Sandbox } from 'e2b'; -const endpoint = process.env.DORMICE_ENDPOINT ?? 'http://127.0.0.1:3676'; +const endpoint = process.env.DORMICE_ENDPOINT ?? 'http://127.0.0.1:3677'; const token = process.env.DORMICE_API_TOKEN; if (!token) { console.error( diff --git a/examples/native-lifecycle.mjs b/examples/native-lifecycle.mjs index 25a5f6c4..331d403e 100644 --- a/examples/native-lifecycle.mjs +++ b/examples/native-lifecycle.mjs @@ -5,7 +5,7 @@ // DORMICE_API_TOKEN=... node examples/native-lifecycle.mjs import { Dormice } from '@dormice/sdk'; -const endpoint = process.env.DORMICE_ENDPOINT ?? 'http://127.0.0.1:3676'; +const endpoint = process.env.DORMICE_ENDPOINT ?? 'http://127.0.0.1:3677'; const token = process.env.DORMICE_API_TOKEN; if (!token) { console.error( diff --git a/examples/resident-agent.mjs b/examples/resident-agent.mjs index 9a52c2ca..eac6725d 100644 --- a/examples/resident-agent.mjs +++ b/examples/resident-agent.mjs @@ -9,7 +9,7 @@ // DORMICE_API_TOKEN=... node examples/resident-agent.mjs import { Dormice } from '@dormice/sdk'; -const endpoint = process.env.DORMICE_ENDPOINT ?? 'http://127.0.0.1:3676'; +const endpoint = process.env.DORMICE_ENDPOINT ?? 'http://127.0.0.1:3677'; const token = process.env.DORMICE_API_TOKEN; if (!token) { console.error( diff --git a/packages/cli/README.md b/packages/cli/README.md index 39763d0a..b12331bb 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -12,14 +12,18 @@ npm install -g @dormice/cli ## Connect -Everything under `dor sandbox` talks to a daemon named by two environment -variables (it complains by name if one is missing): +`dor` talks to the gateway — the fleet's one door — named by two +environment variables (it complains by name if one is missing): ```sh -export DORMICE_ENDPOINT=http://127.0.0.1:3676 +export DORMICE_ENDPOINT=http://127.0.0.1:3677 export DORMICE_API_TOKEN=... ``` +`dor sandbox ls` (and `dor sandbox meta ` without labels) read the +sandbox list, which the gateway does not route yet: point +`DORMICE_ENDPOINT` at the daemon, `http://127.0.0.1:3676`, for those. + ## Commands | Command | What it does | @@ -30,7 +34,7 @@ export DORMICE_API_TOKEN=... | `dor sandbox push [remote]` | Copy a local file into the sandbox | | `dor sandbox pull [local]` | Copy a file out; no local path = raw bytes to stdout | | `dor sandbox rebuild ` | Swap the container, keep `/home/user` — next use starts on the daemon's current base image | -| `dor sandbox release ` | Destroy the sandbox (idempotent) | +| `dor sandbox destroy ` | Destroy the sandbox (idempotent) | `doctor` inspects the local host, never writes, and prints the fix for anything it flags — the checks are the distilled lessons of running the diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 492a92e7..507aa860 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -16,7 +16,7 @@ npm install @dormice/sdk import { Dormice } from '@dormice/sdk'; const client = new Dormice({ - endpoint: 'http://127.0.0.1:3676', // your daemon + endpoint: 'http://127.0.0.1:3677', // the gateway, the fleet's door token: process.env.DORMICE_API_TOKEN!, }); diff --git a/packages/sdk/src/client.ts b/packages/sdk/src/client.ts index 6f0e5818..8c1ed5be 100644 --- a/packages/sdk/src/client.ts +++ b/packages/sdk/src/client.ts @@ -83,9 +83,9 @@ import { Agent, fetch, type Response } from 'undici'; const dispatcher = new Agent({ headersTimeout: 0, bodyTimeout: 0 }); export interface DormiceOptions { - /** Base URL of the daemon, e.g. `http://127.0.0.1:3676`. */ + /** Base URL of the gateway, the fleet's door, e.g. `http://127.0.0.1:3677`. */ endpoint: string; - /** The daemon's DORMICE_API_TOKEN. */ + /** The fleet's DORMICE_API_TOKEN, or an API key minted at the gateway. */ token: string; /** * Per-request timeout. Without one, a wedged daemon would hang the diff --git a/skills/dormice/SKILL.md b/skills/dormice/SKILL.md index c31c6bea..9f2d7eb0 100644 --- a/skills/dormice/SKILL.md +++ b/skills/dormice/SKILL.md @@ -58,8 +58,8 @@ import { Sandbox } from 'e2b'; const sbx = await Sandbox.create({ apiKey: `e2b_${process.env.DORMICE_API_TOKEN}`, - apiUrl: 'http://127.0.0.1:3676/e2b/api', - sandboxUrl: 'http://127.0.0.1:3676/e2b/envd', + apiUrl: 'http://127.0.0.1:3677/e2b/api', + sandboxUrl: 'http://127.0.0.1:3677/e2b/envd', }); await sbx.commands.run('echo hello'); ``` @@ -70,8 +70,8 @@ from e2b import Sandbox sandbox = Sandbox.create( api_key=f"e2b_{os.environ['DORMICE_API_TOKEN']}", - api_url="http://127.0.0.1:3676/e2b/api", - sandbox_url="http://127.0.0.1:3676/e2b/envd", + api_url="http://127.0.0.1:3677/e2b/api", + sandbox_url="http://127.0.0.1:3677/e2b/envd", ) sandbox.commands.run("echo hello") ``` @@ -93,7 +93,7 @@ Every operation is `POST /` with a JSON body and import { Dormice } from '@dormice/sdk'; const client = new Dormice({ - endpoint: 'http://127.0.0.1:3676', + endpoint: 'http://127.0.0.1:3677', token: process.env.DORMICE_API_TOKEN!, }); @@ -112,7 +112,7 @@ await client.destroySandbox('my-agent'); // the only verb that loses data The same loop in curl: ```sh -curl -X POST http://127.0.0.1:3676/acquireSandbox \ +curl -X POST http://127.0.0.1:3677/acquireSandbox \ -H "Authorization: Bearer $DORMICE_API_TOKEN" \ -H "content-type: application/json" \ -d '{"name": "my-agent"}' diff --git a/website/content/docs/cli.mdx b/website/content/docs/cli.mdx index 584d7d02..972815c5 100644 --- a/website/content/docs/cli.mdx +++ b/website/content/docs/cli.mdx @@ -13,10 +13,12 @@ export DORMICE_ENDPOINT=http://127.0.0.1:3677 export DORMICE_API_TOKEN= # on the host: grep ^DORMICE_API_TOKEN /etc/dormice/env ``` -Two commands list rather than act — `dor sandbox ls` and `dor sandbox -meta` — and the gateway does not route the list verbs yet: for those, -point `DORMICE_ENDPOINT` at the daemon (`http://127.0.0.1:3676`) until -it does. Everything else on this page goes to the gateway. +Two forms read the sandbox list — `dor sandbox ls`, and `dor sandbox +meta ` without labels (showing labels reads the list) — and the +gateway does not route the list verbs yet: for those, point +`DORMICE_ENDPOINT` at the daemon (`http://127.0.0.1:3676`) until it +does. Everything else on this page goes to the gateway, `meta` with +labels included. Errors print as one line on stderr (no stack traces at a shell prompt) and exit 1. diff --git a/website/content/docs/e2b-sdks.mdx b/website/content/docs/e2b-sdks.mdx index 2fa2d0c0..7f3d6ecd 100644 --- a/website/content/docs/e2b-sdks.mdx +++ b/website/content/docs/e2b-sdks.mdx @@ -25,8 +25,8 @@ import { Sandbox } from 'e2b'; const sbx = await Sandbox.create({ apiKey: `e2b_${process.env.DORMICE_API_TOKEN}`, - apiUrl: 'http://127.0.0.1:3676/e2b/api', - sandboxUrl: 'http://127.0.0.1:3676/e2b/envd', + apiUrl: 'http://127.0.0.1:3677/e2b/api', + sandboxUrl: 'http://127.0.0.1:3677/e2b/envd', }); await sbx.commands.run('echo hello'); @@ -42,8 +42,8 @@ from e2b import Sandbox sandbox = Sandbox.create( api_key=f"e2b_{os.environ['DORMICE_API_TOKEN']}", - api_url="http://127.0.0.1:3676/e2b/api", - sandbox_url="http://127.0.0.1:3676/e2b/envd", + api_url="http://127.0.0.1:3677/e2b/api", + sandbox_url="http://127.0.0.1:3677/e2b/envd", ) sandbox.commands.run("echo hello") diff --git a/website/content/docs/index.mdx b/website/content/docs/index.mdx index fd8671ea..1c0bf1f6 100644 --- a/website/content/docs/index.mdx +++ b/website/content/docs/index.mdx @@ -19,7 +19,7 @@ Start a sandbox and run a command in a few lines: import { Dormice } from '@dormice/sdk'; const client = new Dormice({ - endpoint: 'http://127.0.0.1:3676', + endpoint: 'http://127.0.0.1:3677', token: process.env.DORMICE_API_TOKEN!, }); diff --git a/website/content/docs/quickstart.mdx b/website/content/docs/quickstart.mdx index b63328c7..7c77cff7 100644 --- a/website/content/docs/quickstart.mdx +++ b/website/content/docs/quickstart.mdx @@ -20,10 +20,11 @@ export DORMICE_ENDPOINT=http://127.0.0.1:3677 export DORMICE_API_TOKEN= # on the host: grep ^DORMICE_API_TOKEN /etc/dormice/env ``` -The gateway answers everything this page does. Two CLI commands list -rather than act — `dor sandbox ls` and `dor sandbox meta` — and the -gateway does not route the list verbs yet: for those, point -`DORMICE_ENDPOINT` at the daemon (`http://127.0.0.1:3676`) until it does. +The gateway answers everything this page does. Two CLI forms read the +sandbox list — `dor sandbox ls`, and `dor sandbox meta ` without +labels — and the gateway does not route the list verbs yet: for those, +point `DORMICE_ENDPOINT` at the daemon (`http://127.0.0.1:3676`) until +it does. ## 2. Get the SDK From ae6f90b72143f33e9e9ce25cef88cb0b93d84230 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 19:45:08 +0800 Subject: [PATCH 41/89] The watchdog's beat is a parameter of the one wait that may beat it; the no-copy sentence says "applied", not "answered" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit untilConfigured() took the beat from the CheckIn options, where a comment promised the ticker would never call it and a test counted zero calls to prove it. As the parameter of the wait itself, the ticker has no beat to call: the rule is the shape, not a comment. The comment also overstated: a bundle that moves the pids cap makes the ticker's once() beat through the sweep — the lifecycle work beating per row, as work does everywhere, which is legitimate and stays. The first-failure sentence for a node without a copy said it does not listen "until the gateway answers with one"; a gateway that answered a bundle the node could not apply (a full disk) leaves the node in the same predicament with the gateway blameless. "Until it has applied one from the gateway" is true in both cases, and the error field names which happened. --- packages/server/src/check-in.test.ts | 28 +++++++--------------- packages/server/src/check-in.ts | 35 ++++++++++++++-------------- packages/server/src/main.ts | 9 ++++--- 3 files changed, 31 insertions(+), 41 deletions(-) diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index 4bd932d7..08d5d8c8 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -322,14 +322,13 @@ describe('CheckIn', () => { }); const { log } = logSpy(); let beats = 0; - const opts = options(gw.endpoint, log, { - beat: () => { - beats += 1; - }, - }); + const beat = () => { + beats += 1; + }; + const opts = options(gw.endpoint, log); const checkIn = new CheckIn(opts); const started = Date.now(); - await checkIn.untilConfigured(); + await checkIn.untilConfigured(beat); expect(readConfigVersion(opts.db)).toBe(2); expect(gw.seen).toHaveLength(3); // Two waits of one interval between the three asks. @@ -338,7 +337,7 @@ describe('CheckIn', () => { // node waiting for its gateway is alive, not stalled. expect(beats).toBe(3); // Holding a copy already: nothing is asked, nothing beats. - await checkIn.untilConfigured(); + await checkIn.untilConfigured(beat); expect(gw.seen).toHaveLength(3); expect(beats).toBe(3); }); @@ -352,7 +351,7 @@ describe('CheckIn', () => { await new CheckIn(options(gw.endpoint, bare.log)).once(); expect(bare.warns).toEqual([ expect.stringMatching( - /^check-in failed; this node holds no configuration copy and does not listen until the gateway answers with one/, + /^check-in failed; this node holds no configuration copy and does not listen until it has applied one from the gateway/, ), ]); const holding = logSpy(); @@ -366,17 +365,10 @@ describe('CheckIn', () => { ]); }); - it('ticks on its interval from start() and stops on stop(); the ticker never beats the watchdog', async () => { + it('ticks on its interval from start() and stops on stop()', async () => { const gw = await gateway(() => answering(1)); const { log } = logSpy(); - let beats = 0; - const checkIn = new CheckIn( - options(gw.endpoint, log, { - beat: () => { - beats += 1; - }, - }), - ); + const checkIn = new CheckIn(options(gw.endpoint, log)); checkIn.start(); const deadline = Date.now() + 5_000; while (gw.seen.length < 2 && Date.now() < deadline) { @@ -387,7 +379,5 @@ describe('CheckIn', () => { const afterStop = gw.seen.length; await new Promise((resolve) => setTimeout(resolve, 1_200)); expect(gw.seen.length).toBe(afterStop); - // A ticker's liveness must never reassure the watchdog (main.ts). - expect(beats).toBe(0); }); }); diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index ddbbd702..38200467 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -52,14 +52,6 @@ export interface CheckInOptions { /** Makes a bundle the gateway answered with real on this node (node-config.ts applyConfig). */ applyConfig: (bundle: NodeConfigBundle) => Promise; log: CheckInLog; - /** - * The heartbeat watchdog's ear, for untilConfigured() alone: a node - * waiting for its first bundle has no lifecycle work to beat, and each - * attempt — answered or not, bounded by CHECK_IN_TIMEOUT_MS — is the - * wait provably alive. The ticker never beats: a ticker's liveness must - * not reassure the watchdog (main.ts has the 2026-08-13 lesson). - */ - beat?: () => void; /** Test seam; production uses the platform's fetch. */ fetchImpl?: typeof fetch; } @@ -188,10 +180,13 @@ export class CheckIn { // holding a copy keeps serving and is merely not placed on; one // without (untilConfigured, at boot) is not listening at all, and // "the gateway forwards nothing here" would name the wrong - // predicament (found by review, 2026-09-14). + // predicament (found by review, 2026-09-14). "Applied", not + // "answered": a gateway that answered a bundle this node could not + // apply leaves it in the same predicament, and the error names + // which of the two happened. const cost = opts.configVersion() === null - ? 'this node holds no configuration copy and does not listen until the gateway answers with one' + ? 'this node holds no configuration copy and does not listen until it has applied one from the gateway' : 'the gateway places nothing here and forwards no new names to this node until it answers again'; opts.log.warn( { gateway: opts.gateway, error: message }, @@ -213,16 +208,22 @@ export class CheckIn { * logged once by once(), so a gateway down for an hour is one line. The * check-ins sent here carry `configVersion: null`, which is what keeps * the gateway from placing on this node before it listens. + * + * `beat` is the heartbeat watchdog's ear, and this wait is the only + * place the check-in may beat it: each attempt — answered or not, + * bounded by CHECK_IN_TIMEOUT_MS — is the wait provably alive, where + * the watchdog, started before boot's awaits, otherwise read a node + * half an hour into waiting for its gateway as a stalled daemon and + * exited it — every thirty minutes, for nothing (found by review, + * 2026-09-14). The ticker (start()) is handed no beat, so a ticker's + * liveness cannot reassure the watchdog (main.ts has the 2026-08-13 + * lesson); the lifecycle work a bundle sets off (the pids sweep in + * applyConfig) beats per row on its own, as work does everywhere. */ - async untilConfigured(): Promise { + async untilConfigured(beat: () => void): Promise { while (!this.closing && this.opts.configVersion() === null) { await this.once(); - // Each attempt is the wait provably alive (CheckInOptions.beat): the - // watchdog starts before boot's awaits, and without a beat it read a - // node half an hour into waiting for its gateway as a stalled daemon - // and exited it — every thirty minutes, for nothing (found by - // review, 2026-09-14). - this.opts.beat?.(); + beat(); if (this.opts.configVersion() !== null) return; await new Promise((resolve) => setTimeout(resolve, this.opts.intervalSeconds * 1000), diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index 15bf6f01..dd024e91 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -213,10 +213,6 @@ const checkIn = new CheckIn({ applyConfig: (bundle) => applyConfig(bundle, { db, executor, locks, swap, log, beat }), log, - // For the wait before listen alone (check-in.ts untilConfigured): the - // ticker itself never beats, for the reason the metrics ticker does not - // (the watchdog's comment above). - beat, }); // The daemon's own upgrade window compares the commit baked into this @@ -281,7 +277,10 @@ if (readConfigVersion(db) === null) { log.info( `no configuration copy in the ledger — asking gateway ${config.DORMICE_GATEWAY_ENDPOINT} before anything else (retrying every ${config.DORMICE_CHECK_IN_INTERVAL_SECONDS}s until it answers)`, ); - await checkIn.untilConfigured(); + // The wait beats the watchdog per attempt (check-in.ts untilConfigured); + // the ticker started after listen is handed no beat, for the reason the + // metrics ticker is not (the watchdog's comment above). + await checkIn.untilConfigured(beat); } { const copy = readNodeConfig(db); From 2f0fa83bb17cc0f9d1fb4745d9c2536d5045b603 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 19:59:06 +0800 Subject: [PATCH 42/89] The sandbox port proxy has a face at the gateway: a sandbox host is forwarded to the node holding the id, Host kept, upgrades too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since install.sh points Caddy at the gateway, a preview URL on the wildcard sandbox domain — the one URL E2B's getHost() hands to browsers — arrived at a door that answered 404: the fleet's one door did not serve it. Now the gateway's serverFactory judges the Host first, against the domain group in its own settings (read per request, so a console edit applies to the next request), and a `-.` host is the proxy face: the node holding the id is found (the cache, then every node asked) and the request goes there whole — Host kept, since the node's own proxy keys on it and dials the container; no credential added, since sandbox traffic is unauthenticated, as E2B's is. WebSocket upgrades ride the same way (forwardUpgrade). The gateway wakes nothing and dials nothing; it adds only where the sandbox is. The 49983 /files form rides through like any other host — the node carves it out onto its signed-URL door and pins the sandbox by the Host, so the gateway holds no second copy of that rule; its own refusals carry CORS on that form alone, keeping the node's promise that every answer to the browser-direct form is readable. Refusals otherwise wear the daemon's proxy answer, { message } 502, so a caller reads one shape from either door. The bare root /files stays 501: nothing in it names a sandbox the gateway could read. The Host grammar moves to @dormice/shared (sandbox-host.ts) so both doors parse one way; the daemon and its signed door import it from there. Exams: four at the gateway (forwarded with Host kept and no credential, whatever path the host spells; a sandbox built behind its back found by asking; the refusal dialect and where CORS goes; upgrades both ways and refused or cut; a domain engaging, an alias joining and clearing disengaging with no restart), and three e2e legs through real processes: the fleet door over HTTP and WebSocket to whichever node holds the sandbox, the E2B getHost round trip through the door, and the domain switch judged at both doors. The ports page points the wildcard site block at the gateway. --- e2e/src/e2b.test.ts | 46 ++-- e2e/src/gateway.test.ts | 67 ++++++ e2e/src/helpers.ts | 30 +++ e2e/src/settings-hot.test.ts | 43 ++-- packages/gateway/src/app.test.ts | 249 +++++++++++++++++++++- packages/gateway/src/app.ts | 17 +- packages/gateway/src/classify.ts | 25 ++- packages/gateway/src/errors.ts | 4 +- packages/gateway/src/forward.ts | 23 +- packages/gateway/src/raw.ts | 202 ++++++++++++++---- packages/server/src/e2b/signed-files.ts | 2 +- packages/server/src/sandbox-proxy.test.ts | 6 +- packages/server/src/sandbox-proxy.ts | 72 ++----- packages/shared/src/index.ts | 1 + packages/shared/src/sandbox-host.ts | 75 +++++++ website/content/docs/ports.mdx | 25 ++- 16 files changed, 697 insertions(+), 190 deletions(-) create mode 100644 packages/shared/src/sandbox-host.ts diff --git a/e2e/src/e2b.test.ts b/e2e/src/e2b.test.ts index f69a5489..5209ffa2 100644 --- a/e2e/src/e2b.test.ts +++ b/e2e/src/e2b.test.ts @@ -1,8 +1,7 @@ -import http from 'node:http'; import { Dormice } from '@dormice/sdk'; import { CommandExitError, Sandbox } from 'e2b'; import { describe, expect, inject, it } from 'vitest'; -import { door, until as poll, settled } from './helpers'; +import { door, until as poll, settled, spoofHost } from './helpers'; // The compatibility promise, verified with the promise's own artifact: the // OFFICIAL e2b package, pointed at the daemon by exactly two URLs (plus its @@ -29,36 +28,12 @@ async function until(check: () => boolean, timeoutMs = 8_000) { } } -/** - * A GET at the daemon with a spoofed Host header — exactly what traffic - * from a wildcard-DNS reverse proxy looks like, no DNS needed (fetch - * refuses to set Host, so this speaks node:http directly). - */ -function throughProxy( +/** A GET with a spoofed Host header, at node A by default or at the door (helpers.ts spoofHost). */ +const throughProxy = ( host: string, path = '/', -): Promise<{ status: number; body: string }> { - const endpoint = new URL(inject('dormiceEndpoint')); - return new Promise((resolve, reject) => { - const req = http.request( - { - host: endpoint.hostname, - port: endpoint.port, - path, - headers: { host }, - }, - (res) => { - let body = ''; - res.on('data', (chunk) => { - body += chunk; - }); - res.on('end', () => resolve({ status: res.statusCode ?? 0, body })); - }, - ); - req.on('error', reject); - req.end(); - }); -} + via = inject('dormiceEndpoint'), +) => spoofHost(via, host, path); describe('official e2b SDK against the daemon', () => { it('creates a sandbox and runs a command', async () => { @@ -777,6 +752,17 @@ describe.runIf(process.env.DORMICE_EXECUTOR !== 'docker')( expect(echo.sandboxId).toBe(sbx.sandboxId); expect(echo.path).toBe('/hello?from=e2e'); expect(echo.host).toBe(host); + // The same URL at the door: the gateway reads the id off the Host, + // finds the node holding it and forwards, Host kept — getHost() + // works through the fleet's one door, which is where the wildcard + // DNS points in production. + const viaDoor = await throughProxy(host, '/hello?from=door', door()); + expect(viaDoor.status).toBe(200); + expect(JSON.parse(viaDoor.body)).toMatchObject({ + sandboxId: sbx.sandboxId, + path: '/hello?from=door', + host, + }); } finally { await sbx.kill(); } diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts index f4448ac0..fa848149 100644 --- a/e2e/src/gateway.test.ts +++ b/e2e/src/gateway.test.ts @@ -11,6 +11,7 @@ import { configSettled, listNodes as listFleetNodes, rpc as post, + spoofHost, until, } from './helpers'; @@ -174,6 +175,72 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { } }); + it('a sandbox host at the fleet door reaches the node holding the sandbox — getHost() through the gateway, over HTTP and a WebSocket upgrade; an id on no node is the proxy’s own 502', async () => { + const created = await viaGateway().acquireSandbox('gw-host'); + try { + const id = created.sandbox.id; + const host = `8000-${id}.sbx.dormice.test`; + // The fake executor's upstream echoes what reached the sandbox: the + // id proves which sandbox answered, the host that the Host was kept. + const res = await spoofHost(gateway(), host, '/hello?via=door'); + expect(res.status).toBe(200); + expect(JSON.parse(res.body)).toMatchObject({ + sandboxId: id, + path: '/hello?via=door', + host, + }); + // The node that does not hold it says so directly — the door asked + // the right one. + const elsewhere = await spoofHost( + nodes().find((n) => n.id !== created.sandbox.nodeId)?.endpoint ?? '', + host, + '/', + ); + expect(elsewhere.status).toBe(502); + expect(JSON.parse(elsewhere.body).message).toContain('not found'); + + // The upgrade half: a raw handshake through the door, the echo back. + const url = new URL(gateway()); + const echoed = await new Promise((resolve, reject) => { + let buffer = ''; + const socket = net.connect(Number(url.port), url.hostname, () => { + socket.write( + [ + 'GET /ws HTTP/1.1', + `Host: 5173-${id}.sbx.dormice.test`, + 'Connection: Upgrade', + 'Upgrade: websocket', + '', + '', + ].join('\r\n'), + ); + }); + socket.on('data', (chunk) => { + buffer += chunk.toString('utf8'); + if (buffer.includes(' 101 ') && !buffer.includes('marco')) { + socket.write('marco'); + } + if (buffer.includes('marco')) socket.end(); + }); + socket.on('close', () => resolve(buffer)); + socket.on('error', reject); + setTimeout(() => reject(new Error('upgrade timed out')), 5_000); + }); + expect(echoed).toContain(' 101 '); + expect(echoed).toContain('marco'); + + const nobody = await spoofHost( + gateway(), + `8000-${randomUUID()}.sbx.dormice.test`, + '/', + ); + expect(nobody.status).toBe(502); + expect(JSON.parse(nobody.body).message).toMatch(/on no node/); + } finally { + await viaGateway().destroySandbox('gw-host'); + } + }); + it('acquire is placed on one node and lands in exactly one ledger; re-acquire finds it there', async () => { const created = await viaGateway().acquireSandbox('gw-place'); try { diff --git a/e2e/src/helpers.ts b/e2e/src/helpers.ts index 42168071..2bed6f6a 100644 --- a/e2e/src/helpers.ts +++ b/e2e/src/helpers.ts @@ -1,3 +1,4 @@ +import http from 'node:http'; import { inject } from 'vitest'; /** @@ -22,6 +23,35 @@ export async function until( } } +/** + * A GET at `via` with a spoofed Host header — exactly what traffic from a + * wildcard-DNS reverse proxy looks like, no DNS needed (fetch refuses to + * set Host, so this speaks node:http directly). Both doors key on the + * Host: the daemon's port proxy dials the sandbox, the gateway's proxy + * face forwards to the node holding it. + */ +export function spoofHost( + via: string, + host: string, + path = '/', +): Promise<{ status: number; body: string }> { + const endpoint = new URL(via); + return new Promise((resolve, reject) => { + const req = http.request( + { host: endpoint.hostname, port: endpoint.port, path, headers: { host } }, + (res) => { + let body = ''; + res.on('data', (chunk) => { + body += chunk; + }); + res.on('end', () => resolve({ status: res.statusCode ?? 0, body })); + }, + ); + req.on('error', reject); + req.end(); + }); +} + export async function rpc( endpoint: string, path: string, diff --git a/e2e/src/settings-hot.test.ts b/e2e/src/settings-hot.test.ts index be040958..9a69cbbf 100644 --- a/e2e/src/settings-hot.test.ts +++ b/e2e/src/settings-hot.test.ts @@ -1,7 +1,6 @@ -import http from 'node:http'; import { Dormice } from '@dormice/sdk'; import { describe, expect, inject, it } from 'vitest'; -import { door, listNodes, settled, until } from './helpers'; +import { door, listNodes, settled, spoofHost, until } from './helpers'; // The fleet-settings hot path for the two knobs that moved into the ledger // on 2026-07-26 — the S3 archive store and the sandbox domain — as they @@ -31,27 +30,12 @@ function sleep(seconds: number) { return new Promise((resolve) => setTimeout(resolve, seconds * 1000)); } -/** A GET at the daemon with a spoofed Host header (fetch refuses to set Host). */ -function throughProxy( +/** A GET with a spoofed Host header, at node A by default or at the door (helpers.ts spoofHost). */ +const throughProxy = ( host: string, path = '/', -): Promise<{ status: number; body: string }> { - const endpoint = new URL(inject('dormiceEndpoint')); - return new Promise((resolve, reject) => { - const req = http.request( - { host: endpoint.hostname, port: endpoint.port, path, headers: { host } }, - (res) => { - let body = ''; - res.on('data', (chunk) => { - body += chunk; - }); - res.on('end', () => resolve({ status: res.statusCode ?? 0, body })); - }, - ); - req.on('error', reject); - req.end(); - }); -} + via = inject('dormiceEndpoint'), +) => spoofHost(via, host, path); describe('the S3 archive store as a live ledger setting', () => { it('reports the store keyless and accepts a probed credential rotation', async () => { @@ -215,15 +199,30 @@ describe('the sandbox domain as a live ledger setting', () => { // The seed domain is out of force: its hosts are plain Fastify // traffic now, and the router answers 404, not the proxy. expect((await throughProxy(seededHost, '/hot')).status).toBe(404); + // The same two hosts at the door: the gateway keys on its own copy + // of the domain group (in force the moment the write returned) and + // forwards a sandbox host to node A, Host kept — so what the door + // says is what the node says, one hop later. + expect( + (await throughProxy(altHost, '/hot?x=1', door())).status, + ).toSatisfy(proxied); + expect((await throughProxy(seededHost, '/hot', door())).status).toBe( + 404, + ); } finally { await dormice.updateSettings({ sandboxDomain: seeded }); await settled(); } - // Restored: the seed domain proxies again, the alt one is gone. + // Restored: the seed domain proxies again, the alt one is gone — at + // both doors. expect((await throughProxy(seededHost, '/hot')).status).toSatisfy( proxied, ); expect((await throughProxy(altHost, '/hot')).status).toBe(404); + expect((await throughProxy(seededHost, '/hot', door())).status).toSatisfy( + proxied, + ); + expect((await throughProxy(altHost, '/hot', door())).status).toBe(404); } finally { await node().destroySandbox('settings-hot-domain'); } diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index db4ba12c..c9f85a8e 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -3,6 +3,7 @@ import http from 'node:http'; import net, { type AddressInfo } from 'node:net'; import { fileURLToPath } from 'node:url'; import { KeyedQueue } from '@dormice/server/keyed-queue'; +import { parseSandboxHost } from '@dormice/shared'; import { pino } from 'pino'; import { afterEach, describe, expect, it } from 'vitest'; import { buildGatewayApp } from './app'; @@ -17,6 +18,9 @@ import { checkInOf, type reading } from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); const TOKEN = 'fleet-token-fleet-token-fleet-token-fleet'; +/** The sandbox domain group the fake nodes' proxies key on — a canonical domain and one inbound alias. */ +const DOMAINS = ['sbx.test', 'alias.test']; +const DOMAIN = DOMAINS[0] as string; /** * A node as the gateway sees one: the daemon's wire for the handful of @@ -28,7 +32,13 @@ class FakeNode { string, { id: string; name: string; files: Map } >(); - readonly hits: Array<{ path: string; auth: string | undefined }> = []; + readonly hits: Array<{ + path: string; + auth: string | undefined; + /** Set on a hit the node's port proxy took (a sandbox Host) — and whether it was an upgrade. */ + host?: string | undefined; + upgrade?: boolean; + }> = []; creates = 0; endpoint = ''; /** How long a destroy takes to answer — a slow node holding the name's slot. */ @@ -43,6 +53,27 @@ class FakeNode { }); req.on('end', () => this.answer(req, res, text)); }); + // The daemon's port proxy takes upgrades for sandbox hosts (a dev + // server's WebSocket); this double's "container" echoes bytes after a + // bare 101. Anything else is cut, as the daemon cuts it. + this.server.on('upgrade', (req, socket) => { + socket.on('error', () => socket.destroy()); + const sandbox = parseSandboxHost(req.headers.host, DOMAINS); + this.hits.push({ + path: req.url ?? '/', + auth: req.headers.authorization, + host: req.headers.host, + upgrade: true, + }); + if (!sandbox || !this.byId(sandbox.sandboxId)) { + socket.end('HTTP/1.1 502 Bad Gateway\r\nconnection: close\r\n\r\n'); + return; + } + socket.write( + 'HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n', + ); + socket.pipe(socket); + }); } async start(): Promise { @@ -78,11 +109,33 @@ class FakeNode { const auth = req.headers.authorization ?? (req.headers['x-api-key'] as string | undefined); - this.hits.push({ path, auth }); const json = (status: number, body: unknown) => { res.writeHead(status, { 'content-type': 'application/json' }); res.end(JSON.stringify(body)); }; + // The daemon's port proxy as the gateway meets it: keyed on the Host + // it was sent, dialing the sandbox it holds — here an echo of what + // arrived (the fake executor's upstream does the same), or the + // daemon's own 502 for an id it lacks. Unauthenticated, as the real + // one is. + const sandboxHost = parseSandboxHost(req.headers.host, DOMAINS); + if (sandboxHost) { + this.hits.push({ path, auth, host: req.headers.host }); + const sandbox = this.byId(sandboxHost.sandboxId); + if (!sandbox) { + return json(502, { + message: `sandbox ${sandboxHost.sandboxId} not found`, + }); + } + return json(200, { + proxied: this.id, + host: req.headers.host, + port: sandboxHost.port, + url, + auth: auth ?? null, + }); + } + this.hits.push({ path, auth }); const body = text ? (JSON.parse(text) as Record) : {}; if (path.startsWith('/e2b/envd/')) { return json(200, { @@ -1088,3 +1141,195 @@ describe('the E2B faces', () => { ); }); }); + +/** + * A request at the door with a spoofed Host — wildcard-DNS traffic as the + * reverse proxy hands it over (fetch refuses to set Host, so node:http + * speaks). + */ +function viaHost( + h: Harness, + host: string, + path = '/', +): Promise<{ + status: number; + headers: http.IncomingHttpHeaders; + body: string; +}> { + const endpoint = new URL(h.endpoint); + return new Promise((resolve, reject) => { + const req = http.request( + { host: endpoint.hostname, port: endpoint.port, path, headers: { host } }, + (res) => { + let body = ''; + res.on('data', (chunk) => { + body += chunk; + }); + res.on('end', () => + resolve({ status: res.statusCode ?? 0, headers: res.headers, body }), + ); + }, + ); + req.on('error', reject); + req.end(); + }); +} + +/** + * An upgrade handshake at the door, raw: what came back before the socket + * closed — a 101 and the echo of `marco`, a refusal's status line, or + * nothing at all for a socket the gateway cut. + */ +function rawUpgrade(h: Harness, host: string): Promise { + const port = Number(new URL(h.endpoint).port); + return new Promise((resolve, reject) => { + let buffer = ''; + const socket = net.connect(port, '127.0.0.1', () => { + socket.write( + [ + 'GET /ws HTTP/1.1', + `Host: ${host}`, + 'Connection: Upgrade', + 'Upgrade: websocket', + '', + '', + ].join('\r\n'), + ); + }); + socket.on('data', (chunk) => { + buffer += chunk.toString('utf8'); + // Handshake done — the fake node's "container" echoes raw bytes back. + if (buffer.includes(' 101 ') && !buffer.includes('marco')) { + socket.write('marco'); + } + if (buffer.includes('marco')) socket.end(); + }); + socket.on('close', () => resolve(buffer)); + socket.on('error', reject); + setTimeout(() => reject(new Error('upgrade timed out')), 5_000); + }); +} + +describe('the sandbox port proxy face', () => { + it('a sandbox host is forwarded to the node holding the id — Host kept, no credential added, the answer relayed as it came; the path is the sandbox’s whatever it spells', async () => { + const h = await gateway(['a', 'b'], { DORMICE_SANDBOX_DOMAIN: DOMAIN }); + const created = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'web' })); + const host = `8000-${created.id}.${DOMAIN}`; + const res = await viaHost(h, host, '/hello?x=1'); + expect(res.status).toBe(200); + expect(JSON.parse(res.body)).toEqual({ + proxied: created.nodeId, + host, + port: 8000, + url: '/hello?x=1', + auth: null, + }); + // A native verb's path under a sandbox host is a path inside the + // sandbox: the door's own router never sees it, exactly as the + // daemon's proxy stands in front of its router. + const verb = await viaHost(h, host, '/acquireSandbox'); + expect(JSON.parse(verb.body)).toMatchObject({ + proxied: created.nodeId, + url: '/acquireSandbox', + }); + // The id was cached by the placement: the other node was never asked. + const elsewhere = h.nodes.find((n) => n.id !== created.nodeId); + expect(elsewhere?.hits.some((hit) => hit.host !== undefined)).toBe(false); + }); + + it('a sandbox built behind the gateway’s back is found by asking; an id on no node gets the daemon’s proxy answer, 502 { message } without CORS — except on the browser-direct file form, which carries it', async () => { + const h = await gateway(['a', 'b'], { DORMICE_SANDBOX_DOMAIN: DOMAIN }); + const staged = await stage(h.nodes[1] as FakeNode, 'behind'); + const res = await viaHost(h, `3000-${staged.id}.${DOMAIN}`, '/'); + expect(res.status).toBe(200); + expect(JSON.parse(res.body)).toMatchObject({ proxied: 'b', port: 3000 }); + + const nobody = await viaHost(h, `8000-${randomUUID()}.${DOMAIN}`, '/'); + expect(nobody.status).toBe(502); + expect(JSON.parse(nobody.body)).toEqual({ + message: expect.stringMatching(/on no node/), + }); + expect(nobody.headers['access-control-allow-origin']).toBeUndefined(); + + // 49983 /files is the signed-URL form a browser posts to directly; the + // daemon promises CORS on every answer to it, and the door keeps the + // promise on its own refusals. + const files = await viaHost( + h, + `49983-${randomUUID()}.${DOMAIN}`, + '/files?signature=x', + ); + expect(files.status).toBe(502); + expect(files.headers['access-control-allow-origin']).toBe('*'); + // For a sandbox that exists the form rides to its node whole — the + // carve-out onto the signed door is the node's own. + const carved = await viaHost( + h, + `49983-${staged.id}.${DOMAIN}`, + '/files?signature=x', + ); + expect(JSON.parse(carved.body)).toMatchObject({ + proxied: 'b', + port: 49983, + url: '/files?signature=x', + }); + }); + + it('WebSocket upgrades ride through to the node both ways, Host kept; an upgrade for an id on no node is refused with a status line; any other upgrade is cut', async () => { + const h = await gateway(['a'], { DORMICE_SANDBOX_DOMAIN: DOMAIN }); + const created = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'ws' })); + const host = `5173-${created.id}.${DOMAIN}`; + const echoed = await rawUpgrade(h, host); + expect(echoed).toContain(' 101 '); + expect(echoed).toContain('marco'); + expect( + h.nodes[0]?.hits.some((hit) => hit.upgrade === true && hit.host === host), + ).toBe(true); + + const refused = await rawUpgrade(h, `5173-${randomUUID()}.${DOMAIN}`); + expect(refused).toMatch(/^HTTP\/1\.1 502 /); + expect(refused).toContain('on no node'); + + // Not a sandbox host: nothing said, the socket closed — stock + // Fastify's behavior for an upgrade it never handles. + expect(await rawUpgrade(h, 'door.example')).toBe(''); + }); + + it('with no domain in force a sandbox host is plain traffic at the router; a domain written at the door engages on the very next request, an alias joins inbound, and clearing disengages — no restart, no check-in to wait for', async () => { + const h = await gateway(['a']); + const created = sandboxOf( + await rpc(h, '/acquireSandbox', { name: 'live' }), + ); + const host = `8000-${created.id}.${DOMAIN}`; + const off = await viaHost(h, host, '/x'); + expect(off.status).toBe(404); + expect(JSON.parse(off.body).message).toMatch(/^route GET \/x not found/); + + expect( + (await rpc(h, '/updateSettings', { sandboxDomain: DOMAIN })).status, + ).toBe(200); + expect(JSON.parse((await viaHost(h, host, '/x')).body)).toMatchObject({ + proxied: 'a', + url: '/x', + }); + + const alias = DOMAINS[1] as string; + expect( + (await rpc(h, '/updateSettings', { sandboxDomainAliases: [alias] })) + .status, + ).toBe(200); + expect( + JSON.parse((await viaHost(h, `8000-${created.id}.${alias}`, '/y')).body), + ).toMatchObject({ proxied: 'a', url: '/y' }); + + expect( + ( + await rpc(h, '/updateSettings', { + sandboxDomain: null, + sandboxDomainAliases: [], + }) + ).status, + ).toBe(200); + expect((await viaHost(h, host, '/x')).status).toBe(404); + }); +}); diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts index 17cbc15c..fb370fbf 100644 --- a/packages/gateway/src/app.ts +++ b/packages/gateway/src/app.ts @@ -1,5 +1,6 @@ import http from 'node:http'; import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import { sandboxDomainsInForce } from '@dormice/shared'; import fastifyCookie from '@fastify/cookie'; import fastify, { type FastifyError, type FastifyServerFactory } from 'fastify'; import { @@ -15,6 +16,7 @@ import { type Config, type ConfigSources, configSources } from './config'; import { getConsoleAccount } from './db/account'; import { isLiveApiKey, verifyApiKeyToken } from './db/api-keys'; import type { Db } from './db/db'; +import { readSettings } from './db/settings'; import { renderError } from './errors'; import type { Finder } from './find'; import type { Fleet } from './fleet'; @@ -119,12 +121,17 @@ export function buildGatewayApp({ typeof logger === 'boolean' ? pino({ enabled: logger }) : logger; const token = config.DORMICE_API_TOKEN; - // The face keyed on a header sits in front of Fastify, exactly as the + // The faces keyed on a header sit in front of Fastify, exactly as the // daemon's port proxy does (server/app.ts): refuse what is not an // origin-form target (classify.ts isOriginForm), triage the raw // request, hand the rest to Fastify. app.inject() bypasses the factory, - // so that face is exercised over real sockets only. + // so those faces are exercised over real sockets only. const raw = createRawFaces({ finder, token, log: loggerInstance }); + // The domain group the proxy face keys on is the gateway's own setting + // (the sandbox domain and its inbound aliases), read per request — a + // point read, and a console edit applies to the very next request here, + // as it does on a node once its copy has arrived. + const domains = () => sandboxDomainsInForce(readSettings(db)); const serverFactory: FastifyServerFactory = (handler) => { const server = http.createServer((req, res) => { if (!isOriginForm(req)) { @@ -135,12 +142,12 @@ export function buildGatewayApp({ }); return; } - const kind = classify(req); + const kind = classify(req, domains()); if (kind.face === 'fastify') handler(req, res); else raw.handleRequest(kind, req, res); }); - server.on('upgrade', (req, socket) => { - raw.handleUpgrade(classify(req), req, socket); + server.on('upgrade', (req, socket, head) => { + raw.handleUpgrade(classify(req, domains()), req, socket, head); }); // The gateway only relays; the node's own request timeout is the one // that should fire on a slow upload, not a second one in front of it. diff --git a/packages/gateway/src/classify.ts b/packages/gateway/src/classify.ts index 94f91adc..09458826 100644 --- a/packages/gateway/src/classify.ts +++ b/packages/gateway/src/classify.ts @@ -1,8 +1,17 @@ +import { parseSandboxHost } from '@dormice/shared'; + /** * Which face a raw request belongs to — the gateway's serverFactory - * triages every request before Fastify's router sees it, because one face - * is keyed on something Fastify cannot route on (a header, on any path). - * Pure, so the order of the tests is a fact here and nowhere else: + * triages every request before Fastify's router sees it, because three + * faces are keyed on something Fastify cannot route on (a header, on any + * path). Pure, so the order of the tests is a fact here and nowhere else: + * proxy Host `-.`, for a domain in the + * group in force (the gateway's own settings: the sandbox + * domain and its inbound aliases, handed in per request) — + * E2B's getHost() URL, the sandbox port proxy. Judged first, + * as on the daemon, where the proxy stands in front of the + * router: whatever path a sandbox host spells is the + * sandbox's, never a verb of the door. * envd /e2b/envd/* — E2B's in-sandbox API, keyed by the * E2b-Sandbox-Id header. * signedRoot exactly /files at the root — the bare signed-URL form, @@ -10,10 +19,9 @@ * without the node's signing secret. * fastify everything else — the native verbs, /e2b/api, /healthz, * the gateway's own verbs. - * The sandbox port proxy (Host `-.`) joins this list - * when the sandbox domain moves into the gateway's settings. */ export type Classified = + | { face: 'proxy'; port: number; sandboxId: string } | { face: 'envd' } | { face: 'signedRoot' } | { face: 'fastify' }; @@ -38,7 +46,12 @@ export function isOriginForm(req: { url?: string }): boolean { return (req.url ?? '').startsWith('/'); } -export function classify(req: { url?: string }): Classified { +export function classify( + req: { url?: string; headers: { host?: string } }, + domains: readonly string[], +): Classified { + const sandbox = parseSandboxHost(req.headers.host, domains); + if (sandbox) return { face: 'proxy', ...sandbox }; const url = req.url ?? ''; const q = url.indexOf('?'); const path = q === -1 ? url : url.slice(0, q); diff --git a/packages/gateway/src/errors.ts b/packages/gateway/src/errors.ts index b310db13..ceaca7a7 100644 --- a/packages/gateway/src/errors.ts +++ b/packages/gateway/src/errors.ts @@ -99,6 +99,8 @@ export async function relay( log: ErrorLog, step: () => Promise, unreachable: (error: UnreachableError) => RenderedError, + /** CORS on the 500 too; the browser-consumable faces (connect, and the proxy's browser-direct file form) need it to read any answer. */ + cors: boolean = dialect === 'connect', ): Promise { try { await step(); @@ -111,7 +113,7 @@ export async function relay( renderError(res, dialect, { status: 500, message: 'the gateway failed while forwarding — see its log', - cors: dialect === 'connect', + cors, }); } } diff --git a/packages/gateway/src/forward.ts b/packages/gateway/src/forward.ts index a867546d..4b4d694c 100644 --- a/packages/gateway/src/forward.ts +++ b/packages/gateway/src/forward.ts @@ -54,11 +54,12 @@ export interface ForwardOptions { /** The request body when Fastify already consumed the stream; omit to stream req itself. */ body?: Buffer | undefined; /** - * Keep the caller's Host header. Only a face keyed on the Host wants - * this (the sandbox port proxy, once it routes through the gateway). - * Everywhere else the Host names the gateway, and carrying it to a node - * whose Caddy binds that very domain gets a 308 to https instead of the - * daemon — so the default lets undici name the node's own endpoint. + * Keep the caller's Host header. Only the face keyed on the Host wants + * this (the sandbox port proxy, raw.ts: the node's own proxy keys on the + * same header). Everywhere else the Host names the gateway, and carrying + * it to a node whose Caddy binds that very domain gets a 308 to https + * instead of the daemon — so the default lets undici name the node's + * own endpoint. */ preserveHost?: boolean; } @@ -306,12 +307,12 @@ export function replay( } /** - * The upgrade path (sandbox WebSockets, once the port proxy routes - * through the gateway): the daemon's own replay (sandbox-proxy.ts - * handleUpgrade) — dial the node, write the request line and rawHeaders - * verbatim, then pipe both ways. Plain TCP: node endpoints are - * private-network http, and a TLS node endpoint is refused here rather - * than half-supported. + * The upgrade path (sandbox WebSockets through the proxy face, raw.ts): + * the daemon's own replay (sandbox-proxy.ts handleUpgrade) — dial the + * node, write the request line and rawHeaders verbatim (the Host among + * them, which the node's proxy keys on), then pipe both ways. Plain TCP: + * node endpoints are private-network http, and a TLS node endpoint is + * refused here rather than half-supported. */ export function forwardUpgrade( req: http.IncomingMessage, diff --git a/packages/gateway/src/raw.ts b/packages/gateway/src/raw.ts index a5611263..0e3f7239 100644 --- a/packages/gateway/src/raw.ts +++ b/packages/gateway/src/raw.ts @@ -1,10 +1,17 @@ -import type http from 'node:http'; +import http from 'node:http'; import type { Duplex } from 'node:stream'; +import { ENVD_PORT } from '@dormice/shared'; import type { Logger } from 'pino'; import type { Classified } from './classify'; -import { relay, renderError, sendPreflight } from './errors'; +import { + type Dialect, + type RenderedError, + relay, + renderError, + sendPreflight, +} from './errors'; import type { Finder, Found } from './find'; -import { forwardStream } from './forward'; +import { forwardStream, forwardUpgrade } from './forward'; export interface RawFacesDeps { finder: Finder; @@ -15,9 +22,22 @@ export interface RawFacesDeps { /** How long a caller refused with "a node did not answer" may wait before asking again — one check-in interval. */ export const RETRY_AFTER_SECONDS = 15; +type ProxyFace = Extract; + /** * The faces Fastify never sees — keyed on a header on any path, judged on * the raw request the serverFactory hands over: + * proxy the sandbox port proxy, keyed by the Host label (E2B's + * getHost() URL, classify.ts): the node holding the id is + * found and the request goes there whole — Host kept, since + * the node's own proxy keys on it and dials the container + * (and on 49983 /files its signed file door pins the sandbox + * by it), no credential added, since sandbox traffic is + * unauthenticated, as E2B's is: a preview URL exists to be + * opened by whoever it is shared with. The node wakes a + * frozen sandbox on traffic and answers "not listening" for + * a port nobody serves; the gateway adds only where it is. + * WebSocket upgrades ride the same way (forwardUpgrade). * envd E2B's in-sandbox API, keyed by E2b-Sandbox-Id; forwarded * with no credential change (the access token is the node's * own HMAC). Preflights are answered here: the node answers @@ -34,80 +54,101 @@ export const RETRY_AFTER_SECONDS = 15; * getHost URL, and the daemon's own proxy says only "not found" for one * it lacks. The detail an operator needs goes to the log; the * authenticated faces and listNodes name nodes freely. + * + * Each face's refusals wear that face's dialect (errors.ts): connect for + * envd, and for the proxy the daemon's proxy answer — { message }, 502 — + * so a caller reads one shape from either door. */ export function createRawFaces({ finder, token, log }: RawFacesDeps) { /** The one sentence per finding for a sandbox id on these faces — generic on purpose (above). */ - function refusal( - res: http.ServerResponse, + function sentence( id: string, found: Exclude, - ): void { + ): RenderedError { switch (found.kind) { case 'conflict': - renderError(res, 'connect', { + return { status: 502, connectCode: 'unavailable', message: `sandbox "${id}" is held by more than one node — routing resumes once an operator destroys one copy (listNodes and the gateway log name them)`, - cors: true, - }); - return; + }; case 'none': - renderError(res, 'connect', { + return { status: 502, connectCode: 'unavailable', message: `sandbox "${id}" is on no node — it may have been destroyed`, - cors: true, - }); - return; + }; case 'unsure': - renderError(res, 'connect', { + return { status: 503, connectCode: 'unavailable', message: `sandbox "${id}": a node did not answer, so its whereabouts cannot be settled — retry`, - cors: true, retryAfterSeconds: RETRY_AFTER_SECONDS, - }); - return; + }; } } /** - * Finds the id and forwards, or answers the refusal — the one path - * every keyed face takes. Nothing here may throw past this point: no - * framework stands behind a raw face, so an escaped rejection would be - * the process's, not the request's (errors.ts relay answers instead). + * Finds the id, or answers the refusal and returns null. The lookup + * itself failing (not a node's silence — the gateway's own bug) is a 500 + * that sends the operator to the log. */ - async function route( - req: http.IncomingMessage, + async function locate( res: http.ServerResponse, id: string, - ): Promise { + dialect: Dialect, + cors: boolean, + face: string, + ): Promise> { let found: Found; try { found = await finder.byId(id); } catch (error) { - log.error(error, 'envd face: the lookup itself failed'); - renderError(res, 'connect', { + log.error(error, `${face} face: the lookup itself failed`); + renderError(res, dialect, { status: 500, - connectCode: 'internal', message: 'the gateway failed while locating the sandbox — see its log', - cors: true, + cors, }); - return; + throw new Refused(); } if (found.kind !== 'one') { - refusal(res, id, found); + renderError(res, dialect, { ...sentence(id, found), cors }); + throw new Refused(); + } + return found; + } + + /** + * Finds the id and forwards, or answers the refusal — the one path + * every keyed face takes. Nothing here may throw past this point: no + * framework stands behind a raw face, so an escaped rejection would be + * the process's, not the request's (errors.ts relay answers instead). + */ + async function route( + req: http.IncomingMessage, + res: http.ServerResponse, + id: string, + face: 'envd' | 'proxy', + cors: boolean, + ): Promise { + const dialect: Dialect = face === 'envd' ? 'connect' : 'native'; + let found: Extract; + try { + found = await locate(res, id, dialect, cors, face); + } catch { return; } const node = found.node; await relay( res, - 'connect', + dialect, log, async () => { await forwardStream(req, res, { target: { endpoint: node.endpoint, token }, credential: 'none', + preserveHost: face === 'proxy', }); }, (error) => { @@ -118,18 +159,55 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { endpoint: node.endpoint, why: error.why, }, - "envd face: the sandbox's node did not answer", + `${face} face: the sandbox's node did not answer`, ); return { status: 502, connectCode: 'unavailable', message: `sandbox "${id}": its node did not answer (${error.why}) — retry`, - cors: true, + cors, }; }, + cors, ); } + /** + * The upgrade half of the proxy face: found the same way, then the + * daemon's own replay (forwardUpgrade). A refusal is a bare status line + * on the socket — there is no response object on an upgrade — in the + * same words as the request half's. + */ + async function upgrade( + kind: ProxyFace, + req: http.IncomingMessage, + socket: Duplex, + head: Buffer, + ): Promise { + let found: Found; + try { + found = await finder.byId(kind.sandboxId); + } catch (error) { + log.error(error, 'proxy face: the lookup itself failed (upgrade)'); + refuseUpgrade(socket, { + status: 500, + message: 'the gateway failed while locating the sandbox — see its log', + }); + return; + } + // The client left while its sandbox was being found (a lookup round + // is up to two seconds): nothing to dial the node for. + if (socket.destroyed) return; + if (found.kind !== 'one') { + refuseUpgrade(socket, sentence(kind.sandboxId, found)); + return; + } + forwardUpgrade(req, socket, head, { + endpoint: found.node.endpoint, + token, + }); + } + return { handleRequest( kind: Exclude, @@ -137,6 +215,20 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { res: http.ServerResponse, ): void { switch (kind.face) { + case 'proxy': { + // The browser-direct file form (49983 /files, the daemon's + // signed door) promises CORS on every answer, refusals included, + // or the browser could not read them; the daemon's proxy answers + // carry none, and neither do the gateway's for any other host. + void route( + req, + res, + kind.sandboxId, + 'proxy', + browserDirect(kind, req), + ); + return; + } case 'envd': { if (req.method === 'OPTIONS') { sendPreflight(req, res); @@ -153,7 +245,7 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { }); return; } - void route(req, res, id); + void route(req, res, id, 'envd', true); return; } case 'signedRoot': { @@ -174,9 +266,10 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { }, handleUpgrade( - _kind: Classified, - _req: http.IncomingMessage, + kind: Classified, + req: http.IncomingMessage, socket: Duplex, + head: Buffer, ): void { // First, before anything else: http.Server drops its own error // listener from the socket before emitting 'upgrade', so a client @@ -184,9 +277,38 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { // whole gateway down — from an unauthenticated face (the daemon's // sandbox-proxy.ts handleUpgrade has the same first line). socket.on('error', () => socket.destroy()); - // No face of this build takes upgrades: sandbox WebSockets ride the - // port proxy, which joins the gateway with the sandbox domain. - socket.destroy(); + // Only the proxy face takes upgrades: sandbox WebSockets (a dev + // server's HMR, a notebook). Everything else is cut, exactly as + // stock Fastify, which never handles upgrades, would. + if (kind.face !== 'proxy') { + socket.destroy(); + return; + } + void upgrade(kind, req, socket, head); }, }; } + +/** Thrown inside route() once the refusal is on the wire — the signal to stop, never seen outside. */ +class Refused extends Error {} + +/** Is this the browser-postable signed-URL form, `49983-./files`? Path-only, query ignored — the daemon's own carve-out test. */ +function browserDirect(kind: ProxyFace, req: http.IncomingMessage): boolean { + if (kind.port !== ENVD_PORT) return false; + const url = req.url ?? ''; + const q = url.indexOf('?'); + return (q === -1 ? url : url.slice(0, q)) === '/files'; +} + +/** A refusal on an upgrade: one status line and a JSON body, before any handshake was replayed (forwardUpgrade's own refusals have the same shape). */ +function refuseUpgrade(socket: Duplex, error: RenderedError): void { + if (socket.destroyed || socket.writableEnded) return; + const body = JSON.stringify({ message: error.message }); + const retry = + error.retryAfterSeconds === undefined + ? '' + : `retry-after: ${error.retryAfterSeconds}\r\n`; + socket.end( + `HTTP/1.1 ${error.status} ${http.STATUS_CODES[error.status] ?? ''}\r\ncontent-type: application/json\r\ncontent-length: ${Buffer.byteLength(body)}\r\n${retry}connection: close\r\n\r\n${body}`, + ); +} diff --git a/packages/server/src/e2b/signed-files.ts b/packages/server/src/e2b/signed-files.ts index d70dc6e6..f3a2d53c 100644 --- a/packages/server/src/e2b/signed-files.ts +++ b/packages/server/src/e2b/signed-files.ts @@ -1,8 +1,8 @@ +import { parseSandboxHost, sandboxDomainsInForce } from '@dormice/shared'; import multipart from '@fastify/multipart'; import type { FastifyRequest } from 'fastify'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { readRuntimeSettings } from '../db/settings'; -import { parseSandboxHost, sandboxDomainsInForce } from '../sandbox-proxy'; import { allowCorsOrigin, sendPreflight } from './cors'; import type { E2bDeps } from './deps'; import { serveFileDownload, serveFileUpload } from './envd/files'; diff --git a/packages/server/src/sandbox-proxy.test.ts b/packages/server/src/sandbox-proxy.test.ts index 21f3c80a..793c90c7 100644 --- a/packages/server/src/sandbox-proxy.test.ts +++ b/packages/server/src/sandbox-proxy.test.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import http from 'node:http'; import net from 'node:net'; import { fileURLToPath } from 'node:url'; +import { parseSandboxHost } from '@dormice/shared'; import { afterEach, describe, expect, it } from 'vitest'; import { buildApp } from './app'; import { loadConfig } from './config'; @@ -11,7 +12,6 @@ import { getOrCreateSigningSecret } from './db/secrets'; import { mintEnvdToken } from './e2b/protocol'; import { FakeExecutor } from './executor/fake'; import { KeyedQueue } from './keyed-queue'; -import { parseSandboxHost } from './sandbox-proxy'; import { scanOnce } from './scanner'; import { configureNode } from './testing'; @@ -131,13 +131,15 @@ describe('sandbox port proxy', () => { return created.sandboxID; } + // The parser lives in @dormice/shared since the gateway's proxy face + // reads the same header; its unit exam stays beside the proxy it serves. it('parses sandbox hosts and nothing else', () => { const id = '01234567-89ab-cdef-0123-456789abcdef'; expect(parseSandboxHost(`8000-${id}.${DOMAIN}`, [DOMAIN])).toEqual({ port: 8000, sandboxId: id, }); - // The header's own :port tail is the daemon's port, not the sandbox's. + // The header's own :port tail is the door's port, not the sandbox's. expect(parseSandboxHost(`8000-${id}.${DOMAIN}:3676`, [DOMAIN])).toEqual({ port: 8000, sandboxId: id, diff --git a/packages/server/src/sandbox-proxy.ts b/packages/server/src/sandbox-proxy.ts index c237e809..07b24232 100644 --- a/packages/server/src/sandbox-proxy.ts +++ b/packages/server/src/sandbox-proxy.ts @@ -2,7 +2,11 @@ import type http from 'node:http'; import { request as httpRequest } from 'node:http'; import net from 'node:net'; import type { Duplex } from 'node:stream'; -import type { RuntimeSettings } from '@dormice/shared'; +import { + ENVD_PORT, + parseSandboxHost, + sandboxDomainsInForce, +} from '@dormice/shared'; import type { Db } from './db/db'; import { findById, touch } from './db/ledger'; import type { SandboxRow } from './db/schema'; @@ -24,7 +28,9 @@ import { wakeSandbox } from './lifecycle'; * The daemon still binds 127.0.0.1 only: public TLS and wildcard DNS are * the operator's reverse proxy's job (Caddy with `flush_interval -1`, * measured on the predecessor system — without it streaming responses - * buffer into one lump). + * buffer into one lump), and since 2026-09-14 that proxy points at the + * gateway, whose proxy face forwards a sandbox host to the node holding + * the sandbox — this code, one hop later. * * Sandbox traffic is deliberately unauthenticated, like E2B's: a preview * URL exists to be opened by whoever it is shared with. What the proxy @@ -32,16 +38,12 @@ import { wakeSandbox } from './lifecycle'; */ /** - * envd's fixed port in E2B's URL grammar: `49983-.` - * reaches the sandbox's envd, never a user process — it is how the SDK's - * uploadUrl/downloadUrl become browser-postable URLs. Dormice runs no envd - * inside the container (the daemon plays that role), so /files on this - * port is carved out of the proxy and lands on the daemon's signed-URL - * file door, with the Host label pinning which sandbox the signature must - * speak for. Every other path keeps the honest proxy answer: nothing - * listens on 49983 inside the sandbox. + * The Host grammar — parseSandboxHost, the domain group in force, + * ENVD_PORT — lives in @dormice/shared (sandbox-host.ts): the gateway's + * proxy face reads the same header to find the node holding the sandbox + * and forwards the request here whole, Host kept, so a host names the + * same sandbox at both doors. */ -export const ENVD_PORT = 49983; /** Path-only match for the carve-out: exactly /files, query ignored. */ function isEnvdFilesRequest(req: http.IncomingMessage): boolean { @@ -50,54 +52,6 @@ function isEnvdFilesRequest(req: http.IncomingMessage): boolean { return (q === -1 ? url : url.slice(0, q)) === '/files'; } -/** - * The domain group inbound matching runs against: the canonical domain - * first, then the inbound-only aliases; empty when the feature is off - * (sandboxDomain null). The one adjudication of "off = never a match" — - * the proxy's per-request getter and the signed-URL host pin both call - * this instead of deciding it themselves. - */ -export function sandboxDomainsInForce( - settings: Pick, -): string[] { - return settings.sandboxDomain - ? [settings.sandboxDomain, ...settings.sandboxDomainAliases] - : []; -} - -/** - * Host header -> { port, sandboxId }, or null when it is not sandbox - * traffic (then the request belongs to Fastify). The port suffix of the - * header itself (`:3676`) is not the sandbox port — the label carries that. - * - * Every domain gets a full parse, never first-suffix-wins: an alias may be - * a subdomain of another listed domain, and a host under it would suffix- - * match the shorter domain first with a dotted label the regex refuses. - */ -export function parseSandboxHost( - hostHeader: string | undefined, - domains: readonly string[], -): { port: number; sandboxId: string } | null { - if (!hostHeader) return null; - const host = hostHeader.replace(/:\d+$/, '').toLowerCase(); - for (const domain of domains) { - // Empty means "no domain in force" — never a match. Explicit, not left - // to the suffix check: `.` + '' would make every dotted host a candidate. - if (!domain) continue; - const suffix = `.${domain.toLowerCase()}`; - if (!host.endsWith(suffix)) continue; - const label = host.slice(0, -suffix.length); - const match = label.match( - /^(\d{1,5})-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/, - ); - if (!match) continue; - const port = Number(match[1]); - if (port < 1 || port > 65535) continue; - return { port, sandboxId: match[2] as string }; - } - return null; -} - export interface SandboxProxyDeps { db: Db; executor: Executor; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index b2fed36c..604cb643 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -15,6 +15,7 @@ export * from './metrics'; export * from './policy'; export * from './rebuild'; export * from './sandbox'; +export * from './sandbox-host'; export * from './settings'; export * from './spec'; export * from './states'; diff --git a/packages/shared/src/sandbox-host.ts b/packages/shared/src/sandbox-host.ts new file mode 100644 index 00000000..460c9a97 --- /dev/null +++ b/packages/shared/src/sandbox-host.ts @@ -0,0 +1,75 @@ +import type { RuntimeSettings } from './settings'; + +/** + * E2B's URL grammar for a sandbox's port, shared by the two doors that key + * on it: the Host header `-.` — what the SDK's + * getHost() builds from the `domain` field a create answers with. The + * daemon's sandbox port proxy (server/sandbox-proxy.ts) reads it to dial + * the container; the gateway's proxy face (gateway/classify.ts, raw.ts) + * reads it to find the node holding the sandbox and forward the request + * there whole, Host kept. One parser, so a host names the same sandbox at + * both doors. + */ + +/** + * envd's fixed port in E2B's URL grammar: `49983-.` + * reaches the sandbox's envd, never a user process — it is how the SDK's + * uploadUrl/downloadUrl become browser-postable URLs. Dormice runs no envd + * inside the container (the daemon plays that role), so /files on this + * port is carved out of the daemon's proxy and lands on its signed-URL + * file door, with the Host label pinning which sandbox the signature must + * speak for (server/e2b/signed-files.ts). Every other path keeps the + * honest proxy answer: nothing listens on 49983 inside the sandbox. The + * gateway forwards this form like any other sandbox host; the node does + * the carving. + */ +export const ENVD_PORT = 49983; + +/** + * The domain group inbound matching runs against: the canonical domain + * first, then the inbound-only aliases; empty when the feature is off + * (sandboxDomain null). The one adjudication of "off = never a match" — + * both proxies' per-request getters and the signed-URL host pin call this + * instead of deciding it themselves. + */ +export function sandboxDomainsInForce( + settings: Pick, +): string[] { + return settings.sandboxDomain + ? [settings.sandboxDomain, ...settings.sandboxDomainAliases] + : []; +} + +/** + * Host header -> { port, sandboxId }, or null when it is not sandbox + * traffic (then the request belongs to the router). The port suffix of the + * header itself (`:3676`, `:3677`) is the door's port, not the sandbox's — + * the label carries that. + * + * Every domain gets a full parse, never first-suffix-wins: an alias may be + * a subdomain of another listed domain, and a host under it would suffix- + * match the shorter domain first with a dotted label the regex refuses. + */ +export function parseSandboxHost( + hostHeader: string | undefined, + domains: readonly string[], +): { port: number; sandboxId: string } | null { + if (!hostHeader) return null; + const host = hostHeader.replace(/:\d+$/, '').toLowerCase(); + for (const domain of domains) { + // Empty means "no domain in force" — never a match. Explicit, not left + // to the suffix check: `.` + '' would make every dotted host a candidate. + if (!domain) continue; + const suffix = `.${domain.toLowerCase()}`; + if (!host.endsWith(suffix)) continue; + const label = host.slice(0, -suffix.length); + const match = label.match( + /^(\d{1,5})-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/, + ); + if (!match) continue; + const port = Number(match[1]); + if (port < 1 || port > 65535) continue; + return { port, sandboxId: match[2] as string }; + } + return null; +} diff --git a/website/content/docs/ports.mdx b/website/content/docs/ports.mdx index 7ab68286..86532696 100644 --- a/website/content/docs/ports.mdx +++ b/website/content/docs/ports.mdx @@ -5,8 +5,9 @@ description: Learn how to reach a server running inside a sandbox from the outsi Anything listening on a port inside a sandbox — a dev server, a notebook, an agent's UI — can be reached from outside through the -daemon's sandbox proxy: each sandbox port gets its own preview URL like -`8000-.sbx.example.com`. +sandbox port proxy: each sandbox port gets its own preview URL like +`8000-.sbx.example.com`. The gateway serves these URLs and +forwards each one to the node running that sandbox. Setting this up takes three ingredients, each explained in [Core concepts](/docs/core-concepts#reverse-proxies-domains-and-https): a @@ -54,17 +55,18 @@ path and operation — never the domain. ## 2. Point DNS and a reverse proxy at the host -The daemon itself only binds `127.0.0.1`. The internet-facing half is +The gateway itself only binds `127.0.0.1`. The internet-facing half is one wildcard DNS record plus a reverse proxy that terminates TLS — pointed -at the daemon, which serves the sandbox port proxy (the gateway takes -this face over in a later step): +at the gateway (`3677`), which reads the sandbox id off the `Host`, +forwards the request with the `Host` intact to the node running that +sandbox, and the node's own proxy dials the port: ```text *.sbx.example.com → your host # Caddyfile *.sbx.example.com { - reverse_proxy 127.0.0.1:3676 { + reverse_proxy 127.0.0.1:3677 { flush_interval -1 } } @@ -88,10 +90,11 @@ const host = sbx.getHost(8000); // '8000-.sbx.example.com' await fetch(`https://${host}/`); ``` -Requests arriving with such a `Host` header are proxied straight into -the sandbox — headers untouched, WebSocket upgrades passed through both -ways. If nothing listens on the port, the proxy answers a 502 naming the -port, not a hang. +Requests arriving with such a `Host` header — at the gateway, or at a +daemon directly — are proxied straight into the sandbox: headers +untouched, WebSocket upgrades passed through both ways. If nothing +listens on the port, the proxy answers a 502 naming the port, not a +hang; a sandbox id no node holds is a 502 too. With no sandbox domain configured, the feature is off: responses carry no domain and the SDK has nothing to build URLs from. @@ -109,5 +112,5 @@ Preview URLs are unauthenticated — the same call E2B made, because these URLs exist to be opened in browsers and shared. The sandbox ID in the hostname is unguessable (a UUID), but that is obscurity, not access control: **treat a preview URL as public while the port is open.** -Everything else on the daemon — native API, E2B API, console — stays +Everything else at both doors — native API, E2B API, console — stays authenticated; only sandbox port traffic passes free. From 387be3c74ab34515338506e4cb80ff63704ab3e2 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 20:50:55 +0800 Subject: [PATCH 43/89] The browser-direct file form's preflight is answered at the door, an absolute-form upgrade is refused as a request is, and finding a sandbox for a keyed face is one value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy face wore CORS on its own refusals to the 49983 /files form so a browser could read them — but forwarded that form's preflight to the node, and for an id on no node the preflight itself earned the 502. No browser reads a refused preflight: it drops the real request, and the readable refusal was never seen (measured on the test machine, 2026-09-14). The gateway now answers the preflight for that form, in the node's own shape (its cors.ts), as the envd face already did; every other sandbox host's OPTIONS is still the sandbox's own. The request path refuses a target that is not origin-form, so the gateway and the node read the same request; the upgrade path did not, and an absolute-form handshake on a sandbox host was replayed into the sandbox verbatim. handleUpgrade now judges the same rule first, in its own medium — a status line — with the one sentence both paths share (ORIGIN_FORM_REQUIRED). locate() said it returned null and threw a Refused instead, and the upgrade half carried its own copy of find-or-refuse. It is one function returning a value now — the node, or the refusal — and each half writes the refusal where it can: on the response, or on the socket. "Exactly /files, query ignored" was written three times (the gateway's classify and raw, the daemon's proxy); with the Host grammar already in @dormice/shared, the browser-direct form is judged there too (isFilesPath, isEnvdFilesForm), so what is browser-direct at one door is browser-direct at the other. Exams: the gateway's proxy-face exams cover the preflight (unknown id 204, known id without a node hit, an ordinary port's OPTIONS forwarded) and the absolute-form handshake (400, never replayed). The E2B echo exam spells the seed domain and polls each door until its proxy answers: it shares its node with settings-hot, whose domain switch reaches the gateway's copy at once and the node's a check-in later, and a 404 in that window was seen in a local run. --- e2e/src/e2b.test.ts | 21 +++-- packages/gateway/src/app.test.ts | 74 +++++++++++++-- packages/gateway/src/app.ts | 8 +- packages/gateway/src/classify.ts | 13 +-- packages/gateway/src/raw.ts | 129 ++++++++++++++------------- packages/server/src/sandbox-proxy.ts | 27 +++--- packages/shared/src/sandbox-host.ts | 29 +++++- 7 files changed, 198 insertions(+), 103 deletions(-) diff --git a/e2e/src/e2b.test.ts b/e2e/src/e2b.test.ts index 5209ffa2..d7570696 100644 --- a/e2e/src/e2b.test.ts +++ b/e2e/src/e2b.test.ts @@ -745,9 +745,21 @@ describe.runIf(process.env.DORMICE_EXECUTOR !== 'docker')( it('a Host-routed request lands inside the sandbox and echoes back', async () => { const sbx = await Sandbox.create(connection()); try { - const host = sbx.getHost(8000); - const res = await throughProxy(host, '/hello?from=e2e'); - expect(res.status).toBe(200); + // The seed domain, spelled out rather than read off getHost(): + // settings-hot.test.ts moves the shared domain for a few seconds + // at a time, a create inside that window is told the other domain, + // and the seed is the steady state that comes back (the getHost + // exam above covers the assembly). Each door is asked until its + // proxy answers: a 404 mid-switch is the router speaking while the + // edit travels — the gateway keys on its own copy at once, the + // node on its next check-in (seen in a local run, 2026-09-14). + const host = `8000-${sbx.sandboxId}.sbx.dormice.test`; + const proxied = (via: string, path: string) => + poll(async () => { + const res = await throughProxy(host, path, via); + return res.status === 200 ? res : undefined; + }); + const res = await proxied(inject('dormiceEndpoint'), '/hello?from=e2e'); const echo = JSON.parse(res.body); expect(echo.sandboxId).toBe(sbx.sandboxId); expect(echo.path).toBe('/hello?from=e2e'); @@ -756,8 +768,7 @@ describe.runIf(process.env.DORMICE_EXECUTOR !== 'docker')( // finds the node holding it and forwards, Host kept — getHost() // works through the fleet's one door, which is where the wildcard // DNS points in production. - const viaDoor = await throughProxy(host, '/hello?from=door', door()); - expect(viaDoor.status).toBe(200); + const viaDoor = await proxied(door(), '/hello?from=door'); expect(JSON.parse(viaDoor.body)).toMatchObject({ sandboxId: sbx.sandboxId, path: '/hello?from=door', diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index c9f85a8e..fd1b4a56 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -1145,12 +1145,14 @@ describe('the E2B faces', () => { /** * A request at the door with a spoofed Host — wildcard-DNS traffic as the * reverse proxy hands it over (fetch refuses to set Host, so node:http - * speaks). + * speaks). `method` and `headers` for the preflight shapes. */ function viaHost( h: Harness, host: string, path = '/', + method = 'GET', + headers: Record = {}, ): Promise<{ status: number; headers: http.IncomingHttpHeaders; @@ -1159,7 +1161,13 @@ function viaHost( const endpoint = new URL(h.endpoint); return new Promise((resolve, reject) => { const req = http.request( - { host: endpoint.hostname, port: endpoint.port, path, headers: { host } }, + { + host: endpoint.hostname, + port: endpoint.port, + path, + method, + headers: { ...headers, host }, + }, (res) => { let body = ''; res.on('data', (chunk) => { @@ -1178,16 +1186,17 @@ function viaHost( /** * An upgrade handshake at the door, raw: what came back before the socket * closed — a 101 and the echo of `marco`, a refusal's status line, or - * nothing at all for a socket the gateway cut. + * nothing at all for a socket the gateway cut. `target` is the request + * target as written on the request line (origin-form by default). */ -function rawUpgrade(h: Harness, host: string): Promise { +function rawUpgrade(h: Harness, host: string, target = '/ws'): Promise { const port = Number(new URL(h.endpoint).port); return new Promise((resolve, reject) => { let buffer = ''; const socket = net.connect(port, '127.0.0.1', () => { socket.write( [ - 'GET /ws HTTP/1.1', + `GET ${target} HTTP/1.1`, `Host: ${host}`, 'Connection: Upgrade', 'Upgrade: websocket', @@ -1237,7 +1246,7 @@ describe('the sandbox port proxy face', () => { expect(elsewhere?.hits.some((hit) => hit.host !== undefined)).toBe(false); }); - it('a sandbox built behind the gateway’s back is found by asking; an id on no node gets the daemon’s proxy answer, 502 { message } without CORS — except on the browser-direct file form, which carries it', async () => { + it('a sandbox built behind the gateway’s back is found by asking; an id on no node gets the daemon’s proxy answer, 502 { message } without CORS — except on the browser-direct file form, which carries it and whose preflight the door answers itself', async () => { const h = await gateway(['a', 'b'], { DORMICE_SANDBOX_DOMAIN: DOMAIN }); const staged = await stage(h.nodes[1] as FakeNode, 'behind'); const res = await viaHost(h, `3000-${staged.id}.${DOMAIN}`, '/'); @@ -1261,6 +1270,47 @@ describe('the sandbox port proxy face', () => { ); expect(files.status).toBe(502); expect(files.headers['access-control-allow-origin']).toBe('*'); + // Its preflight is the door's own answer, in the node's shape, whether + // or not the id is anywhere: a browser sends nothing until the + // preflight passes, so a 502 here would have hidden the refusal above. + const preflight = await viaHost( + h, + `49983-${randomUUID()}.${DOMAIN}`, + '/files', + 'OPTIONS', + { 'access-control-request-headers': 'content-type' }, + ); + expect(preflight.status).toBe(204); + expect(preflight.headers['access-control-allow-origin']).toBe('*'); + expect(preflight.headers['access-control-allow-methods']).toBe( + 'GET, POST, OPTIONS', + ); + expect(preflight.headers['access-control-allow-headers']).toBe( + 'content-type', + ); + // A known id too — and the node is not asked for it. + const nodeB = h.nodes[1] as FakeNode; + const hitsBefore = nodeB.hits.length; + const known = await viaHost( + h, + `49983-${staged.id}.${DOMAIN}`, + '/files', + 'OPTIONS', + ); + expect(known.status).toBe(204); + expect(nodeB.hits.length).toBe(hitsBefore); + // Only that form: an OPTIONS on any other port is the sandbox's own + // and rides to it like any request. + const appOptions = await viaHost( + h, + `3000-${staged.id}.${DOMAIN}`, + '/api', + 'OPTIONS', + ); + expect(JSON.parse(appOptions.body)).toMatchObject({ + proxied: 'b', + url: '/api', + }); // For a sandbox that exists the form rides to its node whole — the // carve-out onto the signed door is the node's own. const carved = await viaHost( @@ -1275,7 +1325,7 @@ describe('the sandbox port proxy face', () => { }); }); - it('WebSocket upgrades ride through to the node both ways, Host kept; an upgrade for an id on no node is refused with a status line; any other upgrade is cut', async () => { + it('WebSocket upgrades ride through to the node both ways, Host kept; an upgrade for an id on no node is refused with a status line, an absolute-form handshake with a 400; any other upgrade is cut', async () => { const h = await gateway(['a'], { DORMICE_SANDBOX_DOMAIN: DOMAIN }); const created = sandboxOf(await rpc(h, '/acquireSandbox', { name: 'ws' })); const host = `5173-${created.id}.${DOMAIN}`; @@ -1290,6 +1340,16 @@ describe('the sandbox port proxy face', () => { expect(refused).toMatch(/^HTTP\/1\.1 502 /); expect(refused).toContain('on no node'); + // The request path's first rule holds on this path too: an absolute- + // form handshake is refused in the same words, as a status line, and + // is never replayed into the sandbox. + const nodeA = h.nodes[0] as FakeNode; + const hitsBefore = nodeA.hits.length; + const absolute = await rawUpgrade(h, host, `http://${host}/ws`); + expect(absolute).toMatch(/^HTTP\/1\.1 400 /); + expect(absolute).toContain('origin-form'); + expect(nodeA.hits.length).toBe(hitsBefore); + // Not a sandbox host: nothing said, the socket closed — stock // Fastify's behavior for an upgrade it never handles. expect(await rawUpgrade(h, 'door.example')).toBe(''); diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts index fb370fbf..ce34de09 100644 --- a/packages/gateway/src/app.ts +++ b/packages/gateway/src/app.ts @@ -11,7 +11,7 @@ import { import { type Logger, pino } from 'pino'; import { z } from 'zod'; import { requireAdminAuth, requireApiAuth, tokensEqual } from './auth'; -import { classify, isOriginForm } from './classify'; +import { classify, isOriginForm, ORIGIN_FORM_REQUIRED } from './classify'; import { type Config, type ConfigSources, configSources } from './config'; import { getConsoleAccount } from './db/account'; import { isLiveApiKey, verifyApiKeyToken } from './db/api-keys'; @@ -137,8 +137,7 @@ export function buildGatewayApp({ if (!isOriginForm(req)) { renderError(res, 'native', { status: 400, - message: - 'request target must be origin-form (a path starting with "/")', + message: ORIGIN_FORM_REQUIRED, }); return; } @@ -146,6 +145,9 @@ export function buildGatewayApp({ if (kind.face === 'fastify') handler(req, res); else raw.handleRequest(kind, req, res); }); + // Upgrades are judged by the same rules (origin form, then the face), + // in raw.ts: a refusal there is a status line on the socket, which + // that module writes. server.on('upgrade', (req, socket, head) => { raw.handleUpgrade(classify(req, domains()), req, socket, head); }); diff --git a/packages/gateway/src/classify.ts b/packages/gateway/src/classify.ts index 09458826..ac6d3e0a 100644 --- a/packages/gateway/src/classify.ts +++ b/packages/gateway/src/classify.ts @@ -1,4 +1,4 @@ -import { parseSandboxHost } from '@dormice/shared'; +import { isFilesPath, parseSandboxHost } from '@dormice/shared'; /** * Which face a raw request belongs to — the gateway's serverFactory @@ -46,16 +46,17 @@ export function isOriginForm(req: { url?: string }): boolean { return (req.url ?? '').startsWith('/'); } +/** The one sentence a non-origin-form target is refused with: a 400 on a request (app.ts), a status line on an upgrade (raw.ts). */ +export const ORIGIN_FORM_REQUIRED = + 'request target must be origin-form (a path starting with "/")'; + export function classify( req: { url?: string; headers: { host?: string } }, domains: readonly string[], ): Classified { const sandbox = parseSandboxHost(req.headers.host, domains); if (sandbox) return { face: 'proxy', ...sandbox }; - const url = req.url ?? ''; - const q = url.indexOf('?'); - const path = q === -1 ? url : url.slice(0, q); - if (url.startsWith('/e2b/envd/')) return { face: 'envd' }; - if (path === '/files') return { face: 'signedRoot' }; + if ((req.url ?? '').startsWith('/e2b/envd/')) return { face: 'envd' }; + if (isFilesPath(req.url)) return { face: 'signedRoot' }; return { face: 'fastify' }; } diff --git a/packages/gateway/src/raw.ts b/packages/gateway/src/raw.ts index 0e3f7239..a2fd18ff 100644 --- a/packages/gateway/src/raw.ts +++ b/packages/gateway/src/raw.ts @@ -1,8 +1,12 @@ import http from 'node:http'; import type { Duplex } from 'node:stream'; -import { ENVD_PORT } from '@dormice/shared'; +import { isEnvdFilesForm } from '@dormice/shared'; import type { Logger } from 'pino'; -import type { Classified } from './classify'; +import { + type Classified, + isOriginForm, + ORIGIN_FORM_REQUIRED, +} from './classify'; import { type Dialect, type RenderedError, @@ -11,6 +15,7 @@ import { sendPreflight, } from './errors'; import type { Finder, Found } from './find'; +import type { NodeState } from './fleet'; import { forwardStream, forwardUpgrade } from './forward'; export interface RawFacesDeps { @@ -24,6 +29,9 @@ export const RETRY_AFTER_SECONDS = 15; type ProxyFace = Extract; +/** What a keyed face learned of a sandbox id: the node to forward to, or the refusal to answer with. */ +type Located = { node: NodeState } | { refusal: RenderedError }; + /** * The faces Fastify never sees — keyed on a header on any path, judged on * the raw request the serverFactory hands over: @@ -38,6 +46,8 @@ type ProxyFace = Extract; * frozen sandbox on traffic and answers "not listening" for * a port nobody serves; the gateway adds only where it is. * WebSocket upgrades ride the same way (forwardUpgrade). + * The one exception is the browser-direct file form's + * preflight, answered at the door (handleRequest below). * envd E2B's in-sandbox API, keyed by E2b-Sandbox-Id; forwarded * with no credential change (the access token is the node's * own HMAC). Preflights are answered here: the node answers @@ -89,41 +99,37 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { } /** - * Finds the id, or answers the refusal and returns null. The lookup - * itself failing (not a node's silence — the gateway's own bug) is a 500 - * that sends the operator to the log. + * Finds the id, or the refusal to answer with — the one adjudication + * both halves of a keyed face make, each writing a refusal in its own + * medium (a response; a status line on an upgrade's socket). A value, + * not an exception: the refusal is an answer, not a failure. The lookup + * itself failing (not a node's silence — the gateway's own bug) is a + * 500 that sends the operator to the log. */ - async function locate( - res: http.ServerResponse, - id: string, - dialect: Dialect, - cors: boolean, - face: string, - ): Promise> { + async function locate(id: string, what: string): Promise { let found: Found; try { found = await finder.byId(id); } catch (error) { - log.error(error, `${face} face: the lookup itself failed`); - renderError(res, dialect, { - status: 500, - message: 'the gateway failed while locating the sandbox — see its log', - cors, - }); - throw new Refused(); + log.error(error, `${what}: the lookup itself failed`); + return { + refusal: { + status: 500, + message: + 'the gateway failed while locating the sandbox — see its log', + }, + }; } - if (found.kind !== 'one') { - renderError(res, dialect, { ...sentence(id, found), cors }); - throw new Refused(); - } - return found; + return found.kind === 'one' + ? { node: found.node } + : { refusal: sentence(id, found) }; } /** * Finds the id and forwards, or answers the refusal — the one path - * every keyed face takes. Nothing here may throw past this point: no - * framework stands behind a raw face, so an escaped rejection would be - * the process's, not the request's (errors.ts relay answers instead). + * every keyed face takes. Nothing here may throw: no framework stands + * behind a raw face, so an escaped rejection would be the process's, + * not the request's (errors.ts relay answers instead). */ async function route( req: http.IncomingMessage, @@ -133,13 +139,12 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { cors: boolean, ): Promise { const dialect: Dialect = face === 'envd' ? 'connect' : 'native'; - let found: Extract; - try { - found = await locate(res, id, dialect, cors, face); - } catch { + const located = await locate(id, `${face} face`); + if ('refusal' in located) { + renderError(res, dialect, { ...located.refusal, cors }); return; } - const node = found.node; + const { node } = located; await relay( res, dialect, @@ -184,26 +189,16 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { socket: Duplex, head: Buffer, ): Promise { - let found: Found; - try { - found = await finder.byId(kind.sandboxId); - } catch (error) { - log.error(error, 'proxy face: the lookup itself failed (upgrade)'); - refuseUpgrade(socket, { - status: 500, - message: 'the gateway failed while locating the sandbox — see its log', - }); - return; - } + const located = await locate(kind.sandboxId, 'proxy face (upgrade)'); // The client left while its sandbox was being found (a lookup round // is up to two seconds): nothing to dial the node for. if (socket.destroyed) return; - if (found.kind !== 'one') { - refuseUpgrade(socket, sentence(kind.sandboxId, found)); + if ('refusal' in located) { + refuseUpgrade(socket, located.refusal); return; } forwardUpgrade(req, socket, head, { - endpoint: found.node.endpoint, + endpoint: located.node.endpoint, token, }); } @@ -220,13 +215,20 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { // signed door) promises CORS on every answer, refusals included, // or the browser could not read them; the daemon's proxy answers // carry none, and neither do the gateway's for any other host. - void route( - req, - res, - kind.sandboxId, - 'proxy', - browserDirect(kind, req), - ); + // Its preflight is answered here, as the envd face's is: a + // preflight is credential-less and asks nothing of the sandbox, + // and the node would answer it in this very shape (its cors.ts). + // Forwarded instead, an id on no node earned it the 502 below — + // which no browser reads on a preflight: it drops the real + // request, and the readable refusal is never seen (found by + // review, 2026-09-14). Every other sandbox host's OPTIONS is the + // sandbox's own: the app inside decides its CORS. + const direct = isEnvdFilesForm(kind.port, req.url); + if (direct && req.method === 'OPTIONS') { + sendPreflight(req, res); + return; + } + void route(req, res, kind.sandboxId, 'proxy', direct); return; } case 'envd': { @@ -277,6 +279,16 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { // whole gateway down — from an unauthenticated face (the daemon's // sandbox-proxy.ts handleUpgrade has the same first line). socket.on('error', () => socket.destroy()); + // The request path's first rule (classify.ts isOriginForm), in this + // path's medium — a status line, there being no response object. + // Judged ahead of the face: an absolute-form handshake on a sandbox + // host was replayed into the sandbox verbatim, the gateway and the + // node reading two different requests (found by review, 2026-09-14, + // reproduced on the test machine). + if (!isOriginForm(req)) { + refuseUpgrade(socket, { status: 400, message: ORIGIN_FORM_REQUIRED }); + return; + } // Only the proxy face takes upgrades: sandbox WebSockets (a dev // server's HMR, a notebook). Everything else is cut, exactly as // stock Fastify, which never handles upgrades, would. @@ -289,17 +301,6 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { }; } -/** Thrown inside route() once the refusal is on the wire — the signal to stop, never seen outside. */ -class Refused extends Error {} - -/** Is this the browser-postable signed-URL form, `49983-./files`? Path-only, query ignored — the daemon's own carve-out test. */ -function browserDirect(kind: ProxyFace, req: http.IncomingMessage): boolean { - if (kind.port !== ENVD_PORT) return false; - const url = req.url ?? ''; - const q = url.indexOf('?'); - return (q === -1 ? url : url.slice(0, q)) === '/files'; -} - /** A refusal on an upgrade: one status line and a JSON body, before any handshake was replayed (forwardUpgrade's own refusals have the same shape). */ function refuseUpgrade(socket: Duplex, error: RenderedError): void { if (socket.destroyed || socket.writableEnded) return; diff --git a/packages/server/src/sandbox-proxy.ts b/packages/server/src/sandbox-proxy.ts index 07b24232..af3c2813 100644 --- a/packages/server/src/sandbox-proxy.ts +++ b/packages/server/src/sandbox-proxy.ts @@ -3,7 +3,7 @@ import { request as httpRequest } from 'node:http'; import net from 'node:net'; import type { Duplex } from 'node:stream'; import { - ENVD_PORT, + isEnvdFilesForm, parseSandboxHost, sandboxDomainsInForce, } from '@dormice/shared'; @@ -38,20 +38,14 @@ import { wakeSandbox } from './lifecycle'; */ /** - * The Host grammar — parseSandboxHost, the domain group in force, - * ENVD_PORT — lives in @dormice/shared (sandbox-host.ts): the gateway's - * proxy face reads the same header to find the node holding the sandbox - * and forwards the request here whole, Host kept, so a host names the - * same sandbox at both doors. + * The Host grammar — parseSandboxHost, the domain group in force, and + * the browser-direct file form isEnvdFilesForm — lives in @dormice/shared + * (sandbox-host.ts): the gateway's proxy face reads the same header to + * find the node holding the sandbox and forwards the request here whole, + * Host kept, so a host names the same sandbox at both doors and the same + * form is browser-direct at both. */ -/** Path-only match for the carve-out: exactly /files, query ignored. */ -function isEnvdFilesRequest(req: http.IncomingMessage): boolean { - const url = req.url ?? ''; - const q = url.indexOf('?'); - return (q === -1 ? url : url.slice(0, q)) === '/files'; -} - export interface SandboxProxyDeps { db: Db; executor: Executor; @@ -127,10 +121,9 @@ export function createSandboxProxy(deps: SandboxProxyDeps): SandboxProxy { matches(req) { const parsed = parseSandboxHost(req.headers.host, domains()); if (!parsed) return false; - // The envd file face on its fixed port belongs to Fastify's signed - // door, not to a dial into the container (see ENVD_PORT). - if (parsed.port === ENVD_PORT && isEnvdFilesRequest(req)) return false; - return true; + // The browser-direct file form on envd's fixed port belongs to + // Fastify's signed door, not to a dial into the container. + return !isEnvdFilesForm(parsed.port, req.url); }, handleRequest(req, res) { diff --git a/packages/shared/src/sandbox-host.ts b/packages/shared/src/sandbox-host.ts index 460c9a97..46a331b5 100644 --- a/packages/shared/src/sandbox-host.ts +++ b/packages/shared/src/sandbox-host.ts @@ -21,10 +21,37 @@ import type { RuntimeSettings } from './settings'; * speak for (server/e2b/signed-files.ts). Every other path keeps the * honest proxy answer: nothing listens on 49983 inside the sandbox. The * gateway forwards this form like any other sandbox host; the node does - * the carving. + * the carving, the gateway answers its preflights and wears CORS on its + * own refusals to it (isEnvdFilesForm below is the one judgment of it). */ export const ENVD_PORT = 49983; +/** + * Exactly `/files`, query ignored — the path of both signed-URL forms, + * the bare daemon root and the 49983 subdomain (isEnvdFilesForm). + */ +export function isFilesPath(url: string | undefined): boolean { + const target = url ?? ''; + const q = target.indexOf('?'); + return (q === -1 ? target : target.slice(0, q)) === '/files'; +} + +/** + * The browser-postable signed-URL form: envd's port in the Host label and + * the /files path — `49983-./files`. Judged here for + * both doors, so what is browser-direct at one is browser-direct at the + * other: the daemon's proxy carves it out of port forwarding onto its + * signed door (server/sandbox-proxy.ts), the gateway answers its + * preflights itself and wears CORS on its own refusals to it + * (gateway/raw.ts). + */ +export function isEnvdFilesForm( + port: number, + url: string | undefined, +): boolean { + return port === ENVD_PORT && isFilesPath(url); +} + /** * The domain group inbound matching runs against: the canonical domain * first, then the inbound-only aliases; empty when the feature is off From 81100f81e64c51b12ab4b0ef3047fe5e421c4ee9 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 22:58:50 +0800 Subject: [PATCH 44/89] The bare signed-URL form is routed at the gateway: a node reads a signature back to its sandbox, lookupSandbox asks by it, and the door forwards to the one that says yes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway answered `/files?…signature=…` at its root with a 501, on the belief that a configured sandbox domain makes the SDK mint the `49983-./files` form instead. It does not: the official package builds uploadUrl/downloadUrl as `new URL('/files', envdDirectUrl)`, and envdDirectUrl is the sandboxUrl it was given whenever one is set — so every signed URL a client pointed at the gateway mints lands on the gateway's root, with no sandbox id in it, whatever domain is in force (e2b 2.31.0, JS and Python alike; reproduced on the test machine). Those URLs are what a browser opens for a preview or a download; the Hong Kong daemon's log holds 26 805 of them since 2026-07-27. Behind the gateway they would all have failed. The signature is readable only by the secret of the node that minted the sandbox's token, so the gateway asks: lookupSandbox gains a third way of naming a sandbox, `{ signed: { operation, query } }` — the query as it arrived and the door's operation — and the node reads it back to the live sandbox whose token signed it (signing.ts sandboxOfSignedQuery: identity only; the expiration and the rest are the door's to judge, on the request that is then forwarded whole). The verb moves into its own plugin, routes/lookup.ts, which holds the signing secret the sandbox verbs have no business with. At the gateway, Finder.bySignature asks every node — there is no key to consult the cache by — caches the yes by id and name, and the signedRoot face takes the same route() as the other keyed faces. Refusals are the door's own: 401 "invalid signature" when no node's live sandbox signed it (a forged one and a destroyed sandbox's read the same at either door), 401 "missing signature query parameter" before any node is asked, 503 with Retry-After while a node is silent — all with CORS, so a browser can read them. raw.ts names what a face looks for as a value (Subject: its label and what "no node holds it" means there), so the id faces and the signature face share one sentence() and one locate(). Tests: the fake node grows a signing scheme and the signed door itself; the gateway suite covers download, upload (the body rides once, to the one node), forged, missing, preflight and a silent node; the finder suite covers the ask and the cache; the node suite covers reading a signature back by sandbox and operation, and not judging the expiration. e2e: the official package minted at the fleet door downloads and uploads at the fleet door; the same through node A's door in both executor modes; the console's browser-formula URL is opened at the door, where the console lives. --- e2e/src/console.test.ts | 15 +- e2e/src/e2b.test.ts | 23 +++ e2e/src/gateway.test.ts | 56 +++++++- packages/gateway/src/app.test.ts | 144 ++++++++++++++++++- packages/gateway/src/find.test.ts | 30 ++++ packages/gateway/src/find.ts | 13 ++ packages/gateway/src/raw.ts | 183 ++++++++++++++++++------ packages/server/src/app.test.ts | 74 ++++++++++ packages/server/src/app.ts | 2 + packages/server/src/e2b/signing.ts | 79 ++++++++-- packages/server/src/routes/lookup.ts | 80 +++++++++++ packages/server/src/routes/sandboxes.ts | 41 ------ packages/shared/src/lookup.ts | 37 ++++- 13 files changed, 656 insertions(+), 121 deletions(-) create mode 100644 packages/server/src/routes/lookup.ts diff --git a/e2e/src/console.test.ts b/e2e/src/console.test.ts index 16222104..6091ba1e 100644 --- a/e2e/src/console.test.ts +++ b/e2e/src/console.test.ts @@ -164,12 +164,13 @@ describe('web console over a real daemon', () => { describe('browser-side signed download URLs (the Office preview foundation)', () => { // The console's preview pane recomputes the file signature in the browser // (envd-client.ts signedDownloadUrl) from the token /envdToken - // hands it. This pins the whole chain end-to-end — console minting at - // the gateway (which asks the sandbox's node), the formula REWRITTEN - // here rather than imported (a black box pins the formula itself, not a - // shared implementation's self-consistency), and the root /files door, - // which is the node's today: the gateway's sandbox-domain face is the - // next step of the move (RULES/协议.md「网关」). + // hands it, and opens it on its own origin — the gateway's. This pins + // the whole chain end-to-end — console minting at the gateway (which + // asks the sandbox's node), the formula REWRITTEN here rather than + // imported (a black box pins the formula itself, not a shared + // implementation's self-consistency), and the root /files door at the + // gateway, which asks every node whose signature it is and forwards to + // the one that signed it. it('a console-minted token signs a working /files URL with the browser formula', async () => { // Continue the account story: re-setup with the token so this describe // owns known credentials regardless of what ran before it. @@ -228,7 +229,7 @@ describe('browser-side signed download URLs (the Office preview foundation)', () ); const signature = `v1_${btoa(String.fromCharCode(...new Uint8Array(digest))).replace(/=+$/, '')}`; const url = (extra = '') => - `${inject('dormiceEndpoint')}/files?path=pixel.png${extra}&signature=${encodeURIComponent(signature)}&signature_expiration=${exp}`; + `${endpoint()}/files?path=pixel.png${extra}&signature=${encodeURIComponent(signature)}&signature_expiration=${exp}`; const res = await fetch(url()); expect(res.status).toBe(200); diff --git a/e2e/src/e2b.test.ts b/e2e/src/e2b.test.ts index d7570696..0a5e0ce6 100644 --- a/e2e/src/e2b.test.ts +++ b/e2e/src/e2b.test.ts @@ -235,6 +235,29 @@ describe('official e2b SDK against the daemon', () => { } }); + it('signed URLs minted through the door work through the door: the form carries no sandbox id, and the door asks the node', async () => { + const sbx = await Sandbox.create({ + ...connection(), + apiUrl: `${door()}/e2b/api`, + sandboxUrl: `${door()}/e2b/envd`, + }); + try { + await sbx.files.write('signed/door.txt', 'through the door\n'); + const url = await sbx.downloadUrl('signed/door.txt'); + // The URL the SDK really builds off a door origin: the door's root + // /files, no sandbox id anywhere — what a browser opens. + expect(url.startsWith(`${door()}/files?`)).toBe(true); + const res = await fetch(url); + expect(res.status).toBe(200); + expect(await res.text()).toBe('through the door\n'); + const forged = new URL(url); + forged.searchParams.set('signature', 'v1_forged'); + expect((await fetch(forged)).status).toBe(401); + } finally { + await sbx.kill(); + } + }); + it('reports info, appears in list, and can be found by metadata', async () => { const sbx = await Sandbox.create({ ...connection(), diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts index fa848149..ecd8f57b 100644 --- a/e2e/src/gateway.test.ts +++ b/e2e/src/gateway.test.ts @@ -497,19 +497,13 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { } }); - it('envd preflights are answered without a header; the bare signed-URL door is a 501 with CORS', async () => { + it('envd preflights are answered without a header; a signed URL the official package mints at the fleet door works at the fleet door — the door asks every node whose signature it is', async () => { const preflight = await fetch(`${gateway()}/e2b/envd/files`, { method: 'OPTIONS', headers: { origin: 'https://app.example' }, }); expect(preflight.status).toBe(204); expect(preflight.headers.get('access-control-allow-origin')).toBe('*'); - const bare = await fetch(`${gateway()}/files?signature=x&path=/y`); - expect(bare.status).toBe(501); - expect(bare.headers.get('access-control-allow-origin')).toBe('*'); - expect(((await bare.json()) as { code: string }).code).toBe( - 'unimplemented', - ); const stranger = await fetch(`${gateway()}/e2b/envd/files`, { headers: { 'e2b-sandbox-id': randomUUID() }, }); @@ -517,6 +511,54 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { expect(((await stranger.json()) as { code: string }).code).toBe( 'unavailable', ); + + // The SDK builds uploadUrl/downloadUrl off the sandboxUrl it was + // given — `/files?…signature=…`, no sandbox id in it, whatever + // domain is in force (pinned here: this is the URL clawsgo's browser + // opens). A bare fetch of it at the door must reach the node whose + // sandbox signed it. + const sbx = await Sandbox.create({ + apiKey: `e2b_${token()}`, + apiUrl: `${gateway()}/e2b/api`, + sandboxUrl: `${gateway()}/e2b/envd`, + metadata: { name: 'gw-signed' }, + }); + try { + await sbx.files.write('signed/hello.txt', 'signed at the door\n'); + const url = await sbx.downloadUrl('signed/hello.txt', { + useSignatureExpiration: 300, + }); + expect(url.startsWith(`${gateway()}/files?`)).toBe(true); + const res = await fetch(url); + expect(res.status).toBe(200); + expect(res.headers.get('access-control-allow-origin')).toBe('*'); + expect(await res.text()).toBe('signed at the door\n'); + + const forged = new URL(url); + forged.searchParams.set('signature', 'v1_forged'); + const refused = await fetch(forged); + expect(refused.status).toBe(401); + expect(refused.headers.get('access-control-allow-origin')).toBe('*'); + expect(await refused.json()).toEqual({ + code: 'unauthenticated', + message: 'invalid signature', + }); + + const uploadUrl = await sbx.uploadUrl(); + const form = new FormData(); + form.append( + 'file', + new Blob(['uploaded at the door\n']), + 'signed/up.txt', + ); + const up = await fetch(uploadUrl, { method: 'POST', body: form }); + expect(up.status).toBe(200); + expect(await sbx.files.read('signed/up.txt')).toBe( + 'uploaded at the door\n', + ); + } finally { + await sbx.kill(); + } }); it('a third node joins at its first check-in; when it dies its sandboxes 502 and new names are a 503 naming it, until an operator removes it', async () => { diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index fd1b4a56..230fdc34 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -97,6 +97,19 @@ class FakeNode { return [...this.sandboxes.values()].find((s) => s.id === id); } + /** + * This double's signing secret is its own id: a signature + * `sig--` speaks for that sandbox here and for nothing + * anywhere else — as a real node's HMAC, keyed by its own secret, does. + */ + bySignature(query: string) { + const signature = new URLSearchParams(query).get('signature') ?? ''; + const prefix = `sig-${this.id}-`; + return signature.startsWith(prefix) + ? this.byId(signature.slice(prefix.length)) + : undefined; + } + // Inferred return type on purpose: the early `return json(...)` exits // read as statements, and an explicit void would flag each one. private answer( @@ -136,6 +149,31 @@ class FakeNode { }); } this.hits.push({ path, auth }); + if (path === '/files') { + // The daemon's signed file door as the gateway meets it: judged by + // the query alone, no headers wanted, CORS on every answer — and + // here an echo of what arrived. + const sandbox = this.bySignature(url.slice(url.indexOf('?') + 1)); + res.writeHead(sandbox ? 200 : 401, { + 'content-type': 'application/json', + 'access-control-allow-origin': '*', + }); + res.end( + JSON.stringify( + sandbox + ? { + signedDoor: this.id, + sandboxId: sandbox.id, + method: req.method, + url, + auth: auth ?? null, + bodyBytes: text.length, + } + : { code: 'unauthenticated', message: 'invalid signature' }, + ), + ); + return; + } const body = text ? (JSON.parse(text) as Record) : {}; if (path.startsWith('/e2b/envd/')) { return json(200, { @@ -187,7 +225,13 @@ class FakeNode { return json(200, { sandboxNames: [] }); } case '/lookupSandbox': { - const sandbox = 'id' in body ? this.byId(body.id as string) : found; + const signed = body.signed as { query: string } | undefined; + const sandbox = + signed !== undefined + ? this.bySignature(signed.query) + : 'id' in body + ? this.byId(body.id as string) + : found; return json( 200, sandbox @@ -1133,11 +1177,13 @@ describe('the E2B faces', () => { }); expect(preflight.status).toBe(204); expect(preflight.headers.get('access-control-allow-origin')).toBe('*'); + // The bare signed form is its own face (below): a signature nobody + // signed is the door's own 401, with CORS. const bare = await fetch(`${h.endpoint}/files?signature=x`); - expect(bare.status).toBe(501); + expect(bare.status).toBe(401); expect(bare.headers.get('access-control-allow-origin')).toBe('*'); expect(((await bare.json()) as { code: string }).code).toBe( - 'unimplemented', + 'unauthenticated', ); }); }); @@ -1393,3 +1439,95 @@ describe('the sandbox port proxy face', () => { expect((await viaHost(h, host, '/x')).status).toBe(404); }); }); + +describe('the bare signed-URL face', () => { + it('a signed /files request at the root is routed by asking every node whose signature it is: download and upload reach the node whose sandbox signed it, whole and credential-less; a signature nobody signed is the door’s own 401, no signature the door’s first rule; a silent node makes it a 503', async () => { + const h = await gateway(['a', 'b']); + const nodeA = h.nodes[0] as FakeNode; + const nodeB = h.nodes[1] as FakeNode; + // Built behind the gateway's back: nothing cached, nothing but the + // signature to go on — the form the SDK's downloadUrl mints off the + // door's origin, whatever domain is in force. + const staged = await stage(nodeB, 'signer'); + const query = `path=out.txt&signature=${encodeURIComponent(`sig-b-${staged.id}`)}&signature_expiration=1`; + const download = await fetch(`${h.endpoint}/files?${query}`); + expect(download.status).toBe(200); + expect(download.headers.get('access-control-allow-origin')).toBe('*'); + expect(await download.json()).toEqual({ + signedDoor: 'b', + sandboxId: staged.id, + method: 'GET', + url: `/files?${query}`, + auth: null, + bodyBytes: 0, + }); + // Every node was asked, once; what b answered is cached by id, so the + // sandbox's other faces now ask nobody. + expect(nodeA.lookups()).toBe(1); + expect(nodeB.lookups()).toBe(1); + expect(h.cache.getById(staged.id)?.nodeId).toBe('b'); + // An upload: the body rides whole to the same node, once — never + // "tried" against each node. A signature is no key the cache holds, + // so each signed request is one round of questions. + const upload = await fetch(`${h.endpoint}/files?${query}`, { + method: 'POST', + headers: { 'content-type': 'application/octet-stream' }, + body: 'x'.repeat(5000), + }); + expect(upload.status).toBe(200); + expect(await upload.json()).toMatchObject({ + signedDoor: 'b', + method: 'POST', + bodyBytes: 5000, + }); + expect(nodeA.lookups()).toBe(2); + expect(nodeB.hits.filter((hit) => hit.path === '/files').length).toBe(2); + expect(nodeA.hits.some((hit) => hit.path === '/files')).toBe(false); + + // A signature no node's sandbox signed: the door's own 401, in its + // words and dialect, readable by a browser. + const forged = await fetch( + `${h.endpoint}/files?path=out.txt&signature=v1_forged`, + ); + expect(forged.status).toBe(401); + expect(forged.headers.get('access-control-allow-origin')).toBe('*'); + expect(await forged.json()).toEqual({ + code: 'unauthenticated', + message: 'invalid signature', + }); + // No signature at all: the door's first rule, and no node is asked. + const asked = nodeA.lookups(); + const bare = await fetch(`${h.endpoint}/files?path=out.txt`); + expect(bare.status).toBe(401); + expect(bare.headers.get('access-control-allow-origin')).toBe('*'); + expect(await bare.json()).toEqual({ + code: 'unauthenticated', + message: 'missing signature query parameter', + }); + expect(nodeA.lookups()).toBe(asked); + // The preflight is the door's own answer, as on the envd face. + const preflight = await fetch(`${h.endpoint}/files`, { + method: 'OPTIONS', + headers: { 'access-control-request-headers': 'content-type' }, + }); + expect(preflight.status).toBe(204); + expect(preflight.headers.get('access-control-allow-headers')).toBe( + 'content-type', + ); + + // A node that does not answer: a signature it may recognize cannot be + // called invalid — retry. One that another node does recognize still + // routes: one yes wins over a silence. + await nodeA.stop(); + const unsure = await fetch( + `${h.endpoint}/files?path=out.txt&signature=v1_forged`, + ); + expect(unsure.status).toBe(503); + expect(unsure.headers.get('retry-after')).toBe('15'); + expect(unsure.headers.get('access-control-allow-origin')).toBe('*'); + expect(((await unsure.json()) as { code: string }).code).toBe( + 'unavailable', + ); + expect((await fetch(`${h.endpoint}/files?${query}`)).status).toBe(200); + }); +}); diff --git a/packages/gateway/src/find.test.ts b/packages/gateway/src/find.test.ts index 1b8a6023..05885523 100644 --- a/packages/gateway/src/find.test.ts +++ b/packages/gateway/src/find.test.ts @@ -11,6 +11,7 @@ import { httpAskNode, LOOKUP_TIMEOUT_MS, type LookupAnswer, + type LookupQuery, } from './lookup'; import { checkInOf } from './testing'; @@ -145,6 +146,35 @@ describe('Finder', () => { expect(warned).toHaveLength(1); }); + it('by signature there is no cache to consult: every node is asked with the query as it came, the one yes is cached by id and name, and the next signed request asks again', async () => { + const fleet = fleetOf('a', 'b'); + const cache = new NameCache(); + const queries: LookupQuery[] = []; + const ask: AskNode = async (node, query) => { + queries.push(query); + return node.id === 'b' + ? { kind: 'found', id: 'sb-9', name: 'signer', state: 'active' } + : { kind: 'absent' }; + }; + const finder = new Finder(fleet, cache, ask, silentLog); + const signed = { + operation: 'read' as const, + query: 'path=out.txt&signature=v1_abc', + }; + const found = await finder.bySignature(signed); + expect(found).toMatchObject({ kind: 'one', id: 'sb-9', name: 'signer' }); + expect(found.kind === 'one' && found.node.id).toBe('b'); + expect(queries).toEqual([{ signed }, { signed }]); + expect(cache.getById('sb-9')?.nodeId).toBe('b'); + expect(cache.getByName('signer')?.id).toBe('sb-9'); + // The other faces now find it without asking; a signature is not a + // key the cache holds, so a signed request is a round of questions. + expect((await finder.byId('sb-9')).kind).toBe('one'); + expect(queries.length).toBe(2); + await finder.bySignature(signed); + expect(queries.length).toBe(4); + }); + it('an empty fleet finds nothing and asks nobody', async () => { const { ask, asked } = scripted({}); expect( diff --git a/packages/gateway/src/find.ts b/packages/gateway/src/find.ts index bc6191ea..4c267c2d 100644 --- a/packages/gateway/src/find.ts +++ b/packages/gateway/src/find.ts @@ -1,3 +1,4 @@ +import type { SignedFileLookup } from '@dormice/shared'; import type { CacheEntry, NameCache } from './cache'; import type { Fleet, NodeState } from './fleet'; import type { AskNode, LookupQuery } from './lookup'; @@ -66,6 +67,18 @@ export class Finder { return this.find(this.cache.getById(id), { id }, false); } + /** + * The bare signed-URL form: the signature is the only identity the + * request carries, and only the secret of the node that minted it reads + * it — so there is no key to consult the cache by, every node is asked, + * and the one whose live sandbox signed it says so (the node's + * lookupSandbox, its signing.ts). What it answers is cached by id and + * name like any other finding, for the sandbox's other faces. + */ + bySignature(signed: SignedFileLookup): Promise { + return this.find(undefined, { signed }, false); + } + private async find( cached: CacheEntry | undefined, query: LookupQuery, diff --git a/packages/gateway/src/raw.ts b/packages/gateway/src/raw.ts index a2fd18ff..9e4a00c1 100644 --- a/packages/gateway/src/raw.ts +++ b/packages/gateway/src/raw.ts @@ -1,6 +1,6 @@ import http from 'node:http'; import type { Duplex } from 'node:stream'; -import { isEnvdFilesForm } from '@dormice/shared'; +import { isEnvdFilesForm, type SignedFileLookup } from '@dormice/shared'; import type { Logger } from 'pino'; import { type Classified, @@ -28,13 +28,49 @@ export interface RawFacesDeps { export const RETRY_AFTER_SECONDS = 15; type ProxyFace = Extract; +type KeyedFace = Exclude['face']; -/** What a keyed face learned of a sandbox id: the node to forward to, or the refusal to answer with. */ -type Located = { node: NodeState } | { refusal: RenderedError }; +/** What a keyed face learned of the sandbox it was asked for: the node to forward to (and the id the sandbox goes by), or the refusal to answer with. */ +type Located = { node: NodeState; id: string } | { refusal: RenderedError }; /** - * The faces Fastify never sees — keyed on a header on any path, judged on - * the raw request the serverFactory hands over: + * What a keyed face is looking for, in the words its refusals use: how + * the sentences name it, and what "no node holds it" means on that face. + */ +interface Subject { + label: string; + none: RenderedError; +} + +/** A sandbox id nobody holds: it may have been destroyed — the daemon's own proxy says "not found" for one it lacks. */ +const byId = (id: string): Subject => ({ + label: `sandbox "${id}"`, + none: { + status: 502, + connectCode: 'unavailable', + message: `sandbox "${id}" is on no node — it may have been destroyed`, + }, +}); + +/** + * A signature no node's live sandbox signed is exactly what the door + * itself calls "invalid signature": a forged one and a destroyed + * sandbox's read the same at either door (the node's signing.ts skips + * dead rows), and a browser-direct upload must be able to read the + * refusal — the door's own 401, its own words. + */ +const bySignature: Subject = { + label: "the signed URL's sandbox", + none: { + status: 401, + connectCode: 'unauthenticated', + message: 'invalid signature', + }, +}; + +/** + * The faces Fastify never sees — keyed on a header on any path, or on a + * signature — judged on the raw request the serverFactory hands over: * proxy the sandbox port proxy, keyed by the Host label (E2B's * getHost() URL, classify.ts): the node holding the id is * found and the request goes there whole — Host kept, since @@ -53,9 +89,17 @@ type Located = { node: NodeState } | { refusal: RenderedError }; * own HMAC). Preflights are answered here: the node answers * them without auth, and a preflight that came back 401 * without CORS would fail every browser-direct upload. - * signedRoot the bare signed-URL form: no sandbox id anywhere the - * gateway can read without the node's signing secret, so an - * honest 501. + * signedRoot the bare signed-URL form, `/files?…signature=…` at the + * root — what the SDK's uploadUrl/downloadUrl mint off the + * API origin they were given, whatever domain is in force, + * and what a browser or a curl then opens with nothing but + * the query. No sandbox id anywhere in it: the signature is + * the identity, and only the secret of the node that minted + * it can read it — so every node is asked whose it is + * (finder.bySignature) and the request goes whole to the one + * whose live sandbox signed it, for its door to judge the + * query again in full. Preflights are answered here, as the + * envd face's are. * * Nobody has authenticated to the gateway on these faces — the node * judges the credential, after the gateway has picked it — so what the @@ -66,13 +110,14 @@ type Located = { node: NodeState } | { refusal: RenderedError }; * authenticated faces and listNodes name nodes freely. * * Each face's refusals wear that face's dialect (errors.ts): connect for - * envd, and for the proxy the daemon's proxy answer — { message }, 502 — - * so a caller reads one shape from either door. + * envd and the signed form (the node's signed door speaks it), and for + * the proxy the daemon's proxy answer — { message }, 502 — so a caller + * reads one shape from either door. */ export function createRawFaces({ finder, token, log }: RawFacesDeps) { - /** The one sentence per finding for a sandbox id on these faces — generic on purpose (above). */ + /** The one sentence per finding on these faces — generic on purpose (above). */ function sentence( - id: string, + subject: Subject, found: Exclude, ): RenderedError { switch (found.kind) { @@ -80,36 +125,36 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { return { status: 502, connectCode: 'unavailable', - message: `sandbox "${id}" is held by more than one node — routing resumes once an operator destroys one copy (listNodes and the gateway log name them)`, + message: `${subject.label} is held by more than one node — routing resumes once an operator destroys one copy (listNodes and the gateway log name them)`, }; case 'none': - return { - status: 502, - connectCode: 'unavailable', - message: `sandbox "${id}" is on no node — it may have been destroyed`, - }; + return subject.none; case 'unsure': return { status: 503, connectCode: 'unavailable', - message: `sandbox "${id}": a node did not answer, so its whereabouts cannot be settled — retry`, + message: `${subject.label}: a node did not answer, so its whereabouts cannot be settled — retry`, retryAfterSeconds: RETRY_AFTER_SECONDS, }; } } /** - * Finds the id, or the refusal to answer with — the one adjudication - * both halves of a keyed face make, each writing a refusal in its own - * medium (a response; a status line on an upgrade's socket). A value, - * not an exception: the refusal is an answer, not a failure. The lookup - * itself failing (not a node's silence — the gateway's own bug) is a - * 500 that sends the operator to the log. + * Finds the sandbox, or the refusal to answer with — the one + * adjudication both halves of a keyed face make, each writing a refusal + * in its own medium (a response; a status line on an upgrade's socket). + * A value, not an exception: the refusal is an answer, not a failure. + * The lookup itself failing (not a node's silence — the gateway's own + * bug) is a 500 that sends the operator to the log. */ - async function locate(id: string, what: string): Promise { + async function locate( + subject: Subject, + find: () => Promise, + what: string, + ): Promise { let found: Found; try { - found = await finder.byId(id); + found = await find(); } catch (error) { log.error(error, `${what}: the lookup itself failed`); return { @@ -121,12 +166,12 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { }; } return found.kind === 'one' - ? { node: found.node } - : { refusal: sentence(id, found) }; + ? { node: found.node, id: found.id } + : { refusal: sentence(subject, found) }; } /** - * Finds the id and forwards, or answers the refusal — the one path + * Finds the sandbox and forwards, or answers the refusal — the one path * every keyed face takes. Nothing here may throw: no framework stands * behind a raw face, so an escaped rejection would be the process's, * not the request's (errors.ts relay answers instead). @@ -134,17 +179,18 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { async function route( req: http.IncomingMessage, res: http.ServerResponse, - id: string, - face: 'envd' | 'proxy', + face: KeyedFace, + subject: Subject, + find: () => Promise, cors: boolean, ): Promise { - const dialect: Dialect = face === 'envd' ? 'connect' : 'native'; - const located = await locate(id, `${face} face`); + const dialect: Dialect = face === 'proxy' ? 'native' : 'connect'; + const located = await locate(subject, find, `${face} face`); if ('refusal' in located) { renderError(res, dialect, { ...located.refusal, cors }); return; } - const { node } = located; + const { node, id } = located; await relay( res, dialect, @@ -169,7 +215,7 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { return { status: 502, connectCode: 'unavailable', - message: `sandbox "${id}": its node did not answer (${error.why}) — retry`, + message: `${subject.label}: its node did not answer (${error.why}) — retry`, cors, }; }, @@ -189,7 +235,11 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { socket: Duplex, head: Buffer, ): Promise { - const located = await locate(kind.sandboxId, 'proxy face (upgrade)'); + const located = await locate( + byId(kind.sandboxId), + () => finder.byId(kind.sandboxId), + 'proxy face (upgrade)', + ); // The client left while its sandbox was being found (a lookup round // is up to two seconds): nothing to dial the node for. if (socket.destroyed) return; @@ -228,7 +278,15 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { sendPreflight(req, res); return; } - void route(req, res, kind.sandboxId, 'proxy', direct); + const { sandboxId } = kind; + void route( + req, + res, + 'proxy', + byId(sandboxId), + () => finder.byId(sandboxId), + direct, + ); return; } case 'envd': { @@ -247,7 +305,7 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { }); return; } - void route(req, res, id, 'envd', true); + void route(req, res, 'envd', byId(id), () => finder.byId(id), true); return; } case 'signedRoot': { @@ -255,13 +313,28 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { sendPreflight(req, res); return; } - renderError(res, 'connect', { - status: 501, - connectCode: 'unimplemented', - message: - 'the bare signed-URL form is not routed by the gateway yet — use the sandbox host form, or the node directly', - cors: true, - }); + // The door's first rule, applied first here too: without a + // signature there is nobody to ask for — every node would say + // no — and the answer is the door's own (signing.ts, in + // validateSigning order: missing before invalid). + const signed = signedFileLookupOf(req); + if (signed === null) { + renderError(res, 'connect', { + status: 401, + connectCode: 'unauthenticated', + message: 'missing signature query parameter', + cors: true, + }); + return; + } + void route( + req, + res, + 'signedRoot', + bySignature, + () => finder.bySignature(signed), + true, + ); return; } } @@ -301,6 +374,24 @@ export function createRawFaces({ finder, token, log }: RawFacesDeps) { }; } +/** + * The question a bare signed file request turns into — the query as it + * arrived, for the node to read beside its own door (the gateway reads + * none of it but the one key that says whether there is a question at + * all), and the door's operation: GET and HEAD read, POST writes, the + * signed door's two routes (the node's signed-files.ts). Null when the + * request carries no signature. + */ +function signedFileLookupOf( + req: http.IncomingMessage, +): SignedFileLookup | null { + const url = req.url ?? ''; + const q = url.indexOf('?'); + const query = q === -1 ? '' : url.slice(q + 1); + if (!new URLSearchParams(query).has('signature')) return null; + return { operation: req.method === 'POST' ? 'write' : 'read', query }; +} + /** A refusal on an upgrade: one status line and a JSON body, before any handshake was replayed (forwardUpgrade's own refusals have the same shape). */ function refuseUpgrade(socket: Duplex, error: RenderedError): void { if (socket.destroyed || socket.writableEnded) return; diff --git a/packages/server/src/app.test.ts b/packages/server/src/app.test.ts index c6179dc3..ee56c6f8 100644 --- a/packages/server/src/app.test.ts +++ b/packages/server/src/app.test.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; @@ -1881,6 +1882,79 @@ describe('POST /lookupSandbox', () => { expect((await rpc(app, '/lookupSandbox', {})).statusCode).toBe(400); }); + it('by signature: the bare signed file query is read back to the sandbox whose token signed it — identity only, the door judges the rest', async () => { + const { app } = testApp(); + const alice = (await acquire(app, { name: 'alice' })).json(); + const bob = (await acquire(app, { name: 'bob' })).json(); + const tokenOf = async (sandboxId: string) => + ( + (await rpc(app, '/envdToken', { sandboxId })).json() as { + envdAccessToken: string; + } + ).envdAccessToken; + // The SDK's formula, rewritten rather than imported: v1_ + base64 + // (padding stripped) of sha256("path:operation:username:token[:exp]"). + const sign = (parts: string[]) => + `v1_${createHash('sha256').update(parts.join(':')).digest('base64').replace(/=+$/, '')}`; + const aliceRead = sign([ + 'a.txt', + 'read', + '', + await tokenOf(alice.sandbox.id), + ]); + const query = (signature: string, more = '') => + `path=a.txt&signature=${encodeURIComponent(signature)}${more}`; + + const byAlice = await rpc(app, '/lookupSandbox', { + signed: { operation: 'read', query: query(aliceRead) }, + }); + expect(byAlice.statusCode).toBe(200); + expect(byAlice.json()).toEqual({ + found: true, + sandbox: { id: alice.sandbox.id, name: 'alice', state: 'active' }, + }); + // The same path signed by bob's token names bob, nobody else. + const bobRead = sign(['a.txt', 'read', '', await tokenOf(bob.sandbox.id)]); + expect( + ( + await rpc(app, '/lookupSandbox', { + signed: { operation: 'read', query: query(bobRead) }, + }) + ).json(), + ).toMatchObject({ found: true, sandbox: { name: 'bob' } }); + // A read signature is not a write signature; a forged one is nobody's; + // no signature is no sandbox. The expiration is not judged here — an + // expired signature still names its sandbox, and the door it is then + // forwarded to says "expired" itself. + for (const signed of [ + { operation: 'write', query: query(aliceRead) }, + { operation: 'read', query: query('v1_forged') }, + { operation: 'read', query: 'path=a.txt' }, + ]) { + expect((await rpc(app, '/lookupSandbox', { signed })).json()).toEqual({ + found: false, + }); + } + const past = Math.floor(Date.now() / 1000) - 60; + const expired = sign([ + 'a.txt', + 'read', + '', + await tokenOf(alice.sandbox.id), + String(past), + ]); + expect( + ( + await rpc(app, '/lookupSandbox', { + signed: { + operation: 'read', + query: query(expired, `&signature_expiration=${past}`), + }, + }) + ).json(), + ).toMatchObject({ found: true, sandbox: { name: 'alice' } }); + }); + it('a name whose slot is busy waits its turn: asked while an acquire is mid-create, it answers found once the row exists', async () => { // A create that parks inside the executor — the daemon's own shape of // "in flight": the acquire holds the name's slot, the container is diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 498e6be4..96909044 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -20,6 +20,7 @@ import type { Executor } from './executor/executor'; import type { KeyedQueue } from './keyed-queue'; import { envdTokenRoutes } from './routes/envd-token'; import { hostRoutes } from './routes/host'; +import { lookupRoutes } from './routes/lookup'; import { sandboxRoutes } from './routes/sandboxes'; import { templateUsersRoutes } from './routes/template-users'; import { upgradeRoutes } from './routes/upgrade'; @@ -207,6 +208,7 @@ export function buildApp({ watchers, archiver, }); + await api.register(lookupRoutes, { db, locks, envdSigningSecret }); await api.register(templateUsersRoutes, { db }); await api.register(hostRoutes, { config, db, executor }); await api.register(upgradeRoutes, { updater }); diff --git a/packages/server/src/e2b/signing.ts b/packages/server/src/e2b/signing.ts index 42ff50f0..305cf9a1 100644 --- a/packages/server/src/e2b/signing.ts +++ b/packages/server/src/e2b/signing.ts @@ -85,6 +85,73 @@ export interface SignedFileQuery { signature_expiration?: string; } +const SIGNED_FILE_QUERY_KEYS = [ + 'path', + 'username', + 'signature', + 'signature_expiration', +] as const; + +/** + * The query half read off a bare query string — the gateway's + * lookupSandbox hands the string over as it arrived on the wire, and it + * is read here, beside the door that reads the real request, so the two + * readings agree. URLSearchParams and Fastify's parser decode the four + * keys alike (`+` a space, percent-escapes resolved). + */ +export function parseSignedFileQuery(raw: string): SignedFileQuery { + const params = new URLSearchParams(raw); + const query: SignedFileQuery = {}; + for (const key of SIGNED_FILE_QUERY_KEYS) { + const value = params.get(key); + if (value !== null) query[key] = value; + } + return query; +} + +/** The signed material a query spells, in the SDK's order; the expiration rides along for the door's own check. */ +function materialOf( + query: SignedFileQuery, + operation: SigningOperation, +): SignatureMaterial { + const expirationUnix = + query.signature_expiration === undefined + ? undefined + : Number(query.signature_expiration); + return { + path: query.path ?? '', + operation, + username: query.username ?? '', + ...(expirationUnix === undefined ? {} : { expirationUnix }), + }; +} + +/** + * Which live sandbox does a bare signed query speak for — identity alone, + * for the gateway's lookupSandbox. A root `/files` request carries no + * sandbox id (the SDK builds the URL off the API origin), the signature + * is the only key in it, and only this node's secret reads it: so the + * gateway asks every node this and forwards to the one that says yes. + * Nothing else is judged here — not the expiration, not the username: the + * door the request is then forwarded to judges the whole query again, in + * validateSigning order (authenticateSignedQuery). No signature, no + * sandbox. + */ +export function sandboxOfSignedQuery( + db: Db, + signingSecret: string, + query: SignedFileQuery, + operation: SigningOperation, +): SandboxRow | undefined { + if (!query.signature) return undefined; + return findRowBySignature( + db, + signingSecret, + materialOf(query, operation), + query.signature, + ); +} + /** * The one adjudication of a signed file query, in real envd's * validateSigning order: missing signature first, then the constant-time @@ -117,16 +184,8 @@ export function authenticateSignedQuery(opts: { 'missing signature query parameter', ); } - const expirationUnix = - query.signature_expiration === undefined - ? undefined - : Number(query.signature_expiration); - const material: SignatureMaterial = { - path: query.path ?? '', - operation, - username: query.username ?? '', - ...(expirationUnix === undefined ? {} : { expirationUnix }), - }; + const material = materialOf(query, operation); + const { expirationUnix } = material; const sandboxId = pinnedSandboxId !== undefined ? matchesToken( diff --git a/packages/server/src/routes/lookup.ts b/packages/server/src/routes/lookup.ts new file mode 100644 index 00000000..6f92fe18 --- /dev/null +++ b/packages/server/src/routes/lookup.ts @@ -0,0 +1,80 @@ +import { + lookupSandboxRequestSchema, + lookupSandboxResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import type { Db } from '../db/db'; +import { findById, findByName } from '../db/ledger'; +import type { SandboxRow } from '../db/schema'; +import { parseSignedFileQuery, sandboxOfSignedQuery } from '../e2b/signing'; +import type { KeyedQueue } from '../keyed-queue'; + +export interface LookupRoutesOptions { + db: Db; + locks: KeyedQueue; + /** + * The signing secret behind envd tokens and signed URLs — the one key + * that reads a bare signature back to the sandbox it speaks for. + */ + envdSigningSecret: string; +} + +/** + * The gateway's one question on its own account: does this node hold the + * sandbox? Named three ways — by name, by id, or by the signature of a + * bare signed file URL (`/files?…signature=…` at the root carries no id: + * the SDK's uploadUrl/downloadUrl build it off the API origin, and only + * the secret that minted the sandbox's token reads the signature back — + * e2b/signing.ts sandboxOfSignedQuery). Read-only — never wakes, never + * touches the idle clock — and truthful about a create in flight, in two + * steps. A row that exists answers at once, whatever its state: a + * restoring sandbox has a row, and waiting for its slot would hold the + * answer for the whole restore, long past the gateway's two-second + * patience — the gateway would read a live sandbox as a node that did not + * answer. No row means an acquire may be writing it right now (create + * first, row second, both under the name's slot), so the answer takes the + * slot itself and looks again: at once when the slot is free (a plain + * no), behind the acquire when it is not. By id or by signature there is + * no slot to wait on (slots are keyed by name), and none is needed: nobody + * can ask about an id, or hold a signature, before the create that minted + * it has answered. + */ +export const lookupRoutes: FastifyPluginAsyncZod = async ( + app, + { db, locks, envdSigningSecret }, +) => { + app.post( + '/lookupSandbox', + { + schema: { + body: lookupSandboxRequestSchema, + response: { 200: lookupSandboxResponseSchema }, + }, + }, + async (request) => { + const query = request.body; + const answer = (row: SandboxRow | undefined) => + row + ? { + found: true as const, + sandbox: { id: row.id, name: row.name, state: row.state }, + } + : { found: false as const }; + if ('signed' in query) { + return answer( + sandboxOfSignedQuery( + db, + envdSigningSecret, + parseSignedFileQuery(query.signed.query), + query.signed.operation, + ), + ); + } + const look = () => + 'name' in query ? findByName(db, query.name) : findById(db, query.id); + const now = look(); + if (now !== undefined || !('name' in query)) return answer(now); + return answer(await locks.run(query.name, async () => look())); + }, + ); +}; diff --git a/packages/server/src/routes/sandboxes.ts b/packages/server/src/routes/sandboxes.ts index 21db1901..032b981f 100644 --- a/packages/server/src/routes/sandboxes.ts +++ b/packages/server/src/routes/sandboxes.ts @@ -19,8 +19,6 @@ import { listSandboxImagesResponseSchema, listSandboxMetricsRequestSchema, listSandboxMetricsResponseSchema, - lookupSandboxRequestSchema, - lookupSandboxResponseSchema, READ_FILES_TOTAL_LIMIT_BYTES, readFileRequestSchema, readFileResponseSchema, @@ -54,7 +52,6 @@ import type { Config } from '../config'; import type { Db } from '../db/db'; import { createSandbox, - findById, findByName, listSandboxes, setDiskGb, @@ -1078,44 +1075,6 @@ export const sandboxRoutes: FastifyPluginAsyncZod< }, ); - // The gateway's one question on its own account: does this node hold - // the sandbox? Read-only — never wakes, never touches the idle clock — - // and truthful about a create in flight, in two steps. A row that - // exists answers at once, whatever its state: a restoring sandbox has a - // row, and waiting for its slot would hold the answer for the whole - // restore, long past the gateway's two-second patience — the gateway - // would read a live sandbox as a node that did not answer. No row means - // an acquire may be writing it right now (create first, row second, - // both under the name's slot), so the answer takes the slot itself and - // looks again: at once when the slot is free (a plain no), behind the - // acquire when it is not. By id there is no slot to wait on (slots are - // keyed by name), and none is needed: nobody can ask about an id before - // the create that minted it has answered. - app.post( - '/lookupSandbox', - { - schema: { - body: lookupSandboxRequestSchema, - response: { 200: lookupSandboxResponseSchema }, - }, - }, - async (request) => { - const query = request.body; - const look = () => - 'name' in query ? findByName(db, query.name) : findById(db, query.id); - const answer = (row: SandboxRow | undefined) => - row - ? { - found: true as const, - sandbox: { id: row.id, name: row.name, state: row.state }, - } - : { found: false as const }; - const now = look(); - if (now !== undefined || !('name' in query)) return answer(now); - return answer(await locks.run(query.name, async () => look())); - }, - ); - app.post( '/destroySandbox', { diff --git a/packages/shared/src/lookup.ts b/packages/shared/src/lookup.ts index 8eb0a281..6eb42476 100644 --- a/packages/shared/src/lookup.ts +++ b/packages/shared/src/lookup.ts @@ -3,13 +3,14 @@ import { sandboxNameSchema } from './sandbox'; import { SANDBOX_STATES } from './states'; /** - * lookupSandbox({ name }) / lookupSandbox({ id }) — "do you hold this - * sandbox?", the one question a gateway asks a node on its own account. - * The gateway keeps no directory of where sandboxes live (a second copy of - * a fact the nodes' ledgers already hold, and every copy drifts); when its - * cache has no answer it asks every node this, in parallel, and routes to - * the one that says yes. Read-only by construction: it never wakes, never - * touches the idle clock, never creates. + * lookupSandbox({ name }) / lookupSandbox({ id }) / lookupSandbox({ signed }) + * — "do you hold this sandbox?", the one question a gateway asks a node + * on its own account. The gateway keeps no directory of where sandboxes + * live (a second copy of a fact the nodes' ledgers already hold, and + * every copy drifts); when its cache has no answer it asks every node + * this, in parallel, and routes to the one that says yes. Read-only by + * construction: it never wakes, never touches the idle clock, never + * creates. * * The node answers inside the name's serialization slot when it must: a * row that exists answers at once, whatever its state; no row while an @@ -21,9 +22,31 @@ import { SANDBOX_STATES } from './states'; * (503 with Retry-After); once the row is written, the retry finds the * sandbox on the node that built it. */ + +/** + * The third way of naming a sandbox: by the signature of a bare signed + * file URL. The SDK's uploadUrl/downloadUrl build `/files?… + * signature=…` off the API origin they were given — no sandbox id + * anywhere in the request, whatever domain is in force — and the + * signature is readable only by the secret of the node that minted the + * sandbox's token (server/e2b/signing.ts). So the gateway hands every node + * the query as it arrived and the door's operation, and the node whose + * live sandbox signed it says so. Identity only: the expiration and the + * rest of the signed material are judged by the door the request is then + * forwarded to, in its own order. + */ +export const signedFileLookupSchema = z.object({ + operation: z.enum(['read', 'write']), + /** The request's query string as sent, without the leading `?`. */ + query: z.string(), +}); + +export type SignedFileLookup = z.infer; + export const lookupSandboxRequestSchema = z.union([ z.object({ name: sandboxNameSchema }), z.object({ id: z.string().min(1) }), + z.object({ signed: signedFileLookupSchema }), ]); export type LookupSandboxRequest = z.infer; From 93ce07a42b79826127367c1cb1f8a978a643a4a7 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Mon, 14 Sep 2026 23:51:48 +0800 Subject: [PATCH 45/89] A node still waiting for its first configuration is not dialled; the bundle line is said once per gap; the bundle reads the settings row once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three leftovers of the second cut's review. A node that has checked in and reported no configuration copy is not listening yet (the daemon fetches its first bundle before it opens its port), so a lookup dialled there was refused at the socket and read as silence — a 503 to every caller of every uncached name for as long as the node booted. Its own reading says what it holds: nothing, and it is a plain no; sandboxes, and it is silence in the operator's words. Placement already refused such a node; the predicate now has one spelling (fleet.ts awaitingFirstConfig) shared by placement, the finder and, next, the merged lists. The check-in's "the bundle rides on this answer" was logged at every check-in of a node that could not apply a bundle: two hundred and forty lines an hour for one situation. Said once per gap now, and once more when the node catches up (the twins-warning discipline). readNodeConfig read the settings row three times for one bundle; one read carries the version, the knobs and the store together. --- packages/gateway/src/app.test.ts | 55 ++++++++++++++++++++++++++ packages/gateway/src/db/node-config.ts | 17 ++++---- packages/gateway/src/db/settings.ts | 34 ++++++++++++---- packages/gateway/src/find.test.ts | 38 ++++++++++++++++++ packages/gateway/src/find.ts | 37 ++++++++++++++--- packages/gateway/src/fleet.ts | 14 +++++++ packages/gateway/src/placement.ts | 10 ++--- packages/gateway/src/routes/nodes.ts | 36 +++++++++++++---- 8 files changed, 209 insertions(+), 32 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 230fdc34..70de3fdf 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -615,6 +615,61 @@ describe('check-in and the node verbs', () => { expect(warned()).toHaveLength(3); }); + it('a node behind on configuration is said to be so once per gap, not at every check-in; catching up is said once', async () => { + const logs: string[] = []; + const h = await gateway(['b'], {}, { logs }); + const node = h.nodes[0]; + if (!node) throw new Error('node lost'); + const rides = () => + logs.filter((l) => l.includes('the bundle rides on this answer')); + const caughtUp = () => + logs.filter((l) => + l.includes('now runs the current configuration version'), + ); + // The harness's first check-in reported version 1 = current: nothing + // said. Three check-ins on version 0 while current is 1: one line. + expect(rides()).toHaveLength(0); + for (let i = 0; i < 3; i += 1) { + await rpc( + h, + '/checkIn', + checkInOf('b', node.endpoint, { configVersion: 0 }), + ); + } + expect(rides()).toHaveLength(1); + expect(JSON.parse(rides()[0] ?? '{}')).toMatchObject({ + nodeId: 'b', + runs: 0, + current: 1, + }); + // An edit widens the gap: news again, once. + await rpc(h, '/updateSettings', { pidsLimit: 512 }); + await rpc( + h, + '/checkIn', + checkInOf('b', node.endpoint, { configVersion: 0 }), + ); + await rpc( + h, + '/checkIn', + checkInOf('b', node.endpoint, { configVersion: 0 }), + ); + expect(rides()).toHaveLength(2); + // The node applies it: said once, then nothing while it stays current. + await rpc( + h, + '/checkIn', + checkInOf('b', node.endpoint, { configVersion: 2 }), + ); + await rpc( + h, + '/checkIn', + checkInOf('b', node.endpoint, { configVersion: 2 }), + ); + expect(caughtUp()).toHaveLength(1); + expect(rides()).toHaveLength(2); + }); + it('right after a gateway start a node not yet heard from cannot be removed; past two default intervals it can', async () => { // A restart: the rows are known, nothing has checked in yet. const fresh = await gateway(['b']); diff --git a/packages/gateway/src/db/node-config.ts b/packages/gateway/src/db/node-config.ts index ad0ac7d2..78875ef7 100644 --- a/packages/gateway/src/db/node-config.ts +++ b/packages/gateway/src/db/node-config.ts @@ -1,26 +1,27 @@ import type { NodeConfigBundle } from '@dormice/shared'; import type { Db } from './db'; -import { readConfigVersion, readS3Settings, readSettings } from './settings'; +import { readSettingsForBundle } from './settings'; import { listTemplates } from './templates'; /** * The bundle a node applies (shared nodeConfigBundleSchema): the settings - * row with the store's keys, the node's own row, every template — read - * back to back on one synchronous connection, so a node never receives - * one version's number with another version's content (nothing runs - * between two better-sqlite3 statements in the same tick). + * row — version, knobs and the store's keys from one read — the node's + * own row, every template; the two statements run back to back on one + * synchronous connection, so a node never receives one version's number + * with another version's content (nothing runs between two better-sqlite3 + * statements in the same tick). */ export function readNodeConfig( db: Db, node: { swapGb: number }, ): NodeConfigBundle { - const settings = readSettings(db); + const { version, settings, s3 } = readSettingsForBundle(db); return { - version: readConfigVersion(db), + version, settings: { sandboxDefaults: settings.sandboxDefaults, defaultPolicy: settings.defaultPolicy, - s3: readS3Settings(db), + s3, sandboxDomain: settings.sandboxDomain, sandboxDomainAliases: settings.sandboxDomainAliases, pidsLimit: settings.pidsLimit, diff --git a/packages/gateway/src/db/settings.ts b/packages/gateway/src/db/settings.ts index 0a769e40..0d758fe7 100644 --- a/packages/gateway/src/db/settings.ts +++ b/packages/gateway/src/db/settings.ts @@ -111,13 +111,8 @@ export function readConfigVersion(db: Db): number { return readRow(db).version; } -/** - * The S3 store in force, keys included — for the probe that guards a - * settings write and for the bundle a node pulls, never for the - * observation wire (readSettings withholds both keys). - */ -export function readS3Settings(db: Db): S3Settings | null { - const row = readRow(db); +/** The six S3 columns read back as one unit, keys included — or null when the store is off. */ +function s3Of(row: SettingsRow): S3Settings | null { if (row.s3Endpoint === null) return null; return { endpoint: row.s3Endpoint, @@ -131,6 +126,31 @@ export function readS3Settings(db: Db): S3Settings | null { }; } +/** + * The S3 store in force, keys included — for the probe that guards a + * settings write, never for the observation wire (readSettings withholds + * both keys). + */ +export function readS3Settings(db: Db): S3Settings | null { + return s3Of(readRow(db)); +} + +/** + * What of this row a node's bundle carries — the version, the knobs and + * the store with its keys — from one read (db/node-config.ts): one + * statement at every check-in that carries a bundle instead of three, and + * no way for the version to come from one write and the content from the + * next. + */ +export function readSettingsForBundle(db: Db): { + version: number; + settings: RuntimeSettings; + s3: S3Settings | null; +} { + const row = readRow(db); + return { version: row.version, settings: toView(row), s3: s3Of(row) }; +} + /** * Counts a configuration change the nodes must hear about. Callers that * change something outside this row (a template, a node's swap target) diff --git a/packages/gateway/src/find.test.ts b/packages/gateway/src/find.test.ts index 05885523..c91f286e 100644 --- a/packages/gateway/src/find.test.ts +++ b/packages/gateway/src/find.test.ts @@ -175,6 +175,44 @@ describe('Finder', () => { expect(queries.length).toBe(4); }); + it('a node that reported no configuration copy is not dialled: empty, it is a no; holding sandboxes, it is silence saying so — and a node not heard from since the start is asked', async () => { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const fleet = new Fleet(db); + fleet.checkIn(checkInOf('a', 'http://a:80'), NOW); + // b is booting: checked in, holds no copy, its port is shut. + fleet.checkIn( + checkInOf('b', 'http://b:80', { configVersion: null, active: 0 }), + NOW, + ); + const { ask, asked } = scripted({}); + const finder = new Finder(fleet, new NameCache(), ask, silentLog); + expect(await finder.byName('new-name')).toEqual({ kind: 'none' }); + expect(asked).toEqual(['a']); + + // The same node, but its ledger holds sandboxes (a node upgraded into + // the fleet, waiting for its first bundle): unreachable, not absent. + asked.length = 0; + fleet.checkIn( + checkInOf('b', 'http://b:80', { configVersion: null, active: 3 }), + NOW, + ); + const unsure = await finder.byName('new-name'); + expect(unsure).toMatchObject({ + kind: 'unsure', + silent: [{ nodeId: 'b', why: expect.stringMatching(/not listening/) }], + }); + expect(asked).toEqual(['a']); + + // A gateway restarted over the same rows: nothing has checked in, + // both are asked — b may be running on the copy it kept. + asked.length = 0; + await new Finder(new Fleet(db), new NameCache(), ask, silentLog).byName( + 'new-name', + ); + expect(asked.sort()).toEqual(['a', 'b']); + }); + it('an empty fleet finds nothing and asks nobody', async () => { const { ask, asked } = scripted({}); expect( diff --git a/packages/gateway/src/find.ts b/packages/gateway/src/find.ts index 4c267c2d..7db0e8a1 100644 --- a/packages/gateway/src/find.ts +++ b/packages/gateway/src/find.ts @@ -1,7 +1,7 @@ import type { SignedFileLookup } from '@dormice/shared'; import type { CacheEntry, NameCache } from './cache'; -import type { Fleet, NodeState } from './fleet'; -import type { AskNode, LookupQuery } from './lookup'; +import { awaitingFirstConfig, type Fleet, type NodeState } from './fleet'; +import type { AskNode, LookupAnswer, LookupQuery } from './lookup'; /** * Where a sandbox is, adjudicated once for every face: @@ -67,6 +67,33 @@ export class Finder { return this.find(this.cache.getById(id), { id }, false); } + /** + * One node's answer to one question — or, for a node that has checked + * in since this gateway started and reported no configuration copy, the + * answer without the question (fleet.ts awaitingFirstConfig): its port + * is shut until its first bundle applies, so a dial there is refused at + * the socket and would read as silence — a 503 to every caller of every + * uncached name for as long as the node boots (left by the second cut's + * review, 2026-09-14). Its reading says what it holds: nothing, and it + * is a plain no; sandboxes, and it is silence in the operator's words — + * they are there, and unreachable until its next check-in says the port + * is open. + */ + private askNode(node: NodeState, query: LookupQuery): Promise { + if (awaitingFirstConfig(node)) { + const total = node.reading?.sandboxes.total ?? 0; + return Promise.resolve( + total === 0 + ? { kind: 'absent' } + : { + kind: 'silent', + why: `not listening — it holds ${total} sandboxes but no configuration copy yet, and its first bundle rides on its next check-in`, + }, + ); + } + return this.ask(node, query); + } + /** * The bare signed-URL form: the signature is the only identity the * request carries, and only the secret of the node that minted it reads @@ -93,7 +120,7 @@ export class Finder { } else if (!confirm) { return { kind: 'one', node, id: cached.id, name: cached.name }; } else { - const answer = await this.ask(node, { id: cached.id }); + const answer = await this.askNode(node, { id: cached.id }); if (answer.kind === 'found') { return { kind: 'one', node, id: answer.id, name: answer.name }; } @@ -115,7 +142,7 @@ export class Finder { const answers = await Promise.all( members.map(async (node) => ({ node, - answer: await this.ask(node, query), + answer: await this.askNode(node, query), })), ); const found = answers.flatMap(({ node, answer }) => @@ -169,7 +196,7 @@ export class Finder { this.cache.evict(entry); return; } - const answer = await this.ask(node, { id: entry.id }); + const answer = await this.askNode(node, { id: entry.id }); if (answer.kind === 'absent') this.cache.evict(entry); } } diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 15f278ab..6dbb7a14 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -64,6 +64,20 @@ export function downReason(node: NodeState, now: Date): string | null { return null; } +/** + * A node that has checked in since this gateway started and reported no + * configuration copy: its daemon fetches its first bundle before it opens + * its port (server/main.ts, CheckIn.untilConfigured), so until its next + * check-in says otherwise nothing dialled there answers — the socket is + * shut. Placement refuses it, a lookup does not dial it (find.ts), a + * merged list does not wait on it (merge.ts). A node not heard from at + * all since the start is not this: it may well be running on a copy it + * kept, and is asked like any other. + */ +export function awaitingFirstConfig(node: NodeState): boolean { + return node.reading !== null && node.configVersion === null; +} + /** What a check-in came to: taken (and whether it joined or moved), or refused with the sentence the node is told (routes/nodes.ts answers 409). */ export type CheckInOutcome = | { node: NodeState; joined: boolean; movedFrom: string | null } diff --git a/packages/gateway/src/placement.ts b/packages/gateway/src/placement.ts index 326e4dd3..53f38e9e 100644 --- a/packages/gateway/src/placement.ts +++ b/packages/gateway/src/placement.ts @@ -1,4 +1,4 @@ -import { downReason, type NodeState } from './fleet'; +import { awaitingFirstConfig, downReason, type NodeState } from './fleet'; export interface PlacementKnobs { /** A reading above this refuses the node. */ @@ -90,10 +90,10 @@ export function pick( continue; } // A node that reported no configuration copy has no defaults to build - // a sandbox from — and is not listening yet: the daemon fetches its - // first bundle before it opens its port (server/main.ts), and a create - // sent there would be refused at the socket. - if (node.configVersion === null) { + // a sandbox from — and is not listening yet (fleet.ts + // awaitingFirstConfig): a create sent there would be refused at the + // socket. + if (awaitingFirstConfig(node)) { refuse( 'holds no configuration copy yet — its first bundle rides on its next check-in', ); diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index e6d14bd4..c11cac37 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -57,6 +57,17 @@ export const checkInRoutes: FastifyPluginAsyncZod< * the same discipline, server/check-in.ts). */ const twinsWarned = new Map(); + /** + * Per node, the gap (version it runs → version current) the bundle was + * last said to ride on — so the line below is said when a node falls + * behind or the gap changes, not at every check-in: a node that cannot + * apply a bundle reports the old version every fifteen seconds and is + * answered the bundle every time (the retry is the protocol, + * server/check-in.ts) — one situation, not two hundred and forty lines + * an hour (left by the second cut's review, 2026-09-14). Catching up is + * said once too: it is the edit's arrival at that node. + */ + const bundleSaid = new Map(); app.post( '/checkIn', @@ -119,15 +130,26 @@ export const checkInRoutes: FastifyPluginAsyncZod< ); } const version = readConfigVersion(db); - if (request.body.configVersion === version) { + const runs = request.body.configVersion; + if (runs === version) { + if (bundleSaid.delete(node.id)) { + request.log.info( + { nodeId: node.id, version }, + 'the node now runs the current configuration version', + ); + } return { configVersion: version }; } - request.log.info( - { nodeId: node.id, runs: request.body.configVersion, current: version }, - request.body.configVersion === null - ? 'a node with no configuration copy checked in; the bundle rides on this answer' - : 'a node runs another configuration version; the bundle rides on this answer', - ); + const gap = `${String(runs)}→${version}`; + if (bundleSaid.get(node.id) !== gap) { + bundleSaid.set(node.id, gap); + request.log.info( + { nodeId: node.id, runs, current: version }, + runs === null + ? 'a node with no configuration copy checked in; the bundle rides on this answer' + : 'a node runs another configuration version; the bundle rides on this answer', + ); + } return { configVersion: version, config: readNodeConfig(db, node) }; }, ); From 179f92c9e4967c9e9b20ca2e672e53801dff8e9e Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 00:10:59 +0800 Subject: [PATCH 46/89] The fleet's observation wire: lists say which nodes they could not include, a machine's reading names its node, the fleet's sums and state history are the gateway's verbs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shapes the third cut builds on, with every caller moved over and nothing merged yet (the gateway still answers 501 for the lists). listSandboxes, listSandboxMetrics and listSandboxImages carry an optional `silent` — the nodes a gateway's answer could not include, with why. A list that quietly lacked a node would pass for the whole fleet, and refusing the whole list for one node would blind the operator when a node is in trouble. The SDK's three methods return the whole response so the array is never mistaken for the whole; `dor sandbox ls` says it under the table. getHostMetrics and getHostMetricsHistory take `nodeId`: N machines' CPU percentages add up to nothing, so a machine's reading names its machine (a fleet of one needs no name). The figures that do add up — the census, the disks' bill — are getFleetMetrics, from the readings the gateway already holds; the check-in reading carries sandboxDisks for it, optional on the wire so a node on the previous build still checks in during a rolling upgrade. getFleetTimeline is getFleetStateHistory (design record #26: a count per state per moment is a state sample, not a snapshot), a gateway verb; the node answers it under the new name for one more step and stops in the next. The pure half of a history verb — window, bucket width, the whole-row bucketing — moves to @dormice/server/history, a subpath the gateway can import without the executor. GATEWAY_ONLY_VERBS lists the verbs that answer at the gateway alone, for the node's 404 and the gateway's suite to share. --- .../fleet-observation-at-the-gateway.md | 7 + e2e/src/archive.test.ts | 4 +- e2e/src/e2b.test.ts | 2 +- e2e/src/gateway.test.ts | 6 +- e2e/src/native.test.ts | 34 ++-- e2e/src/settings-hot.test.ts | 2 +- packages/cli/src/commands.ts | 32 +++- .../overview/components/FleetChart.tsx | 7 +- .../overview/components/FleetStatCards.tsx | 2 +- .../overview/hooks/useFleetTimeline.ts | 4 +- packages/console/src/lib/api.ts | 17 +- packages/console/vite.config.ts | 3 +- packages/gateway/src/routes/native.ts | 2 +- packages/sdk/src/client.test.ts | 12 +- packages/sdk/src/client.ts | 102 +++++++---- packages/server/package.json | 4 + packages/server/src/check-in.test.ts | 31 +++- packages/server/src/check-in.ts | 8 +- packages/server/src/db/metrics.ts | 68 +------- packages/server/src/e2b/control.ts | 8 +- packages/server/src/history.ts | 83 +++++++++ packages/server/src/main.ts | 2 +- packages/server/src/routes/host.ts | 23 +-- .../server/src/routes/observability.test.ts | 19 ++- packages/server/src/routes/sandboxes.ts | 8 +- packages/server/tsup.config.ts | 8 +- packages/shared/src/gateway.ts | 160 ++++++++++++++++++ packages/shared/src/host.ts | 61 +++++-- packages/shared/src/images.ts | 3 + packages/shared/src/list.ts | 13 +- packages/shared/src/metrics.ts | 75 +------- 31 files changed, 523 insertions(+), 287 deletions(-) create mode 100644 .changeset/fleet-observation-at-the-gateway.md create mode 100644 packages/server/src/history.ts diff --git a/.changeset/fleet-observation-at-the-gateway.md b/.changeset/fleet-observation-at-the-gateway.md new file mode 100644 index 00000000..9e7d1b96 --- /dev/null +++ b/.changeset/fleet-observation-at-the-gateway.md @@ -0,0 +1,7 @@ +--- +"@dormice/shared": minor +"@dormice/sdk": minor +"@dormice/cli": minor +--- + +The fleet-wide observation verbs answer at the gateway and say what they could not see. `listSandboxes`, `listSandboxMetrics` and `listSandboxImages` carry an optional `silent` array — the nodes the gateway could not include, with why — and the SDK's three methods now return the whole response instead of the bare array (`.sandboxes`, `.samples`, `.images`). `getHostMetrics` and `getHostMetricsHistory` take an optional `nodeId`: a machine's reading names its machine, the fleet's sums are the new `getFleetMetrics`. `getFleetTimeline` is `getFleetStateHistory` (a count per state per moment is a state sample, not a snapshot). `dor sandbox ls` warns under the table when a node did not answer. diff --git a/e2e/src/archive.test.ts b/e2e/src/archive.test.ts index 3feb9c33..8c174dd9 100644 --- a/e2e/src/archive.test.ts +++ b/e2e/src/archive.test.ts @@ -37,7 +37,7 @@ describe('the archive lifecycle over a real daemon', () => { // idle clock and keep the sandbox warm forever. const deadline = Date.now() + 15_000; for (;;) { - const sandboxes = await dormice.listSandboxes(); + const { sandboxes } = await dormice.listSandboxes(); const mine = sandboxes.find((s) => s.name === 'archive-key'); if (mine?.state === 'archived') break; if (Date.now() > deadline) { @@ -82,7 +82,7 @@ describe('the archive lifecycle over a real daemon', () => { }); const deadline = Date.now() + 15_000; for (;;) { - const sandboxes = await dormice.listSandboxes(); + const { sandboxes } = await dormice.listSandboxes(); const mine = sandboxes.find((s) => s.name === 'archive-destroy-key'); if (mine?.state === 'archived') break; if (Date.now() > deadline) { diff --git a/e2e/src/e2b.test.ts b/e2e/src/e2b.test.ts index 0a5e0ce6..3f9bb8b0 100644 --- a/e2e/src/e2b.test.ts +++ b/e2e/src/e2b.test.ts @@ -597,7 +597,7 @@ describe('official e2b SDK against the daemon', () => { ); // The real wall-clock scanner freezes the sandbox under the open watch. const frozen = async () => { - const sandboxes = await dormice.listSandboxes(); + const { sandboxes } = await dormice.listSandboxes(); return sandboxes.find((s) => s.name === name)?.state; }; const deadline = Date.now() + 15_000; diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts index ecd8f57b..a146ef29 100644 --- a/e2e/src/gateway.test.ts +++ b/e2e/src/gateway.test.ts @@ -248,9 +248,11 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { expect(created.created).toBe(true); expect(['node-b', 'node-c']).toContain(created.sandbox.nodeId); const here = await direct(created.sandbox.nodeId).listSandboxes(); - expect(here.some((s) => s.id === created.sandbox.id)).toBe(true); + expect(here.sandboxes.some((s) => s.id === created.sandbox.id)).toBe( + true, + ); const there = await direct(other(created.sandbox.nodeId)).listSandboxes(); - expect(there.some((s) => s.name === 'gw-place')).toBe(false); + expect(there.sandboxes.some((s) => s.name === 'gw-place')).toBe(false); const again = await viaGateway().acquireSandbox('gw-place'); expect(again.created).toBe(false); diff --git a/e2e/src/native.test.ts b/e2e/src/native.test.ts index f44adbb7..657ebe01 100644 --- a/e2e/src/native.test.ts +++ b/e2e/src/native.test.ts @@ -94,7 +94,9 @@ describe('native API over a real daemon', () => { expect(updated.sandbox.metadata).toEqual({ app: 'assistant' }); const listed = await client().listSandboxes(); - expect(listed.find((s) => s.name === 'meta-key')?.metadata).toEqual({ + expect( + listed.sandboxes.find((s) => s.name === 'meta-key')?.metadata, + ).toEqual({ app: 'assistant', }); @@ -172,7 +174,9 @@ describe('native API over a real daemon', () => { destroyed: true, }); const listed = await client().listSandboxes(); - expect(listed.some((s) => s.id === created.sandbox.id)).toBe(false); + expect(listed.sandboxes.some((s) => s.id === created.sandbox.id)).toBe( + false, + ); expect(await client().destroySandbox('destroy-key')).toEqual({ destroyed: false, }); @@ -352,7 +356,7 @@ describe('native API over a real daemon', () => { }); const result = await client().execCommand('exec-busy-key', 'sleep 3'); expect(result.exitCode).toBe(0); - const observed = (await client().listSandboxes()).find( + const observed = (await client().listSandboxes()).sandboxes.find( (s) => s.name === 'exec-busy-key', ); expect(observed?.state).toBe('active'); @@ -369,7 +373,7 @@ describe('native API over a real daemon', () => { // Watch it actually freeze from outside, on real wall-clock time. const deadline = Date.now() + 15_000; for (;;) { - const cold = (await client().listSandboxes()).find( + const cold = (await client().listSandboxes()).sandboxes.find( (s) => s.name === 'exec-wake-key', ); if (cold?.state === 'frozen') break; @@ -386,7 +390,7 @@ describe('native API over a real daemon', () => { const result = await client().execCommand('exec-wake-key', 'echo woke'); expect(result.exitCode).toBe(0); expect(result.stdout).toBe('woke\n'); - const observed = (await client().listSandboxes()).find( + const observed = (await client().listSandboxes()).sandboxes.find( (s) => s.name === 'exec-wake-key', ); expect(observed?.state).toBe('active'); @@ -410,7 +414,7 @@ describe('native API over a real daemon', () => { // own, in a separate process, on real wall-clock time. const deadline = Date.now() + 15_000; for (;;) { - const asleep = (await client().listSandboxes()).find( + const asleep = (await client().listSandboxes()).sandboxes.find( (s) => s.name === 'sleeper-key', ); if (asleep?.state === 'stopped') break; @@ -511,7 +515,7 @@ describe('native API over a real daemon', () => { template: 'native-tpl', }); expect(created.sandbox.template).toBe('native-tpl'); - const listed = (await client().listSandboxes()).find( + const listed = (await client().listSandboxes()).sandboxes.find( (s) => s.name === 'tpl-key', ); expect(listed?.template).toBe('native-tpl'); @@ -612,7 +616,7 @@ describe('native API over a real daemon', () => { template: 'lineage-tpl', }); const mine = async () => - (await client().listSandboxImages()).find( + (await client().listSandboxImages()).images.find( (e) => e.sandboxName === 'lineage-key', ); @@ -678,7 +682,7 @@ describe('native API over a real daemon', () => { // Watch it actually freeze from outside, on real wall-clock time. const deadline = Date.now() + 15_000; for (;;) { - const cold = (await client().listSandboxes()).find( + const cold = (await client().listSandboxes()).sandboxes.find( (s) => s.name === 'swap-key', ); if (cold?.state === 'frozen') break; @@ -702,7 +706,7 @@ describe('native API over a real daemon', () => { // The deployment check: the lineage row reports the new image and the // upgradable flag has cleared. expect( - (await client().listSandboxImages()).find( + (await client().listSandboxImages()).images.find( (e) => e.sandboxName === 'swap-key', ), ).toMatchObject({ image: 'img:swap-v2', upgradable: false }); @@ -751,7 +755,7 @@ describe('native API over a real daemon', () => { // Watch it actually freeze from outside, on real wall-clock time. const deadline = Date.now() + 15_000; for (;;) { - const cold = (await client().listSandboxes()).find( + const cold = (await client().listSandboxes()).sandboxes.find( (s) => s.name === 'files-wake-key', ); if (cold?.state === 'frozen') break; @@ -765,7 +769,7 @@ describe('native API over a real daemon', () => { const read = await client().readFile('files-wake-key', 'keep.txt'); expect(new TextDecoder().decode(read.content)).toBe('still here'); - const observed = (await client().listSandboxes()).find( + const observed = (await client().listSandboxes()).sandboxes.find( (s) => s.name === 'files-wake-key', ); expect(observed?.state).toBe('active'); @@ -868,17 +872,17 @@ describe('the observability verbs over a real daemon', () => { ).rejects.toMatchObject({ name: 'DormiceApiError', status: 404 }); }); - it('getFleetTimeline reports points and a peak once the fleet was seen', async () => { + it('getFleetStateHistory reports points and a peak once the fleet was seen', async () => { await client().acquireSandbox('obs-timeline-key'); const deadline = Date.now() + 15_000; - let timeline = await client().getFleetTimeline(); + let timeline = await client().getFleetStateHistory(); // Wait for a tick that observed at least one sandbox alive. while ( (timeline.points.length < 1 || (timeline.peak?.active ?? 0) < 1) && Date.now() < deadline ) { await sleep(0.5); - timeline = await client().getFleetTimeline(); + timeline = await client().getFleetStateHistory(); } expect(timeline.points.length).toBeGreaterThanOrEqual(1); expect(timeline.peak?.active).toBeGreaterThanOrEqual(1); diff --git a/e2e/src/settings-hot.test.ts b/e2e/src/settings-hot.test.ts index 9a69cbbf..d871242e 100644 --- a/e2e/src/settings-hot.test.ts +++ b/e2e/src/settings-hot.test.ts @@ -121,7 +121,7 @@ describe('the S3 archive store as a live ledger setting', () => { try { const deadline = Date.now() + 15_000; for (;;) { - const mine = (await node().listSandboxes()).find( + const mine = (await node().listSandboxes()).sandboxes.find( (s) => s.name === 'settings-hot-held', ); if (mine?.state === 'archived') break; diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index dc2ebd93..d93c5f4e 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -71,16 +71,28 @@ function renderTable(headers: string[], rows: string[][]): string { return [line(headers), ...rows.map(line)].join('\n'); } -/** `dor sandbox ls`: every sandbox with its lifecycle state, as plain columns. */ +/** + * `dor sandbox ls`: every sandbox with its lifecycle state, as plain + * columns. Asked of the gateway, the list may lack a node that did not + * answer — said under the table, one line per node, never dropped: an + * operator reading "No sandboxes." while a node is down must be told. + */ export async function sandboxLs(client: Dormice): Promise { - const sandboxes = await client.listSandboxes(); - if (sandboxes.length === 0) { - return 'No sandboxes.'; - } - return renderTable( - COLUMNS.map((column) => column.header), - sandboxes.map((s) => COLUMNS.map((column) => printable(column.value(s)))), + const { sandboxes, silent = [] } = await client.listSandboxes(); + const table = + sandboxes.length === 0 + ? 'No sandboxes.' + : renderTable( + COLUMNS.map((column) => column.header), + sandboxes.map((s) => + COLUMNS.map((column) => printable(column.value(s))), + ), + ); + const warnings = silent.map( + (node) => + `warning: node ${printable(node.nodeId)} did not answer (${printable(node.why)}) — its sandboxes are not listed`, ); + return [table, ...warnings].join('\n'); } /** @@ -113,7 +125,9 @@ export async function sandboxMeta( ): Promise { if (labels === null) { // Read path: the native list is the one read the daemon offers. - const sandbox = (await client.listSandboxes()).find((s) => s.name === name); + const sandbox = (await client.listSandboxes()).sandboxes.find( + (s) => s.name === name, + ); if (!sandbox) { throw new Error(`no sandbox named "${name}" — acquire it first`); } diff --git a/packages/console/src/features/overview/components/FleetChart.tsx b/packages/console/src/features/overview/components/FleetChart.tsx index 56ca6301..0d2b7cdc 100644 --- a/packages/console/src/features/overview/components/FleetChart.tsx +++ b/packages/console/src/features/overview/components/FleetChart.tsx @@ -1,4 +1,7 @@ -import type { GetFleetTimelineResponse, SandboxState } from '@dormice/shared'; +import type { + GetFleetStateHistoryResponse, + SandboxState, +} from '@dormice/shared'; import { Area, AreaChart, CartesianGrid, XAxis, YAxis } from 'recharts'; import { Card, @@ -66,7 +69,7 @@ type ChartRow = { at: number } & Record; * 样本则取中位点距(采样间隔是服务端配置,客户端不猜死值)。 */ function toChartRows( - points: GetFleetTimelineResponse['points'], + points: GetFleetStateHistoryResponse['points'], bucketSeconds: number | null, ): ChartRow[] { const rows: ChartRow[] = points.map((p) => ({ diff --git a/packages/console/src/features/overview/components/FleetStatCards.tsx b/packages/console/src/features/overview/components/FleetStatCards.tsx index 6f8542e3..34777c0e 100644 --- a/packages/console/src/features/overview/components/FleetStatCards.tsx +++ b/packages/console/src/features/overview/components/FleetStatCards.tsx @@ -16,7 +16,7 @@ import { StatCard, StatCardSkeleton } from './StatCard'; * (5 秒一刷的快照 + 窗口内活跃数 sparkline)、窗口峰值、总数、 * 沙箱磁盘账单。容量上限随讨论稿 #23 删(2026-09-14):账本行数不是 * 资源,数据盘水位才是——它有自己的卡。当前值来自 /getHostMetrics;峰值与 sparkline 来自 - * /getFleetTimeline — daemon 采样器 30 秒落一行,峰值由原始行现算, + * /getFleetStateHistory — 网关每次节点报到落一行,峰值由原始行现算, * 分桶抹不掉它。档位由页头的全局切换器驱动。 */ export function FleetStatCards({ range }: { range: TimelineRangeKey }) { diff --git a/packages/console/src/features/overview/hooks/useFleetTimeline.ts b/packages/console/src/features/overview/hooks/useFleetTimeline.ts index f0686328..76c9ed60 100644 --- a/packages/console/src/features/overview/hooks/useFleetTimeline.ts +++ b/packages/console/src/features/overview/hooks/useFleetTimeline.ts @@ -1,5 +1,5 @@ import { keepPreviousData, useQuery } from '@tanstack/react-query'; -import { getFleetTimeline } from '@/lib/api'; +import { getFleetStateHistory } from '@/lib/api'; import { m } from '@/paraglide/messages'; /** @@ -34,7 +34,7 @@ export function useFleetTimeline(range: TimelineRangeKey) { queryFn: () => { const end = Date.now(); const start = end - rangeSpanMs(range); - return getFleetTimeline( + return getFleetStateHistory( new Date(start).toISOString(), new Date(end).toISOString(), ); diff --git a/packages/console/src/lib/api.ts b/packages/console/src/lib/api.ts index de955395..b038b3e8 100644 --- a/packages/console/src/lib/api.ts +++ b/packages/console/src/lib/api.ts @@ -6,7 +6,7 @@ import type { CheckUpgradeResponse, CreateApiKeyResponse, GetConfigResponse, - GetFleetTimelineResponse, + GetFleetStateHistoryResponse, GetHostMetricsHistoryResponse, GetIngressResponse, GetSandboxMetricsHistoryResponse, @@ -14,6 +14,7 @@ import type { GetUpgradeStatusResponse, HostMetricsResponse, LifecyclePolicyOverride, + ListSandboxesResponse, ListSandboxImagesResponse, ListSandboxMetricsResponse, RegisterTemplateResponse, @@ -139,8 +140,7 @@ export const login = (input: { username: string; password: string }) => export const logout = () => rpc<{ loggedIn: false }>('/console/auth/logout', {}, { intercept401: false }); -export const listSandboxes = () => - rpc<{ sandboxes: Sandbox[] }>('/listSandboxes'); +export const listSandboxes = () => rpc('/listSandboxes'); // The host-level observation window: machine health plus fleet aggregates. // Pure observation — the daemon wakes nothing to answer it. @@ -170,11 +170,12 @@ export const getSandboxMetricsHistory = ( end, }); -// Fleet state counts over time — the concurrency curve's data. Bucketed -// points are whole raw snapshots (byState always sums to total); peak is -// computed from raw rows and immune to bucketing. -export const getFleetTimeline = (start: string, end: string) => - rpc('/getFleetTimeline', { start, end }); +// Fleet state counts over time — the concurrency curve's data, kept by the +// gateway one sample per node check-in. Bucketed points are whole raw +// samples (byState always sums to total); peak is computed from raw rows +// and immune to bucketing. +export const getFleetStateHistory = (start: string, end: string) => + rpc('/getFleetStateHistory', { start, end }); // The machine's sampled past — the host health card's trend food. Buckets // keep each field's worst case (max usage, min available) so spikes diff --git a/packages/console/vite.config.ts b/packages/console/vite.config.ts index fed898d4..d91404fd 100644 --- a/packages/console/vite.config.ts +++ b/packages/console/vite.config.ts @@ -59,7 +59,8 @@ export default defineConfig({ '/getSandboxMetricsHistory', '/listSandboxMetrics', '/listSandboxImages', - '/getFleetTimeline', + '/getFleetStateHistory', + '/getFleetMetrics', '/getConfig', '/checkUpgrade', '/applyUpgrade', diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index 93855773..279ed5c5 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -58,7 +58,7 @@ export const UNNAMED_VERBS = [ 'listSandboxes', 'listSandboxMetrics', 'listSandboxImages', - 'getFleetTimeline', + 'getFleetStateHistory', 'getHostMetrics', 'getHostMetricsHistory', 'checkUpgrade', diff --git a/packages/sdk/src/client.test.ts b/packages/sdk/src/client.test.ts index d5799280..ba81fb4b 100644 --- a/packages/sdk/src/client.test.ts +++ b/packages/sdk/src/client.test.ts @@ -197,8 +197,10 @@ describe('Dormice.acquireSandbox over real HTTP', () => { it('lists sandboxes with their lifecycle states', async () => { await client.acquireSandbox('erin'); - const sandboxes = await client.listSandboxes(); + const { sandboxes, silent } = await client.listSandboxes(); const erin = sandboxes.find((s) => s.name === 'erin'); + // A node's own list has nobody to be silent about. + expect(silent).toBeUndefined(); expect(erin?.state).toBe('active'); }); @@ -208,7 +210,7 @@ describe('Dormice.acquireSandbox over real HTTP', () => { expect(metrics.host.cpuCount).toBeGreaterThan(0); expect(metrics.host.memTotalBytes).toBeGreaterThan(0); const listed = await client.listSandboxes(); - expect(metrics.sandboxes.total).toBe(listed.length); + expect(metrics.sandboxes.total).toBe(listed.sandboxes.length); expect(metrics.sandboxDisks.count).toBeGreaterThanOrEqual(1); expect(metrics.sandboxDisks.nominalBytes).toBeGreaterThan( metrics.sandboxDisks.actualBytes, @@ -553,7 +555,7 @@ describe('the observability verbs over real HTTP', () => { it('listSandboxMetrics answers every measurable sandbox in one call', async () => { await client.acquireSandbox('fleet-a'); await client.acquireSandbox('fleet-b'); - const samples = await client.listSandboxMetrics(); + const { samples } = await client.listSandboxMetrics(); const mine = samples.filter((s) => s.sandboxName.startsWith('fleet-')); expect(mine.map((s) => s.sandboxName).sort()).toEqual([ 'fleet-a', @@ -566,6 +568,8 @@ describe('the observability verbs over real HTTP', () => { await client.destroySandbox('fleet-b'); // Released means gone from the measurable set, not null-stuffed. const after = await client.listSandboxMetrics(); - expect(after.filter((s) => s.sandboxName.startsWith('fleet-'))).toEqual([]); + expect( + after.samples.filter((s) => s.sandboxName.startsWith('fleet-')), + ).toEqual([]); }); }); diff --git a/packages/sdk/src/client.ts b/packages/sdk/src/client.ts index 8c1ed5be..9a9702d6 100644 --- a/packages/sdk/src/client.ts +++ b/packages/sdk/src/client.ts @@ -16,13 +16,15 @@ import { execCommandResponseSchema, expandDiskResponseSchema, type GetConfigResponse, - type GetFleetTimelineResponse, + type GetFleetMetricsResponse, + type GetFleetStateHistoryResponse, type GetHostMetricsHistoryResponse, type GetIngressResponse, type GetSandboxMetricsHistoryResponse, type GetUpgradeStatusResponse, getConfigResponseSchema, - getFleetTimelineResponseSchema, + getFleetMetricsResponseSchema, + getFleetStateHistoryResponseSchema, getHostMetricsHistoryResponseSchema, getIngressResponseSchema, getSandboxMetricsHistoryResponseSchema, @@ -31,6 +33,7 @@ import { type HostMetricsResponse, hostMetricsResponseSchema, type LifecyclePolicyOverride, + type ListSandboxesResponse, type ListSandboxImagesResponse, type ListSandboxMetricsResponse, listApiKeysResponseSchema, @@ -48,7 +51,6 @@ import { registerTemplateResponseSchema, removeTemplateResponseSchema, revokeApiKeyResponseSchema, - type Sandbox, type SandboxMetadata, type SandboxMetricsSample, type SandboxSpecOverride, @@ -191,44 +193,71 @@ export class Dormice { return acquireResponseSchema.parse(data); } - /** Every sandbox on the daemon with its current lifecycle state. */ - async listSandboxes(): Promise { + /** + * Every sandbox with its current lifecycle state — `sandboxes`. Asked + * of the gateway, the list is every node's concatenated, and `silent` + * names any node it could not include (down, not listening yet, or too + * slow), so a shorter list is never mistaken for the whole fleet; a + * node's own answer carries no `silent`. + */ + async listSandboxes(): Promise { const data = await this.rpc('listSandboxes', {}); - return listSandboxesResponseSchema.parse(data).sandboxes; + return listSandboxesResponseSchema.parse(data); } /** - * The daemon host's own health in one snapshot: CPU, memory and swap - * (the freeze mechanism's fuel), the data disk, ledger aggregates, and - * what the sparse sandbox disks nominally promise versus actually occupy. - * Pure observation — never wakes a sandbox. Readings the platform cannot - * produce come back null, never invented. + * One machine's own health in one snapshot: CPU, memory and swap (the + * freeze mechanism's fuel), the data disk, its ledger aggregates, and + * what its sparse sandbox disks nominally promise versus actually + * occupy. Pure observation — never wakes a sandbox. Readings the + * platform cannot produce come back null, never invented. At the + * gateway `nodeId` names the machine (listNodes lists them); a fleet of + * one needs none. The fleet's sums are getFleetMetrics. */ - async getHostMetrics(): Promise { - const data = await this.rpc('getHostMetrics', {}); + async getHostMetrics(options?: { + nodeId?: string; + }): Promise { + const data = await this.rpc('getHostMetrics', { + nodeId: options?.nodeId, + }); return hostMetricsResponseSchema.parse(data); } /** - * The host machine's sampled history, sliced by an optional ISO window + * One machine's sampled history, sliced by an optional ISO window * (default: the last 24 hours). Past 360 points the server buckets the * answer — `bucketSeconds` says how wide — keeping each field's worst * case (max usage, min available), so spikes survive. `peak` carries the * window's highest whole-machine CPU percentage from raw rows, immune to * bucketing. Nulls inside a point are honest platform gaps, and a window - * the daemon was down for shows the gap. + * the daemon was down for shows the gap. `nodeId` as in getHostMetrics. */ async getHostMetricsHistory(options?: { + nodeId?: string; start?: string; end?: string; }): Promise { const data = await this.rpc('getHostMetricsHistory', { + nodeId: options?.nodeId, start: options?.start, end: options?.end, }); return getHostMetricsHistoryResponseSchema.parse(data); } + /** + * The fleet's figures that add up, from the nodes' last check-ins: how + * many nodes there are, are reachable and have reported; the sandbox + * census by state; the sandbox disks' bill. Answered by the gateway + * from what it holds — no node is asked. `nodes.reported` says how many + * nodes the sums cover; a node not heard from since the gateway started + * is not in them. + */ + async getFleetMetrics(): Promise { + const data = await this.rpc('getFleetMetrics', {}); + return getFleetMetricsResponseSchema.parse(data); + } + /** * One sandbox's point-in-time resource reading (CPU, memory, disk). * Observation never wakes: a frozen sandbox is measured as it sleeps, @@ -263,42 +292,45 @@ export class Dormice { /** * Fleet state counts over time (default window: the last 24 hours) — * how many sandboxes sat active/frozen/stopped/archived/restoring at - * each sampler tick. Bucketed points are whole raw snapshots (byState - * always sums to total); `peak` carries the window's highest active - * count from raw rows, immune to bucketing. + * each moment, summed over every node; the gateway keeps this history, + * one sample per node check-in. Bucketed points are whole raw samples + * (byState always sums to total); `peak` carries the window's highest + * active count from raw rows, immune to bucketing. */ - async getFleetTimeline(options?: { + async getFleetStateHistory(options?: { start?: string; end?: string; - }): Promise { - const data = await this.rpc('getFleetTimeline', { + }): Promise { + const data = await this.rpc('getFleetStateHistory', { start: options?.start, end: options?.end, }); - return getFleetTimelineResponseSchema.parse(data); + return getFleetStateHistoryResponseSchema.parse(data); } /** - * Every measurable sandbox's reading in one answer — a view over N - * sandboxes costs one request instead of N. Presence means measured: - * only physically running/paused sandboxes appear; colder states are - * absent (getSandboxMetrics's null, expressed as absence). + * Every measurable sandbox's reading in one answer — `samples` — so a + * view over N sandboxes costs one request instead of N. Presence means + * measured: only physically running/paused sandboxes appear; colder + * states are absent (getSandboxMetrics's null, expressed as absence). + * `silent` as in listSandboxes. */ - async listSandboxMetrics(): Promise { + async listSandboxMetrics(): Promise { const data = await this.rpc('listSandboxMetrics', {}); - return listSandboxMetricsResponseSchema.parse(data).samples; + return listSandboxMetricsResponseSchema.parse(data); } /** - * Every sandbox's image lineage in one answer: the image its current - * shell was born from (`image`, null when no shell exists), the image - * its next shell would boot (`nextImage`), and whether a rebuild would - * change anything (`upgradable`). The window answering "which sandboxes - * still run an old image?" after a template is re-registered. + * Every sandbox's image lineage in one answer — `images`: the image its + * current shell was born from (`image`, null when no shell exists), the + * image its next shell would boot (`nextImage`), and whether a rebuild + * would change anything (`upgradable`). The window answering "which + * sandboxes still run an old image?" after a template is re-registered. + * `silent` as in listSandboxes. */ - async listSandboxImages(): Promise { + async listSandboxImages(): Promise { const data = await this.rpc('listSandboxImages', {}); - return listSandboxImagesResponseSchema.parse(data).images; + return listSandboxImagesResponseSchema.parse(data); } /** diff --git a/packages/server/package.json b/packages/server/package.json index bca3d704..aafba06e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -33,6 +33,10 @@ "./s3-store": { "types": "./dist/archive/s3-store.d.ts", "default": "./dist/archive/s3-store.js" + }, + "./history": { + "types": "./dist/history.d.ts", + "default": "./dist/history.js" } }, "files": [ diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index 08d5d8c8..fe1baa4f 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -7,6 +7,7 @@ import { CheckIn, type CheckInOptions, readNodeReading } from './check-in'; import { migrateDb, openDb } from './db/db'; import { createSandbox } from './db/ledger'; import { applyNodeConfig, readConfigVersion } from './db/settings'; +import { FakeExecutor } from './executor/fake'; import { CpuSampler } from './host-metrics'; import { testBundle } from './testing'; @@ -103,7 +104,8 @@ function options( title: 'a commit', committedAt: '2026-09-14T00:00:00.000Z', }, - readReading: () => readNodeReading(db, cpu, '/nonexistent-data-dir'), + readReading: () => + readNodeReading(db, cpu, '/nonexistent-data-dir', new FakeExecutor()), // The daemon's wiring in miniature: the copy is the ledger's, applied // by the pure write (node-config.ts's hooks are its own suite). configVersion: () => readConfigVersion(db), @@ -143,10 +145,16 @@ describe('CheckIn', () => { it('the reading carries the managed swap when the daemon has one', async () => { const db = openDb(':memory:'); migrateDb(db, MIGRATIONS); - const reading = await readNodeReading(db, new CpuSampler(), '/tmp', { - status: async () => ({ activeGb: 16, blocks: [] }), - reconcile: async () => ({ activeGb: 16, blocks: [] }), - }); + const reading = await readNodeReading( + db, + new CpuSampler(), + '/tmp', + new FakeExecutor(), + { + status: async () => ({ activeGb: 16, blocks: [] }), + reconcile: async () => ({ activeGb: 16, blocks: [] }), + }, + ); expect(reading.managedSwap).toEqual({ activeGb: 16 }); }); @@ -168,10 +176,21 @@ describe('CheckIn', () => { spec: undefined, }); } - const reading = await readNodeReading(db, new CpuSampler(), '/tmp'); + const reading = await readNodeReading( + db, + new CpuSampler(), + '/tmp', + new FakeExecutor(), + ); expect(reading.sandboxes.total).toBe(2); expect(reading.sandboxes.byState.active).toBe(2); expect(reading.dataDisk?.path).toBe('/tmp'); + // The disks' bill rides along: what the gateway sums for the fleet. + expect(reading.sandboxDisks).toEqual({ + count: 0, + nominalBytes: 0, + actualBytes: 0, + }); }); it('logs a failing gateway once, and its recovery once — not every tick', async () => { diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index 38200467..d91a8ce1 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -7,12 +7,16 @@ import { } from '@dormice/shared'; import type { Db } from './db/db'; import { countByState, listSandboxes } from './db/ledger'; +import type { Executor } from './executor/executor'; import { type CpuSampler, readHostReading } from './host-metrics'; import type { SwapControl } from './swap'; /** * A node's reading for its check-in: the host half (host-metrics.ts), the - * ledger's census, and what the daemon-managed swap holds — null where + * ledger's census, what the sandbox disks cost (the executor's diskUsage — + * a readdir and a stat per disk, what the console's getHostMetrics poll + * already asked of this node every five seconds; the gateway sums it for + * the fleet instead), and what the daemon-managed swap holds — null where * the daemon manages none (a non-Linux host, the fake executor), which is * how the gateway knows to refuse a swap target for this node. */ @@ -20,12 +24,14 @@ export async function readNodeReading( db: Db, cpu: CpuSampler, dataDir: string, + executor: Executor, swap?: SwapControl, ): Promise { const { byState, total } = countByState(listSandboxes(db)); return { ...(await readHostReading(cpu, dataDir)), sandboxes: { total, byState }, + sandboxDisks: await executor.diskUsage(), managedSwap: swap === undefined ? null : { activeGb: (await swap.status()).activeGb }, }; diff --git a/packages/server/src/db/metrics.ts b/packages/server/src/db/metrics.ts index 458e37cf..6480c734 100644 --- a/packages/server/src/db/metrics.ts +++ b/packages/server/src/db/metrics.ts @@ -10,6 +10,7 @@ import { lte, } from 'drizzle-orm'; import type { SandboxMetrics } from '../executor/executor'; +import { bucketIndex } from '../history'; import type { HostSample } from '../host-metrics'; import type { Db } from './db'; import { @@ -31,14 +32,6 @@ import { */ export const FLEET_SNAPSHOT_KEEP_DAYS = 30; -/** - * The most points a history answer carries. One ceiling for every consumer - * (native verbs and the E2B slice alike): past it the server buckets, so a - * 30-day window costs ~360 points on the wire instead of 86k raw rows no - * chart could draw anyway. - */ -export const MAX_POINTS = 360; - export interface FleetCounts { active: number; frozen: number; @@ -235,46 +228,6 @@ export function queryFleetPeak( return row ?? null; } -/** - * Resolves a history verb's optional ISO window: end defaults to now, start - * to end minus the verb's default span. One resolver for both verbs so - * "defaults" cannot drift apart. Parseability is the request schema's job - * (a malformed timestamp is rejected as a 400 at the door, never NaN here). - */ -export function resolveWindow( - start: string | undefined, - end: string | undefined, - defaultSpanMs: number, - now: Date, -): { startIso: string; endIso: string; startMs: number; endMs: number } { - const endMs = end !== undefined ? Date.parse(end) : now.getTime(); - const startMs = - start !== undefined ? Date.parse(start) : endMs - defaultSpanMs; - return { - startIso: new Date(startMs).toISOString(), - endIso: new Date(endMs).toISOString(), - startMs, - endMs, - }; -} - -/** - * Decides the answer's granularity: raw under MAX_POINTS, bucketed past it. - * The server picks — one ruling for every caller, clients never negotiate. - */ -export function resolveBucketSeconds( - rawCount: number, - startMs: number, - endMs: number, -): number | null { - if (rawCount <= MAX_POINTS) return null; - return Math.max(1, Math.ceil((endMs - startMs) / 1000 / MAX_POINTS)); -} - -function bucketIndex(atIso: string, startMs: number, bucketSeconds: number) { - return Math.floor((Date.parse(atIso) - startMs) / (bucketSeconds * 1000)); -} - /** * Buckets per-sandbox samples by per-field max: someone reading history is * hunting for spikes, and averaging erases exactly what they came for. Each @@ -374,22 +327,3 @@ export function bucketHostSamples( } return [...buckets.entries()].sort(([a], [b]) => a - b).map(([, row]) => row); } - -/** - * Buckets fleet snapshots by keeping each bucket's LAST raw row whole — - * never per-state maxima, which would count a sandbox mid-transition twice - * and break byState summing to total. Every emitted point is a snapshot - * that really happened; the peak travels separately (queryFleetPeak). - */ -export function bucketSnapshots( - rows: FleetSnapshotRow[], - startMs: number, - bucketSeconds: number, -): FleetSnapshotRow[] { - const buckets = new Map(); - for (const row of rows) { - // Rows arrive ascending, so a later row simply overwrites the bucket. - buckets.set(bucketIndex(row.at, startMs, bucketSeconds), row); - } - return [...buckets.entries()].sort(([a], [b]) => a - b).map(([, row]) => row); -} diff --git a/packages/server/src/e2b/control.ts b/packages/server/src/e2b/control.ts index 9f819da0..e5bef817 100644 --- a/packages/server/src/e2b/control.ts +++ b/packages/server/src/e2b/control.ts @@ -11,15 +11,11 @@ import { setPausedByUser, touch, } from '../db/ledger'; -import { - bucketSamples, - querySandboxSamples, - resolveBucketSeconds, - resolveWindow, -} from '../db/metrics'; +import { bucketSamples, querySandboxSamples } from '../db/metrics'; import type { SandboxRow } from '../db/schema'; import { archiveEnabled, readRuntimeSettings } from '../db/settings'; import { findTemplate, resolveImage } from '../db/templates'; +import { resolveBucketSeconds, resolveWindow } from '../history'; import { destroySandbox, freezeSandbox, diff --git a/packages/server/src/history.ts b/packages/server/src/history.ts new file mode 100644 index 00000000..ed090103 --- /dev/null +++ b/packages/server/src/history.ts @@ -0,0 +1,83 @@ +/** + * The pure half of a history verb — how a window's defaults resolve, when + * an answer is bucketed and how wide, and the one bucketing that keeps + * whole rows. Shared by the daemon's own history verbs + * (getSandboxMetricsHistory, getHostMetricsHistory, the E2B metrics slice) + * and by the gateway's getFleetStateHistory over its fleet_state_samples, + * through the `@dormice/server/history` subpath: the gateway reuses the + * rulings without loading the daemon's executor, the way it reuses the + * lock and the queue. Nothing here touches a database. + */ + +/** + * The most points a history answer carries. One ceiling for every consumer + * (native verbs, the E2B slice, the gateway's fleet history): past it the + * server buckets, so a 30-day window costs ~360 points on the wire instead + * of 86k raw rows no chart could draw anyway. + */ +export const MAX_POINTS = 360; + +/** + * Resolves a history verb's optional ISO window: end defaults to now, start + * to end minus the verb's default span. One resolver for every verb so + * "defaults" cannot drift apart. Parseability is the request schema's job + * (a malformed timestamp is rejected as a 400 at the door, never NaN here). + */ +export function resolveWindow( + start: string | undefined, + end: string | undefined, + defaultSpanMs: number, + now: Date, +): { startIso: string; endIso: string; startMs: number; endMs: number } { + const endMs = end !== undefined ? Date.parse(end) : now.getTime(); + const startMs = + start !== undefined ? Date.parse(start) : endMs - defaultSpanMs; + return { + startIso: new Date(startMs).toISOString(), + endIso: new Date(endMs).toISOString(), + startMs, + endMs, + }; +} + +/** + * Decides the answer's granularity: raw under MAX_POINTS, bucketed past it. + * The server picks — one ruling for every caller, clients never negotiate. + */ +export function resolveBucketSeconds( + rawCount: number, + startMs: number, + endMs: number, +): number | null { + if (rawCount <= MAX_POINTS) return null; + return Math.max(1, Math.ceil((endMs - startMs) / 1000 / MAX_POINTS)); +} + +/** Which bucket a row falls in, counted from the window's start. */ +export function bucketIndex( + atIso: string, + startMs: number, + bucketSeconds: number, +): number { + return Math.floor((Date.parse(atIso) - startMs) / (bucketSeconds * 1000)); +} + +/** + * Buckets rows by keeping each bucket's LAST raw row whole — for a row + * whose fields are one consistent observation (a census by state: never + * per-state maxima, which would count a sandbox mid-transition twice and + * break byState summing to total). Every emitted point really happened; + * a window's peak travels separately, computed from the raw rows. Rows + * arrive ascending, so a later row simply overwrites the bucket. + */ +export function bucketLast( + rows: T[], + startMs: number, + bucketSeconds: number, +): T[] { + const buckets = new Map(); + for (const row of rows) { + buckets.set(bucketIndex(row.at, startMs, bucketSeconds), row); + } + return [...buckets.entries()].sort(([a], [b]) => a - b).map(([, row]) => row); +} diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index dd024e91..cfa6bfe2 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -208,7 +208,7 @@ const checkIn = new CheckIn({ intervalSeconds: config.DORMICE_CHECK_IN_INTERVAL_SECONDS, build, readReading: () => - readNodeReading(db, checkInCpu, config.DORMICE_DATA_DIR, swap), + readNodeReading(db, checkInCpu, config.DORMICE_DATA_DIR, executor, swap), configVersion: () => readConfigVersion(db), applyConfig: (bundle) => applyConfig(bundle, { db, executor, locks, swap, log, beat }), diff --git a/packages/server/src/routes/host.ts b/packages/server/src/routes/host.ts index 86f9c1f5..2edffdc4 100644 --- a/packages/server/src/routes/host.ts +++ b/packages/server/src/routes/host.ts @@ -1,6 +1,6 @@ import { - getFleetTimelineRequestSchema, - getFleetTimelineResponseSchema, + getFleetStateHistoryRequestSchema, + getFleetStateHistoryResponseSchema, getHostMetricsHistoryRequestSchema, getHostMetricsHistoryResponseSchema, hostMetricsResponseSchema, @@ -11,15 +11,13 @@ import type { Db } from '../db/db'; import { countByState, listSandboxes } from '../db/ledger'; import { bucketHostSamples, - bucketSnapshots, queryFleetPeak, queryFleetSnapshots, queryHostCpuPeak, queryHostSamples, - resolveBucketSeconds, - resolveWindow, } from '../db/metrics'; import type { Executor } from '../executor/executor'; +import { bucketLast, resolveBucketSeconds, resolveWindow } from '../history'; import { CpuSampler, readHostReading } from '../host-metrics'; export interface HostRoutesOptions { @@ -33,7 +31,9 @@ export interface HostRoutesOptions { * of listSandboxes. Read-only by construction: it reads the ledger, /proc, * statfs and the disk images' metadata, and touches no sandbox and no * lifecycle state (observation is not activity — the same principle that - * keeps listing and metrics from waking anything). + * keeps listing and metrics from waking anything). A request may carry + * `nodeId` (shared getHostMetricsRequestSchema): that is the gateway's + * address of this node, forwarded along, and means nothing here. */ export const hostRoutes: FastifyPluginAsyncZod = async ( app, @@ -129,13 +129,14 @@ export const hostRoutes: FastifyPluginAsyncZod = async ( // the bucket — so byState always sums to total; the concurrency peak is // computed from raw rows and travels beside the points, immune to // bucketing. A window the daemon slept through simply has no rows: the - // gap IS the answer. + // gap IS the answer. (The gateway answers this verb for the fleet from + // its own samples; this node-local answer leaves with the third cut.) app.post( - '/getFleetTimeline', + '/getFleetStateHistory', { schema: { - body: getFleetTimelineRequestSchema, - response: { 200: getFleetTimelineResponseSchema }, + body: getFleetStateHistoryRequestSchema, + response: { 200: getFleetStateHistoryResponseSchema }, }, }, async (request) => { @@ -150,7 +151,7 @@ export const hostRoutes: FastifyPluginAsyncZod = async ( const points = bucketSeconds === null ? rows - : bucketSnapshots(rows, startMs, bucketSeconds); + : bucketLast(rows, startMs, bucketSeconds); return { points: points.map((row) => ({ at: row.at, diff --git a/packages/server/src/routes/observability.test.ts b/packages/server/src/routes/observability.test.ts index 6a45d7d2..d33a345f 100644 --- a/packages/server/src/routes/observability.test.ts +++ b/packages/server/src/routes/observability.test.ts @@ -1,6 +1,6 @@ import { fileURLToPath } from 'node:url'; import { - getFleetTimelineResponseSchema, + getFleetStateHistoryResponseSchema, getHostMetricsHistoryResponseSchema, getSandboxMetricsHistoryResponseSchema, getSandboxMetricsResponseSchema, @@ -11,8 +11,9 @@ import { describe, expect, it } from 'vitest'; import { buildApp } from '../app'; import { loadConfig } from '../config'; import { migrateDb, openDb } from '../db/db'; -import { insertMetricsTick, MAX_POINTS } from '../db/metrics'; +import { insertMetricsTick } from '../db/metrics'; import { FAKE_BASE_IMAGE, FakeExecutor } from '../executor/fake'; +import { MAX_POINTS } from '../history'; import { CpuSampler, type HostSample } from '../host-metrics'; import { KeyedQueue } from '../keyed-queue'; import { freezeSandbox, stopSandbox } from '../lifecycle'; @@ -229,12 +230,12 @@ describe('getSandboxMetricsHistory', () => { }); }); -describe('getFleetTimeline', () => { +describe('getFleetStateHistory', () => { it('answers an empty window with no points and a null peak', async () => { const { app } = testApp(); - const res = await rpc(app, '/getFleetTimeline', {}); + const res = await rpc(app, '/getFleetStateHistory', {}); expect(res.statusCode).toBe(200); - const body = getFleetTimelineResponseSchema.parse(res.json()); + const body = getFleetStateHistoryResponseSchema.parse(res.json()); expect(body).toEqual({ points: [], bucketSeconds: null, peak: null }); }); @@ -246,12 +247,12 @@ describe('getFleetTimeline', () => { await rpc(app, '/acquireSandbox', { name: 'two' }); await sampleOnce(db, executor, new Date(t0 + 30_000), tickOpts()); - const res = await rpc(app, '/getFleetTimeline', { + const res = await rpc(app, '/getFleetStateHistory', { start: new Date(t0 - 1000).toISOString(), end: new Date(t0 + 60_000).toISOString(), }); const { points, bucketSeconds, peak } = - getFleetTimelineResponseSchema.parse(res.json()); + getFleetStateHistoryResponseSchema.parse(res.json()); expect(bucketSeconds).toBe(null); expect(points.map((p) => p.at)).toEqual([ new Date(t0).toISOString(), @@ -306,12 +307,12 @@ describe('getFleetTimeline', () => { retentionHours: 168, }); - const res = await rpc(app, '/getFleetTimeline', { + const res = await rpc(app, '/getFleetStateHistory', { start: new Date(t0).toISOString(), end: new Date(t0 + rows * 30_000).toISOString(), }); const { points, bucketSeconds, peak } = - getFleetTimelineResponseSchema.parse(res.json()); + getFleetStateHistoryResponseSchema.parse(res.json()); expect(bucketSeconds).not.toBe(null); expect(points.length).toBeLessThanOrEqual(MAX_POINTS); expect(peak).toEqual({ diff --git a/packages/server/src/routes/sandboxes.ts b/packages/server/src/routes/sandboxes.ts index 032b981f..990789a4 100644 --- a/packages/server/src/routes/sandboxes.ts +++ b/packages/server/src/routes/sandboxes.ts @@ -61,12 +61,7 @@ import { updateSpec, updateTemplate, } from '../db/ledger'; -import { - bucketSamples, - querySandboxSamples, - resolveBucketSeconds, - resolveWindow, -} from '../db/metrics'; +import { bucketSamples, querySandboxSamples } from '../db/metrics'; import type { SandboxRow } from '../db/schema'; import { archiveEnabled, readRuntimeSettings } from '../db/settings'; import { findTemplate, resolveImage } from '../db/templates'; @@ -78,6 +73,7 @@ import { FileTooLargeError, NotAFileError, } from '../executor/executor'; +import { resolveBucketSeconds, resolveWindow } from '../history'; import { httpError } from '../http-error'; import type { KeyedQueue } from '../keyed-queue'; import { destroySandbox, rebuildSandbox, wakeSandbox } from '../lifecycle'; diff --git a/packages/server/tsup.config.ts b/packages/server/tsup.config.ts index 2dadd3e7..9451ed75 100644 --- a/packages/server/tsup.config.ts +++ b/packages/server/tsup.config.ts @@ -25,9 +25,10 @@ const commitTime = git('log -1 --format=%cI'); export default defineConfig({ // Subpath entries beyond the root: mini-s3 for the e2e harness; auth, - // keyed-queue, lock and shutdown for the gateway, which reuses the - // daemon's small self-contained pieces without loading its executor - // (the root's import graph drags dockerode, execa and the AWS SDK in). + // keyed-queue, lock, shutdown, s3-store and history for the gateway, + // which reuses the daemon's small self-contained pieces without loading + // its executor (the root's import graph drags dockerode, execa and the + // AWS SDK in). entry: [ 'src/index.ts', 'src/main.ts', @@ -37,6 +38,7 @@ export default defineConfig({ 'src/db/lock.ts', 'src/shutdown.ts', 'src/archive/s3-store.ts', + 'src/history.ts', ], format: ['esm'], dts: true, diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts index 3803e4e9..d6b62251 100644 --- a/packages/shared/src/gateway.ts +++ b/packages/shared/src/gateway.ts @@ -2,6 +2,8 @@ import { z } from 'zod'; import { dataDiskSchema, hostReadingSchema, + isoTimestampSchema, + sandboxDisksSchema, sandboxStateCountsSchema, } from './host'; import { lifecyclePolicySchema } from './policy'; @@ -91,6 +93,16 @@ export const nodeReadingSchema = z.object({ managedSwap: z .object({ activeGb: z.number().int().nonnegative() }) .nullable(), + /** + * What this node's sandbox disks cost (host.ts sandboxDisksSchema), so + * the fleet's bill is a sum the gateway already holds (getFleetMetrics) + * and no console poll has to ask the nodes. Optional on the wire for one + * reason: a rolling upgrade takes the gateway first, and a node still on + * a build before the third cut (2026-09-14) reports without it — its + * check-in is taken, not refused with a 400 for the length of the + * upgrade. + */ + sandboxDisks: sandboxDisksSchema.optional(), }); export type NodeReading = z.infer; @@ -257,3 +269,151 @@ export const updateNodeSettingsResponseSchema = z.object({ export type UpdateNodeSettingsResponse = z.infer< typeof updateNodeSettingsResponseSchema >; + +/** + * A node a merged answer could not include, and why. The gateway's + * fleet-wide lists (listSandboxes, listSandboxMetrics, listSandboxImages) + * ask every node and concatenate; a node that is down, not listening yet + * or too slow is left out — and said, here, in the same answer. A list + * that quietly lacked a node would pass for the whole fleet, and refusing + * the whole list for one node would blind the operator exactly when a + * node is in trouble (RULES: a verb tells the truth). Always present in a + * gateway's answer, empty when every node answered; absent from a node's + * own answer, which has nobody to be silent. + */ +export const silentNodeSchema = z.object({ + nodeId: z.string(), + /** In the gateway's words: why the node was not asked (down, not listening yet) or how it failed to answer (the transport's word). */ + why: z.string(), +}); + +export type SilentNode = z.infer; + +/** + * getFleetMetrics() — the fleet's figures that add up, from what the + * gateway already holds: every node's last reading. The sandbox census by + * state and the sandbox disks' bill are sums over nodes; a machine's CPU, + * memory and swap are not, and getHostMetrics names a node for those. + * Costs no node anything — the console polls this every few seconds, and + * a poll that fanned out to every node would make the nodes' one observer + * their heaviest caller (design record #24: the console's polling was 42% + * of every request measured on the Beijing node, 2026-09-12). + * + * `nodes.reported` says how many nodes the sums cover: a node that has + * not checked in since this gateway started has no reading here, and + * until it does the sums are a lower bound — said as such, never rounded + * up. + */ +export const getFleetMetricsRequestSchema = z.object({}); + +export type GetFleetMetricsRequest = z.infer< + typeof getFleetMetricsRequestSchema +>; + +export const getFleetMetricsResponseSchema = z.object({ + nodes: z.object({ + /** Every node the gateway knows — its nodes table. */ + total: z.number().int(), + /** Checked in within two of their own intervals (listNodes' `reachable`). */ + reachable: z.number().int(), + /** Have a reading — checked in since this gateway started. The sums below cover exactly these. */ + reported: z.number().int(), + }), + sandboxes: z.object({ + total: z.number().int(), + byState: sandboxStateCountsSchema, + }), + /** Summed over the reported nodes whose reading carries it (a node on an older build reports none). */ + sandboxDisks: sandboxDisksSchema, +}); + +export type GetFleetMetricsResponse = z.infer< + typeof getFleetMetricsResponseSchema +>; + +/** + * getFleetStateHistory(start?, end?) — how many sandboxes sat in each + * state over time, fleet-wide: the product's own story ("idle is free" is + * visible as active falling while frozen rises). Answered by the gateway + * from its fleet_state_samples — one row per check-in, the sum of every + * node's last census at that moment (design record #26: the one figure no + * single node can compute); a node keeps no fleet history of its own + * since the third cut. Kept 30 days. + * + * Bucketing differs from the per-sandbox verb on purpose: a bucket + * reports its last raw row whole, never per-state maxima — independent + * maxima would double-count a sandbox mid-transition and the stacked + * counts would stop summing to total. The concurrency peak is instead + * computed from the window's raw rows and carried separately in `peak`, + * so no bucketing can flatten it. + */ +export const fleetStatePointSchema = z.object({ + /** ISO 8601 UTC — when the sample was taken. */ + at: z.string(), + byState: sandboxStateCountsSchema, + total: z.number().int(), +}); + +export type FleetStatePoint = z.infer; + +export const getFleetStateHistoryRequestSchema = z.object({ + /** ISO 8601; defaults to 24 hours before `end`. */ + start: isoTimestampSchema.optional(), + /** ISO 8601; defaults to now. */ + end: isoTimestampSchema.optional(), +}); + +export type GetFleetStateHistoryRequest = z.infer< + typeof getFleetStateHistoryRequestSchema +>; + +export const getFleetStateHistoryResponseSchema = z.object({ + /** Ascending by timestamp. */ + points: z.array(fleetStatePointSchema), + /** Null when raw samples were returned unbucketed. */ + bucketSeconds: z.number().int().positive().nullable(), + /** + * Highest active count in the window, from raw rows (not buckets), with + * the earliest instant it was observed. Null when the window holds no + * samples at all. + */ + peak: z + .object({ + active: z.number().int(), + at: z.string(), + }) + .nullable(), +}); + +export type GetFleetStateHistoryResponse = z.infer< + typeof getFleetStateHistoryResponseSchema +>; + +/** + * The native verbs that answer at the gateway and nowhere else: the + * fleet's configuration (keys, settings, templates, domains, nodes) and + * its own observation (getFleetMetrics, getFleetStateHistory). A node has + * no route for them, and its 404 names the gateway (server/app.ts); the + * gateway's suite checks that each is registered — so the two ends of + * this list cannot drift apart. + */ +export const GATEWAY_ONLY_VERBS = [ + 'createApiKey', + 'listApiKeys', + 'updateApiKey', + 'revokeApiKey', + 'getConfig', + 'updateSettings', + 'registerTemplate', + 'listTemplates', + 'removeTemplate', + 'getIngress', + 'setIngress', + 'listNodes', + 'updateNodeSettings', + 'removeNode', + 'getFleetMetrics', + 'getFleetStateHistory', +] as const; + +export type GatewayOnlyVerb = (typeof GATEWAY_ONLY_VERBS)[number]; diff --git a/packages/shared/src/host.ts b/packages/shared/src/host.ts index c6b9ecfd..2a59d6b6 100644 --- a/packages/shared/src/host.ts +++ b/packages/shared/src/host.ts @@ -65,11 +65,43 @@ export const sandboxStateCountsSchema = z.object({ export type SandboxStateCounts = z.infer; /** - * getHostMetrics() — the observation window into the machine itself: is the - * host healthy, and what do the sandboxes collectively cost it? A single - * point-in-time snapshot; for the machine's past see getHostMetricsHistory - * below. Observation never wakes a sandbox and never touches lifecycle. + * What the sandbox disks cost, from the executor: nominal is the summed + * promised sizes, actual is what the sparse images really occupy. The gap + * is the overcommit — the number an operator watches close as the host + * fills, because nothing else caps it. A node's figure in getHostMetrics + * and in its check-in reading (gateway.ts); the fleet's sum in + * getFleetMetrics. */ +export const sandboxDisksSchema = z.object({ + count: z.number().int(), + nominalBytes: z.number(), + actualBytes: z.number(), +}); + +export type SandboxDisks = z.infer; + +/** + * getHostMetrics({ nodeId? }) — the observation window into the machine + * itself: is the host healthy, and what do the sandboxes collectively cost + * it? A single point-in-time snapshot; for the machine's past see + * getHostMetricsHistory below. Observation never wakes a sandbox and never + * touches lifecycle. + * + * A machine's reading is one machine's: N nodes' CPU percentages add up + * to nothing, so at the gateway this verb names its node — `nodeId`, as + * listNodes lists them — and is forwarded there whole. A fleet of one + * needs no name (the single-machine install, unchanged); a fleet of + * several refuses an unnamed request (400) rather than pick a machine. + * A node ignores the field. The fleet's figures that do add up — the + * sandbox census, the disks' bill — are getFleetMetrics (gateway.ts). + */ +export const getHostMetricsRequestSchema = z.object({ + /** Which node's machine. Required at the gateway once the fleet has more than one node; ignored by a node. */ + nodeId: z.string().min(1).optional(), +}); + +export type GetHostMetricsRequest = z.infer; + export const hostMetricsResponseSchema = z.object({ host: hostReadingSchema, dataDisk: dataDiskSchema.nullable(), @@ -78,17 +110,7 @@ export const hostMetricsResponseSchema = z.object({ total: z.number().int(), byState: sandboxStateCountsSchema, }), - /** - * What the sandbox disks cost, from the executor: nominal is the summed - * promised sizes, actual is what the sparse images really occupy. The gap - * is the overcommit — the number an operator watches close as the host - * fills, because nothing else caps it. - */ - sandboxDisks: z.object({ - count: z.number().int(), - nominalBytes: z.number(), - actualBytes: z.number(), - }), + sandboxDisks: sandboxDisksSchema, }); export type HostMetricsResponse = z.infer; @@ -143,16 +165,19 @@ export const hostTimelinePointSchema = z.object({ export type HostTimelinePoint = z.infer; /** - * A parseable timestamp, rejected at the door — same rule as the metrics - * verbs: a malformed start/end must 400, never become NaN arithmetic. + * A parseable timestamp, rejected at the door — the one rule for every + * history window (this file, metrics.ts, gateway.ts): a malformed + * start/end must 400, never become NaN arithmetic. */ -const isoTimestampSchema = z +export const isoTimestampSchema = z .string() .refine((value) => !Number.isNaN(Date.parse(value)), { message: 'must be an ISO 8601 timestamp', }); export const getHostMetricsHistoryRequestSchema = z.object({ + /** Which node's machine — getHostMetricsRequestSchema has the rule. */ + nodeId: z.string().min(1).optional(), /** ISO 8601; defaults to 24 hours before `end`. */ start: isoTimestampSchema.optional(), /** ISO 8601; defaults to now. */ diff --git a/packages/shared/src/images.ts b/packages/shared/src/images.ts index c861c6cd..931ff4ef 100644 --- a/packages/shared/src/images.ts +++ b/packages/shared/src/images.ts @@ -1,4 +1,5 @@ import { z } from 'zod'; +import { silentNodeSchema } from './gateway'; /** * listSandboxImages() — every sandbox's image lineage in one answer: which @@ -46,6 +47,8 @@ export const listSandboxImagesResponseSchema = z.object({ upgradable: z.boolean(), }), ), + /** At the gateway: the nodes this answer could not include (gateway.ts silentNodeSchema). A node's own answer carries none. */ + silent: z.array(silentNodeSchema).optional(), }); export type ListSandboxImagesResponse = z.infer< diff --git a/packages/shared/src/list.ts b/packages/shared/src/list.ts index 5d8fcbeb..543d8993 100644 --- a/packages/shared/src/list.ts +++ b/packages/shared/src/list.ts @@ -1,14 +1,19 @@ import { z } from 'zod'; +import { silentNodeSchema } from './gateway'; import { sandboxSchema } from './sandbox'; /** - * listSandboxes() — the observation window into the ledger: every sandbox on - * this daemon with its current lifecycle state. Takes no input; the caller - * filters. Powers `dor sandbox ls`, the web console, and black-box tests - * that assert cold states from outside. + * listSandboxes() — the observation window into the ledger: every sandbox + * on this node with its current lifecycle state — or, at the gateway, on + * every node that answered, concatenated, with the nodes that did not in + * `silent`. Takes no input; the caller filters. Powers `dor sandbox ls`, + * the web console, and black-box tests that assert cold states from + * outside. */ export const listSandboxesResponseSchema = z.object({ sandboxes: z.array(sandboxSchema), + /** At the gateway: the nodes this answer could not include (gateway.ts silentNodeSchema). A node's own answer carries none. */ + silent: z.array(silentNodeSchema).optional(), }); export type ListSandboxesResponse = z.infer; diff --git a/packages/shared/src/metrics.ts b/packages/shared/src/metrics.ts index 893b3562..6e51d785 100644 --- a/packages/shared/src/metrics.ts +++ b/packages/shared/src/metrics.ts @@ -1,4 +1,6 @@ import { z } from 'zod'; +import { silentNodeSchema } from './gateway'; +import { isoTimestampSchema } from './host'; import { sandboxNameSchema } from './sandbox'; /** @@ -78,6 +80,8 @@ export const listSandboxMetricsResponseSchema = z.object({ sample: sandboxMetricsSampleSchema, }), ), + /** At the gateway: the nodes this answer could not include (gateway.ts silentNodeSchema). A node's own answer carries none. */ + silent: z.array(silentNodeSchema).optional(), }); export type ListSandboxMetricsResponse = z.infer< @@ -104,16 +108,6 @@ export type ListSandboxMetricsResponse = z.infer< * shows the hole instead of interpolating over it. * - An unsampled sandbox answers an empty array — never a made-up reading. */ -/** - * A parseable timestamp, rejected at the door: a malformed start/end would - * otherwise turn into NaN arithmetic deep in the window resolver. - */ -const isoTimestampSchema = z - .string() - .refine((value) => !Number.isNaN(Date.parse(value)), { - message: 'must be an ISO 8601 timestamp', - }); - export const getSandboxMetricsHistoryRequestSchema = z.object({ name: sandboxNameSchema, /** ISO 8601; defaults to one hour before `end`. */ @@ -136,64 +130,3 @@ export const getSandboxMetricsHistoryResponseSchema = z.object({ export type GetSandboxMetricsHistoryResponse = z.infer< typeof getSandboxMetricsHistoryResponseSchema >; - -/** - * getFleetTimeline(start?, end?) — how many sandboxes sat in each state - * over time: the fleet-level sibling of getSandboxMetricsHistory, and the - * product's own story ("idle is free" is visible as active falling while - * frozen rises). One snapshot row per sampler tick, kept 30 days. - * - * Bucketing differs from the per-sandbox verb on purpose: a bucket reports - * its last raw snapshot whole, never per-state maxima — independent maxima - * would double-count a sandbox mid-transition and the stacked counts would - * stop summing to total. The concurrency peak is instead computed from the - * window's raw rows and carried separately in `peak`, so no bucketing can - * flatten it. - */ -export const fleetTimelinePointSchema = z.object({ - /** ISO 8601 UTC — when the snapshot was taken. */ - at: z.string(), - byState: z.object({ - active: z.number().int(), - frozen: z.number().int(), - stopped: z.number().int(), - archived: z.number().int(), - restoring: z.number().int(), - }), - total: z.number().int(), -}); - -export type FleetTimelinePoint = z.infer; - -export const getFleetTimelineRequestSchema = z.object({ - /** ISO 8601; defaults to 24 hours before `end`. */ - start: isoTimestampSchema.optional(), - /** ISO 8601; defaults to now. */ - end: isoTimestampSchema.optional(), -}); - -export type GetFleetTimelineRequest = z.infer< - typeof getFleetTimelineRequestSchema ->; - -export const getFleetTimelineResponseSchema = z.object({ - /** Ascending by timestamp. */ - points: z.array(fleetTimelinePointSchema), - /** Null when raw snapshots were returned unbucketed. */ - bucketSeconds: z.number().int().positive().nullable(), - /** - * Highest active count in the window, from raw rows (not buckets), with - * the earliest instant it was observed. Null when the window holds no - * snapshots at all. - */ - peak: z - .object({ - active: z.number().int(), - at: z.string(), - }) - .nullable(), -}); - -export type GetFleetTimelineResponse = z.infer< - typeof getFleetTimelineResponseSchema ->; From b17ff96f5b542b4799c574480d4f03e995819704 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 00:29:05 +0800 Subject: [PATCH 47/89] The fleet-wide lists are merged at the gateway, the E2B list pages across nodes on one cursor, and a host reading is forwarded to the node it names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit listSandboxes, listSandboxMetrics and listSandboxImages ask every askable node in parallel and concatenate in node-id order; the nodes the answer lacks are named in `silent`. Not asked: a node that is down (a dial would wait the whole timeout on every console poll for as long as it stayed down) and a node awaiting its first configuration (not listening — named when it holds sandboxes, unmentioned when it holds none). Asked regardless for the startup grace: after a gateway restart every node is silent so far, the running ones included. The merge waits ten seconds, not the lookup's two: listSandboxMetrics reads every running container, and "slow is down" was said of finding a sandbox. E2B's GET /v2/sandboxes is one page drawn from every node, newest first, on a cursor that carries every node's own offset (base64url JSON); the last page carries no x-next-token, as the SDK's paginator expects. A node the page would lack is a 503 naming it: this wire is a bare array with nowhere to say what is missing, and a partial page would pass for the fleet. getHostMetrics and getHostMetricsHistory are forwarded to the node the request names; a fleet of one needs no name, a fleet of several is refused (400) rather than have a machine picked, a fleet of none is a 503. The upgrade verbs alone remain 501 until their cut. httpAsk grows the options the list needs (GET, x-api-key, a longer patience) and returns the answer's headers; a body the schema refuses is silence that says so, not a 500 out of the serializer. --- e2e/src/gateway.test.ts | 46 +- packages/gateway/src/app.test.ts | 420 +++++++++++++++++- packages/gateway/src/app.ts | 17 +- packages/gateway/src/cursor.test.ts | 85 ++++ packages/gateway/src/cursor.ts | 88 ++++ packages/gateway/src/find.ts | 16 +- packages/gateway/src/fleet.ts | 14 + packages/gateway/src/lookup.ts | 44 +- packages/gateway/src/merge.test.ts | 138 ++++++ packages/gateway/src/merge.ts | 126 ++++++ packages/gateway/src/routes/api-keys.test.ts | 12 +- packages/gateway/src/routes/e2b.ts | 99 ++++- packages/gateway/src/routes/native.ts | 110 ++++- packages/gateway/src/routes/observe.ts | 84 ++++ packages/gateway/src/routes/templates.test.ts | 6 +- 15 files changed, 1233 insertions(+), 72 deletions(-) create mode 100644 packages/gateway/src/cursor.test.ts create mode 100644 packages/gateway/src/cursor.ts create mode 100644 packages/gateway/src/merge.test.ts create mode 100644 packages/gateway/src/merge.ts create mode 100644 packages/gateway/src/routes/observe.ts diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts index a146ef29..3b749b53 100644 --- a/e2e/src/gateway.test.ts +++ b/e2e/src/gateway.test.ts @@ -431,19 +431,49 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { } }); - it('daemon-addressed verbs are an honest 501; a misspelled verb is a 404', async () => { - await expect(viaGateway().listSandboxes()).rejects.toMatchObject({ + it('the upgrade verbs are an honest 501 until their cut; a misspelled verb is a 404', async () => { + await expect(viaGateway().checkUpgrade()).rejects.toMatchObject({ status: 501, - message: expect.stringMatching(/call the node directly/), + message: expect.stringMatching(/until the upgrade cut/), }); - const e2bList = await fetch(`${gateway()}/e2b/api/v2/sandboxes`, { - headers: { 'x-api-key': `e2b_${token()}` }, - }); - expect(e2bList.status).toBe(501); - expect(((await e2bList.json()) as { code: number }).code).toBe(501); expect((await rpc('/acquireSandbx', { name: 'x' })).status).toBe(404); }); + it("listSandboxes at the door is both nodes' lists with nobody silent; the E2B list pages across both nodes through the official package", async () => { + const onB = await direct('node-b').acquireSandbox('gw-list-b'); + const onC = await direct('node-c').acquireSandbox('gw-list-c'); + try { + const listed = await viaGateway().listSandboxes(); + expect(listed.silent).toEqual([]); + const byName = new Map(listed.sandboxes.map((s) => [s.name, s.nodeId])); + expect(byName.get('gw-list-b')).toBe('node-b'); + expect(byName.get('gw-list-c')).toBe('node-c'); + // A node's own list has nobody to be silent about. + expect((await direct('node-b').listSandboxes()).silent).toBeUndefined(); + + // apiUrl is not in the list options' type (it is in the create's), + // but the paginator's ConnectionConfig reads it all the same — spread + // in, past the literal's excess-property check, exactly as a caller + // configuring a self-hosted door would. + const connection = { + apiKey: `e2b_${token()}`, + apiUrl: `${gateway()}/e2b/api`, + }; + const seen: string[] = []; + const paginator = Sandbox.list({ ...connection, limit: 1 }); + while (paginator.hasNext) { + for (const info of await paginator.nextItems()) + seen.push(info.sandboxId); + } + expect(seen).toContain(onB.sandbox.id); + expect(seen).toContain(onC.sandbox.id); + expect(new Set(seen).size).toBe(seen.length); + } finally { + await viaGateway().destroySandbox('gw-list-b'); + await viaGateway().destroySandbox('gw-list-c'); + } + }); + it('the official e2b package works through the gateway: create, live streaming, files, kill; an unnamed create routes by id', async () => { const connection = { apiKey: `e2b_${token()}`, diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 70de3fdf..cf11acf1 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -13,7 +13,7 @@ import { migrateDb, openDb } from './db/db'; import { ensureSettings } from './db/settings'; import { Finder } from './find'; import { Fleet } from './fleet'; -import { httpAskNode } from './lookup'; +import { type AskVerb, httpAsk, httpAskNode } from './lookup'; import { checkInOf, type reading } from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); @@ -27,10 +27,13 @@ const DOMAIN = DOMAINS[0] as string; * verbs the gateway touches, over a real socket. Every sandbox it holds is * a row in `sandboxes`; every request it received is in `hits`. */ +/** One clock for every fake node's createdAt: a later create is newer wherever it landed, so a merged newest-first page has one right order. */ +let births = 0; + class FakeNode { readonly sandboxes = new Map< string, - { id: string; name: string; files: Map } + { id: string; name: string; createdAt: string; files: Map } >(); readonly hits: Array<{ path: string; @@ -43,6 +46,8 @@ class FakeNode { endpoint = ''; /** How long a destroy takes to answer — a slow node holding the name's slot. */ destroyTakesMs = 0; + /** How long the list verbs take to answer — a busy node the merged lists must not wait forever for. */ + listTakesMs = 0; private readonly server: http.Server; constructor(readonly id: string) { @@ -186,6 +191,40 @@ class FakeNode { if (path.startsWith('/e2b/api/')) { if (auth !== `e2b_${TOKEN}`) return json(401, { code: 401, message: 'invalid API key' }); + if (path === '/e2b/api/v2/sandboxes' && req.method === 'GET') { + // The daemon's v2 list: newest first, offset in nextToken, the + // next offset in the x-next-token header when there is more. + const query = new URLSearchParams(url.slice(url.indexOf('?') + 1)); + const limit = Number(query.get('limit') ?? '100'); + const offset = Number(query.get('nextToken') ?? '0') || 0; + const all = [...this.sandboxes.values()].sort((a, b) => + a.createdAt < b.createdAt ? 1 : -1, + ); + const page = all.slice(offset, offset + limit); + const headers: Record = { + 'content-type': 'application/json', + }; + if (offset + limit < all.length) { + headers['x-next-token'] = String(offset + limit); + } + const answer = () => { + res.writeHead(200, headers); + res.end( + JSON.stringify( + page.map((s) => ({ + sandboxID: s.id, + clientID: this.id, + alias: s.name, + state: 'running', + startedAt: s.createdAt, + })), + ), + ); + }; + if (this.listTakesMs > 0) setTimeout(answer, this.listTakesMs); + else answer(); + return; + } const m = path.match(/^\/e2b\/api\/sandboxes(?:\/([^/]+))?(\/.*)?$/); if (!m) return json(404, { code: 404, message: 'not found' }); const [, id, rest] = m; @@ -224,6 +263,44 @@ class FakeNode { // This double records no template per sandbox: nothing here uses one. return json(200, { sandboxNames: [] }); } + case '/listSandboxes': + case '/listSandboxMetrics': + case '/listSandboxImages': { + // The daemon's three fleet-wide lists, as the gateway merges them. + const all = [...this.sandboxes.values()]; + const answer = () => + json( + 200, + path === '/listSandboxes' + ? { sandboxes: all.map((s) => this.view(s)) } + : path === '/listSandboxMetrics' + ? { + samples: all.map((s) => ({ + sandboxName: s.name, + sandboxId: s.id, + sample: SAMPLE, + })), + } + : { + images: all.map((s) => ({ + sandboxName: s.name, + sandboxId: s.id, + image: 'img:1', + nextImage: 'img:1', + upgradable: false, + })), + }, + ); + if (this.listTakesMs > 0) setTimeout(answer, this.listTakesMs); + else answer(); + return; + } + case '/getHostMetrics': + case '/getHostMetricsHistory': { + // A machine's reading, forwarded whole: an echo says which machine + // answered and what it was sent. + return json(200, { hostOf: this.id, verb: path, body }); + } case '/lookupSandbox': { const signed = body.signed as { query: string } | undefined; const sandbox = @@ -298,12 +375,56 @@ class FakeNode { private create(name: string) { this.creates += 1; - const sandbox = { id: randomUUID(), name, files: new Map() }; + births += 1; + const sandbox = { + id: randomUUID(), + name, + createdAt: new Date( + Date.UTC(2026, 8, 15) + births * 60_000, + ).toISOString(), + files: new Map(), + }; this.sandboxes.set(name, sandbox); return sandbox; } + + /** The daemon's sandbox object for one of this double's rows — enough of it to pass the shared schema. */ + private view(s: { id: string; name: string; createdAt: string }) { + return { + id: s.id, + name: s.name, + state: 'active', + nodeId: this.id, + endpoint: this.endpoint, + policy: { + freezeAfterSeconds: 300, + stopAfterSeconds: null, + archiveAfterSeconds: null, + }, + spec: { cpus: 1, memoryGb: 2, diskGb: 10 }, + template: null, + metadata: {}, + createdAt: s.createdAt, + lastActiveAt: s.createdAt, + lastExit: null, + }; + } } +/** One measurable sandbox's reading, the same for every row of every double. */ +const SAMPLE = { + timestamp: '2026-09-15T00:00:00.000Z', + cpuCount: 1, + cpuUsedPct: 5, + memUsedBytes: 100, + memTotalBytes: 2048, + memCacheBytes: 10, + swapUsedBytes: null, + swapTotalBytes: null, + diskUsedBytes: 1000, + diskTotalBytes: 10_000, +}; + interface Harness { endpoint: string; nodes: FakeNode[]; @@ -311,7 +432,10 @@ interface Harness { fleet: Fleet; checkIn( node: FakeNode, - over?: Parameters[0] & { intervalSeconds?: number }, + over?: Parameters[0] & { + intervalSeconds?: number; + configVersion?: number | null; + }, ): Promise; close(): Promise; } @@ -329,6 +453,8 @@ async function gateway( startedAt?: Date; /** Collects the gateway's own log lines (JSON, one per entry) when a test asserts on what it says. */ logs?: string[]; + /** How the gateway asks nodes on its own account — a test shortens its patience for the slow-node case. */ + ask?: AskVerb; } = {}, ): Promise { const db = openDb(':memory:'); @@ -358,6 +484,7 @@ async function gateway( ? false : pino({ level: 'info' }, { write: (line: string) => logs.push(line) }), build: null, + ask: opts.ask, }); await app.listen({ host: '127.0.0.1', port: 0 }); const endpoint = `http://127.0.0.1:${(app.server.address() as AddressInfo).port}`; @@ -1088,12 +1215,11 @@ describe('using, destroying, and the cache', () => { expect(a.lookups()).toBe(before + 1); }); - it('daemon-addressed verbs are an honest 501, a misspelled verb a 404, a body without a name a 400', async () => { + it('the upgrade verbs are an honest 501, a misspelled verb a 404, a body without a name a 400', async () => { const h = await gateway(['a']); - const listed = await rpc(h, '/listSandboxes'); - expect(listed.status).toBe(501); - expect(message(listed)).toContain('call the node directly'); - expect((await rpc(h, '/getHostMetrics')).status).toBe(501); + const upgrade = await rpc(h, '/checkUpgrade'); + expect(upgrade.status).toBe(501); + expect(message(upgrade)).toContain('call the node directly'); expect((await rpc(h, '/acquireSandbx', { name: 'x' })).status).toBe(404); expect((await rpc(h, '/execCommand', { command: 'x' })).status).toBe(400); expect(h.nodes[0]?.hits).toEqual([]); @@ -1186,7 +1312,15 @@ describe('the E2B faces', () => { expect(h.cache.getById(anon.sandboxID)?.name).toBeNull(); expect((await e2b(h, `/sandboxes/${anon.sandboxID}`)).status).toBe(200); - expect((await e2b(h, '/v2/sandboxes')).status).toBe(501); + // The list is answered here now (its own suite below): the named and + // the unnamed sandbox both, from whichever node holds each. + const listed = await e2b(h, '/v2/sandboxes'); + expect(listed.status).toBe(200); + expect( + ((await listed.json()) as Array<{ sandboxID: string }>).map( + (s) => s.sandboxID, + ), + ).toContain(anon.sandboxID); const wrongKey = await fetch(`${h.endpoint}/e2b/api/sandboxes`, { method: 'POST', headers: { 'x-api-key': 'e2b_wrong', 'content-type': 'application/json' }, @@ -1586,3 +1720,269 @@ describe('the bare signed-URL face', () => { expect((await fetch(`${h.endpoint}/files?${query}`)).status).toBe(200); }); }); + +describe('the fleet-wide lists and the by-node readings', () => { + const namesOf = (r: { body: unknown }) => + (r.body as { sandboxes: Array<{ name: string; nodeId: string }> }) + .sandboxes; + const silentOf = (r: { body: unknown }) => + (r.body as { silent: Array<{ nodeId: string; why: string }> }).silent; + + async function seeded() { + const h = await gateway(['c', 'b']); + // Three names: placement alternates (the emptiest by active density, + // the in-flight count moving it), so both nodes hold some. + for (const name of ['s1', 's2', 's3']) { + expect((await rpc(h, '/acquireSandbox', { name })).status).toBe(200); + } + const [c, b] = h.nodes; + if (!c || !b) throw new Error('nodes lost'); + return { h, b, c }; + } + + it("listSandboxes is every node's list in node-id order, each asked once, with nobody silent", async () => { + const { h, b, c } = await seeded(); + const listed = await rpc(h, '/listSandboxes'); + expect(listed.status).toBe(200); + const nodesInOrder = namesOf(listed).map((s) => s.nodeId); + expect(nodesInOrder).toHaveLength(3); + expect(nodesInOrder).toEqual([...nodesInOrder].sort()); + expect( + namesOf(listed) + .map((s) => s.name) + .sort(), + ).toEqual(['s1', 's2', 's3']); + expect(silentOf(listed)).toEqual([]); + for (const node of [b, c]) { + expect(node.hits.filter((x) => x.path === '/listSandboxes')).toHaveLength( + 1, + ); + } + }); + + it('a node that is down is not dialled and is named silent with the reason; its sandboxes are not in the list', async () => { + const { h, b, c } = await seeded(); + const down = h.fleet.get('c'); + if (!down) throw new Error('node lost'); + down.lastCheckInAt = new Date(Date.now() - 31_000); + const listed = await rpc(h, '/listSandboxes'); + expect(listed.status).toBe(200); + expect(namesOf(listed).every((s) => s.nodeId === 'b')).toBe(true); + expect(namesOf(listed)).toHaveLength(b.sandboxes.size); + expect(silentOf(listed)).toEqual([ + { + nodeId: 'c', + why: expect.stringMatching(/has not checked in for 3\ds/), + }, + ]); + expect(c.hits.filter((x) => x.path === '/listSandboxes')).toHaveLength(0); + }); + + it('a node awaiting its first configuration is not dialled: empty, nothing is said; holding sandboxes, it is named as not listening', async () => { + const h = await gateway(['b', 'c']); + const [b, c] = h.nodes; + if (!b || !c) throw new Error('nodes lost'); + await h.checkIn(c, { configVersion: null, active: 0 }); + let listed = await rpc(h, '/listSandboxes'); + expect(silentOf(listed)).toEqual([]); + await h.checkIn(c, { configVersion: null, active: 5 }); + listed = await rpc(h, '/listSandboxes'); + expect(silentOf(listed)).toEqual([ + { nodeId: 'c', why: expect.stringMatching(/not listening — it holds 5/) }, + ]); + expect(c.hits.filter((x) => x.path === '/listSandboxes')).toHaveLength(0); + expect(b.hits.filter((x) => x.path === '/listSandboxes')).toHaveLength(2); + }); + + it('a node too slow to answer is silent after the merge timeout, and the rest of the list is answered', async () => { + // The gateway's patience shortened to 300ms: the rule, not the wait. + const h = await gateway( + ['b', 'c'], + {}, + { + ask: (node, verb, body, schema, options) => + httpAsk(TOKEN)(node, verb, body, schema, { + ...options, + timeoutMs: 300, + }), + }, + ); + const [b, c] = h.nodes; + if (!b || !c) throw new Error('nodes lost'); + await rpc(h, '/acquireSandbox', { name: 'quick' }); + c.listTakesMs = 2_000; + const started = Date.now(); + const listed = await rpc(h, '/listSandboxes'); + expect(Date.now() - started).toBeLessThan(1_500); + expect(listed.status).toBe(200); + expect(silentOf(listed)).toEqual([ + { nodeId: 'c', why: expect.stringMatching(/timeout|abort/i) }, + ]); + expect(namesOf(listed).map((s) => s.nodeId)).toEqual( + namesOf(listed).map(() => 'b'), + ); + }); + + it('listSandboxMetrics and listSandboxImages merge the same way', async () => { + const { h } = await seeded(); + const metrics = await rpc(h, '/listSandboxMetrics'); + expect(metrics.status).toBe(200); + const samples = metrics.body as { + samples: Array<{ sandboxName: string }>; + silent: unknown[]; + }; + expect(samples.samples.map((s) => s.sandboxName).sort()).toEqual([ + 's1', + 's2', + 's3', + ]); + expect(samples.silent).toEqual([]); + const images = await rpc(h, '/listSandboxImages'); + expect(images.status).toBe(200); + const lineage = images.body as { + images: Array<{ sandboxName: string; upgradable: boolean }>; + silent: unknown[]; + }; + expect(lineage.images.map((i) => i.sandboxName).sort()).toEqual([ + 's1', + 's2', + 's3', + ]); + expect(lineage.images.every((i) => i.upgradable === false)).toBe(true); + expect(lineage.silent).toEqual([]); + }); + + it('a host reading names its node: forwarded to it whole; unnamed in a fleet of several it is a 400 naming them; an unknown id is a 404', async () => { + const h = await gateway(['b', 'c']); + const named = await rpc(h, '/getHostMetrics', { nodeId: 'c' }); + expect(named.status).toBe(200); + expect(named.body).toEqual({ + hostOf: 'c', + verb: '/getHostMetrics', + body: { nodeId: 'c' }, + }); + const history = await rpc(h, '/getHostMetricsHistory', { + nodeId: 'b', + start: '2026-09-14T00:00:00.000Z', + }); + expect(history.body).toMatchObject({ + hostOf: 'b', + verb: '/getHostMetricsHistory', + }); + const unnamed = await rpc(h, '/getHostMetrics', {}); + expect(unnamed.status).toBe(400); + expect(message(unnamed)).toContain('the fleet has 2 nodes (b, c)'); + expect(message(unnamed)).toContain('getFleetMetrics'); + const unknown = await rpc(h, '/getHostMetrics', { nodeId: 'zzz' }); + expect(unknown.status).toBe(404); + expect(message(unknown)).toContain("no node with id 'zzz'"); + const malformed = await rpc(h, '/getHostMetrics', { nodeId: 7 }); + expect(malformed.status).toBe(400); + }); + + it('a fleet of one needs no name; a fleet of none is a 503 with Retry-After', async () => { + const one = await gateway(['b']); + const unnamed = await rpc(one, '/getHostMetrics', {}); + expect(unnamed.status).toBe(200); + expect(unnamed.body).toMatchObject({ hostOf: 'b', body: {} }); + const none = await gateway([]); + const refused = await rpc(none, '/getHostMetrics', {}); + expect(refused.status).toBe(503); + expect(refused.headers.get('retry-after')).toBe('15'); + expect(message(refused)).toContain('no node has checked in yet'); + }); + + it('the upgrade verbs alone are still an honest 501', async () => { + const h = await gateway(['b']); + for (const verb of ['checkUpgrade', 'applyUpgrade', 'getUpgradeStatus']) { + const r = await rpc(h, `/${verb}`, {}); + expect(r.status).toBe(501); + expect(message(r)).toContain('until the upgrade cut'); + } + }); +}); + +describe('the E2B list across nodes', () => { + async function e2bList(h: Harness, query: string) { + const res = await fetch(`${h.endpoint}/e2b/api/v2/sandboxes${query}`, { + headers: { 'x-api-key': `e2b_${TOKEN}` }, + }); + const text = await res.text(); + return { + status: res.status, + body: text ? (JSON.parse(text) as unknown) : null, + next: res.headers.get('x-next-token'), + retryAfter: res.headers.get('retry-after'), + }; + } + const ids = (r: { body: unknown }) => + (r.body as Array<{ sandboxID: string; alias: string }>).map((s) => s.alias); + + it('pages newest first across both nodes on one opaque cursor, and the last page carries no cursor', async () => { + const h = await gateway(['b', 'c']); + for (const name of ['e1', 'e2', 'e3']) { + const created = await fetch(`${h.endpoint}/e2b/api/sandboxes`, { + method: 'POST', + headers: { + 'x-api-key': `e2b_${TOKEN}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ templateID: 'base', metadata: { name } }), + }); + expect(created.status).toBe(201); + } + const [b, c] = h.nodes; + if (!b || !c) throw new Error('nodes lost'); + expect(b.sandboxes.size + c.sandboxes.size).toBe(3); + expect(Math.min(b.sandboxes.size, c.sandboxes.size)).toBeGreaterThan(0); + + const first = await e2bList(h, '?limit=2'); + expect(first.status).toBe(200); + expect(ids(first)).toEqual(['e3', 'e2']); + expect(first.next).not.toBeNull(); + const second = await e2bList(h, `?limit=2&nextToken=${first.next}`); + expect(ids(second)).toEqual(['e1']); + expect(second.next).toBeNull(); + + // One at a time: three pages, every node asked its own offset. + const seen: string[] = []; + let token: string | null = null; + do { + const page = await e2bList( + h, + `?limit=1${token === null ? '' : `&nextToken=${token}`}`, + ); + seen.push(...ids(page)); + token = page.next; + } while (token !== null); + expect(seen).toEqual(['e3', 'e2', 'e1']); + + const whole = await e2bList(h, ''); + expect(ids(whole)).toEqual(['e3', 'e2', 'e1']); + expect(whole.next).toBeNull(); + }); + + it('a cursor it did not mint is a 400; a node the list would lack is a 503 naming it, with Retry-After', async () => { + const h = await gateway(['b', 'c']); + const bad = await e2bList(h, '?nextToken=garbage'); + expect(bad.status).toBe(400); + expect(bad.body).toMatchObject({ + code: 400, + message: expect.stringMatching(/invalid nextToken/), + }); + const tooMany = await e2bList(h, '?limit=5000'); + expect(tooMany.status).toBe(400); + const down = h.fleet.get('c'); + if (!down) throw new Error('node lost'); + down.lastCheckInAt = new Date(Date.now() - 31_000); + const refused = await e2bList(h, ''); + expect(refused.status).toBe(503); + expect(refused.retryAfter).toBe('15'); + expect(refused.body).toMatchObject({ + code: 503, + message: expect.stringMatching( + /node c did not answer \(has not checked in for 3\ds\)/, + ), + }); + }); +}); diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts index ce34de09..bd947d50 100644 --- a/packages/gateway/src/app.ts +++ b/packages/gateway/src/app.ts @@ -31,6 +31,7 @@ import { envdTokenRoutes } from './routes/envd-token'; import { ingressRoutes } from './routes/ingress'; import { nativeRoutes } from './routes/native'; import { checkInRoutes, nodeRoutes } from './routes/nodes'; +import { observeRoutes } from './routes/observe'; import { settingsRoutes } from './routes/settings'; import { templateRoutes } from './routes/templates'; import { type BuildInfo, readBuildInfo } from './version'; @@ -63,7 +64,8 @@ export interface GatewayAppDeps { probeS3?: SettingsProbe; /** * How the gateway asks a node a verb on its own account (removeTemplate's - * templateUsers). Defaults to HTTP under the fleet token; tests script it. + * templateUsers, the merged lists, the E2B list). Defaults to HTTP under + * the fleet token; tests script it, or shorten its patience. */ ask?: AskVerb; /** @@ -226,6 +228,7 @@ export function buildGatewayApp({ ); const knobs = placementKnobs(config); + const askVerb = ask ?? httpAsk(token); // The nodes' gate: the fleet token alone. A key or a session is a // caller's credential, and a check-in is not a call — it is a machine @@ -241,10 +244,13 @@ export function buildGatewayApp({ await nodesFace.register(checkInRoutes, { fleet, db }); }); - // The sandbox gate: everything that addresses a sandbox. + // The sandbox gate: everything that addresses a sandbox — and the + // fleet-wide observation (the merged lists), which every credential + // that may address a sandbox may read, as on a node. app.register(async (api) => { api.addHook('onRequest', apiAuth); await api.register(envdTokenRoutes, { finder, token }); + await api.register(observeRoutes, { fleet, ask: askVerb }); // Its own sub-scope: the byte-preserving body parser it installs must // not reach the gateway's own verbs, which keep Fastify's JSON parsing. await api.register(nativeRoutes, { fleet, finder, locks, knobs, token }); @@ -262,11 +268,7 @@ export function buildGatewayApp({ sources, ...(probeS3 ? { probeS3 } : {}), }); - await admin.register(templateRoutes, { - db, - fleet, - ask: ask ?? httpAsk(token), - }); + await admin.register(templateRoutes, { db, fleet, ask: askVerb }); await admin.register(ingressRoutes, { ingress }); }); @@ -285,6 +287,7 @@ export function buildGatewayApp({ knobs, token, isCredential, + ask: askVerb, prefix: '/e2b/api', }); diff --git a/packages/gateway/src/cursor.test.ts b/packages/gateway/src/cursor.test.ts new file mode 100644 index 00000000..eb150c92 --- /dev/null +++ b/packages/gateway/src/cursor.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest'; +import { decodeCursor, encodeCursor, mergePages } from './cursor'; + +// The E2B list's cursor across nodes and the page it draws — pure. + +const item = (id: string, startedAt: string) => ({ sandboxID: id, startedAt }); + +describe('the cursor', () => { + it('round-trips every node offset and refuses anything it did not mint', () => { + const offsets = { 'node-b': 3, 'node-c': 0 }; + expect(decodeCursor(encodeCursor(offsets))).toEqual(offsets); + expect(decodeCursor('')).toBeNull(); + expect(decodeCursor('12')).toBeNull(); + expect(decodeCursor('not base64 json')).toBeNull(); + expect( + decodeCursor(Buffer.from('{"v":2,"o":{}}').toString('base64url')), + ).toBeNull(); + expect( + decodeCursor(Buffer.from('{"v":1,"o":{"a":-1}}').toString('base64url')), + ).toBeNull(); + expect( + decodeCursor(Buffer.from('{"v":1,"o":{"a":1.5}}').toString('base64url')), + ).toBeNull(); + expect( + decodeCursor(Buffer.from('{"v":1,"o":{}}').toString('base64url')), + ).toEqual({}); + }); +}); + +describe('mergePages', () => { + it('draws one page newest first across nodes, advances each node by what it took, and stops when every node is exhausted', () => { + const b = { + nodeId: 'b', + items: [ + item('b3', '2026-09-15T00:03:00Z'), + item('b1', '2026-09-15T00:01:00Z'), + ], + more: false, + }; + const c = { + nodeId: 'c', + items: [item('c2', '2026-09-15T00:02:00Z')], + more: false, + }; + const first = mergePages([b, c], {}, 2); + expect(first.items.map((i) => i.sandboxID)).toEqual(['b3', 'c2']); + expect(first.next).toEqual({ b: 1, c: 1 }); + // The next page: b answers from offset 1, c has nothing left. + const second = mergePages( + [ + { ...b, items: [b.items[1] as (typeof b.items)[number]] }, + { ...c, items: [] }, + ], + first.next ?? {}, + 2, + ); + expect(second.items.map((i) => i.sandboxID)).toEqual(['b1']); + expect(second.next).toBeNull(); + }); + + it('a node that said it had more beyond the page it sent keeps the cursor alive even when the page took all of it', () => { + const b = { + nodeId: 'b', + items: [item('b9', '2026-09-15T00:09:00Z')], + more: true, + }; + const page = mergePages([b], { b: 4 }, 5); + expect(page.items).toHaveLength(1); + expect(page.next).toEqual({ b: 5 }); + }); + + it('equal timestamps order by id, the same every time', () => { + const at = '2026-09-15T00:00:00Z'; + const page = mergePages( + [ + { nodeId: 'c', items: [item('m', at), item('a', at)], more: false }, + { nodeId: 'b', items: [item('k', at)], more: false }, + ], + {}, + 10, + ); + expect(page.items.map((i) => i.sandboxID)).toEqual(['a', 'k', 'm']); + expect(page.next).toBeNull(); + }); +}); diff --git a/packages/gateway/src/cursor.ts b/packages/gateway/src/cursor.ts new file mode 100644 index 00000000..ea30d800 --- /dev/null +++ b/packages/gateway/src/cursor.ts @@ -0,0 +1,88 @@ +/** + * The E2B list's cursor across nodes. E2B's v2 list pages by an opaque + * `x-next-token` the client hands back verbatim; a node's token is its own + * offset into its own newest-first list (the daemon's e2b/control.ts). At + * the gateway one page is drawn from every node, so the cursor is every + * node's offset at once: base64url of `{ v: 1, o: { : } }`. + * A node not in it starts at 0; a node in it that has since been removed + * is ignored. Opaque to the SDK, which never reads it. + */ +export type Offsets = Record; + +const VERSION = 1; + +export function encodeCursor(offsets: Offsets): string { + return Buffer.from(JSON.stringify({ v: VERSION, o: offsets })).toString( + 'base64url', + ); +} + +/** The offsets a token carries, or null for anything but a token this gateway minted. */ +export function decodeCursor(token: string): Offsets | null { + let parsed: unknown; + try { + parsed = JSON.parse(Buffer.from(token, 'base64url').toString('utf8')); + } catch { + return null; + } + if (typeof parsed !== 'object' || parsed === null) return null; + const { v, o } = parsed as { v?: unknown; o?: unknown }; + if (v !== VERSION || typeof o !== 'object' || o === null) return null; + const offsets: Offsets = {}; + for (const [nodeId, offset] of Object.entries(o)) { + if (!Number.isInteger(offset) || (offset as number) < 0) return null; + offsets[nodeId] = offset as number; + } + return offsets; +} + +/** One node's page: what it answered from its offset, and whether it said there was more after it (its own x-next-token). */ +export interface NodePage { + nodeId: string; + items: T[]; + more: boolean; +} + +export interface MergedPage { + items: T[]; + /** The next page's cursor, or null when every node is exhausted. */ + next: Offsets | null; +} + +/** + * One page across nodes: every node's page, newest first (the daemon's + * order — `startedAt` descending; ties by id so the order is the same + * every time), cut at `limit`. Each node's offset advances by what this + * page took from its page, so what it did not take is at the front of + * that node's next page. There is a next page while any node has items + * this page did not take, or said it had more beyond the page it sent. + */ +export function mergePages( + pages: NodePage[], + offsets: Offsets, + limit: number, +): MergedPage { + const tagged = pages.flatMap((page) => + page.items.map((item) => ({ nodeId: page.nodeId, item })), + ); + tagged.sort((a, b) => + a.item.startedAt === b.item.startedAt + ? a.item.sandboxID.localeCompare(b.item.sandboxID) + : a.item.startedAt < b.item.startedAt + ? 1 + : -1, + ); + const taken = tagged.slice(0, limit); + const used = new Map(); + for (const { nodeId } of taken) used.set(nodeId, (used.get(nodeId) ?? 0) + 1); + const next: Offsets = { ...offsets }; + let more = false; + for (const page of pages) { + const took = used.get(page.nodeId) ?? 0; + next[page.nodeId] = (offsets[page.nodeId] ?? 0) + took; + if (took < page.items.length || (page.more && took === page.items.length)) { + more = true; + } + } + return { items: taken.map((t) => t.item), next: more ? next : null }; +} diff --git a/packages/gateway/src/find.ts b/packages/gateway/src/find.ts index 7db0e8a1..9eab80c2 100644 --- a/packages/gateway/src/find.ts +++ b/packages/gateway/src/find.ts @@ -1,6 +1,11 @@ import type { SignedFileLookup } from '@dormice/shared'; import type { CacheEntry, NameCache } from './cache'; -import { awaitingFirstConfig, type Fleet, type NodeState } from './fleet'; +import { + awaitingFirstConfig, + awaitingFirstConfigWhy, + type Fleet, + type NodeState, +} from './fleet'; import type { AskNode, LookupAnswer, LookupQuery } from './lookup'; /** @@ -81,14 +86,9 @@ export class Finder { */ private askNode(node: NodeState, query: LookupQuery): Promise { if (awaitingFirstConfig(node)) { - const total = node.reading?.sandboxes.total ?? 0; + const why = awaitingFirstConfigWhy(node); return Promise.resolve( - total === 0 - ? { kind: 'absent' } - : { - kind: 'silent', - why: `not listening — it holds ${total} sandboxes but no configuration copy yet, and its first bundle rides on its next check-in`, - }, + why === null ? { kind: 'absent' } : { kind: 'silent', why }, ); } return this.ask(node, query); diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 6dbb7a14..ff01c533 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -78,6 +78,20 @@ export function awaitingFirstConfig(node: NodeState): boolean { return node.reading !== null && node.configVersion === null; } +/** + * What to say of a node awaiting its first configuration, in place of + * asking it: nothing (null) when its reading says it holds no sandbox — + * there is nothing an answer would lack — and, when it holds some, the + * sentence that names them as there and unreachable until its next + * check-in says the port is open. + */ +export function awaitingFirstConfigWhy(node: NodeState): string | null { + const total = node.reading?.sandboxes.total ?? 0; + return total === 0 + ? null + : `not listening — it holds ${total} sandboxes but no configuration copy yet, and its first bundle rides on its next check-in`; +} + /** What a check-in came to: taken (and whether it joined or moved), or refused with the sentence the node is told (routes/nodes.ts answers 409). */ export type CheckInOutcome = | { node: NodeState; joined: boolean; movedFrom: string | null } diff --git a/packages/gateway/src/lookup.ts b/packages/gateway/src/lookup.ts index bed6f169..8ec0d669 100644 --- a/packages/gateway/src/lookup.ts +++ b/packages/gateway/src/lookup.ts @@ -35,17 +35,28 @@ export type AskNode = ( query: LookupQuery, ) => Promise; -/** A node's answer to any verb asked on the gateway's account: parsed, or silence with the transport's word. */ +/** A node's answer to any verb asked on the gateway's account: parsed (with the answer's headers, for the one verb that pages by one), or silence with the transport's word. */ export type Asked = - | { kind: 'answer'; value: T } + | { kind: 'answer'; value: T; headers: Headers } | { kind: 'silent'; why: string }; -/** Asks one node one verb, validated by the schema of its answer. */ +/** How a verb is asked, where the native default does not fit. */ +export interface AskOptions { + /** GET for the E2B control plane's list; POST, the native dialect, by default. */ + method?: 'GET' | 'POST'; + /** The fleet token as the native Bearer (default) or as E2B's x-api-key. */ + credential?: 'bearer' | 'x-api-key'; + /** LOOKUP_TIMEOUT_MS by default — a ledger read; longer for a verb that reads containers (merge.ts). */ + timeoutMs?: number; +} + +/** Asks one node one verb, validated by the schema of its answer. `verb` is the path under the node's endpoint, query string included for a GET. */ export type AskVerb = ( node: AskedNode, verb: string, body: unknown, schema: z.ZodType, + options?: AskOptions, ) => Promise>; /** @@ -80,16 +91,19 @@ export function causeOf(error: unknown): string { * (9, 22, 25, 6000 …) without dialling — no node's front lives on one. */ export function httpAsk(token: string): AskVerb { - return async (node, verb, body, schema) => { + return async (node, verb, body, schema, options = {}) => { + const method = options.method ?? 'POST'; try { const res = await fetch(`${node.endpoint}/${verb}`, { - method: 'POST', + method, headers: { - authorization: `Bearer ${token}`, - 'content-type': 'application/json', + ...(options.credential === 'x-api-key' + ? { 'x-api-key': `e2b_${token}` } + : { authorization: `Bearer ${token}` }), + ...(method === 'POST' ? { 'content-type': 'application/json' } : {}), }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(LOOKUP_TIMEOUT_MS), + body: method === 'POST' ? JSON.stringify(body) : undefined, + signal: AbortSignal.timeout(options.timeoutMs ?? LOOKUP_TIMEOUT_MS), redirect: 'manual', }); if (res.status !== 200) { @@ -99,7 +113,17 @@ export function httpAsk(token: string): AskVerb { why: `${verb} answered ${res.status}: ${text.slice(0, 200)}`, }; } - return { kind: 'answer', value: schema.parse(await res.json()) }; + // A body the schema refuses is a node the gateway cannot read — + // a build too far apart, or not a node at all — and is silence + // that says so, not a 500 out of the gateway's own serializer. + const parsed = schema.safeParse(await res.json()); + if (!parsed.success) { + return { + kind: 'silent', + why: `${verb} answered a body the gateway cannot read (${parsed.error.issues[0]?.message ?? 'schema mismatch'})`, + }; + } + return { kind: 'answer', value: parsed.data, headers: res.headers }; } catch (error) { return { kind: 'silent', why: causeOf(error) }; } diff --git a/packages/gateway/src/merge.test.ts b/packages/gateway/src/merge.test.ts new file mode 100644 index 00000000..67105a2b --- /dev/null +++ b/packages/gateway/src/merge.test.ts @@ -0,0 +1,138 @@ +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { migrateDb, openDb } from './db/db'; +import { Fleet, STARTUP_GRACE_MS } from './fleet'; +import type { AskVerb } from './lookup'; +import { askability, askEach, MERGE_TIMEOUT_MS } from './merge'; +import { checkInOf } from './testing'; + +// Who a merged answer asks, and what it says of the rest — the pure rules, +// with a scripted asker. The wire (a real node, a real timeout) is +// app.test.ts's. + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const NOW = new Date('2026-09-15T00:00:00.000Z'); + +function fleetAt(startedAt: Date) { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + return new Fleet(db, startedAt); +} + +describe('askability', () => { + it('a node that checked in and runs a configuration is asked; one down for two of its intervals is not, with the reason', () => { + const fleet = fleetAt(NOW); + const a = fleet.checkIn(checkInOf('a', 'http://a:80'), NOW); + if ('refused' in a) throw new Error(a.refused); + expect(askability(a.node, NOW, fleet.startedAt)).toEqual({ ask: true }); + const later = new Date(NOW.getTime() + 31_000); + expect(askability(a.node, later, fleet.startedAt)).toEqual({ + ask: false, + why: 'has not checked in for 31s', + }); + }); + + it('a node awaiting its first configuration is not asked: holding nothing, nothing is said; holding sandboxes, it is named as not listening', () => { + const fleet = fleetAt(NOW); + const empty = fleet.checkIn( + checkInOf('b', 'http://b:80', { configVersion: null, active: 0 }), + NOW, + ); + if ('refused' in empty) throw new Error(empty.refused); + expect(askability(empty.node, NOW, fleet.startedAt)).toEqual({ + ask: false, + why: null, + }); + fleet.checkIn( + checkInOf('b', 'http://b:80', { configVersion: null, active: 4 }), + NOW, + ); + expect(askability(empty.node, NOW, fleet.startedAt)).toEqual({ + ask: false, + why: expect.stringMatching(/not listening — it holds 4 sandboxes/), + }); + }); + + it('after a gateway start a node not yet heard from is asked for the grace period, and is down after it', () => { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + new Fleet(db, NOW).checkIn(checkInOf('c', 'http://c:80'), NOW); + // A restart over the same rows: c is known, silent so far. + const restarted = new Fleet(db, NOW); + const c = restarted.get('c'); + if (!c) throw new Error('row lost'); + expect( + askability(c, new Date(NOW.getTime() + STARTUP_GRACE_MS - 1), NOW), + ).toEqual({ ask: true }); + expect( + askability(c, new Date(NOW.getTime() + STARTUP_GRACE_MS), NOW), + ).toEqual({ + ask: false, + why: 'has not checked in since the gateway started', + }); + }); +}); + +describe('askEach', () => { + const answerSchema = z.object({ items: z.array(z.string()) }); + + it('asks every askable node with the merge timeout, keeps every answer in node-id order, and names the silent ones', async () => { + const fleet = fleetAt(NOW); + for (const id of ['c', 'a', 'b', 'd']) { + fleet.checkIn(checkInOf(id, `http://${id}:80`), NOW); + } + // d fell silent; b is booting with sandboxes; nobody dials either. + const d = fleet.get('d'); + if (!d) throw new Error('node lost'); + d.lastCheckInAt = new Date(NOW.getTime() - 40_000); + fleet.checkIn( + checkInOf('b', 'http://b:80', { configVersion: null, active: 2 }), + NOW, + ); + const asked: Array<{ id: string; verb: string; timeoutMs?: number }> = []; + const ask: AskVerb = async (node, verb, _body, schema, options) => { + asked.push({ id: node.id, verb, timeoutMs: options?.timeoutMs }); + if (node.id === 'c') return { kind: 'silent', why: 'ECONNRESET' }; + return { + kind: 'answer', + value: schema.parse({ items: [`${node.id}-1`] }), + headers: new Headers({ 'x-next-token': '7' }), + }; + }; + const merged = await askEach( + fleet, + ask, + NOW, + (node) => `list?node=${node.id}`, + {}, + answerSchema, + ); + expect(asked).toEqual([ + { id: 'c', verb: 'list?node=c', timeoutMs: MERGE_TIMEOUT_MS }, + { id: 'a', verb: 'list?node=a', timeoutMs: MERGE_TIMEOUT_MS }, + ]); + expect(merged.answers.map((a) => a.node.id)).toEqual(['a']); + expect(merged.answers[0]?.value).toEqual({ items: ['a-1'] }); + expect(merged.answers[0]?.headers.get('x-next-token')).toBe('7'); + expect(merged.silent).toEqual([ + { nodeId: 'b', why: expect.stringMatching(/not listening/) }, + { nodeId: 'c', why: 'ECONNRESET' }, + { nodeId: 'd', why: 'has not checked in for 40s' }, + ]); + }); + + it('an empty fleet answers nothing and nobody is silent', async () => { + const merged = await askEach( + fleetAt(NOW), + async () => { + throw new Error('nobody to ask'); + }, + NOW, + 'list', + {}, + answerSchema, + ); + expect(merged).toEqual({ answers: [], silent: [] }); + }); +}); diff --git a/packages/gateway/src/merge.ts b/packages/gateway/src/merge.ts new file mode 100644 index 00000000..33a909ca --- /dev/null +++ b/packages/gateway/src/merge.ts @@ -0,0 +1,126 @@ +import type { SilentNode } from '@dormice/shared'; +import type { z } from 'zod'; +import { + awaitingFirstConfig, + awaitingFirstConfigWhy, + downReason, + type Fleet, + type NodeState, + STARTUP_GRACE_MS, +} from './fleet'; +import type { AskOptions, AskVerb } from './lookup'; + +/** + * How long a merged answer waits for a node. Longer than a lookup's two + * seconds on purpose: a lookup is one ledger read, while listSandboxMetrics + * reads every running container (about a second each, in parallel) and + * listSandboxImages inspects every shell — a busy node legitimately takes + * several seconds, and "slow is down" (design record #35) was said of + * finding a sandbox, where the caller holds a name's slot. Past it the + * node is silent for this answer, and named as such. + */ +export const MERGE_TIMEOUT_MS = 10_000; + +/** + * Whether a node is asked for a merged answer, and if not, what the + * answer says about it (`why` null: nothing — it holds nothing the answer + * could lack). Not asked: + * - a node that is down (fleet.ts downReason: two of its own intervals + * silent). A dial would wait the whole timeout for nothing, on every + * console poll, for as long as it stayed down — it is named with the + * reason placement refuses it; + * - a node awaiting its first configuration (awaitingFirstConfig): not + * listening, so a dial is refused at the socket and would read as + * silence anyway. Named when its reading says it holds sandboxes. + * Asked regardless: a node not heard from since a gateway start, for + * STARTUP_GRACE_MS — every node is silent so far after a restart, the + * running ones included, and its last row's endpoint is most likely a + * live daemon (the removeNode rule, routes/nodes.ts). Past the grace, a + * node still not heard from is down. + */ +export type Askability = { ask: true } | { ask: false; why: string | null }; + +export function askability( + node: NodeState, + now: Date, + startedAt: Date, +): Askability { + if ( + node.lastCheckInAt === null && + now.getTime() - startedAt.getTime() < STARTUP_GRACE_MS + ) { + return { ask: true }; + } + const down = downReason(node, now); + if (down !== null) return { ask: false, why: down }; + if (awaitingFirstConfig(node)) { + return { ask: false, why: awaitingFirstConfigWhy(node) }; + } + return { ask: true }; +} + +export interface Merged { + /** The nodes that answered, in node-id order, with what they said. */ + answers: Array<{ node: NodeState; value: T; headers: Headers }>; + /** The nodes this answer lacks, in node-id order (shared silentNodeSchema). */ + silent: SilentNode[]; +} + +/** + * Asks every askable node one verb in parallel and keeps every answer — + * the fleet-wide lists' one step (routes/observe.ts, the E2B list in + * routes/e2b.ts). Unlike the finder, which wants exactly one yes, a + * merged answer wants everyone, and a node that does not answer is not + * a reason to refuse the rest: the operator reads the fleet's sandboxes + * with one node in trouble, and reads which one. `verb` may depend on + * the node — the E2B list sends each node its own offset. + */ +export async function askEach( + fleet: Fleet, + ask: AskVerb, + now: Date, + verb: string | ((node: NodeState) => string), + body: unknown, + schema: z.ZodType, + options: AskOptions = {}, +): Promise> { + const silent: SilentNode[] = []; + const asked: NodeState[] = []; + for (const node of fleet.all()) { + const judged = askability(node, now, fleet.startedAt); + if (judged.ask) asked.push(node); + else if (judged.why !== null) + silent.push({ nodeId: node.id, why: judged.why }); + } + const results = await Promise.all( + asked.map(async (node) => ({ + node, + asked: await ask( + node, + typeof verb === 'string' ? verb : verb(node), + body, + schema, + { timeoutMs: MERGE_TIMEOUT_MS, ...options }, + ), + })), + ); + const answers: Merged['answers'] = []; + for (const { node, asked: answer } of results) { + if (answer.kind === 'answer') { + answers.push({ node, value: answer.value, headers: answer.headers }); + } else { + silent.push({ nodeId: node.id, why: answer.why }); + } + } + const byId = ( + a: N, + b: N, + ) => idOf(a).localeCompare(idOf(b)); + answers.sort(byId); + silent.sort(byId); + return { answers, silent }; +} + +function idOf(entry: { nodeId: string } | { node: NodeState }): string { + return 'nodeId' in entry ? entry.nodeId : entry.node.id; +} diff --git a/packages/gateway/src/routes/api-keys.test.ts b/packages/gateway/src/routes/api-keys.test.ts index 3884d2b5..bedefe6c 100644 --- a/packages/gateway/src/routes/api-keys.test.ts +++ b/packages/gateway/src/routes/api-keys.test.ts @@ -26,10 +26,9 @@ async function mint(app: TestApp, name: string, expiresAt?: string) { } /** - * Whether a credential opens the sandbox gate. The verb behind it that - * needs no node is one of the fleet-wide verbs the gateway does not route - * yet: its honest 501 is "you are through the door"; a 401 is not. (The - * next cut merges those verbs and this probe moves to a real answer.) + * Whether a credential opens the sandbox gate. The verb behind it needs + * no node: listSandboxes over an empty fleet is an empty list with nobody + * silent — a real answer, "you are through the door"; a 401 is not. */ const useKey = (app: TestApp, token: string, url = '/listSandboxes') => app.inject({ @@ -38,7 +37,7 @@ const useKey = (app: TestApp, token: string, url = '/listSandboxes') => headers: { authorization: `Bearer ${token}` }, payload: {}, }); -const OPENED = 501; +const OPENED = 200; describe('API keys on the gateway', () => { it('mints a 64-hex token, shown once and never stored in the view', async () => { @@ -71,7 +70,8 @@ describe('API keys on the gateway', () => { url: '/e2b/api/v2/sandboxes', headers: { 'x-api-key': `e2b_${token}` }, }); - expect(e2b.statusCode).toBe(501); + expect(e2b.statusCode).toBe(200); + expect(e2b.json()).toEqual([]); expect((await rpc(app, '/revokeApiKey', { id })).json()).toEqual({ revoked: true, diff --git a/packages/gateway/src/routes/e2b.ts b/packages/gateway/src/routes/e2b.ts index 83a7640c..927e807a 100644 --- a/packages/gateway/src/routes/e2b.ts +++ b/packages/gateway/src/routes/e2b.ts @@ -2,10 +2,14 @@ import type { KeyedQueue } from '@dormice/server/keyed-queue'; import { sandboxNameSchema } from '@dormice/shared'; import type { FastifyError, FastifyReply, FastifyRequest } from 'fastify'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { z } from 'zod'; +import { decodeCursor, encodeCursor, mergePages } from '../cursor'; import { relay } from '../errors'; import type { Finder } from '../find'; import type { Fleet, NodeState } from '../fleet'; import { forwardStream, replay } from '../forward'; +import type { AskVerb } from '../lookup'; +import { askEach } from '../merge'; import { type PlacementKnobs, refusalMessage } from '../placement'; import { RETRY_AFTER_SECONDS } from '../raw'; import { @@ -27,8 +31,23 @@ export interface E2bRoutesOptions { token: string; /** The app's one adjudication of a bare credential (fleet token or a live minted key). */ isCredential: (bareToken: string) => boolean; + /** Asks one node one verb on the gateway's account (lookup.ts httpAsk) — the list. */ + ask: AskVerb; } +/** The daemon's own bounds on a page (e2b/control.ts listQuerySchema), judged here first so every node is asked for the same page. */ +const listLimitSchema = z.coerce + .number() + .int() + .positive() + .max(1000) + .default(100); + +/** What the gateway reads of a node's list item: the two fields the merge orders by. Everything else passes through as the node wrote it. */ +const e2bListItemSchema = z + .object({ sandboxID: z.string(), startedAt: z.string() }) + .loose(); + /** * The E2B control plane in front of several nodes: what the official SDK * calls api.e2b.app for, mounted at /e2b/api like the daemon's. Creates @@ -39,7 +58,7 @@ export interface E2bRoutesOptions { */ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( app, - { fleet, finder, locks, knobs, token, isCredential }, + { fleet, finder, locks, knobs, token, isCredential, ask }, ) => { app.addContentTypeParser( 'application/json', @@ -171,13 +190,77 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( return create(request, reply, target, null, body, true); }); - app.get('/v2/sandboxes', async (_request, reply) => - send( - reply, - 501, - 'listing sandboxes is not routed by the gateway yet — call the node directly', - ), - ); + // The list across nodes: every askable node is asked its own page from + // its own offset (cursor.ts carries every node's), the pages are merged + // newest first and cut at the caller's limit. A node the answer would + // lack is a 503 naming it, not a shorter list: this wire is a bare + // array with nowhere to say what is missing, and the SDK's paginator + // would take a partial page for the whole fleet. The daemon judges + // `state` and `metadata`; the gateway reads only limit and the cursor. + app.get('/v2/sandboxes', async (request, reply) => { + const url = request.raw.url ?? ''; + const q = url.indexOf('?'); + const query = new URLSearchParams(q === -1 ? '' : url.slice(q + 1)); + const limit = listLimitSchema.safeParse(query.get('limit') ?? undefined); + if (!limit.success) { + return send( + reply, + 400, + `invalid limit: ${limit.error.issues[0]?.message ?? 'refused'}`, + ); + } + const token_ = query.get('nextToken'); + const offsets = + token_ === null || token_ === '' ? {} : decodeCursor(token_); + if (offsets === null) { + return send( + reply, + 400, + 'invalid nextToken — pass back the x-next-token of the previous page unchanged', + ); + } + query.delete('nextToken'); + query.set('limit', String(limit.data)); + const { answers, silent } = await askEach( + fleet, + ask, + new Date(), + (node) => { + const own = new URLSearchParams(query); + const offset = offsets[node.id] ?? 0; + if (offset > 0) own.set('nextToken', String(offset)); + return `e2b/api/v2/sandboxes?${own.toString()}`; + }, + undefined, + z.array(e2bListItemSchema), + { method: 'GET', credential: 'x-api-key' }, + ); + if (silent.length > 0) { + reply.header('retry-after', String(RETRY_AFTER_SECONDS)); + return send( + reply, + 503, + `the list is incomplete: ${silent + .map((s) => `node ${s.nodeId} did not answer (${s.why})`) + .join( + ', ', + )} — retry after Retry-After, or remove the node if it is gone for good`, + ); + } + const page = mergePages( + answers.map((a) => ({ + nodeId: a.node.id, + items: a.value, + more: a.headers.get('x-next-token') !== null, + })), + offsets, + limit.data, + ); + if (page.next !== null) { + reply.header('x-next-token', encodeCursor(page.next)); + } + return reply.code(200).send(page.items); + }); const byId = async (request: FastifyRequest, reply: FastifyReply) => { const { id } = request.params as { id: string }; diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index 279ed5c5..56947e4f 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -7,7 +7,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { relay } from '../errors'; import type { Finder } from '../find'; -import type { Fleet } from '../fleet'; +import type { Fleet, NodeState } from '../fleet'; import { forwardStream, replay } from '../forward'; import { type PlacementKnobs, refusalMessage } from '../placement'; import { RETRY_AFTER_SECONDS } from '../raw'; @@ -46,21 +46,27 @@ export const NAMED_VERBS = [ ] as const; /** - * The verbs that address the daemon, not a sandbox, and that the gateway - * cannot answer from its own tables: fleet lists, host readings, the - * upgrade. Asking every node and merging comes in a later cut; until then - * each answers an honest 501 naming the alternative, instead of a - * misleading answer from whichever node the gateway happened to pick. - * (Keys, settings, templates and ingress left this list with the - * configuration authority.) + * The verbs that address one machine, by its node id: a host reading is + * one host's (N nodes' CPU percentages add up to nothing — shared + * host.ts has the rule), so the request names its node and is forwarded + * there whole. A fleet of one needs no name: the single-machine install + * asks as it always has. The figures that do add up are getFleetMetrics + * (routes/fleet.ts), from the readings the gateway already holds. */ -export const UNNAMED_VERBS = [ - 'listSandboxes', - 'listSandboxMetrics', - 'listSandboxImages', - 'getFleetStateHistory', +export const BY_NODE_VERBS = [ 'getHostMetrics', 'getHostMetricsHistory', +] as const; + +/** + * The verbs that address the daemon and that the gateway does not route + * yet: the upgrade, whose fleet-wide form (the gateway upgrades itself, + * then rolls the nodes one at a time) is the fourth cut's. Until then + * each answers an honest 501 naming the alternative. (The lists merged + * and the host readings went by node in the third cut; keys, settings, + * templates and ingress left with the configuration authority.) + */ +export const UNNAMED_VERBS = [ 'checkUpgrade', 'applyUpgrade', 'getUpgradeStatus', @@ -90,11 +96,87 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( for (const verb of UNNAMED_VERBS) { app.post(`/${verb}`, async (_request, reply) => reply.code(501).send({ - message: `${verb} is not routed by the gateway yet — call the node directly (this version routes only sandbox-addressed verbs)`, + message: `${verb} is not routed by the gateway until the upgrade cut — call the node directly`, }), ); } + for (const verb of BY_NODE_VERBS) { + app.post(`/${verb}`, async (request, reply) => { + const body = request.body as Buffer | undefined; + const parsed = parseJson(body) as { nodeId?: unknown } | undefined; + const nodeId = parsed?.nodeId; + if ( + nodeId !== undefined && + (typeof nodeId !== 'string' || nodeId === '') + ) { + return reply + .code(400) + .send({ message: 'nodeId must be a non-empty string when given' }); + } + const chosen = nodeFor(nodeId); + if ('refusal' in chosen) { + if (chosen.retryAfter) { + reply.header('retry-after', String(RETRY_AFTER_SECONDS)); + } + return reply.code(chosen.status).send({ message: chosen.refusal }); + } + const { node } = chosen; + return forwarded(request, reply, ' — retry', async () => { + await forwardStream(request.raw, reply.raw, { + target: { endpoint: node.endpoint, token }, + credential: 'bearer', + body, + }); + }); + }); + } + + /** + * The machine a by-node verb is about. Named: that node, or 404 for an + * id the fleet has no row for. Unnamed: the one node of a fleet of one; + * a fleet of several is refused (400) rather than have a machine picked + * for the caller — the answer would read as the fleet's; a fleet of + * none has no machine to read (503, a node joins at its first check-in). + * Whether the node answers is the forward's to find out: a node that is + * down earns the node-did-not-answer 502 like any forwarded verb. + */ + function nodeFor( + nodeId: string | undefined, + ): + | { node: NodeState } + | { status: number; refusal: string; retryAfter?: boolean } { + if (nodeId !== undefined) { + const node = fleet.get(nodeId); + return node !== undefined + ? { node } + : { + status: 404, + refusal: `no node with id '${nodeId}' — listNodes shows which exist`, + }; + } + const all = fleet.all(); + const only = all[0]; + if (all.length === 1 && only !== undefined) return { node: only }; + if (all.length === 0) { + return { + status: 503, + refusal: + "no node has checked in yet — a machine's reading needs a machine, and a node joins the fleet at its first check-in", + retryAfter: true, + }; + } + return { + status: 400, + refusal: `the fleet has ${all.length} nodes (${all + .map((n) => n.id) + .sort() + .join( + ', ', + )}) — name one with nodeId (listNodes lists them); the figures that add up across the fleet are getFleetMetrics`, + }; + } + for (const verb of NAMED_VERBS) { const bodyLimit = verb === 'writeFile' || verb === 'writeFiles' diff --git a/packages/gateway/src/routes/observe.ts b/packages/gateway/src/routes/observe.ts new file mode 100644 index 00000000..1a8bafbb --- /dev/null +++ b/packages/gateway/src/routes/observe.ts @@ -0,0 +1,84 @@ +import { + listSandboxesResponseSchema, + listSandboxImagesResponseSchema, + listSandboxMetricsResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import type { Fleet } from '../fleet'; +import type { AskVerb } from '../lookup'; +import { askEach } from '../merge'; + +export interface ObserveRoutesOptions { + fleet: Fleet; + /** Asks one node one verb on the gateway's account (lookup.ts httpAsk). */ + ask: AskVerb; +} + +/** + * The fleet-wide lists: the three observation verbs whose answer is every + * node's answer put together — listSandboxes, listSandboxMetrics, + * listSandboxImages. Every askable node is asked in parallel (merge.ts), + * the arrays are concatenated in node-id order with each node's own + * order kept (the caller filters and sorts, as the wire has always said), + * and the nodes the answer lacks are named in `silent` — always present + * here, empty when nobody was missing. A node's own answer has nobody to + * be silent about and carries none. + * + * Behind the sandbox gate, like the node's: observation is what every + * credential that addresses a sandbox may do. Nothing here wakes a + * sandbox; the nodes' verbs never did. + */ +export const observeRoutes: FastifyPluginAsyncZod< + ObserveRoutesOptions +> = async (app, { fleet, ask }) => { + app.post( + '/listSandboxes', + { schema: { response: { 200: listSandboxesResponseSchema } } }, + async () => { + const { answers, silent } = await askEach( + fleet, + ask, + new Date(), + 'listSandboxes', + {}, + listSandboxesResponseSchema, + ); + return { + sandboxes: answers.flatMap((a) => a.value.sandboxes), + silent, + }; + }, + ); + + app.post( + '/listSandboxMetrics', + { schema: { response: { 200: listSandboxMetricsResponseSchema } } }, + async () => { + const { answers, silent } = await askEach( + fleet, + ask, + new Date(), + 'listSandboxMetrics', + {}, + listSandboxMetricsResponseSchema, + ); + return { samples: answers.flatMap((a) => a.value.samples), silent }; + }, + ); + + app.post( + '/listSandboxImages', + { schema: { response: { 200: listSandboxImagesResponseSchema } } }, + async () => { + const { answers, silent } = await askEach( + fleet, + ask, + new Date(), + 'listSandboxImages', + {}, + listSandboxImagesResponseSchema, + ); + return { images: answers.flatMap((a) => a.value.images), silent }; + }, + ); +}; diff --git a/packages/gateway/src/routes/templates.test.ts b/packages/gateway/src/routes/templates.test.ts index 22ecd97f..9649f77e 100644 --- a/packages/gateway/src/routes/templates.test.ts +++ b/packages/gateway/src/routes/templates.test.ts @@ -18,7 +18,11 @@ function templatesGateway(users: Record) { if (answer === undefined || answer === 'silent') { return { kind: 'silent', why: 'ECONNREFUSED' }; } - return { kind: 'answer', value: schema.parse({ sandboxNames: answer }) }; + return { + kind: 'answer', + value: schema.parse({ sandboxNames: answer }), + headers: new Headers(), + }; }; const { app, db, fleet } = testGateway({}, { askVerb }); let host = 1; From 62e101d6e03532244e92d03a439774b48fc37eef Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 00:45:35 +0800 Subject: [PATCH 48/89] The fleet's state history is the gateway's: one sample per check-in, summed over every node; the node writes no fleet row and names the gateway for the verbs that live there MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fleet_state_samples (gateway migration 0002) holds the fleet's census by state over time — the one figure no single node can compute (design record #26) — written on each check-in once the reporting node's reading is in, from every node's last reading (a down node's sandboxes are still there). No ticker: the gateway only listens, and the check-ins are its clock. Within the startup grace nothing is written while a known node has not checked in, or every restart would draw the fleet collapsing to the first node heard and climbing back. Kept 30 days, pruned with the write. getFleetMetrics answers the present from the same readings (nodes total / reachable / reported, the census, the disks' bill) without asking a node; getFleetStateHistory answers the past from the samples, bucketed by whole rows with the peak from raw rows, through @dormice/server/history. The node's sampler writes no fleet row and its getFleetStateHistory route is gone; fleet_snapshots stays as a legacy table, read by nothing, until the fourth cut's import has carried a production node's last 30 days into the gateway (a single-node fleet's history is the same figure) — the DROP ships with that import. A node's 404 for a verb that answers at the gateway alone, and for the console, names the gateway. --- e2e/src/gateway.test.ts | 41 ++ e2e/src/native.test.ts | 12 +- .../drizzle/0002_fleet-state-samples.sql | 11 + .../gateway/drizzle/meta/0002_snapshot.json | 439 ++++++++++++++++++ packages/gateway/drizzle/meta/_journal.json | 7 + packages/gateway/src/app.ts | 2 + packages/gateway/src/db/fleet-samples.ts | 93 ++++ packages/gateway/src/db/schema.ts | 51 +- packages/gateway/src/fleet.ts | 60 ++- packages/gateway/src/routes/fleet.test.ts | 292 ++++++++++++ packages/gateway/src/routes/fleet.ts | 105 +++++ packages/gateway/src/routes/nodes.ts | 4 + packages/gateway/src/testing.ts | 4 +- packages/server/src/app.test.ts | 14 + packages/server/src/app.ts | 20 +- packages/server/src/config.ts | 2 +- packages/server/src/db/metrics.ts | 85 +--- packages/server/src/db/schema.ts | 34 +- packages/server/src/metrics-sampler.test.ts | 61 +-- packages/server/src/metrics-sampler.ts | 13 +- packages/server/src/routes/host.ts | 52 +-- .../server/src/routes/observability.test.ts | 124 ----- 22 files changed, 1199 insertions(+), 327 deletions(-) create mode 100644 packages/gateway/drizzle/0002_fleet-state-samples.sql create mode 100644 packages/gateway/drizzle/meta/0002_snapshot.json create mode 100644 packages/gateway/src/db/fleet-samples.ts create mode 100644 packages/gateway/src/routes/fleet.test.ts create mode 100644 packages/gateway/src/routes/fleet.ts diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts index 3b749b53..f17b644e 100644 --- a/e2e/src/gateway.test.ts +++ b/e2e/src/gateway.test.ts @@ -431,6 +431,47 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { } }); + it('getFleetMetrics sums both nodes from their check-ins without asking them; getFleetStateHistory grows a point per check-in with a peak', async () => { + const metrics = await viaGateway().getFleetMetrics(); + expect(metrics.nodes).toEqual({ total: 2, reachable: 2, reported: 2 }); + const own = await Promise.all( + nodes().map((n) => direct(n.id).getHostMetrics()), + ); + expect(metrics.sandboxes.total).toBe( + own.reduce((sum, m) => sum + m.sandboxes.total, 0), + ); + expect(metrics.sandboxDisks.count).toBe( + own.reduce((sum, m) => sum + m.sandboxDisks.count, 0), + ); + // A check-in a second: a couple of them bring points and a peak. + const history = await until(async () => { + const h = await viaGateway().getFleetStateHistory(); + return h.points.length >= 2 && h.peak !== null ? h : undefined; + }); + for (const point of history.points) { + const sum = Object.values(point.byState).reduce((a, b) => a + b, 0); + expect(sum).toBe(point.total); + } + expect(history.bucketSeconds).toBeNull(); + }); + + it('a host reading at the door names its node; unnamed in a fleet of two it is a 400 naming both', async () => { + const b = await viaGateway().getHostMetrics({ nodeId: 'node-b' }); + const own = await direct('node-b').getHostMetrics(); + expect(b.host.cpuCount).toBe(own.host.cpuCount); + expect(b.sandboxes.total).toBe(own.sandboxes.total); + await expect(viaGateway().getHostMetrics()).rejects.toMatchObject({ + status: 400, + message: expect.stringMatching( + /the fleet has 2 nodes \(node-b, node-c\)/, + ), + }); + const history = await viaGateway().getHostMetricsHistory({ + nodeId: 'node-c', + }); + expect(history.points.length).toBeGreaterThanOrEqual(1); + }); + it('the upgrade verbs are an honest 501 until their cut; a misspelled verb is a 404', async () => { await expect(viaGateway().checkUpgrade()).rejects.toMatchObject({ status: 501, diff --git a/e2e/src/native.test.ts b/e2e/src/native.test.ts index 657ebe01..4ebabf2b 100644 --- a/e2e/src/native.test.ts +++ b/e2e/src/native.test.ts @@ -872,17 +872,23 @@ describe('the observability verbs over a real daemon', () => { ).rejects.toMatchObject({ name: 'DormiceApiError', status: 404 }); }); - it('getFleetStateHistory reports points and a peak once the fleet was seen', async () => { + it('getFleetStateHistory at the door reports points and a peak once the fleet was seen; a node answers no fleet history of its own', async () => { await client().acquireSandbox('obs-timeline-key'); + // The node's own answer: none — the fleet's history is the gateway's, + // and the node's 404 says where it lives. + await expect(client().getFleetStateHistory()).rejects.toMatchObject({ + status: 404, + message: expect.stringMatching(/answers at the gateway/), + }); const deadline = Date.now() + 15_000; - let timeline = await client().getFleetStateHistory(); + let timeline = await viaDoor().getFleetStateHistory(); // Wait for a tick that observed at least one sandbox alive. while ( (timeline.points.length < 1 || (timeline.peak?.active ?? 0) < 1) && Date.now() < deadline ) { await sleep(0.5); - timeline = await client().getFleetStateHistory(); + timeline = await viaDoor().getFleetStateHistory(); } expect(timeline.points.length).toBeGreaterThanOrEqual(1); expect(timeline.peak?.active).toBeGreaterThanOrEqual(1); diff --git a/packages/gateway/drizzle/0002_fleet-state-samples.sql b/packages/gateway/drizzle/0002_fleet-state-samples.sql new file mode 100644 index 00000000..af3c2333 --- /dev/null +++ b/packages/gateway/drizzle/0002_fleet-state-samples.sql @@ -0,0 +1,11 @@ +CREATE TABLE `fleet_state_samples` ( + `at` text NOT NULL, + `active` integer NOT NULL, + `frozen` integer NOT NULL, + `stopped` integer NOT NULL, + `archived` integer NOT NULL, + `restoring` integer NOT NULL, + `total` integer NOT NULL +); +--> statement-breakpoint +CREATE INDEX `fleet_state_samples_at_idx` ON `fleet_state_samples` (`at`); \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/0002_snapshot.json b/packages/gateway/drizzle/meta/0002_snapshot.json new file mode 100644 index 00000000..a070d01a --- /dev/null +++ b/packages/gateway/drizzle/meta/0002_snapshot.json @@ -0,0 +1,439 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "eba4fadc-ca03-41c6-b5a1-72adf1b2552c", + "prevId": "2821eca7-9ec2-43c6-a788-77c858d4360d", + "tables": { + "api_keys": { + "name": "api_keys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + "key_hash" + ], + "isUnique": true + }, + "api_keys_active_name_idx": { + "name": "api_keys_active_name_idx", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"api_keys\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "console_account": { + "name": "console_account", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_secret": { + "name": "session_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "fleet_state_samples": { + "name": "fleet_state_samples", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active": { + "name": "active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "frozen": { + "name": "frozen", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stopped": { + "name": "stopped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "restoring": { + "name": "restoring", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "fleet_state_samples_at_idx": { + "name": "fleet_state_samples_at_idx", + "columns": [ + "at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "nodes": { + "name": "nodes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "added_at": { + "name": "added_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_gb": { + "name": "swap_gb", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_cpus": { + "name": "sandbox_cpus", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_memory_gb": { + "name": "sandbox_memory_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_disk_gb": { + "name": "sandbox_disk_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_freeze_after_seconds": { + "name": "default_freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_stop_after_seconds": { + "name": "default_stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_archive_after_seconds": { + "name": "default_archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_endpoint": { + "name": "s3_endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_access_key_id": { + "name": "s3_access_key_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_secret_access_key": { + "name": "s3_secret_access_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_region": { + "name": "s3_region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_force_path_style": { + "name": "s3_force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain": { + "name": "sandbox_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain_aliases": { + "name": "sandbox_domain_aliases", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pids_limit": { + "name": "pids_limit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/_journal.json b/packages/gateway/drizzle/meta/_journal.json index b37272a1..36e3069b 100644 --- a/packages/gateway/drizzle/meta/_journal.json +++ b/packages/gateway/drizzle/meta/_journal.json @@ -15,6 +15,13 @@ "when": 1789371415369, "tag": "0001_configuration-authority", "breakpoints": true + }, + { + "idx": 2, + "version": "6", + "when": 1789403526928, + "tag": "0002_fleet-state-samples", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts index bd947d50..6d653abe 100644 --- a/packages/gateway/src/app.ts +++ b/packages/gateway/src/app.ts @@ -28,6 +28,7 @@ import { apiKeyRoutes } from './routes/api-keys'; import { consoleRoutes } from './routes/console'; import { e2bControlRoutes } from './routes/e2b'; import { envdTokenRoutes } from './routes/envd-token'; +import { fleetRoutes } from './routes/fleet'; import { ingressRoutes } from './routes/ingress'; import { nativeRoutes } from './routes/native'; import { checkInRoutes, nodeRoutes } from './routes/nodes'; @@ -251,6 +252,7 @@ export function buildGatewayApp({ api.addHook('onRequest', apiAuth); await api.register(envdTokenRoutes, { finder, token }); await api.register(observeRoutes, { fleet, ask: askVerb }); + await api.register(fleetRoutes, { db, fleet }); // Its own sub-scope: the byte-preserving body parser it installs must // not reach the gateway's own verbs, which keep Fastify's JSON parsing. await api.register(nativeRoutes, { fleet, finder, locks, knobs, token }); diff --git a/packages/gateway/src/db/fleet-samples.ts b/packages/gateway/src/db/fleet-samples.ts new file mode 100644 index 00000000..168f5f04 --- /dev/null +++ b/packages/gateway/src/db/fleet-samples.ts @@ -0,0 +1,93 @@ +import { and, asc, desc, gte, lt, lte } from 'drizzle-orm'; +import { type Fleet, STARTUP_GRACE_MS, sumReadings } from '../fleet'; +import type { Db } from './db'; +import { type FleetStateSampleRow, fleetStateSamples } from './schema'; + +/** + * How long fleet state samples live. Not a knob: the dashboard's widest + * range (30 days) defines the need, and at one small row per check-in the + * table stays tens of megabytes for a fleet of ten (the node's old fleet + * table had the same ruling). + */ +export const FLEET_SAMPLE_KEEP_DAYS = 30; + +/** + * One sample of the fleet's state, written on a check-in (routes/nodes.ts) + * once the reporting node's reading is in: the sum over every node that + * has a reading — a node that is down contributes its last one; its + * sandboxes are still there. Prune rides the same transaction, as on the + * node's sampler. + * + * Not written while the gateway is still getting to know its fleet: + * within STARTUP_GRACE_MS of a start, as long as any node from the rows + * has not checked in yet. A restarted gateway hears from its nodes one by + * one over an interval, and a sum written after the first would draw the + * fleet collapsing to that node's share and climbing back — a false dip + * on the curve at every gateway restart. Past the grace, a node still + * silent is down, and the sum is written without it (a lower bound, as + * getFleetMetrics says of the same figure). Answers whether a row was + * written. + */ +export function recordFleetSample(db: Db, fleet: Fleet, now: Date): boolean { + const nodes = fleet.all(); + const settling = now.getTime() - fleet.startedAt.getTime() < STARTUP_GRACE_MS; + if (settling && nodes.some((node) => node.reading === null)) return false; + const { sandboxes } = sumReadings(nodes); + const cutoff = new Date( + now.getTime() - FLEET_SAMPLE_KEEP_DAYS * 86_400_000, + ).toISOString(); + db.transaction((tx) => { + tx.insert(fleetStateSamples) + .values({ + at: now.toISOString(), + ...sandboxes.byState, + total: sandboxes.total, + }) + .run(); + tx.delete(fleetStateSamples).where(lt(fleetStateSamples.at, cutoff)).run(); + }); + return true; +} + +/** Ascending slice — ISO strings compare lexicographically as time. */ +export function queryFleetSamples( + db: Db, + startIso: string, + endIso: string, +): FleetStateSampleRow[] { + return db + .select() + .from(fleetStateSamples) + .where( + and( + gte(fleetStateSamples.at, startIso), + lte(fleetStateSamples.at, endIso), + ), + ) + .orderBy(asc(fleetStateSamples.at)) + .all(); +} + +/** + * The window's concurrency peak, from raw rows so no bucketing can flatten + * it: highest active count, and the earliest instant it was observed. + */ +export function queryFleetPeak( + db: Db, + startIso: string, + endIso: string, +): { active: number; at: string } | null { + const row = db + .select({ active: fleetStateSamples.active, at: fleetStateSamples.at }) + .from(fleetStateSamples) + .where( + and( + gte(fleetStateSamples.at, startIso), + lte(fleetStateSamples.at, endIso), + ), + ) + .orderBy(desc(fleetStateSamples.active), asc(fleetStateSamples.at)) + .limit(1) + .get(); + return row ?? null; +} diff --git a/packages/gateway/src/db/schema.ts b/packages/gateway/src/db/schema.ts index e6e993fe..21a35614 100644 --- a/packages/gateway/src/db/schema.ts +++ b/packages/gateway/src/db/schema.ts @@ -1,5 +1,6 @@ import { sql } from 'drizzle-orm'; import { + index, integer, real, sqliteTable, @@ -10,12 +11,14 @@ import { /** * The gateway's tables are "how the fleet is configured and who may * enter" — never a sandbox's state, which lives in the ledger of the node - * that runs it and is asked for when needed (find.ts). Five tables: the - * nodes that have ever checked in, the fleet-wide settings row, the - * templates, the API keys and the console account. The last four moved - * here from the daemon with the configuration authority (design record - * #22, 2026-09-13): one authority, one edit, every node pulls it at its - * next check-in and keeps a copy in its own ledger. + * that runs it and is asked for when needed (find.ts). Five tables of + * configuration: the nodes that have ever checked in, the fleet-wide + * settings row, the templates, the API keys and the console account. The + * last four moved here from the daemon with the configuration authority + * (design record #22, 2026-09-13): one authority, one edit, every node + * pulls it at its next check-in and keeps a copy in its own ledger. And + * one table of observation, fleet_state_samples: the one figure no single + * node can compute (design record #26). */ /** @@ -186,3 +189,39 @@ export const consoleAccount = sqliteTable('console_account', { }); export type ConsoleAccountRow = typeof consoleAccount.$inferSelect; + +/** + * The fleet's state counts over time — how many sandboxes sat in each + * state across every node — one row per node check-in (routes/nodes.ts, + * db/fleet-samples.ts): the sum of every node's last census at that + * moment, the data behind the console's concurrency curve and its peak. + * Written on the check-in rather than by a ticker of the gateway's own, + * which has none: the gateway only listens and compares, and the + * check-ins are its clock. The one figure no single node can compute + * (design record #26) — each node's own machine history stays on that + * node (host_metrics_samples), and the nodes wrote no fleet history of + * their own since the third cut. Kept 30 days, the dashboard's widest + * range; pruned with every write. + * + * Five explicit state columns instead of a JSON blob, as on the node's + * old table: the window peak is max(active) in one SQL aggregate, and the + * stacked chart needs each state addressable. `total` is stored + * redundantly so readers never re-derive it. `at` is indexed, not unique: + * two nodes may check in within the same millisecond. + */ +export const fleetStateSamples = sqliteTable( + 'fleet_state_samples', + { + /** ISO 8601 UTC — when the check-in that produced this sum arrived. */ + at: text('at').notNull(), + active: integer('active').notNull(), + frozen: integer('frozen').notNull(), + stopped: integer('stopped').notNull(), + archived: integer('archived').notNull(), + restoring: integer('restoring').notNull(), + total: integer('total').notNull(), + }, + (table) => [index('fleet_state_samples_at_idx').on(table.at)], +); + +export type FleetStateSampleRow = typeof fleetStateSamples.$inferSelect; diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index ff01c533..252fa40d 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -1,4 +1,10 @@ -import type { BuildInfo, CheckInRequest, NodeReading } from '@dormice/shared'; +import type { + BuildInfo, + CheckInRequest, + NodeReading, + SandboxDisks, + SandboxStateCounts, +} from '@dormice/shared'; import { eq } from 'drizzle-orm'; import type { Db } from './db/db'; import { nodes } from './db/schema'; @@ -92,6 +98,58 @@ export function awaitingFirstConfigWhy(node: NodeState): string | null { : `not listening — it holds ${total} sandboxes but no configuration copy yet, and its first bundle rides on its next check-in`; } +/** + * The figures that add up across the fleet, summed over the nodes that + * have a reading — the census by state and the sandbox disks' bill — and + * how many nodes that is. A node without a reading (not heard from since + * this gateway started) contributes nothing, and `reported` says so: the + * sums are a lower bound until every node has spoken. A node that is + * down but did report contributes its last reading — its sandboxes are + * still there, merely out of reach. One function for the check-in's + * sample (db/fleet-samples.ts) and getFleetMetrics (routes/fleet.ts), so + * the curve and the number under it can never disagree. + */ +export function sumReadings(nodes: readonly NodeState[]): { + reported: number; + sandboxes: { total: number; byState: SandboxStateCounts }; + sandboxDisks: SandboxDisks; +} { + const byState: SandboxStateCounts = { + active: 0, + frozen: 0, + stopped: 0, + archived: 0, + restoring: 0, + }; + const sandboxDisks: SandboxDisks = { + count: 0, + nominalBytes: 0, + actualBytes: 0, + }; + let total = 0; + let reported = 0; + for (const node of nodes) { + if (node.reading === null) continue; + reported += 1; + total += node.reading.sandboxes.total; + for (const state of Object.keys(byState) as Array< + keyof SandboxStateCounts + >) { + byState[state] += node.reading.sandboxes.byState[state]; + } + // Optional on the wire for the rolling upgrade (shared gateway.ts): + // a node on the previous build reports no disks, and its share is + // simply not in the sum. + const disks = node.reading.sandboxDisks; + if (disks !== undefined) { + sandboxDisks.count += disks.count; + sandboxDisks.nominalBytes += disks.nominalBytes; + sandboxDisks.actualBytes += disks.actualBytes; + } + } + return { reported, sandboxes: { total, byState }, sandboxDisks }; +} + /** What a check-in came to: taken (and whether it joined or moved), or refused with the sentence the node is told (routes/nodes.ts answers 409). */ export type CheckInOutcome = | { node: NodeState; joined: boolean; movedFrom: string | null } diff --git a/packages/gateway/src/routes/fleet.test.ts b/packages/gateway/src/routes/fleet.test.ts new file mode 100644 index 00000000..c9b52336 --- /dev/null +++ b/packages/gateway/src/routes/fleet.test.ts @@ -0,0 +1,292 @@ +import { + getFleetMetricsResponseSchema, + getFleetStateHistoryResponseSchema, +} from '@dormice/shared'; +import { describe, expect, it } from 'vitest'; +import { fleetStateSamples } from '../db/schema'; +import { STARTUP_GRACE_MS } from '../fleet'; +import { checkInOf, TEST_TOKEN, testGateway } from '../testing'; + +// The fleet's own observation: what the check-ins write, what the two +// verbs read back — over app.inject(), the check-ins posted as a node +// posts them. + +const authed = { authorization: `Bearer ${TEST_TOKEN}` }; +type App = ReturnType['app']; + +function rpc(app: App, url: string, payload: object = {}) { + return app.inject({ method: 'POST', url, headers: authed, payload }); +} + +async function checkIn( + app: App, + id: string, + over: Parameters[2] = {}, +) { + const res = await rpc( + app, + '/checkIn', + checkInOf(id, `http://${id}:80`, over), + ); + expect(res.statusCode).toBe(200); +} + +/** A gateway started long enough ago that no startup grace applies — against the real clock, which the check-in route reads. */ +const SETTLED = new Date(Date.now() - STARTUP_GRACE_MS - 60_000); + +function samples(db: ReturnType['db']) { + return db + .select() + .from(fleetStateSamples) + .orderBy(fleetStateSamples.at) + .all(); +} + +describe('the fleet state sample a check-in writes', () => { + it('each check-in writes one row: the sum over every node with a reading, a down node counted by its last reading', async () => { + const { app, db, fleet } = testGateway({}, { startedAt: SETTLED }); + await checkIn(app, 'b', { active: 3, frozen: 1 }); + await checkIn(app, 'c', { active: 2, archived: 4 }); + const rows = samples(db); + expect(rows).toHaveLength(2); + // The first check-in knew only b; the second knew both. + expect(rows[0]).toMatchObject({ + active: 3, + frozen: 1, + archived: 0, + total: 4, + }); + expect(rows[1]).toMatchObject({ + active: 5, + frozen: 1, + archived: 4, + total: 10, + }); + // c falls silent: its sandboxes are still there, and b's next check-in + // sums c's last reading in. + const c = fleet.get('c'); + if (!c) throw new Error('node lost'); + c.lastCheckInAt = new Date(Date.now() - 40_000); + await checkIn(app, 'b', { active: 3, frozen: 1 }); + expect(samples(db)[2]).toMatchObject({ active: 5, total: 10 }); + // Removed, it is counted no more. + expect((await rpc(app, '/removeNode', { id: 'c' })).json()).toEqual({ + removed: true, + }); + await checkIn(app, 'b', { active: 3, frozen: 1 }); + expect(samples(db)[3]).toMatchObject({ active: 3, total: 4 }); + }); + + it('within the startup grace no row is written while a known node has not checked in; past it the sum is written without it', async () => { + // A node known from the rows but not heard from since this start: a + // check-in's reading, then the memory a restart leaves — the row and + // nothing else (fleet.ts's constructor shape). + const silence = (fleet: ReturnType['fleet']) => { + fleet.checkIn(checkInOf('c', 'http://c:80', { active: 1 })); + const c = fleet.get('c'); + if (!c) throw new Error('node lost'); + c.reading = null; + c.lastCheckInAt = null; + c.intervalSeconds = null; + }; + const fresh = testGateway({}, { startedAt: new Date() }); + silence(fresh.fleet); + await checkIn(fresh.app, 'b', { active: 2 }); + expect(samples(fresh.db)).toHaveLength(0); + + const settled = testGateway({}, { startedAt: SETTLED }); + silence(settled.fleet); + await checkIn(settled.app, 'b', { active: 2 }); + expect(samples(settled.db)).toEqual([ + expect.objectContaining({ active: 2, total: 2 }), + ]); + }); + + it('rows older than 30 days are pruned with the write', async () => { + const { app, db } = testGateway({}, { startedAt: SETTLED }); + db.insert(fleetStateSamples) + .values({ + at: new Date(Date.now() - 31 * 86_400_000).toISOString(), + active: 9, + frozen: 0, + stopped: 0, + archived: 0, + restoring: 0, + total: 9, + }) + .run(); + await checkIn(app, 'b', { active: 1 }); + const rows = samples(db); + expect(rows).toHaveLength(1); + expect(rows[0]?.active).toBe(1); + }); +}); + +describe('getFleetMetrics', () => { + it('sums the census and the disks over the reported nodes, and says how many nodes that is', async () => { + const { app, fleet } = testGateway({}, { startedAt: SETTLED }); + await checkIn(app, 'b', { active: 3, frozen: 1 }); + await checkIn(app, 'c', { active: 2, archived: 4 }); + // A third node known from a previous run, not heard from: in total, + // not in reported, not in the sums. + fleet.checkIn(checkInOf('d', 'http://d:80')); + const d = fleet.get('d'); + if (!d) throw new Error('node lost'); + d.reading = null; + d.lastCheckInAt = null; + d.intervalSeconds = null; + const res = await rpc(app, '/getFleetMetrics'); + expect(res.statusCode).toBe(200); + const body = getFleetMetricsResponseSchema.parse(res.json()); + expect(body.nodes).toEqual({ total: 3, reachable: 2, reported: 2 }); + expect(body.sandboxes).toEqual({ + total: 10, + byState: { active: 5, frozen: 1, stopped: 0, archived: 4, restoring: 0 }, + }); + // The scaffolding's reading carries no disks (a node on the previous + // build): the bill is an honest zero, not a refusal. + expect(body.sandboxDisks).toEqual({ + count: 0, + nominalBytes: 0, + actualBytes: 0, + }); + }); + + it('sums the disks a reading carries', async () => { + const { app } = testGateway({}, { startedAt: SETTLED }); + const withDisks = (id: string, count: number) => ({ + ...checkInOf(id, `http://${id}:80`), + reading: { + ...checkInOf(id, `http://${id}:80`).reading, + sandboxDisks: { count, nominalBytes: count * 10, actualBytes: count }, + }, + }); + expect((await rpc(app, '/checkIn', withDisks('b', 2))).statusCode).toBe( + 200, + ); + expect((await rpc(app, '/checkIn', withDisks('c', 3))).statusCode).toBe( + 200, + ); + const body = getFleetMetricsResponseSchema.parse( + (await rpc(app, '/getFleetMetrics')).json(), + ); + expect(body.sandboxDisks).toEqual({ + count: 5, + nominalBytes: 50, + actualBytes: 5, + }); + }); + + it('is behind the sandbox gate: a minted key reads it, no token does not', async () => { + const { app } = testGateway({}, { startedAt: SETTLED }); + const minted = (await rpc(app, '/createApiKey', { name: 'ci' })).json(); + const keyed = await app.inject({ + method: 'POST', + url: '/getFleetMetrics', + headers: { authorization: `Bearer ${minted.token}` }, + payload: {}, + }); + expect(keyed.statusCode).toBe(200); + const bare = await app.inject({ + method: 'POST', + url: '/getFleetMetrics', + payload: {}, + }); + expect(bare.statusCode).toBe(401); + }); +}); + +describe('getFleetStateHistory', () => { + it('answers an empty window with no points and a null peak; then the samples ascending, byState summing to total, the peak from raw rows', async () => { + const { app, db } = testGateway({}, { startedAt: SETTLED }); + const empty = getFleetStateHistoryResponseSchema.parse( + (await rpc(app, '/getFleetStateHistory', {})).json(), + ); + expect(empty).toEqual({ points: [], bucketSeconds: null, peak: null }); + + const t0 = Date.parse('2026-09-15T10:00:00.000Z'); + const row = (offsetMs: number, active: number) => ({ + at: new Date(t0 + offsetMs).toISOString(), + active, + frozen: 1, + stopped: 0, + archived: 0, + restoring: 0, + total: active + 1, + }); + db.insert(fleetStateSamples) + .values([row(0, 2), row(15_000, 7), row(30_000, 3)]) + .run(); + const body = getFleetStateHistoryResponseSchema.parse( + ( + await rpc(app, '/getFleetStateHistory', { + start: new Date(t0 - 1000).toISOString(), + end: new Date(t0 + 60_000).toISOString(), + }) + ).json(), + ); + expect(body.bucketSeconds).toBeNull(); + expect(body.points.map((p) => p.byState.active)).toEqual([2, 7, 3]); + for (const point of body.points) { + const sum = Object.values(point.byState).reduce((a, b) => a + b, 0); + expect(sum).toBe(point.total); + } + expect(body.peak).toEqual({ + active: 7, + at: new Date(t0 + 15_000).toISOString(), + }); + }); + + it("buckets past 360 points by keeping each bucket's last whole row, and the peak survives bucketing", async () => { + const { app, db } = testGateway({}, { startedAt: SETTLED }); + const t0 = Date.parse('2026-09-15T00:00:00.000Z'); + const rows = 400; + const values = []; + for (let i = 0; i < rows; i += 1) { + values.push({ + at: new Date(t0 + i * 15_000).toISOString(), + active: 1, + frozen: 0, + stopped: 0, + archived: 0, + restoring: 0, + total: 1, + }); + } + // A spike squeezed between two grid rows of its own bucket. + values.push({ + at: new Date(t0 + 200 * 15_000 + 500).toISOString(), + active: 9, + frozen: 0, + stopped: 0, + archived: 0, + restoring: 0, + total: 9, + }); + db.insert(fleetStateSamples).values(values).run(); + const body = getFleetStateHistoryResponseSchema.parse( + ( + await rpc(app, '/getFleetStateHistory', { + start: new Date(t0).toISOString(), + end: new Date(t0 + rows * 15_000).toISOString(), + }) + ).json(), + ); + expect(body.bucketSeconds).not.toBeNull(); + expect(body.points.length).toBeLessThanOrEqual(360); + expect(body.peak).toEqual({ + active: 9, + at: new Date(t0 + 200 * 15_000 + 500).toISOString(), + }); + for (const point of body.points) { + const sum = Object.values(point.byState).reduce((a, b) => a + b, 0); + expect(sum).toBe(point.total); + } + }); + + it('rejects an unparseable timestamp at the door', async () => { + const { app } = testGateway(); + const res = await rpc(app, '/getFleetStateHistory', { start: 'yesterday' }); + expect(res.statusCode).toBe(400); + }); +}); diff --git a/packages/gateway/src/routes/fleet.ts b/packages/gateway/src/routes/fleet.ts new file mode 100644 index 00000000..ec637b2b --- /dev/null +++ b/packages/gateway/src/routes/fleet.ts @@ -0,0 +1,105 @@ +import { + bucketLast, + resolveBucketSeconds, + resolveWindow, +} from '@dormice/server/history'; +import { + getFleetMetricsRequestSchema, + getFleetMetricsResponseSchema, + getFleetStateHistoryRequestSchema, + getFleetStateHistoryResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import type { Db } from '../db/db'; +import { queryFleetPeak, queryFleetSamples } from '../db/fleet-samples'; +import { downReason, type Fleet, sumReadings } from '../fleet'; + +export interface FleetRoutesOptions { + db: Db; + fleet: Fleet; +} + +/** + * The fleet's own observation: the figures that add up, answered from + * what the gateway already holds and never by asking a node (design + * record #24 — the console polls these every few seconds, and a poll + * that fanned out would make the fleet's one observer its heaviest + * caller). getFleetMetrics is the present, from every node's last + * reading; getFleetStateHistory is the past, from the samples the + * check-ins wrote (db/fleet-samples.ts). Behind the sandbox gate, as + * observation is on a node. + */ +export const fleetRoutes: FastifyPluginAsyncZod = async ( + app, + { db, fleet }, +) => { + app.post( + '/getFleetMetrics', + { + schema: { + body: getFleetMetricsRequestSchema, + response: { 200: getFleetMetricsResponseSchema }, + }, + }, + async () => { + const now = new Date(); + const nodes = fleet.all(); + const { reported, sandboxes, sandboxDisks } = sumReadings(nodes); + return { + nodes: { + total: nodes.length, + reachable: nodes.filter((node) => downReason(node, now) === null) + .length, + reported, + }, + sandboxes, + sandboxDisks, + }; + }, + ); + + // The fleet's past: state counts per check-in, sliced and (past 360 + // points) bucketed. Buckets carry whole raw samples — the last one in + // the bucket — so byState always sums to total; the concurrency peak is + // computed from raw rows and travels beside the points, immune to + // bucketing. A window the gateway was down for has no rows: the gap IS + // the answer. + app.post( + '/getFleetStateHistory', + { + schema: { + body: getFleetStateHistoryRequestSchema, + response: { 200: getFleetStateHistoryResponseSchema }, + }, + }, + async (request) => { + const { startIso, endIso, startMs, endMs } = resolveWindow( + request.body.start, + request.body.end, + 24 * 3600_000, + new Date(), + ); + const rows = queryFleetSamples(db, startIso, endIso); + const bucketSeconds = resolveBucketSeconds(rows.length, startMs, endMs); + const points = + bucketSeconds === null + ? rows + : bucketLast(rows, startMs, bucketSeconds); + return { + points: points.map((row) => ({ + at: row.at, + byState: { + active: row.active, + frozen: row.frozen, + stopped: row.stopped, + archived: row.archived, + restoring: row.restoring, + }, + total: row.total, + })), + bucketSeconds, + peak: queryFleetPeak(db, startIso, endIso), + }; + }, + ); +}; diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index c11cac37..47baed59 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -12,6 +12,7 @@ import { import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import type { NameCache } from '../cache'; import type { Db } from '../db/db'; +import { recordFleetSample } from '../db/fleet-samples'; import { readNodeConfig } from '../db/node-config'; import { readConfigVersion } from '../db/settings'; import { @@ -87,6 +88,9 @@ export const checkInRoutes: FastifyPluginAsyncZod< throw refusal(409, outcome.refused); } const { node, joined, movedFrom } = outcome; + // The fleet's state, sampled now that this node's reading is in + // (db/fleet-samples.ts has when a sample is not written). + recordFleetSample(db, fleet, new Date()); if (joined) { request.log.info( { nodeId: node.id, endpoint: node.endpoint }, diff --git a/packages/gateway/src/testing.ts b/packages/gateway/src/testing.ts index 2d663faa..7ad92b0a 100644 --- a/packages/gateway/src/testing.ts +++ b/packages/gateway/src/testing.ts @@ -107,6 +107,8 @@ export function testGateway( ingress?: Ingress; /** Forged by default: the suites here are about the settings machinery, not S3's availability. */ probeS3?: NonNullable[0]['probeS3']>; + /** When this gateway "started" — what the startup grace is judged against (fleet.ts STARTUP_GRACE_MS). */ + startedAt?: Date; } = {}, ) { const db = openDb(':memory:'); @@ -118,7 +120,7 @@ export function testGateway( }; const config = loadConfig(rawEnv); ensureSettings(db, config); - const fleet = new Fleet(db); + const fleet = new Fleet(db, opts.startedAt); // Under the fleet token the config carries: a suite that embeds a real // node beside this gateway (the SDK's) gives both the same token, and // the lookups must present it, not the scaffolding's default. diff --git a/packages/server/src/app.test.ts b/packages/server/src/app.test.ts index ee56c6f8..d299283e 100644 --- a/packages/server/src/app.test.ts +++ b/packages/server/src/app.test.ts @@ -248,6 +248,20 @@ describe('error shape', () => { expect(res.statusCode).toBe(404); expect(Object.keys(res.json())).toEqual(['message']); }); + + it('a verb that answers at the gateway alone is a 404 naming the gateway; so is the console', async () => { + const { app } = testApp(); + for (const verb of ['listApiKeys', 'getConfig', 'getFleetMetrics']) { + const res = await rpc(app, `/${verb}`); + expect(res.statusCode).toBe(404); + expect(res.json().message).toBe( + `/${verb} answers at the gateway (http://127.0.0.1:3677), not on a node`, + ); + } + const page = await app.inject({ method: 'GET', url: '/console/' }); + expect(page.statusCode).toBe(404); + expect(page.json().message).toMatch(/^\/console\/ answers at the gateway/); + }); }); describe('concurrent acquires', () => { diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 96909044..234e971d 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -1,5 +1,6 @@ import http from 'node:http'; import nodePath from 'node:path'; +import { GATEWAY_ONLY_VERBS } from '@dormice/shared'; import fastify, { type FastifyError, type FastifyServerFactory } from 'fastify'; import { serializerCompiler, @@ -162,10 +163,23 @@ export function buildApp({ } reply.code(status).send({ message: error.message }); }); + // A verb that answers at the gateway alone, asked of a node: the 404 + // names the door. The emergency path is ssh to a node and curl its + // loopback with the fleet token, and an operator on it asking for the + // keys or the settings should be sent to where they live, not left + // with a plain "not found" (shared GATEWAY_ONLY_VERBS; the console too, + // which a node has not served since the second cut). app.setNotFoundHandler((request, reply) => { - reply - .code(404) - .send({ message: `route ${request.method} ${request.url} not found` }); + const path = request.url.split('?')[0] ?? request.url; + const atGateway = + (GATEWAY_ONLY_VERBS as readonly string[]).includes(path.slice(1)) || + path === '/console' || + path.startsWith('/console/'); + reply.code(404).send({ + message: atGateway + ? `${path} answers at the gateway (${config.DORMICE_GATEWAY_ENDPOINT}), not on a node` + : `route ${request.method} ${request.url} not found`, + }); }); // Liveness probe: open by design (probes have no secrets), everything diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index f40ca14d..decaad37 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -45,7 +45,7 @@ const envSchema = z.object({ .default(30), /** * How long per-sandbox samples live (fleet rows are fixed at 30 days — - * FLEET_SNAPSHOT_KEEP_DAYS). A knob because volume scales with the fleet: + * HOST_SAMPLE_KEEP_DAYS). A knob because volume scales with the fleet: * at the 30s default, a worst-case 100 always-hot sandboxes over the * default 7 days is ~2M rows — fine for SQLite, but the operator of such * a box may want to trade history depth for disk. diff --git a/packages/server/src/db/metrics.ts b/packages/server/src/db/metrics.ts index 6480c734..4b20f43e 100644 --- a/packages/server/src/db/metrics.ts +++ b/packages/server/src/db/metrics.ts @@ -14,8 +14,6 @@ import { bucketIndex } from '../history'; import type { HostSample } from '../host-metrics'; import type { Db } from './db'; import { - type FleetSnapshotRow, - fleetSnapshots, type HostMetricsSampleRow, hostMetricsSamples, type SandboxMetricsSampleRow, @@ -24,38 +22,31 @@ import { } from './schema'; /** - * How long fleet snapshots and host samples live. Not a knob: the - * dashboard's widest range (30 days) defines the need, and at one small row - * per tick each table stays a few megabytes — nobody sizes an explanation - * window (ACTIVITY_KEEP's reasoning). Per-sandbox samples DO get a knob + * How long host samples live. Not a knob: the dashboard's widest range + * (30 days) defines the need, and at one small row per tick the table + * stays a few megabytes — nobody sizes an explanation window + * (ACTIVITY_KEEP's reasoning). Per-sandbox samples DO get a knob * (DORMICE_METRICS_RETENTION_HOURS): their volume scales with fleet size. + * (The fleet's state history lives at the gateway since the third cut, + * under the same 30 days.) */ -export const FLEET_SNAPSHOT_KEEP_DAYS = 30; - -export interface FleetCounts { - active: number; - frozen: number; - stopped: number; - archived: number; - restoring: number; - total: number; -} +export const HOST_SAMPLE_KEEP_DAYS = 30; export interface MetricsTickInput { /** ISO 8601 UTC — the tick's single timestamp, shared by every row. */ at: string; - fleetCounts: FleetCounts; host: HostSample; samples: Array<{ sandboxId: string; metrics: SandboxMetrics }>; retentionHours: number; } /** - * Writes one sampler tick — fleet snapshot, per-sandbox samples, and both - * retention prunes — in a single synchronous transaction: a tick's data is - * either fully visible or not at all. Sampling (async, ~1s per container) - * happened before this call; better-sqlite3 transactions cannot span an - * await, so "collect first, write once" is not a choice but a law. + * Writes one sampler tick — the host sample, per-sandbox samples, and + * both retention prunes — in a single synchronous transaction: a tick's + * data is either fully visible or not at all. Sampling (async, ~1s per + * container) happened before this call; better-sqlite3 transactions + * cannot span an await, so "collect first, write once" is not a choice + * but a law. * * Samples are re-filtered against the ledger inside the transaction: a * sandbox destroyed while its reading was in flight must not leave orphan @@ -65,15 +56,11 @@ export function insertMetricsTick(db: Db, input: MetricsTickInput): void { const sampleCutoff = new Date( Date.parse(input.at) - input.retentionHours * 3600_000, ).toISOString(); - const fleetCutoff = new Date( - Date.parse(input.at) - FLEET_SNAPSHOT_KEEP_DAYS * 86_400_000, + const hostCutoff = new Date( + Date.parse(input.at) - HOST_SAMPLE_KEEP_DAYS * 86_400_000, ).toISOString(); db.transaction((tx) => { - tx.insert(fleetSnapshots) - .values({ at: input.at, ...input.fleetCounts }) - .run(); - tx.insert(hostMetricsSamples) .values({ at: input.at, ...input.host }) .run(); @@ -103,16 +90,15 @@ export function insertMetricsTick(db: Db, input: MetricsTickInput): void { tx.delete(sandboxMetricsSamples) .where(lt(sandboxMetricsSamples.at, sampleCutoff)) .run(); - tx.delete(fleetSnapshots).where(lt(fleetSnapshots.at, fleetCutoff)).run(); tx.delete(hostMetricsSamples) - .where(lt(hostMetricsSamples.at, fleetCutoff)) + .where(lt(hostMetricsSamples.at, hostCutoff)) .run(); }); } /** * The destroy cascade: a sandbox whose disk is gone has no owner for its - * history. Fleet snapshots are untouched — they belong to no sandbox. + * history. Host samples are untouched — they belong to no sandbox. */ export function deleteSandboxMetricsSamples(db: Db, sandboxId: string): void { db.delete(sandboxMetricsSamples) @@ -191,43 +177,6 @@ export function queryHostCpuPeak( : null; } -/** Ascending slice of fleet snapshots. */ -export function queryFleetSnapshots( - db: Db, - startIso: string, - endIso: string, -): FleetSnapshotRow[] { - return db - .select() - .from(fleetSnapshots) - .where( - and(gte(fleetSnapshots.at, startIso), lte(fleetSnapshots.at, endIso)), - ) - .orderBy(asc(fleetSnapshots.at)) - .all(); -} - -/** - * The window's concurrency peak, from raw rows so no bucketing can flatten - * it: highest active count, and the earliest instant it was observed. - */ -export function queryFleetPeak( - db: Db, - startIso: string, - endIso: string, -): { active: number; at: string } | null { - const row = db - .select({ active: fleetSnapshots.active, at: fleetSnapshots.at }) - .from(fleetSnapshots) - .where( - and(gte(fleetSnapshots.at, startIso), lte(fleetSnapshots.at, endIso)), - ) - .orderBy(desc(fleetSnapshots.active), asc(fleetSnapshots.at)) - .limit(1) - .get(); - return row ?? null; -} - /** * Buckets per-sandbox samples by per-field max: someone reading history is * hunting for spikes, and averaging erases exactly what they came for. Each diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index e4833ab5..23b4a43c 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -116,11 +116,12 @@ export type TemplateRow = typeof templates.$inferSelect; * compatibility contract — the E2B metrics endpoint slices by start/end — * and because nothing outside the ledger can measure per-sandbox. * - * Three tables, not one: this, fleet_snapshots and host_metrics_samples - * differ in unit of meaning (one sandbox's resources vs the fleet's state - * counts vs the machine's own resources), retention - * (DORMICE_METRICS_RETENTION_HOURS vs a fixed 30 days for the other two) - * and deletion path (destroy cascades here, never there). + * Two tables, not one: this and host_metrics_samples differ in unit of + * meaning (one sandbox's resources vs the machine's own), retention + * (DORMICE_METRICS_RETENTION_HOURS vs a fixed 30 days) and deletion path + * (destroy cascades here, never there). The fleet's state counts are the + * gateway's table since the third cut (fleet_snapshots below is the + * legacy). * * Keyed by the sandbox's platform id, not its name: rebuild replaces the * shell but keeps the id, so history stays continuous across rebuilds; @@ -159,15 +160,17 @@ export const sandboxMetricsSamples = sqliteTable( export type SandboxMetricsSampleRow = typeof sandboxMetricsSamples.$inferSelect; /** - * Fleet state counts over time, one row per sampler tick: the data behind - * the console's concurrency curve and peak. Owned by no sandbox — destroy - * never touches it — and kept a fixed 30 days (the dashboard's widest - * range defines the need; like ACTIVITY_KEEP, nobody tunes the size of an - * explanation window). - * - * Five explicit state columns instead of a JSON blob: the window peak is - * max(active) in one SQL aggregate, and the stacked chart needs each state - * addressable. `total` is stored redundantly so readers never re-derive it. + * LEGACY, read by nothing on the node since the third cut (2026-09-15): + * the fleet's state counts per sampler tick, as this node sampled them + * while it was a product of its own. The fleet's history is the gateway's + * table now (gateway db/schema.ts fleet_state_samples, summed over every + * node at each check-in), and the sampler writes here no more. The table + * and its rows stay until the fourth cut's import tool has carried a + * production node's last 30 days into the gateway — a single-node fleet's + * history is the same figure — so the console's 30-day curve does not + * break at the cut-over; the DROP ships with that import, beside + * api_keys and console_account. Not a bug to delete early: the import + * reads it. */ export const fleetSnapshots = sqliteTable('fleet_snapshots', { /** ISO 8601 UTC; one row per tick, so time itself is the key. */ @@ -189,8 +192,7 @@ export type FleetSnapshotRow = typeof fleetSnapshots.$inferSelect; * self-hosted single box nobody runs Prometheus, and overcommit-by- * observation — the platform's own capacity story — is impossible without * a peak to look at. Owned by no sandbox (destroy never touches it), kept - * a fixed 30 days like fleet_snapshots and for the same reason: the - * dashboard's widest range defines the need. + * a fixed 30 days: the dashboard's widest range defines the need. * * Nullable columns are honest platform gaps, never zeros: cpu_used_pct is * null on the tick after a daemon start (a delta needs two samples), swap diff --git a/packages/server/src/metrics-sampler.test.ts b/packages/server/src/metrics-sampler.test.ts index 67d477db..c3551381 100644 --- a/packages/server/src/metrics-sampler.test.ts +++ b/packages/server/src/metrics-sampler.test.ts @@ -4,12 +4,8 @@ import { describe, expect, it } from 'vitest'; import { buildApp } from './app'; import { loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; -import { FLEET_SNAPSHOT_KEEP_DAYS, insertMetricsTick } from './db/metrics'; -import { - fleetSnapshots, - hostMetricsSamples, - sandboxMetricsSamples, -} from './db/schema'; +import { HOST_SAMPLE_KEEP_DAYS, insertMetricsTick } from './db/metrics'; +import { hostMetricsSamples, sandboxMetricsSamples } from './db/schema'; import { FakeExecutor } from './executor/fake'; import { CpuSampler, type HostSample } from './host-metrics'; import { KeyedQueue } from './keyed-queue'; @@ -79,16 +75,12 @@ function sampleRows(db: ReturnType['db']) { return db.select().from(sandboxMetricsSamples).all(); } -function fleetRows(db: ReturnType['db']) { - return db.select().from(fleetSnapshots).all(); -} - function hostRows(db: ReturnType['db']) { return db.select().from(hostMetricsSamples).all(); } describe('sampleOnce', () => { - it('writes one fleet row and one sample per measurable sandbox', async () => { + it('writes one host row and one sample per measurable sandbox', async () => { const { app, db, executor } = harness(); await acquire(app, 'hot'); const napping = await acquire(app, 'napping'); @@ -102,18 +94,6 @@ describe('sampleOnce', () => { // Active and frozen are measured; stopped has no container and is not. expect(result).toEqual({ sampled: 2, skipped: 0 }); - const fleet = fleetRows(db); - expect(fleet).toEqual([ - { - at: now.toISOString(), - active: 1, - frozen: 1, - stopped: 1, - archived: 0, - restoring: 0, - total: 3, - }, - ]); const samples = sampleRows(db); expect(samples).toHaveLength(2); for (const row of samples) { @@ -146,12 +126,12 @@ describe('sampleOnce', () => { const result = await sampleOnce(db, executor, now, tickOpts()); expect(result).toEqual({ sampled: 1, skipped: 1 }); expect(sampleRows(db)).toHaveLength(1); - // The fleet row still lands: state counts come from the ledger, not - // from what happened to be measurable. - expect(fleetRows(db)).toHaveLength(1); + // The host row still lands: the machine's reading does not depend on + // what happened to be measurable. + expect(hostRows(db)).toHaveLength(1); }); - it('prunes samples past retention and fleet rows past 30 days', async () => { + it('prunes samples past retention and host rows past 30 days', async () => { const { app, db, executor } = harness(); await acquire(app, 'steady'); @@ -159,25 +139,20 @@ describe('sampleOnce', () => { await sampleOnce(db, executor, early, tickOpts()); // One retention window plus a minute later: the early sample must fall, - // the early fleet row (well within 30 days) must survive. + // the early host row (well within 30 days) must survive. const later = new Date(early.getTime() + 168 * 3600_000 + 60_000); await sampleOnce(db, executor, later, tickOpts()); expect(sampleRows(db).map((r) => r.at)).toEqual([later.toISOString()]); - expect(fleetRows(db).map((r) => r.at)).toEqual([ + expect(hostRows(db).map((r) => r.at)).toEqual([ early.toISOString(), later.toISOString(), ]); - // Past the fleet's own 30-day window the early fleet row falls too, - // and the host samples share that window exactly. + // Past the host table's own 30-day window the early row falls too. const ancientCutoff = new Date( - early.getTime() + (FLEET_SNAPSHOT_KEEP_DAYS * 24 + 1) * 3600_000, + early.getTime() + (HOST_SAMPLE_KEEP_DAYS * 24 + 1) * 3600_000, ); await sampleOnce(db, executor, ancientCutoff, tickOpts()); - expect(fleetRows(db).map((r) => r.at)).toEqual([ - later.toISOString(), - ancientCutoff.toISOString(), - ]); expect(hostRows(db).map((r) => r.at)).toEqual([ later.toISOString(), ancientCutoff.toISOString(), @@ -207,8 +182,8 @@ describe('sampleOnce', () => { const remaining = sampleRows(db); expect(remaining).toHaveLength(1); expect(remaining[0]?.sandboxId).not.toBe(victim.id); - // Fleet snapshots belong to no sandbox: untouched. - expect(fleetRows(db)).toHaveLength(1); + // Host samples belong to no sandbox: untouched. + expect(hostRows(db)).toHaveLength(1); }); it('drops a sample whose sandbox was destroyed mid-read (no orphan rows)', () => { @@ -218,14 +193,6 @@ describe('sampleOnce', () => { insertMetricsTick(db, { at: '2026-07-15T10:00:00.000Z', host: HOST, - fleetCounts: { - active: 0, - frozen: 0, - stopped: 0, - archived: 0, - restoring: 0, - total: 0, - }, samples: [ { sandboxId: 'ghost', @@ -249,6 +216,6 @@ describe('sampleOnce', () => { .from(sandboxMetricsSamples) .get() as { n: number }; expect(total.n).toBe(0); - expect(fleetRows(db)).toHaveLength(1); + expect(hostRows(db)).toHaveLength(1); }); }); diff --git a/packages/server/src/metrics-sampler.ts b/packages/server/src/metrics-sampler.ts index 6db5e232..05b266dd 100644 --- a/packages/server/src/metrics-sampler.ts +++ b/packages/server/src/metrics-sampler.ts @@ -1,13 +1,16 @@ import type { Db } from './db/db'; -import { countByState, listSandboxes } from './db/ledger'; +import { listSandboxes } from './db/ledger'; import { insertMetricsTick } from './db/metrics'; import type { Executor, SandboxMetrics } from './executor/executor'; import { type CpuSampler, readHostSample } from './host-metrics'; /** - * The metrics sampler: one tick reads every measurable sandbox, the - * fleet's state census and the host machine itself, and persists all - * three. This is the daemon keeping history — a reversal of the original + * The metrics sampler: one tick reads every measurable sandbox and the + * host machine itself, and persists both. (The fleet's state census went + * to the gateway with the third cut, 2026-09-15: it is summed there over + * every node at each check-in — the one figure no single node can + * compute — and the node keeps no fleet history of its own.) This is the + * daemon keeping history — a reversal of the original * "observation window, not a monitoring system" stance, overturned for * three reasons (2026-07-15): the E2B metrics endpoint's start/end slice * is a compatibility contract we were answering with a single sample; @@ -58,7 +61,6 @@ export async function sampleOnce( }, ): Promise { const rows = listSandboxes(db); - const { byState, total } = countByState(rows); const measurable = rows.filter( (row) => row.state === 'active' || row.state === 'frozen', ); @@ -82,7 +84,6 @@ export async function sampleOnce( const host = await readHostSample(opts.hostCpu, opts.dataDir); insertMetricsTick(db, { at: now.toISOString(), - fleetCounts: { ...byState, total }, host, samples, retentionHours: opts.retentionHours, diff --git a/packages/server/src/routes/host.ts b/packages/server/src/routes/host.ts index 2edffdc4..d6b100d4 100644 --- a/packages/server/src/routes/host.ts +++ b/packages/server/src/routes/host.ts @@ -1,6 +1,4 @@ import { - getFleetStateHistoryRequestSchema, - getFleetStateHistoryResponseSchema, getHostMetricsHistoryRequestSchema, getHostMetricsHistoryResponseSchema, hostMetricsResponseSchema, @@ -11,13 +9,11 @@ import type { Db } from '../db/db'; import { countByState, listSandboxes } from '../db/ledger'; import { bucketHostSamples, - queryFleetPeak, - queryFleetSnapshots, queryHostCpuPeak, queryHostSamples, } from '../db/metrics'; import type { Executor } from '../executor/executor'; -import { bucketLast, resolveBucketSeconds, resolveWindow } from '../history'; +import { resolveBucketSeconds, resolveWindow } from '../history'; import { CpuSampler, readHostReading } from '../host-metrics'; export interface HostRoutesOptions { @@ -123,50 +119,4 @@ export const hostRoutes: FastifyPluginAsyncZod = async ( }; }, ); - - // The fleet's past: state counts per sampler tick, sliced and (past 360 - // points) bucketed. Buckets carry whole raw snapshots — the last one in - // the bucket — so byState always sums to total; the concurrency peak is - // computed from raw rows and travels beside the points, immune to - // bucketing. A window the daemon slept through simply has no rows: the - // gap IS the answer. (The gateway answers this verb for the fleet from - // its own samples; this node-local answer leaves with the third cut.) - app.post( - '/getFleetStateHistory', - { - schema: { - body: getFleetStateHistoryRequestSchema, - response: { 200: getFleetStateHistoryResponseSchema }, - }, - }, - async (request) => { - const { startIso, endIso, startMs, endMs } = resolveWindow( - request.body.start, - request.body.end, - 24 * 3600_000, - new Date(), - ); - const rows = queryFleetSnapshots(db, startIso, endIso); - const bucketSeconds = resolveBucketSeconds(rows.length, startMs, endMs); - const points = - bucketSeconds === null - ? rows - : bucketLast(rows, startMs, bucketSeconds); - return { - points: points.map((row) => ({ - at: row.at, - byState: { - active: row.active, - frozen: row.frozen, - stopped: row.stopped, - archived: row.archived, - restoring: row.restoring, - }, - total: row.total, - })), - bucketSeconds, - peak: queryFleetPeak(db, startIso, endIso), - }; - }, - ); }; diff --git a/packages/server/src/routes/observability.test.ts b/packages/server/src/routes/observability.test.ts index d33a345f..939cd504 100644 --- a/packages/server/src/routes/observability.test.ts +++ b/packages/server/src/routes/observability.test.ts @@ -1,6 +1,5 @@ import { fileURLToPath } from 'node:url'; import { - getFleetStateHistoryResponseSchema, getHostMetricsHistoryResponseSchema, getSandboxMetricsHistoryResponseSchema, getSandboxMetricsResponseSchema, @@ -196,14 +195,6 @@ describe('getSandboxMetricsHistory', () => { insertMetricsTick(db, { at: new Date(t0 + i * 30_000).toISOString(), host: HOST, - fleetCounts: { - active: 1, - frozen: 0, - stopped: 0, - archived: 0, - restoring: 0, - total: 1, - }, // One reading spikes; every neighbor idles. Averaging would bury it. samples: [ { @@ -230,103 +221,6 @@ describe('getSandboxMetricsHistory', () => { }); }); -describe('getFleetStateHistory', () => { - it('answers an empty window with no points and a null peak', async () => { - const { app } = testApp(); - const res = await rpc(app, '/getFleetStateHistory', {}); - expect(res.statusCode).toBe(200); - const body = getFleetStateHistoryResponseSchema.parse(res.json()); - expect(body).toEqual({ points: [], bucketSeconds: null, peak: null }); - }); - - it('returns snapshots ascending with byState summing to total', async () => { - const { app, db, executor } = testApp(); - await rpc(app, '/acquireSandbox', { name: 'one' }); - const t0 = Date.parse('2026-07-15T10:00:00.000Z'); - await sampleOnce(db, executor, new Date(t0), tickOpts()); - await rpc(app, '/acquireSandbox', { name: 'two' }); - await sampleOnce(db, executor, new Date(t0 + 30_000), tickOpts()); - - const res = await rpc(app, '/getFleetStateHistory', { - start: new Date(t0 - 1000).toISOString(), - end: new Date(t0 + 60_000).toISOString(), - }); - const { points, bucketSeconds, peak } = - getFleetStateHistoryResponseSchema.parse(res.json()); - expect(bucketSeconds).toBe(null); - expect(points.map((p) => p.at)).toEqual([ - new Date(t0).toISOString(), - new Date(t0 + 30_000).toISOString(), - ]); - for (const point of points) { - const sum = Object.values(point.byState).reduce((a, b) => a + b, 0); - expect(sum).toBe(point.total); - } - expect(peak).toEqual({ - active: 2, - at: new Date(t0 + 30_000).toISOString(), - }); - }); - - it('computes the peak from raw rows — bucketing cannot flatten it', async () => { - const { app, db } = testApp(); - const t0 = Date.parse('2026-07-15T00:00:00.000Z'); - const counts = (active: number) => ({ - active, - frozen: 0, - stopped: 0, - archived: 0, - restoring: 0, - total: active, - }); - const rows = MAX_POINTS + 40; - for (let i = 0; i < rows; i += 1) { - insertMetricsTick(db, { - at: new Date(t0 + i * 30_000).toISOString(), - host: HOST, - fleetCounts: counts(1), - samples: [], - retentionHours: 168, - }); - } - // A spike squeezed between two grid rows of its own bucket: the bucket - // keeps its LAST whole snapshot, so no point ever shows 9 — the peak - // field is the only honest carrier. - insertMetricsTick(db, { - at: new Date(t0 + 200 * 30_000 + 1000).toISOString(), - host: HOST, - fleetCounts: counts(9), - samples: [], - retentionHours: 168, - }); - insertMetricsTick(db, { - at: new Date(t0 + 200 * 30_000 + 2000).toISOString(), - host: HOST, - fleetCounts: counts(1), - samples: [], - retentionHours: 168, - }); - - const res = await rpc(app, '/getFleetStateHistory', { - start: new Date(t0).toISOString(), - end: new Date(t0 + rows * 30_000).toISOString(), - }); - const { points, bucketSeconds, peak } = - getFleetStateHistoryResponseSchema.parse(res.json()); - expect(bucketSeconds).not.toBe(null); - expect(points.length).toBeLessThanOrEqual(MAX_POINTS); - expect(peak).toEqual({ - active: 9, - at: new Date(t0 + 200 * 30_000 + 1000).toISOString(), - }); - // Whole-snapshot buckets: sums still hold after bucketing. - for (const point of points) { - const sum = Object.values(point.byState).reduce((a, b) => a + b, 0); - expect(sum).toBe(point.total); - } - }); -}); - describe('getHostMetricsHistory', () => { it('answers an empty window with no points and a null peak', async () => { const { app } = testApp(); @@ -381,14 +275,6 @@ describe('getHostMetricsHistory', () => { at: new Date(t0 + i * 30_000).toISOString(), // One reading spikes; every neighbor idles. Averaging would bury it. host: hostReading(i === 200 ? 95 : 5), - fleetCounts: { - active: 0, - frozen: 0, - stopped: 0, - archived: 0, - restoring: 0, - total: 0, - }, samples: [], retentionHours: 168, }); @@ -416,25 +302,15 @@ describe('getHostMetricsHistory', () => { it('a null-CPU tick never competes for the peak', async () => { const { app, db } = testApp(); const t0 = Date.parse('2026-07-15T10:00:00.000Z'); - const counts = { - active: 0, - frozen: 0, - stopped: 0, - archived: 0, - restoring: 0, - total: 0, - }; insertMetricsTick(db, { at: new Date(t0).toISOString(), host: hostReading(null), - fleetCounts: counts, samples: [], retentionHours: 168, }); insertMetricsTick(db, { at: new Date(t0 + 30_000).toISOString(), host: hostReading(40), - fleetCounts: counts, samples: [], retentionHours: 168, }); From 4b63ac53540db300d6a11f16f77967198386faa7 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 01:00:37 +0800 Subject: [PATCH 49/89] The console reads the fleet at the gateway: the overview from the fleet's own verbs, a nodes page with each machine, the sandbox list saying which node it lacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The overview's stat cards and chart read getFleetMetrics and getFleetStateHistory — the gateway's sums and samples, no fan-out per poll. The host health card, which had no subject in a fleet ("this machine" is one of several), becomes the nodes card: one row per node with its CPU, memory and data-disk levels and its running count, from the readings the gateway already holds, and a footnote when a node has not reported and the figures above are a lower bound. A nodes page under Ops, first: every node's reachability, build (marked when it differs from the gateway's), the three levels, its census, and its configuration version against the gateway's current one (in sync, behind by N, or not open yet). The row menu sets the one per-node knob (extra swap, restored here from the settings page it left with the second cut) and removes a node — only an unreachable one; the gateway refuses the rest with its reason, relayed. A node's own page carries the moved health card with its trends and CPU peak, by nodeId, beside its details. Swap usage stays off the table: nine columns is the width the layout has. The sandbox list shows a banner naming the nodes its answer lacks. Messages: a nodes domain in ten locales, the host card's keys moved with the card, check:messages aligned. --- packages/console/messages/de/nodes.json | 75 ++++ packages/console/messages/de/overview.json | 21 +- packages/console/messages/de/sandboxes.json | 3 +- packages/console/messages/de/shell.json | 1 + packages/console/messages/en/nodes.json | 75 ++++ packages/console/messages/en/overview.json | 21 +- packages/console/messages/en/sandboxes.json | 3 +- packages/console/messages/en/shell.json | 1 + packages/console/messages/es/nodes.json | 75 ++++ packages/console/messages/es/overview.json | 21 +- packages/console/messages/es/sandboxes.json | 3 +- packages/console/messages/es/shell.json | 1 + packages/console/messages/fr/nodes.json | 75 ++++ packages/console/messages/fr/overview.json | 21 +- packages/console/messages/fr/sandboxes.json | 3 +- packages/console/messages/fr/shell.json | 1 + packages/console/messages/ja/nodes.json | 75 ++++ packages/console/messages/ja/overview.json | 21 +- packages/console/messages/ja/sandboxes.json | 3 +- packages/console/messages/ja/shell.json | 1 + packages/console/messages/ko/nodes.json | 75 ++++ packages/console/messages/ko/overview.json | 21 +- packages/console/messages/ko/sandboxes.json | 3 +- packages/console/messages/ko/shell.json | 1 + packages/console/messages/pt-BR/nodes.json | 75 ++++ packages/console/messages/pt-BR/overview.json | 21 +- .../console/messages/pt-BR/sandboxes.json | 3 +- packages/console/messages/pt-BR/shell.json | 1 + packages/console/messages/ru/nodes.json | 75 ++++ packages/console/messages/ru/overview.json | 21 +- packages/console/messages/ru/sandboxes.json | 3 +- packages/console/messages/ru/shell.json | 1 + packages/console/messages/zh-CN/nodes.json | 75 ++++ packages/console/messages/zh-CN/overview.json | 21 +- .../console/messages/zh-CN/sandboxes.json | 3 +- packages/console/messages/zh-CN/shell.json | 1 + packages/console/messages/zh-TW/nodes.json | 75 ++++ packages/console/messages/zh-TW/overview.json | 21 +- .../console/messages/zh-TW/sandboxes.json | 3 +- packages/console/messages/zh-TW/shell.json | 1 + packages/console/project.inlang/settings.json | 1 + packages/console/src/components/nav.ts | 3 + .../features/nodes/components/NodeBadges.tsx | 71 ++++ .../components/NodeHealthCard.tsx} | 63 +-- .../nodes/components/RemoveNodeDialog.tsx | 59 +++ .../features/nodes/components/SwapDialog.tsx | 118 ++++++ .../src/features/nodes/hooks/useNodes.ts | 114 ++++++ .../features/nodes/pages/NodeDetailPage.tsx | 150 +++++++ .../src/features/nodes/pages/NodesPage.tsx | 371 ++++++++++++++++++ .../overview/components/FleetStatCards.tsx | 11 +- .../overview/components/NodesCard.tsx | 167 ++++++++ .../overview/components/SandboxDisksCard.tsx | 7 +- .../overview/hooks/useFleetMetrics.ts | 17 + .../features/overview/hooks/useHostMetrics.ts | 14 - .../overview/hooks/useHostTimeline.ts | 25 -- .../features/overview/pages/OverviewPage.tsx | 12 +- .../features/sandboxes/hooks/useSandboxes.ts | 4 +- .../sandboxes/pages/SandboxesPage.tsx | 20 +- packages/console/src/lib/api.ts | 64 ++- packages/console/src/routeTree.gen.ts | 42 ++ .../console/src/routes/_app/nodes/$id.tsx | 6 + .../console/src/routes/_app/nodes/index.tsx | 6 + packages/console/vite.config.ts | 1 + 63 files changed, 2104 insertions(+), 242 deletions(-) create mode 100644 packages/console/messages/de/nodes.json create mode 100644 packages/console/messages/en/nodes.json create mode 100644 packages/console/messages/es/nodes.json create mode 100644 packages/console/messages/fr/nodes.json create mode 100644 packages/console/messages/ja/nodes.json create mode 100644 packages/console/messages/ko/nodes.json create mode 100644 packages/console/messages/pt-BR/nodes.json create mode 100644 packages/console/messages/ru/nodes.json create mode 100644 packages/console/messages/zh-CN/nodes.json create mode 100644 packages/console/messages/zh-TW/nodes.json create mode 100644 packages/console/src/features/nodes/components/NodeBadges.tsx rename packages/console/src/features/{overview/components/HostHealthCard.tsx => nodes/components/NodeHealthCard.tsx} (75%) create mode 100644 packages/console/src/features/nodes/components/RemoveNodeDialog.tsx create mode 100644 packages/console/src/features/nodes/components/SwapDialog.tsx create mode 100644 packages/console/src/features/nodes/hooks/useNodes.ts create mode 100644 packages/console/src/features/nodes/pages/NodeDetailPage.tsx create mode 100644 packages/console/src/features/nodes/pages/NodesPage.tsx create mode 100644 packages/console/src/features/overview/components/NodesCard.tsx create mode 100644 packages/console/src/features/overview/hooks/useFleetMetrics.ts delete mode 100644 packages/console/src/features/overview/hooks/useHostMetrics.ts delete mode 100644 packages/console/src/features/overview/hooks/useHostTimeline.ts create mode 100644 packages/console/src/routes/_app/nodes/$id.tsx create mode 100644 packages/console/src/routes/_app/nodes/index.tsx diff --git a/packages/console/messages/de/nodes.json b/packages/console/messages/de/nodes.json new file mode 100644 index 00000000..2158163e --- /dev/null +++ b/packages/console/messages/de/nodes.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://inlang.com/schema/inlang-message-format", + "nodes_page_title": "Knoten", + "nodes_gateway_title": "Gateway", + "nodes_gateway_desc": "Die eine Tür der Flotte; ein Knoten tritt beim ersten Check-in bei", + "nodes_gateway_build": "Build {commit}", + "nodes_gateway_build_unknown": "Build-Identität unbekannt", + "nodes_gateway_fleet": "{total} Knoten · {reachable} erreichbar", + "nodes_gateway_config": "Konfiguration v{version}", + "nodes_col_node": "Knoten", + "nodes_col_status": "Status", + "nodes_col_build": "Build", + "nodes_col_cpu": "CPU", + "nodes_col_memory": "Speicher", + "nodes_col_disk": "Datenlaufwerk", + "nodes_col_sandboxes": "Sandboxes", + "nodes_col_config": "Konfig", + "nodes_col_actions": "Aktionen", + "nodes_reachable": "Erreichbar", + "nodes_unreachable": "Nicht erreichbar", + "nodes_last_check_in": "Letzter Check-in {ago}", + "nodes_never_checked_in": "Kein Check-in seit dem Start des Gateways", + "nodes_build_differs": "Weicht vom Gateway ab", + "nodes_no_reading": "Noch keine Messung", + "nodes_sandboxes_cell": "{active} laufend · {frozen} eingefroren · {total} gesamt", + "nodes_config_synced": "Synchron", + "nodes_config_behind": "{n} zurück", + "nodes_config_none": "Noch nicht offen", + "nodes_config_none_hint": "Noch keine Konfigurationskopie; ein Knoten lauscht erst, wenn er seine erste hat", + "nodes_swap_managed": "Verwalteter Swap {active} GiB", + "nodes_swap_target": "Ziel {target} GiB", + "nodes_swap_reconciling": "Wird angewendet", + "nodes_swap_unsupported": "Dieser Knoten kann keinen Swap verwalten", + "nodes_row_actions_aria": "Aktionen für {id}", + "nodes_menu_swap": "Zusätzlichen Swap setzen", + "nodes_menu_remove": "Knoten entfernen", + "nodes_menu_remove_hint": "Nur ein nicht erreichbarer Knoten kann entfernt werden", + "nodes_swap_dialog_title": "Zusätzlicher Swap: {id}", + "nodes_swap_dialog_desc": "Wie viele GiB Swap der Daemon des Knotens auf seinem eigenen Datenlaufwerk verwaltet, zusätzlich zum Swap des Hosts. Vergrößern wird beim nächsten Check-in eingehängt; Verkleinern wartet auf den nächsten Neustart dieses Hosts, ein aktiver Swap-Block wird nie ausgehängt.", + "nodes_swap_field": "Zusätzlicher Swap (GiB)", + "nodes_swap_field_hint": "0 = keiner.", + "nodes_swap_saved": "Swap-Ziel für {id} auf {gb} GiB gesetzt, wird beim nächsten Check-in angewendet", + "nodes_remove_title": "Knoten „{id}“ entfernen?", + "nodes_remove_desc": "Das ist die Erklärung, dass er nie zurückkommt: sein Eintrag wird gelöscht, seine Sandboxes werden nicht mehr gesucht, und ein Name, der nur dort lebte, wird wieder ein neuer Name. Ein versehentlich entfernter Knoten kommt beim nächsten Check-in zurück.", + "nodes_remove_success": "Knoten „{id}“ entfernt", + "nodes_remove_absent": "„{id}“ war gar nicht da", + "nodes_empty_title": "Noch kein Knoten hat sich gemeldet", + "nodes_empty_description": "Ein Knoten tritt der Flotte beim ersten Check-in bei. Installiere den Daemon, richte DORMICE_GATEWAY_ENDPOINT auf dieses Gateway, und er erscheint hier innerhalb von 15 Sekunden.", + "nodes_loading": "Knoten werden geladen", + "nodes_detail_info_title": "Details", + "nodes_detail_endpoint": "Endpunkt", + "nodes_detail_added_at": "Beigetreten", + "nodes_detail_interval": "Check-in-Intervall", + "nodes_detail_interval_value": "Alle {n} s", + "nodes_detail_build": "Build", + "nodes_detail_config_version": "Konfigurationsversion", + "nodes_detail_swap_target": "Swap-Ziel", + "nodes_detail_swap_active": "Swap eingehängt", + "nodes_detail_not_found_title": "Kein Knoten mit der ID „{id}“", + "nodes_detail_not_found_desc": "Er wurde vielleicht entfernt oder hat sich noch nicht gemeldet.", + "nodes_detail_back": "Zurück zu den Knoten", + "nodes_host_title": "Host-Status", + "nodes_host_desc": "Ressourcenpegel und Verlauf dieser Maschine", + "nodes_host_cpu_cores": "{n} Kerne", + "nodes_host_cpu_peak": "{n} Kerne · Fensterspitze {pct}% ({ago})", + "nodes_host_mem_label": "Speicher", + "nodes_host_mem_hint": "Gesamt {total} · {available} frei", + "nodes_host_swap_unreadable": "Swap ist auf dieser Plattform nicht lesbar", + "nodes_host_swap_unconfigured": "Nicht konfiguriert", + "nodes_host_swap_warning": "Einfrieren braucht Swap — siehe dor doctor", + "nodes_host_swap_hint": "Gesamt {total} · hier wohnen die eingefrorenen Sandboxes", + "nodes_host_disk_label": "Datenlaufwerk", + "nodes_host_disk_missing": "Dieser Host hat kein Datenverzeichnis", + "nodes_host_disk_hint": "Gesamt {total} · {available} übrig" +} diff --git a/packages/console/messages/de/overview.json b/packages/console/messages/de/overview.json index 047ae4d7..c3a01b6a 100644 --- a/packages/console/messages/de/overview.json +++ b/packages/console/messages/de/overview.json @@ -22,22 +22,15 @@ "overview_fleet_chart_desc": "Sandbox-Anzahl je Zustand im Zeitverlauf — fällt Aktiv, während Eingefroren steigt, sehen Sie „Leerlauf kostet nichts“ bei der Arbeit.", "overview_fleet_chart_empty_title": "Noch kein Verlauf in diesem Fenster", "overview_fleet_chart_empty_desc": "Der Daemon schreibt alle 30 Sekunden einen Messpunkt; ab zwei Punkten wird gezeichnet. Während der Daemon aus ist, gibt es keine Messpunkte — die Kurve bricht dann ehrlich ab.", - "overview_host_title": "Host-Status", - "overview_host_desc": "Ressourcenpegel und Verlauf dieser Maschine", - "overview_host_cpu_cores": "{n} Kerne", - "overview_host_cpu_peak": "{n} Kerne · Fensterspitze {pct}% ({ago})", - "overview_host_mem_label": "Speicher", - "overview_host_mem_hint": "Gesamt {total} · {available} frei", - "overview_host_swap_unreadable": "Swap ist auf dieser Plattform nicht lesbar", - "overview_host_swap_unconfigured": "Nicht konfiguriert", - "overview_host_swap_warning": "Einfrieren braucht Swap — siehe dor doctor", - "overview_host_swap_hint": "Gesamt {total} · hier wohnen die eingefrorenen Sandboxes", - "overview_host_disk_label": "Datenlaufwerk", - "overview_host_disk_missing": "Dieser Host hat kein Datenverzeichnis", - "overview_host_disk_hint": "Gesamt {total} · {available} übrig", "overview_quick_title": "Schnell verbinden", "overview_quick_desc_1": "Das offizielle ", "overview_quick_desc_2": " Paket verbindet nach dem Tausch von nur zwei URLs; der Token steht in ", "overview_quick_desc_3": " auf dem Daemon-Host.", - "overview_quick_all_methods": "Alle Verbindungswege" + "overview_quick_all_methods": "Alle Verbindungswege", + "overview_nodes_title": "Knoten", + "overview_nodes_desc": "Jede Maschine jetzt; Verläufe auf der Knotenseite", + "overview_nodes_reported": "{reported} von {total} Knoten haben gemeldet; die Zahlen oben sind eine Untergrenze", + "overview_nodes_running": "{n} laufend", + "overview_nodes_empty": "Noch kein Knoten hat sich gemeldet", + "overview_nodes_all": "Alle Knoten" } diff --git a/packages/console/messages/de/sandboxes.json b/packages/console/messages/de/sandboxes.json index 138ceb99..b4e1d58b 100644 --- a/packages/console/messages/de/sandboxes.json +++ b/packages/console/messages/de/sandboxes.json @@ -112,5 +112,6 @@ "sandboxes_cpu_value": "{v}% (pro einzelner vCPU)", "sandboxes_footer_live": "Aktuelle Werte werden alle 5 Sekunden aktualisiert; diese Messung entstand {ago}.", "sandboxes_footer_history": "Den Verlauf misst und speichert der Daemon im Hintergrund; eine schlafende Sandbox wird weiter gemessen (Beobachten weckt nie), nach einem Stopp reißt die Kurve ehrlich ab.", - "sandboxes_footer_bucketed": "Langes Fenster: aggregiert in {n}-Minuten-Buckets, jeder Punkt ist die Spitze seines Buckets." + "sandboxes_footer_bucketed": "Langes Fenster: aggregiert in {n}-Minuten-Buckets, jeder Punkt ist die Spitze seines Buckets.", + "sandboxes_silent_nodes": "Knoten {nodes} hat nicht geantwortet; seine Sandboxes fehlen in der Liste" } diff --git a/packages/console/messages/de/shell.json b/packages/console/messages/de/shell.json index 1d46cdb4..bfe50dfe 100644 --- a/packages/console/messages/de/shell.json +++ b/packages/console/messages/de/shell.json @@ -7,6 +7,7 @@ "shell_nav_sandboxes": "Sandboxes", "shell_nav_templates": "Vorlagen", "shell_nav_api_keys": "API-Schlüssel", + "shell_nav_nodes": "Knoten", "shell_nav_domains": "Domains", "shell_nav_doctor": "Doctor", "shell_nav_settings": "Einstellungen", diff --git a/packages/console/messages/en/nodes.json b/packages/console/messages/en/nodes.json new file mode 100644 index 00000000..b16e387d --- /dev/null +++ b/packages/console/messages/en/nodes.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://inlang.com/schema/inlang-message-format", + "nodes_page_title": "Nodes", + "nodes_gateway_title": "Gateway", + "nodes_gateway_desc": "The fleet's one door; a node joins at its first check-in", + "nodes_gateway_build": "Build {commit}", + "nodes_gateway_build_unknown": "Build identity unknown", + "nodes_gateway_fleet": "{total} nodes · {reachable} reachable", + "nodes_gateway_config": "Configuration v{version}", + "nodes_col_node": "Node", + "nodes_col_status": "Status", + "nodes_col_build": "Build", + "nodes_col_cpu": "CPU", + "nodes_col_memory": "Memory", + "nodes_col_disk": "Data disk", + "nodes_col_sandboxes": "Sandboxes", + "nodes_col_config": "Config", + "nodes_col_actions": "Actions", + "nodes_reachable": "Reachable", + "nodes_unreachable": "Unreachable", + "nodes_last_check_in": "Last check-in {ago}", + "nodes_never_checked_in": "No check-in since the gateway started", + "nodes_build_differs": "Differs from the gateway", + "nodes_no_reading": "No reading yet", + "nodes_sandboxes_cell": "{active} running · {frozen} frozen · {total} total", + "nodes_config_synced": "In sync", + "nodes_config_behind": "{n} behind", + "nodes_config_none": "Not open yet", + "nodes_config_none_hint": "No configuration copy yet; a node does not listen until it has its first one", + "nodes_swap_managed": "Managed swap {active} GiB", + "nodes_swap_target": "Target {target} GiB", + "nodes_swap_reconciling": "Applying", + "nodes_swap_unsupported": "This node cannot manage swap", + "nodes_row_actions_aria": "Actions for {id}", + "nodes_menu_swap": "Set extra swap", + "nodes_menu_remove": "Remove node", + "nodes_menu_remove_hint": "Only an unreachable node can be removed", + "nodes_swap_dialog_title": "Extra swap: {id}", + "nodes_swap_dialog_desc": "How many GiB of swap the node's daemon manages on its own data disk, on top of the host's own. Growing mounts at the next check-in; shrinking waits for that host's next reboot, a running swap block is never unmounted.", + "nodes_swap_field": "Extra swap (GiB)", + "nodes_swap_field_hint": "0 = none.", + "nodes_swap_saved": "Swap target for {id} set to {gb} GiB, applied at its next check-in", + "nodes_remove_title": "Remove node “{id}”?", + "nodes_remove_desc": "This says it is gone for good: its record is deleted, its sandboxes are no longer looked for, and a name that lived only there becomes a new name again. A node removed by mistake comes back at its next check-in.", + "nodes_remove_success": "Node “{id}” removed", + "nodes_remove_absent": "“{id}” was not there to begin with", + "nodes_empty_title": "No node has checked in yet", + "nodes_empty_description": "A node joins the fleet at its first check-in. Install the daemon, point DORMICE_GATEWAY_ENDPOINT at this gateway, and it appears here within 15 seconds.", + "nodes_loading": "Loading nodes", + "nodes_detail_info_title": "Details", + "nodes_detail_endpoint": "Endpoint", + "nodes_detail_added_at": "Joined", + "nodes_detail_interval": "Check-in interval", + "nodes_detail_interval_value": "Every {n}s", + "nodes_detail_build": "Build", + "nodes_detail_config_version": "Configuration version", + "nodes_detail_swap_target": "Swap target", + "nodes_detail_swap_active": "Swap mounted", + "nodes_detail_not_found_title": "No node with id “{id}”", + "nodes_detail_not_found_desc": "It may have been removed, or it has not checked in yet.", + "nodes_detail_back": "Back to nodes", + "nodes_host_title": "Host health", + "nodes_host_desc": "Resource levels and trends of this machine", + "nodes_host_cpu_cores": "{n} cores", + "nodes_host_cpu_peak": "{n} cores · window peak {pct}% ({ago})", + "nodes_host_mem_label": "Memory", + "nodes_host_mem_hint": "Total {total} · {available} free", + "nodes_host_swap_unreadable": "Swap is not readable on this platform", + "nodes_host_swap_unconfigured": "Not set", + "nodes_host_swap_warning": "Freezing needs swap — see dor doctor", + "nodes_host_swap_hint": "Total {total} · frozen sandboxes live here", + "nodes_host_disk_label": "Data disk", + "nodes_host_disk_missing": "This host has no data directory", + "nodes_host_disk_hint": "Total {total} · {available} left" +} diff --git a/packages/console/messages/en/overview.json b/packages/console/messages/en/overview.json index d6fdbb58..6a57b682 100644 --- a/packages/console/messages/en/overview.json +++ b/packages/console/messages/en/overview.json @@ -22,22 +22,15 @@ "overview_fleet_chart_desc": "Sandbox counts by state over time — active falling while frozen rises is idle-is-free at work.", "overview_fleet_chart_empty_title": "No trend to draw in this window", "overview_fleet_chart_empty_desc": "The daemon records a sample every 30 seconds and drawing starts at two points; while the daemon is down there are no samples, so the curve honestly breaks.", - "overview_host_title": "Host health", - "overview_host_desc": "Resource levels and trends of this machine", - "overview_host_cpu_cores": "{n} cores", - "overview_host_cpu_peak": "{n} cores · window peak {pct}% ({ago})", - "overview_host_mem_label": "Memory", - "overview_host_mem_hint": "Total {total} · {available} free", - "overview_host_swap_unreadable": "Swap is not readable on this platform", - "overview_host_swap_unconfigured": "Not set", - "overview_host_swap_warning": "Freezing needs swap — see dor doctor", - "overview_host_swap_hint": "Total {total} · frozen sandboxes live here", - "overview_host_disk_label": "Data disk", - "overview_host_disk_missing": "This host has no data directory", - "overview_host_disk_hint": "Total {total} · {available} left", "overview_quick_title": "Quick connect", "overview_quick_desc_1": "The official ", "overview_quick_desc_2": " package connects with just two URL swaps; the token is in ", "overview_quick_desc_3": " on the daemon host.", - "overview_quick_all_methods": "All connection methods" + "overview_quick_all_methods": "All connection methods", + "overview_nodes_title": "Nodes", + "overview_nodes_desc": "Each machine right now; trends live on the node page", + "overview_nodes_reported": "{reported} of {total} nodes have reported; the figures above are a lower bound", + "overview_nodes_running": "{n} running", + "overview_nodes_empty": "No node has checked in yet", + "overview_nodes_all": "All nodes" } diff --git a/packages/console/messages/en/sandboxes.json b/packages/console/messages/en/sandboxes.json index 915259ac..21a1b5d7 100644 --- a/packages/console/messages/en/sandboxes.json +++ b/packages/console/messages/en/sandboxes.json @@ -112,5 +112,6 @@ "sandboxes_cpu_value": "{v}% (per single vCPU)", "sandboxes_footer_live": "Live values refresh every 5 seconds; this reading is from {ago}.", "sandboxes_footer_history": "History is sampled and stored by the daemon in the background; a sleeping sandbox is still measured (observing never wakes it), and after a stop the curve honestly cuts off.", - "sandboxes_footer_bucketed": "Long window: aggregated into {n}-minute buckets, each point is the peak within its bucket." + "sandboxes_footer_bucketed": "Long window: aggregated into {n}-minute buckets, each point is the peak within its bucket.", + "sandboxes_silent_nodes": "Node {nodes} did not answer; its sandboxes are not listed" } diff --git a/packages/console/messages/en/shell.json b/packages/console/messages/en/shell.json index a830791a..2766232a 100644 --- a/packages/console/messages/en/shell.json +++ b/packages/console/messages/en/shell.json @@ -7,6 +7,7 @@ "shell_nav_sandboxes": "Sandboxes", "shell_nav_templates": "Templates", "shell_nav_api_keys": "API Keys", + "shell_nav_nodes": "Nodes", "shell_nav_domains": "Domains", "shell_nav_doctor": "Doctor", "shell_nav_settings": "Settings", diff --git a/packages/console/messages/es/nodes.json b/packages/console/messages/es/nodes.json new file mode 100644 index 00000000..fe979ce9 --- /dev/null +++ b/packages/console/messages/es/nodes.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://inlang.com/schema/inlang-message-format", + "nodes_page_title": "Nodos", + "nodes_gateway_title": "Gateway", + "nodes_gateway_desc": "La única puerta de la flota; un nodo se une en su primer check-in", + "nodes_gateway_build": "Build {commit}", + "nodes_gateway_build_unknown": "Identidad del build desconocida", + "nodes_gateway_fleet": "{total} nodos · {reachable} alcanzables", + "nodes_gateway_config": "Configuración v{version}", + "nodes_col_node": "Nodo", + "nodes_col_status": "Estado", + "nodes_col_build": "Build", + "nodes_col_cpu": "CPU", + "nodes_col_memory": "Memoria", + "nodes_col_disk": "Disco de datos", + "nodes_col_sandboxes": "Sandboxes", + "nodes_col_config": "Config", + "nodes_col_actions": "Acciones", + "nodes_reachable": "Alcanzable", + "nodes_unreachable": "Inalcanzable", + "nodes_last_check_in": "Último check-in {ago}", + "nodes_never_checked_in": "Sin check-in desde que arrancó el gateway", + "nodes_build_differs": "Distinto del gateway", + "nodes_no_reading": "Aún sin lectura", + "nodes_sandboxes_cell": "{active} en ejecución · {frozen} congelados · {total} en total", + "nodes_config_synced": "Sincronizado", + "nodes_config_behind": "{n} por detrás", + "nodes_config_none": "Aún no abierto", + "nodes_config_none_hint": "Aún no tiene copia de la configuración; un nodo no escucha hasta tener la primera", + "nodes_swap_managed": "Swap gestionado {active} GiB", + "nodes_swap_target": "Objetivo {target} GiB", + "nodes_swap_reconciling": "Aplicando", + "nodes_swap_unsupported": "Este nodo no puede gestionar swap", + "nodes_row_actions_aria": "Acciones para {id}", + "nodes_menu_swap": "Fijar swap adicional", + "nodes_menu_remove": "Quitar nodo", + "nodes_menu_remove_hint": "Solo se puede quitar un nodo inalcanzable", + "nodes_swap_dialog_title": "Swap adicional: {id}", + "nodes_swap_dialog_desc": "Cuántos GiB de swap gestiona el daemon del nodo en su propio disco de datos, además del swap del host. Ampliar se monta en el siguiente check-in; reducir espera al próximo reinicio de ese host, un bloque de swap en uso nunca se desmonta.", + "nodes_swap_field": "Swap adicional (GiB)", + "nodes_swap_field_hint": "0 = ninguno.", + "nodes_swap_saved": "Objetivo de swap de {id} fijado en {gb} GiB, se aplica en su siguiente check-in", + "nodes_remove_title": "¿Quitar el nodo «{id}»?", + "nodes_remove_desc": "Esto declara que no volverá jamás: se borra su registro, sus sandboxes dejan de buscarse y un nombre que solo vivía allí vuelve a ser un nombre nuevo. Un nodo quitado por error vuelve en su siguiente check-in.", + "nodes_remove_success": "Nodo «{id}» quitado", + "nodes_remove_absent": "«{id}» no estaba", + "nodes_empty_title": "Ningún nodo ha hecho check-in todavía", + "nodes_empty_description": "Un nodo se une a la flota en su primer check-in. Instala el daemon, apunta DORMICE_GATEWAY_ENDPOINT a este gateway y aparecerá aquí en 15 segundos.", + "nodes_loading": "Cargando nodos", + "nodes_detail_info_title": "Detalles", + "nodes_detail_endpoint": "Endpoint", + "nodes_detail_added_at": "Se unió", + "nodes_detail_interval": "Intervalo de check-in", + "nodes_detail_interval_value": "Cada {n} s", + "nodes_detail_build": "Build", + "nodes_detail_config_version": "Versión de configuración", + "nodes_detail_swap_target": "Objetivo de swap", + "nodes_detail_swap_active": "Swap montado", + "nodes_detail_not_found_title": "No hay ningún nodo con id «{id}»", + "nodes_detail_not_found_desc": "Puede que se haya quitado o que aún no haya hecho check-in.", + "nodes_detail_back": "Volver a los nodos", + "nodes_host_title": "Salud del host", + "nodes_host_desc": "Nivel de recursos y tendencias de esta máquina", + "nodes_host_cpu_cores": "{n} núcleos", + "nodes_host_cpu_peak": "{n} núcleos · pico de la ventana {pct}% ({ago})", + "nodes_host_mem_label": "Memoria", + "nodes_host_mem_hint": "Total {total} · {available} libres", + "nodes_host_swap_unreadable": "En esta plataforma no se puede leer el swap", + "nodes_host_swap_unconfigured": "Sin configurar", + "nodes_host_swap_warning": "Congelar necesita swap — ver dor doctor", + "nodes_host_swap_hint": "Total {total} · aquí viven los sandboxes congelados", + "nodes_host_disk_label": "Disco de datos", + "nodes_host_disk_missing": "Este host no tiene directorio de datos", + "nodes_host_disk_hint": "Total {total} · quedan {available}" +} diff --git a/packages/console/messages/es/overview.json b/packages/console/messages/es/overview.json index 012dbdd4..0f07eaa0 100644 --- a/packages/console/messages/es/overview.json +++ b/packages/console/messages/es/overview.json @@ -22,22 +22,15 @@ "overview_fleet_chart_desc": "Cantidad de sandboxes por estado a lo largo del tiempo: cuando los activos bajan y los congelados suben, es «inactivo significa gratis» en acción.", "overview_fleet_chart_empty_title": "No hay tendencia que dibujar en esta ventana", "overview_fleet_chart_empty_desc": "El daemon registra una muestra cada 30 segundos y el trazado empieza con dos puntos; mientras el daemon está detenido no hay muestras, así que la curva se corta con honestidad.", - "overview_host_title": "Salud del host", - "overview_host_desc": "Nivel de recursos y tendencias de esta máquina", - "overview_host_cpu_cores": "{n} núcleos", - "overview_host_cpu_peak": "{n} núcleos · pico de la ventana {pct}% ({ago})", - "overview_host_mem_label": "Memoria", - "overview_host_mem_hint": "Total {total} · {available} libres", - "overview_host_swap_unreadable": "En esta plataforma no se puede leer el swap", - "overview_host_swap_unconfigured": "Sin configurar", - "overview_host_swap_warning": "Congelar necesita swap — ver dor doctor", - "overview_host_swap_hint": "Total {total} · aquí viven los sandboxes congelados", - "overview_host_disk_label": "Disco de datos", - "overview_host_disk_missing": "Este host no tiene directorio de datos", - "overview_host_disk_hint": "Total {total} · quedan {available}", "overview_quick_title": "Conexión rápida", "overview_quick_desc_1": "El paquete oficial ", "overview_quick_desc_2": " se conecta cambiando solo dos URL; el token está en ", "overview_quick_desc_3": " en el host del daemon.", - "overview_quick_all_methods": "Todas las formas de conexión" + "overview_quick_all_methods": "Todas las formas de conexión", + "overview_nodes_title": "Nodos", + "overview_nodes_desc": "Cada máquina ahora mismo; las tendencias, en la página del nodo", + "overview_nodes_reported": "{reported} de {total} nodos han informado; las cifras de arriba son un mínimo", + "overview_nodes_running": "{n} en ejecución", + "overview_nodes_empty": "Ningún nodo ha hecho check-in todavía", + "overview_nodes_all": "Todos los nodos" } diff --git a/packages/console/messages/es/sandboxes.json b/packages/console/messages/es/sandboxes.json index 89e50546..d79d1377 100644 --- a/packages/console/messages/es/sandboxes.json +++ b/packages/console/messages/es/sandboxes.json @@ -112,5 +112,6 @@ "sandboxes_cpu_value": "{v}% (por vCPU)", "sandboxes_footer_live": "Los valores en vivo se actualizan cada 5 segundos; esta lectura es de {ago}.", "sandboxes_footer_history": "El daemon toma y guarda las muestras en segundo plano; un sandbox dormido se sigue midiendo (observar nunca lo despierta) y tras una parada la curva se corta con honestidad.", - "sandboxes_footer_bucketed": "Ventana larga: se agrupa en tramos de {n} minutos y cada punto es el pico de su tramo." + "sandboxes_footer_bucketed": "Ventana larga: se agrupa en tramos de {n} minutos y cada punto es el pico de su tramo.", + "sandboxes_silent_nodes": "El nodo {nodes} no respondió; sus sandboxes no aparecen en la lista" } diff --git a/packages/console/messages/es/shell.json b/packages/console/messages/es/shell.json index f178eae3..16fa1a7a 100644 --- a/packages/console/messages/es/shell.json +++ b/packages/console/messages/es/shell.json @@ -7,6 +7,7 @@ "shell_nav_sandboxes": "Sandboxes", "shell_nav_templates": "Plantillas", "shell_nav_api_keys": "Claves de API", + "shell_nav_nodes": "Nodos", "shell_nav_domains": "Dominios", "shell_nav_doctor": "Doctor", "shell_nav_settings": "Ajustes", diff --git a/packages/console/messages/fr/nodes.json b/packages/console/messages/fr/nodes.json new file mode 100644 index 00000000..7254f3ba --- /dev/null +++ b/packages/console/messages/fr/nodes.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://inlang.com/schema/inlang-message-format", + "nodes_page_title": "Nœuds", + "nodes_gateway_title": "Gateway", + "nodes_gateway_desc": "L'unique porte de la flotte ; un nœud la rejoint à son premier check-in", + "nodes_gateway_build": "Build {commit}", + "nodes_gateway_build_unknown": "Identité du build inconnue", + "nodes_gateway_fleet": "{total} nœuds · {reachable} joignables", + "nodes_gateway_config": "Configuration v{version}", + "nodes_col_node": "Nœud", + "nodes_col_status": "État", + "nodes_col_build": "Build", + "nodes_col_cpu": "CPU", + "nodes_col_memory": "Mémoire", + "nodes_col_disk": "Disque de données", + "nodes_col_sandboxes": "Sandbox", + "nodes_col_config": "Config", + "nodes_col_actions": "Actions", + "nodes_reachable": "Joignable", + "nodes_unreachable": "Injoignable", + "nodes_last_check_in": "Dernier check-in {ago}", + "nodes_never_checked_in": "Aucun check-in depuis le démarrage de la gateway", + "nodes_build_differs": "Diffère de la gateway", + "nodes_no_reading": "Pas encore de mesure", + "nodes_sandboxes_cell": "{active} en cours · {frozen} gelées · {total} au total", + "nodes_config_synced": "Synchronisé", + "nodes_config_behind": "{n} de retard", + "nodes_config_none": "Pas encore ouvert", + "nodes_config_none_hint": "Pas encore de copie de la configuration ; un nœud n'écoute pas avant d'avoir la première", + "nodes_swap_managed": "Swap géré {active} GiB", + "nodes_swap_target": "Cible {target} GiB", + "nodes_swap_reconciling": "Application en cours", + "nodes_swap_unsupported": "Ce nœud ne peut pas gérer de swap", + "nodes_row_actions_aria": "Actions pour {id}", + "nodes_menu_swap": "Définir le swap supplémentaire", + "nodes_menu_remove": "Retirer le nœud", + "nodes_menu_remove_hint": "Seul un nœud injoignable peut être retiré", + "nodes_swap_dialog_title": "Swap supplémentaire : {id}", + "nodes_swap_dialog_desc": "Combien de GiB de swap le daemon du nœud gère sur son propre disque de données, en plus du swap de l'hôte. Agrandir se monte au prochain check-in ; réduire attend le prochain redémarrage de cet hôte, un bloc de swap en service n'est jamais démonté.", + "nodes_swap_field": "Swap supplémentaire (GiB)", + "nodes_swap_field_hint": "0 = aucun.", + "nodes_swap_saved": "Cible de swap de {id} fixée à {gb} GiB, appliquée à son prochain check-in", + "nodes_remove_title": "Retirer le nœud « {id} » ?", + "nodes_remove_desc": "C'est déclarer qu'il ne reviendra jamais : son enregistrement est supprimé, ses sandbox ne sont plus recherchées et un nom qui ne vivait que là redevient un nom nouveau. Un nœud retiré par erreur revient à son prochain check-in.", + "nodes_remove_success": "Nœud « {id} » retiré", + "nodes_remove_absent": "« {id} » n'était pas là", + "nodes_empty_title": "Aucun nœud n'a encore fait de check-in", + "nodes_empty_description": "Un nœud rejoint la flotte à son premier check-in. Installez le daemon, pointez DORMICE_GATEWAY_ENDPOINT vers cette gateway et il apparaît ici en 15 secondes.", + "nodes_loading": "Chargement des nœuds", + "nodes_detail_info_title": "Détails", + "nodes_detail_endpoint": "Endpoint", + "nodes_detail_added_at": "Rejoint", + "nodes_detail_interval": "Intervalle de check-in", + "nodes_detail_interval_value": "Toutes les {n} s", + "nodes_detail_build": "Build", + "nodes_detail_config_version": "Version de configuration", + "nodes_detail_swap_target": "Cible de swap", + "nodes_detail_swap_active": "Swap monté", + "nodes_detail_not_found_title": "Aucun nœud avec l'id « {id} »", + "nodes_detail_not_found_desc": "Il a peut-être été retiré, ou n'a pas encore fait de check-in.", + "nodes_detail_back": "Retour aux nœuds", + "nodes_host_title": "Santé de l'hôte", + "nodes_host_desc": "Niveaux de ressources et tendances de cette machine", + "nodes_host_cpu_cores": "{n} cœurs", + "nodes_host_cpu_peak": "{n} cœurs · pic sur la fenêtre {pct} % ({ago})", + "nodes_host_mem_label": "Mémoire", + "nodes_host_mem_hint": "Total {total} · {available} disponibles", + "nodes_host_swap_unreadable": "Le swap n'est pas lisible sur cette plateforme", + "nodes_host_swap_unconfigured": "Non configuré", + "nodes_host_swap_warning": "Le gel dépend du swap — voir dor doctor", + "nodes_host_swap_hint": "Total {total} · les sandbox gelées vivent ici", + "nodes_host_disk_label": "Disque de données", + "nodes_host_disk_missing": "Cet hôte n'a pas de répertoire de données", + "nodes_host_disk_hint": "Total {total} · {available} restants" +} diff --git a/packages/console/messages/fr/overview.json b/packages/console/messages/fr/overview.json index b7eff2a6..dbbdc614 100644 --- a/packages/console/messages/fr/overview.json +++ b/packages/console/messages/fr/overview.json @@ -22,22 +22,15 @@ "overview_fleet_chart_desc": "Nombre de sandbox par état au fil du temps — quand les actives baissent et que les gelées montent, c'est « l'inactivité ne coûte rien » à l'œuvre.", "overview_fleet_chart_empty_title": "Aucune tendance à tracer sur cette fenêtre", "overview_fleet_chart_empty_desc": "Le daemon enregistre un échantillon toutes les 30 secondes et le tracé démarre à deux points ; pendant que le daemon est arrêté il n'y a pas d'échantillons, la courbe s'interrompt donc honnêtement.", - "overview_host_title": "Santé de l'hôte", - "overview_host_desc": "Niveaux de ressources et tendances de cette machine", - "overview_host_cpu_cores": "{n} cœurs", - "overview_host_cpu_peak": "{n} cœurs · pic sur la fenêtre {pct} % ({ago})", - "overview_host_mem_label": "Mémoire", - "overview_host_mem_hint": "Total {total} · {available} disponibles", - "overview_host_swap_unreadable": "Le swap n'est pas lisible sur cette plateforme", - "overview_host_swap_unconfigured": "Non configuré", - "overview_host_swap_warning": "Le gel dépend du swap — voir dor doctor", - "overview_host_swap_hint": "Total {total} · les sandbox gelées vivent ici", - "overview_host_disk_label": "Disque de données", - "overview_host_disk_missing": "Cet hôte n'a pas de répertoire de données", - "overview_host_disk_hint": "Total {total} · {available} restants", "overview_quick_title": "Connexion rapide", "overview_quick_desc_1": "Le package officiel ", "overview_quick_desc_2": " se connecte en changeant seulement deux URL ; le token se trouve dans ", "overview_quick_desc_3": " sur l'hôte du daemon.", - "overview_quick_all_methods": "Toutes les méthodes de connexion" + "overview_quick_all_methods": "Toutes les méthodes de connexion", + "overview_nodes_title": "Nœuds", + "overview_nodes_desc": "Chaque machine maintenant ; les tendances sont sur la page du nœud", + "overview_nodes_reported": "{reported} nœuds sur {total} ont rapporté ; les chiffres ci-dessus sont une borne basse", + "overview_nodes_running": "{n} en cours", + "overview_nodes_empty": "Aucun nœud n'a encore fait de check-in", + "overview_nodes_all": "Tous les nœuds" } diff --git a/packages/console/messages/fr/sandboxes.json b/packages/console/messages/fr/sandboxes.json index 5ca7cee8..a5f7a12e 100644 --- a/packages/console/messages/fr/sandboxes.json +++ b/packages/console/messages/fr/sandboxes.json @@ -112,5 +112,6 @@ "sandboxes_cpu_value": "{v} % (par vCPU)", "sandboxes_footer_live": "Les valeurs en direct se rafraîchissent toutes les 5 secondes ; cette lecture date de {ago}.", "sandboxes_footer_history": "L'historique est échantillonné et stocké par le daemon en arrière-plan ; une sandbox endormie est quand même mesurée (observer ne réveille jamais), et après un arrêt la courbe s'interrompt honnêtement.", - "sandboxes_footer_bucketed": "Fenêtre longue : agrégation par paniers de {n} minutes, chaque point est le pic de son panier." + "sandboxes_footer_bucketed": "Fenêtre longue : agrégation par paniers de {n} minutes, chaque point est le pic de son panier.", + "sandboxes_silent_nodes": "Le nœud {nodes} n'a pas répondu ; ses sandbox ne sont pas listées" } diff --git a/packages/console/messages/fr/shell.json b/packages/console/messages/fr/shell.json index b4ab5557..105e7c17 100644 --- a/packages/console/messages/fr/shell.json +++ b/packages/console/messages/fr/shell.json @@ -7,6 +7,7 @@ "shell_nav_sandboxes": "Sandbox", "shell_nav_templates": "Modèles", "shell_nav_api_keys": "Clés API", + "shell_nav_nodes": "Nœuds", "shell_nav_domains": "Domaines", "shell_nav_doctor": "Doctor", "shell_nav_settings": "Paramètres", diff --git a/packages/console/messages/ja/nodes.json b/packages/console/messages/ja/nodes.json new file mode 100644 index 00000000..d35bad5c --- /dev/null +++ b/packages/console/messages/ja/nodes.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://inlang.com/schema/inlang-message-format", + "nodes_page_title": "ノード", + "nodes_gateway_title": "ゲートウェイ", + "nodes_gateway_desc": "フリート唯一の入口。ノードは最初のチェックインで加わります", + "nodes_gateway_build": "ビルド {commit}", + "nodes_gateway_build_unknown": "ビルドの識別情報が不明", + "nodes_gateway_fleet": "{total} ノード · 到達可能 {reachable}", + "nodes_gateway_config": "設定 v{version}", + "nodes_col_node": "ノード", + "nodes_col_status": "状態", + "nodes_col_build": "ビルド", + "nodes_col_cpu": "CPU", + "nodes_col_memory": "メモリ", + "nodes_col_disk": "データディスク", + "nodes_col_sandboxes": "サンドボックス", + "nodes_col_config": "設定", + "nodes_col_actions": "操作", + "nodes_reachable": "到達可能", + "nodes_unreachable": "到達不能", + "nodes_last_check_in": "最終チェックイン {ago}", + "nodes_never_checked_in": "ゲートウェイ起動後にチェックインなし", + "nodes_build_differs": "ゲートウェイと異なる", + "nodes_no_reading": "まだ読み取りなし", + "nodes_sandboxes_cell": "実行中 {active} · 凍結 {frozen} · 合計 {total}", + "nodes_config_synced": "同期済み", + "nodes_config_behind": "{n} 版遅れ", + "nodes_config_none": "未開放", + "nodes_config_none_hint": "設定のコピーがまだありません。最初の 1 部を受け取るまでノードは待ち受けません", + "nodes_swap_managed": "自管理 swap {active} GiB", + "nodes_swap_target": "目標 {target} GiB", + "nodes_swap_reconciling": "適用中", + "nodes_swap_unsupported": "このノードは swap を自管理できません", + "nodes_row_actions_aria": "{id} の操作", + "nodes_menu_swap": "追加 swap を設定", + "nodes_menu_remove": "ノードを削除", + "nodes_menu_remove_hint": "到達不能なノードだけ削除できます", + "nodes_swap_dialog_title": "追加 swap:{id}", + "nodes_swap_dialog_desc": "ノードの daemon が自分のデータディスク上で、ホスト本来の swap に加えて何 GiB の swap を管理するか。増量は次のチェックインでマウントされ、減量はそのホストの次回再起動まで待ちます。稼働中の swap ブロックは決してアンマウントしません。", + "nodes_swap_field": "追加 swap(GiB)", + "nodes_swap_field_hint": "0 = 追加なし。", + "nodes_swap_saved": "{id} の swap 目標を {gb} GiB に設定しました。次のチェックインで適用されます", + "nodes_remove_title": "ノード「{id}」を削除しますか?", + "nodes_remove_desc": "これは「二度と戻らない」の宣言です。記録を削除し、その上のサンドボックスは探されなくなり、そこにだけ住んでいた名前は再び新しい名前になります。誤って削除したノードは次のチェックインで自動的に戻ります。", + "nodes_remove_success": "ノード「{id}」を削除しました", + "nodes_remove_absent": "「{id}」は元から存在しません", + "nodes_empty_title": "まだノードのチェックインがありません", + "nodes_empty_description": "ノードは最初のチェックインで自動的にフリートに加わります。daemon を導入し、DORMICE_GATEWAY_ENDPOINT をこのゲートウェイに向ければ、15 秒以内にここに現れます。", + "nodes_loading": "ノードを読み込み中", + "nodes_detail_info_title": "情報", + "nodes_detail_endpoint": "エンドポイント", + "nodes_detail_added_at": "参加", + "nodes_detail_interval": "チェックイン間隔", + "nodes_detail_interval_value": "{n} 秒ごと", + "nodes_detail_build": "ビルド", + "nodes_detail_config_version": "設定バージョン", + "nodes_detail_swap_target": "swap 目標", + "nodes_detail_swap_active": "swap マウント中", + "nodes_detail_not_found_title": "id「{id}」のノードはありません", + "nodes_detail_not_found_desc": "削除されたか、まだチェックインしていない可能性があります。", + "nodes_detail_back": "ノード一覧へ戻る", + "nodes_host_title": "ホストの健全性", + "nodes_host_desc": "このマシンのリソース使用状況と推移", + "nodes_host_cpu_cores": "{n} コア", + "nodes_host_cpu_peak": "{n} コア · 期間内ピーク {pct}%({ago})", + "nodes_host_mem_label": "メモリ", + "nodes_host_mem_hint": "合計 {total} · 空き {available}", + "nodes_host_swap_unreadable": "このプラットフォームでは swap を読み取れません", + "nodes_host_swap_unconfigured": "未設定", + "nodes_host_swap_warning": "凍結には swap が必要です — dor doctor を参照", + "nodes_host_swap_hint": "合計 {total} · 凍結中のサンドボックスはここに退避されます", + "nodes_host_disk_label": "データディスク", + "nodes_host_disk_missing": "このホストにはデータディレクトリがありません", + "nodes_host_disk_hint": "合計 {total} · 残り {available}" +} diff --git a/packages/console/messages/ja/overview.json b/packages/console/messages/ja/overview.json index d96b94c2..f3b5320b 100644 --- a/packages/console/messages/ja/overview.json +++ b/packages/console/messages/ja/overview.json @@ -22,22 +22,15 @@ "overview_fleet_chart_desc": "状態別サンドボックス数の時間変化です — アクティブが下がり凍結が増えていくのは、「アイドルは無料」が機能している証拠です。", "overview_fleet_chart_empty_title": "この期間にはまだ描ける推移がありません", "overview_fleet_chart_empty_desc": "daemon は 30 秒ごとにサンプルを記録し、2 点たまると描画が始まります。daemon の停止中はサンプルがないため、曲線は正直に途切れます。", - "overview_host_title": "ホストの健全性", - "overview_host_desc": "このマシンのリソース使用状況と推移", - "overview_host_cpu_cores": "{n} コア", - "overview_host_cpu_peak": "{n} コア · 期間内ピーク {pct}%({ago})", - "overview_host_mem_label": "メモリ", - "overview_host_mem_hint": "合計 {total} · 空き {available}", - "overview_host_swap_unreadable": "このプラットフォームでは swap を読み取れません", - "overview_host_swap_unconfigured": "未設定", - "overview_host_swap_warning": "凍結には swap が必要です — dor doctor を参照", - "overview_host_swap_hint": "合計 {total} · 凍結中のサンドボックスはここに退避されます", - "overview_host_disk_label": "データディスク", - "overview_host_disk_missing": "このホストにはデータディレクトリがありません", - "overview_host_disk_hint": "合計 {total} · 残り {available}", "overview_quick_title": "クイック接続", "overview_quick_desc_1": "公式の ", "overview_quick_desc_2": " パッケージは URL を 2 つ差し替えるだけで直結できます。トークンは daemon ホストの ", "overview_quick_desc_3": " にあります。", - "overview_quick_all_methods": "すべての接続方法" + "overview_quick_all_methods": "すべての接続方法", + "overview_nodes_title": "ノード", + "overview_nodes_desc": "各マシンの現在の水位。推移はノードページに", + "overview_nodes_reported": "{total} ノード中 {reported} が報告済み。上の数値は下限です", + "overview_nodes_running": "実行中 {n}", + "overview_nodes_empty": "まだノードのチェックインがありません", + "overview_nodes_all": "すべてのノード" } diff --git a/packages/console/messages/ja/sandboxes.json b/packages/console/messages/ja/sandboxes.json index 3e0c950b..d78db886 100644 --- a/packages/console/messages/ja/sandboxes.json +++ b/packages/console/messages/ja/sandboxes.json @@ -112,5 +112,6 @@ "sandboxes_cpu_value": "{v}%(vCPU 1 基あたり)", "sandboxes_footer_live": "現在値は 5 秒ごとに更新されます。今回の読み取りは {ago} のものです。", "sandboxes_footer_history": "履歴は daemon がバックグラウンドでサンプリングして保存します。スリープ中のサンドボックスも計測され(観察してもウェイクアップしません)、停止後は曲線が正直に途切れます。", - "sandboxes_footer_bucketed": "期間が長いため {n} 分単位のバケットに集約しています。各点はバケット内のピーク値です。" + "sandboxes_footer_bucketed": "期間が長いため {n} 分単位のバケットに集約しています。各点はバケット内のピーク値です。", + "sandboxes_silent_nodes": "ノード {nodes} が応答しませんでした。その上のサンドボックスは一覧にありません" } diff --git a/packages/console/messages/ja/shell.json b/packages/console/messages/ja/shell.json index a41402ad..2205475f 100644 --- a/packages/console/messages/ja/shell.json +++ b/packages/console/messages/ja/shell.json @@ -7,6 +7,7 @@ "shell_nav_sandboxes": "サンドボックス", "shell_nav_templates": "テンプレート", "shell_nav_api_keys": "API キー", + "shell_nav_nodes": "ノード", "shell_nav_domains": "ドメイン", "shell_nav_doctor": "診断", "shell_nav_settings": "設定", diff --git a/packages/console/messages/ko/nodes.json b/packages/console/messages/ko/nodes.json new file mode 100644 index 00000000..046ad107 --- /dev/null +++ b/packages/console/messages/ko/nodes.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://inlang.com/schema/inlang-message-format", + "nodes_page_title": "노드", + "nodes_gateway_title": "게이트웨이", + "nodes_gateway_desc": "플릿의 유일한 문. 노드는 첫 체크인에서 합류합니다", + "nodes_gateway_build": "빌드 {commit}", + "nodes_gateway_build_unknown": "빌드 신원 미확인", + "nodes_gateway_fleet": "노드 {total}대 · 도달 가능 {reachable}대", + "nodes_gateway_config": "설정 v{version}", + "nodes_col_node": "노드", + "nodes_col_status": "상태", + "nodes_col_build": "빌드", + "nodes_col_cpu": "CPU", + "nodes_col_memory": "메모리", + "nodes_col_disk": "데이터 디스크", + "nodes_col_sandboxes": "샌드박스", + "nodes_col_config": "설정", + "nodes_col_actions": "작업", + "nodes_reachable": "도달 가능", + "nodes_unreachable": "도달 불가", + "nodes_last_check_in": "마지막 체크인 {ago}", + "nodes_never_checked_in": "게이트웨이 시작 후 체크인 없음", + "nodes_build_differs": "게이트웨이와 다름", + "nodes_no_reading": "아직 읽기 없음", + "nodes_sandboxes_cell": "실행 {active} · 동결 {frozen} · 전체 {total}", + "nodes_config_synced": "동기화됨", + "nodes_config_behind": "{n}판 뒤처짐", + "nodes_config_none": "아직 열리지 않음", + "nodes_config_none_hint": "설정 복사본이 아직 없습니다. 첫 복사본을 받기 전까지 노드는 수신하지 않습니다", + "nodes_swap_managed": "자체 관리 swap {active} GiB", + "nodes_swap_target": "목표 {target} GiB", + "nodes_swap_reconciling": "적용 중", + "nodes_swap_unsupported": "이 노드는 swap을 자체 관리할 수 없습니다", + "nodes_row_actions_aria": "{id}의 작업", + "nodes_menu_swap": "추가 swap 설정", + "nodes_menu_remove": "노드 제거", + "nodes_menu_remove_hint": "도달 불가한 노드만 제거할 수 있습니다", + "nodes_swap_dialog_title": "추가 swap: {id}", + "nodes_swap_dialog_desc": "노드 daemon이 호스트 자체 swap 위에 자기 데이터 디스크에서 몇 GiB의 swap을 추가로 관리할지 정합니다. 늘리면 다음 체크인에 마운트되고, 줄이면 그 호스트의 다음 재시작을 기다립니다. 사용 중인 swap 블록은 절대 언마운트하지 않습니다.", + "nodes_swap_field": "추가 swap (GiB)", + "nodes_swap_field_hint": "0 = 추가 없음.", + "nodes_swap_saved": "{id}의 swap 목표를 {gb} GiB로 설정했습니다. 다음 체크인에 적용됩니다", + "nodes_remove_title": "노드 “{id}”를 제거할까요?", + "nodes_remove_desc": "“영원히 돌아오지 않는다”는 선언입니다. 기록을 지우고, 그 위의 샌드박스는 더 이상 찾지 않으며, 그곳에만 있던 이름은 다시 새 이름이 됩니다. 실수로 제거한 노드는 다음 체크인에 자동으로 돌아옵니다.", + "nodes_remove_success": "노드 “{id}”를 제거했습니다", + "nodes_remove_absent": "“{id}”는 원래 없었습니다", + "nodes_empty_title": "아직 체크인한 노드가 없습니다", + "nodes_empty_description": "노드는 첫 체크인에서 자동으로 플릿에 합류합니다. daemon을 설치하고 DORMICE_GATEWAY_ENDPOINT를 이 게이트웨이로 향하게 하면 15초 안에 여기에 나타납니다.", + "nodes_loading": "노드 불러오는 중", + "nodes_detail_info_title": "정보", + "nodes_detail_endpoint": "엔드포인트", + "nodes_detail_added_at": "합류", + "nodes_detail_interval": "체크인 간격", + "nodes_detail_interval_value": "{n}초마다", + "nodes_detail_build": "빌드", + "nodes_detail_config_version": "설정 버전", + "nodes_detail_swap_target": "swap 목표", + "nodes_detail_swap_active": "마운트된 swap", + "nodes_detail_not_found_title": "id가 “{id}”인 노드가 없습니다", + "nodes_detail_not_found_desc": "제거되었거나 아직 체크인하지 않았을 수 있습니다.", + "nodes_detail_back": "노드 목록으로", + "nodes_host_title": "호스트 상태", + "nodes_host_desc": "이 머신의 리소스 수위와 추이", + "nodes_host_cpu_cores": "{n}코어", + "nodes_host_cpu_peak": "{n}코어 · 기간 최고 {pct}% ({ago})", + "nodes_host_mem_label": "메모리", + "nodes_host_mem_hint": "전체 {total} · 사용 가능 {available}", + "nodes_host_swap_unreadable": "이 플랫폼에서는 swap을 읽을 수 없습니다", + "nodes_host_swap_unconfigured": "미설정", + "nodes_host_swap_warning": "동결은 swap에 의존 — dor doctor 참조", + "nodes_host_swap_hint": "전체 {total} · 동결된 샌드박스가 여기에 삽니다", + "nodes_host_disk_label": "데이터 디스크", + "nodes_host_disk_missing": "이 호스트에는 데이터 디렉터리가 없습니다", + "nodes_host_disk_hint": "전체 {total} · 남음 {available}" +} diff --git a/packages/console/messages/ko/overview.json b/packages/console/messages/ko/overview.json index f6722baf..f4243c7c 100644 --- a/packages/console/messages/ko/overview.json +++ b/packages/console/messages/ko/overview.json @@ -22,22 +22,15 @@ "overview_fleet_chart_desc": "상태별 샌드박스 수의 시간에 따른 변화입니다 — 활성이 내려가고 동결이 올라가는 것이 바로 '유휴는 무료'가 작동하는 모습입니다.", "overview_fleet_chart_empty_title": "이 기간에는 아직 그릴 추이가 없습니다", "overview_fleet_chart_empty_desc": "daemon은 30초마다 샘플을 기록하며 두 점이 모이면 그리기 시작합니다. daemon이 중지된 시간대에는 샘플이 없어 곡선이 정직하게 끊어집니다.", - "overview_host_title": "호스트 상태", - "overview_host_desc": "이 머신의 리소스 수위와 추이", - "overview_host_cpu_cores": "{n}코어", - "overview_host_cpu_peak": "{n}코어 · 기간 최고 {pct}% ({ago})", - "overview_host_mem_label": "메모리", - "overview_host_mem_hint": "전체 {total} · 사용 가능 {available}", - "overview_host_swap_unreadable": "이 플랫폼에서는 swap을 읽을 수 없습니다", - "overview_host_swap_unconfigured": "미설정", - "overview_host_swap_warning": "동결은 swap에 의존 — dor doctor 참조", - "overview_host_swap_hint": "전체 {total} · 동결된 샌드박스가 여기에 삽니다", - "overview_host_disk_label": "데이터 디스크", - "overview_host_disk_missing": "이 호스트에는 데이터 디렉터리가 없습니다", - "overview_host_disk_hint": "전체 {total} · 남음 {available}", "overview_quick_title": "빠른 연결", "overview_quick_desc_1": "공식 ", "overview_quick_desc_2": " 패키지는 URL 두 개만 바꾸면 바로 연결됩니다. 토큰은 daemon 호스트의 ", "overview_quick_desc_3": "에 있습니다.", - "overview_quick_all_methods": "모든 연결 방법" + "overview_quick_all_methods": "모든 연결 방법", + "overview_nodes_title": "노드", + "overview_nodes_desc": "각 머신의 현재 수위. 추이는 노드 페이지에", + "overview_nodes_reported": "노드 {total}대 중 {reported}대가 보고했습니다. 위 숫자는 하한입니다", + "overview_nodes_running": "실행 {n}", + "overview_nodes_empty": "아직 체크인한 노드가 없습니다", + "overview_nodes_all": "모든 노드" } diff --git a/packages/console/messages/ko/sandboxes.json b/packages/console/messages/ko/sandboxes.json index 89218a75..3608ba45 100644 --- a/packages/console/messages/ko/sandboxes.json +++ b/packages/console/messages/ko/sandboxes.json @@ -112,5 +112,6 @@ "sandboxes_cpu_value": "{v}% (단일 vCPU 기준)", "sandboxes_footer_live": "현재 값은 5초마다 새로 고쳐지며, 이번 판독값은 {ago} 시점의 것입니다.", "sandboxes_footer_history": "히스토리는 daemon이 백그라운드에서 샘플링해 저장합니다. 잠든 샌드박스도 그대로 측정되며(관찰은 절대 깨우지 않음), 중지 후에는 곡선이 정직하게 끊깁니다.", - "sandboxes_footer_bucketed": "기간이 길어 {n}분 버킷으로 집계했으며, 각 점은 버킷 내 최고치입니다." + "sandboxes_footer_bucketed": "기간이 길어 {n}분 버킷으로 집계했으며, 각 점은 버킷 내 최고치입니다.", + "sandboxes_silent_nodes": "노드 {nodes}가 응답하지 않았습니다. 그 위의 샌드박스는 목록에 없습니다" } diff --git a/packages/console/messages/ko/shell.json b/packages/console/messages/ko/shell.json index ace12051..168eb93b 100644 --- a/packages/console/messages/ko/shell.json +++ b/packages/console/messages/ko/shell.json @@ -7,6 +7,7 @@ "shell_nav_sandboxes": "샌드박스", "shell_nav_templates": "템플릿", "shell_nav_api_keys": "API 키", + "shell_nav_nodes": "노드", "shell_nav_domains": "도메인", "shell_nav_doctor": "진단", "shell_nav_settings": "설정", diff --git a/packages/console/messages/pt-BR/nodes.json b/packages/console/messages/pt-BR/nodes.json new file mode 100644 index 00000000..39eeec89 --- /dev/null +++ b/packages/console/messages/pt-BR/nodes.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://inlang.com/schema/inlang-message-format", + "nodes_page_title": "Nós", + "nodes_gateway_title": "Gateway", + "nodes_gateway_desc": "A única porta da frota; um nó entra no primeiro check-in", + "nodes_gateway_build": "Build {commit}", + "nodes_gateway_build_unknown": "Identidade do build desconhecida", + "nodes_gateway_fleet": "{total} nós · {reachable} alcançáveis", + "nodes_gateway_config": "Configuração v{version}", + "nodes_col_node": "Nó", + "nodes_col_status": "Estado", + "nodes_col_build": "Build", + "nodes_col_cpu": "CPU", + "nodes_col_memory": "Memória", + "nodes_col_disk": "Disco de dados", + "nodes_col_sandboxes": "Sandboxes", + "nodes_col_config": "Config", + "nodes_col_actions": "Ações", + "nodes_reachable": "Alcançável", + "nodes_unreachable": "Inalcançável", + "nodes_last_check_in": "Último check-in {ago}", + "nodes_never_checked_in": "Sem check-in desde que o gateway iniciou", + "nodes_build_differs": "Difere do gateway", + "nodes_no_reading": "Ainda sem leitura", + "nodes_sandboxes_cell": "{active} em execução · {frozen} congelados · {total} no total", + "nodes_config_synced": "Sincronizado", + "nodes_config_behind": "{n} atrás", + "nodes_config_none": "Ainda não aberto", + "nodes_config_none_hint": "Ainda sem cópia da configuração; um nó não escuta até receber a primeira", + "nodes_swap_managed": "Swap gerenciado {active} GiB", + "nodes_swap_target": "Meta {target} GiB", + "nodes_swap_reconciling": "Aplicando", + "nodes_swap_unsupported": "Este nó não consegue gerenciar swap", + "nodes_row_actions_aria": "Ações de {id}", + "nodes_menu_swap": "Definir swap adicional", + "nodes_menu_remove": "Remover nó", + "nodes_menu_remove_hint": "Só um nó inalcançável pode ser removido", + "nodes_swap_dialog_title": "Swap adicional: {id}", + "nodes_swap_dialog_desc": "Quantos GiB de swap o daemon do nó gerencia no próprio disco de dados, além do swap do host. Aumentar monta no próximo check-in; reduzir espera a próxima reinicialização desse host, um bloco de swap em uso nunca é desmontado.", + "nodes_swap_field": "Swap adicional (GiB)", + "nodes_swap_field_hint": "0 = nenhum.", + "nodes_swap_saved": "Meta de swap de {id} definida em {gb} GiB, aplicada no próximo check-in", + "nodes_remove_title": "Remover o nó “{id}”?", + "nodes_remove_desc": "Isto declara que ele nunca mais volta: o registro é apagado, seus sandboxes deixam de ser procurados e um nome que só vivia lá volta a ser um nome novo. Um nó removido por engano volta no próximo check-in.", + "nodes_remove_success": "Nó “{id}” removido", + "nodes_remove_absent": "“{id}” não estava lá", + "nodes_empty_title": "Nenhum nó fez check-in ainda", + "nodes_empty_description": "Um nó entra na frota no primeiro check-in. Instale o daemon, aponte DORMICE_GATEWAY_ENDPOINT para este gateway e ele aparece aqui em 15 segundos.", + "nodes_loading": "Carregando nós", + "nodes_detail_info_title": "Detalhes", + "nodes_detail_endpoint": "Endpoint", + "nodes_detail_added_at": "Entrou", + "nodes_detail_interval": "Intervalo de check-in", + "nodes_detail_interval_value": "A cada {n} s", + "nodes_detail_build": "Build", + "nodes_detail_config_version": "Versão da configuração", + "nodes_detail_swap_target": "Meta de swap", + "nodes_detail_swap_active": "Swap montado", + "nodes_detail_not_found_title": "Nenhum nó com id “{id}”", + "nodes_detail_not_found_desc": "Pode ter sido removido, ou ainda não fez check-in.", + "nodes_detail_back": "Voltar aos nós", + "nodes_host_title": "Saúde do host", + "nodes_host_desc": "Níveis e tendências dos recursos desta máquina", + "nodes_host_cpu_cores": "{n} núcleos", + "nodes_host_cpu_peak": "{n} núcleos · pico na janela {pct}% ({ago})", + "nodes_host_mem_label": "Memória", + "nodes_host_mem_hint": "Total {total} · {available} livres", + "nodes_host_swap_unreadable": "Não é possível ler o swap nesta plataforma", + "nodes_host_swap_unconfigured": "Não configurado", + "nodes_host_swap_warning": "Congelar depende de swap — veja dor doctor", + "nodes_host_swap_hint": "Total {total} · sandboxes congelados moram aqui", + "nodes_host_disk_label": "Disco de dados", + "nodes_host_disk_missing": "Este host não tem diretório de dados", + "nodes_host_disk_hint": "Total {total} · {available} restantes" +} diff --git a/packages/console/messages/pt-BR/overview.json b/packages/console/messages/pt-BR/overview.json index e5d48fbc..2baf5ded 100644 --- a/packages/console/messages/pt-BR/overview.json +++ b/packages/console/messages/pt-BR/overview.json @@ -22,22 +22,15 @@ "overview_fleet_chart_desc": "Contagem de sandboxes por estado ao longo do tempo — ativos caindo enquanto congelados sobem é o ocioso-é-grátis em ação.", "overview_fleet_chart_empty_title": "Nada para desenhar nesta janela", "overview_fleet_chart_empty_desc": "O daemon registra uma amostra a cada 30 segundos e o desenho começa com dois pontos; enquanto o daemon está fora do ar não há amostras, então a curva se interrompe honestamente.", - "overview_host_title": "Saúde do host", - "overview_host_desc": "Níveis e tendências dos recursos desta máquina", - "overview_host_cpu_cores": "{n} núcleos", - "overview_host_cpu_peak": "{n} núcleos · pico na janela {pct}% ({ago})", - "overview_host_mem_label": "Memória", - "overview_host_mem_hint": "Total {total} · {available} livres", - "overview_host_swap_unreadable": "Não é possível ler o swap nesta plataforma", - "overview_host_swap_unconfigured": "Não configurado", - "overview_host_swap_warning": "Congelar depende de swap — veja dor doctor", - "overview_host_swap_hint": "Total {total} · sandboxes congelados moram aqui", - "overview_host_disk_label": "Disco de dados", - "overview_host_disk_missing": "Este host não tem diretório de dados", - "overview_host_disk_hint": "Total {total} · {available} restantes", "overview_quick_title": "Conexão rápida", "overview_quick_desc_1": "O pacote oficial ", "overview_quick_desc_2": " conecta trocando apenas duas URL; o token está em ", "overview_quick_desc_3": " no host do daemon.", - "overview_quick_all_methods": "Todos os métodos de conexão" + "overview_quick_all_methods": "Todos os métodos de conexão", + "overview_nodes_title": "Nós", + "overview_nodes_desc": "Cada máquina agora; as tendências ficam na página do nó", + "overview_nodes_reported": "{reported} de {total} nós reportaram; os números acima são um limite inferior", + "overview_nodes_running": "{n} em execução", + "overview_nodes_empty": "Nenhum nó fez check-in ainda", + "overview_nodes_all": "Todos os nós" } diff --git a/packages/console/messages/pt-BR/sandboxes.json b/packages/console/messages/pt-BR/sandboxes.json index cb4bcd48..9b74445d 100644 --- a/packages/console/messages/pt-BR/sandboxes.json +++ b/packages/console/messages/pt-BR/sandboxes.json @@ -112,5 +112,6 @@ "sandboxes_cpu_value": "{v}% (por vCPU)", "sandboxes_footer_live": "Os valores ao vivo atualizam a cada 5 segundos; esta leitura é de {ago}.", "sandboxes_footer_history": "O histórico é amostrado e gravado pelo daemon em segundo plano; um sandbox adormecido continua sendo medido (observar nunca acorda) e, depois de parar, a curva se interrompe honestamente.", - "sandboxes_footer_bucketed": "Janela longa: agregado em blocos de {n} minutos; cada ponto é o pico dentro do seu bloco." + "sandboxes_footer_bucketed": "Janela longa: agregado em blocos de {n} minutos; cada ponto é o pico dentro do seu bloco.", + "sandboxes_silent_nodes": "O nó {nodes} não respondeu; seus sandboxes não estão na lista" } diff --git a/packages/console/messages/pt-BR/shell.json b/packages/console/messages/pt-BR/shell.json index 6072febf..5d3cfe57 100644 --- a/packages/console/messages/pt-BR/shell.json +++ b/packages/console/messages/pt-BR/shell.json @@ -7,6 +7,7 @@ "shell_nav_sandboxes": "Sandboxes", "shell_nav_templates": "Templates", "shell_nav_api_keys": "Chaves de API", + "shell_nav_nodes": "Nós", "shell_nav_domains": "Domínios", "shell_nav_doctor": "Diagnóstico", "shell_nav_settings": "Configurações", diff --git a/packages/console/messages/ru/nodes.json b/packages/console/messages/ru/nodes.json new file mode 100644 index 00000000..e4653879 --- /dev/null +++ b/packages/console/messages/ru/nodes.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://inlang.com/schema/inlang-message-format", + "nodes_page_title": "Узлы", + "nodes_gateway_title": "Шлюз", + "nodes_gateway_desc": "Единственная дверь флота; узел вступает при первом чек-ине", + "nodes_gateway_build": "Сборка {commit}", + "nodes_gateway_build_unknown": "Идентичность сборки неизвестна", + "nodes_gateway_fleet": "Узлов: {total} · доступно: {reachable}", + "nodes_gateway_config": "Конфигурация v{version}", + "nodes_col_node": "Узел", + "nodes_col_status": "Состояние", + "nodes_col_build": "Сборка", + "nodes_col_cpu": "CPU", + "nodes_col_memory": "Память", + "nodes_col_disk": "Диск данных", + "nodes_col_sandboxes": "Песочницы", + "nodes_col_config": "Конфиг", + "nodes_col_actions": "Действия", + "nodes_reachable": "Доступен", + "nodes_unreachable": "Недоступен", + "nodes_last_check_in": "Последний чек-ин {ago}", + "nodes_never_checked_in": "Ни одного чек-ина с запуска шлюза", + "nodes_build_differs": "Отличается от шлюза", + "nodes_no_reading": "Показаний пока нет", + "nodes_sandboxes_cell": "Работают: {active} · заморожены: {frozen} · всего: {total}", + "nodes_config_synced": "Синхронизирован", + "nodes_config_behind": "Отстаёт на {n}", + "nodes_config_none": "Ещё не открыт", + "nodes_config_none_hint": "Копии конфигурации ещё нет; узел не слушает, пока не получит первую", + "nodes_swap_managed": "Управляемый swap {active} GiB", + "nodes_swap_target": "Цель {target} GiB", + "nodes_swap_reconciling": "Применяется", + "nodes_swap_unsupported": "Этот узел не умеет управлять swap", + "nodes_row_actions_aria": "Действия для {id}", + "nodes_menu_swap": "Задать дополнительный swap", + "nodes_menu_remove": "Удалить узел", + "nodes_menu_remove_hint": "Удалить можно только недоступный узел", + "nodes_swap_dialog_title": "Дополнительный swap: {id}", + "nodes_swap_dialog_desc": "Сколько GiB swap демон узла держит на своём диске данных поверх swap самого хоста. Увеличение монтируется при следующем чек-ине; уменьшение ждёт следующей перезагрузки этого хоста, работающий блок swap никогда не отмонтируется.", + "nodes_swap_field": "Дополнительный swap (GiB)", + "nodes_swap_field_hint": "0 = без дополнительного.", + "nodes_swap_saved": "Цель swap для {id} задана: {gb} GiB, применится при следующем чек-ине", + "nodes_remove_title": "Удалить узел «{id}»?", + "nodes_remove_desc": "Это заявление, что он больше не вернётся: запись удаляется, его песочницы больше не ищутся, а имя, жившее только там, снова становится новым. Удалённый по ошибке узел вернётся при следующем чек-ине.", + "nodes_remove_success": "Узел «{id}» удалён", + "nodes_remove_absent": "«{id}» и так не было", + "nodes_empty_title": "Ни один узел ещё не отметился", + "nodes_empty_description": "Узел вступает во флот при первом чек-ине. Установите демон, направьте DORMICE_GATEWAY_ENDPOINT на этот шлюз, и он появится здесь в течение 15 секунд.", + "nodes_loading": "Загрузка узлов", + "nodes_detail_info_title": "Сведения", + "nodes_detail_endpoint": "Эндпоинт", + "nodes_detail_added_at": "Вступил", + "nodes_detail_interval": "Интервал чек-ина", + "nodes_detail_interval_value": "Каждые {n} с", + "nodes_detail_build": "Сборка", + "nodes_detail_config_version": "Версия конфигурации", + "nodes_detail_swap_target": "Цель swap", + "nodes_detail_swap_active": "Смонтированный swap", + "nodes_detail_not_found_title": "Узла с id «{id}» нет", + "nodes_detail_not_found_desc": "Возможно, он удалён или ещё не отметился.", + "nodes_detail_back": "К списку узлов", + "nodes_host_title": "Здоровье хоста", + "nodes_host_desc": "Уровни ресурсов этой машины и их динамика", + "nodes_host_cpu_cores": "Ядер: {n}", + "nodes_host_cpu_peak": "Ядер: {n} · пик за окно {pct}% ({ago})", + "nodes_host_mem_label": "Память", + "nodes_host_mem_hint": "Всего {total} · доступно {available}", + "nodes_host_swap_unreadable": "На этой платформе swap не читается", + "nodes_host_swap_unconfigured": "Не настроен", + "nodes_host_swap_warning": "Заморозке нужен swap — см. dor doctor", + "nodes_host_swap_hint": "Всего {total} · здесь живут замороженные песочницы", + "nodes_host_disk_label": "Диск данных", + "nodes_host_disk_missing": "На этом хосте нет каталога данных", + "nodes_host_disk_hint": "Всего {total} · свободно {available}" +} diff --git a/packages/console/messages/ru/overview.json b/packages/console/messages/ru/overview.json index 03fc6e67..fff6616d 100644 --- a/packages/console/messages/ru/overview.json +++ b/packages/console/messages/ru/overview.json @@ -22,22 +22,15 @@ "overview_fleet_chart_desc": "Число песочниц в каждом состоянии во времени — активные падают, замороженные растут: так работает «простой ничего не стоит».", "overview_fleet_chart_empty_title": "В этом окне пока нечего рисовать", "overview_fleet_chart_empty_desc": "Daemon записывает замер каждые 30 секунд, график начинается с двух точек; пока daemon остановлен, замеров нет, и кривая честно прерывается.", - "overview_host_title": "Здоровье хоста", - "overview_host_desc": "Уровни ресурсов этой машины и их динамика", - "overview_host_cpu_cores": "Ядер: {n}", - "overview_host_cpu_peak": "Ядер: {n} · пик за окно {pct}% ({ago})", - "overview_host_mem_label": "Память", - "overview_host_mem_hint": "Всего {total} · доступно {available}", - "overview_host_swap_unreadable": "На этой платформе swap не читается", - "overview_host_swap_unconfigured": "Не настроен", - "overview_host_swap_warning": "Заморозке нужен swap — см. dor doctor", - "overview_host_swap_hint": "Всего {total} · здесь живут замороженные песочницы", - "overview_host_disk_label": "Диск данных", - "overview_host_disk_missing": "На этом хосте нет каталога данных", - "overview_host_disk_hint": "Всего {total} · свободно {available}", "overview_quick_title": "Быстрое подключение", "overview_quick_desc_1": "Официальный пакет ", "overview_quick_desc_2": " подключается заменой всего двух URL; токен лежит в ", "overview_quick_desc_3": " на хосте daemon.", - "overview_quick_all_methods": "Все способы подключения" + "overview_quick_all_methods": "Все способы подключения", + "overview_nodes_title": "Узлы", + "overview_nodes_desc": "Каждая машина сейчас; динамика на странице узла", + "overview_nodes_reported": "Отчитались {reported} из {total} узлов; цифры выше — нижняя граница", + "overview_nodes_running": "Работают: {n}", + "overview_nodes_empty": "Ни один узел ещё не отметился", + "overview_nodes_all": "Все узлы" } diff --git a/packages/console/messages/ru/sandboxes.json b/packages/console/messages/ru/sandboxes.json index 22359946..79ce6694 100644 --- a/packages/console/messages/ru/sandboxes.json +++ b/packages/console/messages/ru/sandboxes.json @@ -112,5 +112,6 @@ "sandboxes_cpu_value": "{v}% (на один vCPU)", "sandboxes_footer_live": "Текущие значения обновляются каждые 5 секунд; это показание снято {ago}.", "sandboxes_footer_history": "Историю снимает и хранит daemon в фоне; спящая песочница всё равно измеряется (наблюдение не будит), а после остановки кривая честно обрывается.", - "sandboxes_footer_bucketed": "Окно длинное: данные агрегированы по корзинам в {n} мин., каждая точка — пик внутри корзины." + "sandboxes_footer_bucketed": "Окно длинное: данные агрегированы по корзинам в {n} мин., каждая точка — пик внутри корзины.", + "sandboxes_silent_nodes": "Узел {nodes} не ответил; его песочниц нет в списке" } diff --git a/packages/console/messages/ru/shell.json b/packages/console/messages/ru/shell.json index 2dc6d702..32337527 100644 --- a/packages/console/messages/ru/shell.json +++ b/packages/console/messages/ru/shell.json @@ -7,6 +7,7 @@ "shell_nav_sandboxes": "Песочницы", "shell_nav_templates": "Шаблоны", "shell_nav_api_keys": "API-ключи", + "shell_nav_nodes": "Узлы", "shell_nav_domains": "Домены", "shell_nav_doctor": "Диагностика", "shell_nav_settings": "Настройки", diff --git a/packages/console/messages/zh-CN/nodes.json b/packages/console/messages/zh-CN/nodes.json new file mode 100644 index 00000000..eb2386c4 --- /dev/null +++ b/packages/console/messages/zh-CN/nodes.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://inlang.com/schema/inlang-message-format", + "nodes_page_title": "节点", + "nodes_gateway_title": "网关", + "nodes_gateway_desc": "舰队唯一的门;节点在第一次报到时加入", + "nodes_gateway_build": "构建 {commit}", + "nodes_gateway_build_unknown": "构建身份未知", + "nodes_gateway_fleet": "{total} 台节点 · {reachable} 台可达", + "nodes_gateway_config": "配置第 {version} 版", + "nodes_col_node": "节点", + "nodes_col_status": "状态", + "nodes_col_build": "版本", + "nodes_col_cpu": "CPU", + "nodes_col_memory": "内存", + "nodes_col_disk": "数据盘", + "nodes_col_sandboxes": "沙箱", + "nodes_col_config": "配置", + "nodes_col_actions": "操作", + "nodes_reachable": "可达", + "nodes_unreachable": "不可达", + "nodes_last_check_in": "最后报到 {ago}", + "nodes_never_checked_in": "网关启动后尚未报到", + "nodes_build_differs": "与网关不同", + "nodes_no_reading": "尚无读数", + "nodes_sandboxes_cell": "运行 {active} · 冻结 {frozen} · 共 {total}", + "nodes_config_synced": "已同步", + "nodes_config_behind": "落后 {n} 版", + "nodes_config_none": "未开门", + "nodes_config_none_hint": "还没有配置副本;拿到第一份之前节点不监听", + "nodes_swap_managed": "自管 swap {active} GiB", + "nodes_swap_target": "目标 {target} GiB", + "nodes_swap_reconciling": "兑现中", + "nodes_swap_unsupported": "此节点不能自管 swap", + "nodes_row_actions_aria": "{id} 的操作", + "nodes_menu_swap": "设置追加 swap", + "nodes_menu_remove": "移除节点", + "nodes_menu_remove_hint": "只能移除不可达的节点", + "nodes_swap_dialog_title": "追加 swap:{id}", + "nodes_swap_dialog_desc": "节点 daemon 在自己的数据盘上额外管多少 GiB swap,叠在宿主自带的 swap 之上。增容在下一次报到时挂上;缩容等那台宿主下次重启,运行中的 swap 块绝不卸载。", + "nodes_swap_field": "追加 swap(GiB)", + "nodes_swap_field_hint": "0 = 不追加。", + "nodes_swap_saved": "{id} 的 swap 目标已设为 {gb} GiB,下一次报到时应用", + "nodes_remove_title": "移除节点「{id}」?", + "nodes_remove_desc": "这是「它永远不回来了」的声明:删掉它的记录,它上面的沙箱不再被寻找,只住在它上面的名字重新变成新名字。误删的节点下次报到会自动回来。", + "nodes_remove_success": "已移除节点「{id}」", + "nodes_remove_absent": "「{id}」本来就不在", + "nodes_empty_title": "还没有节点报到", + "nodes_empty_description": "节点在第一次报到时自动加入舰队。装好 daemon、把 DORMICE_GATEWAY_ENDPOINT 指向这个网关,15 秒内它会出现在这里。", + "nodes_loading": "读取节点", + "nodes_detail_info_title": "信息", + "nodes_detail_endpoint": "端点", + "nodes_detail_added_at": "加入于", + "nodes_detail_interval": "报到间隔", + "nodes_detail_interval_value": "每 {n} 秒", + "nodes_detail_build": "版本", + "nodes_detail_config_version": "配置版本", + "nodes_detail_swap_target": "swap 目标", + "nodes_detail_swap_active": "swap 现挂", + "nodes_detail_not_found_title": "没有 id 为「{id}」的节点", + "nodes_detail_not_found_desc": "可能已被移除,或者它还没有报到过。", + "nodes_detail_back": "回到节点列表", + "nodes_host_title": "宿主健康", + "nodes_host_desc": "这台机器的资源水位与走势", + "nodes_host_cpu_cores": "{n} 核", + "nodes_host_cpu_peak": "{n} 核 · 窗口峰值 {pct}%({ago})", + "nodes_host_mem_label": "内存", + "nodes_host_mem_hint": "共 {total} · 可用 {available}", + "nodes_host_swap_unreadable": "此平台读不到 swap", + "nodes_host_swap_unconfigured": "未配置", + "nodes_host_swap_warning": "冻结依赖 swap — 见 dor doctor", + "nodes_host_swap_hint": "共 {total} · 冻结的沙箱住在这里", + "nodes_host_disk_label": "数据盘", + "nodes_host_disk_missing": "这台主机没有数据目录", + "nodes_host_disk_hint": "共 {total} · 剩 {available}" +} diff --git a/packages/console/messages/zh-CN/overview.json b/packages/console/messages/zh-CN/overview.json index 601c21d1..8ef14ddf 100644 --- a/packages/console/messages/zh-CN/overview.json +++ b/packages/console/messages/zh-CN/overview.json @@ -22,22 +22,15 @@ "overview_fleet_chart_desc": "各状态沙箱数量随时间的变化 — 活跃掉下去、冻结涨上来,就是「空闲即免费」在发生。", "overview_fleet_chart_empty_title": "窗口内还没有走势可画", "overview_fleet_chart_empty_desc": "daemon 每 30 秒落一次采样,攒够两个点就开始画;daemon 停机的时段没有采样,曲线会如实断开。", - "overview_host_title": "宿主健康", - "overview_host_desc": "这台机器的资源水位与走势", - "overview_host_cpu_cores": "{n} 核", - "overview_host_cpu_peak": "{n} 核 · 窗口峰值 {pct}%({ago})", - "overview_host_mem_label": "内存", - "overview_host_mem_hint": "共 {total} · 可用 {available}", - "overview_host_swap_unreadable": "此平台读不到 swap", - "overview_host_swap_unconfigured": "未配置", - "overview_host_swap_warning": "冻结依赖 swap — 见 dor doctor", - "overview_host_swap_hint": "共 {total} · 冻结的沙箱住在这里", - "overview_host_disk_label": "数据盘", - "overview_host_disk_missing": "这台主机没有数据目录", - "overview_host_disk_hint": "共 {total} · 剩 {available}", "overview_quick_title": "快速接入", "overview_quick_desc_1": "官方 ", "overview_quick_desc_2": " 包换两个 URL 直连;token 在 daemon 主机的 ", "overview_quick_desc_3": " 里。", - "overview_quick_all_methods": "全部接入方式" + "overview_quick_all_methods": "全部接入方式", + "overview_nodes_title": "节点", + "overview_nodes_desc": "每台机器此刻的水位;走势在节点页", + "overview_nodes_reported": "{total} 台中 {reported} 台已报到,以上数字是下界", + "overview_nodes_running": "运行 {n}", + "overview_nodes_empty": "还没有节点报到", + "overview_nodes_all": "全部节点" } diff --git a/packages/console/messages/zh-CN/sandboxes.json b/packages/console/messages/zh-CN/sandboxes.json index b30922be..32fa2b6f 100644 --- a/packages/console/messages/zh-CN/sandboxes.json +++ b/packages/console/messages/zh-CN/sandboxes.json @@ -112,5 +112,6 @@ "sandboxes_cpu_value": "{v}%(按单 vCPU 计)", "sandboxes_footer_live": "当前值 5 秒刷新,本次读数取自 {ago}。", "sandboxes_footer_history": "历史由 daemon 后台采样落库;沙箱睡着照样测(观察不唤醒),停止后曲线如实断流。", - "sandboxes_footer_bucketed": "窗口较长,已按每 {n} 分钟一桶聚合,每点为桶内峰值。" + "sandboxes_footer_bucketed": "窗口较长,已按每 {n} 分钟一桶聚合,每点为桶内峰值。", + "sandboxes_silent_nodes": "节点 {nodes} 没有应答,它们上面的沙箱未列出" } diff --git a/packages/console/messages/zh-CN/shell.json b/packages/console/messages/zh-CN/shell.json index 4d70037d..be16a572 100644 --- a/packages/console/messages/zh-CN/shell.json +++ b/packages/console/messages/zh-CN/shell.json @@ -7,6 +7,7 @@ "shell_nav_sandboxes": "沙箱", "shell_nav_templates": "模板", "shell_nav_api_keys": "API 密钥", + "shell_nav_nodes": "节点", "shell_nav_domains": "域名", "shell_nav_doctor": "体检", "shell_nav_settings": "设置", diff --git a/packages/console/messages/zh-TW/nodes.json b/packages/console/messages/zh-TW/nodes.json new file mode 100644 index 00000000..54ecd88e --- /dev/null +++ b/packages/console/messages/zh-TW/nodes.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://inlang.com/schema/inlang-message-format", + "nodes_page_title": "節點", + "nodes_gateway_title": "網關", + "nodes_gateway_desc": "艦隊唯一的門;節點在第一次報到時加入", + "nodes_gateway_build": "建置 {commit}", + "nodes_gateway_build_unknown": "建置身分未知", + "nodes_gateway_fleet": "{total} 台節點 · {reachable} 台可達", + "nodes_gateway_config": "設定第 {version} 版", + "nodes_col_node": "節點", + "nodes_col_status": "狀態", + "nodes_col_build": "版本", + "nodes_col_cpu": "CPU", + "nodes_col_memory": "記憶體", + "nodes_col_disk": "資料磁碟", + "nodes_col_sandboxes": "沙箱", + "nodes_col_config": "設定", + "nodes_col_actions": "操作", + "nodes_reachable": "可達", + "nodes_unreachable": "不可達", + "nodes_last_check_in": "最後報到 {ago}", + "nodes_never_checked_in": "網關啟動後尚未報到", + "nodes_build_differs": "與網關不同", + "nodes_no_reading": "尚無讀數", + "nodes_sandboxes_cell": "執行中 {active} · 凍結 {frozen} · 共 {total}", + "nodes_config_synced": "已同步", + "nodes_config_behind": "落後 {n} 版", + "nodes_config_none": "未開門", + "nodes_config_none_hint": "還沒有設定副本;拿到第一份之前節點不監聽", + "nodes_swap_managed": "自管 swap {active} GiB", + "nodes_swap_target": "目標 {target} GiB", + "nodes_swap_reconciling": "兌現中", + "nodes_swap_unsupported": "此節點不能自管 swap", + "nodes_row_actions_aria": "{id} 的操作", + "nodes_menu_swap": "設定追加 swap", + "nodes_menu_remove": "移除節點", + "nodes_menu_remove_hint": "只能移除不可達的節點", + "nodes_swap_dialog_title": "追加 swap:{id}", + "nodes_swap_dialog_desc": "節點 daemon 在自己的資料磁碟上額外管理多少 GiB swap,疊在主機自帶的 swap 之上。增容在下一次報到時掛上;縮容等那台主機下次重啟,執行中的 swap 區塊絕不卸載。", + "nodes_swap_field": "追加 swap(GiB)", + "nodes_swap_field_hint": "0 = 不追加。", + "nodes_swap_saved": "{id} 的 swap 目標已設為 {gb} GiB,下一次報到時套用", + "nodes_remove_title": "移除節點「{id}」?", + "nodes_remove_desc": "這是「它永遠不回來了」的聲明:刪掉它的紀錄,它上面的沙箱不再被尋找,只住在它上面的名字重新變成新名字。誤刪的節點下次報到會自動回來。", + "nodes_remove_success": "已移除節點「{id}」", + "nodes_remove_absent": "「{id}」本來就不在", + "nodes_empty_title": "還沒有節點報到", + "nodes_empty_description": "節點在第一次報到時自動加入艦隊。裝好 daemon、把 DORMICE_GATEWAY_ENDPOINT 指向這個網關,15 秒內它會出現在這裡。", + "nodes_loading": "讀取節點", + "nodes_detail_info_title": "資訊", + "nodes_detail_endpoint": "端點", + "nodes_detail_added_at": "加入於", + "nodes_detail_interval": "報到間隔", + "nodes_detail_interval_value": "每 {n} 秒", + "nodes_detail_build": "版本", + "nodes_detail_config_version": "設定版本", + "nodes_detail_swap_target": "swap 目標", + "nodes_detail_swap_active": "swap 現掛", + "nodes_detail_not_found_title": "沒有 id 為「{id}」的節點", + "nodes_detail_not_found_desc": "可能已被移除,或者它還沒有報到過。", + "nodes_detail_back": "回到節點列表", + "nodes_host_title": "主機健康", + "nodes_host_desc": "這台機器的資源水位與走勢", + "nodes_host_cpu_cores": "{n} 核心", + "nodes_host_cpu_peak": "{n} 核心 · 區間尖峰 {pct}%({ago})", + "nodes_host_mem_label": "記憶體", + "nodes_host_mem_hint": "共 {total} · 可用 {available}", + "nodes_host_swap_unreadable": "此平台讀不到 swap", + "nodes_host_swap_unconfigured": "未設定", + "nodes_host_swap_warning": "凍結依賴 swap — 見 dor doctor", + "nodes_host_swap_hint": "共 {total} · 凍結的沙箱住在這裡", + "nodes_host_disk_label": "資料磁碟", + "nodes_host_disk_missing": "這台主機沒有資料目錄", + "nodes_host_disk_hint": "共 {total} · 剩 {available}" +} diff --git a/packages/console/messages/zh-TW/overview.json b/packages/console/messages/zh-TW/overview.json index 4d9db046..00aa5a4d 100644 --- a/packages/console/messages/zh-TW/overview.json +++ b/packages/console/messages/zh-TW/overview.json @@ -22,22 +22,15 @@ "overview_fleet_chart_desc": "各狀態沙箱數量隨時間的變化 — 活躍往下掉、凍結往上升,就是「閒置即免費」正在發生。", "overview_fleet_chart_empty_title": "此區間內還沒有走勢可畫", "overview_fleet_chart_empty_desc": "daemon 每 30 秒記錄一次取樣,湊滿兩個點就開始畫;daemon 停機的時段沒有取樣,曲線會如實中斷。", - "overview_host_title": "主機健康", - "overview_host_desc": "這台機器的資源水位與走勢", - "overview_host_cpu_cores": "{n} 核心", - "overview_host_cpu_peak": "{n} 核心 · 區間尖峰 {pct}%({ago})", - "overview_host_mem_label": "記憶體", - "overview_host_mem_hint": "共 {total} · 可用 {available}", - "overview_host_swap_unreadable": "此平台讀不到 swap", - "overview_host_swap_unconfigured": "未設定", - "overview_host_swap_warning": "凍結依賴 swap — 見 dor doctor", - "overview_host_swap_hint": "共 {total} · 凍結的沙箱住在這裡", - "overview_host_disk_label": "資料磁碟", - "overview_host_disk_missing": "這台主機沒有資料目錄", - "overview_host_disk_hint": "共 {total} · 剩 {available}", "overview_quick_title": "快速串接", "overview_quick_desc_1": "官方 ", "overview_quick_desc_2": " 套件只要換兩個 URL 就能直連;token 在 daemon 主機的 ", "overview_quick_desc_3": " 裡。", - "overview_quick_all_methods": "所有串接方式" + "overview_quick_all_methods": "所有串接方式", + "overview_nodes_title": "節點", + "overview_nodes_desc": "每台機器此刻的水位;走勢在節點頁", + "overview_nodes_reported": "{total} 台中 {reported} 台已報到,以上數字是下界", + "overview_nodes_running": "執行中 {n}", + "overview_nodes_empty": "還沒有節點報到", + "overview_nodes_all": "全部節點" } diff --git a/packages/console/messages/zh-TW/sandboxes.json b/packages/console/messages/zh-TW/sandboxes.json index a0bdce6a..acca75d3 100644 --- a/packages/console/messages/zh-TW/sandboxes.json +++ b/packages/console/messages/zh-TW/sandboxes.json @@ -112,5 +112,6 @@ "sandboxes_cpu_value": "{v}%(以單一 vCPU 計)", "sandboxes_footer_live": "目前值每 5 秒重新整理,本次讀數取自 {ago}。", "sandboxes_footer_history": "歷史由 daemon 在背景取樣並寫入資料庫;沙箱睡著照樣量測(觀察不喚醒),停止後曲線如實中斷。", - "sandboxes_footer_bucketed": "時間範圍較長,已以每 {n} 分鐘為一桶彙整,每點為桶內尖峰。" + "sandboxes_footer_bucketed": "時間範圍較長,已以每 {n} 分鐘為一桶彙整,每點為桶內尖峰。", + "sandboxes_silent_nodes": "節點 {nodes} 沒有回應,它們上面的沙箱未列出" } diff --git a/packages/console/messages/zh-TW/shell.json b/packages/console/messages/zh-TW/shell.json index 66fd135a..90b4d719 100644 --- a/packages/console/messages/zh-TW/shell.json +++ b/packages/console/messages/zh-TW/shell.json @@ -7,6 +7,7 @@ "shell_nav_sandboxes": "沙箱", "shell_nav_templates": "範本", "shell_nav_api_keys": "API 金鑰", + "shell_nav_nodes": "節點", "shell_nav_domains": "網域", "shell_nav_doctor": "健檢", "shell_nav_settings": "設定", diff --git a/packages/console/project.inlang/settings.json b/packages/console/project.inlang/settings.json index b491f50e..d1ff670c 100644 --- a/packages/console/project.inlang/settings.json +++ b/packages/console/project.inlang/settings.json @@ -25,6 +25,7 @@ "./messages/{locale}/apikeys.json", "./messages/{locale}/templates.json", "./messages/{locale}/domains.json", + "./messages/{locale}/nodes.json", "./messages/{locale}/settings.json", "./messages/{locale}/connect.json" ] diff --git a/packages/console/src/components/nav.ts b/packages/console/src/components/nav.ts index 482e4004..5fb7f4a4 100644 --- a/packages/console/src/components/nav.ts +++ b/packages/console/src/components/nav.ts @@ -1,4 +1,5 @@ import { + CloudServerIcon, DashboardSquare01Icon, GitCommitIcon, Globe02Icon, @@ -44,6 +45,8 @@ export const NAV_GROUPS: Array<{ id: 'ops', label: m.shell_nav_group_ops, items: [ + // 节点页第一(2026-09-15 集群刀 3):运维先看机器,再看钥匙。 + { to: '/nodes', label: m.shell_nav_nodes, icon: CloudServerIcon }, { to: '/api-keys', label: m.shell_nav_api_keys, icon: Key01Icon }, { to: '/domains', label: m.shell_nav_domains, icon: Globe02Icon }, { diff --git a/packages/console/src/features/nodes/components/NodeBadges.tsx b/packages/console/src/features/nodes/components/NodeBadges.tsx new file mode 100644 index 00000000..2d0e44eb --- /dev/null +++ b/packages/console/src/features/nodes/components/NodeBadges.tsx @@ -0,0 +1,71 @@ +import type { NodeView } from '@dormice/shared'; +import { Badge } from '@/components/ui/badge'; +import { ago } from '@/features/sandboxes/format'; +import { cn } from '@/lib/utils'; +import { m } from '@/paraglide/messages'; + +/** + * 可达/不可达:网关的裁决(两个自报间隔内报到过),这里只显示。点的 + * 颜色与沙箱五态的点同一套语言;最后报到时刻挂在 title 里。 + */ +export function ReachableBadge({ node }: { node: NodeView }) { + return ( + + + {node.reachable ? m.nodes_reachable() : m.nodes_unreachable()} + + ); +} + +/** + * 配置漂移标记:节点报的版本对网关当前版本(getConfig.configVersion)。 + * 相等=已同步;落后=琥珀,并说落后几版(下一拍就拉齐,一直落后是那台 + * 应用不上);null=还没有副本,节点在拿到第一份之前不开门。网关版本还 + * 没读到时只显示节点自报的数字,不猜。 + */ +export function ConfigBadge({ + version, + current, +}: { + version: number | null; + current: number | undefined; +}) { + if (version === null) { + return ( + + {m.nodes_config_none()} + + ); + } + if (current === undefined || version > current) { + return v{version}; + } + if (version === current) { + return ( + + {m.nodes_config_synced()} + + ); + } + return ( + + {m.nodes_config_behind({ n: current - version })} + + ); +} diff --git a/packages/console/src/features/overview/components/HostHealthCard.tsx b/packages/console/src/features/nodes/components/NodeHealthCard.tsx similarity index 75% rename from packages/console/src/features/overview/components/HostHealthCard.tsx rename to packages/console/src/features/nodes/components/NodeHealthCard.tsx index 1b02d234..b10f61d1 100644 --- a/packages/console/src/features/overview/components/HostHealthCard.tsx +++ b/packages/console/src/features/nodes/components/NodeHealthCard.tsx @@ -15,14 +15,13 @@ import { CardTitle, } from '@/components/ui/card'; import { Skeleton } from '@/components/ui/skeleton'; +import { Sparkline } from '@/features/overview/components/Sparkline'; +import type { TimelineRangeKey } from '@/features/overview/hooks/useFleetTimeline'; import { ago } from '@/features/sandboxes/format'; import { formatBytes, pctOf } from '@/lib/format'; import { cn } from '@/lib/utils'; import { m } from '@/paraglide/messages'; -import type { TimelineRangeKey } from '../hooks/useFleetTimeline'; -import { useHostMetrics } from '../hooks/useHostMetrics'; -import { useHostTimeline } from '../hooks/useHostTimeline'; -import { Sparkline } from './Sparkline'; +import { useNodeHostMetrics, useNodeHostTimeline } from '../hooks/useNodes'; function HostRow({ icon, @@ -68,30 +67,34 @@ function HostRow({ } /** - * 宿主健康竖卡:主图旁边的机器体征列(对位 openasi 的 ModelStatus 卡)。 - * Swap 和数据盘与 CPU、内存平起平坐,因为在这个平台上它们才是要命的 - * 两个:swap 满了「空闲即免费」就完了,数据盘满了连创建都完了。沙箱 - * 磁盘的账单在页面底部一行 — 这张卡只讲机器本身。 + * 一台节点的机器体征竖卡(2026-09-15 集群刀 3 从总览搬来:总览在多机 + * 世界里没有「这台机器」,宿主的事按节点问)。Swap 和数据盘与 CPU、 + * 内存平起平坐,因为在这个平台上它们才是要命的两个:swap 满了「空闲 + * 即免费」就完了,数据盘满了连创建都完了。沙箱磁盘的账单在总览 — 这 + * 张卡只讲机器本身。 * - * 每行的数值是即时读数(5s 轮询),行内 sparkline 与 CPU 峰值来自 - * getHostMetricsHistory,跟随页面全局档位 — 即时值答"现在怎么样", - * 走势答"这个窗口里发生过什么",峰值从原始行来,分桶抹不平它。 + * 每行的数值是即时读数(5s 轮询,按 nodeId 转到那一台),行内 sparkline + * 与 CPU 峰值来自那台的 getHostMetricsHistory,跟随页面档位 — 即时值答 + * "现在怎么样",走势答"这个窗口里发生过什么",峰值从原始行来,分桶抹 + * 不平它。 */ -export function HostHealthCard({ +export function NodeHealthCard({ + nodeId, range, className, }: { + nodeId: string; range: TimelineRangeKey; className?: string; }) { - const query = useHostMetrics(); - const history = useHostTimeline(range); + const query = useNodeHostMetrics(nodeId); + const history = useNodeHostTimeline(nodeId, range); return ( - {m.overview_host_title()} - {m.overview_host_desc()} + {m.nodes_host_title()} + {m.nodes_host_desc()} @@ -104,8 +107,8 @@ function HostHealthRows({ query, history, }: { - query: ReturnType; - history: ReturnType; + query: ReturnType; + history: ReturnType; }) { if (query.isError) { return ( @@ -162,8 +165,8 @@ function HostHealthRows({ } hint={ peak === null - ? m.overview_host_cpu_cores({ n: host.cpuCount }) - : m.overview_host_cpu_peak({ + ? m.nodes_host_cpu_cores({ n: host.cpuCount }) + : m.nodes_host_cpu_peak({ n: host.cpuCount, pct: Math.round(peak.cpuUsedPct), ago: ago(peak.at), @@ -174,9 +177,9 @@ function HostHealthRows({ /> ) : host.swap.totalBytes === 0 ? ( - {m.overview_host_swap_warning()} + {m.nodes_host_swap_warning()} } /> @@ -206,7 +209,7 @@ function HostHealthRows({ icon={SnowIcon} label="Swap" value={formatBytes(host.swap.usedBytes)} - hint={m.overview_host_swap_hint({ + hint={m.nodes_host_swap_hint({ total: formatBytes(host.swap.totalBytes), })} pct={pctOf(host.swap.usedBytes, host.swap.totalBytes)} @@ -216,18 +219,18 @@ function HostHealthRows({ {dataDisk === null ? ( ) : ( - {m.overview_host_disk_hint({ + {m.nodes_host_disk_hint({ total: formatBytes(dataDisk.totalBytes), available: formatBytes(dataDisk.availableBytes), })} diff --git a/packages/console/src/features/nodes/components/RemoveNodeDialog.tsx b/packages/console/src/features/nodes/components/RemoveNodeDialog.tsx new file mode 100644 index 00000000..6b4d7e15 --- /dev/null +++ b/packages/console/src/features/nodes/components/RemoveNodeDialog.tsx @@ -0,0 +1,59 @@ +import { toast } from 'sonner'; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from '@/components/ui/alert-dialog'; +import { m } from '@/paraglide/messages'; +import { useRemoveNode } from '../hooks/useNodes'; + +/** + * 移除节点的确认:「它永远不回来了」的声明,不是停机。网关自己把关 — + * 还在报到的节点拒 409 并说明为什么(先停 daemon、等两个间隔),原话 + * 就地 toast;误删的节点下次报到自动回来,所以这里不需要二次确认。 + */ +export function RemoveNodeDialog({ + id, + open, + onOpenChange, +}: { + id: string; + open: boolean; + onOpenChange: (open: boolean) => void; +}) { + const mutation = useRemoveNode(); + const remove = () => + mutation.mutate(id, { + onSuccess: ({ removed }) => + toast.success( + removed + ? m.nodes_remove_success({ id }) + : m.nodes_remove_absent({ id }), + ), + onError: (error) => toast.error(error.message), + }); + + return ( + + + + {m.nodes_remove_title({ id })} + + {m.nodes_remove_desc()} + + + + {m.common_cancel()} + + {m.nodes_menu_remove()} + + + + + ); +} diff --git a/packages/console/src/features/nodes/components/SwapDialog.tsx b/packages/console/src/features/nodes/components/SwapDialog.tsx new file mode 100644 index 00000000..45878fc0 --- /dev/null +++ b/packages/console/src/features/nodes/components/SwapDialog.tsx @@ -0,0 +1,118 @@ +import type { NodeView } from '@dormice/shared'; +import { useState } from 'react'; +import { toast } from 'sonner'; +import { Button } from '@/components/ui/button'; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; +import { + Field, + FieldDescription, + FieldError, + FieldGroup, + FieldLabel, +} from '@/components/ui/field'; +import { Input } from '@/components/ui/input'; +import { Spinner } from '@/components/ui/spinner'; +import { m } from '@/paraglide/messages'; +import { useUpdateNodeSettings } from '../hooks/useNodes'; + +/** + * 追加 swap 弹窗:唯一的每节点旋钮(2026-09-14 集群刀 2 从设置页搬来, + * 刀 3 在这里按台恢复)。值住在网关的 nodes 行,那台节点下一拍拿到全份 + * 后 swap.reconcile:增容立即挂上,缩容等宿主重启(运行中的 swap 块绝不 + * 卸载,server/swap.ts 的规矩)。管不了 swap 的节点(非 Linux、fake 执行 + * 器)网关会拒 400 — 读数里 managedSwap 为 null 时这里先不让提交,把原因 + * 说在输入框下面。受控弹窗:触发它的行菜单关闭即卸载。 + */ +export function SwapDialog({ + node, + open, + onOpenChange, +}: { + node: NodeView; + open: boolean; + onOpenChange: (open: boolean) => void; +}) { + const [gb, setGb] = useState(String(node.swapGb)); + const mutation = useUpdateNodeSettings(); + const unsupported = + node.reading !== null && node.reading.managedSwap === null; + const parsed = Number(gb); + const valid = + gb.trim() !== '' && Number.isInteger(parsed) && parsed >= 0 && !unsupported; + + return ( +

{ + onOpenChange(next); + if (next) { + setGb(String(node.swapGb)); + mutation.reset(); + } + }} + > + + + + {m.nodes_swap_dialog_title({ id: node.id })} + + {m.nodes_swap_dialog_desc()} + +
{ + event.preventDefault(); + mutation.mutate( + { id: node.id, swapGb: parsed }, + { + onSuccess: () => { + toast.success( + m.nodes_swap_saved({ id: node.id, gb: parsed }), + ); + onOpenChange(false); + }, + }, + ); + }} + > + + + + {m.nodes_swap_field()} + + setGb(event.target.value)} + disabled={unsupported} + /> + + {unsupported + ? m.nodes_swap_unsupported() + : m.nodes_swap_field_hint()} + + + {mutation.isError && ( + {mutation.error.message} + )} + + + + +
+ + + ); +} diff --git a/packages/console/src/features/nodes/hooks/useNodes.ts b/packages/console/src/features/nodes/hooks/useNodes.ts new file mode 100644 index 00000000..f3e203f1 --- /dev/null +++ b/packages/console/src/features/nodes/hooks/useNodes.ts @@ -0,0 +1,114 @@ +import { + keepPreviousData, + useMutation, + useQuery, + useQueryClient, +} from '@tanstack/react-query'; +import { + rangeSpanMs, + type TimelineRangeKey, +} from '@/features/overview/hooks/useFleetTimeline'; +import { + gatewayHealth, + getHostMetrics, + getHostMetricsHistory, + listNodes, + removeNode, + updateNodeSettings, +} from '@/lib/api'; + +/** + * The nodes as the gateway knows them, one file (the sandboxes' rule): + * every read and write about a node. The list is what the gateway holds + * in memory from the check-ins — polling it costs no node anything, so + * 5s is the overview's cadence, not the sandbox list's 2s. + */ +export function useNodes() { + return useQuery({ + queryKey: ['nodes'], + queryFn: listNodes, + refetchInterval: 5000, + retry: false, + }); +} + +/** One node, selected from the same list cache — the list is the gateway's one read about nodes. */ +export function useNode(id: string) { + const query = useNodes(); + return { ...query, node: query.data?.nodes.find((n) => n.id === id) }; +} + +/** + * The gateway's own build, off /healthz (open, no token): the nodes page + * marks a node whose build differs from it. A build changes when the + * gateway restarts — refetch on focus is plenty. + */ +export function useGatewayHealth() { + return useQuery({ + queryKey: ['gatewayHealth'], + queryFn: gatewayHealth, + staleTime: 30_000, + retry: false, + }); +} + +/** + * One machine's live reading, by node: CPU is a delta between consecutive + * requests, and 5s keeps that reading meaningful where 2s would mostly + * sample noise. Only mounted on that node's page — a poll per node per + * page open, never per node per fleet. + */ +export function useNodeHostMetrics(nodeId: string) { + return useQuery({ + queryKey: ['hostMetrics', nodeId], + queryFn: () => getHostMetrics(nodeId), + refetchInterval: 5000, + retry: false, + }); +} + +/** + * One machine's history, following the page's range — the fleet + * timeline's cadence exactly: 30s (the node's sampling interval; faster + * only rereads the same rows), the window computed per queryFn so a + * page left open slides, keepPreviousData across a range switch. + */ +export function useNodeHostTimeline(nodeId: string, range: TimelineRangeKey) { + return useQuery({ + queryKey: ['hostTimeline', nodeId, range], + queryFn: () => { + const end = Date.now(); + const start = end - rangeSpanMs(range); + return getHostMetricsHistory( + nodeId, + new Date(start).toISOString(), + new Date(end).toISOString(), + ); + }, + refetchInterval: 30_000, + retry: false, + placeholderData: keepPreviousData, + }); +} + +/** The one per-node knob; the config version counts up with it, so the settings page's version refreshes too. */ +export function useUpdateNodeSettings() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (args: { id: string; swapGb: number }) => + updateNodeSettings(args.id, args.swapGb), + onSettled: () => { + void queryClient.invalidateQueries({ queryKey: ['nodes'] }); + void queryClient.invalidateQueries({ queryKey: ['config'] }); + }, + }); +} + +/** Removal is the gateway's ruling: a node still checking in is refused (409) with the reason, relayed as it came. */ +export function useRemoveNode() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (id: string) => removeNode(id), + onSettled: () => queryClient.invalidateQueries({ queryKey: ['nodes'] }), + }); +} diff --git a/packages/console/src/features/nodes/pages/NodeDetailPage.tsx b/packages/console/src/features/nodes/pages/NodeDetailPage.tsx new file mode 100644 index 00000000..828073ee --- /dev/null +++ b/packages/console/src/features/nodes/pages/NodeDetailPage.tsx @@ -0,0 +1,150 @@ +import { Link, useParams } from '@tanstack/react-router'; +import { useState } from 'react'; +import { Button } from '@/components/ui/button'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { + Empty, + EmptyContent, + EmptyDescription, + EmptyHeader, + EmptyTitle, +} from '@/components/ui/empty'; +import { RangeSwitcher } from '@/features/overview/components/RangeSwitcher'; +import type { TimelineRangeKey } from '@/features/overview/hooks/useFleetTimeline'; +import { useConfig } from '@/features/settings/hooks/useConfig'; +import { formatDateTime } from '@/lib/datetime'; +import { m } from '@/paraglide/messages'; +import { ConfigBadge, ReachableBadge } from '../components/NodeBadges'; +import { NodeHealthCard } from '../components/NodeHealthCard'; +import { useNode } from '../hooks/useNodes'; + +/** + * 一台节点的页面:机器体征卡(即时读数 + 走势 + 窗口峰值,按 nodeId + * 问那一台)配一张信息卡(端点、加入时刻、报到间隔、版本、配置版本、 + * swap 目标与现挂)。基础信息从 5 秒轮询的节点列表缓存里选(列表是 + * 网关关于节点的唯一读);档位切换器与总览同款,驱动体征卡的走势。 + * 纯观察:动作(swap、移除)在列表页的行菜单。 + */ +export function NodeDetailPage() { + const { id } = useParams({ from: '/_app/nodes/$id' }); + const { node, isSuccess } = useNode(id); + const config = useConfig(); + const [range, setRange] = useState('24h'); + + if (!node) { + return isSuccess ? ( + + + {m.nodes_detail_not_found_title({ id })} + {m.nodes_detail_not_found_desc()} + + + + + + ) : null; + } + + const reading = node.reading; + return ( +
+
+
+

{node.id}

+ +
+ +
+
+ + + + {m.nodes_detail_info_title()} + + +
+ + {node.endpoint} + + + {formatDateTime(node.addedAt)} + + + {node.intervalSeconds === null + ? '—' + : m.nodes_detail_interval_value({ n: node.intervalSeconds })} + + + {node.build === null ? ( + m.common_unknown() + ) : ( + + {node.build.commit} + + )} + + + + + + {reading === null + ? m.nodes_no_reading() + : m.nodes_sandboxes_cell({ + active: reading.sandboxes.byState.active, + frozen: reading.sandboxes.byState.frozen, + total: reading.sandboxes.total, + })} + + + {reading !== null && reading.managedSwap === null + ? m.nodes_swap_unsupported() + : m.nodes_swap_target({ target: node.swapGb })} + + + {reading === null || reading.managedSwap === null + ? '—' + : reading.managedSwap.activeGb === node.swapGb + ? m.nodes_swap_managed({ + active: reading.managedSwap.activeGb, + }) + : `${m.nodes_swap_managed({ active: reading.managedSwap.activeGb })} · ${m.nodes_swap_reconciling()}`} + +
+ + +
+
+ ); +} + +/** 键值一行,工作台信息卡的同款解剖:左键右值,值 mono 截断。 */ +function InfoRow({ + label, + children, +}: { + label: string; + children: React.ReactNode; +}) { + return ( +
+
{label}
+
{children}
+
+ ); +} diff --git a/packages/console/src/features/nodes/pages/NodesPage.tsx b/packages/console/src/features/nodes/pages/NodesPage.tsx new file mode 100644 index 00000000..0d2902af --- /dev/null +++ b/packages/console/src/features/nodes/pages/NodesPage.tsx @@ -0,0 +1,371 @@ +import type { NodeView } from '@dormice/shared'; +import { + CloudServerIcon, + Delete02Icon, + MoreHorizontalIcon, + SnowIcon, +} from '@hugeicons/core-free-icons'; +import { HugeiconsIcon } from '@hugeicons/react'; +import { Link } from '@tanstack/react-router'; +import { useState } from 'react'; +import { DataTable } from '@/components/DataTable'; +import { Meter } from '@/components/Meter'; +import { Alert, AlertDescription } from '@/components/ui/alert'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu'; +import { + Empty, + EmptyDescription, + EmptyHeader, + EmptyMedia, + EmptyTitle, +} from '@/components/ui/empty'; +import { Spinner } from '@/components/ui/spinner'; +import { + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { useConfig } from '@/features/settings/hooks/useConfig'; +import { formatDateTime } from '@/lib/datetime'; +import { formatBytes, pctOf } from '@/lib/format'; +import { cn } from '@/lib/utils'; +import { m } from '@/paraglide/messages'; +import { ConfigBadge, ReachableBadge } from '../components/NodeBadges'; +import { RemoveNodeDialog } from '../components/RemoveNodeDialog'; +import { SwapDialog } from '../components/SwapDialog'; +import { useGatewayHealth, useNodes } from '../hooks/useNodes'; + +/** + * 节点页(2026-09-15 集群刀 3,讨论稿 #24):每台机器一行 — 可达、版本、 + * CPU/内存/数据盘三条量表、沙箱计数、配置漂移 — 读的是网关手里每台最近 + * 一次报到的读数(listNodes,5s 轮询,不扇出到任何节点);页头一张网关卡 + * (它自己的构建、几台节点、配置第几版)。走势与峰值在每台的详情页。 + * 行操作:追加 swap(唯一的每节点旋钮)与移除(只对不可达的节点开放 — + * 网关对还在报到的节点拒 409,菜单项先把这条规矩写在脸上)。 + * + * 列宽军备(RULES/前端.md):max-w-6xl 里九列刚好,swap 用量不进表 — + * 详情页有它,弹窗里也看得到。 + */ +export function NodesPage() { + const query = useNodes(); + const gateway = useGatewayHealth(); + const config = useConfig(); + const nodes = [...(query.data?.nodes ?? [])].sort((a, b) => + a.id.localeCompare(b.id), + ); + const currentVersion = config.data?.configVersion; + const gatewayCommit = gateway.data?.build?.commit; + + return ( +
+
+

{m.nodes_page_title()}

+
+ + n.reachable).length} + version={currentVersion} + /> + + {query.isError && ( + + {query.error.message} + + )} + + {query.isPending && ( +
+ {m.nodes_loading()} +
+ )} + + {query.isSuccess && nodes.length === 0 && ( + + + + + + {m.nodes_empty_title()} + {m.nodes_empty_description()} + + + )} + + {nodes.length > 0 && ( + + + + {m.nodes_col_node()} + {m.nodes_col_status()} + {m.nodes_col_build()} + {m.nodes_col_cpu()} + + {m.nodes_col_memory()} + + {m.nodes_col_disk()} + {m.nodes_col_sandboxes()} + {m.nodes_col_config()} + + {m.nodes_col_actions()} + + + + + {nodes.map((node) => ( + + ))} + + + )} +
+ ); +} + +/** + * 网关自己的一张卡:构建、节点几台几台可达、配置第几版 — 网关是舰队 + * 唯一的门,它的版本是节点「与网关不同」标记的参照。 + */ +function GatewayCard({ + build, + loaded, + total, + reachable, + version, +}: { + build: { commit: string; title: string; committedAt: string } | null; + loaded: boolean; + total: number; + reachable: number; + version: number | undefined; +}) { + return ( + + + {m.nodes_gateway_title()} + {m.nodes_gateway_desc()} + + + + {build + ? m.nodes_gateway_build({ commit: build.commit }) + : loaded + ? m.nodes_gateway_build_unknown() + : '—'} + + + {m.nodes_gateway_fleet({ total, reachable })} + + + {version === undefined ? '—' : m.nodes_gateway_config({ version })} + + + + ); +} + +/** + * 资源列的一格:量表是主角,数字是 text-xs 注脚(沙箱列表的主次对调, + * 2026-07-18 用户拍板);null = 这台还没报过读数。 + */ +function MeterCell({ pct, value }: { pct: number | null; value: string }) { + return ( + = 90 + ? 'text-red-600 dark:text-red-400' + : pct !== null && pct >= 75 + ? 'text-amber-600 dark:text-amber-400' + : 'text-muted-foreground', + )} + > + {value} +
+ +
+ + ); +} + +function NodeRow({ + node, + currentVersion, + gatewayCommit, +}: { + node: NodeView; + currentVersion: number | undefined; + gatewayCommit: string | undefined; +}) { + const reading = node.reading; + const memUsed = + reading === null + ? null + : reading.host.memTotalBytes - reading.host.memAvailableBytes; + return ( + + + + {node.id} + + + + + + + {node.build === null ? ( + {m.common_unknown()} + ) : ( + + {node.build.commit} + {gatewayCommit !== undefined && + node.build.commit !== gatewayCommit && ( + + {m.nodes_build_differs()} + + )} + + )} + + + + + + {reading === null + ? m.nodes_no_reading() + : m.nodes_sandboxes_cell({ + active: reading.sandboxes.byState.active, + frozen: reading.sandboxes.byState.frozen, + total: reading.sandboxes.total, + })} + + + + + + + + + ); +} + +/** + * 行操作收进「⋯」菜单;两个弹窗挂在菜单外受控 — 菜单关闭即卸载, + * 放里面会跟着消失(RULES/前端.md)。 + */ +function NodeRowMenu({ node }: { node: NodeView }) { + const [swapOpen, setSwapOpen] = useState(false); + const [removeOpen, setRemoveOpen] = useState(false); + return ( + <> + + + + + } + /> + + setSwapOpen(true)} + > + + {m.nodes_menu_swap()} + + setRemoveOpen(true)} + > + + {m.nodes_menu_remove()} + + + + + + + ); +} diff --git a/packages/console/src/features/overview/components/FleetStatCards.tsx b/packages/console/src/features/overview/components/FleetStatCards.tsx index 34777c0e..d7e86fea 100644 --- a/packages/console/src/features/overview/components/FleetStatCards.tsx +++ b/packages/console/src/features/overview/components/FleetStatCards.tsx @@ -1,12 +1,12 @@ import { Alert, AlertDescription } from '@/components/ui/alert'; import { m } from '@/paraglide/messages'; import { fullClock } from '../format'; +import { useFleetMetrics } from '../hooks/useFleetMetrics'; import { TIMELINE_RANGES, type TimelineRangeKey, useFleetTimeline, } from '../hooks/useFleetTimeline'; -import { useHostMetrics } from '../hooks/useHostMetrics'; import { SandboxDisksCard } from './SandboxDisksCard'; import { Sparkline } from './Sparkline'; import { StatCard, StatCardSkeleton } from './StatCard'; @@ -15,12 +15,13 @@ import { StatCard, StatCardSkeleton } from './StatCard'; * 舰队四卡(openasi 顶排版式,2026-07-16 沙箱磁盘上顶):当前活跃 * (5 秒一刷的快照 + 窗口内活跃数 sparkline)、窗口峰值、总数、 * 沙箱磁盘账单。容量上限随讨论稿 #23 删(2026-09-14):账本行数不是 - * 资源,数据盘水位才是——它有自己的卡。当前值来自 /getHostMetrics;峰值与 sparkline 来自 - * /getFleetStateHistory — 网关每次节点报到落一行,峰值由原始行现算, - * 分桶抹不掉它。档位由页头的全局切换器驱动。 + * 资源,数据盘水位才是——它有自己的卡。当前值来自 /getFleetMetrics + * (2026-09-15 刀 3:网关把每台节点最近一次报到的读数加总,不扇出); + * 峰值与 sparkline 来自 /getFleetStateHistory — 网关每次节点报到落一 + * 行,峰值由原始行现算,分桶抹不掉它。档位由页头的全局切换器驱动。 */ export function FleetStatCards({ range }: { range: TimelineRangeKey }) { - const host = useHostMetrics(); + const host = useFleetMetrics(); const timeline = useFleetTimeline(range); const rangeLabel = TIMELINE_RANGES.find((r) => r.key === range)?.label() ?? range; diff --git a/packages/console/src/features/overview/components/NodesCard.tsx b/packages/console/src/features/overview/components/NodesCard.tsx new file mode 100644 index 00000000..bf3b608e --- /dev/null +++ b/packages/console/src/features/overview/components/NodesCard.tsx @@ -0,0 +1,167 @@ +import { Link } from '@tanstack/react-router'; +import { Meter } from '@/components/Meter'; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from '@/components/ui/card'; +import { Skeleton } from '@/components/ui/skeleton'; +import { useNodes } from '@/features/nodes/hooks/useNodes'; +import { formatBytes, pctOf } from '@/lib/format'; +import { cn } from '@/lib/utils'; +import { m } from '@/paraglide/messages'; + +/** + * 节点竖卡:主图旁边每台机器一行(2026-09-15 集群刀 3,取代宿主健康 + * 卡 — 舰队里没有「这台机器」)。每行三条量表 CPU/内存/数据盘 + 运行中 + * 沙箱数,读的是网关手里每台最近一次报到的读数(listNodes,5s 轮询, + * 不扇出到任何节点);走势与峰值不在这里 — 点进节点页,那里按台问。 + * 单机也是一行:一台节点的舰队,产品故事不因机器数变形。 + * + * 有节点还没报到时,卡头改说「N 台中 M 台已报到」— 顶排四卡加的是 + * 已报到那几台的数,下界要说出来。 + */ +export function NodesCard({ className }: { className?: string }) { + const query = useNodes(); + const nodes = [...(query.data?.nodes ?? [])].sort((a, b) => + a.id.localeCompare(b.id), + ); + const reported = nodes.filter((n) => n.reading !== null).length; + + return ( + + + {m.overview_nodes_title()} + + {nodes.length > 0 && reported < nodes.length + ? m.overview_nodes_reported({ total: nodes.length, reported }) + : m.overview_nodes_desc()} + + + + {query.isError ? ( +
+ {query.error.message} +
+ ) : !query.data ? ( + ['a', 'b'].map((slot) => ( +
+ + + + +
+ )) + ) : nodes.length === 0 ? ( +
+ {m.overview_nodes_empty()} +
+ ) : ( + nodes.map((node) => ) + )} + + + + {m.overview_nodes_all()} + + + + ); +} + +function NodeRow({ + node, +}: { + node: NonNullable['data']>['nodes'][number]; +}) { + const reading = node.reading; + const memUsed = + reading === null + ? null + : reading.host.memTotalBytes - reading.host.memAvailableBytes; + return ( +
+
+ + + {node.id} + + + {reading === null + ? m.nodes_no_reading() + : m.overview_nodes_running({ + n: reading.sandboxes.byState.active, + })} + +
+ + + +
+ ); +} + +function MiniMeter({ + label, + pct, + value, +}: { + label: string; + pct: number | null; + value: string; +}) { + return ( +
+ {label} +
+ +
+ + {value} + +
+ ); +} diff --git a/packages/console/src/features/overview/components/SandboxDisksCard.tsx b/packages/console/src/features/overview/components/SandboxDisksCard.tsx index 8a099449..0a07abfd 100644 --- a/packages/console/src/features/overview/components/SandboxDisksCard.tsx +++ b/packages/console/src/features/overview/components/SandboxDisksCard.tsx @@ -2,16 +2,17 @@ import { Meter } from '@/components/Meter'; import { Alert, AlertDescription } from '@/components/ui/alert'; import { formatBytes, pctOf } from '@/lib/format'; import { m } from '@/paraglide/messages'; -import { useHostMetrics } from '../hooks/useHostMetrics'; +import { useFleetMetrics } from '../hooks/useFleetMetrics'; import { StatCard, StatCardSkeleton } from './StatCard'; /** * 沙箱磁盘账单卡:这群沙箱的盘许诺了多少、实际占了多少 — 稀疏镜像 * 只为真实内容付费,这两个数的差就是超卖的空间。机器本身的体征在 - * 宿主健康卡;这张卡讲的是沙箱群欠机器多少。 + * 各节点页;这张卡讲的是全舰队的沙箱欠机器多少(网关把每台报到读数 + * 里的账单加总)。 */ export function SandboxDisksCard() { - const query = useHostMetrics(); + const query = useFleetMetrics(); if (query.isError) { return ( diff --git a/packages/console/src/features/overview/hooks/useFleetMetrics.ts b/packages/console/src/features/overview/hooks/useFleetMetrics.ts new file mode 100644 index 00000000..cf62ad87 --- /dev/null +++ b/packages/console/src/features/overview/hooks/useFleetMetrics.ts @@ -0,0 +1,17 @@ +import { useQuery } from '@tanstack/react-query'; +import { getFleetMetrics } from '@/lib/api'; + +/** + * The fleet's present — the figures that add up, from the nodes' last + * check-ins. The gateway answers from memory, so this poll reaches no + * node: the one observer of the fleet must not be its heaviest caller + * (design record #24). 5s like the old host poll it replaces. + */ +export function useFleetMetrics() { + return useQuery({ + queryKey: ['fleetMetrics'], + queryFn: getFleetMetrics, + refetchInterval: 5000, + retry: false, + }); +} diff --git a/packages/console/src/features/overview/hooks/useHostMetrics.ts b/packages/console/src/features/overview/hooks/useHostMetrics.ts deleted file mode 100644 index 1b9597d5..00000000 --- a/packages/console/src/features/overview/hooks/useHostMetrics.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { useQuery } from '@tanstack/react-query'; -import { getHostMetrics } from '@/lib/api'; - -export function useHostMetrics() { - return useQuery({ - queryKey: ['hostMetrics'], - queryFn: getHostMetrics, - // Slower than the sandbox list on purpose: CPU usage is a delta between - // consecutive requests, and a 5s interval keeps that reading meaningful - // where a 2s one would mostly sample noise. - refetchInterval: 5000, - retry: false, - }); -} diff --git a/packages/console/src/features/overview/hooks/useHostTimeline.ts b/packages/console/src/features/overview/hooks/useHostTimeline.ts deleted file mode 100644 index 6d703e1e..00000000 --- a/packages/console/src/features/overview/hooks/useHostTimeline.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { keepPreviousData, useQuery } from '@tanstack/react-query'; -import { getHostMetricsHistory } from '@/lib/api'; -import { rangeSpanMs, type TimelineRangeKey } from './useFleetTimeline'; - -/** - * 宿主机的走势,跟随总览页的全局档位 — 节奏与 useFleetTimeline 完全同款: - * 30 秒一刷(同 daemon 采样间隔,更快只是重复读同一批样本)、窗口在 - * queryFn 里现算随时间滑动、切档位时 keepPreviousData 顶住不塌骨架。 - */ -export function useHostTimeline(range: TimelineRangeKey) { - return useQuery({ - queryKey: ['hostTimeline', range], - queryFn: () => { - const end = Date.now(); - const start = end - rangeSpanMs(range); - return getHostMetricsHistory( - new Date(start).toISOString(), - new Date(end).toISOString(), - ); - }, - refetchInterval: 30_000, - retry: false, - placeholderData: keepPreviousData, - }); -} diff --git a/packages/console/src/features/overview/pages/OverviewPage.tsx b/packages/console/src/features/overview/pages/OverviewPage.tsx index 6a1a3951..63f4ad85 100644 --- a/packages/console/src/features/overview/pages/OverviewPage.tsx +++ b/packages/console/src/features/overview/pages/OverviewPage.tsx @@ -2,17 +2,19 @@ import { useState } from 'react'; import { m } from '@/paraglide/messages'; import { FleetChart } from '../components/FleetChart'; import { FleetStatCards } from '../components/FleetStatCards'; -import { HostHealthCard } from '../components/HostHealthCard'; +import { NodesCard } from '../components/NodesCard'; import { QuickConnectCard } from '../components/QuickConnectCard'; import { RangeSwitcher } from '../components/RangeSwitcher'; import type { TimelineRangeKey } from '../hooks/useFleetTimeline'; /** - * 回答三个问题:沙箱群现在多忙、一段时间以来多忙(daemon 采样器落库 - * 的舰队时间线,含窗口峰值)、这台机器还好吗。版式仿 openasi 仪表盘 + * 回答三个问题:沙箱群现在多忙、一段时间以来多忙(网关按报到落库的 + * 舰队走势,含窗口峰值)、机器们还好吗。版式仿 openasi 仪表盘 * (2026-07-16 用户拍板对齐页头与容器):max-w-6xl 限宽居中,页头一行 * 标题 + 全局档位,顶排四张统计卡(沙箱磁盘也在其中),主图区 3:1 - * (走势图配宿主健康竖卡),底部快速接入(「换两个 URL 直连」放上第一屏)。 + * (走势图配节点竖卡 — 2026-09-15 刀 3 由宿主健康卡改来:舰队里没有 + * 「这台机器」,每台一行、点进节点页看走势),底部快速接入(「换两个 + * URL 直连」放上第一屏)。 */ export function OverviewPage() { const [range, setRange] = useState('24h'); @@ -26,7 +28,7 @@ export function OverviewPage() {
- +
diff --git a/packages/console/src/features/sandboxes/hooks/useSandboxes.ts b/packages/console/src/features/sandboxes/hooks/useSandboxes.ts index 73a564d5..831df3a4 100644 --- a/packages/console/src/features/sandboxes/hooks/useSandboxes.ts +++ b/packages/console/src/features/sandboxes/hooks/useSandboxes.ts @@ -83,9 +83,9 @@ export function useSandboxMetricsHistory(name: string, spanMs: number) { * 5 秒一拍与列表的 2 秒分开定:daemon 侧一次 docker stats 读数约一秒, * 这口锅比读 SQLite 贵。只在列表页挂载时跑,页面一关轮询即停。 */ -export function useFleetMetrics() { +export function useListSandboxMetrics() { return useQuery({ - queryKey: ['fleet-metrics'], + queryKey: ['sandbox-metrics-list'], queryFn: listSandboxMetrics, refetchInterval: 5000, retry: false, diff --git a/packages/console/src/features/sandboxes/pages/SandboxesPage.tsx b/packages/console/src/features/sandboxes/pages/SandboxesPage.tsx index 1a76f270..bf10cf4f 100644 --- a/packages/console/src/features/sandboxes/pages/SandboxesPage.tsx +++ b/packages/console/src/features/sandboxes/pages/SandboxesPage.tsx @@ -69,7 +69,7 @@ import { SandboxStateBadge } from '../components/SandboxStateBadge'; import { UpgradableBadge } from '../components/UpgradableBadge'; import { ago, stateLabel } from '../format'; import { - useFleetMetrics, + useListSandboxMetrics, useSandboxes, useSandboxImages, } from '../hooks/useSandboxes'; @@ -313,7 +313,7 @@ export function SandboxesPage() { const query = useSandboxes(); const sandboxes = query.data?.sandboxes ?? []; // 资源快照批量拉(一个请求管全表);读不到就整列出 —,不挡列表本身。 - const fleet = useFleetMetrics(); + const fleet = useListSandboxMetrics(); const metricsOf = useMemo( () => new Map( @@ -321,6 +321,8 @@ export function SandboxesPage() { ), [fleet.data], ); + // 网关的列表可能缺一台节点(它没应答):说出来,不许当成整个舰队。 + const silent = query.data?.silent ?? []; // 镜像血统批量拉,同一口径:拉不到就不出标记,不挡列表。 const images = useSandboxImages(); const lineageOf = useMemo( @@ -395,6 +397,20 @@ export function SandboxesPage() { + {silent.length > 0 && ( + + `${s.nodeId}: ${s.why}`).join('\n')} + > + {m.sandboxes_silent_nodes({ + nodes: silent + .map((s) => s.nodeId) + .join(m.common_name_separator()), + })} + + + )} +
diff --git a/packages/console/src/lib/api.ts b/packages/console/src/lib/api.ts index b038b3e8..0b27feb7 100644 --- a/packages/console/src/lib/api.ts +++ b/packages/console/src/lib/api.ts @@ -6,6 +6,7 @@ import type { CheckUpgradeResponse, CreateApiKeyResponse, GetConfigResponse, + GetFleetMetricsResponse, GetFleetStateHistoryResponse, GetHostMetricsHistoryResponse, GetIngressResponse, @@ -14,13 +15,16 @@ import type { GetUpgradeStatusResponse, HostMetricsResponse, LifecyclePolicyOverride, + ListNodesResponse, ListSandboxesResponse, ListSandboxImagesResponse, ListSandboxMetricsResponse, RegisterTemplateResponse, + RemoveNodeResponse, Sandbox, SetIngressResponse, Template, + UpdateNodeSettingsResponse, UpdateSettingsRequest, UpdateSettingsResponse, } from '@dormice/shared'; @@ -142,9 +146,18 @@ export const logout = () => export const listSandboxes = () => rpc('/listSandboxes'); -// The host-level observation window: machine health plus fleet aggregates. -// Pure observation — the daemon wakes nothing to answer it. -export const getHostMetrics = () => rpc('/getHostMetrics'); +// One machine's observation window: its health, its ledger's census, its +// disks' bill. Named by node since the third cut (2026-09-15): the gateway +// forwards it to that node. Pure observation — nothing wakes to answer it. +export const getHostMetrics = (nodeId: string) => + rpc('/getHostMetrics', { nodeId }); + +// The fleet's figures that add up — how many nodes, the census by state, +// the disks' bill — from the nodes' last check-ins, answered by the gateway +// without asking a node. `nodes.reported` says how many nodes the sums +// cover; until every node has reported they are a lower bound. +export const getFleetMetrics = () => + rpc('/getFleetMetrics'); // One sandbox's point-in-time reading. Same principle: a frozen sandbox is // measured as it sleeps, a stopped one answers sample: null — never woken. @@ -177,11 +190,50 @@ export const getSandboxMetricsHistory = ( export const getFleetStateHistory = (start: string, end: string) => rpc('/getFleetStateHistory', { start, end }); -// The machine's sampled past — the host health card's trend food. Buckets +// One machine's sampled past — the node health card's trend food. Buckets // keep each field's worst case (max usage, min available) so spikes // survive; peak carries the window's raw CPU high point. -export const getHostMetricsHistory = (start: string, end: string) => - rpc('/getHostMetricsHistory', { start, end }); +export const getHostMetricsHistory = ( + nodeId: string, + start: string, + end: string, +) => + rpc('/getHostMetricsHistory', { + nodeId, + start, + end, + }); + +// The nodes as the gateway knows them: every node that ever checked in, +// with its last reading, build and configuration version (the drift +// marker is this against getConfig's configVersion). Costs no node +// anything — the gateway answers from memory. +export const listNodes = () => rpc('/listNodes'); + +// The one per-node knob: how much swap that node's daemon manages on its +// own data disk. Applied by the node at its next check-in. +export const updateNodeSettings = (id: string, swapGb: number) => + rpc('/updateNodeSettings', { id, swapGb }); + +// The operator's word that a node is gone for good; the gateway refuses it +// (409) for a node still checking in — the message is relayed as it came. +export const removeNode = (id: string) => + rpc('/removeNode', { id }); + +// The gateway's liveness answer, open by design: its build identity, so +// the nodes page can say which node runs a build other than the gateway's. +export async function gatewayHealth(): Promise<{ + status: 'ok'; + build: { commit: string; title: string; committedAt: string } | null; +}> { + const res = await fetch('/healthz'); + if (!res.ok) + throw new ApiError(`/healthz failed with ${res.status}`, res.status); + return (await res.json()) as { + status: 'ok'; + build: { commit: string; title: string; committedAt: string } | null; + }; +} // Every sandbox's image lineage: the born image of the current shell (null // when no shell exists), the image the next shell would boot, and whether a diff --git a/packages/console/src/routeTree.gen.ts b/packages/console/src/routeTree.gen.ts index 2441d7eb..03d80574 100644 --- a/packages/console/src/routeTree.gen.ts +++ b/packages/console/src/routeTree.gen.ts @@ -20,8 +20,10 @@ import { Route as AppDoctorRouteImport } from './routes/_app/doctor' import { Route as AppConnectRouteImport } from './routes/_app/connect' import { Route as AppApiKeysRouteImport } from './routes/_app/api-keys' import { Route as AppSandboxesIndexRouteImport } from './routes/_app/sandboxes/index' +import { Route as AppNodesIndexRouteImport } from './routes/_app/nodes/index' import { Route as AppTempSplatRouteImport } from './routes/_app/temp.$' import { Route as AppSandboxesNameRouteImport } from './routes/_app/sandboxes/$name' +import { Route as AppNodesIdRouteImport } from './routes/_app/nodes/$id' const LoginRoute = LoginRouteImport.update({ id: '/login', @@ -77,6 +79,11 @@ const AppSandboxesIndexRoute = AppSandboxesIndexRouteImport.update({ path: '/sandboxes/', getParentRoute: () => AppRoute, } as any) +const AppNodesIndexRoute = AppNodesIndexRouteImport.update({ + id: '/nodes/', + path: '/nodes/', + getParentRoute: () => AppRoute, +} as any) const AppTempSplatRoute = AppTempSplatRouteImport.update({ id: '/temp/$', path: '/temp/$', @@ -87,6 +94,11 @@ const AppSandboxesNameRoute = AppSandboxesNameRouteImport.update({ path: '/sandboxes/$name', getParentRoute: () => AppRoute, } as any) +const AppNodesIdRoute = AppNodesIdRouteImport.update({ + id: '/nodes/$id', + path: '/nodes/$id', + getParentRoute: () => AppRoute, +} as any) export interface FileRoutesByFullPath { '/': typeof AppIndexRoute @@ -98,8 +110,10 @@ export interface FileRoutesByFullPath { '/settings': typeof AppSettingsRoute '/templates': typeof AppTemplatesRoute '/version': typeof AppVersionRoute + '/nodes/$id': typeof AppNodesIdRoute '/sandboxes/$name': typeof AppSandboxesNameRoute '/temp/$': typeof AppTempSplatRoute + '/nodes/': typeof AppNodesIndexRoute '/sandboxes/': typeof AppSandboxesIndexRoute } export interface FileRoutesByTo { @@ -112,8 +126,10 @@ export interface FileRoutesByTo { '/templates': typeof AppTemplatesRoute '/version': typeof AppVersionRoute '/': typeof AppIndexRoute + '/nodes/$id': typeof AppNodesIdRoute '/sandboxes/$name': typeof AppSandboxesNameRoute '/temp/$': typeof AppTempSplatRoute + '/nodes': typeof AppNodesIndexRoute '/sandboxes': typeof AppSandboxesIndexRoute } export interface FileRoutesById { @@ -128,8 +144,10 @@ export interface FileRoutesById { '/_app/templates': typeof AppTemplatesRoute '/_app/version': typeof AppVersionRoute '/_app/': typeof AppIndexRoute + '/_app/nodes/$id': typeof AppNodesIdRoute '/_app/sandboxes/$name': typeof AppSandboxesNameRoute '/_app/temp/$': typeof AppTempSplatRoute + '/_app/nodes/': typeof AppNodesIndexRoute '/_app/sandboxes/': typeof AppSandboxesIndexRoute } export interface FileRouteTypes { @@ -144,8 +162,10 @@ export interface FileRouteTypes { | '/settings' | '/templates' | '/version' + | '/nodes/$id' | '/sandboxes/$name' | '/temp/$' + | '/nodes/' | '/sandboxes/' fileRoutesByTo: FileRoutesByTo to: @@ -158,8 +178,10 @@ export interface FileRouteTypes { | '/templates' | '/version' | '/' + | '/nodes/$id' | '/sandboxes/$name' | '/temp/$' + | '/nodes' | '/sandboxes' id: | '__root__' @@ -173,8 +195,10 @@ export interface FileRouteTypes { | '/_app/templates' | '/_app/version' | '/_app/' + | '/_app/nodes/$id' | '/_app/sandboxes/$name' | '/_app/temp/$' + | '/_app/nodes/' | '/_app/sandboxes/' fileRoutesById: FileRoutesById } @@ -262,6 +286,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AppSandboxesIndexRouteImport parentRoute: typeof AppRoute } + '/_app/nodes/': { + id: '/_app/nodes/' + path: '/nodes' + fullPath: '/nodes/' + preLoaderRoute: typeof AppNodesIndexRouteImport + parentRoute: typeof AppRoute + } '/_app/temp/$': { id: '/_app/temp/$' path: '/temp/$' @@ -276,6 +307,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AppSandboxesNameRouteImport parentRoute: typeof AppRoute } + '/_app/nodes/$id': { + id: '/_app/nodes/$id' + path: '/nodes/$id' + fullPath: '/nodes/$id' + preLoaderRoute: typeof AppNodesIdRouteImport + parentRoute: typeof AppRoute + } } } @@ -288,8 +326,10 @@ interface AppRouteChildren { AppTemplatesRoute: typeof AppTemplatesRoute AppVersionRoute: typeof AppVersionRoute AppIndexRoute: typeof AppIndexRoute + AppNodesIdRoute: typeof AppNodesIdRoute AppSandboxesNameRoute: typeof AppSandboxesNameRoute AppTempSplatRoute: typeof AppTempSplatRoute + AppNodesIndexRoute: typeof AppNodesIndexRoute AppSandboxesIndexRoute: typeof AppSandboxesIndexRoute } @@ -302,8 +342,10 @@ const AppRouteChildren: AppRouteChildren = { AppTemplatesRoute: AppTemplatesRoute, AppVersionRoute: AppVersionRoute, AppIndexRoute: AppIndexRoute, + AppNodesIdRoute: AppNodesIdRoute, AppSandboxesNameRoute: AppSandboxesNameRoute, AppTempSplatRoute: AppTempSplatRoute, + AppNodesIndexRoute: AppNodesIndexRoute, AppSandboxesIndexRoute: AppSandboxesIndexRoute, } diff --git a/packages/console/src/routes/_app/nodes/$id.tsx b/packages/console/src/routes/_app/nodes/$id.tsx new file mode 100644 index 00000000..2df77e8c --- /dev/null +++ b/packages/console/src/routes/_app/nodes/$id.tsx @@ -0,0 +1,6 @@ +import { createFileRoute } from '@tanstack/react-router'; +import { NodeDetailPage } from '@/features/nodes/pages/NodeDetailPage'; + +export const Route = createFileRoute('/_app/nodes/$id')({ + component: NodeDetailPage, +}); diff --git a/packages/console/src/routes/_app/nodes/index.tsx b/packages/console/src/routes/_app/nodes/index.tsx new file mode 100644 index 00000000..06bc1a99 --- /dev/null +++ b/packages/console/src/routes/_app/nodes/index.tsx @@ -0,0 +1,6 @@ +import { createFileRoute } from '@tanstack/react-router'; +import { NodesPage } from '@/features/nodes/pages/NodesPage'; + +export const Route = createFileRoute('/_app/nodes/')({ + component: NodesPage, +}); diff --git a/packages/console/vite.config.ts b/packages/console/vite.config.ts index d91404fd..e6817e72 100644 --- a/packages/console/vite.config.ts +++ b/packages/console/vite.config.ts @@ -68,6 +68,7 @@ export default defineConfig({ '/updateSettings', '/getIngress', '/setIngress', + '/healthz', '/listNodes', '/removeNode', '/updateNodeSettings', From edb52e555c6a9ec320f3829216d771a6cd8f27ce Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 01:04:14 +0800 Subject: [PATCH 50/89] The fleet exam reads a partial list from the door when a node dies, and dor sandbox ls at the door lists the fleet When node-d dies, listSandboxes at the door lacks its sandbox and names node-d silent with the reason it was not even asked, the E2B list is a 503 naming it, and after removeNode nobody is silent. The CLI's sandbox ls pointed at node A's gateway lists node A's sandboxes with no warning line. --- e2e/src/cli.test.ts | 12 ++++++++++++ e2e/src/gateway.test.ts | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/e2e/src/cli.test.ts b/e2e/src/cli.test.ts index 61befa80..dc579a75 100644 --- a/e2e/src/cli.test.ts +++ b/e2e/src/cli.test.ts @@ -59,6 +59,18 @@ describe('dor CLI against a real daemon', () => { expect(stdout).toContain(created.sandbox.id); }); + it("sandbox ls at the door lists the fleet — node A's sandboxes, nobody silent, no warning line", async () => { + const sdk = new Dormice({ + endpoint: inject('dormiceEndpoint'), + token: inject('dormiceToken'), + }); + const created = await sdk.acquireSandbox('cli-ls-door-key'); + const { stdout } = await door('sandbox', 'ls'); + expect(stdout).toMatch(/cli-ls-door-key\s{2,}active/); + expect(stdout).toContain(created.sandbox.id); + expect(stdout).not.toContain('warning:'); + }); + it('sandbox meta shows, replaces and clears labels through the real binary', async () => { const sdk = new Dormice({ endpoint: inject('dormiceEndpoint'), diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts index f17b644e..64c41892 100644 --- a/e2e/src/gateway.test.ts +++ b/e2e/src/gateway.test.ts @@ -702,11 +702,30 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { ? true : undefined, ); + // The merged list is the other nodes' and says so: d's sandbox is + // not in it, and d is named silent with the reason it was not even + // asked — not a dial that waited the whole merge timeout. + const partial = await viaGateway().listSandboxes(); + expect(partial.sandboxes.some((s) => s.name === 'gw-on-d')).toBe(false); + expect(partial.silent).toEqual([ + { nodeId: 'node-d', why: expect.stringMatching(/has not checked in/) }, + ]); + // The E2B list has nowhere to say what it lacks: a 503 naming d. + const e2bList = await fetch(`${gateway()}/e2b/api/v2/sandboxes`, { + headers: { 'x-api-key': `e2b_${token()}` }, + }); + expect(e2bList.status).toBe(503); + expect((await e2bList.json()) as object).toMatchObject({ + code: 503, + message: expect.stringMatching(/node node-d did not answer/), + }); expect((await rpc('/removeNode', { id: 'node-d' })).body).toEqual({ removed: true, }); expect((await listNodes()).some((n) => n.id === 'node-d')).toBe(false); + // Removed, it is nobody's silence: the list is whole again. + expect((await viaGateway().listSandboxes()).silent).toEqual([]); const placed = await viaGateway().acquireSandbox('gw-while-d-down'); try { expect(['node-b', 'node-c']).toContain(placed.sandbox.nodeId); From b061b39c707ad99bdab286836db688c7eb5c75ce Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 01:11:15 +0800 Subject: [PATCH 51/89] The docs say which door answers the lists and the host reading, and describe the fleet's own figures and the nodes page The HTTP API page's door legend no longer sends clients to the daemon for the list verbs: the gateway merges them and names in `silent` the node it could not include, forwards a host reading to the node named, and answers getFleetMetrics and getFleetStateHistory itself; only the upgrade verbs remain a 501. The metrics page shows the list methods' object shape, getHostMetrics by node, and the fleet's sums and census history; the console page describes the overview's nodes card and the nodes page; the CLI page drops the "point ls at the daemon" workaround. The skill and the SDK README follow. --- packages/sdk/README.md | 6 ++-- skills/dormice/SKILL.md | 14 +++++---- website/content/docs/cli.mdx | 9 ++---- website/content/docs/console.mdx | 30 ++++++++++++++------ website/content/docs/http-api.mdx | 29 +++++++++++-------- website/content/docs/metrics.mdx | 47 +++++++++++++++++++++++++------ 6 files changed, 93 insertions(+), 42 deletions(-) diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 507aa860..603a3540 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -46,7 +46,7 @@ lifecycle policy is set at creation: `freezeAfterSeconds`, `stopAfterSeconds` | Method | What it does | | --- | --- | | `acquireSandbox(userKey, { policy?, template? })` | Create or wake the sandbox behind a key (idempotent); both options apply only when this call creates it | -| `listSandboxes()` | Every sandbox with its current lifecycle state | +| `listSandboxes()` | `{ sandboxes, silent? }` — every sandbox with its current lifecycle state; at the gateway, every node's, `silent` naming a node it could not include | | `execCommand(userKey, command, opts?)` | Run a shell command; buffered stdout/stderr and the real exit code | | `writeFiles(userKey, files)` | Write files onto the sandbox disk (relative paths land under `/home/user`) | | `readFile(userKey, path)` | Read a file back as bytes | @@ -55,7 +55,9 @@ lifecycle policy is set at creation: `freezeAfterSeconds`, `stopAfterSeconds` | `registerTemplate(name, image)` | Name a Docker image on the host as a template (an upsert — re-register to upgrade) | | `listTemplates()` | Every registered template | | `removeTemplate(name)` | Remove a template's registration; refused (409) while sandboxes use it | -| `getHostMetrics()` | One snapshot of the host: CPU, memory, swap, data disk, sandbox disks, ledger totals | +| `getHostMetrics({ nodeId? })` | One machine's snapshot: CPU, memory, swap, data disk, sandbox disks, ledger totals — at the gateway, `nodeId` names the machine | +| `getFleetMetrics()` | The fleet's sums from the nodes' check-ins: nodes total / reachable / reported, the sandbox census, the disks' bill | +| `getFleetStateHistory({ start?, end? })` | The fleet's census over time with the window's concurrency peak | A non-zero exit code is a result, not an error. API failures throw `DormiceApiError` carrying the HTTP status and the daemon's message. diff --git a/skills/dormice/SKILL.md b/skills/dormice/SKILL.md index 9f2d7eb0..a77a8b17 100644 --- a/skills/dormice/SKILL.md +++ b/skills/dormice/SKILL.md @@ -131,11 +131,15 @@ nodes learn a template at their next check-in), gateway: revocable peers of the API token with optional expiry and a reversible disable switch; the create response shows the key once, never again; these four verbs accept only the token), -`getHostMetrics`, `getSandboxMetrics` / `listSandboxMetrics` (live -resource samples; never wake anything), `listSandboxImages` (who still -runs an old template image) — the list and host verbs at the daemon — -`getConfig` / `updateSettings` (the fleet's settings at the gateway, -secrets redacted; applied by every node at its next check-in), +`getHostMetrics` (one machine's reading; at the gateway name the node +with `nodeId`, a fleet of one needs none), `getSandboxMetrics` / +`listSandboxMetrics` (live resource samples; never wake anything), +`listSandboxImages` (who still runs an old template image) — the lists +at the gateway are every node's, with `silent` naming a node it could +not include — `getFleetMetrics` / `getFleetStateHistory` (the fleet's +sums and its census over time, answered by the gateway from the nodes' +check-ins), `getConfig` / `updateSettings` (the fleet's settings at the +gateway, secrets redacted; applied by every node at its next check-in), `getIngress` / `setIngress` (bind domains on the gateway's managed reverse proxy), `listNodes` (every node and what it last reported). `execCommand` takes diff --git a/website/content/docs/cli.mdx b/website/content/docs/cli.mdx index 972815c5..b94db581 100644 --- a/website/content/docs/cli.mdx +++ b/website/content/docs/cli.mdx @@ -13,12 +13,9 @@ export DORMICE_ENDPOINT=http://127.0.0.1:3677 export DORMICE_API_TOKEN= # on the host: grep ^DORMICE_API_TOKEN /etc/dormice/env ``` -Two forms read the sandbox list — `dor sandbox ls`, and `dor sandbox -meta ` without labels (showing labels reads the list) — and the -gateway does not route the list verbs yet: for those, point -`DORMICE_ENDPOINT` at the daemon (`http://127.0.0.1:3676`) until it -does. Everything else on this page goes to the gateway, `meta` with -labels included. +Everything on this page goes to the gateway. `dor sandbox ls` lists +every node's sandboxes; when a node did not answer, a `warning:` line +under the table names it and says its sandboxes are not listed. Errors print as one line on stderr (no stack traces at a shell prompt) and exit 1. diff --git a/website/content/docs/console.mdx b/website/content/docs/console.mdx index cc7303bb..59e76514 100644 --- a/website/content/docs/console.mdx +++ b/website/content/docs/console.mdx @@ -36,14 +36,28 @@ account (via the token) invalidates every existing session. ## Overview -The "is this machine healthy" page: CPU, memory, **swap** and the **data -disk** as first-class cards — swap filling up means "idle is free" is -ending, and a full data disk is the real capacity ceiling. Below them: -sandbox counts by lifecycle state and the disk-overcommit figure -(promised vs actually occupied — the same numbers as -[`getHostMetrics`](/docs/metrics)). Usage bars turn amber at 75% and red -at 90%; a host without swap gets an amber warning pointing at -`dor doctor`. +The "how busy is the fleet" page: sandboxes active now, the window's +concurrency peak, the total by lifecycle state and the disk-overcommit +figure (promised vs actually occupied — the same numbers as +[`getFleetMetrics`](/docs/metrics)), the stacked concurrency curve over +the chosen window, and a nodes card with each machine's CPU, memory and +data-disk levels. When a node has not reported since the gateway +started, the card says so — the figures above are then a lower bound. + +## Nodes + +Every node that has ever checked in, one row each: reachable or not, +the build it runs (marked when it differs from the gateway's), CPU, +memory and data-disk levels, its sandbox census, and its configuration +version against the gateway's — in sync, behind, or not open yet. The +row menu sets the one per-node knob (extra swap the node's daemon +manages on its own data disk) and removes a node that is gone for good; +the gateway refuses to remove one still checking in. A node's own page +shows its **swap** and **data disk** beside CPU and memory, each with +its trend and the window's CPU peak — swap filling up means "idle is +free" is ending, and a full data disk is the real capacity ceiling. +Usage bars turn amber at 75% and red at 90%; a host without swap gets an +amber warning pointing at `dor doctor`. ## Sandboxes diff --git a/website/content/docs/http-api.mdx b/website/content/docs/http-api.mdx index 328c4fab..93091f6d 100644 --- a/website/content/docs/http-api.mdx +++ b/website/content/docs/http-api.mdx @@ -16,14 +16,17 @@ curl -X POST http://127.0.0.1:3677/acquireSandbox \ Two doors answer this wire. The **gateway** (`127.0.0.1:3677`) is the fleet's one door: it answers the configuration verbs itself — templates, -API keys, settings, domains, nodes — and forwards every per-sandbox verb -to the node that holds the sandbox (placing a new name on a node first). -The **daemon** (`127.0.0.1:3676`) is a node: it answers the per-sandbox -verbs and the list and observation verbs directly. Until the gateway -answers the fleet-wide list and observation verbs itself (the next step -of the move), point clients at the daemon for those; the gateway answers -them with an honest `501` naming the node meanwhile. The rows below say -which door. +API keys, settings, domains, nodes — and the fleet's own figures +(`getFleetMetrics`, `getFleetStateHistory`); it forwards every +per-sandbox verb to the node that holds the sandbox (placing a new name +on a node first), merges the fleet-wide lists from every node (saying in +`silent` which node it could not include), and forwards a host reading +to the node the request names. The **daemon** (`127.0.0.1:3676`) is a +node: it answers the per-sandbox, list and host verbs for itself; asked +for a verb that lives at the gateway, it answers `404` naming the +gateway. Point clients at the gateway; the rows below say which door +answers what. The three upgrade verbs are the one thing the gateway does +not route yet (an honest `501`). Two rules cover the whole surface: @@ -45,7 +48,7 @@ The [E2B compatibility surface](/docs/e2b-sdks) is a separate wire under | `POST /acquireSandbox` | both | create, wake, or restore — idempotent per name; at the gateway a new name is placed on a node first | 400 invalid policy/unknown template; 503 with `Retry-After` when no node can take a new sandbox (each node and its reason named) | | `POST /updatePolicy` | both | patch an existing sandbox's [lifecycle policy](/docs/lifecycle#change-the-policy-later) in place; never wakes, never resets the idle clock | 404 unknown name, 400 invalid merged policy | | `POST /updateMetadata` | both | replace an existing sandbox's label set wholesale (`{}` clears); never wakes, never resets the idle clock | 404 unknown name | -| `POST /listSandboxes` | daemon | every sandbox and its state; never wakes anything | — | +| `POST /listSandboxes` | both | every sandbox and its state; never wakes anything. At the gateway: every node's list, with `silent` naming any node it could not include (down, not listening yet, or too slow) — a shorter list is never mistaken for the whole fleet | — | | `POST /execCommand` | both | run a command, buffered | 404 unknown name, 409 archived/restoring | | `POST /writeFiles` | both | batch write, base64 in JSON | 404/409 as above; body over 48 MiB refused | | `POST /writeFile` | both | write one file — the single form of `writeFiles` | same as `writeFiles` | @@ -60,10 +63,12 @@ The [E2B compatibility surface](/docs/e2b-sdks) is a separate wire under | `POST /listApiKeys` | gateway | every key ever minted, revoked ones included, newest first — no secrets | — | | `POST /updateApiKey` | gateway | edit a key by `id`: rename, change/clear `expiresAt`, park/resume via `disabled` — absent fields untouched | 404 unknown id; 409 revoked row or name collision | | `POST /revokeApiKey` | gateway | soft-revoke a key by `id`; idempotent (`revoked: false` when none was) | — | -| `POST /getHostMetrics` | daemon | host snapshot; never wakes anything | — | +| `POST /getHostMetrics` | both | one machine's snapshot; never wakes anything. At the gateway, `nodeId` names the machine (a fleet of one needs none) | 400 at the gateway when the fleet has several nodes and none is named; 404 unknown `nodeId` | +| `POST /getFleetMetrics` | gateway | the figures that add up across the fleet — nodes (total / reachable / reported), the sandbox census by state, the sandbox disks' bill — from the nodes' last check-ins, asking nobody; `nodes.reported` says how many nodes the sums cover | — | +| `POST /getFleetStateHistory` | gateway | how many sandboxes sat in each state over time, one sample per node check-in, with the window's concurrency `peak`; bucketed past 360 points by whole rows | 400 unparseable `start`/`end` | | `POST /getSandboxMetrics` | both | one sandbox's live CPU/memory/disk sample; `sample` is `null` when nothing is running | 404 unknown name | -| `POST /listSandboxMetrics` | daemon | every measurable sandbox's sample in one answer | — | -| `POST /listSandboxImages` | daemon | each sandbox's born image vs its template's current one | — | +| `POST /listSandboxMetrics` | both | every measurable sandbox's sample in one answer; at the gateway, every node's, with `silent` as in `listSandboxes` | — | +| `POST /listSandboxImages` | both | each sandbox's born image vs its template's current one; at the gateway, every node's, with `silent` as in `listSandboxes` | — | | `POST /getConfig` | gateway | effective configuration: the gateway's env knobs (read-only; secrets reported present-or-absent, value never sent), the live fleet `settings`, and `configVersion` — the number every node reports back once it runs this configuration | — | | `POST /updateSettings` | gateway | rewrite the fleet settings (new-sandbox defaults, default policy, the S3 archive store, the sandbox domain, the per-sandbox pids cap) — no restart; the version counts up and every node applies the bundle at its next check-in (within 15 seconds); each provided group replaces that group whole. The `s3` group takes all six fields every time (keys included — they are never read back) and is probed with a real write-read-delete round trip before anything is saved; `s3: null` turns archiving off. `sandboxDomain` takes a bare hostname or `null`; `sandboxDomainAliases` takes the full list of inbound-only alias hostnames (`[]` clears) — swapping the canonical domain is one call carrying both fields, see [Exposing ports](/docs/ports). `pidsLimit` (floor 256, never unlimited) reaches new containers at birth, running sandboxes in place as their node takes the bundle, and frozen or stopped ones at their next wake — no rebuild | 400 empty patch, a `pidsLimit` below 256, a default policy that asks to archive with no store in force, a failed probe (S3 answered 4xx), clearing/moving the store while sandboxes are archived in it (counted across the fleet from the nodes' check-ins), or an alias list that contradicts the post-patch state (duplicates, the canonical domain listed as an alias, or aliases left standing with no canonical domain); 502 when the probed store is unreachable; 503 with `Retry-After` when a node has not checked in since the gateway started, so the archived count is unknown | | `POST /getIngress` | gateway | domains bound on the gateway's managed reverse proxy, with live DNS and certificate probes | — | diff --git a/website/content/docs/metrics.mdx b/website/content/docs/metrics.mdx index 1050d324..2e1d0490 100644 --- a/website/content/docs/metrics.mdx +++ b/website/content/docs/metrics.mdx @@ -55,21 +55,26 @@ array with one sample. Either way: a dashboard over N sandboxes costs one call, not N: ```ts -const samples = await client.listSandboxMetrics(); -// [{ sandboxName, sandboxId, sample }, …] +const { samples, silent } = await client.listSandboxMetrics(); +// samples: [{ sandboxName, sandboxId, sample }, …] ``` Only sandboxes with a running or frozen container appear; colder sandboxes are simply absent — the same honesty as `null`, expressed as -absence. +absence. Asked of the gateway, the answer is every node's put together, +and `silent` names any node it could not include (down, not listening +yet, or too slow), so a shorter list is never mistaken for the whole +fleet; `listSandboxes` and `listSandboxImages` carry the same field. ## Get metrics for the host The native `getHostMetrics()` answers the operator's question — is this -machine okay, and what do the sandboxes collectively cost it? +machine okay, and what do the sandboxes collectively cost it? A machine's +reading is one machine's: at the gateway, name the node (`listNodes` +lists them); a fleet of one needs no name. ```ts -const m = await client.getHostMetrics(); +const m = await client.getHostMetrics({ nodeId: 'node-1' }); ``` The snapshot has four sections: @@ -91,7 +96,31 @@ The snapshot has four sections: - `swap.totalBytes: 0` is a real reading of a machine with no swap; `null` means the platform offers no reading at all. -The web console's Overview page renders exactly this snapshot — CPU, -memory, swap, the data disk, sandbox states, and the overcommit meter — -refreshed every few seconds. Readings a platform cannot produce show as -an amber hint pointing at `dor doctor`, never a fake number. +The web console's node page renders exactly this snapshot for one node +— CPU, memory, swap and the data disk, each with its trend — refreshed +every few seconds. Readings a platform cannot produce show as an amber +hint pointing at `dor doctor`, never a fake number. + +## Get the fleet's figures + +Some figures add up across machines and some do not. A machine's CPU +percentage is one machine's; the sandbox census and the disks' bill are +sums. `getFleetMetrics()` answers the sums from the nodes' last +check-ins — the gateway asks no node to answer it, so a dashboard may +poll it freely: + +```ts +const fleet = await client.getFleetMetrics(); +// { nodes: { total, reachable, reported }, +// sandboxes: { total, byState }, sandboxDisks } +``` + +`nodes.reported` says how many nodes the sums cover. A node that has not +checked in since the gateway started has no reading yet; until it does, +the sums are a lower bound, and the console's overview says so. + +`getFleetStateHistory({ start?, end? })` is the same census over time — +one sample per node check-in, kept 30 days, bucketed past 360 points by +whole rows so the states always sum to the total, with the window's +concurrency `peak` computed from the raw rows. It feeds the overview's +concurrency curve. From 14675eeb007baac3ab02fe4b4b208ba0c6c8cfcf Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 01:17:20 +0800 Subject: [PATCH 52/89] install.sh's closing note no longer sends the sandbox list to the daemon: the gateway routes it since the third cut --- deploy/install.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/deploy/install.sh b/deploy/install.sh index c45493ce..bd11989c 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -857,10 +857,9 @@ printf '\nDormice is installed.\n' printf ' API token: grep ^DORMICE_API_TOKEN %s\n' "$ENV_FILE" printf ' gateway logs: journalctl -u dormice-gateway -f (the door: console, keys, settings, templates)\n' printf ' daemon logs: journalctl -u dormice -f (the node: sandboxes)\n' -printf ' CLI: export DORMICE_ENDPOINT=http://127.0.0.1:%s DORMICE_API_TOKEN=; dor template ls\n' "$GATEWAY_PORT" -printf ' (the gateway is the door; dor sandbox ls, and dor sandbox meta without labels, read\n' -printf ' the sandbox list, which the gateway does not route yet — point DORMICE_ENDPOINT at the daemon,\n' -printf ' 127.0.0.1:%s, for those)\n' "$PORT" +printf ' CLI: export DORMICE_ENDPOINT=http://127.0.0.1:%s DORMICE_API_TOKEN=; dor sandbox ls\n' "$GATEWAY_PORT" +printf ' (the gateway is the door for every verb; a node answers only the sandbox and host verbs\n' +printf ' for itself on 127.0.0.1:%s)\n' "$PORT" printf ' Both processes listen on 127.0.0.1 only, by design — exposing them is a reverse proxy'"'"'s job.\n' if [ "$(systemctl is-active caddy 2>/dev/null)" = active ]; then printf ' console: http:///console (Caddy on :80 -> the gateway; open your cloud firewall for\n' From 95dedc14548c0cf7c66eb6ab67dcb3c74fa1a764 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 03:12:25 +0800 Subject: [PATCH 53/89] The gateway's way of asking a node is ask.ts, askEach takes one request, and dor sandbox ls says on stderr which node it lacks lookup.ts had outgrown its name: since the third cut it carries httpAsk, the one transport for every verb the gateway asks a node on its own account (the lookup, templateUsers, the merged lists, the E2B list's page), so the file is ask.ts and the lookup is one caller among four. askEach took seven positional arguments; it takes the fleet, the asker and one request object (verb, body, schema, options, now). The three fleet-wide list routes stay written out: a generic helper over z.ZodType loses the type provider's return type (TS2345), and fifteen lines are not worth a cast. dor sandbox ls prints its table on stdout and the "node X did not answer" lines on stderr, so a pipe never reads a warning as a row. --- packages/cli/src/commands.test.ts | 10 ++-- packages/cli/src/commands.ts | 14 ++++-- packages/cli/src/main.ts | 6 ++- packages/gateway/src/app.test.ts | 2 +- packages/gateway/src/app.ts | 9 ++-- packages/gateway/src/{lookup.ts => ask.ts} | 20 ++++---- packages/gateway/src/find.test.ts | 10 ++-- packages/gateway/src/find.ts | 4 +- packages/gateway/src/forward.ts | 2 +- packages/gateway/src/index.ts | 2 +- packages/gateway/src/main.ts | 2 +- packages/gateway/src/merge.test.ts | 21 ++++---- packages/gateway/src/merge.ts | 34 ++++++++----- packages/gateway/src/routes/e2b.ts | 18 +++---- packages/gateway/src/routes/observe.ts | 48 +++++++++---------- packages/gateway/src/routes/templates.test.ts | 4 +- packages/gateway/src/routes/templates.ts | 4 +- packages/gateway/src/testing.ts | 2 +- website/content/docs/cli.mdx | 3 +- 19 files changed, 115 insertions(+), 100 deletions(-) rename packages/gateway/src/{lookup.ts => ask.ts} (87%) diff --git a/packages/cli/src/commands.test.ts b/packages/cli/src/commands.test.ts index c77f9fac..b5bff5d6 100644 --- a/packages/cli/src/commands.test.ts +++ b/packages/cli/src/commands.test.ts @@ -118,7 +118,10 @@ describe('clientFromEnv', () => { describe('sandbox commands over real HTTP', () => { // Runs first: the daemon starts with an empty ledger. it('ls reports an empty daemon honestly', async () => { - expect(await sandboxLs(client)).toBe('No sandboxes.'); + expect(await sandboxLs(client)).toEqual({ + table: 'No sandboxes.', + warnings: [], + }); }); it('ls renders one aligned row per sandbox', async () => { @@ -127,7 +130,8 @@ describe('sandbox commands over real HTTP', () => { }); await client.acquireSandbox('bob'); - const output = await sandboxLs(client); + const { table: output, warnings } = await sandboxLs(client); + expect(warnings).toEqual([]); const lines = output.split('\n'); expect(lines[0]).toMatch( /^NAME\s{2,}STATE\s{2,}ID\s{2,}LAST ACTIVE\s{2,}METADATA$/, @@ -162,7 +166,7 @@ describe('sandbox commands over real HTTP', () => { // The protocol keeps name opaque, so an ESC sequence is a legal key; // printed raw it would rewrite the operator's terminal. await client.acquireSandbox('evil\u001b[31mkey'); - const output = await sandboxLs(client); + const { table: output } = await sandboxLs(client); expect(output).not.toContain('\u001b'); expect(output).toContain('evil?[31mkey'); await client.destroySandbox('evil\u001b[31mkey'); diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index d93c5f4e..83997cc1 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -73,11 +73,15 @@ function renderTable(headers: string[], rows: string[][]): string { /** * `dor sandbox ls`: every sandbox with its lifecycle state, as plain - * columns. Asked of the gateway, the list may lack a node that did not - * answer — said under the table, one line per node, never dropped: an - * operator reading "No sandboxes." while a node is down must be told. + * columns (`table`, for stdout). Asked of the gateway, the list may lack + * a node that did not answer — said in `warnings`, one line per node, + * never dropped: an operator reading "No sandboxes." while a node is down + * must be told. Kept apart from the table so main.ts can send them to + * stderr: `dor sandbox ls | grep …` must not read a warning as a row. */ -export async function sandboxLs(client: Dormice): Promise { +export async function sandboxLs( + client: Dormice, +): Promise<{ table: string; warnings: string[] }> { const { sandboxes, silent = [] } = await client.listSandboxes(); const table = sandboxes.length === 0 @@ -92,7 +96,7 @@ export async function sandboxLs(client: Dormice): Promise { (node) => `warning: node ${printable(node.nodeId)} did not answer (${printable(node.why)}) — its sandboxes are not listed`, ); - return [table, ...warnings].join('\n'); + return { table, warnings }; } /** diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index e4898cef..0316bd2a 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -52,7 +52,11 @@ sandbox .command('ls') .description('List every sandbox with its current lifecycle state') .action(async () => { - console.log(await sandboxLs(clientFromEnv(process.env))); + const { table, warnings } = await sandboxLs(clientFromEnv(process.env)); + console.log(table); + // A node the list lacks is said on stderr: the table stays a table for + // a pipe, and the warning still reaches the operator's terminal. + for (const warning of warnings) console.error(warning); }); sandbox diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index cf11acf1..a10962a8 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -7,13 +7,13 @@ import { parseSandboxHost } from '@dormice/shared'; import { pino } from 'pino'; import { afterEach, describe, expect, it } from 'vitest'; import { buildGatewayApp } from './app'; +import { type AskVerb, httpAsk, httpAskNode } from './ask'; import { NameCache } from './cache'; import { loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; import { ensureSettings } from './db/settings'; import { Finder } from './find'; import { Fleet } from './fleet'; -import { type AskVerb, httpAsk, httpAskNode } from './lookup'; import { checkInOf, type reading } from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts index 6d653abe..fe8330d5 100644 --- a/packages/gateway/src/app.ts +++ b/packages/gateway/src/app.ts @@ -10,6 +10,7 @@ import { } from 'fastify-type-provider-zod'; import { type Logger, pino } from 'pino'; import { z } from 'zod'; +import { type AskVerb, httpAsk } from './ask'; import { requireAdminAuth, requireApiAuth, tokensEqual } from './auth'; import { classify, isOriginForm, ORIGIN_FORM_REQUIRED } from './classify'; import { type Config, type ConfigSources, configSources } from './config'; @@ -21,7 +22,6 @@ import { renderError } from './errors'; import type { Finder } from './find'; import type { Fleet } from './fleet'; import type { Ingress } from './ingress'; -import { type AskVerb, httpAsk } from './lookup'; import type { PlacementKnobs } from './placement'; import { createRawFaces } from './raw'; import { apiKeyRoutes } from './routes/api-keys'; @@ -64,9 +64,10 @@ export interface GatewayAppDeps { /** Test seam over updateSettings' S3 round-trip probe; production probes for real. */ probeS3?: SettingsProbe; /** - * How the gateway asks a node a verb on its own account (removeTemplate's - * templateUsers, the merged lists, the E2B list). Defaults to HTTP under - * the fleet token; tests script it, or shorten its patience. + * How the gateway asks a node a verb on its own account (ask.ts: + * removeTemplate's templateUsers, the merged lists, the E2B list). + * Defaults to HTTP under the fleet token; tests script it, or shorten + * its patience. */ ask?: AskVerb; /** diff --git a/packages/gateway/src/lookup.ts b/packages/gateway/src/ask.ts similarity index 87% rename from packages/gateway/src/lookup.ts rename to packages/gateway/src/ask.ts index 8ec0d669..2cd08004 100644 --- a/packages/gateway/src/lookup.ts +++ b/packages/gateway/src/ask.ts @@ -6,14 +6,18 @@ import { import type { z } from 'zod'; /** - * Asking one node a question on the gateway's own account — the daemon's - * lookupSandbox ("do you hold this sandbox?") and templateUsers ("which - * of yours still use this template?"), the two read-only verbs the - * gateway sends that are not a caller's request forwarded raw. Two - * seconds, not more: a node that cannot answer a ledger read in two - * seconds is a node in trouble, and the caller is waiting on the whole - * round. There is no second, slower deadline — slow is down (design - * record #35). + * Asking one node a question on the gateway's own account — a read-only + * verb the gateway sends that is not a caller's request forwarded raw: + * the finder's lookupSandbox ("do you hold this sandbox?"), removeTemplate's + * templateUsers, the merged lists (merge.ts), the E2B list's page. One + * transport (httpAsk) under the fleet token; the callers differ in what + * they ask and how long they wait. + * + * The lookup's deadline. Two seconds, not more: a node that cannot answer + * a ledger read in two seconds is a node in trouble, and the caller is + * waiting on the whole round. There is no second, slower deadline for a + * lookup — slow is down (design record #35). A verb that reads containers + * waits longer (merge.ts MERGE_TIMEOUT_MS). */ export const LOOKUP_TIMEOUT_MS = 2_000; diff --git a/packages/gateway/src/find.test.ts b/packages/gateway/src/find.test.ts index c91f286e..9077d848 100644 --- a/packages/gateway/src/find.test.ts +++ b/packages/gateway/src/find.test.ts @@ -2,17 +2,17 @@ import http from 'node:http'; import type { AddressInfo } from 'node:net'; import { fileURLToPath } from 'node:url'; import { afterEach, describe, expect, it } from 'vitest'; -import { NameCache } from './cache'; -import { migrateDb, openDb } from './db/db'; -import { Finder } from './find'; -import { Fleet } from './fleet'; import { type AskNode, httpAskNode, LOOKUP_TIMEOUT_MS, type LookupAnswer, type LookupQuery, -} from './lookup'; +} from './ask'; +import { NameCache } from './cache'; +import { migrateDb, openDb } from './db/db'; +import { Finder } from './find'; +import { Fleet } from './fleet'; import { checkInOf } from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); diff --git a/packages/gateway/src/find.ts b/packages/gateway/src/find.ts index 9eab80c2..269ed3f2 100644 --- a/packages/gateway/src/find.ts +++ b/packages/gateway/src/find.ts @@ -1,4 +1,5 @@ import type { SignedFileLookup } from '@dormice/shared'; +import type { AskNode, LookupAnswer, LookupQuery } from './ask'; import type { CacheEntry, NameCache } from './cache'; import { awaitingFirstConfig, @@ -6,7 +7,6 @@ import { type Fleet, type NodeState, } from './fleet'; -import type { AskNode, LookupAnswer, LookupQuery } from './lookup'; /** * Where a sandbox is, adjudicated once for every face: @@ -32,7 +32,7 @@ export interface FinderLog { /** * Finds a sandbox by asking. The cache answers first; on a miss every - * node in the fleet is asked in parallel (lookup.ts), and exactly one + * node in the fleet is asked in parallel (ask.ts), and exactly one * "yes" wins — the sandbox is wherever it says it is, whether or not some * other node was slow to say no. Down nodes are asked like any other: * the reading a node last reported says where NOT to place, never where diff --git a/packages/gateway/src/forward.ts b/packages/gateway/src/forward.ts index 4b4d694c..80f172f8 100644 --- a/packages/gateway/src/forward.ts +++ b/packages/gateway/src/forward.ts @@ -3,7 +3,7 @@ import net from 'node:net'; import type { Duplex } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { Agent } from 'undici'; -import { causeOf } from './lookup'; +import { causeOf } from './ask'; /** * The one place the gateway talks to a node on a caller's behalf. Bytes diff --git a/packages/gateway/src/index.ts b/packages/gateway/src/index.ts index 0c275dbe..b0b28cb9 100644 --- a/packages/gateway/src/index.ts +++ b/packages/gateway/src/index.ts @@ -6,11 +6,11 @@ * gateway lives in main.ts. */ export { buildGatewayApp, type GatewayAppDeps } from './app'; +export { type AskNode, type AskVerb, httpAsk, httpAskNode } from './ask'; export { NameCache } from './cache'; export { type Config, loadConfig } from './config'; export { type Db, migrateDb, openDb } from './db/db'; export { ensureSettings } from './db/settings'; export { Finder } from './find'; export { Fleet } from './fleet'; -export { type AskNode, type AskVerb, httpAsk, httpAskNode } from './lookup'; export { checkInOf, reading, TEST_TOKEN, testGateway } from './testing'; diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts index da982859..05e7208e 100644 --- a/packages/gateway/src/main.ts +++ b/packages/gateway/src/main.ts @@ -6,6 +6,7 @@ import { closeWithGrace, trackConnections } from '@dormice/server/shutdown'; import { pino } from 'pino'; import { z } from 'zod'; import { buildGatewayApp } from './app'; +import { httpAskNode } from './ask'; import { NameCache } from './cache'; import { loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; @@ -13,7 +14,6 @@ import { ensureSettings } from './db/settings'; import { Finder } from './find'; import { Fleet } from './fleet'; import { Ingress } from './ingress'; -import { httpAskNode } from './lookup'; import { readBuildInfo } from './version'; const log = pino(); diff --git a/packages/gateway/src/merge.test.ts b/packages/gateway/src/merge.test.ts index 67105a2b..fe2e3793 100644 --- a/packages/gateway/src/merge.test.ts +++ b/packages/gateway/src/merge.test.ts @@ -1,9 +1,9 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; import { z } from 'zod'; +import type { AskVerb } from './ask'; import { migrateDb, openDb } from './db/db'; import { Fleet, STARTUP_GRACE_MS } from './fleet'; -import type { AskVerb } from './lookup'; import { askability, askEach, MERGE_TIMEOUT_MS } from './merge'; import { checkInOf } from './testing'; @@ -100,14 +100,12 @@ describe('askEach', () => { headers: new Headers({ 'x-next-token': '7' }), }; }; - const merged = await askEach( - fleet, - ask, - NOW, - (node) => `list?node=${node.id}`, - {}, - answerSchema, - ); + const merged = await askEach(fleet, ask, { + verb: (node) => `list?node=${node.id}`, + body: {}, + schema: answerSchema, + now: NOW, + }); expect(asked).toEqual([ { id: 'c', verb: 'list?node=c', timeoutMs: MERGE_TIMEOUT_MS }, { id: 'a', verb: 'list?node=a', timeoutMs: MERGE_TIMEOUT_MS }, @@ -128,10 +126,7 @@ describe('askEach', () => { async () => { throw new Error('nobody to ask'); }, - NOW, - 'list', - {}, - answerSchema, + { verb: 'list', body: {}, schema: answerSchema, now: NOW }, ); expect(merged).toEqual({ answers: [], silent: [] }); }); diff --git a/packages/gateway/src/merge.ts b/packages/gateway/src/merge.ts index 33a909ca..4c77b7ba 100644 --- a/packages/gateway/src/merge.ts +++ b/packages/gateway/src/merge.ts @@ -1,5 +1,6 @@ import type { SilentNode } from '@dormice/shared'; import type { z } from 'zod'; +import type { AskOptions, AskVerb } from './ask'; import { awaitingFirstConfig, awaitingFirstConfigWhy, @@ -8,7 +9,6 @@ import { type NodeState, STARTUP_GRACE_MS, } from './fleet'; -import type { AskOptions, AskVerb } from './lookup'; /** * How long a merged answer waits for a node. Longer than a lookup's two @@ -66,24 +66,34 @@ export interface Merged { silent: SilentNode[]; } +/** What every node is asked. */ +export interface EachAsk { + /** The path under each node's endpoint (query string included for a GET) — or a function of the node: the E2B list sends each node its own offset. */ + verb: string | ((node: NodeState) => string); + /** The POST body; none for a GET. */ + body?: unknown; + /** The shape of one node's answer; a body that fails it is silence (ask.ts httpAsk). */ + schema: z.ZodType; + /** How the verb is asked (ask.ts AskOptions); the timeout defaults to MERGE_TIMEOUT_MS. */ + options?: AskOptions; + /** The instant askability is judged at; now by default, injected by tests. */ + now?: Date; +} + /** * Asks every askable node one verb in parallel and keeps every answer — * the fleet-wide lists' one step (routes/observe.ts, the E2B list in * routes/e2b.ts). Unlike the finder, which wants exactly one yes, a * merged answer wants everyone, and a node that does not answer is not * a reason to refuse the rest: the operator reads the fleet's sandboxes - * with one node in trouble, and reads which one. `verb` may depend on - * the node — the E2B list sends each node its own offset. + * with one node in trouble, and reads which one. */ export async function askEach( fleet: Fleet, ask: AskVerb, - now: Date, - verb: string | ((node: NodeState) => string), - body: unknown, - schema: z.ZodType, - options: AskOptions = {}, + each: EachAsk, ): Promise> { + const now = each.now ?? new Date(); const silent: SilentNode[] = []; const asked: NodeState[] = []; for (const node of fleet.all()) { @@ -97,10 +107,10 @@ export async function askEach( node, asked: await ask( node, - typeof verb === 'string' ? verb : verb(node), - body, - schema, - { timeoutMs: MERGE_TIMEOUT_MS, ...options }, + typeof each.verb === 'string' ? each.verb : each.verb(node), + each.body, + each.schema, + { timeoutMs: MERGE_TIMEOUT_MS, ...each.options }, ), })), ); diff --git a/packages/gateway/src/routes/e2b.ts b/packages/gateway/src/routes/e2b.ts index 927e807a..5fa9f5d8 100644 --- a/packages/gateway/src/routes/e2b.ts +++ b/packages/gateway/src/routes/e2b.ts @@ -3,12 +3,12 @@ import { sandboxNameSchema } from '@dormice/shared'; import type { FastifyError, FastifyReply, FastifyRequest } from 'fastify'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { z } from 'zod'; +import type { AskVerb } from '../ask'; import { decodeCursor, encodeCursor, mergePages } from '../cursor'; import { relay } from '../errors'; import type { Finder } from '../find'; import type { Fleet, NodeState } from '../fleet'; import { forwardStream, replay } from '../forward'; -import type { AskVerb } from '../lookup'; import { askEach } from '../merge'; import { type PlacementKnobs, refusalMessage } from '../placement'; import { RETRY_AFTER_SECONDS } from '../raw'; @@ -31,7 +31,7 @@ export interface E2bRoutesOptions { token: string; /** The app's one adjudication of a bare credential (fleet token or a live minted key). */ isCredential: (bareToken: string) => boolean; - /** Asks one node one verb on the gateway's account (lookup.ts httpAsk) — the list. */ + /** Asks one node one verb on the gateway's account (ask.ts httpAsk) — the list. */ ask: AskVerb; } @@ -221,20 +221,16 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( } query.delete('nextToken'); query.set('limit', String(limit.data)); - const { answers, silent } = await askEach( - fleet, - ask, - new Date(), - (node) => { + const { answers, silent } = await askEach(fleet, ask, { + verb: (node) => { const own = new URLSearchParams(query); const offset = offsets[node.id] ?? 0; if (offset > 0) own.set('nextToken', String(offset)); return `e2b/api/v2/sandboxes?${own.toString()}`; }, - undefined, - z.array(e2bListItemSchema), - { method: 'GET', credential: 'x-api-key' }, - ); + schema: z.array(e2bListItemSchema), + options: { method: 'GET', credential: 'x-api-key' }, + }); if (silent.length > 0) { reply.header('retry-after', String(RETRY_AFTER_SECONDS)); return send( diff --git a/packages/gateway/src/routes/observe.ts b/packages/gateway/src/routes/observe.ts index 1a8bafbb..f18b02fa 100644 --- a/packages/gateway/src/routes/observe.ts +++ b/packages/gateway/src/routes/observe.ts @@ -4,13 +4,13 @@ import { listSandboxMetricsResponseSchema, } from '@dormice/shared'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import type { AskVerb } from '../ask'; import type { Fleet } from '../fleet'; -import type { AskVerb } from '../lookup'; import { askEach } from '../merge'; export interface ObserveRoutesOptions { fleet: Fleet; - /** Asks one node one verb on the gateway's account (lookup.ts httpAsk). */ + /** Asks one node one verb on the gateway's account (ask.ts httpAsk). */ ask: AskVerb; } @@ -31,18 +31,20 @@ export interface ObserveRoutesOptions { export const observeRoutes: FastifyPluginAsyncZod< ObserveRoutesOptions > = async (app, { fleet, ask }) => { + // Three routes written out rather than one generic helper: the type + // provider derives each handler's return type from its concrete schema + // and cannot resolve it through a generic `z.ZodType` (tried at the + // third cut's review, 2026-09-15 — TS2345 on every route), and the + // fifteen lines a helper would save are not worth a cast. app.post( '/listSandboxes', { schema: { response: { 200: listSandboxesResponseSchema } } }, async () => { - const { answers, silent } = await askEach( - fleet, - ask, - new Date(), - 'listSandboxes', - {}, - listSandboxesResponseSchema, - ); + const { answers, silent } = await askEach(fleet, ask, { + verb: 'listSandboxes', + body: {}, + schema: listSandboxesResponseSchema, + }); return { sandboxes: answers.flatMap((a) => a.value.sandboxes), silent, @@ -54,14 +56,11 @@ export const observeRoutes: FastifyPluginAsyncZod< '/listSandboxMetrics', { schema: { response: { 200: listSandboxMetricsResponseSchema } } }, async () => { - const { answers, silent } = await askEach( - fleet, - ask, - new Date(), - 'listSandboxMetrics', - {}, - listSandboxMetricsResponseSchema, - ); + const { answers, silent } = await askEach(fleet, ask, { + verb: 'listSandboxMetrics', + body: {}, + schema: listSandboxMetricsResponseSchema, + }); return { samples: answers.flatMap((a) => a.value.samples), silent }; }, ); @@ -70,14 +69,11 @@ export const observeRoutes: FastifyPluginAsyncZod< '/listSandboxImages', { schema: { response: { 200: listSandboxImagesResponseSchema } } }, async () => { - const { answers, silent } = await askEach( - fleet, - ask, - new Date(), - 'listSandboxImages', - {}, - listSandboxImagesResponseSchema, - ); + const { answers, silent } = await askEach(fleet, ask, { + verb: 'listSandboxImages', + body: {}, + schema: listSandboxImagesResponseSchema, + }); return { images: answers.flatMap((a) => a.value.images), silent }; }, ); diff --git a/packages/gateway/src/routes/templates.test.ts b/packages/gateway/src/routes/templates.test.ts index 9649f77e..44ae7720 100644 --- a/packages/gateway/src/routes/templates.test.ts +++ b/packages/gateway/src/routes/templates.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; +import type { AskVerb } from '../ask'; import { readConfigVersion } from '../db/settings'; -import type { AskVerb } from '../lookup'; import { checkInOf, TEST_TOKEN, testGateway } from '../testing'; const authed = { authorization: `Bearer ${TEST_TOKEN}` }; @@ -8,7 +8,7 @@ const authed = { authorization: `Bearer ${TEST_TOKEN}` }; /** * A fleet whose nodes answer templateUsers from a script: which names * each node reports for a template, or silence. The route is about the - * decision, not the transport (lookup.ts httpAsk is the transport, and + * decision, not the transport (ask.ts httpAsk is the transport, and * app.test.ts exercises it over sockets). */ function templatesGateway(users: Record) { diff --git a/packages/gateway/src/routes/templates.ts b/packages/gateway/src/routes/templates.ts index 62903f9e..1a202254 100644 --- a/packages/gateway/src/routes/templates.ts +++ b/packages/gateway/src/routes/templates.ts @@ -8,6 +8,7 @@ import { } from '@dormice/shared'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import { z } from 'zod'; +import type { AskVerb } from '../ask'; import type { Db } from '../db/db'; import { listTemplates, @@ -15,13 +16,12 @@ import { removeTemplate, } from '../db/templates'; import type { Fleet } from '../fleet'; -import type { AskVerb } from '../lookup'; import { RETRY_AFTER_SECONDS } from '../raw'; export interface TemplateRoutesOptions { db: Db; fleet: Fleet; - /** Asks one node one verb on the gateway's account (lookup.ts httpAsk). */ + /** Asks one node one verb on the gateway's account (ask.ts httpAsk). */ ask: AskVerb; } diff --git a/packages/gateway/src/testing.ts b/packages/gateway/src/testing.ts index 7ad92b0a..331e9de8 100644 --- a/packages/gateway/src/testing.ts +++ b/packages/gateway/src/testing.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url'; import { KeyedQueue } from '@dormice/server/keyed-queue'; import type { CheckInRequest, NodeReading } from '@dormice/shared'; import { buildGatewayApp } from './app'; +import { type AskNode, type AskVerb, httpAskNode } from './ask'; import { NameCache } from './cache'; import { configSources, loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; @@ -9,7 +10,6 @@ import { ensureSettings } from './db/settings'; import { Finder } from './find'; import { Fleet } from './fleet'; import type { Ingress } from './ingress'; -import { type AskNode, type AskVerb, httpAskNode } from './lookup'; /** * Test scaffolding shared by the gateway's suites — and, through index.ts, diff --git a/website/content/docs/cli.mdx b/website/content/docs/cli.mdx index b94db581..5dc04003 100644 --- a/website/content/docs/cli.mdx +++ b/website/content/docs/cli.mdx @@ -15,7 +15,8 @@ export DORMICE_API_TOKEN= # on the host: grep ^DORMICE_API_TOKEN /etc/d Everything on this page goes to the gateway. `dor sandbox ls` lists every node's sandboxes; when a node did not answer, a `warning:` line -under the table names it and says its sandboxes are not listed. +on stderr names it and says its sandboxes are not listed — the table on +stdout stays a table, so a pipe never reads the warning as a row. Errors print as one line on stderr (no stack traces at a shell prompt) and exit 1. From 6c71ecebae8d108aa680fbd758d13057f4788d61 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 03:12:50 +0800 Subject: [PATCH 54/89] The fleet's state history is sampled on the gateway's own clock, one row an interval however many nodes report, and no check-in waits on the write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The third cut wrote one fleet_state_samples row per node check-in. Two things were wrong with that. The table grew with the fleet — N nodes, N times the rows, and getFleetStateHistory reads the whole window into memory before bucketing — and the write ran inside the check-in handler, so a full disk turned every check-in into a 500 and held the configuration bundle that rides on its answer hostage to an observation row. The gateway now runs a chained ticker of its own (DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS, 30 by default, the node sampler's density, so a single node's imported history joins at the same spacing; the exam sets 1): each tick sums the readings the check-ins left in memory into one row, writes nothing while no node has a reading or while the startup grace still waits on a known node, logs a failed write and tries again next tick. The check-in handler writes nothing. --- e2e/src/gateway.test.ts | 5 +- e2e/src/setup/daemon.ts | 2 + packages/console/messages/de/settings.json | 1 + packages/console/messages/en/settings.json | 1 + packages/console/messages/es/settings.json | 1 + packages/console/messages/fr/settings.json | 1 + packages/console/messages/ja/settings.json | 1 + packages/console/messages/ko/settings.json | 1 + packages/console/messages/pt-BR/settings.json | 1 + packages/console/messages/ru/settings.json | 1 + packages/console/messages/zh-CN/settings.json | 1 + packages/console/messages/zh-TW/settings.json | 1 + .../overview/components/FleetStatCards.tsx | 4 +- .../overview/hooks/useFleetTimeline.ts | 4 +- .../features/settings/pages/SettingsPage.tsx | 1 + packages/console/src/lib/api.ts | 2 +- packages/gateway/src/config.test.ts | 8 ++ packages/gateway/src/config.ts | 15 ++++ packages/gateway/src/db/fleet-samples.ts | 45 ++++++----- packages/gateway/src/db/schema.ts | 20 +++-- packages/gateway/src/fleet.ts | 6 +- packages/gateway/src/main.ts | 28 +++++++ packages/gateway/src/routes/fleet.test.ts | 77 +++++++++++-------- packages/gateway/src/routes/fleet.ts | 6 +- packages/gateway/src/routes/nodes.ts | 4 - packages/sdk/src/client.ts | 2 +- packages/shared/src/gateway.ts | 9 ++- website/content/docs/configuration.mdx | 1 + website/content/docs/http-api.mdx | 2 +- website/content/docs/metrics.mdx | 7 +- 30 files changed, 170 insertions(+), 88 deletions(-) diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts index 64c41892..d3c6299d 100644 --- a/e2e/src/gateway.test.ts +++ b/e2e/src/gateway.test.ts @@ -431,7 +431,7 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { } }); - it('getFleetMetrics sums both nodes from their check-ins without asking them; getFleetStateHistory grows a point per check-in with a peak', async () => { + it('getFleetMetrics sums both nodes from their check-ins without asking them; getFleetStateHistory grows a point per sampler tick with a peak', async () => { const metrics = await viaGateway().getFleetMetrics(); expect(metrics.nodes).toEqual({ total: 2, reachable: 2, reported: 2 }); const own = await Promise.all( @@ -443,7 +443,8 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { expect(metrics.sandboxDisks.count).toBe( own.reduce((sum, m) => sum + m.sandboxDisks.count, 0), ); - // A check-in a second: a couple of them bring points and a peak. + // The gateway samples every second in the exam: a couple of ticks + // bring points and a peak. const history = await until(async () => { const h = await viaGateway().getFleetStateHistory(); return h.points.length >= 2 && h.peak !== null ? h : undefined; diff --git a/e2e/src/setup/daemon.ts b/e2e/src/setup/daemon.ts index 07ce330d..62a4b75b 100644 --- a/e2e/src/setup/daemon.ts +++ b/e2e/src/setup/daemon.ts @@ -82,6 +82,8 @@ async function bootGateway(spec: GatewaySpec) { // CPU gate is opened wide and the disk floor is off. DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: '100', DORMICE_GATEWAY_NODE_MIN_DISK_GB: '0', + // A fleet history with points inside a test's patience. + DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS: '1', // A wildcard sandbox domain so getHost() and the port proxy are // exercised — no DNS needed, tests spoof the Host header locally. // Every exam starts on a fresh database, so the seed lands every run, diff --git a/packages/console/messages/de/settings.json b/packages/console/messages/de/settings.json index afade226..8e4a5841 100644 --- a/packages/console/messages/de/settings.json +++ b/packages/console/messages/de/settings.json @@ -6,6 +6,7 @@ "settings_hint_node_cpu_limit": "Platzierungsschranke: Ein Knoten, dessen letzter Check-in eine Gesamt-CPU über diesem Prozentsatz meldete, nimmt keine neuen Sandboxes an", "settings_hint_node_active_limit": "Platzierungsschranke: Ein Knoten mit so vielen aktiven Sandboxes (Platzierungen seit dem Check-in eingerechnet) nimmt keine weiteren an; eingefrorene zählen nicht", "settings_hint_node_min_disk": "Platzierungsschranke: Ein Knoten mit weniger als so vielen GiB frei auf der Datenplatte nimmt keine neuen Sandboxes an — eine volle Datenplatte hält alle Sandboxes des Knotens auf einmal an", + "settings_hint_sample_interval": "Das eigene Abtastintervall des Gateways in Sekunden: so oft summiert es den letzten Zensus jedes Knotens zu einer Zeile Flottenverlauf — die Daten der Übersichtskurve; Standard 30, das Intervall der Knoten-Sampler", "settings_hint_sandbox_disk": "Erststart-Saatwert für die Standard-Datenträgerquote — der wirksame Wert steht oben bei den Betriebsreglern", "settings_hint_sandbox_cpus": "Erststart-Saatwert für die Standard-CPU-Quote — der wirksame Wert steht oben bei den Betriebsreglern", "settings_hint_sandbox_memory": "Erststart-Saatwert für das Standard-Speicherlimit — der wirksame Wert steht oben bei den Betriebsreglern", diff --git a/packages/console/messages/en/settings.json b/packages/console/messages/en/settings.json index b6e3b95f..4f5abba4 100644 --- a/packages/console/messages/en/settings.json +++ b/packages/console/messages/en/settings.json @@ -6,6 +6,7 @@ "settings_hint_node_cpu_limit": "Placement gate: a node whose last check-in reported whole-machine CPU above this percentage takes no new sandboxes", "settings_hint_node_active_limit": "Placement gate: a node with this many active sandboxes (placements since its check-in included) takes no more; frozen ones are not counted", "settings_hint_node_min_disk": "Placement gate: a node whose data disk has less than this many GiB free takes no new sandboxes — a full data disk stops every sandbox on the node at once", + "settings_hint_sample_interval": "The gateway's own sampling interval, in seconds: every this long it sums every node's last census into one fleet history row — the overview curve's data; 30 by default, the nodes' own sampling interval", "settings_hint_sandbox_disk": "First-boot seed for the default disk quota — the effective value is in the knobs above", "settings_hint_sandbox_cpus": "First-boot seed for the default CPU quota — the effective value is in the knobs above", "settings_hint_sandbox_memory": "First-boot seed for the default memory limit — the effective value is in the knobs above", diff --git a/packages/console/messages/es/settings.json b/packages/console/messages/es/settings.json index 44ad5c49..60ebe7ab 100644 --- a/packages/console/messages/es/settings.json +++ b/packages/console/messages/es/settings.json @@ -6,6 +6,7 @@ "settings_hint_node_cpu_limit": "Puerta de colocación: un nodo cuyo último registro informó una CPU total por encima de este porcentaje no acepta sandboxes nuevos", "settings_hint_node_active_limit": "Puerta de colocación: un nodo con este número de sandboxes activos (incluidas las colocaciones desde su registro) no acepta más; los congelados no cuentan", "settings_hint_node_min_disk": "Puerta de colocación: un nodo con menos de estos GiB libres en el disco de datos no acepta sandboxes nuevos — un disco de datos lleno detiene de golpe todos los sandboxes del nodo", + "settings_hint_sample_interval": "Intervalo de muestreo propio del gateway, en segundos: cada tanto suma el último censo de cada nodo en una fila del historial de la flota — los datos de la curva del resumen; 30 por defecto, el mismo intervalo que el muestreador de los nodos", "settings_hint_sandbox_disk": "Valor semilla del primer arranque para la cuota de disco predeterminada — el valor efectivo está en los parámetros de arriba", "settings_hint_sandbox_cpus": "Valor semilla del primer arranque para la cuota de CPU predeterminada — el valor efectivo está en los parámetros de arriba", "settings_hint_sandbox_memory": "Valor semilla del primer arranque para el límite de memoria predeterminado — el valor efectivo está en los parámetros de arriba", diff --git a/packages/console/messages/fr/settings.json b/packages/console/messages/fr/settings.json index cbb68e4e..d6c96e9c 100644 --- a/packages/console/messages/fr/settings.json +++ b/packages/console/messages/fr/settings.json @@ -6,6 +6,7 @@ "settings_hint_node_cpu_limit": "Porte de placement : un nœud dont le dernier pointage indiquait un CPU machine au-dessus de ce pourcentage ne prend plus de nouvelles sandboxes", "settings_hint_node_active_limit": "Porte de placement : un nœud avec autant de sandboxes actives (placements depuis son pointage inclus) n'en prend plus ; les gelées ne comptent pas", "settings_hint_node_min_disk": "Porte de placement : un nœud dont le disque de données a moins de ce nombre de Gio libres ne prend plus de nouvelles sandboxes — un disque de données plein arrête d'un coup toutes les sandboxes du nœud", + "settings_hint_sample_interval": "Intervalle d'échantillonnage propre à la passerelle, en secondes : à cette cadence elle additionne le dernier recensement de chaque nœud en une ligne d'historique de la flotte — les données de la courbe de la vue d'ensemble ; 30 par défaut, l'intervalle de l'échantillonneur des nœuds", "settings_hint_sandbox_disk": "Valeur d'amorçage du quota disque par défaut — la valeur effective est dans les réglages ci-dessus", "settings_hint_sandbox_cpus": "Valeur d'amorçage du quota CPU par défaut — la valeur effective est dans les réglages ci-dessus", "settings_hint_sandbox_memory": "Valeur d'amorçage de la limite mémoire par défaut — la valeur effective est dans les réglages ci-dessus", diff --git a/packages/console/messages/ja/settings.json b/packages/console/messages/ja/settings.json index 5aa6928d..66f94371 100644 --- a/packages/console/messages/ja/settings.json +++ b/packages/console/messages/ja/settings.json @@ -6,6 +6,7 @@ "settings_hint_node_cpu_limit": "配置ゲート:直近のチェックインでマシン全体の CPU がこの割合を超えたノードは新しいサンドボックスを受け付けません", "settings_hint_node_active_limit": "配置ゲート:アクティブなサンドボックス数(チェックイン後の配置分を含む)がこの数に達したノードは新規を受け付けません。凍結中は数えません", "settings_hint_node_min_disk": "配置ゲート:データディスクの空きがこの GiB を下回るノードは新規を受け付けません — データディスクが満杯になるとノード上の全サンドボックスが一斉に止まります", + "settings_hint_sample_interval": "ゲートウェイ自身のサンプリング間隔(秒):この間隔で各ノードの最新チェックインの状態別カウントを合計して 1 行書き込みます — 概要の推移グラフのデータ源。既定の 30 はノードのサンプラーと同じ", "settings_hint_sandbox_disk": "既定ディスククォータの初回起動シード値 — 有効値は上の運用設定にあります", "settings_hint_sandbox_cpus": "既定 CPU クォータの初回起動シード値 — 有効値は上の運用設定にあります", "settings_hint_sandbox_memory": "既定メモリ上限の初回起動シード値 — 有効値は上の運用設定にあります", diff --git a/packages/console/messages/ko/settings.json b/packages/console/messages/ko/settings.json index 3d7b3f0b..ca2a7f88 100644 --- a/packages/console/messages/ko/settings.json +++ b/packages/console/messages/ko/settings.json @@ -6,6 +6,7 @@ "settings_hint_node_cpu_limit": "배치 게이트: 마지막 체크인에서 머신 전체 CPU가 이 비율을 넘은 노드는 새 샌드박스를 받지 않습니다", "settings_hint_node_active_limit": "배치 게이트: 활성 샌드박스 수(체크인 이후 배치된 것 포함)가 이 값에 도달한 노드는 더 받지 않습니다. 동결된 것은 세지 않습니다", "settings_hint_node_min_disk": "배치 게이트: 데이터 디스크 여유 공간이 이 GiB 미만인 노드는 새 샌드박스를 받지 않습니다 — 데이터 디스크가 가득 차면 노드의 모든 샌드박스가 한꺼번에 멈춥니다", + "settings_hint_sample_interval": "게이트웨이 자체 샘플링 간격(초): 이 간격마다 각 노드의 마지막 체크인 상태별 개수를 합산해 한 행을 기록합니다 — 개요 추이 그래프의 데이터 원본. 기본값 30은 노드 샘플러와 같습니다", "settings_hint_sandbox_disk": "기본 디스크 할당량의 첫 부팅 시드 값 — 유효 값은 위의 운영 노브에 있습니다", "settings_hint_sandbox_cpus": "기본 CPU 할당량의 첫 부팅 시드 값 — 유효 값은 위의 운영 노브에 있습니다", "settings_hint_sandbox_memory": "기본 메모리 상한의 첫 부팅 시드 값 — 유효 값은 위의 운영 노브에 있습니다", diff --git a/packages/console/messages/pt-BR/settings.json b/packages/console/messages/pt-BR/settings.json index c77d9d6c..d15cd38c 100644 --- a/packages/console/messages/pt-BR/settings.json +++ b/packages/console/messages/pt-BR/settings.json @@ -6,6 +6,7 @@ "settings_hint_node_cpu_limit": "Portão de alocação: um nó cujo último check-in informou CPU total acima desta porcentagem não recebe novos sandboxes", "settings_hint_node_active_limit": "Portão de alocação: um nó com esta quantidade de sandboxes ativos (incluindo alocações desde o check-in) não recebe mais; congelados não contam", "settings_hint_node_min_disk": "Portão de alocação: um nó com menos que estes GiB livres no disco de dados não recebe novos sandboxes — um disco de dados cheio para todos os sandboxes do nó de uma vez", + "settings_hint_sample_interval": "Intervalo de amostragem do próprio gateway, em segundos: a cada tanto ele soma o último censo de cada nó em uma linha do histórico da frota — os dados da curva da visão geral; 30 por padrão, o mesmo intervalo do amostrador dos nós", "settings_hint_sandbox_disk": "Semente de primeira inicialização para a cota padrão de disco — o valor efetivo está nos controles acima", "settings_hint_sandbox_cpus": "Semente de primeira inicialização para a cota padrão de CPU — o valor efetivo está nos controles acima", "settings_hint_sandbox_memory": "Semente de primeira inicialização para o limite padrão de memória — o valor efetivo está nos controles acima", diff --git a/packages/console/messages/ru/settings.json b/packages/console/messages/ru/settings.json index c46318a4..51cb4b33 100644 --- a/packages/console/messages/ru/settings.json +++ b/packages/console/messages/ru/settings.json @@ -6,6 +6,7 @@ "settings_hint_node_cpu_limit": "Шлагбаум размещения: узел, у которого в последнем отчёте загрузка CPU выше этого процента, не получает новых песочниц", "settings_hint_node_active_limit": "Шлагбаум размещения: узел с таким числом активных песочниц (включая размещённые после отчёта) новых не получает; замороженные не считаются", "settings_hint_node_min_disk": "Шлагбаум размещения: узел, у которого на диске данных свободно меньше этого числа ГиБ, не получает новых песочниц — заполненный диск данных останавливает все песочницы узла разом", + "settings_hint_sample_interval": "Собственный интервал выборки шлюза, в секундах: с этой частотой он суммирует последнюю перепись каждого узла в одну строку истории флота — данные кривой на обзоре; по умолчанию 30, интервал сэмплера узлов", "settings_hint_sandbox_disk": "Стартовое значение дисковой квоты по умолчанию — действующее значение в настройках выше", "settings_hint_sandbox_cpus": "Стартовое значение квоты CPU по умолчанию — действующее значение в настройках выше", "settings_hint_sandbox_memory": "Стартовое значение лимита памяти по умолчанию — действующее значение в настройках выше", diff --git a/packages/console/messages/zh-CN/settings.json b/packages/console/messages/zh-CN/settings.json index 167e1dfb..f4205496 100644 --- a/packages/console/messages/zh-CN/settings.json +++ b/packages/console/messages/zh-CN/settings.json @@ -6,6 +6,7 @@ "settings_hint_node_cpu_limit": "落位闸:节点上一次报到的整机 CPU 高于此百分比就不接新沙箱", "settings_hint_node_active_limit": "落位闸:节点上活跃沙箱数(含报到后已落位的)达到此数就不接新沙箱;冻结的不算", "settings_hint_node_min_disk": "落位闸:节点数据盘剩余空间低于此 GiB 就不接新沙箱 — 数据盘写满会让整节点的沙箱一起停摆", + "settings_hint_sample_interval": "网关自己的采样间隔(秒):每隔这么久把各节点最近一次报到的五态计数加总写一行,总览走势图的数据源;默认 30 与节点采样器同步", "settings_hint_sandbox_disk": "默认磁盘配额的首启种子值 — 生效值在上方运营旋钮里", "settings_hint_sandbox_cpus": "默认 CPU 配额的首启种子值 — 生效值在上方运营旋钮里", "settings_hint_sandbox_memory": "默认内存上限的首启种子值 — 生效值在上方运营旋钮里", diff --git a/packages/console/messages/zh-TW/settings.json b/packages/console/messages/zh-TW/settings.json index 14ec40c3..ec5a2efd 100644 --- a/packages/console/messages/zh-TW/settings.json +++ b/packages/console/messages/zh-TW/settings.json @@ -6,6 +6,7 @@ "settings_hint_node_cpu_limit": "落位閘:節點上一次報到的整機 CPU 高於此百分比就不接新沙箱", "settings_hint_node_active_limit": "落位閘:節點上活躍沙箱數(含報到後已落位的)達到此數就不接新沙箱;凍結的不算", "settings_hint_node_min_disk": "落位閘:節點資料碟剩餘空間低於此 GiB 就不接新沙箱 — 資料碟寫滿會讓整節點的沙箱一起停擺", + "settings_hint_sample_interval": "網關自己的採樣間隔(秒):每隔這麼久把各節點最近一次報到的五態計數加總寫一行,總覽走勢圖的資料來源;預設 30 與節點採樣器同步", "settings_hint_sandbox_disk": "預設磁碟配額的首次啟動種子值 — 生效值在上方維運旋鈕裡", "settings_hint_sandbox_cpus": "預設 CPU 配額的首次啟動種子值 — 生效值在上方維運旋鈕裡", "settings_hint_sandbox_memory": "預設記憶體上限的首次啟動種子值 — 生效值在上方維運旋鈕裡", diff --git a/packages/console/src/features/overview/components/FleetStatCards.tsx b/packages/console/src/features/overview/components/FleetStatCards.tsx index d7e86fea..c0a80580 100644 --- a/packages/console/src/features/overview/components/FleetStatCards.tsx +++ b/packages/console/src/features/overview/components/FleetStatCards.tsx @@ -17,8 +17,8 @@ import { StatCard, StatCardSkeleton } from './StatCard'; * 沙箱磁盘账单。容量上限随讨论稿 #23 删(2026-09-14):账本行数不是 * 资源,数据盘水位才是——它有自己的卡。当前值来自 /getFleetMetrics * (2026-09-15 刀 3:网关把每台节点最近一次报到的读数加总,不扇出); - * 峰值与 sparkline 来自 /getFleetStateHistory — 网关每次节点报到落一 - * 行,峰值由原始行现算,分桶抹不掉它。档位由页头的全局切换器驱动。 + * 峰值与 sparkline 来自 /getFleetStateHistory — 网关自己的采样器 30 秒 + * 落一行,峰值由原始行现算,分桶抹不掉它。档位由页头的全局切换器驱动。 */ export function FleetStatCards({ range }: { range: TimelineRangeKey }) { const host = useFleetMetrics(); diff --git a/packages/console/src/features/overview/hooks/useFleetTimeline.ts b/packages/console/src/features/overview/hooks/useFleetTimeline.ts index 76c9ed60..783eaaa5 100644 --- a/packages/console/src/features/overview/hooks/useFleetTimeline.ts +++ b/packages/console/src/features/overview/hooks/useFleetTimeline.ts @@ -23,8 +23,8 @@ export function rangeSpanMs(key: TimelineRangeKey): number { } /** - * 舰队时间线,跟随档位轮询。30 秒一刷 — 与 daemon 的默认采样间隔同步, - * 更快只是重复读到同一批快照。窗口在每次 queryFn 里现算,所以长开的 + * 舰队时间线,跟随档位轮询。30 秒一刷 — 与网关采样器的默认间隔同步, + * 更快只是重复读到同一批样本。窗口在每次 queryFn 里现算,所以长开的 * 页面窗口会随时间滑动。切换档位时沿用上一档的数据顶住新答案到来 * (keepPreviousData),整卡不塌回骨架屏。 */ diff --git a/packages/console/src/features/settings/pages/SettingsPage.tsx b/packages/console/src/features/settings/pages/SettingsPage.tsx index dd7317fe..841baeb8 100644 --- a/packages/console/src/features/settings/pages/SettingsPage.tsx +++ b/packages/console/src/features/settings/pages/SettingsPage.tsx @@ -37,6 +37,7 @@ const KEY_HINTS: Record string> = { DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: m.settings_hint_node_cpu_limit, DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: m.settings_hint_node_active_limit, DORMICE_GATEWAY_NODE_MIN_DISK_GB: m.settings_hint_node_min_disk, + DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS: m.settings_hint_sample_interval, DORMICE_SANDBOX_DISK_GB: m.settings_hint_sandbox_disk, DORMICE_SANDBOX_CPUS: m.settings_hint_sandbox_cpus, DORMICE_SANDBOX_MEMORY_GB: m.settings_hint_sandbox_memory, diff --git a/packages/console/src/lib/api.ts b/packages/console/src/lib/api.ts index 0b27feb7..9200945e 100644 --- a/packages/console/src/lib/api.ts +++ b/packages/console/src/lib/api.ts @@ -184,7 +184,7 @@ export const getSandboxMetricsHistory = ( }); // Fleet state counts over time — the concurrency curve's data, kept by the -// gateway one sample per node check-in. Bucketed points are whole raw +// gateway's own sampler (30s by default). Bucketed points are whole raw // samples (byState always sums to total); peak is computed from raw rows // and immune to bucketing. export const getFleetStateHistory = (start: string, end: string) => diff --git a/packages/gateway/src/config.test.ts b/packages/gateway/src/config.test.ts index 186aa5cb..4267bb80 100644 --- a/packages/gateway/src/config.test.ts +++ b/packages/gateway/src/config.test.ts @@ -13,6 +13,7 @@ describe('loadConfig', () => { expect(config.DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT).toBe(70); expect(config.DORMICE_GATEWAY_NODE_ACTIVE_LIMIT).toBe(400); expect(config.DORMICE_GATEWAY_NODE_MIN_DISK_GB).toBe(10); + expect(config.DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS).toBe(30); }); it('requires the fleet token, at least 32 characters, naming the variable', () => { @@ -48,5 +49,12 @@ describe('loadConfig', () => { expect(() => loadConfig({ ...TOKEN, DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: '0' }), ).toThrow(); + expect( + loadConfig({ ...TOKEN, DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS: '1' }) + .DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS, + ).toBe(1); + expect(() => + loadConfig({ ...TOKEN, DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS: '0' }), + ).toThrow(); }); }); diff --git a/packages/gateway/src/config.ts b/packages/gateway/src/config.ts index 55e60027..ecdba32f 100644 --- a/packages/gateway/src/config.ts +++ b/packages/gateway/src/config.ts @@ -80,6 +80,20 @@ const envSchema = z.object({ * names from landing on the same full box. */ DORMICE_GATEWAY_NODE_MIN_DISK_GB: z.coerce.number().nonnegative().default(10), + /** + * How often the gateway writes one row of the fleet's state census — + * the data behind the console's concurrency curve (db/fleet-samples.ts). + * The gateway's one clock of its own: a row per tick keeps the table + * the same size for a fleet of one and a fleet of ten, where a row per + * node check-in grew with the fleet. 30, the daemon's own sampling + * interval, so a single node's imported history and the gateway's join + * at the same density; the exam sets 1. + */ + DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS: z.coerce + .number() + .int() + .positive() + .default(30), // ---- first-boot seeds of the settings table, in the daemon's words ---- DORMICE_SANDBOX_DISK_GB: z.coerce.number().positive().default(10), DORMICE_SANDBOX_CPUS: z.coerce.number().positive().default(1), @@ -192,6 +206,7 @@ export const CONFIG_KEYS: Record = { DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT: { sensitive: false }, DORMICE_GATEWAY_NODE_ACTIVE_LIMIT: { sensitive: false }, DORMICE_GATEWAY_NODE_MIN_DISK_GB: { sensitive: false }, + DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS: { sensitive: false }, DORMICE_SANDBOX_DISK_GB: { sensitive: false }, DORMICE_SANDBOX_CPUS: { sensitive: false }, DORMICE_SANDBOX_MEMORY_GB: { sensitive: false }, diff --git a/packages/gateway/src/db/fleet-samples.ts b/packages/gateway/src/db/fleet-samples.ts index 168f5f04..53601a67 100644 --- a/packages/gateway/src/db/fleet-samples.ts +++ b/packages/gateway/src/db/fleet-samples.ts @@ -5,34 +5,41 @@ import { type FleetStateSampleRow, fleetStateSamples } from './schema'; /** * How long fleet state samples live. Not a knob: the dashboard's widest - * range (30 days) defines the need, and at one small row per check-in the - * table stays tens of megabytes for a fleet of ten (the node's old fleet - * table had the same ruling). + * range (30 days) defines the need, and at one small row per tick the + * table stays a few megabytes whatever the fleet's size (the node's old + * fleet table had the same ruling). */ export const FLEET_SAMPLE_KEEP_DAYS = 30; /** - * One sample of the fleet's state, written on a check-in (routes/nodes.ts) - * once the reporting node's reading is in: the sum over every node that - * has a reading — a node that is down contributes its last one; its - * sandboxes are still there. Prune rides the same transaction, as on the - * node's sampler. + * One sample of the fleet's state, written by the gateway's own ticker + * (main.ts, every DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS): the sum over + * every node that has a reading — a node that is down contributes its + * last one; its sandboxes are still there. Prune rides the same + * transaction, as on the node's sampler. A tick of the gateway's, not the + * check-in itself (the third cut first wrote a row per check-in; its + * review moved it, 2026-09-15): one figure, one row an interval, however + * many nodes report it — and a write that fails, a full disk, fails a + * sample the next tick retries, never a node's check-in and the + * configuration bundle riding on its answer. * - * Not written while the gateway is still getting to know its fleet: - * within STARTUP_GRACE_MS of a start, as long as any node from the rows - * has not checked in yet. A restarted gateway hears from its nodes one by - * one over an interval, and a sum written after the first would draw the - * fleet collapsing to that node's share and climbing back — a false dip - * on the curve at every gateway restart. Past the grace, a node still - * silent is down, and the sum is written without it (a lower bound, as - * getFleetMetrics says of the same figure). Answers whether a row was - * written. + * Not written when there is nothing true to write: no node has a reading + * (nothing has checked in since this start — the fleet's sandboxes are + * on the nodes' disks, unknown here, and a zero row would draw a cliff + * the fleet did not fall off); or, within STARTUP_GRACE_MS of a start, + * while any node from the rows has not checked in yet — a restarted + * gateway hears from its nodes one by one over an interval, and a sum + * written after the first would draw the fleet collapsing to that node's + * share and climbing back, a false dip at every gateway restart. Past the + * grace, a node still silent is down, and the sum is written without it + * (a lower bound, as getFleetMetrics says of the same figure). Answers + * whether a row was written. */ export function recordFleetSample(db: Db, fleet: Fleet, now: Date): boolean { const nodes = fleet.all(); + const { reported, sandboxes } = sumReadings(nodes); const settling = now.getTime() - fleet.startedAt.getTime() < STARTUP_GRACE_MS; - if (settling && nodes.some((node) => node.reading === null)) return false; - const { sandboxes } = sumReadings(nodes); + if (reported === 0 || (settling && reported < nodes.length)) return false; const cutoff = new Date( now.getTime() - FLEET_SAMPLE_KEEP_DAYS * 86_400_000, ).toISOString(); diff --git a/packages/gateway/src/db/schema.ts b/packages/gateway/src/db/schema.ts index 21a35614..47d96bcf 100644 --- a/packages/gateway/src/db/schema.ts +++ b/packages/gateway/src/db/schema.ts @@ -192,22 +192,20 @@ export type ConsoleAccountRow = typeof consoleAccount.$inferSelect; /** * The fleet's state counts over time — how many sandboxes sat in each - * state across every node — one row per node check-in (routes/nodes.ts, - * db/fleet-samples.ts): the sum of every node's last census at that - * moment, the data behind the console's concurrency curve and its peak. - * Written on the check-in rather than by a ticker of the gateway's own, - * which has none: the gateway only listens and compares, and the - * check-ins are its clock. The one figure no single node can compute - * (design record #26) — each node's own machine history stays on that - * node (host_metrics_samples), and the nodes wrote no fleet history of - * their own since the third cut. Kept 30 days, the dashboard's widest - * range; pruned with every write. + * state across every node — one row per tick of the gateway's sampler + * (main.ts, db/fleet-samples.ts; DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS, + * 30 by default): the sum of every node's last census at that moment, + * the data behind the console's concurrency curve and its peak. The one + * figure no single node can compute (design record #26) — each node's + * own machine history stays on that node (host_metrics_samples), and the + * nodes write no fleet history of their own since the third cut. Kept 30 + * days, the dashboard's widest range; pruned with every write. * * Five explicit state columns instead of a JSON blob, as on the node's * old table: the window peak is max(active) in one SQL aggregate, and the * stacked chart needs each state addressable. `total` is stored * redundantly so readers never re-derive it. `at` is indexed, not unique: - * two nodes may check in within the same millisecond. + * the fourth cut's import lays a single node's old rows beside these. */ export const fleetStateSamples = sqliteTable( 'fleet_state_samples', diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 252fa40d..966f09eb 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -105,9 +105,9 @@ export function awaitingFirstConfigWhy(node: NodeState): string | null { * this gateway started) contributes nothing, and `reported` says so: the * sums are a lower bound until every node has spoken. A node that is * down but did report contributes its last reading — its sandboxes are - * still there, merely out of reach. One function for the check-in's - * sample (db/fleet-samples.ts) and getFleetMetrics (routes/fleet.ts), so - * the curve and the number under it can never disagree. + * still there, merely out of reach. One function for the sampler's row + * (db/fleet-samples.ts) and getFleetMetrics (routes/fleet.ts), so the + * curve and the number under it can never disagree. */ export function sumReadings(nodes: readonly NodeState[]): { reported: number; diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts index 05e7208e..0fe04345 100644 --- a/packages/gateway/src/main.ts +++ b/packages/gateway/src/main.ts @@ -10,6 +10,7 @@ import { httpAskNode } from './ask'; import { NameCache } from './cache'; import { loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; +import { recordFleetSample } from './db/fleet-samples'; import { ensureSettings } from './db/settings'; import { Finder } from './find'; import { Fleet } from './fleet'; @@ -143,9 +144,36 @@ log.info( const SHUTDOWN_GRACE_MS = 10_000; const connections = trackConnections(app.server); let closing = false; + +// The fleet's state sampler — the gateway's one clock of its own, the +// daemon's metrics ticker in shape (server/main.ts): every interval one +// row of the fleet's census, summed from the readings the check-ins left +// in memory (db/fleet-samples.ts says when no row is true enough to +// write). The first shot fires at once, like the daemon's: a restart's +// gap in the curve should equal the downtime, not downtime plus an +// interval. Same failure stance: log, never fatal, the next tick retries — +// and no check-in ever waits on this write. +let sampleTimer: NodeJS.Timeout | undefined; +function sampleTick() { + try { + recordFleetSample(db, fleet, new Date()); + } catch (error) { + app.log.error(error, 'fleet state sample failed'); + } finally { + if (!closing) { + sampleTimer = setTimeout( + sampleTick, + config.DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS * 1000, + ); + } + } +} +sampleTimer = setTimeout(sampleTick, 0); + const close = async (signal: NodeJS.Signals) => { if (closing) return; closing = true; + clearTimeout(sampleTimer); process.removeListener('SIGTERM', onSigterm); process.removeListener('SIGINT', onSigint); app.log.info( diff --git a/packages/gateway/src/routes/fleet.test.ts b/packages/gateway/src/routes/fleet.test.ts index c9b52336..61196fba 100644 --- a/packages/gateway/src/routes/fleet.test.ts +++ b/packages/gateway/src/routes/fleet.test.ts @@ -3,13 +3,14 @@ import { getFleetStateHistoryResponseSchema, } from '@dormice/shared'; import { describe, expect, it } from 'vitest'; +import { recordFleetSample } from '../db/fleet-samples'; import { fleetStateSamples } from '../db/schema'; import { STARTUP_GRACE_MS } from '../fleet'; import { checkInOf, TEST_TOKEN, testGateway } from '../testing'; -// The fleet's own observation: what the check-ins write, what the two -// verbs read back — over app.inject(), the check-ins posted as a node -// posts them. +// The fleet's own observation: what the sampler writes (recordFleetSample +// is the ticker's unit; main.ts owns the clock), what the two verbs read +// back — over app.inject(), the check-ins posted as a node posts them. const authed = { authorization: `Bearer ${TEST_TOKEN}` }; type App = ReturnType['app']; @@ -42,42 +43,39 @@ function samples(db: ReturnType['db']) { .all(); } -describe('the fleet state sample a check-in writes', () => { - it('each check-in writes one row: the sum over every node with a reading, a down node counted by its last reading', async () => { +describe('the fleet state sample the sampler writes', () => { + it('a tick writes one row: the sum over every node with a reading, a down node counted by its last reading — and a check-in writes none', async () => { const { app, db, fleet } = testGateway({}, { startedAt: SETTLED }); await checkIn(app, 'b', { active: 3, frozen: 1 }); await checkIn(app, 'c', { active: 2, archived: 4 }); - const rows = samples(db); - expect(rows).toHaveLength(2); - // The first check-in knew only b; the second knew both. - expect(rows[0]).toMatchObject({ - active: 3, - frozen: 1, - archived: 0, - total: 4, - }); - expect(rows[1]).toMatchObject({ - active: 5, - frozen: 1, - archived: 4, - total: 10, - }); - // c falls silent: its sandboxes are still there, and b's next check-in - // sums c's last reading in. + // The check-ins themselves wrote nothing: the sampler's clock is the + // gateway's own, and no check-in waits on a history write. + expect(samples(db)).toHaveLength(0); + expect(recordFleetSample(db, fleet, new Date())).toBe(true); + expect(samples(db)).toEqual([ + expect.objectContaining({ + active: 5, + frozen: 1, + archived: 4, + total: 10, + }), + ]); + // c falls silent: its sandboxes are still there, and its last reading + // stays in the sum. const c = fleet.get('c'); if (!c) throw new Error('node lost'); c.lastCheckInAt = new Date(Date.now() - 40_000); - await checkIn(app, 'b', { active: 3, frozen: 1 }); - expect(samples(db)[2]).toMatchObject({ active: 5, total: 10 }); + recordFleetSample(db, fleet, new Date()); + expect(samples(db)[1]).toMatchObject({ active: 5, total: 10 }); // Removed, it is counted no more. expect((await rpc(app, '/removeNode', { id: 'c' })).json()).toEqual({ removed: true, }); - await checkIn(app, 'b', { active: 3, frozen: 1 }); - expect(samples(db)[3]).toMatchObject({ active: 3, total: 4 }); + recordFleetSample(db, fleet, new Date()); + expect(samples(db)[2]).toMatchObject({ active: 3, total: 4 }); }); - it('within the startup grace no row is written while a known node has not checked in; past it the sum is written without it', async () => { + it('no row while no node has a reading; within the startup grace none while a known node has not checked in; past it the sum is written without it', () => { // A node known from the rows but not heard from since this start: a // check-in's reading, then the memory a restart leaves — the row and // nothing else (fleet.ts's constructor shape). @@ -89,21 +87,35 @@ describe('the fleet state sample a check-in writes', () => { c.lastCheckInAt = null; c.intervalSeconds = null; }; + // Nobody has reported: nothing true to write, however long ago the + // gateway started — and an empty fleet writes nothing either. + const unheard = testGateway({}, { startedAt: SETTLED }); + expect(recordFleetSample(unheard.db, unheard.fleet, new Date())).toBe( + false, + ); + silence(unheard.fleet); + expect(recordFleetSample(unheard.db, unheard.fleet, new Date())).toBe( + false, + ); + expect(samples(unheard.db)).toHaveLength(0); + const fresh = testGateway({}, { startedAt: new Date() }); silence(fresh.fleet); - await checkIn(fresh.app, 'b', { active: 2 }); + fresh.fleet.checkIn(checkInOf('b', 'http://b:80', { active: 2 })); + expect(recordFleetSample(fresh.db, fresh.fleet, new Date())).toBe(false); expect(samples(fresh.db)).toHaveLength(0); const settled = testGateway({}, { startedAt: SETTLED }); silence(settled.fleet); - await checkIn(settled.app, 'b', { active: 2 }); + settled.fleet.checkIn(checkInOf('b', 'http://b:80', { active: 2 })); + expect(recordFleetSample(settled.db, settled.fleet, new Date())).toBe(true); expect(samples(settled.db)).toEqual([ expect.objectContaining({ active: 2, total: 2 }), ]); }); - it('rows older than 30 days are pruned with the write', async () => { - const { app, db } = testGateway({}, { startedAt: SETTLED }); + it('rows older than 30 days are pruned with the write', () => { + const { db, fleet } = testGateway({}, { startedAt: SETTLED }); db.insert(fleetStateSamples) .values({ at: new Date(Date.now() - 31 * 86_400_000).toISOString(), @@ -115,7 +127,8 @@ describe('the fleet state sample a check-in writes', () => { total: 9, }) .run(); - await checkIn(app, 'b', { active: 1 }); + fleet.checkIn(checkInOf('b', 'http://b:80', { active: 1 })); + recordFleetSample(db, fleet, new Date()); const rows = samples(db); expect(rows).toHaveLength(1); expect(rows[0]?.active).toBe(1); diff --git a/packages/gateway/src/routes/fleet.ts b/packages/gateway/src/routes/fleet.ts index ec637b2b..d7b0bf04 100644 --- a/packages/gateway/src/routes/fleet.ts +++ b/packages/gateway/src/routes/fleet.ts @@ -25,8 +25,8 @@ export interface FleetRoutesOptions { * record #24 — the console polls these every few seconds, and a poll * that fanned out would make the fleet's one observer its heaviest * caller). getFleetMetrics is the present, from every node's last - * reading; getFleetStateHistory is the past, from the samples the - * check-ins wrote (db/fleet-samples.ts). Behind the sandbox gate, as + * reading; getFleetStateHistory is the past, from the rows the gateway's + * own sampler wrote (db/fleet-samples.ts). Behind the sandbox gate, as * observation is on a node. */ export const fleetRoutes: FastifyPluginAsyncZod = async ( @@ -58,7 +58,7 @@ export const fleetRoutes: FastifyPluginAsyncZod = async ( }, ); - // The fleet's past: state counts per check-in, sliced and (past 360 + // The fleet's past: state counts per sampler tick, sliced and (past 360 // points) bucketed. Buckets carry whole raw samples — the last one in // the bucket — so byState always sums to total; the concurrency peak is // computed from raw rows and travels beside the points, immune to diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index 47baed59..c11cac37 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -12,7 +12,6 @@ import { import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import type { NameCache } from '../cache'; import type { Db } from '../db/db'; -import { recordFleetSample } from '../db/fleet-samples'; import { readNodeConfig } from '../db/node-config'; import { readConfigVersion } from '../db/settings'; import { @@ -88,9 +87,6 @@ export const checkInRoutes: FastifyPluginAsyncZod< throw refusal(409, outcome.refused); } const { node, joined, movedFrom } = outcome; - // The fleet's state, sampled now that this node's reading is in - // (db/fleet-samples.ts has when a sample is not written). - recordFleetSample(db, fleet, new Date()); if (joined) { request.log.info( { nodeId: node.id, endpoint: node.endpoint }, diff --git a/packages/sdk/src/client.ts b/packages/sdk/src/client.ts index 9a9702d6..2dc21dcb 100644 --- a/packages/sdk/src/client.ts +++ b/packages/sdk/src/client.ts @@ -293,7 +293,7 @@ export class Dormice { * Fleet state counts over time (default window: the last 24 hours) — * how many sandboxes sat active/frozen/stopped/archived/restoring at * each moment, summed over every node; the gateway keeps this history, - * one sample per node check-in. Bucketed points are whole raw samples + * one sample every 30 seconds by default. Bucketed points are whole raw samples * (byState always sums to total); `peak` carries the window's highest * active count from raw rows, immune to bucketing. */ diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts index d6b62251..6fcaf2ed 100644 --- a/packages/shared/src/gateway.ts +++ b/packages/shared/src/gateway.ts @@ -335,10 +335,11 @@ export type GetFleetMetricsResponse = z.infer< * getFleetStateHistory(start?, end?) — how many sandboxes sat in each * state over time, fleet-wide: the product's own story ("idle is free" is * visible as active falling while frozen rises). Answered by the gateway - * from its fleet_state_samples — one row per check-in, the sum of every - * node's last census at that moment (design record #26: the one figure no - * single node can compute); a node keeps no fleet history of its own - * since the third cut. Kept 30 days. + * from its fleet_state_samples — one row per tick of its own sampler + * (DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS, 30 by default), the sum of + * every node's last census at that moment (design record #26: the one + * figure no single node can compute); a node keeps no fleet history of + * its own since the third cut. Kept 30 days. * * Bucketing differs from the per-sandbox verb on purpose: a bucket * reports its last raw row whole, never per-state maxima — independent diff --git a/website/content/docs/configuration.mdx b/website/content/docs/configuration.mdx index cf4e9691..b3ecfc55 100644 --- a/website/content/docs/configuration.mdx +++ b/website/content/docs/configuration.mdx @@ -49,6 +49,7 @@ first-boot seeds of the fleet settings. | `DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT` | `70` | Placement gate: a node whose last check-in reported whole-machine CPU above this takes no new sandboxes. | | `DORMICE_GATEWAY_NODE_ACTIVE_LIMIT` | `400` | Placement gate: a node with this many active sandboxes (placements since its check-in included) takes no more; frozen ones are not counted. | | `DORMICE_GATEWAY_NODE_MIN_DISK_GB` | `10` | Placement gate: a node whose data disk has less than this free takes no new sandboxes — a full data disk stops every sandbox on the node at once. | +| `DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS` | `30` | How often the gateway writes one row of the fleet's state census (the overview's concurrency curve, `getFleetStateHistory`), summed from the nodes' last check-ins. One row per tick whatever the fleet's size; kept 30 days. | ### Fleet settings — first-boot seeds diff --git a/website/content/docs/http-api.mdx b/website/content/docs/http-api.mdx index 93091f6d..502b343c 100644 --- a/website/content/docs/http-api.mdx +++ b/website/content/docs/http-api.mdx @@ -65,7 +65,7 @@ The [E2B compatibility surface](/docs/e2b-sdks) is a separate wire under | `POST /revokeApiKey` | gateway | soft-revoke a key by `id`; idempotent (`revoked: false` when none was) | — | | `POST /getHostMetrics` | both | one machine's snapshot; never wakes anything. At the gateway, `nodeId` names the machine (a fleet of one needs none) | 400 at the gateway when the fleet has several nodes and none is named; 404 unknown `nodeId` | | `POST /getFleetMetrics` | gateway | the figures that add up across the fleet — nodes (total / reachable / reported), the sandbox census by state, the sandbox disks' bill — from the nodes' last check-ins, asking nobody; `nodes.reported` says how many nodes the sums cover | — | -| `POST /getFleetStateHistory` | gateway | how many sandboxes sat in each state over time, one sample per node check-in, with the window's concurrency `peak`; bucketed past 360 points by whole rows | 400 unparseable `start`/`end` | +| `POST /getFleetStateHistory` | gateway | how many sandboxes sat in each state over time — one sample per tick of the gateway's own sampler (`DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS`, 30 by default), summed from the nodes' last check-ins — with the window's concurrency `peak`; bucketed past 360 points by whole rows | 400 unparseable `start`/`end` | | `POST /getSandboxMetrics` | both | one sandbox's live CPU/memory/disk sample; `sample` is `null` when nothing is running | 404 unknown name | | `POST /listSandboxMetrics` | both | every measurable sandbox's sample in one answer; at the gateway, every node's, with `silent` as in `listSandboxes` | — | | `POST /listSandboxImages` | both | each sandbox's born image vs its template's current one; at the gateway, every node's, with `silent` as in `listSandboxes` | — | diff --git a/website/content/docs/metrics.mdx b/website/content/docs/metrics.mdx index 2e1d0490..435606c7 100644 --- a/website/content/docs/metrics.mdx +++ b/website/content/docs/metrics.mdx @@ -120,7 +120,8 @@ checked in since the gateway started has no reading yet; until it does, the sums are a lower bound, and the console's overview says so. `getFleetStateHistory({ start?, end? })` is the same census over time — -one sample per node check-in, kept 30 days, bucketed past 360 points by -whole rows so the states always sum to the total, with the window's -concurrency `peak` computed from the raw rows. It feeds the overview's +one sample every 30 seconds (the gateway's own sampler, +`DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS`), kept 30 days, bucketed past +360 points by whole rows so the states always sum to the total, with the +window's concurrency `peak` computed from the raw rows. It feeds the overview's concurrency curve. From fd743967f1263370ae1120a577a81d3a8cf7c2bc Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 03:43:36 +0800 Subject: [PATCH 55/89] Every interval knob stops at a day, dor sandbox ls exits 1 for a partial list, and the sampler's comment says what a restart costs the curve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The second review of the third cut, five small honesties. The gateway's sampler comment borrowed the daemon's reason for firing its first shot at once — "a restart's gap equals the downtime" — which is false here: a gateway just started holds no readings, the nodes have yet to report them, so the first shot writes nothing and the first row lands at the first tick after every known node has checked in. Measured on the test machine: 0.6s of downtime, a 54s gap in the curve. The comment now says so, and the fleet_state_samples.at column no longer claims to be a check-in's arrival. The four interval knobs (the gateway's sampler, the daemon's scanner, metrics sampler and check-in) had no ceiling. Past 2^31-1 ms Node's setTimeout does not wait: it warns and fires after one millisecond, which for a sampler is a row every millisecond. A day is the ceiling, refused at boot with the variable named. dor sandbox ls printed the warning about a silent node on stderr and exited 0, so a script counting rows took the partial list for the whole fleet. It exits 1 now, as ls does for a directory it could not read, with the rest still listed on stdout; the fleet exam runs the built CLI against the door while node-d is down and checks all three channels. The fleet exam's history test asserted an unbucketed answer over the default day-long window, true only while the fleet's gateway had lived under 360 seconds at a row a second. It asks for the last ten seconds. --- e2e/src/gateway.test.ts | 35 ++++++++++++++++++++++++-- packages/cli/src/main.ts | 8 ++++-- packages/gateway/src/config.test.ts | 14 +++++++++++ packages/gateway/src/config.ts | 16 +++++++++++- packages/gateway/src/db/schema.ts | 2 +- packages/gateway/src/main.ts | 13 +++++++--- packages/server/src/config.test.ts | 15 +++++++++++ packages/server/src/config.ts | 21 +++++++++++++++- website/content/docs/cli.mdx | 6 +++-- website/content/docs/configuration.mdx | 6 ++--- 10 files changed, 120 insertions(+), 16 deletions(-) diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts index d3c6299d..e22e4d04 100644 --- a/e2e/src/gateway.test.ts +++ b/e2e/src/gateway.test.ts @@ -1,9 +1,11 @@ -import { spawn } from 'node:child_process'; +import { execFile, spawn } from 'node:child_process'; import { randomBytes, randomUUID } from 'node:crypto'; import { mkdtemp, rm } from 'node:fs/promises'; import net, { type AddressInfo } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; import { Dormice } from '@dormice/sdk'; import { Sandbox } from 'e2b'; import { describe, expect, inject, it } from 'vitest'; @@ -35,6 +37,19 @@ function direct(id: string) { } const other = (id: string) => (id === 'node-b' ? 'node-c' : 'node-b'); +/** The built CLI, run as an operator runs it, pointed at the fleet's door. */ +const CLI = fileURLToPath( + new URL('../../packages/cli/dist/main.js', import.meta.url), +); +const dor = (...args: string[]) => + promisify(execFile)('node', [CLI, ...args], { + env: { + ...process.env, + DORMICE_ENDPOINT: gateway(), + DORMICE_API_TOKEN: token(), + }, + }); + const rpc = ( path: string, payload: unknown = {}, @@ -445,8 +460,13 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { ); // The gateway samples every second in the exam: a couple of ticks // bring points and a peak. + // Over the last ten seconds, not the default day: what is asserted of + // bucketing below is true of a window, not of how long the fleet's + // gateway has lived (a row a second) by the time the suite is here. const history = await until(async () => { - const h = await viaGateway().getFleetStateHistory(); + const h = await viaGateway().getFleetStateHistory({ + start: new Date(Date.now() - 10_000).toISOString(), + }); return h.points.length >= 2 && h.peak !== null ? h : undefined; }); for (const point of history.points) { @@ -720,6 +740,17 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { code: 503, message: expect.stringMatching(/node node-d did not answer/), }); + // The CLI at the same door: the table on stdout without the warning + // in it, the warning on stderr naming d, and exit 1 — a partial + // listing is not a success, as ls exits 1 for a directory it could + // not read. + await expect(dor('sandbox', 'ls')).rejects.toMatchObject({ + code: 1, + stdout: expect.not.stringContaining('warning:'), + stderr: expect.stringMatching( + /^warning: node node-d did not answer \(has not checked in for \d+s\) — its sandboxes are not listed\n$/, + ), + }); expect((await rpc('/removeNode', { id: 'node-d' })).body).toEqual({ removed: true, diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 0316bd2a..45c79b73 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -54,9 +54,13 @@ sandbox .action(async () => { const { table, warnings } = await sandboxLs(clientFromEnv(process.env)); console.log(table); - // A node the list lacks is said on stderr: the table stays a table for - // a pipe, and the warning still reaches the operator's terminal. + // A node the list lacks is said on stderr, and in the exit code: the + // table stays a table for a pipe, the warning still reaches the + // operator's terminal, and a script counting rows is told the list is + // partial — as ls exits 1 for a directory it could not read, with the + // rest listed. for (const warning of warnings) console.error(warning); + if (warnings.length > 0) process.exitCode = 1; }); sandbox diff --git a/packages/gateway/src/config.test.ts b/packages/gateway/src/config.test.ts index 4267bb80..9ffa802c 100644 --- a/packages/gateway/src/config.test.ts +++ b/packages/gateway/src/config.test.ts @@ -56,5 +56,19 @@ describe('loadConfig', () => { expect(() => loadConfig({ ...TOKEN, DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS: '0' }), ).toThrow(); + // Past 2^31-1 ms Node's setTimeout fires after one millisecond instead + // of waiting: a day is the ceiling, refused at boot. + expect( + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS: '86400', + }).DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS, + ).toBe(86_400); + expect(() => + loadConfig({ + ...TOKEN, + DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS: '86401', + }), + ).toThrow(); }); }); diff --git a/packages/gateway/src/config.ts b/packages/gateway/src/config.ts index ecdba32f..76278a7f 100644 --- a/packages/gateway/src/config.ts +++ b/packages/gateway/src/config.ts @@ -21,6 +21,18 @@ import { z } from 'zod'; * daemon's old names so an operator moving a single machine onto the * gateway copies the lines across, nothing more. */ + +/** + * The ceiling on an interval knob: one day. A tick a day is the slowest + * cadence that still means anything — and past 2^31-1 ms (24.8 days) + * Node's setTimeout does not wait at all: it warns and fires after one + * millisecond (TimeoutOverflowWarning), which for a sampler is a row + * every millisecond. Refused at boot instead (found by review, + * 2026-09-15; the daemon's interval knobs carry the same rule, + * server/config.ts). + */ +const MAX_INTERVAL_SECONDS = 86_400; + const envSchema = z.object({ DORMICE_GATEWAY_PORT: z.coerce.number().int().min(1).max(65535).default(3677), /** @@ -87,12 +99,14 @@ const envSchema = z.object({ * the same size for a fleet of one and a fleet of ten, where a row per * node check-in grew with the fleet. 30, the daemon's own sampling * interval, so a single node's imported history and the gateway's join - * at the same density; the exam sets 1. + * at the same density; the exam sets 1. At most a day + * (MAX_INTERVAL_SECONDS). */ DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS: z.coerce .number() .int() .positive() + .max(MAX_INTERVAL_SECONDS) .default(30), // ---- first-boot seeds of the settings table, in the daemon's words ---- DORMICE_SANDBOX_DISK_GB: z.coerce.number().positive().default(10), diff --git a/packages/gateway/src/db/schema.ts b/packages/gateway/src/db/schema.ts index 47d96bcf..581160d3 100644 --- a/packages/gateway/src/db/schema.ts +++ b/packages/gateway/src/db/schema.ts @@ -210,7 +210,7 @@ export type ConsoleAccountRow = typeof consoleAccount.$inferSelect; export const fleetStateSamples = sqliteTable( 'fleet_state_samples', { - /** ISO 8601 UTC — when the check-in that produced this sum arrived. */ + /** ISO 8601 UTC — the instant of the sampler tick that summed the readings. */ at: text('at').notNull(), active: integer('active').notNull(), frozen: integer('frozen').notNull(), diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts index 0fe04345..7418e96d 100644 --- a/packages/gateway/src/main.ts +++ b/packages/gateway/src/main.ts @@ -149,10 +149,15 @@ let closing = false; // daemon's metrics ticker in shape (server/main.ts): every interval one // row of the fleet's census, summed from the readings the check-ins left // in memory (db/fleet-samples.ts says when no row is true enough to -// write). The first shot fires at once, like the daemon's: a restart's -// gap in the curve should equal the downtime, not downtime plus an -// interval. Same failure stance: log, never fatal, the next tick retries — -// and no check-in ever waits on this write. +// write). The first shot fires at once and, on a gateway just started, +// writes nothing: the readings are the nodes' to report, and none has +// yet. So the first row after a restart lands at the first tick after +// every known node has checked in, and the curve's gap is the downtime +// plus at most one check-in interval and one sample interval (measured +// 2026-09-15: 0.6s down, a 54s gap) — not the daemon's "gap equals +// downtime", whose figures sit in its own ledger at boot where the +// gateway's sit in the nodes' mouths. Same failure stance: log, never +// fatal, the next tick retries — and no check-in ever waits on this write. let sampleTimer: NodeJS.Timeout | undefined; function sampleTick() { try { diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index aa6a5c8d..be2460e1 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -74,6 +74,21 @@ describe('the metrics sampler knobs', () => { loadConfig({ ...TOKEN, DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS: '0' }), ).toThrow(); }); + + it('refuses an interval past a day on every ticker knob — past 2^31-1 ms Node would fire it every millisecond instead', () => { + for (const knob of [ + 'DORMICE_SCAN_INTERVAL_SECONDS', + 'DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS', + 'DORMICE_CHECK_IN_INTERVAL_SECONDS', + ]) { + const atTheCeiling = loadConfig({ ...TOKEN, [knob]: '86400' }) as Record< + string, + unknown + >; + expect(atTheCeiling[knob]).toBe(86_400); + expect(() => loadConfig({ ...TOKEN, [knob]: '86401' })).toThrow(); + } + }); }); describe('the knobs that moved to the gateway', () => { diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index decaad37..b801400f 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -22,6 +22,18 @@ import { z } from 'zod'; * namespace — bare names like PORT collide with whatever else the operator * has exported. */ + +/** + * The ceiling on the three interval knobs: one day. A tick a day is the + * slowest cadence that still means anything — and past 2^31-1 ms (24.8 + * days) Node's setTimeout does not wait at all: it warns and fires after + * one millisecond (TimeoutOverflowWarning) — a sweep, a sample or a + * check-in every millisecond. Refused at boot instead (found by review, + * 2026-09-15; the gateway's sampler knob carries the same rule, + * gateway/config.ts). + */ +const MAX_INTERVAL_SECONDS = 86_400; + const envSchema = z.object({ DORMICE_PORT: z.coerce.number().int().min(1).max(65535).default(3676), DORMICE_DB_PATH: z.string().default('data/dormice.db'), @@ -33,7 +45,12 @@ const envSchema = z.object({ */ DORMICE_NODE_ID: z.string().min(1).default('node-1'), /** How often the idle scanner sweeps the ledger. */ - DORMICE_SCAN_INTERVAL_SECONDS: z.coerce.number().int().positive().default(60), + DORMICE_SCAN_INTERVAL_SECONDS: z.coerce + .number() + .int() + .positive() + .max(MAX_INTERVAL_SECONDS) + .default(60), /** * How often the metrics sampler persists a reading per measurable sandbox * plus one fleet state-count row — the resolution of every history curve. @@ -42,6 +59,7 @@ const envSchema = z.object({ .number() .int() .positive() + .max(MAX_INTERVAL_SECONDS) .default(30), /** * How long per-sandbox samples live (fleet rows are fixed at 30 days — @@ -136,6 +154,7 @@ const envSchema = z.object({ .number() .int() .positive() + .max(MAX_INTERVAL_SECONDS) .default(15), }); diff --git a/website/content/docs/cli.mdx b/website/content/docs/cli.mdx index 5dc04003..9955fe03 100644 --- a/website/content/docs/cli.mdx +++ b/website/content/docs/cli.mdx @@ -15,8 +15,10 @@ export DORMICE_API_TOKEN= # on the host: grep ^DORMICE_API_TOKEN /etc/d Everything on this page goes to the gateway. `dor sandbox ls` lists every node's sandboxes; when a node did not answer, a `warning:` line -on stderr names it and says its sandboxes are not listed — the table on -stdout stays a table, so a pipe never reads the warning as a row. +on stderr names it and says its sandboxes are not listed, and the +command exits 1 — the table on stdout stays a table, so a pipe never +reads the warning as a row, and a script never takes the partial list +for the whole fleet. Errors print as one line on stderr (no stack traces at a shell prompt) and exit 1. diff --git a/website/content/docs/configuration.mdx b/website/content/docs/configuration.mdx index b3ecfc55..e4c6d25d 100644 --- a/website/content/docs/configuration.mdx +++ b/website/content/docs/configuration.mdx @@ -49,7 +49,7 @@ first-boot seeds of the fleet settings. | `DORMICE_GATEWAY_NODE_CPU_LIMIT_PCT` | `70` | Placement gate: a node whose last check-in reported whole-machine CPU above this takes no new sandboxes. | | `DORMICE_GATEWAY_NODE_ACTIVE_LIMIT` | `400` | Placement gate: a node with this many active sandboxes (placements since its check-in included) takes no more; frozen ones are not counted. | | `DORMICE_GATEWAY_NODE_MIN_DISK_GB` | `10` | Placement gate: a node whose data disk has less than this free takes no new sandboxes — a full data disk stops every sandbox on the node at once. | -| `DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS` | `30` | How often the gateway writes one row of the fleet's state census (the overview's concurrency curve, `getFleetStateHistory`), summed from the nodes' last check-ins. One row per tick whatever the fleet's size; kept 30 days. | +| `DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS` | `30` | How often the gateway writes one row of the fleet's state census (the overview's concurrency curve, `getFleetStateHistory`), summed from the nodes' last check-ins. One row per tick whatever the fleet's size; kept 30 days. At most `86400` (a day). | ### Fleet settings — first-boot seeds @@ -115,7 +115,7 @@ is a fleet setting. | `DORMICE_NODE_ID` | `node-1` | This node's name — in its sandboxes' rows and to its gateway, which tells nodes apart by it. The default serves a node beside its gateway; a node whose gateway is on another machine must state its own. | | `DORMICE_GATEWAY_ENDPOINT` | `http://127.0.0.1:3677` | The gateway this daemon is a node of. It checks in there every `DORMICE_CHECK_IN_INTERVAL_SECONDS` — readings, build, where it can be reached, which configuration version it runs — and takes the fleet's configuration from the answer. A daemon with no copy yet waits for its gateway before it listens. | | `DORMICE_NODE_ENDPOINT` | `http://127.0.0.1:` | Where the gateway reaches this node. Required (an origin: scheme, host, port — no path) when the gateway is on another machine. | -| `DORMICE_CHECK_IN_INTERVAL_SECONDS` | `15` | How often the node checks in; the gateway reads two missed check-ins as down. | +| `DORMICE_CHECK_IN_INTERVAL_SECONDS` | `15` | How often the node checks in; the gateway reads two missed check-ins as down. At most `86400` (a day). | ### Executor @@ -129,7 +129,7 @@ is a fleet setting. | Variable | Default | What it does | | --- | --- | --- | -| `DORMICE_SCAN_INTERVAL_SECONDS` | `60` | How often the idle scanner runs. Each sweep moves an idle sandbox down at most one state — see [Sandbox lifecycle](/docs/lifecycle). | +| `DORMICE_SCAN_INTERVAL_SECONDS` | `60` | How often the idle scanner runs. Each sweep moves an idle sandbox down at most one state — see [Sandbox lifecycle](/docs/lifecycle). At most `86400` (a day). | | `DORMICE_RECLAIM_TIMEOUT_SECONDS` | `45` | Upper bound on the memory-reclaim step of a freeze. Hitting it is expected on stubborn workloads, not a failure. | ## Client-side variables From 6a37f968d35286aa7009367418aaff0564f16018 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 04:01:52 +0800 Subject: [PATCH 56/89] The reclaim timeout is capped like the intervals, and the gateway's sampler waits one interval before its first tick MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DORMICE_RECLAIM_TIMEOUT_SECONDS becomes execa's timeout, which rides the same Node timer as setTimeout: past 2^31-1 ms it fires after one millisecond, so every freeze's memory.reclaim would be killed at once and logged as the expected cut-short — idle would stop being free, with no error anywhere. The one-day ceiling now covers every knob that becomes a timer, under a name that says so (MAX_TIMER_SECONDS); the test loops over all four. The gateway's sampler no longer fires at boot: that shot could never write (the readings are the nodes' to report, and none has yet), so the comment explained a tick that did nothing. The first tick is one interval after boot, which is when the first row could land anyway; the measured 54s gap after a 0.6s restart is unchanged. --- packages/gateway/src/config.ts | 18 +++++++------- packages/gateway/src/main.ts | 29 ++++++++++------------ packages/server/src/config.test.ts | 5 +++- packages/server/src/config.ts | 33 +++++++++++++++++--------- website/content/docs/configuration.mdx | 2 +- 5 files changed, 49 insertions(+), 38 deletions(-) diff --git a/packages/gateway/src/config.ts b/packages/gateway/src/config.ts index 76278a7f..0cacfb8a 100644 --- a/packages/gateway/src/config.ts +++ b/packages/gateway/src/config.ts @@ -23,15 +23,15 @@ import { z } from 'zod'; */ /** - * The ceiling on an interval knob: one day. A tick a day is the slowest - * cadence that still means anything — and past 2^31-1 ms (24.8 days) - * Node's setTimeout does not wait at all: it warns and fires after one - * millisecond (TimeoutOverflowWarning), which for a sampler is a row - * every millisecond. Refused at boot instead (found by review, - * 2026-09-15; the daemon's interval knobs carry the same rule, + * The ceiling on a knob that becomes a Node timer: one day. A tick a day + * is the slowest cadence that still means anything — and past 2^31-1 ms + * (24.8 days) Node's setTimeout does not wait at all: it warns and fires + * after one millisecond (TimeoutOverflowWarning), which for a sampler is + * a row every millisecond. Refused at boot instead (found by review, + * 2026-09-15; the daemon's timer knobs carry the same rule, * server/config.ts). */ -const MAX_INTERVAL_SECONDS = 86_400; +const MAX_TIMER_SECONDS = 86_400; const envSchema = z.object({ DORMICE_GATEWAY_PORT: z.coerce.number().int().min(1).max(65535).default(3677), @@ -100,13 +100,13 @@ const envSchema = z.object({ * node check-in grew with the fleet. 30, the daemon's own sampling * interval, so a single node's imported history and the gateway's join * at the same density; the exam sets 1. At most a day - * (MAX_INTERVAL_SECONDS). + * (MAX_TIMER_SECONDS). */ DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS: z.coerce .number() .int() .positive() - .max(MAX_INTERVAL_SECONDS) + .max(MAX_TIMER_SECONDS) .default(30), // ---- first-boot seeds of the settings table, in the daemon's words ---- DORMICE_SANDBOX_DISK_GB: z.coerce.number().positive().default(10), diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts index 7418e96d..b6908d24 100644 --- a/packages/gateway/src/main.ts +++ b/packages/gateway/src/main.ts @@ -149,15 +149,17 @@ let closing = false; // daemon's metrics ticker in shape (server/main.ts): every interval one // row of the fleet's census, summed from the readings the check-ins left // in memory (db/fleet-samples.ts says when no row is true enough to -// write). The first shot fires at once and, on a gateway just started, -// writes nothing: the readings are the nodes' to report, and none has -// yet. So the first row after a restart lands at the first tick after -// every known node has checked in, and the curve's gap is the downtime -// plus at most one check-in interval and one sample interval (measured -// 2026-09-15: 0.6s down, a 54s gap) — not the daemon's "gap equals -// downtime", whose figures sit in its own ledger at boot where the -// gateway's sit in the nodes' mouths. Same failure stance: log, never -// fatal, the next tick retries — and no check-in ever waits on this write. +// write). The first tick is one interval after boot, not at once as the +// daemon's: the readings are the nodes' to report, none has yet, and a +// shot at boot would write nothing. So the first row after a restart is +// the first tick after every known node has checked in, and the curve's +// gap is the downtime plus at most one check-in interval and one sample +// interval (measured 2026-09-15: 0.6s down, a 54s gap) — not the +// daemon's "gap equals downtime", whose figures sit in its own ledger at +// boot where the gateway's sit in the nodes' mouths. Same failure +// stance: log, never fatal, the next tick retries — and no check-in ever +// waits on this write. +const sampleIntervalMs = config.DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS * 1000; let sampleTimer: NodeJS.Timeout | undefined; function sampleTick() { try { @@ -165,15 +167,10 @@ function sampleTick() { } catch (error) { app.log.error(error, 'fleet state sample failed'); } finally { - if (!closing) { - sampleTimer = setTimeout( - sampleTick, - config.DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS * 1000, - ); - } + if (!closing) sampleTimer = setTimeout(sampleTick, sampleIntervalMs); } } -sampleTimer = setTimeout(sampleTick, 0); +sampleTimer = setTimeout(sampleTick, sampleIntervalMs); const close = async (signal: NodeJS.Signals) => { if (closing) return; diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index be2460e1..cf4bbfd9 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -75,11 +75,14 @@ describe('the metrics sampler knobs', () => { ).toThrow(); }); - it('refuses an interval past a day on every ticker knob — past 2^31-1 ms Node would fire it every millisecond instead', () => { + it('refuses a value past a day on every knob that becomes a timer — past 2^31-1 ms Node would fire it after one millisecond instead', () => { for (const knob of [ 'DORMICE_SCAN_INTERVAL_SECONDS', 'DORMICE_METRICS_SAMPLE_INTERVAL_SECONDS', 'DORMICE_CHECK_IN_INTERVAL_SECONDS', + // execa's timeout is the same timer: overflowed, every freeze's + // memory.reclaim would be killed after one millisecond. + 'DORMICE_RECLAIM_TIMEOUT_SECONDS', ]) { const atTheCeiling = loadConfig({ ...TOKEN, [knob]: '86400' }) as Record< string, diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index b801400f..f3971edf 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -24,15 +24,18 @@ import { z } from 'zod'; */ /** - * The ceiling on the three interval knobs: one day. A tick a day is the - * slowest cadence that still means anything — and past 2^31-1 ms (24.8 - * days) Node's setTimeout does not wait at all: it warns and fires after - * one millisecond (TimeoutOverflowWarning) — a sweep, a sample or a - * check-in every millisecond. Refused at boot instead (found by review, - * 2026-09-15; the gateway's sampler knob carries the same rule, - * gateway/config.ts). + * The ceiling on every knob that becomes a Node timer — the three + * intervals and the reclaim timeout: one day. A day is the slowest + * cadence, and the longest bound, that still means anything — and past + * 2^31-1 ms (24.8 days) Node's timers do not wait at all: setTimeout + * warns and fires after one millisecond (TimeoutOverflowWarning), and + * execa's timeout rides the same timer. A sweep, a sample or a check-in + * every millisecond; a memory.reclaim killed at once on every freeze and + * logged as the expected cut-short, the idle sandboxes' memory never + * squeezed out. Refused at boot instead (found by review, 2026-09-15; + * the gateway's sampler knob carries the same rule, gateway/config.ts). */ -const MAX_INTERVAL_SECONDS = 86_400; +const MAX_TIMER_SECONDS = 86_400; const envSchema = z.object({ DORMICE_PORT: z.coerce.number().int().min(1).max(65535).default(3676), @@ -49,7 +52,7 @@ const envSchema = z.object({ .number() .int() .positive() - .max(MAX_INTERVAL_SECONDS) + .max(MAX_TIMER_SECONDS) .default(60), /** * How often the metrics sampler persists a reading per measurable sandbox @@ -59,7 +62,7 @@ const envSchema = z.object({ .number() .int() .positive() - .max(MAX_INTERVAL_SECONDS) + .max(MAX_TIMER_SECONDS) .default(30), /** * How long per-sandbox samples live (fleet rows are fixed at 30 days — @@ -91,10 +94,18 @@ const envSchema = z.object({ DORMICE_BASE_IMAGE: z.string().optional(), /** Sandbox disk images and their mount points live here (docker executor only). */ DORMICE_DATA_DIR: z.string().default('/var/lib/dormice'), + /** + * Upper bound on a freeze's memory.reclaim write (executor/docker.ts + * reclaimMemory) — a SIGKILL deadline on the writer, so a Node timer + * too, and capped like the intervals (MAX_TIMER_SECONDS): overflowed, + * every reclaim would be killed after one millisecond and logged as + * the expected cut-short, and idle would stop being free. + */ DORMICE_RECLAIM_TIMEOUT_SECONDS: z.coerce .number() .int() .positive() + .max(MAX_TIMER_SECONDS) .default(45), /** * The gateway this daemon is a node of — its intranet address, e.g. @@ -154,7 +165,7 @@ const envSchema = z.object({ .number() .int() .positive() - .max(MAX_INTERVAL_SECONDS) + .max(MAX_TIMER_SECONDS) .default(15), }); diff --git a/website/content/docs/configuration.mdx b/website/content/docs/configuration.mdx index e4c6d25d..bb5374e5 100644 --- a/website/content/docs/configuration.mdx +++ b/website/content/docs/configuration.mdx @@ -130,7 +130,7 @@ is a fleet setting. | Variable | Default | What it does | | --- | --- | --- | | `DORMICE_SCAN_INTERVAL_SECONDS` | `60` | How often the idle scanner runs. Each sweep moves an idle sandbox down at most one state — see [Sandbox lifecycle](/docs/lifecycle). At most `86400` (a day). | -| `DORMICE_RECLAIM_TIMEOUT_SECONDS` | `45` | Upper bound on the memory-reclaim step of a freeze. Hitting it is expected on stubborn workloads, not a failure. | +| `DORMICE_RECLAIM_TIMEOUT_SECONDS` | `45` | Upper bound on the memory-reclaim step of a freeze. Hitting it is expected on stubborn workloads, not a failure. At most `86400` (a day). | ## Client-side variables From 5890ea0944b747e5b50950b09e2d9e74f673b640 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 15:53:14 +0800 Subject: [PATCH 57/89] A node's last check-in lives on its row, and the gateway's four startup graces go with it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nodes table gains last_check_in_at, interval_seconds, config_version, build and reading (migration 0003). Every check-in writes them back, best-effort: a failed UPDATE is said once, memory is updated and the check-in is answered, bundle included — the row is what this process leaves the next one, and the configuration bundle must not wait on it. A join's INSERT still has to land. At start the Fleet loads each node as of its last check-in, the two JSON columns read back through the wire schemas (an unreadable one is null, said once, filled in next check-in). With that, "silent since the gateway started" is no longer a state a running node can be in, and the thirty-second STARTUP_GRACE_MS the third cut carried in four places — merged lists, the sampler's settling, removeNode's 409, placement's refusal word — is deleted along with Fleet.startedAt and the testing seam. A row that never checked in (the import will pre-create one) is "has never checked in". The sampler fires at boot again: its first row after a restart is the fleet as of just before it, so the curve's gap is the downtime. --- .../gateway/drizzle/0003_last-check-in.sql | 5 + .../gateway/drizzle/meta/0003_snapshot.json | 474 ++++++++++++++++++ packages/gateway/drizzle/meta/_journal.json | 7 + packages/gateway/src/app.test.ts | 42 +- packages/gateway/src/db/fleet-samples.ts | 28 +- packages/gateway/src/db/schema.ts | 35 +- packages/gateway/src/find.test.ts | 27 +- packages/gateway/src/find.ts | 6 +- packages/gateway/src/fleet.test.ts | 153 +++++- packages/gateway/src/fleet.ts | 292 +++++++---- packages/gateway/src/main.ts | 31 +- packages/gateway/src/merge.test.ts | 58 +-- packages/gateway/src/merge.ts | 31 +- packages/gateway/src/placement.test.ts | 30 +- packages/gateway/src/routes/fleet.test.ts | 82 ++- packages/gateway/src/routes/nodes.test.ts | 21 +- packages/gateway/src/routes/nodes.ts | 37 +- packages/gateway/src/routes/settings.test.ts | 9 +- packages/gateway/src/routes/settings.ts | 9 +- packages/gateway/src/testing.ts | 4 +- packages/sdk/src/client.ts | 4 +- packages/shared/src/gateway.ts | 11 +- website/content/docs/http-api.mdx | 2 +- website/content/docs/metrics.mdx | 7 +- 24 files changed, 1071 insertions(+), 334 deletions(-) create mode 100644 packages/gateway/drizzle/0003_last-check-in.sql create mode 100644 packages/gateway/drizzle/meta/0003_snapshot.json diff --git a/packages/gateway/drizzle/0003_last-check-in.sql b/packages/gateway/drizzle/0003_last-check-in.sql new file mode 100644 index 00000000..c55aa510 --- /dev/null +++ b/packages/gateway/drizzle/0003_last-check-in.sql @@ -0,0 +1,5 @@ +ALTER TABLE `nodes` ADD `last_check_in_at` text;--> statement-breakpoint +ALTER TABLE `nodes` ADD `interval_seconds` integer;--> statement-breakpoint +ALTER TABLE `nodes` ADD `config_version` integer;--> statement-breakpoint +ALTER TABLE `nodes` ADD `build` text;--> statement-breakpoint +ALTER TABLE `nodes` ADD `reading` text; \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/0003_snapshot.json b/packages/gateway/drizzle/meta/0003_snapshot.json new file mode 100644 index 00000000..e69a0d79 --- /dev/null +++ b/packages/gateway/drizzle/meta/0003_snapshot.json @@ -0,0 +1,474 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "8fba703e-7fc7-4bfc-aa8d-903f01a049d6", + "prevId": "eba4fadc-ca03-41c6-b5a1-72adf1b2552c", + "tables": { + "api_keys": { + "name": "api_keys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + "key_hash" + ], + "isUnique": true + }, + "api_keys_active_name_idx": { + "name": "api_keys_active_name_idx", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"api_keys\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "console_account": { + "name": "console_account", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_secret": { + "name": "session_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "fleet_state_samples": { + "name": "fleet_state_samples", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active": { + "name": "active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "frozen": { + "name": "frozen", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stopped": { + "name": "stopped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "restoring": { + "name": "restoring", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "fleet_state_samples_at_idx": { + "name": "fleet_state_samples_at_idx", + "columns": [ + "at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "nodes": { + "name": "nodes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "added_at": { + "name": "added_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_gb": { + "name": "swap_gb", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_check_in_at": { + "name": "last_check_in_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "interval_seconds": { + "name": "interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config_version": { + "name": "config_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "build": { + "name": "build", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reading": { + "name": "reading", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_cpus": { + "name": "sandbox_cpus", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_memory_gb": { + "name": "sandbox_memory_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_disk_gb": { + "name": "sandbox_disk_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_freeze_after_seconds": { + "name": "default_freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_stop_after_seconds": { + "name": "default_stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_archive_after_seconds": { + "name": "default_archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_endpoint": { + "name": "s3_endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_access_key_id": { + "name": "s3_access_key_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_secret_access_key": { + "name": "s3_secret_access_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_region": { + "name": "s3_region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_force_path_style": { + "name": "s3_force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain": { + "name": "sandbox_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain_aliases": { + "name": "sandbox_domain_aliases", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pids_limit": { + "name": "pids_limit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/_journal.json b/packages/gateway/drizzle/meta/_journal.json index 36e3069b..0f7d5393 100644 --- a/packages/gateway/drizzle/meta/_journal.json +++ b/packages/gateway/drizzle/meta/_journal.json @@ -22,6 +22,13 @@ "when": 1789403526928, "tag": "0002_fleet-state-samples", "breakpoints": true + }, + { + "idx": 3, + "version": "6", + "when": 1789458250873, + "tag": "0003_last-check-in", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index a10962a8..25c20f7b 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -449,8 +449,6 @@ async function gateway( nodeIds: string[], env: Record = {}, opts: { - /** When the gateway "started" — the removeNode startup grace is judged against it. */ - startedAt?: Date; /** Collects the gateway's own log lines (JSON, one per entry) when a test asserts on what it says. */ logs?: string[]; /** How the gateway asks nodes on its own account — a test shortens its patience for the slow-node case. */ @@ -467,7 +465,7 @@ async function gateway( ...env, }); ensureSettings(db, config); - const fleet = new Fleet(db, opts.startedAt); + const fleet = new Fleet(db); const cache = new NameCache(); const finder = new Finder(fleet, cache, httpAskNode(TOKEN), { warn: () => {}, @@ -797,35 +795,25 @@ describe('check-in and the node verbs', () => { expect(rides()).toHaveLength(2); }); - it('right after a gateway start a node not yet heard from cannot be removed; past two default intervals it can', async () => { - // A restart: the rows are known, nothing has checked in yet. - const fresh = await gateway(['b']); - const b = fresh.fleet.get('b'); + it("a row that never checked in (the import pre-creates one) is removed at once; a node that checked in seconds ago is refused — as of its row after a restart too (fleet.test), with no grace for the gateway's own age", async () => { + const h = await gateway(['b']); + // The row as the import leaves it, in the shape fleet.ts loads for it: + // known, never heard from, nothing to protect. + const b = h.fleet.get('b'); if (!b) throw new Error('node lost'); b.lastCheckInAt = null; b.intervalSeconds = null; - const early = await rpc(fresh, '/removeNode', { id: 'b' }); - expect(early.status).toBe(409); - expect(message(early)).toMatch( - /^the gateway started \ds ago and has not heard from node b yet/, - ); - expect(fresh.fleet.get('b')).toBeDefined(); - // The same silence thirty-one seconds into the gateway's life is a - // node that is down. - const settled = await gateway( - ['b'], - {}, - { - startedAt: new Date(Date.now() - 31_000), - }, - ); - const quiet = settled.fleet.get('b'); - if (!quiet) throw new Error('node lost'); - quiet.lastCheckInAt = null; - quiet.intervalSeconds = null; - expect((await rpc(settled, '/removeNode', { id: 'b' })).body).toEqual({ + b.reading = null; + expect((await rpc(h, '/removeNode', { id: 'b' })).body).toEqual({ removed: true, }); + // Checked in seconds ago — to this process, or per its row to the one + // before: the same refusal. + await h.checkIn(h.nodes[0] as FakeNode); + const live = await rpc(h, '/removeNode', { id: 'b' }); + expect(live.status).toBe(409); + expect(message(live)).toMatch(/^node b checked in \ds ago — it is running/); + expect(h.fleet.get('b')).toBeDefined(); }); }); diff --git a/packages/gateway/src/db/fleet-samples.ts b/packages/gateway/src/db/fleet-samples.ts index 53601a67..ef03abed 100644 --- a/packages/gateway/src/db/fleet-samples.ts +++ b/packages/gateway/src/db/fleet-samples.ts @@ -1,5 +1,5 @@ import { and, asc, desc, gte, lt, lte } from 'drizzle-orm'; -import { type Fleet, STARTUP_GRACE_MS, sumReadings } from '../fleet'; +import { type Fleet, sumReadings } from '../fleet'; import type { Db } from './db'; import { type FleetStateSampleRow, fleetStateSamples } from './schema'; @@ -24,22 +24,20 @@ export const FLEET_SAMPLE_KEEP_DAYS = 30; * configuration bundle riding on its answer. * * Not written when there is nothing true to write: no node has a reading - * (nothing has checked in since this start — the fleet's sandboxes are - * on the nodes' disks, unknown here, and a zero row would draw a cliff - * the fleet did not fall off); or, within STARTUP_GRACE_MS of a start, - * while any node from the rows has not checked in yet — a restarted - * gateway hears from its nodes one by one over an interval, and a sum - * written after the first would draw the fleet collapsing to that node's - * share and climbing back, a false dip at every gateway restart. Past the - * grace, a node still silent is down, and the sum is written without it - * (a lower bound, as getFleetMetrics says of the same figure). Answers - * whether a row was written. + * — an empty fleet, or rows that never checked in (the import's) — the + * fleet's sandboxes are on the nodes' disks, unknown here, and a zero + * row would draw a cliff the fleet did not fall off. A restart draws no + * false dip either: every node's last reading comes back from its row + * (fleet.ts), so the first tick after a restart sums the whole fleet as + * of before it. The third cut, with the readings in memory only, held + * the write for a startup grace while any known node had not checked in + * — a sum written after the first would have drawn the fleet collapsing + * to that node's share and climbing back; the rows made the rule + * unnecessary (fourth cut). Answers whether a row was written. */ export function recordFleetSample(db: Db, fleet: Fleet, now: Date): boolean { - const nodes = fleet.all(); - const { reported, sandboxes } = sumReadings(nodes); - const settling = now.getTime() - fleet.startedAt.getTime() < STARTUP_GRACE_MS; - if (reported === 0 || (settling && reported < nodes.length)) return false; + const { reported, sandboxes } = sumReadings(fleet.all()); + if (reported === 0) return false; const cutoff = new Date( now.getTime() - FLEET_SAMPLE_KEEP_DAYS * 86_400_000, ).toISOString(); diff --git a/packages/gateway/src/db/schema.ts b/packages/gateway/src/db/schema.ts index 581160d3..db9bdaf5 100644 --- a/packages/gateway/src/db/schema.ts +++ b/packages/gateway/src/db/schema.ts @@ -23,16 +23,21 @@ import { /** * Every node that has ever checked in (routes/nodes.ts): its id, the - * address the gateway forwards to, when it first appeared, and the one - * per-node setting — how much swap its daemon manages on its own disk. - * Written by the nodes themselves at their first check-in — there is no - * registration verb and no nodes file, so "which nodes exist" has exactly - * one home — and deleted only by an operator's removeNode. Persistent, not - * memory, for one reason: a node that is down must still be known after a - * gateway restart, or a name that lives only there would be placed anew - * elsewhere and come back as a conflict when the node returns. Everything - * the node last reported (its reading, build, check-in time) is memory: - * fifteen seconds later it is reported again. + * address the gateway forwards to, when it first appeared, the one + * per-node setting — how much swap its daemon manages on its own disk — + * and, since the fourth cut, what it last reported: when and at what + * interval, running which configuration version and build, and its + * reading (two JSON columns in the wire schemas' shapes). Written by the + * nodes themselves at every check-in — there is no registration verb and + * no nodes file, so "which nodes exist" has exactly one home — and + * deleted only by an operator's removeNode. Persistent, not memory, for + * two reasons: a node that is down must still be known after a gateway + * restart, or a name that lives only there would be placed anew elsewhere + * and come back as a conflict when the node returns; and a restarted + * gateway must judge its nodes by their last check-in, not by its own age + * — with the check-in in memory only (the third cut), "silent since the + * gateway started" was true of every node for up to an interval after + * every restart, and four rules carried a thirty-second grace for it. */ export const nodes = sqliteTable('nodes', { /** DORMICE_NODE_ID as the node states it — the `nodeId` in every sandbox answer. */ @@ -49,6 +54,16 @@ export const nodes = sqliteTable('nodes', { * 0 = manage none, the only value that fits every host at birth. */ swapGb: integer('swap_gb').notNull().default(0), + /** ISO 8601 UTC — the last check-in taken; null for a row that has never checked in (the import pre-creates one). */ + lastCheckInAt: text('last_check_in_at'), + /** The interval the node stated at that check-in — the yardstick for "two missed". */ + intervalSeconds: integer('interval_seconds'), + /** The configuration version the node said it runs; null = no copy yet, or never said. */ + configVersion: integer('config_version'), + /** JSON, shared buildInfoSchema; null = a dist built outside a checkout, or never checked in. */ + build: text('build'), + /** JSON, shared nodeReadingSchema; null until the first check-in. */ + reading: text('reading'), }); export type NodeRow = typeof nodes.$inferSelect; diff --git a/packages/gateway/src/find.test.ts b/packages/gateway/src/find.test.ts index 9077d848..3f141fa9 100644 --- a/packages/gateway/src/find.test.ts +++ b/packages/gateway/src/find.test.ts @@ -11,6 +11,7 @@ import { } from './ask'; import { NameCache } from './cache'; import { migrateDb, openDb } from './db/db'; +import { nodes } from './db/schema'; import { Finder } from './find'; import { Fleet } from './fleet'; import { checkInOf } from './testing'; @@ -175,7 +176,7 @@ describe('Finder', () => { expect(queries.length).toBe(4); }); - it('a node that reported no configuration copy is not dialled: empty, it is a no; holding sandboxes, it is silence saying so — and a node not heard from since the start is asked', async () => { + it('a node that reported no configuration copy is not dialled: empty, it is a no; holding sandboxes, it is silence saying so — after a restart too, from its row; a row that never checked in is asked', async () => { const db = openDb(':memory:'); migrateDb(db, MIGRATIONS); const fleet = new Fleet(db); @@ -204,13 +205,25 @@ describe('Finder', () => { }); expect(asked).toEqual(['a']); - // A gateway restarted over the same rows: nothing has checked in, - // both are asked — b may be running on the copy it kept. + // A gateway restarted over the same rows judges b by what its row + // kept — no copy, three sandboxes: still not dialled, still named — + // and asks a row that never checked in (the import pre-creates one) + // like any other node: nothing is known of it. asked.length = 0; - await new Finder(new Fleet(db), new NameCache(), ask, silentLog).byName( - 'new-name', - ); - expect(asked.sort()).toEqual(['a', 'b']); + db.insert(nodes) + .values({ id: 'n', endpoint: 'http://n:80', addedAt: NOW.toISOString() }) + .run(); + const restarted = await new Finder( + new Fleet(db), + new NameCache(), + ask, + silentLog, + ).byName('new-name'); + expect(restarted).toMatchObject({ + kind: 'unsure', + silent: [{ nodeId: 'b', why: expect.stringMatching(/not listening/) }], + }); + expect(asked.sort()).toEqual(['a', 'n']); }); it('an empty fleet finds nothing and asks nobody', async () => { diff --git a/packages/gateway/src/find.ts b/packages/gateway/src/find.ts index 269ed3f2..09f9e995 100644 --- a/packages/gateway/src/find.ts +++ b/packages/gateway/src/find.ts @@ -73,9 +73,9 @@ export class Finder { } /** - * One node's answer to one question — or, for a node that has checked - * in since this gateway started and reported no configuration copy, the - * answer without the question (fleet.ts awaitingFirstConfig): its port + * One node's answer to one question — or, for a node whose last + * check-in reported no configuration copy, the answer without the + * question (fleet.ts awaitingFirstConfig): its port * is shut until its first bundle applies, so a dial there is refused at * the socket and would read as silence — a 503 to every caller of every * uncached name for as long as the node boots (left by the second cut's diff --git a/packages/gateway/src/fleet.test.ts b/packages/gateway/src/fleet.test.ts index f6f5ae53..a39024e5 100644 --- a/packages/gateway/src/fleet.test.ts +++ b/packages/gateway/src/fleet.test.ts @@ -1,7 +1,8 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; import { migrateDb, openDb } from './db/db'; -import { type CheckInOutcome, downReason, Fleet } from './fleet'; +import { nodes } from './db/schema'; +import { type CheckInOutcome, downReason, Fleet, type FleetLog } from './fleet'; import { checkInOf } from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); @@ -19,19 +20,34 @@ function db() { return handle; } +/** A log that keeps what the fleet said, level and structured part included. */ +function collector() { + const lines: Array<{ + level: string; + obj: Record; + msg: string; + }> = []; + const line = (level: string) => (obj: object, msg: string) => + lines.push({ level, obj: obj as Record, msg }); + const log: FleetLog = { + info: line('info'), + warn: line('warn'), + error: line('error'), + }; + return { lines, log }; +} + describe('Fleet', () => { - it('a first check-in joins the node and persists its row; a gateway restart still knows it, unreached until it checks in again', () => { + it('a first check-in joins the node and persists what it said; a gateway restart reads the node back as of that check-in and judges it by it', () => { const handle = db(); - const fleet = new Fleet(handle, NOW); + const fleet = new Fleet(handle); expect(fleet.all()).toEqual([]); - expect(fleet.startedAt).toBe(NOW); - // Left unsaid, the start is now. - expect(Date.now() - new Fleet(handle).startedAt.getTime()).toBeLessThan( - 5_000, - ); const { node, joined } = taken( fleet.checkIn( - checkInOf('node-b', 'http://10.0.0.7:80', { intervalSeconds: 15 }), + checkInOf('node-b', 'http://10.0.0.7:80', { + intervalSeconds: 15, + active: 4, + }), NOW, ), ); @@ -41,17 +57,124 @@ describe('Fleet', () => { expect(node.reading?.host.cpuCount).toBe(8); expect(downReason(node, NOW)).toBeNull(); - // The same database after a restart: the row is there, the memory is not. + // The same database after a restart: the row carries the check-in, + // and the node is judged by it — up a moment on, down two of its + // intervals on — never by how long this gateway has been running. const restarted = new Fleet(handle); const known = restarted.get('node-b'); - expect(known?.endpoint).toBe('http://10.0.0.7:80'); - expect(known?.addedAt).toBe(NOW.toISOString()); - expect(known?.reading).toBeNull(); - expect(downReason(known as NonNullable, NOW)).toBe( - 'has not checked in since the gateway started', + if (!known) throw new Error('row lost'); + expect(known).toMatchObject({ + endpoint: 'http://10.0.0.7:80', + addedAt: NOW.toISOString(), + lastCheckInAt: NOW, + intervalSeconds: 15, + configVersion: 1, + build: { + commit: 'abc1234', + title: 'a commit', + committedAt: '2026-09-14T00:00:00.000Z', + }, + placedSinceCheckIn: 0, + }); + expect(known.reading).toEqual(node.reading); + expect(downReason(known, new Date(NOW.getTime() + 29_000))).toBeNull(); + expect(downReason(known, new Date(NOW.getTime() + 31_000))).toBe( + 'has not checked in for 31s', ); }); + it('a row that never checked in (the import pre-creates one) is known and down, with the word for it; a row whose JSON no longer reads is loaded without it, said once, and filled in by the next check-in', () => { + const handle = db(); + handle + .insert(nodes) + .values({ + id: 'node-1', + endpoint: 'http://127.0.0.1:3676', + addedAt: NOW.toISOString(), + swapGb: 8, + }) + .run(); + handle + .insert(nodes) + .values({ + id: 'node-x', + endpoint: 'http://x:80', + addedAt: NOW.toISOString(), + lastCheckInAt: NOW.toISOString(), + intervalSeconds: 15, + configVersion: 1, + build: '{"commit":', + reading: JSON.stringify({ host: 'not a reading' }), + }) + .run(); + const { lines, log } = collector(); + const fleet = new Fleet(handle, log); + const fresh = fleet.get('node-1'); + if (!fresh) throw new Error('row lost'); + expect(fresh).toMatchObject({ + swapGb: 8, + lastCheckInAt: null, + intervalSeconds: null, + configVersion: null, + build: null, + reading: null, + }); + expect(downReason(fresh, NOW)).toBe('has never checked in'); + expect(fleet.get('node-x')).toMatchObject({ + lastCheckInAt: NOW, + intervalSeconds: 15, + build: null, + reading: null, + }); + expect(lines.map((l) => [l.level, l.obj.nodeId, l.obj.column])).toEqual([ + ['warn', 'node-x', 'build'], + ['warn', 'node-x', 'reading'], + ]); + taken(fleet.checkIn(checkInOf('node-x', 'http://x:80'), NOW)); + expect(new Fleet(handle).get('node-x')?.reading?.host.cpuCount).toBe(8); + }); + + it('a check-in whose row cannot be written is taken all the same — memory updated, one error line until the writes succeed again; a join must land', () => { + const handle = db(); + const { lines, log } = collector(); + const fleet = new Fleet(handle, log); + const at = (seconds: number) => new Date(NOW.getTime() + seconds * 1000); + const report = (active: number, when: Date) => + taken( + fleet.checkIn( + checkInOf('node-b', 'http://10.0.0.7:80', { active }), + when, + ), + ).node; + report(1, NOW); + // Every write refused from here — a full disk's shape. + handle.$client.pragma('query_only = 1'); + const node = report(2, at(15)); + expect(node.lastCheckInAt).toEqual(at(15)); + expect(node.reading?.sandboxes.byState.active).toBe(2); + report(3, at(30)); + expect(lines.map((l) => [l.level, l.obj.nodeId])).toEqual([ + ['error', 'node-b'], + ]); + // The row still says the last check-in that was written. + expect( + new Fleet(handle).get('node-b')?.reading?.sandboxes.byState.active, + ).toBe(1); + handle.$client.pragma('query_only = 0'); + report(4, at(45)); + expect(lines.map((l) => l.level)).toEqual(['error', 'info']); + expect( + new Fleet(handle).get('node-b')?.reading?.sandboxes.byState.active, + ).toBe(4); + // A join, by contrast, throws: a node no row holds is unknown to the + // next gateway process. + handle.$client.pragma('query_only = 1'); + expect(() => + fleet.checkIn(checkInOf('node-c', 'http://c:80'), NOW), + ).toThrow(); + expect(fleet.get('node-c')).toBeUndefined(); + }); + it('a later check-in is not a join; a changed endpoint inside the interval is refused as a second daemon, an interval later written through as a move; the placement counter restarts', () => { const handle = db(); const fleet = new Fleet(handle); diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 966f09eb..12c93324 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -1,25 +1,33 @@ -import type { - BuildInfo, - CheckInRequest, - NodeReading, - SandboxDisks, - SandboxStateCounts, +import { + type BuildInfo, + buildInfoSchema, + type CheckInRequest, + type NodeReading, + nodeReadingSchema, + type SandboxDisks, + type SandboxStateCounts, } from '@dormice/shared'; import { eq } from 'drizzle-orm'; +import type { z } from 'zod'; import type { Db } from './db/db'; -import { nodes } from './db/schema'; +import { type NodeRow, nodes } from './db/schema'; import { bumpConfigVersion } from './db/settings'; /** - * A node as the gateway knows it: the persistent row (id, endpoint, - * addedAt) and what it last reported (memory — reported again at the next - * check-in, gone with the process and rightly so). placedSinceCheckIn - * counts the sandboxes the gateway sent here since the last reading, so a - * burst inside one interval is counted against the node before its next - * reading shows it (placement.ts); placedIds are the ones among them whose - * create answered with an id, so a sandbox placed and destroyed inside one - * interval — a short job, the exam's churn — is taken off the count again - * instead of holding a slot the reading will never show (routes/destroy.ts). + * A node as the gateway knows it: its row — id, endpoint, addedAt, the + * swap target, and what it last reported: when and at what interval, the + * configuration version, the build, the reading (written back at every + * check-in since the fourth cut, so a restarted gateway starts from the + * last check-in and not from nothing) — and the gateway's own counters. + * placedSinceCheckIn counts the sandboxes the gateway sent here since the + * last reading, so a burst inside one interval is counted against the + * node before its next reading shows it (placement.ts); placedIds are the + * ones among them whose create answered with an id, so a sandbox placed + * and destroyed inside one interval — a short job, the exam's churn — is + * taken off the count again instead of holding a slot the reading will + * never show (routes/destroy.ts). The counters are memory: they count + * what this process did since the reading, and a restarted one has done + * nothing yet. */ export interface NodeState { readonly id: string; @@ -29,6 +37,7 @@ export interface NodeState { swapGb: number; /** The configuration version the node last reported running; null before it has said. */ configVersion: number | null; + /** The last check-in taken; null for a row that has never checked in (the import pre-creates one). */ lastCheckInAt: Date | null; intervalSeconds: number | null; build: BuildInfo | null; @@ -38,30 +47,35 @@ export interface NodeState { } /** - * How long after a gateway start a node that has not checked in yet is - * still presumed alive. A restarted gateway knows its nodes from their - * rows and nothing else: every lastCheckInAt is null, and "has not checked - * in since the gateway started" is true of a healthy node for up to one - * of its intervals. Judged by downReason alone, removeNode would let an - * operator delete a running node in that window (found by review, - * 2026-09-14). Two of the daemon's default intervals - * (DORMICE_CHECK_IN_INTERVAL_SECONDS, 15): the gateway cannot know a - * node's own interval before it has heard from it once. + * What the fleet says for itself — a row it could not read back, a row it + * could not write. Pino's shape (main.ts passes the gateway's logger); + * silent by default, for the suites that embed a fleet. */ -export const STARTUP_GRACE_MS = 2 * 15 * 1000; +export interface FleetLog { + info(obj: object, msg: string): void; + warn(obj: object, msg: string): void; + error(obj: object, msg: string): void; +} + +const SILENT_LOG: FleetLog = { info() {}, warn() {}, error() {} }; /** * Why a node is not to be placed on right now, or null when it is fine: - * never checked in since this gateway started, or silent for two of its - * own intervals — the interval it stated in its last check-in, so the - * exam's one-second nodes and production's fifteen-second nodes are judged - * by the same rule. Two, not one: a check-in delayed by a busy event loop - * or a slow reading is normal; two in a row missing is a node in trouble. - * The same word answers listNodes' `reachable`. + * never checked in — a row the import pre-created, before the node's + * first check-in — or silent for two of its own intervals: the interval + * it stated in its last check-in, so the exam's one-second nodes and + * production's fifteen-second nodes are judged by the same rule. Two, + * not one: a check-in delayed by a busy event loop or a slow reading is + * normal; two in a row missing is a node in trouble. The last check-in + * is the row's as much as memory's (Fleet has why), so a restarted + * gateway reads a node that checked in seconds before the restart as up + * and one silent since long before it as down — and no rule here needs + * to know how old the gateway is. The same word answers listNodes' + * `reachable`. */ export function downReason(node: NodeState, now: Date): string | null { if (node.lastCheckInAt === null || node.intervalSeconds === null) { - return 'has not checked in since the gateway started'; + return 'has never checked in'; } const silentMs = now.getTime() - node.lastCheckInAt.getTime(); if (silentMs > 2 * node.intervalSeconds * 1000) { @@ -71,14 +85,13 @@ export function downReason(node: NodeState, now: Date): string | null { } /** - * A node that has checked in since this gateway started and reported no - * configuration copy: its daemon fetches its first bundle before it opens - * its port (server/main.ts, CheckIn.untilConfigured), so until its next - * check-in says otherwise nothing dialled there answers — the socket is - * shut. Placement refuses it, a lookup does not dial it (find.ts), a - * merged list does not wait on it (merge.ts). A node not heard from at - * all since the start is not this: it may well be running on a copy it - * kept, and is asked like any other. + * A node whose last check-in reported no configuration copy: its daemon + * fetches its first bundle before it opens its port (server/main.ts, + * CheckIn.untilConfigured), so until its next check-in says otherwise + * nothing dialled there answers — the socket is shut. Placement refuses + * it, a lookup does not dial it (find.ts), a merged list does not wait on + * it (merge.ts). A node that has never checked in is not this: nothing + * is known of it, and it is asked like any other. */ export function awaitingFirstConfig(node: NodeState): boolean { return node.reading !== null && node.configVersion === null; @@ -101,13 +114,15 @@ export function awaitingFirstConfigWhy(node: NodeState): string | null { /** * The figures that add up across the fleet, summed over the nodes that * have a reading — the census by state and the sandbox disks' bill — and - * how many nodes that is. A node without a reading (not heard from since - * this gateway started) contributes nothing, and `reported` says so: the - * sums are a lower bound until every node has spoken. A node that is - * down but did report contributes its last reading — its sandboxes are - * still there, merely out of reach. One function for the sampler's row - * (db/fleet-samples.ts) and getFleetMetrics (routes/fleet.ts), so the - * curve and the number under it can never disagree. + * how many nodes that is. A node without a reading (never checked in — a + * row the import pre-created) contributes nothing, and `reported` says + * so: the sums are a lower bound until every node has spoken once. A node + * that is down but did report contributes its last reading — its + * sandboxes are still there, merely out of reach — and so does every + * node right after a gateway restart, from its row. One function for the + * sampler's row (db/fleet-samples.ts) and getFleetMetrics + * (routes/fleet.ts), so the curve and the number under it can never + * disagree. */ export function sumReadings(nodes: readonly NodeState[]): { reported: number; @@ -155,37 +170,91 @@ export type CheckInOutcome = | { node: NodeState; joined: boolean; movedFrom: string | null } | { refused: string }; +/** The row's share of one check-in: what the node said, as the columns hold it (load reads them back). */ +type RowShare = Pick< + typeof nodes.$inferInsert, + | 'endpoint' + | 'lastCheckInAt' + | 'intervalSeconds' + | 'configVersion' + | 'build' + | 'reading' +>; + /** - * The fleet: every node that has ever checked in. Rows come from the - * database at start (a node that is down must still be known — its names - * are not new names); everything else fills in as the nodes report. + * The fleet: every node that has ever checked in, and the rows the import + * pre-creates. Rows come from the database at start and carry what each + * node last reported — a node that is down must still be known (its + * names are not new names), and a restarted gateway must judge its nodes + * by their last check-in, not by its own age: the third cut kept the + * check-in in memory only, and every restart began with a thirty-second + * grace in four places (placement, merged lists, the sampler, removeNode) + * to cover for "silent since the gateway started" being true of every + * node, the running ones included, for up to an interval. Memory is this + * process's truth; the row is what it leaves the next one. */ export class Fleet { private readonly members = new Map(); - - /** When this gateway process started — the yardstick for STARTUP_GRACE_MS. */ - readonly startedAt: Date; + /** The nodes whose row the last check-in could not write — said once when it starts, once when it stops (persist). */ + private readonly unwritten = new Set(); constructor( private readonly db: Db, - startedAt: Date = new Date(), + private readonly log: FleetLog = SILENT_LOG, ) { - this.startedAt = startedAt; for (const row of db.select().from(nodes).all()) { - this.members.set(row.id, { - id: row.id, - endpoint: row.endpoint, - addedAt: row.addedAt, - swapGb: row.swapGb, - configVersion: null, - lastCheckInAt: null, - intervalSeconds: null, - build: null, - reading: null, - placedSinceCheckIn: 0, - placedIds: new Set(), - }); + this.members.set(row.id, this.load(row)); + } + } + + /** + * A row as memory. The two JSON columns are read back through the + * shapes they were written from (the wire schemas): a column that fails + * them — a hand edit, a build that wrote a shape this one no longer + * reads — is null, said once here, and filled in again at the node's + * next check-in; never a gateway that refuses to start over one node's + * row. + */ + private load(row: NodeRow): NodeState { + return { + id: row.id, + endpoint: row.endpoint, + addedAt: row.addedAt, + swapGb: row.swapGb, + configVersion: row.configVersion, + lastCheckInAt: + row.lastCheckInAt === null ? null : new Date(row.lastCheckInAt), + intervalSeconds: row.intervalSeconds, + build: this.parseJson(row.id, 'build', row.build, buildInfoSchema), + reading: this.parseJson( + row.id, + 'reading', + row.reading, + nodeReadingSchema, + ), + placedSinceCheckIn: 0, + placedIds: new Set(), + }; + } + + private parseJson( + nodeId: string, + column: string, + json: string | null, + schema: z.ZodType, + ): T | null { + if (json === null) return null; + try { + const parsed = schema.safeParse(JSON.parse(json)); + if (parsed.success) return parsed.data; + } catch { + // Not JSON at all: said below, like a shape the schema refuses. } + this.log.warn( + { nodeId, column }, + "a column on the node's row is not readable and is dropped until its next check-in fills it in", + ); + return null; } all(): NodeState[] { @@ -214,16 +283,30 @@ export class Fleet { * row is written after the container is up) is in neither figure until * the next reading — one interval of slack, self-correcting, the same * as before. + * + * Everything the node said goes to its row as well as to memory. A join + * is the one write that must land — a node no row holds is unknown to + * the next gateway process, and its names are new names there — so its + * INSERT throws (a 500 the node retries an interval later). Every later + * check-in's UPDATE is best-effort (persist has why). */ checkIn(report: CheckInRequest, now = new Date()): CheckInOutcome { let node = this.members.get(report.nodeId); let joined = false; let movedFrom: string | null = null; + const said: RowShare = { + endpoint: report.endpoint, + lastCheckInAt: now.toISOString(), + intervalSeconds: report.intervalSeconds, + configVersion: report.configVersion, + build: report.build === null ? null : JSON.stringify(report.build), + reading: JSON.stringify(report.reading), + }; if (node === undefined) { const addedAt = now.toISOString(); this.db .insert(nodes) - .values({ id: report.nodeId, endpoint: report.endpoint, addedAt }) + .values({ id: report.nodeId, addedAt, ...said }) .run(); node = { id: report.nodeId, @@ -240,28 +323,26 @@ export class Fleet { }; this.members.set(node.id, node); joined = true; - } else if (node.endpoint !== report.endpoint) { - if ( - node.lastCheckInAt !== null && - node.intervalSeconds !== null && - now.getTime() - node.lastCheckInAt.getTime() < - node.intervalSeconds * 1000 - ) { - const ago = Math.round( - (now.getTime() - node.lastCheckInAt.getTime()) / 1000, - ); - return { - refused: `node ${report.nodeId} checked in from ${node.endpoint} ${ago}s ago and now from ${report.endpoint} — two daemons share one DORMICE_NODE_ID (give this one its own), or the node just moved (then its next check-in, an interval later, is taken)`, - }; + } else { + if (node.endpoint !== report.endpoint) { + if ( + node.lastCheckInAt !== null && + node.intervalSeconds !== null && + now.getTime() - node.lastCheckInAt.getTime() < + node.intervalSeconds * 1000 + ) { + const ago = Math.round( + (now.getTime() - node.lastCheckInAt.getTime()) / 1000, + ); + return { + refused: `node ${report.nodeId} checked in from ${node.endpoint} ${ago}s ago and now from ${report.endpoint} — two daemons share one DORMICE_NODE_ID (give this one its own), or the node just moved (then its next check-in, an interval later, is taken)`, + }; + } + movedFrom = node.endpoint; } - this.db - .update(nodes) - .set({ endpoint: report.endpoint }) - .where(eq(nodes.id, report.nodeId)) - .run(); - movedFrom = node.endpoint; - node.endpoint = report.endpoint; + this.persist(node.id, said); } + node.endpoint = report.endpoint; node.lastCheckInAt = now; node.intervalSeconds = report.intervalSeconds; node.build = report.build; @@ -272,6 +353,37 @@ export class Fleet { return { node, joined, movedFrom }; } + /** + * The row's share of a check-in, best-effort: a write that fails — a + * full disk, a file gone read-only — is said once, and the check-in is + * taken all the same. Memory is this process's truth and the row is + * what it leaves the next one; the configuration bundle riding on the + * check-in's answer must not wait on the record of who reported what + * (the sampler's stance, db/fleet-samples.ts: an observation's write + * has no say over the control plane). Said once, not per check-in: one + * failing disk is one situation, not four lines a minute per node — + * and said again when the writes succeed, so the log has both ends. + */ + private persist(id: string, said: RowShare): void { + try { + this.db.update(nodes).set(said).where(eq(nodes.id, id)).run(); + if (this.unwritten.delete(id)) { + this.log.info( + { nodeId: id }, + "the node's row takes its check-ins again", + ); + } + } catch (error) { + if (!this.unwritten.has(id)) { + this.unwritten.add(id); + this.log.error( + { nodeId: id, err: error }, + "the node's row could not be written at its check-in; the check-in is taken in memory, and a gateway restart would know the node only as of its last written check-in", + ); + } + } + } + /** * The one per-node setting, written to the row and counted as a * configuration change in the same transaction, so the node that pulls diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts index b6908d24..92a1de99 100644 --- a/packages/gateway/src/main.ts +++ b/packages/gateway/src/main.ts @@ -62,9 +62,10 @@ migrateDb(db, fileURLToPath(new URL('../drizzle', import.meta.url))); // The fleet's settings row, seeded from the env exactly once (db/settings.ts). ensureSettings(db, config); -// The fleet from the nodes table (a node that is down is still a node); -// the cache and the readings fill in as nodes report and callers ask. -const fleet = new Fleet(db); +// The fleet from the nodes table, each node as of its last check-in (a +// node that is down is still a node, and a restart forgets nothing the +// rows hold); the cache fills in as callers ask. +const fleet = new Fleet(db, log); const finder = new Finder( fleet, new NameCache(), @@ -147,18 +148,16 @@ let closing = false; // The fleet's state sampler — the gateway's one clock of its own, the // daemon's metrics ticker in shape (server/main.ts): every interval one -// row of the fleet's census, summed from the readings the check-ins left -// in memory (db/fleet-samples.ts says when no row is true enough to -// write). The first tick is one interval after boot, not at once as the -// daemon's: the readings are the nodes' to report, none has yet, and a -// shot at boot would write nothing. So the first row after a restart is -// the first tick after every known node has checked in, and the curve's -// gap is the downtime plus at most one check-in interval and one sample -// interval (measured 2026-09-15: 0.6s down, a 54s gap) — not the -// daemon's "gap equals downtime", whose figures sit in its own ledger at -// boot where the gateway's sit in the nodes' mouths. Same failure -// stance: log, never fatal, the next tick retries — and no check-in ever -// waits on this write. +// row of the fleet's census, summed from every node's last reading +// (db/fleet-samples.ts says when no row is true enough to write). The +// first tick is at boot, as the daemon's: the readings come back from +// the nodes' rows (fleet.ts), so the first row after a restart is the +// fleet as of just before it and the curve's gap is the downtime — the +// third cut, with the readings in memory only, had to wait an interval +// for the nodes to report again (measured 2026-09-15: 0.6s down, a 54s +// gap) and deleted the boot shot as necessarily empty; the rows turned +// that around. Same failure stance: log, never fatal, the next tick +// retries — and no check-in ever waits on this write. const sampleIntervalMs = config.DORMICE_GATEWAY_SAMPLE_INTERVAL_SECONDS * 1000; let sampleTimer: NodeJS.Timeout | undefined; function sampleTick() { @@ -170,7 +169,7 @@ function sampleTick() { if (!closing) sampleTimer = setTimeout(sampleTick, sampleIntervalMs); } } -sampleTimer = setTimeout(sampleTick, sampleIntervalMs); +sampleTimer = setTimeout(sampleTick, 0); const close = async (signal: NodeJS.Signals) => { if (closing) return; diff --git a/packages/gateway/src/merge.test.ts b/packages/gateway/src/merge.test.ts index fe2e3793..85c3c09c 100644 --- a/packages/gateway/src/merge.test.ts +++ b/packages/gateway/src/merge.test.ts @@ -3,7 +3,8 @@ import { describe, expect, it } from 'vitest'; import { z } from 'zod'; import type { AskVerb } from './ask'; import { migrateDb, openDb } from './db/db'; -import { Fleet, STARTUP_GRACE_MS } from './fleet'; +import { nodes } from './db/schema'; +import { Fleet } from './fleet'; import { askability, askEach, MERGE_TIMEOUT_MS } from './merge'; import { checkInOf } from './testing'; @@ -14,62 +15,63 @@ import { checkInOf } from './testing'; const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); const NOW = new Date('2026-09-15T00:00:00.000Z'); -function fleetAt(startedAt: Date) { +function fleetOver() { const db = openDb(':memory:'); migrateDb(db, MIGRATIONS); - return new Fleet(db, startedAt); + return { db, fleet: new Fleet(db) }; } describe('askability', () => { it('a node that checked in and runs a configuration is asked; one down for two of its intervals is not, with the reason', () => { - const fleet = fleetAt(NOW); + const { fleet } = fleetOver(); const a = fleet.checkIn(checkInOf('a', 'http://a:80'), NOW); if ('refused' in a) throw new Error(a.refused); - expect(askability(a.node, NOW, fleet.startedAt)).toEqual({ ask: true }); + expect(askability(a.node, NOW)).toEqual({ ask: true }); const later = new Date(NOW.getTime() + 31_000); - expect(askability(a.node, later, fleet.startedAt)).toEqual({ + expect(askability(a.node, later)).toEqual({ ask: false, why: 'has not checked in for 31s', }); }); it('a node awaiting its first configuration is not asked: holding nothing, nothing is said; holding sandboxes, it is named as not listening', () => { - const fleet = fleetAt(NOW); + const { fleet } = fleetOver(); const empty = fleet.checkIn( checkInOf('b', 'http://b:80', { configVersion: null, active: 0 }), NOW, ); if ('refused' in empty) throw new Error(empty.refused); - expect(askability(empty.node, NOW, fleet.startedAt)).toEqual({ - ask: false, - why: null, - }); + expect(askability(empty.node, NOW)).toEqual({ ask: false, why: null }); fleet.checkIn( checkInOf('b', 'http://b:80', { configVersion: null, active: 4 }), NOW, ); - expect(askability(empty.node, NOW, fleet.startedAt)).toEqual({ + expect(askability(empty.node, NOW)).toEqual({ ask: false, why: expect.stringMatching(/not listening — it holds 4 sandboxes/), }); }); - it('after a gateway start a node not yet heard from is asked for the grace period, and is down after it', () => { - const db = openDb(':memory:'); - migrateDb(db, MIGRATIONS); - new Fleet(db, NOW).checkIn(checkInOf('c', 'http://c:80'), NOW); - // A restart over the same rows: c is known, silent so far. - const restarted = new Fleet(db, NOW); + it('after a gateway restart a node is judged by the check-in its row kept: fresh, it is asked at once; two of its intervals silent, it is not; a row that never checked in is named as such', () => { + const { db, fleet } = fleetOver(); + fleet.checkIn(checkInOf('c', 'http://c:80'), NOW); + db.insert(nodes) + .values({ id: 'n', endpoint: 'http://n:80', addedAt: NOW.toISOString() }) + .run(); + const restarted = new Fleet(db); const c = restarted.get('c'); - if (!c) throw new Error('row lost'); - expect( - askability(c, new Date(NOW.getTime() + STARTUP_GRACE_MS - 1), NOW), - ).toEqual({ ask: true }); - expect( - askability(c, new Date(NOW.getTime() + STARTUP_GRACE_MS), NOW), - ).toEqual({ + const n = restarted.get('n'); + if (!c || !n) throw new Error('row lost'); + expect(askability(c, new Date(NOW.getTime() + 29_000))).toEqual({ + ask: true, + }); + expect(askability(c, new Date(NOW.getTime() + 31_000))).toEqual({ + ask: false, + why: 'has not checked in for 31s', + }); + expect(askability(n, NOW)).toEqual({ ask: false, - why: 'has not checked in since the gateway started', + why: 'has never checked in', }); }); }); @@ -78,7 +80,7 @@ describe('askEach', () => { const answerSchema = z.object({ items: z.array(z.string()) }); it('asks every askable node with the merge timeout, keeps every answer in node-id order, and names the silent ones', async () => { - const fleet = fleetAt(NOW); + const { fleet } = fleetOver(); for (const id of ['c', 'a', 'b', 'd']) { fleet.checkIn(checkInOf(id, `http://${id}:80`), NOW); } @@ -122,7 +124,7 @@ describe('askEach', () => { it('an empty fleet answers nothing and nobody is silent', async () => { const merged = await askEach( - fleetAt(NOW), + fleetOver().fleet, async () => { throw new Error('nobody to ask'); }, diff --git a/packages/gateway/src/merge.ts b/packages/gateway/src/merge.ts index 4c77b7ba..86c5d933 100644 --- a/packages/gateway/src/merge.ts +++ b/packages/gateway/src/merge.ts @@ -7,7 +7,6 @@ import { downReason, type Fleet, type NodeState, - STARTUP_GRACE_MS, } from './fleet'; /** @@ -26,31 +25,21 @@ export const MERGE_TIMEOUT_MS = 10_000; * answer says about it (`why` null: nothing — it holds nothing the answer * could lack). Not asked: * - a node that is down (fleet.ts downReason: two of its own intervals - * silent). A dial would wait the whole timeout for nothing, on every - * console poll, for as long as it stayed down — it is named with the - * reason placement refuses it; + * silent, or never checked in). A dial would wait the whole timeout + * for nothing, on every console poll, for as long as it stayed down — + * it is named with the reason placement refuses it; * - a node awaiting its first configuration (awaitingFirstConfig): not * listening, so a dial is refused at the socket and would read as * silence anyway. Named when its reading says it holds sandboxes. - * Asked regardless: a node not heard from since a gateway start, for - * STARTUP_GRACE_MS — every node is silent so far after a restart, the - * running ones included, and its last row's endpoint is most likely a - * live daemon (the removeNode rule, routes/nodes.ts). Past the grace, a - * node still not heard from is down. + * Judged from the row after a gateway restart as from memory before one + * (fleet.ts): a node that checked in seconds before the restart is asked + * at once. The third cut, holding the check-in in memory only, asked + * every node not yet heard from for a thirty-second grace after a start; + * the rows made the grace unnecessary (fourth cut). */ export type Askability = { ask: true } | { ask: false; why: string | null }; -export function askability( - node: NodeState, - now: Date, - startedAt: Date, -): Askability { - if ( - node.lastCheckInAt === null && - now.getTime() - startedAt.getTime() < STARTUP_GRACE_MS - ) { - return { ask: true }; - } +export function askability(node: NodeState, now: Date): Askability { const down = downReason(node, now); if (down !== null) return { ask: false, why: down }; if (awaitingFirstConfig(node)) { @@ -97,7 +86,7 @@ export async function askEach( const silent: SilentNode[] = []; const asked: NodeState[] = []; for (const node of fleet.all()) { - const judged = askability(node, now, fleet.startedAt); + const judged = askability(node, now); if (judged.ask) asked.push(node); else if (judged.why !== null) silent.push({ nodeId: node.id, why: judged.why }); diff --git a/packages/gateway/src/placement.test.ts b/packages/gateway/src/placement.test.ts index 165cf6de..2822997e 100644 --- a/packages/gateway/src/placement.test.ts +++ b/packages/gateway/src/placement.test.ts @@ -1,6 +1,7 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; import { type Db, migrateDb, openDb } from './db/db'; +import { nodes } from './db/schema'; import { Fleet, type NodeState } from './fleet'; import { type PlacementKnobs, pick, refusalMessage } from './placement'; import { checkInOf, type reading } from './testing'; @@ -130,7 +131,7 @@ describe('pick', () => { expect(alternating).toEqual(['c', 'd', 'c', 'd', 'c', 'd']); }); - it('refuses a node that never checked in since the start, or is silent for two of its intervals', () => { + it('refuses a node silent for two of its intervals, or one that never checked in — from the rows after a restart as from memory before', () => { const { db, fleet: f } = fleet(); const stale = node(f, 'stale', { intervalSeconds: 15, @@ -140,18 +141,31 @@ describe('pick', () => { intervalSeconds: 15, checkedInAt: new Date(NOW.getTime() - 29_000), }); - // The same row after a gateway restart: known, but nothing reported yet. - const restarted = new Fleet(db).get('fresh'); - if (!restarted) throw new Error('row lost'); - const result = pick([stale, restarted], KNOBS, NOW); - expect(result.node).toBeNull(); + const before = pick([stale, fresh], KNOBS, NOW); + expect(before.node?.id).toBe('fresh'); + expect(before.refused).toEqual([ + { nodeId: 'stale', reason: 'has not checked in for 31s' }, + ]); + // The same rows after a gateway restart, plus one the import + // pre-created: each judged by the check-in its row kept — the fresh + // one is placed on at once, the stale one refused as before, the + // never-heard-from one refused with the word for it. + db.insert(nodes) + .values({ + id: 'new', + endpoint: 'http://new:80', + addedAt: NOW.toISOString(), + }) + .run(); + const restarted = new Fleet(db); + const result = pick(restarted.all(), KNOBS, NOW); + expect(result.node?.id).toBe('fresh'); expect( Object.fromEntries(result.refused.map((r) => [r.nodeId, r.reason])), ).toEqual({ stale: 'has not checked in for 31s', - fresh: 'has not checked in since the gateway started', + new: 'has never checked in', }); - expect(pick([fresh], KNOBS, NOW).node?.id).toBe('fresh'); }); it('when every node refuses, each refusal names the node and its reason for the 503; no node at all says so', () => { diff --git a/packages/gateway/src/routes/fleet.test.ts b/packages/gateway/src/routes/fleet.test.ts index 61196fba..54a5282c 100644 --- a/packages/gateway/src/routes/fleet.test.ts +++ b/packages/gateway/src/routes/fleet.test.ts @@ -4,8 +4,8 @@ import { } from '@dormice/shared'; import { describe, expect, it } from 'vitest'; import { recordFleetSample } from '../db/fleet-samples'; -import { fleetStateSamples } from '../db/schema'; -import { STARTUP_GRACE_MS } from '../fleet'; +import { fleetStateSamples, nodes } from '../db/schema'; +import { Fleet } from '../fleet'; import { checkInOf, TEST_TOKEN, testGateway } from '../testing'; // The fleet's own observation: what the sampler writes (recordFleetSample @@ -32,9 +32,6 @@ async function checkIn( expect(res.statusCode).toBe(200); } -/** A gateway started long enough ago that no startup grace applies — against the real clock, which the check-in route reads. */ -const SETTLED = new Date(Date.now() - STARTUP_GRACE_MS - 60_000); - function samples(db: ReturnType['db']) { return db .select() @@ -45,7 +42,7 @@ function samples(db: ReturnType['db']) { describe('the fleet state sample the sampler writes', () => { it('a tick writes one row: the sum over every node with a reading, a down node counted by its last reading — and a check-in writes none', async () => { - const { app, db, fleet } = testGateway({}, { startedAt: SETTLED }); + const { app, db, fleet } = testGateway(); await checkIn(app, 'b', { active: 3, frozen: 1 }); await checkIn(app, 'c', { active: 2, archived: 4 }); // The check-ins themselves wrote nothing: the sampler's clock is the @@ -75,47 +72,41 @@ describe('the fleet state sample the sampler writes', () => { expect(samples(db)[2]).toMatchObject({ active: 3, total: 4 }); }); - it('no row while no node has a reading; within the startup grace none while a known node has not checked in; past it the sum is written without it', () => { - // A node known from the rows but not heard from since this start: a - // check-in's reading, then the memory a restart leaves — the row and - // nothing else (fleet.ts's constructor shape). - const silence = (fleet: ReturnType['fleet']) => { - fleet.checkIn(checkInOf('c', 'http://c:80', { active: 1 })); - const c = fleet.get('c'); - if (!c) throw new Error('node lost'); - c.reading = null; - c.lastCheckInAt = null; - c.intervalSeconds = null; - }; - // Nobody has reported: nothing true to write, however long ago the - // gateway started — and an empty fleet writes nothing either. - const unheard = testGateway({}, { startedAt: SETTLED }); - expect(recordFleetSample(unheard.db, unheard.fleet, new Date())).toBe( - false, - ); - silence(unheard.fleet); + it('no row while no node has a reading — an empty fleet, or rows that never checked in; after a restart the first tick sums every node from the reading its row kept', async () => { + // Nobody has ever reported: nothing true to write — an empty fleet, + // then a row as the import leaves it, known and never heard from. + const unheard = testGateway(); expect(recordFleetSample(unheard.db, unheard.fleet, new Date())).toBe( false, ); + unheard.db + .insert(nodes) + .values({ + id: 'n', + endpoint: 'http://n:80', + addedAt: new Date().toISOString(), + }) + .run(); + const preCreated = new Fleet(unheard.db); + expect(preCreated.get('n')?.reading).toBeNull(); + expect(recordFleetSample(unheard.db, preCreated, new Date())).toBe(false); expect(samples(unheard.db)).toHaveLength(0); - const fresh = testGateway({}, { startedAt: new Date() }); - silence(fresh.fleet); - fresh.fleet.checkIn(checkInOf('b', 'http://b:80', { active: 2 })); - expect(recordFleetSample(fresh.db, fresh.fleet, new Date())).toBe(false); - expect(samples(fresh.db)).toHaveLength(0); - - const settled = testGateway({}, { startedAt: SETTLED }); - silence(settled.fleet); - settled.fleet.checkIn(checkInOf('b', 'http://b:80', { active: 2 })); - expect(recordFleetSample(settled.db, settled.fleet, new Date())).toBe(true); - expect(samples(settled.db)).toEqual([ - expect.objectContaining({ active: 2, total: 2 }), + // Two nodes checked in, then the gateway restarts: nothing has spoken + // to the new process yet, and its first tick still sums both, from + // their rows — no dip, no wait. + const { app, db } = testGateway(); + await checkIn(app, 'b', { active: 2 }); + await checkIn(app, 'c', { active: 3, frozen: 1 }); + const restarted = new Fleet(db); + expect(recordFleetSample(db, restarted, new Date())).toBe(true); + expect(samples(db)).toEqual([ + expect.objectContaining({ active: 5, frozen: 1, total: 6 }), ]); }); it('rows older than 30 days are pruned with the write', () => { - const { db, fleet } = testGateway({}, { startedAt: SETTLED }); + const { db, fleet } = testGateway(); db.insert(fleetStateSamples) .values({ at: new Date(Date.now() - 31 * 86_400_000).toISOString(), @@ -137,11 +128,12 @@ describe('the fleet state sample the sampler writes', () => { describe('getFleetMetrics', () => { it('sums the census and the disks over the reported nodes, and says how many nodes that is', async () => { - const { app, fleet } = testGateway({}, { startedAt: SETTLED }); + const { app, fleet } = testGateway(); await checkIn(app, 'b', { active: 3, frozen: 1 }); await checkIn(app, 'c', { active: 2, archived: 4 }); - // A third node known from a previous run, not heard from: in total, - // not in reported, not in the sums. + // A third node whose row the import pre-created, never heard from + // (the shape fleet.ts loads for it): in total, not in reported, not + // in the sums. fleet.checkIn(checkInOf('d', 'http://d:80')); const d = fleet.get('d'); if (!d) throw new Error('node lost'); @@ -166,7 +158,7 @@ describe('getFleetMetrics', () => { }); it('sums the disks a reading carries', async () => { - const { app } = testGateway({}, { startedAt: SETTLED }); + const { app } = testGateway(); const withDisks = (id: string, count: number) => ({ ...checkInOf(id, `http://${id}:80`), reading: { @@ -191,7 +183,7 @@ describe('getFleetMetrics', () => { }); it('is behind the sandbox gate: a minted key reads it, no token does not', async () => { - const { app } = testGateway({}, { startedAt: SETTLED }); + const { app } = testGateway(); const minted = (await rpc(app, '/createApiKey', { name: 'ci' })).json(); const keyed = await app.inject({ method: 'POST', @@ -211,7 +203,7 @@ describe('getFleetMetrics', () => { describe('getFleetStateHistory', () => { it('answers an empty window with no points and a null peak; then the samples ascending, byState summing to total, the peak from raw rows', async () => { - const { app, db } = testGateway({}, { startedAt: SETTLED }); + const { app, db } = testGateway(); const empty = getFleetStateHistoryResponseSchema.parse( (await rpc(app, '/getFleetStateHistory', {})).json(), ); @@ -251,7 +243,7 @@ describe('getFleetStateHistory', () => { }); it("buckets past 360 points by keeping each bucket's last whole row, and the peak survives bucketing", async () => { - const { app, db } = testGateway({}, { startedAt: SETTLED }); + const { app, db } = testGateway(); const t0 = Date.parse('2026-09-15T00:00:00.000Z'); const rows = 400; const values = []; diff --git a/packages/gateway/src/routes/nodes.test.ts b/packages/gateway/src/routes/nodes.test.ts index a8b0ccd2..aab51782 100644 --- a/packages/gateway/src/routes/nodes.test.ts +++ b/packages/gateway/src/routes/nodes.test.ts @@ -117,6 +117,18 @@ describe('the check-in as the configuration pull', () => { (await checkIn(app, 'b', { configVersion: 9 })).config, ).toBeDefined(); }); + + it('a check-in the row cannot take is answered all the same, bundle included: the write is best-effort, memory is the truth', async () => { + const { app, db } = testGateway(); + await checkIn(app, 'b', { configVersion: 1 }); + // Every write refused from here — a full disk's shape. + db.$client.pragma('query_only = 1'); + const answer = await checkIn(app, 'b', { configVersion: null, active: 7 }); + expect(answer.config?.version).toBe(1); + const listed = (await nodes(app)).find((n) => n.id === 'b'); + expect(listed?.configVersion).toBeNull(); + expect(listed?.reading?.sandboxes.byState.active).toBe(7); + }); }); describe('updateNodeSettings', () => { @@ -128,17 +140,18 @@ describe('updateNodeSettings', () => { }); expect(unknown.statusCode).toBe(404); - // Known from a row, silent since this gateway started: capability unknown. + // A row that never checked in (the import pre-creates one; the shape + // fleet.ts loads for it): capability unknown. fleet.checkIn(checkInOf('b', 'http://10.0.0.7:80')); const silent = fleet.get('b'); if (!silent) throw new Error('no node b'); silent.reading = null; + silent.lastCheckInAt = null; + silent.intervalSeconds = null; const early = await rpc(app, '/updateNodeSettings', { id: 'b', swapGb: 8 }); expect(early.statusCode).toBe(503); expect(early.headers['retry-after']).toBe('15'); - expect(early.json().message).toMatch( - /has not checked in since the gateway started/, - ); + expect(early.json().message).toMatch(/has never checked in/); fleet.checkIn(checkInOf('b', 'http://10.0.0.7:80', { managedSwap: null })); const unable = await rpc(app, '/updateNodeSettings', { diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index c11cac37..dd44092f 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -14,12 +14,7 @@ import type { NameCache } from '../cache'; import type { Db } from '../db/db'; import { readNodeConfig } from '../db/node-config'; import { readConfigVersion } from '../db/settings'; -import { - downReason, - type Fleet, - type NodeState, - STARTUP_GRACE_MS, -} from '../fleet'; +import { downReason, type Fleet, type NodeState } from '../fleet'; import { RETRY_AFTER_SECONDS } from '../raw'; export interface CheckInRoutesOptions { @@ -199,13 +194,15 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( // word, carried in its reading (shared nodeReadingSchema managedSwap): // a target for a daemon that cannot honor it would sit in the row // forever, applied by nothing and shown by listNodes as if it were - // real. Unknown (the node has not reported since this gateway - // started) is unknown, not a guess either way. + // real. Unknown (the node has never checked in — a row the import + // pre-created) is unknown, not a guess either way; a node's last + // reading outlives a gateway restart on its row (fleet.ts), so this + // is never said of a node that has reported once. if (node.reading === null) { reply.header('retry-after', String(RETRY_AFTER_SECONDS)); throw refusal( 503, - `node ${id} has not checked in since the gateway started, so whether its daemon manages swap is unknown — retry after its next check-in`, + `node ${id} has never checked in, so whether its daemon manages swap is unknown — retry after its first check-in`, ); } if (node.reading.managedSwap === null) { @@ -239,22 +236,16 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( // two nodes, a 409 an operator clears by hand. Stop the daemon // first; two of its intervals of silence is what "down" means // (fleet.ts downReason), and a down node is removable (found by - // review, 2026-09-14). + // review, 2026-09-14). Judged from the row after a gateway restart + // as from memory before one: a node that checked in seconds before + // the restart is refused here at once, with no grace for the + // gateway's own youth (the third cut's STARTUP_GRACE_MS, deleted + // with the rows in the fourth). A row that has never checked in — + // the import pre-creates one — has nothing here to protect. const node = fleet.get(request.body.id); - if (node !== undefined) { + if (node !== undefined && node.lastCheckInAt !== null) { const now = new Date(); - // Right after a gateway start every node is silent so far, the - // running ones included: they are heard from within one interval. - // Until two default intervals have passed, "not heard from" is - // not "down" (fleet.ts STARTUP_GRACE_MS). - const sinceStart = now.getTime() - fleet.startedAt.getTime(); - if (node.lastCheckInAt === null && sinceStart < STARTUP_GRACE_MS) { - throw refusal( - 409, - `the gateway started ${Math.round(sinceStart / 1000)}s ago and has not heard from node ${node.id} yet — a running node checks in within its interval, so silence this early proves nothing; wait ${STARTUP_GRACE_MS / 1000}s from the gateway's start, then remove it`, - ); - } - if (downReason(node, now) === null && node.lastCheckInAt !== null) { + if (downReason(node, now) === null) { const ago = Math.round( (now.getTime() - node.lastCheckInAt.getTime()) / 1000, ); diff --git a/packages/gateway/src/routes/settings.test.ts b/packages/gateway/src/routes/settings.test.ts index 2bed6ef9..2fdfa4d3 100644 --- a/packages/gateway/src/routes/settings.test.ts +++ b/packages/gateway/src/routes/settings.test.ts @@ -315,18 +315,19 @@ describe('updateSettings: the S3 archive store', () => { expect((await settingsOf(h.app)).s3?.bucket).toBe('seed-bucket'); }); - it('a node that has not reported since the gateway started makes the count unknown: 503 with Retry-After, not a guess', async () => { + it('a node that has never reported makes the count unknown: 503 with Retry-After, not a guess', async () => { const h = testGateway(S3_ENV); reporting(h, 'b', { archived: 0 }); - // A node known from its row (a previous gateway life) that has not - // checked in yet: its disks cannot be counted. + // A row the import pre-created, before the node's first check-in (the + // shape fleet.ts loads for it): its disks cannot be counted. const silent = reporting(h, 'c'); silent.reading = null; silent.lastCheckInAt = null; + silent.intervalSeconds = null; const res = await rpc(h.app, '/updateSettings', { s3: null }); expect(res.statusCode).toBe(503); expect(res.headers['retry-after']).toBe('15'); - expect(res.json().message).toMatch(/node c has not checked in/); + expect(res.json().message).toMatch(/node c has never checked in/); expect((await settingsOf(h.app)).s3?.bucket).toBe('seed-bucket'); // Once it has reported (nothing archived there), the clear goes through. reporting(h, 'c'); diff --git a/packages/gateway/src/routes/settings.ts b/packages/gateway/src/routes/settings.ts index d10bd869..09777aef 100644 --- a/packages/gateway/src/routes/settings.ts +++ b/packages/gateway/src/routes/settings.ts @@ -42,9 +42,10 @@ export interface SettingsRoutesOptions { * ledgers. The gateway holds no sandbox state, but every node's last * check-in carries its census by state, so the count of archived and * restoring sandboxes across the fleet is at hand — for every node that - * has reported since this gateway started. One that has not is a node - * whose disks cannot be counted, and the write refuses (503, retry after - * its next check-in) rather than guess. + * has ever reported (its last reading outlives a gateway restart on its + * row). One that never has — a row the import pre-created, before the + * node's first check-in — is a node whose disks cannot be counted, and + * the write refuses (503, retry after that check-in) rather than guess. */ export const settingsRoutes: FastifyPluginAsyncZod< SettingsRoutesOptions @@ -173,7 +174,7 @@ export const settingsRoutes: FastifyPluginAsyncZod< if ('unknown' in held) { reply.header('retry-after', '15'); return reply.code(503).send({ - message: `${held.unknown.map((id) => `node ${id}`).join(', ')} ${held.unknown.length === 1 ? 'has' : 'have'} not checked in since the gateway started, so the sandboxes archived in the current store cannot be counted — retry after ${held.unknown.length === 1 ? 'its' : 'their'} next check-in, or remove ${held.unknown.length === 1 ? 'it' : 'them'} if gone for good`, + message: `${held.unknown.map((id) => `node ${id}`).join(', ')} ${held.unknown.length === 1 ? 'has' : 'have'} never checked in, so the sandboxes archived in the current store cannot be counted — retry after ${held.unknown.length === 1 ? 'its' : 'their'} first check-in, or remove ${held.unknown.length === 1 ? 'it' : 'them'} if gone for good`, }); } if (held.count > 0) { diff --git a/packages/gateway/src/testing.ts b/packages/gateway/src/testing.ts index 331e9de8..82ec8847 100644 --- a/packages/gateway/src/testing.ts +++ b/packages/gateway/src/testing.ts @@ -107,8 +107,6 @@ export function testGateway( ingress?: Ingress; /** Forged by default: the suites here are about the settings machinery, not S3's availability. */ probeS3?: NonNullable[0]['probeS3']>; - /** When this gateway "started" — what the startup grace is judged against (fleet.ts STARTUP_GRACE_MS). */ - startedAt?: Date; } = {}, ) { const db = openDb(':memory:'); @@ -120,7 +118,7 @@ export function testGateway( }; const config = loadConfig(rawEnv); ensureSettings(db, config); - const fleet = new Fleet(db, opts.startedAt); + const fleet = new Fleet(db); // Under the fleet token the config carries: a suite that embeds a real // node beside this gateway (the SDK's) gives both the same token, and // the lookups must present it, not the scaffolding's default. diff --git a/packages/sdk/src/client.ts b/packages/sdk/src/client.ts index 2dc21dcb..0f0eeed2 100644 --- a/packages/sdk/src/client.ts +++ b/packages/sdk/src/client.ts @@ -250,8 +250,8 @@ export class Dormice { * many nodes there are, are reachable and have reported; the sandbox * census by state; the sandbox disks' bill. Answered by the gateway * from what it holds — no node is asked. `nodes.reported` says how many - * nodes the sums cover; a node not heard from since the gateway started - * is not in them. + * nodes the sums cover; a node that has never checked in is not in + * them. */ async getFleetMetrics(): Promise { const data = await this.rpc('getFleetMetrics', {}); diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts index 6fcaf2ed..65f9229a 100644 --- a/packages/shared/src/gateway.ts +++ b/packages/shared/src/gateway.ts @@ -199,7 +199,7 @@ export const nodeViewSchema = z.object({ swapGb: z.number().int().nonnegative(), /** The configuration version the node last reported it runs; null until it has said (or before it pulls one). */ configVersion: z.number().int().nullable(), - /** ISO 8601 UTC — null only right after a gateway start, before the node's next check-in. */ + /** ISO 8601 UTC — the last check-in taken, kept across gateway restarts; null only for a node that has never checked in. */ lastCheckInAt: z.iso.datetime().nullable(), intervalSeconds: z.number().int().positive().nullable(), /** Checked in within two of its own intervals. */ @@ -300,9 +300,10 @@ export type SilentNode = z.infer; * of every request measured on the Beijing node, 2026-09-12). * * `nodes.reported` says how many nodes the sums cover: a node that has - * not checked in since this gateway started has no reading here, and - * until it does the sums are a lower bound — said as such, never rounded - * up. + * never checked in (a row the import pre-created) has no reading here, + * and until it does the sums are a lower bound — said as such, never + * rounded up. A gateway restart loses no reading: each node's last one + * is on its row. */ export const getFleetMetricsRequestSchema = z.object({}); @@ -316,7 +317,7 @@ export const getFleetMetricsResponseSchema = z.object({ total: z.number().int(), /** Checked in within two of their own intervals (listNodes' `reachable`). */ reachable: z.number().int(), - /** Have a reading — checked in since this gateway started. The sums below cover exactly these. */ + /** Have a reading — have checked in at least once (a restart keeps it). The sums below cover exactly these. */ reported: z.number().int(), }), sandboxes: z.object({ diff --git a/website/content/docs/http-api.mdx b/website/content/docs/http-api.mdx index 502b343c..3a62ceeb 100644 --- a/website/content/docs/http-api.mdx +++ b/website/content/docs/http-api.mdx @@ -70,7 +70,7 @@ The [E2B compatibility surface](/docs/e2b-sdks) is a separate wire under | `POST /listSandboxMetrics` | both | every measurable sandbox's sample in one answer; at the gateway, every node's, with `silent` as in `listSandboxes` | — | | `POST /listSandboxImages` | both | each sandbox's born image vs its template's current one; at the gateway, every node's, with `silent` as in `listSandboxes` | — | | `POST /getConfig` | gateway | effective configuration: the gateway's env knobs (read-only; secrets reported present-or-absent, value never sent), the live fleet `settings`, and `configVersion` — the number every node reports back once it runs this configuration | — | -| `POST /updateSettings` | gateway | rewrite the fleet settings (new-sandbox defaults, default policy, the S3 archive store, the sandbox domain, the per-sandbox pids cap) — no restart; the version counts up and every node applies the bundle at its next check-in (within 15 seconds); each provided group replaces that group whole. The `s3` group takes all six fields every time (keys included — they are never read back) and is probed with a real write-read-delete round trip before anything is saved; `s3: null` turns archiving off. `sandboxDomain` takes a bare hostname or `null`; `sandboxDomainAliases` takes the full list of inbound-only alias hostnames (`[]` clears) — swapping the canonical domain is one call carrying both fields, see [Exposing ports](/docs/ports). `pidsLimit` (floor 256, never unlimited) reaches new containers at birth, running sandboxes in place as their node takes the bundle, and frozen or stopped ones at their next wake — no rebuild | 400 empty patch, a `pidsLimit` below 256, a default policy that asks to archive with no store in force, a failed probe (S3 answered 4xx), clearing/moving the store while sandboxes are archived in it (counted across the fleet from the nodes' check-ins), or an alias list that contradicts the post-patch state (duplicates, the canonical domain listed as an alias, or aliases left standing with no canonical domain); 502 when the probed store is unreachable; 503 with `Retry-After` when a node has not checked in since the gateway started, so the archived count is unknown | +| `POST /updateSettings` | gateway | rewrite the fleet settings (new-sandbox defaults, default policy, the S3 archive store, the sandbox domain, the per-sandbox pids cap) — no restart; the version counts up and every node applies the bundle at its next check-in (within 15 seconds); each provided group replaces that group whole. The `s3` group takes all six fields every time (keys included — they are never read back) and is probed with a real write-read-delete round trip before anything is saved; `s3: null` turns archiving off. `sandboxDomain` takes a bare hostname or `null`; `sandboxDomainAliases` takes the full list of inbound-only alias hostnames (`[]` clears) — swapping the canonical domain is one call carrying both fields, see [Exposing ports](/docs/ports). `pidsLimit` (floor 256, never unlimited) reaches new containers at birth, running sandboxes in place as their node takes the bundle, and frozen or stopped ones at their next wake — no rebuild | 400 empty patch, a `pidsLimit` below 256, a default policy that asks to archive with no store in force, a failed probe (S3 answered 4xx), clearing/moving the store while sandboxes are archived in it (counted across the fleet from the nodes' check-ins), or an alias list that contradicts the post-patch state (duplicates, the canonical domain listed as an alias, or aliases left standing with no canonical domain); 502 when the probed store is unreachable; 503 with `Retry-After` when a node has never checked in, so the archived count is unknown | | `POST /getIngress` | gateway | domains bound on the gateway's managed reverse proxy, with live DNS and certificate probes | — | | `POST /setIngress` | gateway | rewrite the managed proxy config to exactly this domain list (empty list unbinds all) | 400 when the gateway manages no proxy | | `POST /listNodes` | gateway | every node that ever checked in: endpoint, reachable, last check-in, build, reading, the configuration version it runs, its swap target | — | diff --git a/website/content/docs/metrics.mdx b/website/content/docs/metrics.mdx index 435606c7..be68bba6 100644 --- a/website/content/docs/metrics.mdx +++ b/website/content/docs/metrics.mdx @@ -115,9 +115,10 @@ const fleet = await client.getFleetMetrics(); // sandboxes: { total, byState }, sandboxDisks } ``` -`nodes.reported` says how many nodes the sums cover. A node that has not -checked in since the gateway started has no reading yet; until it does, -the sums are a lower bound, and the console's overview says so. +`nodes.reported` says how many nodes the sums cover. A node that has never +checked in has no reading yet; until it does, the sums are a lower bound, +and the console's overview says so. A gateway restart keeps every node's +last reading. `getFleetStateHistory({ start?, end? })` is the same census over time — one sample every 30 seconds (the gateway's own sampler, From 5c794b8be4e9a75d388d4e842614edac2937d180 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 16:02:44 +0800 Subject: [PATCH 58/89] Neither process logs a request that succeeded; one that ended in an error status is one line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fastify's two lines per request are off in the daemon and the gateway (logController with disableRequestLogging — the top-level option is deprecated in fastify 5). A console open against the Beijing node polled the observation verbs every few seconds, the daemon wrote 74 lines a second and journald kept 22 hours. An onResponse hook says what is worth saying: one line for every response with a status of 400 or more — method, path, status, elapsed milliseconds — info for a 4xx, warn for a 5xx (whose error and stack remain the error handler's line; this one names the request now that "incoming request" is gone). The path goes without its query: signed URLs and envd access tokens travel there. Hijacked forwards and direct 503 sends count too — the hook runs when the raw response ends. Slow requests are not logged on purpose: an execCommand or an attached stream is legitimately long, and a hung Docker surfaces as a 5xx through the deadlines. --- packages/gateway/src/app.test.ts | 44 +++++++++++++++++ packages/gateway/src/app.ts | 30 +++++++++++- packages/server/src/app.test.ts | 62 ++++++++++++++++++++++++ packages/server/src/app.ts | 37 +++++++++++++- website/content/docs/troubleshooting.mdx | 4 +- 5 files changed, 174 insertions(+), 3 deletions(-) diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 25c20f7b..2e71c244 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -1974,3 +1974,47 @@ describe('the E2B list across nodes', () => { }); }); }); + +describe('the request log', () => { + it("says nothing of a 2xx; one line for a 4xx (info) or a 5xx (warn) naming method, path and status, the query left out; Fastify's own two lines per request are off", async () => { + const logs: string[] = []; + const h = await gateway(['b'], {}, { logs }); + // The harness's check-ins and a probe are 2xx: nothing said of them. + expect((await fetch(`${h.endpoint}/healthz`)).status).toBe(200); + const said = () => + logs + .map((l) => JSON.parse(l) as Record) + .filter((l) => l.msg === 'request ended in an error status'); + expect(said()).toEqual([]); + expect( + logs.some( + (l) => + l.includes('incoming request') || l.includes('request completed'), + ), + ).toBe(false); + expect((await rpc(h, '/noSuchVerb?signature=secret-sig')).status).toBe(404); + // Every node down: a new name is the placement's own 503, sent + // directly, not through the error handler — logged all the same. + const b = h.fleet.get('b'); + if (!b) throw new Error('node lost'); + b.lastCheckInAt = new Date(Date.now() - 31_000); + expect((await rpc(h, '/acquireSandbox', { name: 'nowhere' })).status).toBe( + 503, + ); + expect(said()).toEqual([ + expect.objectContaining({ + level: 30, + method: 'POST', + path: '/noSuchVerb', + statusCode: 404, + }), + expect.objectContaining({ + level: 40, + method: 'POST', + path: '/acquireSandbox', + statusCode: 503, + }), + ]); + expect(logs.join('\n')).not.toContain('secret-sig'); + }); +}); diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts index fe8330d5..d6bed52d 100644 --- a/packages/gateway/src/app.ts +++ b/packages/gateway/src/app.ts @@ -2,7 +2,11 @@ import http from 'node:http'; import type { KeyedQueue } from '@dormice/server/keyed-queue'; import { sandboxDomainsInForce } from '@dormice/shared'; import fastifyCookie from '@fastify/cookie'; -import fastify, { type FastifyError, type FastifyServerFactory } from 'fastify'; +import fastify, { + type FastifyError, + type FastifyServerFactory, + LogController, +} from 'fastify'; import { serializerCompiler, validatorCompiler, @@ -163,6 +167,9 @@ export function buildGatewayApp({ const app = fastify({ loggerInstance, serverFactory, + // Fastify's two lines per request are off, as on the daemon + // (server/app.ts); the onResponse hook below says what is worth saying. + logController: new LogController({ disableRequestLogging: true }), }).withTypeProvider(); app.setValidatorCompiler(validatorCompiler); app.setSerializerCompiler(serializerCompiler); @@ -183,6 +190,27 @@ export function buildGatewayApp({ .send({ message: `route ${request.method} ${request.url} not found` }); }); + // One line per request that ended in an error status, none for the + // rest — the daemon's rule (server/app.ts has the measurement), at the + // door: a 4xx names the caller's mistake (info), a 5xx is ours or a + // node's (warn; the error handler's line has the error when it was + // ours). A forwarded answer counts by the status the node gave it — + // the hook runs when the raw response ends, hijacked or not. The path + // without its query: signatures and access tokens travel there. + app.addHook('onResponse', async (request, reply) => { + const status = reply.statusCode; + if (status < 400) return; + request.log[status >= 500 ? 'warn' : 'info']( + { + method: request.method, + path: request.url.split('?')[0], + statusCode: status, + elapsedMs: Math.round(reply.elapsedTime), + }, + 'request ended in an error status', + ); + }); + // Liveness, open by design; the build identity so an operator can tell // which commit answers without a token. app.get( diff --git a/packages/server/src/app.test.ts b/packages/server/src/app.test.ts index d299283e..1ab5fe29 100644 --- a/packages/server/src/app.test.ts +++ b/packages/server/src/app.test.ts @@ -8,6 +8,7 @@ import { FILE_SIZE_LIMIT_BYTES, hostMetricsResponseSchema, } from '@dormice/shared'; +import { pino } from 'pino'; import { describe, expect, it } from 'vitest'; import { buildApp } from './app'; import { Archiver } from './archive/archiver'; @@ -2035,3 +2036,64 @@ describe('POST /lookupSandbox', () => { await held; }); }); + +describe('the request log', () => { + it("says nothing of a request that succeeded, one line naming method, path and status for one that did not, the query left out — and Fastify's own two lines per request are off", async () => { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const config = loadConfig({ + DORMICE_DB_PATH: ':memory:', + DORMICE_NODE_ID: 'node-test', + DORMICE_API_TOKEN: TOKEN, + }); + configureNode(db, {}); + const lines: string[] = []; + const app = buildApp({ + config, + db, + executor: new FakeExecutor(), + locks: new KeyedQueue(), + logger: pino( + { level: 'info' }, + { write: (line: string) => lines.push(line) }, + ), + }); + expect( + (await app.inject({ method: 'GET', url: '/healthz' })).statusCode, + ).toBe(200); + expect((await rpc(app, '/listSandboxes')).statusCode).toBe(200); + const said = () => + lines + .map((l) => JSON.parse(l) as Record) + .filter((l) => l.msg === 'request ended in an error status'); + expect(said()).toEqual([]); + expect( + lines.some( + (l) => + l.includes('incoming request') || l.includes('request completed'), + ), + ).toBe(false); + // A signed URL's signature lives in the query: not in the log. + expect( + (await rpc(app, '/noSuchVerb?signature=secret-sig')).statusCode, + ).toBe(404); + expect( + (await app.inject({ method: 'POST', url: '/listSandboxes' })).statusCode, + ).toBe(401); + expect(said()).toEqual([ + expect.objectContaining({ + level: 30, + method: 'POST', + path: '/noSuchVerb', + statusCode: 404, + }), + expect.objectContaining({ + level: 30, + method: 'POST', + path: '/listSandboxes', + statusCode: 401, + }), + ]); + expect(lines.join('\n')).not.toContain('secret-sig'); + }); +}); diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 234e971d..a626704a 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -1,7 +1,11 @@ import http from 'node:http'; import nodePath from 'node:path'; import { GATEWAY_ONLY_VERBS } from '@dormice/shared'; -import fastify, { type FastifyError, type FastifyServerFactory } from 'fastify'; +import fastify, { + type FastifyError, + type FastifyServerFactory, + LogController, +} from 'fastify'; import { serializerCompiler, validatorCompiler, @@ -133,6 +137,10 @@ export function buildApp({ const app = fastify({ loggerInstance, serverFactory, + // Fastify's two lines per request ("incoming request", "request + // completed") are off; the onResponse hook below says what a request + // is worth saying. + logController: new LogController({ disableRequestLogging: true }), }).withTypeProvider(); app.setValidatorCompiler(validatorCompiler); app.setSerializerCompiler(serializerCompiler); @@ -163,6 +171,33 @@ export function buildApp({ } reply.code(status).send({ message: error.message }); }); + + // One line per request that ended in an error status, none for the + // rest: a healthy daemon's log is what happened to it, not every poll + // it answered (with Fastify's own two lines per request the Beijing + // node wrote 74 a second and journald kept 22 hours, measured + // 2026-09-15; they are off above). A 4xx names the caller's mistake + // (info); a 5xx is ours (warn) — its error and stack are the error + // handler's line, this one carries what that line lacks with the + // request logging off: which request. The path without its query: a + // signed URL's signature and an envd access token travel there + // (security rule 5). Nothing about slow requests, on purpose: an + // execCommand or an attached process stream is legitimately long, and + // a Docker that hangs surfaces as a 5xx through the deadlines + // (executor/deadline.ts). + app.addHook('onResponse', async (request, reply) => { + const status = reply.statusCode; + if (status < 400) return; + request.log[status >= 500 ? 'warn' : 'info']( + { + method: request.method, + path: request.url.split('?')[0], + statusCode: status, + elapsedMs: Math.round(reply.elapsedTime), + }, + 'request ended in an error status', + ); + }); // A verb that answers at the gateway alone, asked of a node: the 404 // names the door. The emergency path is ssh to a node and curl its // loopback with the fleet token, and an operator on it asking for the diff --git a/website/content/docs/troubleshooting.mdx b/website/content/docs/troubleshooting.mdx index 4750eba4..9c0151c8 100644 --- a/website/content/docs/troubleshooting.mdx +++ b/website/content/docs/troubleshooting.mdx @@ -5,7 +5,9 @@ description: Symptoms, causes, and fixes — every entry here was hit on real ha Three places answer most questions before this page does: the daemon's log (`journalctl -u dormice -f` — error messages are written to be -read), `dor doctor` (the environment, +read; a request that succeeded leaves no line, one that ended in a 4xx +or 5xx leaves exactly one, naming the method, the path and the status), +`dor doctor` (the environment, [check by check](/docs/doctor)), and the console's Overview or `getHostMetrics` (the machine's [capacity](/docs/metrics)). From bc997602e7cfb9a6d9134533b76089f8fcc501c7 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 16:31:53 +0800 Subject: [PATCH 59/89] The base image is a fleet setting, and a node pulls an image it lacks from the fleet's registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit settings gains baseImage and registryAddress (gateway migration 0004, node copy migration 0026), seeded from the gateway's DORMICE_BASE_IMAGE and DORMICE_REGISTRY_ADDRESS; a table seeded before the two columns has them filled once from the env while empty, counted as a configuration change. baseImage is written by updateSettings (never null — a fleet re-points its base, it cannot forget it) and edited from a new row on the console's settings card; registryAddress is read-only on the wire in this cut. Both ride the bundle, defaulting to null so a node on this build takes a bundle from a gateway on the previous one. On the node the executors' base image becomes a live view (baseImage(), the resources/pidsLimit shape): the copy's, or the node's own DORMICE_BASE_IMAGE while the fleet names none — said at boot and at each bundle that names none — or a refusal that says where to set it. The docker executor no longer requires DORMICE_BASE_IMAGE in its config. Images are bare references. Before every birth the docker executor inspects the image and, when the host lacks it, pulls / under the fleet credential and tags it back under the bare name, so the shell records the same name a local build would and no existing shell reads as upgradable after the move. A reference naming its own registry is pulled as written. Each applied bundle prefetches the base and every template image in the background; a pull that fails is one warning and the bundle stands. --- e2e/src/setup/daemon.ts | 5 + packages/console/messages/de/settings.json | 11 + packages/console/messages/en/settings.json | 11 + packages/console/messages/es/settings.json | 11 + packages/console/messages/fr/settings.json | 11 + packages/console/messages/ja/settings.json | 11 + packages/console/messages/ko/settings.json | 11 + packages/console/messages/pt-BR/settings.json | 11 + packages/console/messages/ru/settings.json | 11 + packages/console/messages/zh-CN/settings.json | 11 + packages/console/messages/zh-TW/settings.json | 11 + .../components/RuntimeSettingsCard.tsx | 94 ++- .../features/settings/pages/SettingsPage.tsx | 2 + .../gateway/drizzle/0004_fleet-base-image.sql | 2 + .../gateway/drizzle/meta/0004_snapshot.json | 488 +++++++++++ packages/gateway/drizzle/meta/_journal.json | 7 + packages/gateway/src/config.ts | 26 + packages/gateway/src/db/node-config.ts | 2 + packages/gateway/src/db/schema.ts | 9 + packages/gateway/src/db/settings.test.ts | 38 +- packages/gateway/src/db/settings.ts | 90 +- packages/gateway/src/routes/nodes.test.ts | 2 + packages/gateway/src/routes/settings.test.ts | 40 + packages/gateway/src/routes/settings.ts | 3 + .../server/drizzle/0026_fleet-base-image.sql | 2 + .../server/drizzle/meta/0026_snapshot.json | 794 ++++++++++++++++++ packages/server/drizzle/meta/_journal.json | 7 + packages/server/src/app.test.ts | 2 +- packages/server/src/config.test.ts | 14 +- packages/server/src/config.ts | 22 +- packages/server/src/db/schema.ts | 11 + packages/server/src/db/settings.ts | 6 + packages/server/src/db/templates.ts | 36 +- packages/server/src/e2b/compat.test.ts | 16 +- packages/server/src/e2b/control.ts | 9 +- .../server/src/executor/docker-exitof.test.ts | 3 +- .../server/src/executor/docker-images.test.ts | 207 +++++ .../src/executor/docker.contract.test.ts | 11 +- packages/server/src/executor/docker.ts | 183 +++- packages/server/src/executor/executor.ts | 37 +- packages/server/src/executor/fake.test.ts | 16 + packages/server/src/executor/fake.ts | 35 +- packages/server/src/lifecycle.ts | 2 +- packages/server/src/main.ts | 52 +- packages/server/src/node-config.test.ts | 125 ++- packages/server/src/node-config.ts | 56 +- packages/server/src/routes/sandboxes.ts | 6 +- packages/server/src/testing.ts | 13 + packages/shared/src/gateway.ts | 9 + packages/shared/src/settings.ts | 36 +- 50 files changed, 2524 insertions(+), 104 deletions(-) create mode 100644 packages/gateway/drizzle/0004_fleet-base-image.sql create mode 100644 packages/gateway/drizzle/meta/0004_snapshot.json create mode 100644 packages/server/drizzle/0026_fleet-base-image.sql create mode 100644 packages/server/drizzle/meta/0026_snapshot.json create mode 100644 packages/server/src/executor/docker-images.test.ts diff --git a/e2e/src/setup/daemon.ts b/e2e/src/setup/daemon.ts index 62a4b75b..036720d7 100644 --- a/e2e/src/setup/daemon.ts +++ b/e2e/src/setup/daemon.ts @@ -98,6 +98,11 @@ async function bootGateway(spec: GatewaySpec) { DORMICE_S3_ACCESS_KEY_ID: 'e2e-key', DORMICE_S3_SECRET_ACCESS_KEY: 'e2e-secret', DORMICE_S3_FORCE_PATH_STYLE: 'true', + // The fleet's base image — a fleet setting since the fourth cut, so + // the nodes take it from their check-in. In docker mode the exported + // real image (the documented real-machine run); in fake mode any name, + // the fake's own default keeps the suites' assertions honest. + DORMICE_BASE_IMAGE: process.env.DORMICE_BASE_IMAGE ?? 'fake-base', ...spec.extraEnv, }; const child = spawn('node', [GATEWAY_MAIN], { diff --git a/packages/console/messages/de/settings.json b/packages/console/messages/de/settings.json index 8e4a5841..ce2fa363 100644 --- a/packages/console/messages/de/settings.json +++ b/packages/console/messages/de/settings.json @@ -12,6 +12,8 @@ "settings_hint_sandbox_memory": "Erststart-Saatwert für das Standard-Speicherlimit — der wirksame Wert steht oben bei den Betriebsreglern", "settings_hint_sandbox_pids_limit": "Erststart-Seed für das pids-Limit — der gültige Wert steht in der Regler-Karte oben", "settings_hint_sandbox_domain": "Erststart-Saatwert für die kanonische Sandbox-Domain — die wirksame Konfiguration (inklusive Aliasse) steht auf der Domains-Seite in der Sandbox-Domain-Karte", + "settings_hint_base_image": "Erststart-Saatwert für das Basis-Image der Flotte — der wirksame Wert steht in der Reglerkarte oben", + "settings_hint_registry_address": "Die Image-Registry der Flotte (Host:Port), die install.sh neben dem Gateway betreibt; ein Knoten ohne ein Image zieht es von hier", "settings_hint_ingress_file": "Vom Gateway verwaltete Caddy-Konfigurationsdatei; nötig, um auf der Domains-Seite Domains zu binden", "settings_hint_ingress_reload_cmd": "Reload-Befehl nach Änderungen an der Proxy-Konfiguration; Standard ist caddy reload auf der verwalteten Datei selbst", "settings_hint_s3_endpoint": "Erststart-Saatwert für den Archiv-Endpoint — die wirksame Konfiguration steht oben in der Archivspeicher-Karte", @@ -48,6 +50,10 @@ "settings_row_policy": "Standard-Lebenszyklus-Richtlinie", "settings_row_pids": "pids-Limit je Sandbox", "settings_row_pids_value": "{n} (Prozess- und Thread-Budget der Sandbox auf dem Host; bei Erreichen endet die ganze Sandbox)", + "settings_row_base_image": "Basis-Image", + "settings_row_base_image_registry": "{image} · ein Knoten ohne dieses Image zieht es aus der Registry {registry}", + "settings_row_base_image_local": "{image} · die Flotte hat keine Registry; jeder Knoten muss dieses Image selbst haben", + "settings_base_image_unset": "Nicht gesetzt — jeder Knoten fällt auf das DORMICE_BASE_IMAGE seiner eigenen env zurück; hier einmal setzen, und die ganze Flotte teilt es", "settings_defaults_dialog_title": "Standardquoten neuer Sandboxes anpassen", "settings_defaults_dialog_desc": "CPU/Speicher greifen, wenn das nächste Mal ein Container geboren wird (auch bei Bestands-Sandboxes nach einem Kaltstart); die Datenträgergröße wird bei der Geburt des Datenträgers festgelegt (Ersterstellung und Archiv-Wiederherstellung) — der Datenträger ist die Sandbox selbst und wird nie an Ort und Stelle umdimensioniert. Vor dem Verkleinern prüfen, dass der Wert nicht unter dem echten Inhalt archivierter Sandboxes liegt.", "settings_defaults_saved": "Standardquoten neuer Sandboxes aktualisiert", @@ -61,6 +67,11 @@ "settings_pids_saved": "pids-Limit der Sandbox geändert auf {value}", "settings_pids_label": "pids-Limit je Sandbox", "settings_pids_field_desc": "Mindestens {min}. Dies ist die Bremse, die eine Fork-Bombe in ihrer eigenen Sandbox hält, und kann nicht unbegrenzt sein; ein Browser plus einige Agent-Sitzungen überschreiten bereits 512.", + "settings_base_image_dialog_title": "Basis-Image wechseln", + "settings_base_image_dialog_desc": "Jede Sandbox ohne Template startet aus diesem Image. Ein neuer Tag richtet die Basis neu aus: bestehende Sandboxen ohne Template wechseln beim nächsten Kaltstart auf ihn (dieselbe Semantik wie das Umzeigen eines Templates); laufende und eingefrorene Sandboxen bleiben unberührt. Das neue Image zuerst in die Flotten-Registry pushen — ein Knoten, dem es fehlt, zieht es automatisch.", + "settings_base_image_label": "Image-Referenz", + "settings_base_image_field_desc": "Eine nackte Referenz wie dormice-base:20260831 — ohne Registry-Host; ein Knoten stellt die Registry-Adresse beim Ziehen voran.", + "settings_base_image_saved": "Basis-Image geändert auf {image}", "settings_policy_dialog_title": "Standard-Lebenszyklus-Richtlinie anpassen", "settings_policy_dialog_desc": "Betrifft nur Sandboxes, die künftige acquire-Aufrufe neu erstellen — Bestands-Sandboxes behalten ihre eigenen Richtlinien; passen Sie diese in der Sandbox-Liste gesammelt an.", "settings_policy_saved": "Standard-Lebenszyklus-Richtlinie aktualisiert", diff --git a/packages/console/messages/en/settings.json b/packages/console/messages/en/settings.json index 4f5abba4..d3eae1a7 100644 --- a/packages/console/messages/en/settings.json +++ b/packages/console/messages/en/settings.json @@ -12,6 +12,8 @@ "settings_hint_sandbox_memory": "First-boot seed for the default memory limit — the effective value is in the knobs above", "settings_hint_sandbox_pids_limit": "First-boot seed for the pids cap — the value in force lives in the knobs card above", "settings_hint_sandbox_domain": "First-boot seed for the canonical sandbox domain — the configuration in force (aliases included) lives on the Domains page", + "settings_hint_base_image": "First-boot seed for the fleet's base image — the value in force lives in the knobs card above", + "settings_hint_registry_address": "The fleet's image registry (host:port), the one install.sh runs beside the gateway; a node missing an image pulls it from here", "settings_hint_ingress_file": "Caddy config file managed by the gateway; required for binding domains on the Domains page", "settings_hint_ingress_reload_cmd": "Reload command after proxy config changes; defaults to caddy reload on the managed file itself", "settings_hint_s3_endpoint": "First-boot seed for the archive endpoint — the store in force lives in the archive card above", @@ -48,6 +50,10 @@ "settings_row_policy": "Default lifecycle policy", "settings_row_pids": "Sandbox pids cap", "settings_row_pids_value": "{n} (the sandbox's host-side process + thread budget; hitting it exits the whole sandbox)", + "settings_row_base_image": "Base image", + "settings_row_base_image_registry": "{image} · a node missing it pulls from registry {registry}", + "settings_row_base_image_local": "{image} · the fleet has no registry; every node must have this image itself", + "settings_base_image_unset": "Not set — each node falls back to the DORMICE_BASE_IMAGE in its own env; set it here once for the whole fleet", "settings_defaults_dialog_title": "Adjust default quotas for new sandboxes", "settings_defaults_dialog_desc": "CPU/memory take effect the next time a container is born (including existing sandboxes cold-started after a stop); disk size is fixed when the disk is born (first creation and archive restore) — the disk is the sandbox itself and is never resized in place, so before lowering it make sure it is not smaller than an archived sandbox's real content.", "settings_defaults_saved": "New sandbox default quotas updated", @@ -61,6 +67,11 @@ "settings_pids_saved": "Sandbox pids cap changed to {value}", "settings_pids_label": "pids cap per sandbox", "settings_pids_field_desc": "Minimum {min}. This is the brake that keeps a fork bomb inside its own sandbox and cannot be unlimited; a browser plus a few agent sessions is enough to pass 512.", + "settings_base_image_dialog_title": "Change the base image", + "settings_base_image_dialog_desc": "Every sandbox without a template boots from this image. A new tag re-points the base: existing template-less sandboxes swap onto it at their next cold wake (the same semantics as re-pointing a template); running and frozen sandboxes are untouched. Push the new image to the fleet registry first — a node missing it pulls it automatically.", + "settings_base_image_label": "Image reference", + "settings_base_image_field_desc": "A bare reference like dormice-base:20260831 — no registry host; a node prepends the registry address when it pulls.", + "settings_base_image_saved": "Base image changed to {image}", "settings_policy_dialog_title": "Adjust default lifecycle policy", "settings_policy_dialog_desc": "Only affects sandboxes created by future acquire calls — existing sandboxes keep their own policies; adjust them in bulk from the sandbox list.", "settings_policy_saved": "Default lifecycle policy updated", diff --git a/packages/console/messages/es/settings.json b/packages/console/messages/es/settings.json index 60ebe7ab..7ed5f16d 100644 --- a/packages/console/messages/es/settings.json +++ b/packages/console/messages/es/settings.json @@ -12,6 +12,8 @@ "settings_hint_sandbox_memory": "Valor semilla del primer arranque para el límite de memoria predeterminado — el valor efectivo está en los parámetros de arriba", "settings_hint_sandbox_pids_limit": "Semilla de primer arranque del límite pids; el valor en vigor está en la tarjeta de ajustes de arriba", "settings_hint_sandbox_domain": "Valor semilla del primer arranque para el dominio canónico de sandbox — la configuración en vigor (alias incluidos) está en la tarjeta de dominio de sandbox de la página Dominios", + "settings_hint_base_image": "Valor semilla del primer arranque para la imagen base de la flota — el valor en vigor está en la tarjeta de parámetros de arriba", + "settings_hint_registry_address": "El registro de imágenes de la flota (host:puerto), el que install.sh levanta junto al gateway; un nodo al que le falte una imagen la descarga de aquí", "settings_hint_ingress_file": "Archivo de configuración de Caddy gestionado por el gateway; hace falta para vincular dominios en la página Dominios", "settings_hint_ingress_reload_cmd": "Comando de recarga tras cambiar la configuración del proxy; por defecto es caddy reload sobre el propio archivo gestionado", "settings_hint_s3_endpoint": "Valor semilla del primer arranque para el Endpoint de archivado — el almacenamiento en vigor está en la tarjeta de archivado de arriba", @@ -48,6 +50,10 @@ "settings_row_policy": "Política de ciclo de vida predeterminada", "settings_row_pids": "Límite pids por sandbox", "settings_row_pids_value": "{n} (presupuesto de procesos y hilos del sandbox en el host; al alcanzarlo, todo el sandbox termina)", + "settings_row_base_image": "Imagen base", + "settings_row_base_image_registry": "{image} · un nodo que no la tenga la descarga del registro {registry}", + "settings_row_base_image_local": "{image} · la flota no tiene registro; cada nodo debe tener esta imagen por su cuenta", + "settings_base_image_unset": "Sin configurar — cada nodo recurre al DORMICE_BASE_IMAGE de su propio env; configúrala aquí una vez para toda la flota", "settings_defaults_dialog_title": "Ajustar las cuotas predeterminadas de los sandboxes nuevos", "settings_defaults_dialog_desc": "La CPU y la memoria se aplican en el próximo nacimiento de un contenedor (incluidos los sandboxes existentes que arrancan en frío tras una parada); el tamaño del disco queda fijado al nacer el disco (primera creación y restauración de archivado) — el disco es el sandbox en sí y nunca se redimensiona en el sitio, así que antes de bajarlo asegúrate de que no quede por debajo del contenido real de un sandbox archivado.", "settings_defaults_saved": "Cuotas predeterminadas de los sandboxes nuevos actualizadas", @@ -61,6 +67,11 @@ "settings_pids_saved": "Límite pids del sandbox cambiado a {value}", "settings_pids_label": "Límite pids por sandbox", "settings_pids_field_desc": "Mínimo {min}. Es el freno que mantiene una bomba fork dentro de su propio sandbox y no puede ser ilimitado; un navegador más unas pocas sesiones de agente bastan para superar 512.", + "settings_base_image_dialog_title": "Cambiar la imagen base", + "settings_base_image_dialog_desc": "Todo sandbox sin plantilla arranca desde esta imagen. Una etiqueta nueva reapunta la base: los sandboxes sin plantilla existentes cambian a ella en su próximo arranque en frío (la misma semántica que reapuntar una plantilla); los sandboxes en ejecución o congelados no se tocan. Sube primero la imagen nueva al registro de la flota — un nodo al que le falte la descarga automáticamente.", + "settings_base_image_label": "Referencia de imagen", + "settings_base_image_field_desc": "Una referencia simple como dormice-base:20260831 — sin host de registro; el nodo antepone la dirección del registro al descargar.", + "settings_base_image_saved": "Imagen base cambiada a {image}", "settings_policy_dialog_title": "Ajustar la política de ciclo de vida predeterminada", "settings_policy_dialog_desc": "Solo afecta a los sandboxes creados por futuras llamadas a acquire — los sandboxes existentes conservan su propia política; ajústalas en lote desde la lista de sandboxes.", "settings_policy_saved": "Política de ciclo de vida predeterminada actualizada", diff --git a/packages/console/messages/fr/settings.json b/packages/console/messages/fr/settings.json index d6c96e9c..67fc5f6a 100644 --- a/packages/console/messages/fr/settings.json +++ b/packages/console/messages/fr/settings.json @@ -12,6 +12,8 @@ "settings_hint_sandbox_memory": "Valeur d'amorçage de la limite mémoire par défaut — la valeur effective est dans les réglages ci-dessus", "settings_hint_sandbox_pids_limit": "Graine de premier démarrage de la limite pids ; la valeur en vigueur est dans la carte des réglages ci-dessus", "settings_hint_sandbox_domain": "Valeur d'amorçage du domaine canonique des sandbox — la configuration effective (alias compris) est dans la carte du domaine des sandbox, page Domaines", + "settings_hint_base_image": "Valeur d'amorçage de l'image de base de la flotte — la valeur effective est dans la carte des réglages ci-dessus", + "settings_hint_registry_address": "Le registre d'images de la flotte (hôte:port), celui qu'install.sh lance à côté de la gateway ; un nœud à qui il manque une image la récupère ici", "settings_hint_ingress_file": "Fichier de configuration Caddy géré par la passerelle ; requis pour lier des domaines depuis la page Domaines", "settings_hint_ingress_reload_cmd": "Commande de rechargement après modification de la configuration du proxy ; par défaut, caddy reload sur le fichier géré lui-même", "settings_hint_s3_endpoint": "Valeur d'amorçage de l'endpoint d'archivage — le stockage effectif est dans la carte d'archivage ci-dessus", @@ -48,6 +50,10 @@ "settings_row_policy": "Politique de cycle de vie par défaut", "settings_row_pids": "Limite pids par sandbox", "settings_row_pids_value": "{n} (budget de processus et threads de la sandbox côté hôte ; l'atteindre termine toute la sandbox)", + "settings_row_base_image": "Image de base", + "settings_row_base_image_registry": "{image} · un nœud qui ne l'a pas la récupère depuis le registre {registry}", + "settings_row_base_image_local": "{image} · la flotte n'a pas de registre ; chaque nœud doit posséder cette image lui-même", + "settings_base_image_unset": "Non définie — chaque nœud se rabat sur le DORMICE_BASE_IMAGE de son propre env ; définissez-la ici une fois pour toute la flotte", "settings_defaults_dialog_title": "Ajuster les quotas par défaut des nouvelles sandbox", "settings_defaults_dialog_desc": "CPU et mémoire prennent effet à la prochaine naissance d'un conteneur (y compris les sandbox existantes redémarrées à froid après un arrêt) ; la taille du disque est figée à la naissance du disque (première création et restauration d'archive) — le disque est la sandbox elle-même et n'est jamais redimensionné en place ; avant de la réduire, vérifiez qu'elle n'est pas inférieure au contenu réel d'une sandbox archivée.", "settings_defaults_saved": "Quotas par défaut des nouvelles sandbox mis à jour", @@ -61,6 +67,11 @@ "settings_pids_saved": "Limite pids de la sandbox changée à {value}", "settings_pids_label": "Limite pids par sandbox", "settings_pids_field_desc": "Minimum {min}. C'est le frein qui garde une fork bomb dans sa propre sandbox et il ne peut pas être illimité ; un navigateur plus quelques sessions d'agent suffisent à dépasser 512.", + "settings_base_image_dialog_title": "Changer l'image de base", + "settings_base_image_dialog_desc": "Toute sandbox sans modèle démarre depuis cette image. Un nouveau tag repointe la base : les sandbox sans modèle existantes basculent dessus à leur prochain réveil à froid (même sémantique que le repointage d'un modèle) ; les sandbox en cours et gelées ne sont pas touchées. Poussez d'abord la nouvelle image dans le registre de la flotte — un nœud à qui elle manque la récupère automatiquement.", + "settings_base_image_label": "Référence d'image", + "settings_base_image_field_desc": "Une référence nue comme dormice-base:20260831 — sans hôte de registre ; le nœud préfixe l'adresse du registre au moment de la récupération.", + "settings_base_image_saved": "Image de base changée en {image}", "settings_policy_dialog_title": "Ajuster la politique de cycle de vie par défaut", "settings_policy_dialog_desc": "N'affecte que les sandbox créées par les futurs appels acquire — les sandbox existantes gardent chacune leur politique ; ajustez-les en masse depuis la liste des sandbox.", "settings_policy_saved": "Politique de cycle de vie par défaut mise à jour", diff --git a/packages/console/messages/ja/settings.json b/packages/console/messages/ja/settings.json index 66f94371..89331da0 100644 --- a/packages/console/messages/ja/settings.json +++ b/packages/console/messages/ja/settings.json @@ -12,6 +12,8 @@ "settings_hint_sandbox_memory": "既定メモリ上限の初回起動シード値 — 有効値は上の運用設定にあります", "settings_hint_sandbox_pids_limit": "pids 上限の初回起動シード値 — 有効な値は上のノブカードにあります", "settings_hint_sandbox_domain": "サンドボックスの正規ドメインの初回起動シード値 — 有効な設定(エイリアス含む)は「ドメイン」ページのサンドボックスドメインカードにあります", + "settings_hint_base_image": "フリートのベースイメージの初回起動シード値 — 有効な値は上のノブカードにあります", + "settings_hint_registry_address": "フリートのイメージレジストリ(ホスト:ポート)。install.sh がゲートウェイ機の横で起動するもので、イメージを持たないノードはここから取得します", "settings_hint_ingress_file": "ゲートウェイが管理する Caddy 設定ファイル。設定すると「ドメイン」ページからウェブでドメインをバインドできます", "settings_hint_ingress_reload_cmd": "プロキシ設定変更後のリロードコマンド。未設定なら既定で管理ファイル自体を caddy reload します", "settings_hint_s3_endpoint": "アーカイブ Endpoint の初回起動シード値 — 有効な設定は上のアーカイブストレージカードにあります", @@ -48,6 +50,10 @@ "settings_row_policy": "既定のライフサイクルポリシー", "settings_row_pids": "サンドボックスの pids 上限", "settings_row_pids_value": "{n}(サンドボックスのホスト側プロセス+スレッド予算。上限到達で全体が終了)", + "settings_row_base_image": "ベースイメージ", + "settings_row_base_image_registry": "{image} · 持たないノードはレジストリ {registry} から取得", + "settings_row_base_image_local": "{image} · フリートにレジストリがなく、各ノードがこのイメージを自前で用意する必要があります", + "settings_base_image_unset": "未設定 — 各ノードは自身の env の DORMICE_BASE_IMAGE にフォールバックします。ここで一度設定すればフリート全体で共有されます", "settings_defaults_dialog_title": "新規サンドボックスの既定クォータの調整", "settings_defaults_dialog_desc": "CPU とメモリは次にコンテナが生成されるときに反映されます(停止後にコールドスタートする既存サンドボックスを含む)。ディスクはディスク作成時に確定します(初回作成とアーカイブからの復元)— ディスクはサンドボックスの本体であり、その場でのサイズ変更は決して行われないため、縮小する前にアーカイブ済みサンドボックスの実際の内容量を下回らないか確認してください。", "settings_defaults_saved": "新規サンドボックスの既定クォータを更新しました", @@ -61,6 +67,11 @@ "settings_pids_saved": "サンドボックスの pids 上限を {value} に変更しました", "settings_pids_label": "サンドボックスごとの pids 上限", "settings_pids_field_desc": "最小 {min}。fork 爆弾を自身のサンドボックス内に閉じ込めるブレーキであり、無制限にはできません。ブラウザに数個の agent セッションを加えるだけで 512 を超えます。", + "settings_base_image_dialog_title": "ベースイメージを変更", + "settings_base_image_dialog_desc": "テンプレートのないサンドボックスはすべてこのイメージから起動します。新しいタグにすることがベースの世代交代です:既存のテンプレートなしサンドボックスは次のコールド復帰時に新しいシェルへ切り替わり(テンプレートの再指定と同じ意味論)、実行中・凍結中のサンドボックスには影響しません。新しいイメージは先にフリートのレジストリへ push してください。持たないノードは自動で取得します。", + "settings_base_image_label": "イメージ参照", + "settings_base_image_field_desc": "dormice-base:20260831 のような裸の参照 — レジストリのホスト名は付けません。ノードが取得時にレジストリアドレスを前置します。", + "settings_base_image_saved": "ベースイメージを {image} に変更しました", "settings_policy_dialog_title": "既定のライフサイクルポリシーの調整", "settings_policy_dialog_desc": "影響するのは今後 acquire で新規作成されるサンドボックスだけです — 既存のサンドボックスはそれぞれのポリシーを保持します。サンドボックス一覧から一括調整してください。", "settings_policy_saved": "既定のライフサイクルポリシーを更新しました", diff --git a/packages/console/messages/ko/settings.json b/packages/console/messages/ko/settings.json index ca2a7f88..8af88dcd 100644 --- a/packages/console/messages/ko/settings.json +++ b/packages/console/messages/ko/settings.json @@ -12,6 +12,8 @@ "settings_hint_sandbox_memory": "기본 메모리 상한의 첫 부팅 시드 값 — 유효 값은 위의 운영 노브에 있습니다", "settings_hint_sandbox_pids_limit": "pids 상한의 최초 부팅 시드 값 — 유효 값은 위 노브 카드에 있습니다", "settings_hint_sandbox_domain": "샌드박스 정식 도메인의 첫 부팅 시드 값 — 유효 설정(별칭 포함)은 '도메인' 페이지의 샌드박스 도메인 카드에 있습니다", + "settings_hint_base_image": "플릿 기본 이미지의 첫 부팅 시드 값 — 유효 값은 위의 운영 노브 카드에 있습니다", + "settings_hint_registry_address": "플릿 이미지 레지스트리(호스트:포트). install.sh가 게이트웨이 머신 옆에 띄우는 것으로, 이미지가 없는 노드는 여기서 가져옵니다", "settings_hint_ingress_file": "게이트웨이가 관리하는 Caddy 설정 파일. 설정해야 '도메인' 페이지에서 웹으로 도메인을 바인딩할 수 있습니다", "settings_hint_ingress_reload_cmd": "프록시 설정 변경 후 실행할 리로드 명령. 미설정 시 기본값은 관리 파일 자체에 대한 caddy reload", "settings_hint_s3_endpoint": "아카이브 Endpoint의 첫 부팅 시드 값 — 유효 설정은 위의 아카이브 스토리지 카드에 있습니다", @@ -48,6 +50,10 @@ "settings_row_policy": "기본 수명 주기 정책", "settings_row_pids": "샌드박스 pids 상한", "settings_row_pids_value": "{n}(샌드박스의 호스트 측 프로세스+스레드 예산, 상한 도달 시 전체 종료)", + "settings_row_base_image": "기본 이미지", + "settings_row_base_image_registry": "{image} · 없는 노드는 레지스트리 {registry}에서 가져옵니다", + "settings_row_base_image_local": "{image} · 플릿에 레지스트리가 없어 각 노드가 이 이미지를 직접 갖고 있어야 합니다", + "settings_base_image_unset": "설정되지 않음 — 각 노드는 자신의 env에 있는 DORMICE_BASE_IMAGE로 되돌아갑니다. 여기서 한 번 설정하면 플릿 전체가 공유합니다", "settings_defaults_dialog_title": "새 샌드박스의 기본 할당량 조정", "settings_defaults_dialog_desc": "CPU/메모리는 다음에 컨테이너가 태어날 때 적용됩니다(중지 후 콜드 스타트되는 기존 샌드박스 포함). 디스크는 디스크가 태어날 때 확정됩니다(최초 생성과 아카이브 복원) — 디스크는 샌드박스의 본체라서 절대 제자리에서 크기를 바꾸지 않으니, 줄이기 전에 아카이브된 샌드박스의 실제 내용보다 작아지지 않는지 확인하세요.", "settings_defaults_saved": "새 샌드박스 기본 할당량이 업데이트되었습니다", @@ -61,6 +67,11 @@ "settings_pids_saved": "샌드박스 pids 상한이 {value}로 변경되었습니다", "settings_pids_label": "샌드박스별 pids 상한", "settings_pids_field_desc": "최소 {min}. fork 폭탄을 자기 샌드박스 안에 가두는 차단기이므로 무제한으로 둘 수 없습니다. 브라우저 하나에 agent 세션 몇 개만 더해도 512를 넘습니다.", + "settings_base_image_dialog_title": "기본 이미지 변경", + "settings_base_image_dialog_desc": "템플릿이 없는 샌드박스는 모두 이 이미지에서 시작합니다. 새 태그로 바꾸는 것이 곧 기본 이미지의 세대교체입니다: 기존의 템플릿 없는 샌드박스는 다음 콜드 웨이크 때 새 셸로 교체되고(템플릿 재지정과 같은 의미), 실행 중·동결 중인 샌드박스는 영향을 받지 않습니다. 새 이미지는 먼저 플릿 레지스트리에 push하세요. 없는 노드는 자동으로 가져옵니다.", + "settings_base_image_label": "이미지 참조", + "settings_base_image_field_desc": "dormice-base:20260831처럼 레지스트리 호스트명이 없는 참조 — 노드가 가져올 때 레지스트리 주소를 앞에 붙입니다.", + "settings_base_image_saved": "기본 이미지가 {image}로 변경되었습니다", "settings_policy_dialog_title": "기본 수명 주기 정책 조정", "settings_policy_dialog_desc": "이후 acquire로 새로 만드는 샌드박스에만 적용됩니다 — 기존 샌드박스는 각자의 정책을 유지하니, 샌드박스 목록에서 일괄 조정하세요.", "settings_policy_saved": "기본 수명 주기 정책이 업데이트되었습니다", diff --git a/packages/console/messages/pt-BR/settings.json b/packages/console/messages/pt-BR/settings.json index d15cd38c..a7101ab3 100644 --- a/packages/console/messages/pt-BR/settings.json +++ b/packages/console/messages/pt-BR/settings.json @@ -12,6 +12,8 @@ "settings_hint_sandbox_memory": "Semente de primeira inicialização para o limite padrão de memória — o valor efetivo está nos controles acima", "settings_hint_sandbox_pids_limit": "Semente de primeira inicialização do limite pids; o valor em vigor está no cartão de ajustes acima", "settings_hint_sandbox_domain": "Semente de primeira inicialização para o domínio canônico de sandbox — a configuração em vigor (incluindo aliases) está no cartão de domínio de sandbox da página Domínios", + "settings_hint_base_image": "Semente de primeira inicialização para a imagem base da frota — o valor em vigor está no cartão de controles acima", + "settings_hint_registry_address": "O registro de imagens da frota (host:porta), o que o install.sh sobe ao lado do gateway; um nó sem uma imagem a baixa daqui", "settings_hint_ingress_file": "Arquivo de configuração do Caddy gerenciado pelo gateway; necessário para vincular domínios na página Domínios", "settings_hint_ingress_reload_cmd": "Comando de recarga após mudanças na configuração do proxy; o padrão é caddy reload no próprio arquivo gerenciado", "settings_hint_s3_endpoint": "Semente de primeira inicialização para o Endpoint de arquivamento — o armazenamento em vigor está no cartão de arquivamento acima", @@ -48,6 +50,10 @@ "settings_row_policy": "Política padrão de ciclo de vida", "settings_row_pids": "Limite pids por sandbox", "settings_row_pids_value": "{n} (orçamento de processos e threads do sandbox no host; ao atingi-lo, todo o sandbox encerra)", + "settings_row_base_image": "Imagem base", + "settings_row_base_image_registry": "{image} · um nó que não a tenha baixa do registro {registry}", + "settings_row_base_image_local": "{image} · a frota não tem registro; cada nó precisa ter esta imagem por conta própria", + "settings_base_image_unset": "Não definida — cada nó recorre ao DORMICE_BASE_IMAGE do seu próprio env; defina aqui uma vez para toda a frota", "settings_defaults_dialog_title": "Ajustar cotas padrão de novos sandboxes", "settings_defaults_dialog_desc": "CPU/memória entram em vigor no próximo nascimento de contêiner (inclusive sandboxes existentes que fazem cold start após parar); o tamanho do disco é fixado quando o disco nasce (primeira criação e restauração de arquivamento) — o disco é o próprio sandbox e nunca é redimensionado no lugar; antes de reduzir, garanta que não fique menor que o conteúdo real de um sandbox arquivado.", "settings_defaults_saved": "Cotas padrão de novos sandboxes atualizadas", @@ -61,6 +67,11 @@ "settings_pids_saved": "Limite pids do sandbox alterado para {value}", "settings_pids_label": "Limite pids por sandbox", "settings_pids_field_desc": "Mínimo {min}. É o freio que mantém uma fork bomb dentro do próprio sandbox e não pode ser ilimitado; um navegador mais algumas sessões de agente já passam de 512.", + "settings_base_image_dialog_title": "Trocar a imagem base", + "settings_base_image_dialog_desc": "Todo sandbox sem template inicia a partir desta imagem. Uma tag nova reaponta a base: os sandboxes sem template existentes trocam para ela no próximo despertar a frio (a mesma semântica de reapontar um template); sandboxes em execução ou congelados não são tocados. Envie a imagem nova ao registro da frota primeiro — um nó que não a tenha a baixa automaticamente.", + "settings_base_image_label": "Referência da imagem", + "settings_base_image_field_desc": "Uma referência simples como dormice-base:20260831 — sem host de registro; o nó acrescenta o endereço do registro ao baixar.", + "settings_base_image_saved": "Imagem base alterada para {image}", "settings_policy_dialog_title": "Ajustar política padrão de ciclo de vida", "settings_policy_dialog_desc": "Afeta só os sandboxes criados por acquire daqui em diante — os existentes mantêm suas próprias políticas; ajuste-os em lote pela lista de sandboxes.", "settings_policy_saved": "Política padrão de ciclo de vida atualizada", diff --git a/packages/console/messages/ru/settings.json b/packages/console/messages/ru/settings.json index 51cb4b33..9c796535 100644 --- a/packages/console/messages/ru/settings.json +++ b/packages/console/messages/ru/settings.json @@ -12,6 +12,8 @@ "settings_hint_sandbox_memory": "Стартовое значение лимита памяти по умолчанию — действующее значение в настройках выше", "settings_hint_sandbox_pids_limit": "Стартовое значение лимита pids при первом запуске; действующее значение в карточке настроек выше", "settings_hint_sandbox_domain": "Стартовое значение канонического домена песочниц — действующая конфигурация (включая алиасы) находится в карточке домена песочниц на странице «Домены»", + "settings_hint_base_image": "Стартовое значение базового образа флота — действующее значение находится в карточке настроек выше", + "settings_hint_registry_address": "Реестр образов флота (хост:порт), который install.sh поднимает рядом со шлюзом; узел, у которого нет образа, берёт его отсюда", "settings_hint_ingress_file": "Файл конфигурации Caddy под управлением шлюза; без него привязка доменов на странице «Домены» недоступна", "settings_hint_ingress_reload_cmd": "Команда перезагрузки после изменения конфигурации прокси; по умолчанию caddy reload на самом управляемом файле", "settings_hint_s3_endpoint": "Адрес объектного хранилища для архивов; архивация включается, только когда заданы все четыре параметра", @@ -48,6 +50,10 @@ "settings_row_policy": "Политика жизненного цикла по умолчанию", "settings_row_pids": "Лимит pids на песочницу", "settings_row_pids_value": "{n} (бюджет процессов и потоков песочницы на хосте; при достижении вся песочница завершается)", + "settings_row_base_image": "Базовый образ", + "settings_row_base_image_registry": "{image} · узел без него загружает его из реестра {registry}", + "settings_row_base_image_local": "{image} · у флота нет реестра; каждый узел должен иметь этот образ сам", + "settings_base_image_unset": "Не задан — каждый узел откатывается к DORMICE_BASE_IMAGE из своего env; задайте здесь один раз для всего флота", "settings_defaults_dialog_title": "Квоты новых песочниц по умолчанию", "settings_defaults_dialog_desc": "CPU и память применяются при следующем рождении контейнера (включая холодный старт существующих песочниц после остановки); размер диска фиксируется при рождении диска (первое создание и восстановление из архива) — диск и есть песочница, на месте он никогда не меняется, поэтому перед уменьшением убедитесь, что новый размер не меньше реального содержимого архивированных песочниц.", "settings_defaults_saved": "Квоты новых песочниц по умолчанию обновлены", @@ -61,6 +67,11 @@ "settings_pids_saved": "Лимит pids песочницы изменён на {value}", "settings_pids_label": "Лимит pids на песочницу", "settings_pids_field_desc": "Минимум {min}. Это предохранитель, удерживающий fork-бомбу внутри её песочницы, и он не может быть безлимитным; браузера и нескольких сессий агента достаточно, чтобы превысить 512.", + "settings_base_image_dialog_title": "Сменить базовый образ", + "settings_base_image_dialog_desc": "Каждая песочница без шаблона запускается из этого образа. Новый тег перенаправляет базу: существующие песочницы без шаблона переключатся на него при следующем холодном пробуждении (та же семантика, что и перенаправление шаблона); работающие и замороженные песочницы не затрагиваются. Сначала отправьте новый образ в реестр флота — узел, у которого его нет, загрузит его автоматически.", + "settings_base_image_label": "Ссылка на образ", + "settings_base_image_field_desc": "Простая ссылка вида dormice-base:20260831 — без хоста реестра; узел добавит адрес реестра при загрузке.", + "settings_base_image_saved": "Базовый образ изменён на {image}", "settings_policy_dialog_title": "Политика жизненного цикла по умолчанию", "settings_policy_dialog_desc": "Влияет только на песочницы, создаваемые будущими вызовами acquire — существующие сохраняют свои политики; их меняют пакетно в списке песочниц.", "settings_policy_saved": "Политика жизненного цикла по умолчанию обновлена", diff --git a/packages/console/messages/zh-CN/settings.json b/packages/console/messages/zh-CN/settings.json index f4205496..389f3961 100644 --- a/packages/console/messages/zh-CN/settings.json +++ b/packages/console/messages/zh-CN/settings.json @@ -12,6 +12,8 @@ "settings_hint_sandbox_memory": "默认内存上限的首启种子值 — 生效值在上方运营旋钮里", "settings_hint_sandbox_pids_limit": "pids 上限的首启种子值 — 生效值在上方运营旋钮里", "settings_hint_sandbox_domain": "沙箱规范域名的首启种子值 — 生效配置(含别名)在「域名」页的沙箱域名卡里", + "settings_hint_base_image": "舰队基础镜像的首启种子值 — 生效值在上方运营旋钮里", + "settings_hint_registry_address": "舰队镜像仓库地址(主机:端口),install.sh 在网关机旁开的那一个;节点缺镜像时从这里拉取", "settings_hint_ingress_file": "网关接管的 Caddy 配置文件;配了才能在「域名」页网页绑定域名", "settings_hint_ingress_reload_cmd": "改完代理配置后的重载命令;不配默认 caddy reload 托管文件本身", "settings_hint_s3_endpoint": "归档端点的首启种子值 — 生效配置在上方归档存储卡里", @@ -48,6 +50,10 @@ "settings_row_policy": "默认生命周期策略", "settings_row_pids": "沙箱 pids 上限", "settings_row_pids_value": "{n}(整箱在宿主侧的进程+线程预算,撞顶即整箱退出)", + "settings_row_base_image": "基础镜像", + "settings_row_base_image_registry": "{image} · 节点缺它时从仓库 {registry} 拉取", + "settings_row_base_image_local": "{image} · 舰队没有镜像仓库,每台节点须自备此镜像", + "settings_base_image_unset": "未设置 — 各节点回退到自己 env 里的 DORMICE_BASE_IMAGE;在这里设一次,全舰队共用", "settings_defaults_dialog_title": "调整新沙箱的默认配额", "settings_defaults_dialog_desc": "CPU/内存在下一次容器出生时生效(含停止后冷启动的存量沙箱);磁盘在磁盘出生时定型(首次创建与归档恢复) — 磁盘是沙箱的本体,永不原地改尺寸,调小前注意别小于归档沙箱的真实内容。", "settings_defaults_saved": "新沙箱默认配额已更新", @@ -61,6 +67,11 @@ "settings_pids_saved": "沙箱 pids 上限已改为 {value}", "settings_pids_label": "每个沙箱的 pids 上限", "settings_pids_field_desc": "最低 {min}。这是防 fork 炸弹波及整机的闸,不能设为无上限;一个浏览器加几个 agent 会话就能超过 512。", + "settings_base_image_dialog_title": "更换基础镜像", + "settings_base_image_dialog_desc": "没有模板的沙箱都从这个镜像出壳。改成新标签就是给基底换代:存量无模板沙箱在下一次冷唤醒时换壳跟上(与模板重指同一语义),运行中和冻结中的沙箱不受影响。新镜像要先推进舰队镜像仓库,节点缺它时会自动拉取。", + "settings_base_image_label": "镜像引用", + "settings_base_image_field_desc": "裸引用,如 dormice-base:20260831 — 不带仓库主机名,节点拉取时自动加上仓库地址。", + "settings_base_image_saved": "基础镜像已改为 {image}", "settings_policy_dialog_title": "调整默认生命周期策略", "settings_policy_dialog_desc": "只影响之后 acquire 新建的沙箱 — 存量沙箱各有各的策略,去沙箱列表批量调。", "settings_policy_saved": "默认生命周期策略已更新", diff --git a/packages/console/messages/zh-TW/settings.json b/packages/console/messages/zh-TW/settings.json index ec5a2efd..7b05f074 100644 --- a/packages/console/messages/zh-TW/settings.json +++ b/packages/console/messages/zh-TW/settings.json @@ -12,6 +12,8 @@ "settings_hint_sandbox_memory": "預設記憶體上限的首次啟動種子值 — 生效值在上方維運旋鈕裡", "settings_hint_sandbox_pids_limit": "pids 上限的首啟種子值 — 生效值在上方營運旋鈕裡", "settings_hint_sandbox_domain": "沙箱正規網域的首次啟動種子值 — 生效設定(含別名)在「網域」頁的沙箱網域卡裡", + "settings_hint_base_image": "艦隊基礎映像的首次啟動種子值 — 生效值在上方維運旋鈕裡", + "settings_hint_registry_address": "艦隊映像倉庫位址(主機:連接埠),install.sh 在閘道機旁開的那一個;節點缺映像時從這裡拉取", "settings_hint_ingress_file": "網關接管的 Caddy 設定檔;設定了才能在「網域」頁用網頁綁定網域", "settings_hint_ingress_reload_cmd": "改完代理設定後的重新載入指令;不設定則預設 caddy reload 託管檔案本身", "settings_hint_s3_endpoint": "封存物件儲存的位址;四件套齊了封存才啟用", @@ -48,6 +50,10 @@ "settings_row_policy": "預設生命週期策略", "settings_row_pids": "沙箱 pids 上限", "settings_row_pids_value": "{n}(整箱在宿主側的程序+執行緒預算,撞頂即整箱退出)", + "settings_row_base_image": "基礎映像", + "settings_row_base_image_registry": "{image} · 節點缺它時從倉庫 {registry} 拉取", + "settings_row_base_image_local": "{image} · 艦隊沒有映像倉庫,每台節點須自備此映像", + "settings_base_image_unset": "未設定 — 各節點回退到自己 env 裡的 DORMICE_BASE_IMAGE;在這裡設一次,全艦隊共用", "settings_defaults_dialog_title": "調整新沙箱的預設配額", "settings_defaults_dialog_desc": "CPU/記憶體在下一次容器出生時生效(含停止後冷啟動的既有沙箱);磁碟在磁碟出生時定型(首次建立與封存還原)— 磁碟是沙箱的本體,永不原地改尺寸,調小前注意別小於已封存沙箱的真實內容。", "settings_defaults_saved": "新沙箱預設配額已更新", @@ -61,6 +67,11 @@ "settings_pids_saved": "沙箱 pids 上限已改為 {value}", "settings_pids_label": "每個沙箱的 pids 上限", "settings_pids_field_desc": "最低 {min}。這是防 fork 炸彈波及整機的閘,不能設為無上限;一個瀏覽器加幾個 agent 工作階段就能超過 512。", + "settings_base_image_dialog_title": "更換基礎映像", + "settings_base_image_dialog_desc": "沒有範本的沙箱都從這個映像出殼。改成新標籤就是給基底換代:既有的無範本沙箱在下一次冷喚醒時換殼跟上(與範本重指同一語義),執行中和凍結中的沙箱不受影響。新映像要先推進艦隊映像倉庫,節點缺它時會自動拉取。", + "settings_base_image_label": "映像參照", + "settings_base_image_field_desc": "裸參照,如 dormice-base:20260831 — 不帶倉庫主機名,節點拉取時自動加上倉庫位址。", + "settings_base_image_saved": "基礎映像已改為 {image}", "settings_policy_dialog_title": "調整預設生命週期策略", "settings_policy_dialog_desc": "只影響之後 acquire 新建的沙箱 — 既有沙箱各有各的策略,去沙箱列表批次調。", "settings_policy_saved": "預設生命週期策略已更新", diff --git a/packages/console/src/features/settings/components/RuntimeSettingsCard.tsx b/packages/console/src/features/settings/components/RuntimeSettingsCard.tsx index cab7f401..8906b3d9 100644 --- a/packages/console/src/features/settings/components/RuntimeSettingsCard.tsx +++ b/packages/console/src/features/settings/components/RuntimeSettingsCard.tsx @@ -34,9 +34,11 @@ import { useUpdateSettings } from '../hooks/useUpdateSettings'; * 一个弹窗,给哪组就整组替换(updatePolicy 的规矩:界面上看到什么就写 * 下什么)。改的是"之后"不是"已经":容量上限管下一次创建,默认配额管 * 下一次出生的磁盘/容器,默认策略管下一次 acquire 创建的沙箱 — 存量 - * 沙箱一根汗毛都不动,这句话在每个弹窗里都说清。一个例外:pids 上限会 + * 沙箱一根汗毛都不动,这句话在每个弹窗里都说清。两个例外:pids 上限会 * 触达存量沙箱 — 保存时就地扫一遍运行中的壳(docker update,箱内无感), - * 冻结/停止的在下一次唤醒跟上,都不重建。归档存储与沙箱域名不在这张卡: + * 冻结/停止的在下一次唤醒跟上,都不重建;基础镜像(2026-09-15 刀 4 从 + * 节点 env 升为舰队设置)改了=给基底换代,存量无模板沙箱在下一次冷唤醒 + * 换壳跟上,与模板重指同一语义,弹窗照实说。归档存储与沙箱域名不在这张卡: * 前者是独立的归档卡(六字段撑不进一行的形制),后者语义归域名页。追加 * swap 是每台机器自己的旋钮,2026-09-14 随集群刀 2 搬去节点页(刀 3)。 */ @@ -417,6 +419,89 @@ function DefaultPolicyDialog({ ); } +/** + * 基础镜像:单字段弹窗。校验只有"像一个镜像引用"(非空、无空格)— 网关 + * 不查镜像存在性(注册模板也不查:镜像可以晚于配置出现,节点缺它时从舰 + * 队仓库拉),前端更不该替它猜。 + */ +function BaseImageDialog({ settings }: { settings: RuntimeSettings }) { + const [open, setOpen] = useState(false); + const [value, setValue] = useState(''); + const { pending, error, setError, submit } = useUpdateSettings(() => + setOpen(false), + ); + + const trimmed = value.trim(); + const valid = trimmed !== '' && !/\s/.test(trimmed); + + return ( + { + setOpen(next); + if (next) { + setValue(settings.baseImage ?? ''); + setError(null); + } + }} + > + + + + {m.settings_base_image_dialog_title()} + + {m.settings_base_image_dialog_desc()} + + +
{ + event.preventDefault(); + void submit( + { baseImage: trimmed }, + m.settings_base_image_saved({ image: trimmed }), + ); + }} + > + + + + {m.settings_base_image_label()} + + setValue(event.target.value)} + /> + + {m.settings_base_image_field_desc()} + + + {error && {error}} + + + + +
+ + + ); +} + +/** 基础镜像那一行的值:设了就说节点缺它时去哪拉;没设就说各节点在靠自己的 env。 */ +function baseImageLine(settings: RuntimeSettings): string { + if (settings.baseImage === null) return m.settings_base_image_unset(); + return settings.registryAddress === null + ? m.settings_row_base_image_local({ image: settings.baseImage }) + : m.settings_row_base_image_registry({ + image: settings.baseImage, + registry: settings.registryAddress, + }); +} + export function RuntimeSettingsCard({ data }: { data: GetConfigResponse }) { const { settings } = data; return ( @@ -457,6 +542,11 @@ export function RuntimeSettingsCard({ data }: { data: GetConfigResponse }) { value={m.settings_row_pids_value({ n: settings.pidsLimit })} dialog={} /> + } + />
); diff --git a/packages/console/src/features/settings/pages/SettingsPage.tsx b/packages/console/src/features/settings/pages/SettingsPage.tsx index 841baeb8..52e60924 100644 --- a/packages/console/src/features/settings/pages/SettingsPage.tsx +++ b/packages/console/src/features/settings/pages/SettingsPage.tsx @@ -43,6 +43,8 @@ const KEY_HINTS: Record string> = { DORMICE_SANDBOX_MEMORY_GB: m.settings_hint_sandbox_memory, DORMICE_SANDBOX_PIDS_LIMIT: m.settings_hint_sandbox_pids_limit, DORMICE_SANDBOX_DOMAIN: m.settings_hint_sandbox_domain, + DORMICE_BASE_IMAGE: m.settings_hint_base_image, + DORMICE_REGISTRY_ADDRESS: m.settings_hint_registry_address, DORMICE_INGRESS_FILE: m.settings_hint_ingress_file, DORMICE_INGRESS_RELOAD_CMD: m.settings_hint_ingress_reload_cmd, DORMICE_S3_ENDPOINT: m.settings_hint_s3_endpoint, diff --git a/packages/gateway/drizzle/0004_fleet-base-image.sql b/packages/gateway/drizzle/0004_fleet-base-image.sql new file mode 100644 index 00000000..603758ca --- /dev/null +++ b/packages/gateway/drizzle/0004_fleet-base-image.sql @@ -0,0 +1,2 @@ +ALTER TABLE `settings` ADD `base_image` text;--> statement-breakpoint +ALTER TABLE `settings` ADD `registry_address` text; \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/0004_snapshot.json b/packages/gateway/drizzle/meta/0004_snapshot.json new file mode 100644 index 00000000..9db8a52a --- /dev/null +++ b/packages/gateway/drizzle/meta/0004_snapshot.json @@ -0,0 +1,488 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "54d3f360-1488-4db9-99a7-2d393ba2dd1f", + "prevId": "8fba703e-7fc7-4bfc-aa8d-903f01a049d6", + "tables": { + "api_keys": { + "name": "api_keys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + "key_hash" + ], + "isUnique": true + }, + "api_keys_active_name_idx": { + "name": "api_keys_active_name_idx", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"api_keys\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "console_account": { + "name": "console_account", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_secret": { + "name": "session_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "fleet_state_samples": { + "name": "fleet_state_samples", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active": { + "name": "active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "frozen": { + "name": "frozen", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stopped": { + "name": "stopped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "restoring": { + "name": "restoring", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "fleet_state_samples_at_idx": { + "name": "fleet_state_samples_at_idx", + "columns": [ + "at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "nodes": { + "name": "nodes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "added_at": { + "name": "added_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_gb": { + "name": "swap_gb", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_check_in_at": { + "name": "last_check_in_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "interval_seconds": { + "name": "interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config_version": { + "name": "config_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "build": { + "name": "build", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reading": { + "name": "reading", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_cpus": { + "name": "sandbox_cpus", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_memory_gb": { + "name": "sandbox_memory_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_disk_gb": { + "name": "sandbox_disk_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_freeze_after_seconds": { + "name": "default_freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_stop_after_seconds": { + "name": "default_stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_archive_after_seconds": { + "name": "default_archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_endpoint": { + "name": "s3_endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_access_key_id": { + "name": "s3_access_key_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_secret_access_key": { + "name": "s3_secret_access_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_region": { + "name": "s3_region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_force_path_style": { + "name": "s3_force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain": { + "name": "sandbox_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain_aliases": { + "name": "sandbox_domain_aliases", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pids_limit": { + "name": "pids_limit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_image": { + "name": "base_image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registry_address": { + "name": "registry_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/_journal.json b/packages/gateway/drizzle/meta/_journal.json index 0f7d5393..7b1be0a2 100644 --- a/packages/gateway/drizzle/meta/_journal.json +++ b/packages/gateway/drizzle/meta/_journal.json @@ -29,6 +29,13 @@ "when": 1789458250873, "tag": "0003_last-check-in", "breakpoints": true + }, + { + "idx": 4, + "version": "6", + "when": 1789459862989, + "tag": "0004_fleet-base-image", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/gateway/src/config.ts b/packages/gateway/src/config.ts index 0cacfb8a..a8e438ca 100644 --- a/packages/gateway/src/config.ts +++ b/packages/gateway/src/config.ts @@ -158,6 +158,30 @@ const envSchema = z.object({ DORMICE_S3_REGION: z.string().default('us-east-1'), /** Path-style addressing: MinIO needs true; the clouds route by subdomain. */ DORMICE_S3_FORCE_PATH_STYLE: z.stringbool().default(false), + /** + * The fleet's base image — the seed of settings.baseImage (shared + * settings.ts has the knob): a bare reference the nodes boot + * template-less sandboxes from, and pull from the fleet's registry when + * a host lacks it. The knob's old home was every node's own env + * (2026-09-15, fourth cut): a fleet shares one base, and a second + * machine must need no setting of its own to know it. install.sh writes + * the tag it built here; a gateway.env from before the knob gets the + * line appended. + */ + DORMICE_BASE_IMAGE: z.string().regex(/^\S+$/).optional(), + /** + * The fleet's image registry, host and port — the seed of + * settings.registryAddress: where a node pulls an image it lacks. + * install.sh runs one beside the gateway on the machine's intranet + * address, port 5000, and writes that here; unset = no registry. + */ + DORMICE_REGISTRY_ADDRESS: z + .string() + .regex(/^[A-Za-z0-9.-]+(:\d{1,5})?$/, { + error: + 'DORMICE_REGISTRY_ADDRESS is a registry host and port, e.g. 10.0.0.5:5000 — no scheme, no path', + }) + .optional(), /** * The Caddy config file the gateway owns — the switch for web-based * domain binding (setIngress rewrites the file, reloads Caddy, Caddy @@ -232,6 +256,8 @@ export const CONFIG_KEYS: Record = { DORMICE_S3_SECRET_ACCESS_KEY: { sensitive: true }, DORMICE_S3_REGION: { sensitive: false }, DORMICE_S3_FORCE_PATH_STYLE: { sensitive: false }, + DORMICE_BASE_IMAGE: { sensitive: false }, + DORMICE_REGISTRY_ADDRESS: { sensitive: false }, DORMICE_INGRESS_FILE: { sensitive: false }, DORMICE_INGRESS_RELOAD_CMD: { sensitive: false }, }; diff --git a/packages/gateway/src/db/node-config.ts b/packages/gateway/src/db/node-config.ts index 78875ef7..c51ed57b 100644 --- a/packages/gateway/src/db/node-config.ts +++ b/packages/gateway/src/db/node-config.ts @@ -25,6 +25,8 @@ export function readNodeConfig( sandboxDomain: settings.sandboxDomain, sandboxDomainAliases: settings.sandboxDomainAliases, pidsLimit: settings.pidsLimit, + baseImage: settings.baseImage, + registryAddress: settings.registryAddress, }, node: { swapGb: node.swapGb }, templates: listTemplates(db), diff --git a/packages/gateway/src/db/schema.ts b/packages/gateway/src/db/schema.ts index db9bdaf5..c978352d 100644 --- a/packages/gateway/src/db/schema.ts +++ b/packages/gateway/src/db/schema.ts @@ -108,6 +108,15 @@ export const settings = sqliteTable('settings', { sandboxDomainAliases: text('sandbox_domain_aliases').notNull(), /** The pids cgroup cap on every sandbox container, fleet-wide. */ pidsLimit: integer('pids_limit').notNull(), + /** + * The fleet's base image, a bare reference (shared settings.ts + * baseImage); NULL = none set. Born after the row (the fourth cut), so + * a table seeded before it holds NULL here until the env seed fills it + * once or the console writes it (db/settings.ts ensureSettings). + */ + baseImage: text('base_image'), + /** The fleet's image registry, host:port (shared settings.ts registryAddress); NULL = no registry. Born with baseImage, filled the same way. */ + registryAddress: text('registry_address'), /** Null until the first updateSettings: "still exactly the seed" is information. */ updatedAt: text('updated_at'), }); diff --git a/packages/gateway/src/db/settings.test.ts b/packages/gateway/src/db/settings.test.ts index e4c831d5..4e59f3a3 100644 --- a/packages/gateway/src/db/settings.test.ts +++ b/packages/gateway/src/db/settings.test.ts @@ -53,12 +53,44 @@ describe('the settings row', () => { sandboxDomain: null, sandboxDomainAliases: [], pidsLimit: 4096, + baseImage: null, + registryAddress: null, updatedAt: null, }); expect(readConfigVersion(db)).toBe(1); expect(readS3Settings(db)).toBeNull(); }); + it('a column born after the row is filled once from the env while empty, counted as a change; a value already there stands', () => { + // A table seeded before the base image and the registry were knobs. + const { db, config } = seeded(); + expect(readSettings(db).baseImage).toBeNull(); + ensureSettings(db, { + ...config, + DORMICE_BASE_IMAGE: 'dormice-base:20260831', + DORMICE_REGISTRY_ADDRESS: '10.0.0.5:5000', + }); + expect(readSettings(db)).toMatchObject({ + baseImage: 'dormice-base:20260831', + registryAddress: '10.0.0.5:5000', + }); + // The nodes must hear of it: one version up, for both columns at once. + expect(readConfigVersion(db)).toBe(2); + // From here the table wins, as for every other seed. + ensureSettings(db, { + ...config, + DORMICE_BASE_IMAGE: 'dormice-base:20260901', + DORMICE_REGISTRY_ADDRESS: '10.0.0.6:5000', + }); + expect(readSettings(db).baseImage).toBe('dormice-base:20260831'); + expect(readSettings(db).registryAddress).toBe('10.0.0.5:5000'); + expect(readConfigVersion(db)).toBe(2); + // An env with no seed fills nothing and counts nothing. + const bare = seeded(); + ensureSettings(bare.db, bare.config); + expect(readConfigVersion(bare.db)).toBe(1); + }); + it('an S3 seed turns the archive default on and keeps the keys for the bundle, never for the view', () => { const { db } = seeded({ ...S3_ENV, @@ -109,6 +141,10 @@ describe('the settings row', () => { expect(after.s3?.bucket).toBe('seed-bucket'); expect(after.updatedAt).toBe(NOW.toISOString()); expect(readConfigVersion(db)).toBe(2); + expect( + writeSettings(db, { baseImage: 'dormice-base:20260901' }, NOW).baseImage, + ).toBe('dormice-base:20260901'); + expect(readConfigVersion(db)).toBe(3); // Clearing the store is all six columns at once; the keys go with it. writeSettings( db, @@ -117,7 +153,7 @@ describe('the settings row', () => { ); expect(readSettings(db).s3).toBeNull(); expect(readS3Settings(db)).toBeNull(); - expect(readConfigVersion(db)).toBe(3); + expect(readConfigVersion(db)).toBe(4); }); it('bumpConfigVersion counts up by one and answers the new version', () => { diff --git a/packages/gateway/src/db/settings.ts b/packages/gateway/src/db/settings.ts index 0d758fe7..cd6e3ab7 100644 --- a/packages/gateway/src/db/settings.ts +++ b/packages/gateway/src/db/settings.ts @@ -13,36 +13,77 @@ import { type SettingsRow, settings } from './schema'; /** The console_account fixed-id pattern: "at most one row" as a schema fact. */ const SETTINGS_ROW_ID = 1; +/** + * The settings row as the env seeds it, whole (the import reuses it as the + * row a node's old settings overwrite, import.ts). The archive default is + * adjudicated here: an S3 seed present means new sandboxes archive after a + * week, absent means never. Version 1 is the seed; every change counts up + * from there, and the nodes compare against it at each check-in. + */ +export function seedRow(config: Config): typeof settings.$inferInsert { + const s3 = s3Seed(config); + return { + id: SETTINGS_ROW_ID, + version: 1, + sandboxCpus: config.DORMICE_SANDBOX_CPUS, + sandboxMemoryGb: config.DORMICE_SANDBOX_MEMORY_GB, + sandboxDiskGb: config.DORMICE_SANDBOX_DISK_GB, + defaultFreezeAfterSeconds: DEFAULT_LIFECYCLE_POLICY.freezeAfterSeconds, + defaultStopAfterSeconds: DEFAULT_LIFECYCLE_POLICY.stopAfterSeconds, + defaultArchiveAfterSeconds: s3 ? ARCHIVE_DEFAULT_SECONDS : null, + ...s3Columns(s3), + sandboxDomain: config.DORMICE_SANDBOX_DOMAIN ?? null, + sandboxDomainAliases: '[]', + pidsLimit: config.DORMICE_SANDBOX_PIDS_LIMIT, + baseImage: config.DORMICE_BASE_IMAGE ?? null, + registryAddress: config.DORMICE_REGISTRY_ADDRESS ?? null, + updatedAt: null, + }; +} + /** * Seeds the settings row from the env at the gateway's first start — * insert-or-nothing, so every later start finds the row and leaves it * alone: the table is the one truth from then on, and a later env edit of * a seed is deliberately ignored (the daemon's discipline since 2026-07-19, - * shared/settings.ts has the line). The archive default is adjudicated - * here: an S3 seed present means new sandboxes archive after a week, - * absent means never. Version 1 is the seed; every change counts up from - * there, and the nodes compare against it at each check-in. + * shared/settings.ts has the line). + * + * One exception, for columns born after the row: the base image and the + * registry (fourth cut, 2026-09-15). A table seeded before them holds + * NULL there, and NULL is "never set", not "set to nothing" — so a seed + * present in the env fills an empty column once, and a value the console + * wrote (or an earlier seed) stands. Counted as a configuration change + * when it fills anything: the nodes take the bundle at their next + * check-in and stop leaning on their own env for the base image. */ export function ensureSettings(db: Db, config: Config): void { - const s3 = s3Seed(config); - db.insert(settings) - .values({ - id: SETTINGS_ROW_ID, - version: 1, - sandboxCpus: config.DORMICE_SANDBOX_CPUS, - sandboxMemoryGb: config.DORMICE_SANDBOX_MEMORY_GB, - sandboxDiskGb: config.DORMICE_SANDBOX_DISK_GB, - defaultFreezeAfterSeconds: DEFAULT_LIFECYCLE_POLICY.freezeAfterSeconds, - defaultStopAfterSeconds: DEFAULT_LIFECYCLE_POLICY.stopAfterSeconds, - defaultArchiveAfterSeconds: s3 ? ARCHIVE_DEFAULT_SECONDS : null, - ...s3Columns(s3), - sandboxDomain: config.DORMICE_SANDBOX_DOMAIN ?? null, - sandboxDomainAliases: '[]', - pidsLimit: config.DORMICE_SANDBOX_PIDS_LIMIT, - updatedAt: null, - }) - .onConflictDoNothing() - .run(); + const seed = seedRow(config); + db.transaction((tx) => { + tx.insert(settings).values(seed).onConflictDoNothing().run(); + const row = tx + .select({ + baseImage: settings.baseImage, + registryAddress: settings.registryAddress, + }) + .from(settings) + .where(eq(settings.id, SETTINGS_ROW_ID)) + .get(); + const late = { + ...(row?.baseImage === null && seed.baseImage !== null + ? { baseImage: seed.baseImage } + : {}), + ...(row?.registryAddress === null && seed.registryAddress !== null + ? { registryAddress: seed.registryAddress } + : {}), + }; + if (Object.keys(late).length > 0) { + tx.update(settings) + .set(late) + .where(eq(settings.id, SETTINGS_ROW_ID)) + .run(); + bumpConfigVersion(tx); + } + }); } /** The six S3 columns as one unit: a store, or all NULL = off. */ @@ -97,6 +138,8 @@ function toView(row: SettingsRow): RuntimeSettings { // throw right here, not read as "no aliases". sandboxDomainAliases: JSON.parse(row.sandboxDomainAliases) as string[], pidsLimit: row.pidsLimit, + baseImage: row.baseImage, + registryAddress: row.registryAddress, updatedAt: row.updatedAt, }; } @@ -208,6 +251,7 @@ export function writeSettings( ? { sandboxDomainAliases: JSON.stringify(patch.sandboxDomainAliases) } : {}), ...(patch.pidsLimit !== undefined ? { pidsLimit: patch.pidsLimit } : {}), + ...(patch.baseImage !== undefined ? { baseImage: patch.baseImage } : {}), version: sql`${settings.version} + 1`, updatedAt: now.toISOString(), }) diff --git a/packages/gateway/src/routes/nodes.test.ts b/packages/gateway/src/routes/nodes.test.ts index aab51782..f34aa857 100644 --- a/packages/gateway/src/routes/nodes.test.ts +++ b/packages/gateway/src/routes/nodes.test.ts @@ -79,6 +79,8 @@ describe('the check-in as the configuration pull', () => { sandboxDomain: 'sbx.example.com', sandboxDomainAliases: [], pidsLimit: 512, + baseImage: null, + registryAddress: null, }); expect(config.node).toEqual({ swapGb: 0 }); expect(config.templates).toMatchObject([{ name: 'py', image: 'img-a' }]); diff --git a/packages/gateway/src/routes/settings.test.ts b/packages/gateway/src/routes/settings.test.ts index 2fdfa4d3..324389a2 100644 --- a/packages/gateway/src/routes/settings.test.ts +++ b/packages/gateway/src/routes/settings.test.ts @@ -164,6 +164,46 @@ describe('updateSettings on the gateway', () => { expect((await rpc(app, '/updateSettings', {})).statusCode).toBe(400); }); + it('re-points the base image, seeded from the env and carried to the nodes in the bundle; the registry is read-only over the wire', async () => { + const { app, db } = testGateway({ + DORMICE_BASE_IMAGE: 'dormice-base:20260831', + DORMICE_REGISTRY_ADDRESS: '10.0.0.5:5000', + }); + expect(await settingsOf(app)).toMatchObject({ + baseImage: 'dormice-base:20260831', + registryAddress: '10.0.0.5:5000', + }); + const res = await rpc(app, '/updateSettings', { + baseImage: 'dormice-base:20260901', + }); + expect(res.statusCode).toBe(200); + expect( + updateSettingsResponseSchema.parse(res.json()).settings.baseImage, + ).toBe('dormice-base:20260901'); + expect(readConfigVersion(db)).toBe(2); + const bundle = await rpc( + app, + '/checkIn', + checkInOf('b', 'http://10.0.0.7:80', { configVersion: 1 }), + ); + expect(bundle.json().config.settings).toMatchObject({ + baseImage: 'dormice-base:20260901', + registryAddress: '10.0.0.5:5000', + }); + // Not a reference: refused at the door, the table stands. + expect( + (await rpc(app, '/updateSettings', { baseImage: 'two words' })) + .statusCode, + ).toBe(400); + // The registry is not a wire knob in this cut: an unknown key is + // stripped, and a patch of it alone is the empty patch. + expect( + (await rpc(app, '/updateSettings', { registryAddress: '1.2.3.4:5000' })) + .statusCode, + ).toBe(400); + expect((await settingsOf(app)).registryAddress).toBe('10.0.0.5:5000'); + }); + it('a new default policy is stored for the nodes to hand to their next acquire', async () => { const { app } = testGateway(); const res = await rpc(app, '/updateSettings', { diff --git a/packages/gateway/src/routes/settings.ts b/packages/gateway/src/routes/settings.ts index 09777aef..c0cdaed0 100644 --- a/packages/gateway/src/routes/settings.ts +++ b/packages/gateway/src/routes/settings.ts @@ -248,6 +248,9 @@ export const settingsRoutes: FastifyPluginAsyncZod< ...(patch.pidsLimit !== undefined ? [`pidsLimit=${patch.pidsLimit}`] : []), + ...(patch.baseImage !== undefined + ? [`baseImage=${patch.baseImage}`] + : []), ].join(', ')}; the nodes apply it at their next check-in`, ); return { settings }; diff --git a/packages/server/drizzle/0026_fleet-base-image.sql b/packages/server/drizzle/0026_fleet-base-image.sql new file mode 100644 index 00000000..d9868c9f --- /dev/null +++ b/packages/server/drizzle/0026_fleet-base-image.sql @@ -0,0 +1,2 @@ +ALTER TABLE `runtime_settings` ADD `base_image` text;--> statement-breakpoint +ALTER TABLE `runtime_settings` ADD `registry_address` text; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0026_snapshot.json b/packages/server/drizzle/meta/0026_snapshot.json new file mode 100644 index 00000000..3ab6dd66 --- /dev/null +++ b/packages/server/drizzle/meta/0026_snapshot.json @@ -0,0 +1,794 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "8cdf1b9f-73e5-46d1-a0cc-bf448d1887d9", + "prevId": "2769cadc-8aa9-42b5-b4c5-0b60d0138113", + "tables": { + "api_keys": { + "name": "api_keys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + "key_hash" + ], + "isUnique": true + }, + "api_keys_active_name_idx": { + "name": "api_keys_active_name_idx", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"api_keys\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "console_account": { + "name": "console_account", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_secret": { + "name": "session_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "daemon_secrets": { + "name": "daemon_secrets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "envd_signing_secret": { + "name": "envd_signing_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "fleet_snapshots": { + "name": "fleet_snapshots", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "active": { + "name": "active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "frozen": { + "name": "frozen", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stopped": { + "name": "stopped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "restoring": { + "name": "restoring", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "host_metrics_samples": { + "name": "host_metrics_samples", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "cpu_used_pct": { + "name": "cpu_used_pct", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_available_bytes": { + "name": "mem_available_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_total_bytes": { + "name": "swap_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_used_bytes": { + "name": "swap_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_total_bytes": { + "name": "disk_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_used_bytes": { + "name": "disk_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_available_bytes": { + "name": "disk_available_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "runtime_settings": { + "name": "runtime_settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_version": { + "name": "config_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config_applied_at": { + "name": "config_applied_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_cpus": { + "name": "sandbox_cpus", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_memory_gb": { + "name": "sandbox_memory_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_disk_gb": { + "name": "sandbox_disk_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_freeze_after_seconds": { + "name": "default_freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_stop_after_seconds": { + "name": "default_stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_archive_after_seconds": { + "name": "default_archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_gb": { + "name": "swap_gb", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "s3_endpoint": { + "name": "s3_endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_access_key_id": { + "name": "s3_access_key_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_secret_access_key": { + "name": "s3_secret_access_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_region": { + "name": "s3_region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_force_path_style": { + "name": "s3_force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain": { + "name": "sandbox_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain_aliases": { + "name": "sandbox_domain_aliases", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pids_limit": { + "name": "pids_limit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_image": { + "name": "base_image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registry_address": { + "name": "registry_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sandbox_metrics_samples": { + "name": "sandbox_metrics_samples", + "columns": { + "sandbox_id": { + "name": "sandbox_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "at": { + "name": "at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cpu_count": { + "name": "cpu_count", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cpu_used_pct": { + "name": "cpu_used_pct", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_used_bytes": { + "name": "mem_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_cache_bytes": { + "name": "mem_cache_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_used_bytes": { + "name": "swap_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_total_bytes": { + "name": "swap_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_used_bytes": { + "name": "disk_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "disk_total_bytes": { + "name": "disk_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "sandbox_metrics_samples_sandbox_at_idx": { + "name": "sandbox_metrics_samples_sandbox_at_idx", + "columns": [ + "sandbox_id", + "at" + ], + "isUnique": false + }, + "sandbox_metrics_samples_at_idx": { + "name": "sandbox_metrics_samples_at_idx", + "columns": [ + "at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sandboxes": { + "name": "sandboxes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "freeze_after_seconds": { + "name": "freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stop_after_seconds": { + "name": "stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "archive_after_seconds": { + "name": "archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpus": { + "name": "cpus", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "memory_gb": { + "name": "memory_gb", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_gb": { + "name": "disk_gb", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_active_at": { + "name": "last_active_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_exit_at": { + "name": "last_exit_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_exit_code": { + "name": "last_exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_exit_cause": { + "name": "last_exit_cause", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "envs": { + "name": "envs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "on_deadline": { + "name": "on_deadline", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "paused_by_user": { + "name": "paused_by_user", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "sandboxes_name_unique": { + "name": "sandboxes_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index 938835ce..e62752d5 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -183,6 +183,13 @@ "when": 1789375849689, "tag": "0025_config-copy", "breakpoints": true + }, + { + "idx": 26, + "version": "6", + "when": 1789460306678, + "tag": "0026_fleet-base-image", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/server/src/app.test.ts b/packages/server/src/app.test.ts index 1ab5fe29..990bf3ea 100644 --- a/packages/server/src/app.test.ts +++ b/packages/server/src/app.test.ts @@ -1754,7 +1754,7 @@ describe('cold wakes converge onto the current image', () => { const woken = (await acquire(app, { name: 'alice' })).json().sandbox; expect(woken.state).toBe('active'); - expect(await executor.imageOf(created.id)).toBe(executor.baseImage); + expect(await executor.imageOf(created.id)).toBe(executor.baseImage()); expect(executor.removedShells).toEqual([]); }); diff --git a/packages/server/src/config.test.ts b/packages/server/src/config.test.ts index cf4bbfd9..b67f9579 100644 --- a/packages/server/src/config.test.ts +++ b/packages/server/src/config.test.ts @@ -8,10 +8,16 @@ describe('loadConfig executor knobs', () => { expect(loadConfig(TOKEN).DORMICE_EXECUTOR).toBe('fake'); }); - it('rejects the docker executor without a base image', () => { - expect(() => loadConfig({ ...TOKEN, DORMICE_EXECUTOR: 'docker' })).toThrow( - /DORMICE_BASE_IMAGE is required/, - ); + it('accepts the docker executor without a base image: the fleet settings name it, the env is the fallback', () => { + const config = loadConfig({ + ...TOKEN, + DORMICE_EXECUTOR: 'docker', + DORMICE_DB_PATH: '/var/lib/dormice/dormice.db', + }); + expect(config.DORMICE_BASE_IMAGE).toBeUndefined(); + expect(() => + loadConfig({ ...TOKEN, DORMICE_BASE_IMAGE: 'has a space:1' }), + ).toThrow(); }); it('accepts the docker executor with a base image and absolute paths', () => { diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index f3971edf..7cfd494a 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -90,8 +90,18 @@ const envSchema = z.object({ * default so a bare `pnpm dev` works on any machine. */ DORMICE_EXECUTOR: z.enum(['fake', 'docker']).default('fake'), - /** Image sandboxes boot from, e.g. dormice-base:20260708. Required by the docker executor. */ - DORMICE_BASE_IMAGE: z.string().optional(), + /** + * The image template-less sandboxes boot from — a fleet setting since + * the fourth cut (shared settings.ts baseImage: every node shares one + * base, pulled from the fleet registry), and this variable is the + * node's fallback while the fleet's settings name none (a node upgraded + * before its gateway learned the knob; db/templates.ts + * resolveBaseImage). main.ts says at boot which of the two is in force. + * Until 2026-09-15 this was the knob's only home and required in docker + * mode; a node without it now refuses only at the moment a sandbox + * would need a base image and the fleet has none. + */ + DORMICE_BASE_IMAGE: z.string().regex(/^\S+$/).optional(), /** Sandbox disk images and their mount points live here (docker executor only). */ DORMICE_DATA_DIR: z.string().default('/var/lib/dormice'), /** @@ -191,14 +201,6 @@ function isLoopbackUrl(url: string): boolean | null { } const checkedSchema = envSchema - .refine( - (cfg) => cfg.DORMICE_EXECUTOR !== 'docker' || !!cfg.DORMICE_BASE_IMAGE, - { - message: - 'DORMICE_BASE_IMAGE is required when DORMICE_EXECUTOR=docker — build one from images/Dockerfile', - path: ['DORMICE_BASE_IMAGE'], - }, - ) // Production discipline for real sandboxes: a relative ledger path // silently depends on the start directory, and a wrong start directory // means an empty ledger facing real sandboxes — the exact catastrophe diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 23b4a43c..ce24734d 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -321,6 +321,17 @@ export const runtimeSettings = sqliteTable('runtime_settings', { * for rows from before the move. */ pidsLimit: integer('pids_limit'), + /** + * The fleet's base image (shared settings.ts baseImage), in the copy + * since the fourth cut (2026-09-15): what a template-less sandbox boots + * from, pulled from the fleet registry when this host lacks it. NULL = + * the fleet names none — this node then falls back to its own + * DORMICE_BASE_IMAGE, the knob's old home (db/templates.ts + * resolveBaseImage). + */ + baseImage: text('base_image'), + /** The fleet's image registry, host:port (shared settings.ts registryAddress); NULL = no registry, a missing image is a plain error. */ + registryAddress: text('registry_address'), /** The old single-machine row's last edit — the gateway's timestamp now; kept for the cut-4 import, never written by the node. */ updatedAt: text('updated_at'), }); diff --git a/packages/server/src/db/settings.ts b/packages/server/src/db/settings.ts index 2459a542..e20aebbd 100644 --- a/packages/server/src/db/settings.ts +++ b/packages/server/src/db/settings.ts @@ -86,6 +86,8 @@ export function applyNodeConfig( sandboxDomain: settings.sandboxDomain, sandboxDomainAliases: JSON.stringify(settings.sandboxDomainAliases), pidsLimit: settings.pidsLimit, + baseImage: settings.baseImage, + registryAddress: settings.registryAddress, }; const { id: _id, ...set } = row; db.transaction((tx) => { @@ -117,6 +119,8 @@ export function readNodeConfig(db: Db): NodeConfigBundle { sandboxDomain: view.sandboxDomain, sandboxDomainAliases: view.sandboxDomainAliases, pidsLimit: view.pidsLimit, + baseImage: view.baseImage, + registryAddress: view.registryAddress, }, node: { swapGb: row.swapGb }, templates: db.select().from(templates).orderBy(templates.name).all(), @@ -161,6 +165,8 @@ function toView(row: RuntimeSettingsRow): NodeSettings { sandboxDomainAliases: JSON.parse(row.sandboxDomainAliases!) as string[], pidsLimit: row.pidsLimit!, // biome-ignore-end lint/style/noNonNullAssertion: a copy writes every column (applyNodeConfig), and readRow refuses anything that is not a copy + baseImage: row.baseImage, + registryAddress: row.registryAddress, }; } diff --git a/packages/server/src/db/templates.ts b/packages/server/src/db/templates.ts index 06c43dcb..870fbd32 100644 --- a/packages/server/src/db/templates.ts +++ b/packages/server/src/db/templates.ts @@ -1,6 +1,7 @@ import { eq } from 'drizzle-orm'; import type { Db } from './db'; import { sandboxes, type TemplateRow, templates } from './schema'; +import { readRuntimeSettings } from './settings'; /** * Readers over the node's copy of the templates table (schema.ts). The @@ -28,14 +29,39 @@ export function sandboxNamesUsingTemplate(db: Db, name: string): string[] { .map((row) => row.name); } +/** + * The image a template-less sandbox boots from, as this node resolves it: + * the fleet's base image from the copy (shared settings.ts baseImage — a + * fleet setting since the fourth cut, 2026-09-15), or, while the fleet + * names none, the node's own DORMICE_BASE_IMAGE — the knob's old home, + * kept as the fallback so a node upgraded before its gateway learned the + * knob keeps building sandboxes (main.ts warns about the fallback at + * boot). Neither is a refusal that says where to set it: a sandbox built + * from a guessed image would be the wrong sandbox. The executors consult + * this through their baseImage closure (main.ts), read at each birth — + * the same live-view shape as the resource knobs, so a console edit + * reaches the next birth without a restart. + */ +export function resolveBaseImage( + db: Db, + envFallback: string | undefined, +): string { + const fleet = readRuntimeSettings(db).baseImage; + if (fleet !== null) return fleet; + if (envFallback !== undefined) return envFallback; + throw new Error( + 'no base image: the fleet settings name none and DORMICE_BASE_IMAGE is not set on this node — set baseImage at the gateway (console › settings, or DORMICE_BASE_IMAGE in its gateway.env before its first start)', + ); +} + /** * The single arbiter turning a sandbox row's template into the image its * next shell boots. Null means the base image — expressed as undefined so - * the executor falls back to its own configured default. A registered name - * resolves to the template's *current* image; a missing row means the - * removal guard was bypassed (a template removed while this node was out - * of the fleet), which is worth an honest crash, not a silent fallback to - * the wrong image. + * the executor resolves it through its own live view (resolveBaseImage + * above, wired in main.ts). A registered name resolves to the template's + * *current* image; a missing row means the removal guard was bypassed (a + * template removed while this node was out of the fleet), which is worth + * an honest crash, not a silent fallback to the wrong image. */ export function resolveImage( db: Db, diff --git a/packages/server/src/e2b/compat.test.ts b/packages/server/src/e2b/compat.test.ts index 46b84efc..e40d8ed3 100644 --- a/packages/server/src/e2b/compat.test.ts +++ b/packages/server/src/e2b/compat.test.ts @@ -2590,24 +2590,20 @@ describe('E2B templates', () => { }); }); - it("'base', the configured base image name, and absence all mean the base image", async () => { - const t = testApp( - new FakeExecutor(), - {}, - { - DORMICE_BASE_IMAGE: 'dormice-base:test', - }, - ); + it("'base', the fleet's base image name, and absence all mean the base image", async () => { + // The fake's own base stands in for the fleet's (the executor's live + // view is what the face asks; main.ts wires the copy behind it). + const t = testApp(); for (const payload of [ {}, { templateID: 'base' }, - { templateID: 'dormice-base:test' }, + { templateID: FAKE_BASE_IMAGE }, ]) { const res = await control(t, 'POST', '/sandboxes', payload); expect(res.statusCode).toBe(201); const body = res.json(); // Echo keeps the pre-templates shape: the base image name, no alias. - expect(body.templateID).toBe('dormice-base:test'); + expect(body.templateID).toBe(FAKE_BASE_IMAGE); expect(body.alias).toBeUndefined(); expect(await t.executor.imageOf(body.sandboxID)).toBe(FAKE_BASE_IMAGE); } diff --git a/packages/server/src/e2b/control.ts b/packages/server/src/e2b/control.ts index e5bef817..e6a9e713 100644 --- a/packages/server/src/e2b/control.ts +++ b/packages/server/src/e2b/control.ts @@ -167,11 +167,12 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( // What the views report as the sandbox's template. E2B's alias is the // template's human name — present only when a registered template was - // used; a base sandbox echoes the base image name (or 'base') as its - // templateID, the honest pre-templates behavior kept for round-trips. + // used; a base sandbox echoes the base image's name as its templateID, + // the honest pre-templates behavior kept for round-trips (the fleet's + // base image, resolved live — the executor's view over the copy). function templateFields(row: SandboxRow) { return { - templateID: row.template ?? config.DORMICE_BASE_IMAGE ?? 'base', + templateID: row.template ?? executor.baseImage(), ...(row.template ? { alias: row.template } : {}), }; } @@ -291,7 +292,7 @@ export const e2bControlRoutes: FastifyPluginAsyncZod = async ( template = body.templateID; } else if ( body.templateID !== 'base' && - body.templateID !== config.DORMICE_BASE_IMAGE + body.templateID !== executor.baseImage() ) { throw apiError(404, `template '${body.templateID}' not found`); } diff --git a/packages/server/src/executor/docker-exitof.test.ts b/packages/server/src/executor/docker-exitof.test.ts index 5cb02357..c0b4d8ca 100644 --- a/packages/server/src/executor/docker-exitof.test.ts +++ b/packages/server/src/executor/docker-exitof.test.ts @@ -80,7 +80,8 @@ function stubDocker( function executor(docker: Docker, cgroupRoot?: string): DockerExecutor { return new DockerExecutor( { - baseImage: 'unused', + baseImage: () => 'unused', + registry: { address: () => null, username: 'dormice', password: 'x' }, dataDir: '/nonexistent', resources: () => ({ diskSizeGb: 1, cpus: 1, memoryGb: 1 }), pidsLimit: () => 4096, diff --git a/packages/server/src/executor/docker-images.test.ts b/packages/server/src/executor/docker-images.test.ts new file mode 100644 index 00000000..473b8a5d --- /dev/null +++ b/packages/server/src/executor/docker-images.test.ts @@ -0,0 +1,207 @@ +import type Docker from 'dockerode'; +import { describe, expect, it } from 'vitest'; +import { DockerExecutor, namesRegistry, splitRepoTag } from './docker'; + +/** + * ensureImage's branches against a stub Docker client — the contract exam + * cannot reach them (it has no registry to pull from): an image the host + * has is left alone; a bare image the host lacks comes from the fleet + * registry under the fleet credential and is tagged back under its bare + * name; an image naming its own registry is pulled as written, no + * credential; no registry is a refusal that says both ways out; a pull + * that fails names the push command; two callers share one pull. The + * real pull is the test machine's (a tagged probe image through the + * fleet registry, 手册). + */ + +interface Stub { + docker: Docker; + calls: { + inspected: string[]; + pulled: Array<{ source: string; auth: unknown }>; + tagged: Array<{ source: string; repo: string; tag: string }>; + }; + /** Which images "the host has" — inspect answers 200 for these, 404 otherwise. */ + present: Set; +} + +function stubDocker(opts: { failPull?: string } = {}): Stub { + const present = new Set(); + const calls: Stub['calls'] = { inspected: [], pulled: [], tagged: [] }; + let release: (() => void) | null = null; + const docker = { + getImage(name: string) { + return { + async inspect() { + calls.inspected.push(name); + if (!present.has(name)) { + throw Object.assign(new Error('no such image'), { + statusCode: 404, + }); + } + return { Id: 'sha256:abc' }; + }, + async tag(o: { repo: string; tag: string }) { + calls.tagged.push({ source: name, ...o }); + present.add(`${o.repo}:${o.tag}`); + }, + }; + }, + async pull(source: string, o: { authconfig?: unknown }) { + calls.pulled.push({ source, auth: o.authconfig }); + return { source }; + }, + modem: { + followProgress( + stream: { source: string }, + done: (err: Error | null) => void, + ) { + if (opts.failPull === stream.source) { + done(new Error('manifest unknown: manifest unknown')); + return; + } + // Held until the test releases it, so two callers can be seen to + // share the one pull; released at once when nobody holds it. + const finish = () => { + present.add(stream.source); + done(null); + }; + if (release === null) { + release = finish; + setTimeout(() => { + if (release === finish) { + release = null; + finish(); + } + }, 20); + } else { + finish(); + } + }, + }, + }; + return { docker: docker as unknown as Docker, calls, present }; +} + +function executor(stub: Stub, registry: string | null): DockerExecutor { + return new DockerExecutor( + { + baseImage: () => 'dormice-base:20260831', + registry: { + address: () => registry, + username: 'dormice', + password: 'fleet-token-fleet-token-fleet-token-fleet', + }, + dataDir: '/nonexistent', + resources: () => ({ diskSizeGb: 1, cpus: 1, memoryGb: 1 }), + pidsLimit: () => 4096, + reclaimTimeoutSeconds: 1, + }, + stub.docker, + ); +} + +describe('image references', () => { + it('namesRegistry: a first component with a dot, a colon or "localhost" is a host; a bare name is not, whatever its tag', () => { + expect(namesRegistry('dormice-base:20260831')).toBe(false); + expect(namesRegistry('clawsgo_20260808_base:20260907')).toBe(false); + expect(namesRegistry('library/python:3.12')).toBe(false); + expect(namesRegistry('docker.io/library/python:3.12')).toBe(true); + expect(namesRegistry('ghcr.io/x/y')).toBe(true); + expect(namesRegistry('10.0.0.5:5000/dormice-base:20260831')).toBe(true); + expect(namesRegistry('localhost/x')).toBe(true); + expect(namesRegistry('registry:5000/x')).toBe(true); + }); + + it('splitRepoTag: repository and tag, latest when none; a digest is refused', () => { + expect(splitRepoTag('dormice-base:20260831')).toEqual({ + repo: 'dormice-base', + tag: '20260831', + }); + expect(splitRepoTag('dormice-base')).toEqual({ + repo: 'dormice-base', + tag: 'latest', + }); + expect(splitRepoTag('team/app:v1')).toEqual({ + repo: 'team/app', + tag: 'v1', + }); + expect(() => splitRepoTag('x@sha256:abcd')).toThrow(/pinned by digest/); + }); +}); + +describe('DockerExecutor.ensureImage', () => { + it('an image the host has is present, nothing pulled', async () => { + const stub = stubDocker(); + stub.present.add('dormice-base:20260831'); + expect( + await executor(stub, '10.0.0.5:5000').ensureImage( + 'dormice-base:20260831', + ), + ).toBe('present'); + expect(stub.calls.pulled).toEqual([]); + }); + + it('a bare image the host lacks is pulled from the fleet registry under the fleet credential and tagged back under its bare name', async () => { + const stub = stubDocker(); + expect(await executor(stub, '10.0.0.5:5000').ensureImage('tpl:1')).toBe( + 'pulled', + ); + expect(stub.calls.pulled).toEqual([ + { + source: '10.0.0.5:5000/tpl:1', + auth: { + username: 'dormice', + password: 'fleet-token-fleet-token-fleet-token-fleet', + serveraddress: '10.0.0.5:5000', + }, + }, + ]); + expect(stub.calls.tagged).toEqual([ + { source: '10.0.0.5:5000/tpl:1', repo: 'tpl', tag: '1' }, + ]); + expect(stub.present.has('tpl:1')).toBe(true); + }); + + it('an image naming its own registry is pulled as written, with no credential and no re-tag', async () => { + const stub = stubDocker(); + expect( + await executor(stub, '10.0.0.5:5000').ensureImage( + 'docker.io/library/python:3.12', + ), + ).toBe('pulled'); + expect(stub.calls.pulled).toEqual([ + { source: 'docker.io/library/python:3.12', auth: undefined }, + ]); + expect(stub.calls.tagged).toEqual([]); + }); + + it('no registry and a bare image the host lacks is a refusal naming both ways out', async () => { + const stub = stubDocker(); + await expect(executor(stub, null).ensureImage('tpl:1')).rejects.toThrow( + /image tpl:1 is not on this host and the fleet has no registry — build or docker pull it under that name on this host, or run the fleet registry/, + ); + expect(stub.calls.pulled).toEqual([]); + }); + + it("a pull the registry refuses names the image, the registry's word and the push command", async () => { + const stub = stubDocker({ failPull: '10.0.0.5:5000/tpl:1' }); + await expect( + executor(stub, '10.0.0.5:5000').ensureImage('tpl:1'), + ).rejects.toThrow( + /image tpl:1 is not on this host, and pulling 10\.0\.0\.5:5000\/tpl:1 from the fleet registry failed: manifest unknown.*docker tag tpl:1 10\.0\.0\.5:5000\/tpl:1 && docker push 10\.0\.0\.5:5000\/tpl:1/, + ); + expect(stub.calls.tagged).toEqual([]); + }); + + it('two callers asking for the same missing image share one pull', async () => { + const stub = stubDocker(); + const ex = executor(stub, '10.0.0.5:5000'); + const [a, b] = await Promise.all([ + ex.ensureImage('tpl:1'), + ex.ensureImage('tpl:1'), + ]); + expect([a, b]).toEqual(['pulled', 'pulled']); + expect(stub.calls.pulled).toHaveLength(1); + }); +}); diff --git a/packages/server/src/executor/docker.contract.test.ts b/packages/server/src/executor/docker.contract.test.ts index 862a9cab..cfa59cbd 100644 --- a/packages/server/src/executor/docker.contract.test.ts +++ b/packages/server/src/executor/docker.contract.test.ts @@ -33,7 +33,10 @@ if (process.env.DORMICE_DOCKER_CONTRACT === '1' && image) { async () => { const dataDir = await mkdtemp(path.join(tmpdir(), 'dormice-contract-')); const executor = new DockerExecutor({ - baseImage: image, + baseImage: () => image, + // No fleet registry on the exam host: every image the contract + // names is present (the base, and its alias tag below). + registry: { address: () => null, username: 'dormice', password: 'x' }, dataDir, // Small and fast: the contract exercises lifecycle, not capacity. // A static closure, not a ledger read: the contract exam runs the @@ -82,7 +85,8 @@ if (process.env.DORMICE_DOCKER_CONTRACT === '1' && image) { const dataDir = await mkdtemp(path.join(tmpdir(), 'dormice-contract-')); const withCap = (pidsLimit: number) => new DockerExecutor({ - baseImage: image, + baseImage: () => image, + registry: { address: () => null, username: 'dormice', password: 'x' }, dataDir, resources: () => ({ diskSizeGb: 1, cpus: 1, memoryGb: 1 }), pidsLimit: () => pidsLimit, @@ -175,7 +179,8 @@ if (process.env.DORMICE_DOCKER_CONTRACT === '1' && image) { const dyingShell = async (memoryGb: number) => { const dataDir = await mkdtemp(path.join(tmpdir(), 'dormice-contract-')); const executor = new DockerExecutor({ - baseImage: image, + baseImage: () => image, + registry: { address: () => null, username: 'dormice', password: 'x' }, dataDir, resources: () => ({ diskSizeGb: 1, cpus: 1, memoryGb }), pidsLimit: () => 256, diff --git a/packages/server/src/executor/docker.ts b/packages/server/src/executor/docker.ts index 99880ae5..b4c380b5 100644 --- a/packages/server/src/executor/docker.ts +++ b/packages/server/src/executor/docker.ts @@ -97,9 +97,33 @@ import { WatchProcessLifecycle } from './watch-lifecycle'; */ export const SANDBOX_LABEL = 'dormice.sandbox'; +/** + * Where a node pulls an image its host lacks (ensureImage): the fleet's + * registry, live from the copy — null while the fleet runs none — and + * the credential it takes, the fleet token as the password (one secret + * for the whole fleet, design record #34; install.sh writes the same + * pair into the registry's htpasswd). Presented per pull, never written + * to any file on the node. + */ +export interface ImageRegistry { + address: () => string | null; + username: string; + password: string; +} + +/** How long one pull may take: an hour — a template of tens of GiB over an intranet is minutes, a registry that hangs must still be given up on. */ +export const PULL_DEADLINE_SECONDS = 3600; + export interface DockerExecutorOptions { - /** Image every sandbox boots from, e.g. dormice-base:20260708. */ - baseImage: string; + /** + * Live view of the fleet's base image (db/templates.ts resolveBaseImage, + * wired in main.ts), read at each birth that names no image and by the + * wake's staleness verdict — a fleet setting since the fourth cut, so a + * console re-point reaches the next birth without a restart. Same shape + * and reason as resources below. + */ + baseImage: () => string; + registry: ImageRegistry; /** Sparse disk images and their mount points live under this directory. */ dataDir: string; /** @@ -125,6 +149,38 @@ export function containerName(sandboxId: string): string { return `sbx-${sandboxId}`; } +/** + * Whether an image reference names its registry — Docker's own rule: the + * first path component is a host when it contains a dot or a colon, or is + * `localhost`. `docker.io/library/python:3.12` and `ghcr.io/x/y` do; the + * fleet's own `dormice-base:20260831` and `clawsgo_20260808_base:20260907` + * do not (a bare reference; its colon is the tag's). + */ +export function namesRegistry(image: string): boolean { + const slash = image.indexOf('/'); + if (slash === -1) return false; + const first = image.slice(0, slash); + return first.includes('.') || first.includes(':') || first === 'localhost'; +} + +/** + * A bare reference split for `docker tag`: repository and tag, `latest` + * when none is written. A digest reference cannot be re-tagged under a + * name and is refused — templates are named by tag here. + */ +export function splitRepoTag(image: string): { repo: string; tag: string } { + if (image.includes('@')) { + throw new Error( + `image ${image} is pinned by digest — the fleet's images are named by tag (repository:tag), which the registry pull tags back under`, + ); + } + const lastSlash = image.lastIndexOf('/'); + const colon = image.lastIndexOf(':'); + return colon > lastSlash + ? { repo: image.slice(0, colon), tag: image.slice(colon + 1) } + : { repo: image, tag: 'latest' }; +} + /** One `docker inspect`, reduced to what the executor's verbs decide on. */ interface Inspected { id: string; @@ -263,8 +319,107 @@ export class DockerExecutor implements Executor { } } - get baseImage(): string { - return this.opts.baseImage; + baseImage(): string { + return this.opts.baseImage(); + } + + /** Pulls in flight, by image: a birth and the prefetch asking for the same image share one pull. */ + private readonly pulls = new Map>(); + + async ensureImage(image: string): Promise<'present' | 'pulled'> { + if (await this.imagePresent(image)) return 'present'; + let pull = this.pulls.get(image); + if (pull === undefined) { + pull = this.pullImage(image).finally(() => this.pulls.delete(image)); + this.pulls.set(image, pull); + } + await pull; + return 'pulled'; + } + + private async imagePresent(image: string): Promise { + try { + await deadline( + this.docker.getImage(image).inspect(), + QUERY_DEADLINE_SECONDS, + `inspect of image ${image}`, + ); + return true; + } catch (err) { + if (isDockerApiError(err) && err.statusCode === 404) return false; + throw err; + } + } + + /** + * The pull behind ensureImage. A bare reference comes from the fleet's + * registry as `
/` under the fleet credential and is + * tagged back under the bare name: the shell is born from the bare name + * (Config.Image), the same name a locally built image carries, so the + * wake's staleness verdict and listSandboxImages keep comparing names + * and the fleet's move onto a registry re-marks no existing shell as + * upgradable. The registry-prefixed tag stays beside it — it says where + * the image came from, and costs nothing. A reference naming its own + * registry is pulled as written, no credential: a public image is a + * template like any other. No registry and a bare image the host lacks + * is a refusal that says both ways out. + */ + private async pullImage(image: string): Promise { + const registry = this.opts.registry.address(); + const fromFleet = !namesRegistry(image); + if (fromFleet && registry === null) { + throw new Error( + `image ${image} is not on this host and the fleet has no registry — build or docker pull it under that name on this host, or run the fleet registry (install.sh runs one beside the gateway) and push it there`, + ); + } + const source = fromFleet ? `${registry}/${image}` : image; + this.log(`pulling image ${source}`); + const started = Date.now(); + try { + const stream = await deadline( + this.docker.pull( + source, + fromFleet + ? { + authconfig: { + username: this.opts.registry.username, + password: this.opts.registry.password, + serveraddress: registry, + }, + } + : {}, + ), + QUERY_DEADLINE_SECONDS, + `pull of ${source}`, + ); + await deadline( + new Promise((resolve, reject) => { + this.docker.modem.followProgress(stream, (err: Error | null) => + err ? reject(err) : resolve(), + ); + }), + PULL_DEADLINE_SECONDS, + `pull of ${source}`, + ); + } catch (err) { + const why = err instanceof Error ? err.message : String(err); + throw new Error( + fromFleet + ? `image ${image} is not on this host, and pulling ${source} from the fleet registry failed: ${why} — push it from a machine that has it: docker tag ${image} ${source} && docker push ${source}` + : `image ${image} is not on this host, and pulling it failed: ${why}`, + ); + } + if (fromFleet) { + const { repo, tag } = splitRepoTag(image); + await deadline( + this.docker.getImage(source).tag({ repo, tag }), + QUERY_DEADLINE_SECONDS, + `tag of ${source}`, + ); + } + this.log( + `pulled image ${source} in ${Math.round((Date.now() - started) / 1000)}s${fromFleet ? `, tagged ${image}` : ''}`, + ); } async create(sandboxId: string, opts?: CreateOptions): Promise { @@ -1747,13 +1902,19 @@ export class DockerExecutor implements Executor { sandboxId: string, opts?: ShellOptions, ): Promise { - const image = opts?.image; + const image = opts?.image ?? this.opts.baseImage(); + // The image first, so a host that lacks it pulls it here rather than + // failing the birth: one image inspect per birth (a millisecond on the + // local socket) buys a node that needs no image staged by hand. Slow + // when it pulls — a first sandbox of a large template waits — and the + // prefetch (node-config.ts) is what makes that rare. + await this.ensureImage(image); let container: Docker.Container; try { container = await deadline( this.docker.createContainer({ name: containerName(sandboxId), - Image: image ?? this.opts.baseImage, + Image: image, Cmd: ['sleep', 'infinity'], Labels: { [SANDBOX_LABEL]: sandboxId }, HostConfig: { @@ -1798,13 +1959,13 @@ export class DockerExecutor implements Executor { if (isDockerApiError(err) && err.statusCode === 409) { throw new Error(`container ${sandboxId} already exists`); } - // Registration never checks image existence (the image may arrive - // later), so this is where a missing one honestly surfaces. Named - // here — dockerode's own 404 would otherwise leak out as this API's - // "sandbox not found" status. + // ensureImage just saw the image; a 404 here is an image removed in + // the milliseconds between (an operator's prune). Named — dockerode's + // own 404 would otherwise leak out as this API's "sandbox not found" + // status — and the retry pulls it again. if (isDockerApiError(err) && err.statusCode === 404) { throw new Error( - `image ${image ?? this.opts.baseImage} is not on this host — docker pull or build it, then retry`, + `image ${image} vanished from this host between the check and the container's creation — retry`, ); } throw err; diff --git a/packages/server/src/executor/executor.ts b/packages/server/src/executor/executor.ts index bff1005a..48b375ab 100644 --- a/packages/server/src/executor/executor.ts +++ b/packages/server/src/executor/executor.ts @@ -260,7 +260,13 @@ export interface SandboxResources { * either goes through removeContainer + start, the rebuild path). */ export interface ShellOptions { - /** Image reference on this host; absent means the executor's configured base image. */ + /** + * Image reference, a bare one (`dormice-base:20260831`) or one naming + * its registry; absent means the fleet's base image, the executor's + * live view of it (baseImage()). An image the host lacks is pulled + * first (ensureImage) — the docker executor from the fleet registry, + * the fake from thin air. + */ image?: string; /** CPU allowance of this shell; absent means the executor's live default (resources()). */ cpus?: number; @@ -360,12 +366,29 @@ export interface ImportDiskOptions { */ export interface Executor { /** - * The image shells boot from when create/start name none. The executor is - * the one authority on its own default — config knows it only in docker - * mode, and callers comparing born images against "what would boot next" - * must not guess. - */ - readonly baseImage: string; + * The image shells boot from when create/start name none: the fleet's + * base image, resolved live at each call (main.ts wires the ledger copy + * with the env fallback, db/templates.ts resolveBaseImage) — the same + * live-view shape as the resource knobs, so a console edit reaches the + * next birth and the next wake's verdict without a restart. Callers + * comparing a born image against "what would boot next" ask this; it + * throws where the fleet names no base image and the node has no + * fallback, the honest answer at the moment a sandbox would need one. + */ + baseImage(): string; + /** + * Makes an image available on this host, pulling it when absent: from + * the fleet's registry for a bare reference (then tagged under the bare + * name, so a shell born from it records the same name a shell built from + * a local build would — listSandboxImages compares names), as written + * for a reference naming its own registry. Answers which it was. Called + * ahead of a birth (node-config.ts prefetches a bundle's images) and at + * a birth whose image is still missing — a template of tens of GiB must + * not make the first sandbox that needs it wait if it can be helped. + * Throws, naming the image, the registry and the push command, when + * neither the host nor the registry has it. + */ + ensureImage(image: string): Promise<'present' | 'pulled'>; /** * Brings a brand-new sandbox up to running. `image` picks what the shell * boots from (a template's current image); absent means the executor's diff --git a/packages/server/src/executor/fake.test.ts b/packages/server/src/executor/fake.test.ts index 82f04d96..f4f898bf 100644 --- a/packages/server/src/executor/fake.test.ts +++ b/packages/server/src/executor/fake.test.ts @@ -28,4 +28,20 @@ describe('FakeExecutor test hooks', () => { expect(executor.stateOf('a')).toBe('running'); expect(executor.stateOf('ghost')).toBeUndefined(); }); + + it('boots the live base image when a birth names none, and ensureImage pulls an image once and answers present after', async () => { + let base = 'base:1'; + const executor = new FakeExecutor(undefined, undefined, () => base); + await executor.create('a'); + expect(await executor.imageOf('a')).toBe('base:1'); + base = 'base:2'; + expect(executor.baseImage()).toBe('base:2'); + await executor.create('b'); + expect(await executor.imageOf('b')).toBe('base:2'); + // The base is always on the host; anything else is pulled once. + expect(await executor.ensureImage('base:2')).toBe('present'); + expect(await executor.ensureImage('tpl:1')).toBe('pulled'); + expect(await executor.ensureImage('tpl:1')).toBe('present'); + expect(executor.pulled).toEqual(['tpl:1']); + }); }); diff --git a/packages/server/src/executor/fake.ts b/packages/server/src/executor/fake.ts index 045fbff0..09da1325 100644 --- a/packages/server/src/executor/fake.ts +++ b/packages/server/src/executor/fake.ts @@ -211,8 +211,6 @@ class FakeProcessIO { * the fake too. */ export class FakeExecutor implements Executor { - readonly baseImage = FAKE_BASE_IMAGE; - /** * Live view of the resource knobs, the docker executor's own contract: * read at each birth, so a ledger edit reaches the next shell and disk. @@ -229,8 +227,37 @@ export class FakeExecutor implements Executor { * the daemons and tests that never move it. */ private readonly pidsLimit: () => number = () => FAKE_PIDS_LIMIT, + /** + * Live view of the fleet's base image, the docker executor's third + * closure (main.ts wires the copy with the env fallback): what a birth + * that names no image boots. The default serves the suites, whose + * copies mostly name none. + */ + private readonly base: () => string = () => FAKE_BASE_IMAGE, ) {} + baseImage(): string { + return this.base(); + } + + /** + * Images "on this host", the docker executor's image store: the base + * image is always there (install.sh builds it), and whatever ensureImage + * pulled. A birth does not consult it — the fake plays whatever image it + * is asked to, and the suites name images freely — so the store exists + * for ensureImage's word alone. + */ + private readonly present = new Set(); + /** Every image ensureImage pulled, in order — the prefetch's footprint, for the suites. */ + readonly pulled: string[] = []; + + async ensureImage(image: string): Promise<'present' | 'pulled'> { + if (image === this.base() || this.present.has(image)) return 'present'; + this.present.add(image); + this.pulled.push(image); + return 'pulled'; + } + /** ShellOptions -> the limits a shell is born with, the docker rounding rules. */ private bornLimits(opts?: ShellOptions): ShellLimits { return { @@ -410,7 +437,7 @@ export class FakeExecutor implements Executor { this.diskNominal.set(sandboxId, this.bornDiskBytes(opts?.diskGb)); this.fs.set(sandboxId, seededDisk()); this.containers.set(sandboxId, 'running'); - this.images.set(sandboxId, opts?.image ?? FAKE_BASE_IMAGE); + this.images.set(sandboxId, opts?.image ?? this.base()); this.limits.set(sandboxId, this.bornLimits(opts)); this.pids.set(sandboxId, this.pidsLimit()); } @@ -451,7 +478,7 @@ export class FakeExecutor implements Executor { throw new Error(`disk ${sandboxId} is absent, cannot start`); } this.containers.set(sandboxId, 'running'); - this.images.set(sandboxId, opts?.image ?? FAKE_BASE_IMAGE); + this.images.set(sandboxId, opts?.image ?? this.base()); this.limits.set(sandboxId, this.bornLimits(opts)); this.pids.set(sandboxId, this.pidsLimit()); this.exits.delete(sandboxId); diff --git a/packages/server/src/lifecycle.ts b/packages/server/src/lifecycle.ts index 3cb675b5..7250a963 100644 --- a/packages/server/src/lifecycle.ts +++ b/packages/server/src/lifecycle.ts @@ -230,7 +230,7 @@ export async function wakeSandbox( } case 'frozen': case 'stopped': { - const next = resolveImage(db, row.template) ?? executor.baseImage; + const next = resolveImage(db, row.template) ?? executor.baseImage(); const born = await executor.imageOf(row.id); // The spec in force, in the runtime's integer units — what a shell // built right now would be born with. diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index cfa6bfe2..cf041245 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -17,6 +17,7 @@ import { readRuntimeSettings, readSwapTarget, } from './db/settings'; +import { resolveBaseImage } from './db/templates'; import { WatcherTable } from './e2b/watcher-table'; import { DockerExecutor } from './executor/docker'; import type { Executor } from './executor/executor'; @@ -97,16 +98,24 @@ function buildExecutor(cfg: Config, log: (msg: string) => void): Executor { // Live too: a console edit reaches the next birth and the next wake's // in-place convergence without a restart. const pidsLimit = () => readRuntimeSettings(db).pidsLimit; + // Live, the same way: the fleet's base image from the copy, this node's + // DORMICE_BASE_IMAGE while the fleet names none (db/templates.ts + // resolveBaseImage has the rule; the boot log below says which is in + // force). + const baseImage = () => resolveBaseImage(db, cfg.DORMICE_BASE_IMAGE); if (cfg.DORMICE_EXECUTOR === 'fake') { - return new FakeExecutor(resources, pidsLimit); - } - if (!cfg.DORMICE_BASE_IMAGE) { - // loadConfig already rejected this combination; the check only narrows - // the type here. - throw new Error('DORMICE_BASE_IMAGE is required for the docker executor'); + return new FakeExecutor(resources, pidsLimit, baseImage); } return new DockerExecutor({ - baseImage: cfg.DORMICE_BASE_IMAGE, + baseImage, + // Where an image this host lacks comes from: the fleet registry named + // in the copy, under the fleet token — the one credential of the fleet + // (design record #34), doubling as the registry's password. + registry: { + address: () => readRuntimeSettings(db).registryAddress, + username: 'dormice', + password: cfg.DORMICE_API_TOKEN, + }, dataDir: cfg.DORMICE_DATA_DIR, resources, pidsLimit, @@ -211,7 +220,15 @@ const checkIn = new CheckIn({ readNodeReading(db, checkInCpu, config.DORMICE_DATA_DIR, executor, swap), configVersion: () => readConfigVersion(db), applyConfig: (bundle) => - applyConfig(bundle, { db, executor, locks, swap, log, beat }), + applyConfig(bundle, { + db, + executor, + locks, + swap, + log, + beat, + baseImageFallback: config.DORMICE_BASE_IMAGE, + }), log, }); @@ -297,6 +314,25 @@ if (readConfigVersion(db) === null) { ? `running configuration v${copy.version}; archiver disabled: no S3 store in the fleet settings (configure one in the console)` : `running configuration v${copy.version}; archiver enabled: bucket ${copy.settings.s3.bucket} at ${copy.settings.s3.endpoint}`, ); + // Which base image is in force, and where it came from: the fleet's + // (the knob's home since the fourth cut), or this node's own env while + // the fleet names none — said at boot, once, so a node running on the + // fallback is never a surprise (db/templates.ts resolveBaseImage). + if (copy.settings.baseImage !== null) { + if (config.DORMICE_BASE_IMAGE !== undefined) { + log.info( + `base image ${copy.settings.baseImage} (the fleet's setting); DORMICE_BASE_IMAGE in this node's env is a fallback only and can be removed`, + ); + } + } else if (config.DORMICE_BASE_IMAGE !== undefined) { + log.warn( + `the fleet settings name no base image — template-less sandboxes on this node boot DORMICE_BASE_IMAGE=${config.DORMICE_BASE_IMAGE} from its env; set baseImage at the gateway (console › settings) so every node shares one`, + ); + } else { + log.warn( + 'the fleet settings name no base image and DORMICE_BASE_IMAGE is not set — a template-less sandbox cannot be built here until baseImage is set at the gateway (console › settings)', + ); + } } if (archiver.enabled()) { await archiver.init(); diff --git a/packages/server/src/node-config.test.ts b/packages/server/src/node-config.test.ts index bed72fb3..5bdfb797 100644 --- a/packages/server/src/node-config.test.ts +++ b/packages/server/src/node-config.test.ts @@ -14,10 +14,10 @@ import { readS3Settings, readSwapTarget, } from './db/settings'; -import { findTemplate } from './db/templates'; -import { FakeExecutor } from './executor/fake'; +import { findTemplate, resolveBaseImage } from './db/templates'; +import { FAKE_BASE_IMAGE, FakeExecutor } from './executor/fake'; import { KeyedQueue } from './keyed-queue'; -import { applyConfig } from './node-config'; +import { applyConfig, prefetchImages } from './node-config'; import type { SwapControl, SwapStatus } from './swap'; import { TEST_S3, testBundle } from './testing'; @@ -286,3 +286,122 @@ describe('applyConfig: the bundle made real on the host', () => { expect(readSwapTarget(db)).toBe(16); }); }); + +describe('the base image: a fleet setting, the env a fallback', () => { + it("resolveBaseImage answers the copy's, the env's while the copy names none, and refuses with where to set it when neither does", () => { + const db = ledger(); + applyNodeConfig(db, testBundle({ baseImage: 'dormice-base:20260831' }, 1)); + expect(resolveBaseImage(db, 'dormice-base:20260718')).toBe( + 'dormice-base:20260831', + ); + expect(resolveBaseImage(db, undefined)).toBe('dormice-base:20260831'); + applyNodeConfig(db, testBundle({ baseImage: null }, 2)); + expect(resolveBaseImage(db, 'dormice-base:20260718')).toBe( + 'dormice-base:20260718', + ); + expect(() => resolveBaseImage(db, undefined)).toThrow( + /no base image: the fleet settings name none and DORMICE_BASE_IMAGE is not set on this node — set baseImage at the gateway/, + ); + // The copy carries the registry beside it, for the pull. + applyNodeConfig( + db, + testBundle({ baseImage: 'b:1', registryAddress: '10.0.0.5:5000' }, 3), + ); + expect(readRuntimeSettings(db)).toMatchObject({ + baseImage: 'b:1', + registryAddress: '10.0.0.5:5000', + }); + }); + + it('a bundle naming no base image is a warning when the node has a fallback, nothing when it has none or the fleet names one', async () => { + const db = ledger(); + const executor = new FakeExecutor(); + const apply = async ( + bundle: ReturnType, + fallback?: string, + ) => { + const { log, lines } = logSpy(); + await applyConfig(bundle, { + db, + executor, + locks: new KeyedQueue(), + log, + baseImageFallback: fallback, + }); + return lines.filter((l) => l.level === 'warn').map((l) => l.msg); + }; + expect(await apply(testBundle({}, 1), 'dormice-base:20260718')).toEqual([ + expect.stringMatching(/the fleet settings name no base image/), + ]); + expect(await apply(testBundle({}, 2))).toEqual([]); + expect( + await apply(testBundle({ baseImage: 'b:1' }, 3), 'dormice-base:20260718'), + ).toEqual([]); + }); + + it("prefetchImages pulls what the bundle names and the host lacks — the base and every template's image, once each — and a pull that fails is one warning, the rest still fetched", async () => { + const executor = new FakeExecutor(); + const { log, lines } = logSpy(); + const bundle = testBundle( + { + baseImage: FAKE_BASE_IMAGE, + templates: [ + { name: 'py', image: 'img-py:1' }, + { name: 'node', image: 'img-node:1' }, + { name: 'py-too', image: 'img-py:1' }, + ], + }, + 1, + ); + await prefetchImages(bundle, executor, log); + // The base is on the host (install.sh builds it); the two template + // images were not, and the one named twice was pulled once. + expect(executor.pulled).toEqual(['img-py:1', 'img-node:1']); + expect( + lines.map((l) => [l.level, (l.obj as { image: string }).image]), + ).toEqual([ + ['info', 'img-py:1'], + ['info', 'img-node:1'], + ]); + // Nothing new: nothing pulled, nothing said. + lines.length = 0; + await prefetchImages(bundle, executor, log); + expect(executor.pulled).toEqual(['img-py:1', 'img-node:1']); + expect(lines).toEqual([]); + + const failing = new FakeExecutor(); + vi.spyOn(failing, 'ensureImage').mockImplementation(async (image) => { + if (image === 'img-gone:1') throw new Error('manifest unknown'); + return 'pulled'; + }); + const { log: log2, lines: lines2 } = logSpy(); + await prefetchImages( + testBundle( + { + templates: [ + { name: 'gone', image: 'img-gone:1' }, + { name: 'ok', image: 'img-ok:1' }, + ], + }, + 1, + ), + failing, + log2, + ); + expect(lines2.map((l) => l.level)).toEqual(['warn', 'info']); + expect(lines2[0]?.msg).toMatch(/could not be fetched ahead/); + expect((lines2[0]?.obj as { image: string }).image).toBe('img-gone:1'); + }); + + it("applyConfig starts the prefetch and does not wait for it: the bundle's images arrive after the copy is applied", async () => { + const db = ledger(); + const executor = new FakeExecutor(); + const { log } = logSpy(); + await applyConfig( + testBundle({ templates: [{ name: 'py', image: 'img-py:1' }] }, 1), + { db, executor, locks: new KeyedQueue(), log }, + ); + expect(readConfigVersion(db)).toBe(1); + await vi.waitFor(() => expect(executor.pulled).toEqual(['img-py:1'])); + }); +}); diff --git a/packages/server/src/node-config.ts b/packages/server/src/node-config.ts index c5cbe9d2..3b554491 100644 --- a/packages/server/src/node-config.ts +++ b/packages/server/src/node-config.ts @@ -17,6 +17,8 @@ export interface ConfigApplierDeps { locks: KeyedQueue; /** Managed swap, where the host has it (main.ts decides); absent, the target is stored and nothing else. */ swap?: SwapControl; + /** This node's own DORMICE_BASE_IMAGE, the base image's old home — what a bundle naming none leaves in force, said as a warning. */ + baseImageFallback?: string; log: { info(obj: unknown, msg: string): void; warn(obj: unknown, msg: string): void; @@ -41,12 +43,17 @@ export interface ConfigApplierDeps { * either way; a shell the runtime refuses converges at its next wake, a * swap block that failed to mount is retried at the next boot — capacity, * not correctness. + * + * And the bundle's images are fetched ahead (prefetchImages), in the + * background: the check-in that carried the bundle is not held for a + * pull — a template of tens of GiB would take minutes, and a node that + * stops checking in for minutes is read as down. */ export async function applyConfig( bundle: NodeConfigBundle, deps: ConfigApplierDeps, ): Promise { - const { db, executor, locks, swap, log, beat } = deps; + const { db, executor, locks, swap, log, beat, baseImageFallback } = deps; const before = readConfigVersion(db) === null ? null @@ -69,6 +76,13 @@ export async function applyConfig( ? 'first configuration copy applied from the gateway' : 'configuration applied from the gateway', ); + if (bundle.settings.baseImage === null && baseImageFallback !== undefined) { + log.warn( + { fallback: baseImageFallback }, + 'the fleet settings name no base image — template-less sandboxes on this node boot DORMICE_BASE_IMAGE from its env; set baseImage at the gateway (console › settings) so every node shares one', + ); + } + void prefetchImages(bundle, executor, log); if (before === null) return; if (bundle.settings.pidsLimit !== before.pidsLimit) { const sweep = await sweepPidsLimit(db, executor, locks, beat); @@ -89,3 +103,43 @@ export async function applyConfig( } } } + +/** + * Every image a bundle names — the fleet's base image and each template's + * — made present on this host ahead of the first sandbox that needs it + * (executor ensureImage: pulled from the fleet registry when absent, a + * word when present). One at a time, in the background of the check-in + * that brought the bundle: a first sandbox that finds its image already + * here starts in seconds where the pull alone would have taken minutes, + * and a node joining the fleet stages its images with no operator's + * hand. A pull that fails is one warning, naming the image and why, and + * the bundle stands: the first sandbox that needs the image tries the + * pull again and fails with the same words if it still cannot be had. + * Two bundles in quick succession run two of these; the executor shares + * one pull per image between them. + */ +export async function prefetchImages( + bundle: NodeConfigBundle, + executor: Executor, + log: ConfigApplierDeps['log'], +): Promise { + const wanted = new Set(); + if (bundle.settings.baseImage !== null) wanted.add(bundle.settings.baseImage); + for (const template of bundle.templates) wanted.add(template.image); + for (const image of wanted) { + try { + const outcome = await executor.ensureImage(image); + if (outcome === 'pulled') { + log.info( + { image }, + 'image pulled ahead of the first sandbox to need it', + ); + } + } catch (error) { + log.warn( + { image, err: error }, + 'an image the configuration names could not be fetched ahead; the first sandbox to need it pulls again, and fails with the reason if it still cannot be had', + ); + } + } +} diff --git a/packages/server/src/routes/sandboxes.ts b/packages/server/src/routes/sandboxes.ts index 990789a4..1fda4e7d 100644 --- a/packages/server/src/routes/sandboxes.ts +++ b/packages/server/src/routes/sandboxes.ts @@ -500,10 +500,10 @@ export const sandboxRoutes: FastifyPluginAsyncZod< const images = await Promise.all( listSandboxes(db).map(async (row) => { // resolveImage is the one arbiter of template -> image; undefined - // means "the executor's own base image", and the executor is the - // one authority on what that is (config only knows in docker mode). + // means the fleet's base image, which the executor resolves live + // (its baseImage view over the copy, main.ts). const nextImage = - resolveImage(db, row.template) ?? executor.baseImage; + resolveImage(db, row.template) ?? executor.baseImage(); let image: string | null = null; if (row.state !== 'archived' && row.state !== 'restoring') { try { diff --git a/packages/server/src/testing.ts b/packages/server/src/testing.ts index 073bd77c..87e86836 100644 --- a/packages/server/src/testing.ts +++ b/packages/server/src/testing.ts @@ -30,6 +30,9 @@ export interface TestConfig { sandboxDomain?: string | null; sandboxDomainAliases?: string[]; pidsLimit?: number; + /** The fleet's base image; null (the default) = the fleet names none and the executor's own default stands in. */ + baseImage?: string | null; + registryAddress?: string | null; swapGb?: number; /** Replaces the whole template list; timestamps are stamped now. */ templates?: Array<{ name: string; image: string }>; @@ -68,6 +71,8 @@ export function testBundle( sandboxDomain: over.sandboxDomain ?? null, sandboxDomainAliases: over.sandboxDomainAliases ?? [], pidsLimit: over.pidsLimit ?? 4096, + baseImage: over.baseImage ?? null, + registryAddress: over.registryAddress ?? null, }, node: { swapGb: over.swapGb ?? 0 }, templates: (over.templates ?? []).map((t) => ({ @@ -112,6 +117,14 @@ export function configureNode(db: Db, over: TestConfig = {}): NodeConfigBundle { over.sandboxDomainAliases ?? current.settings.sandboxDomainAliases, pidsLimit: over.pidsLimit ?? current.settings.pidsLimit, + baseImage: + over.baseImage !== undefined + ? over.baseImage + : current.settings.baseImage, + registryAddress: + over.registryAddress !== undefined + ? over.registryAddress + : current.settings.registryAddress, }, node: { swapGb: over.swapGb ?? current.node.swapGb }, templates: diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts index 65f9229a..f9f75f05 100644 --- a/packages/shared/src/gateway.ts +++ b/packages/shared/src/gateway.ts @@ -164,6 +164,15 @@ export const nodeConfigBundleSchema = z.object({ sandboxDomain: z.string().regex(bareHostnameRegex).nullable(), sandboxDomainAliases: z.array(z.string().regex(bareHostnameRegex)), pidsLimit: z.number().int().min(PIDS_LIMIT_MIN), + /** + * The fleet's base image and its registry (settings.ts has both). + * Default null, not required: a rolling upgrade takes the gateway + * first, and a node on this build must take a bundle from a gateway + * on the previous one — it then falls back to its own env for the + * base image, said as a warning (server/node-config.ts). + */ + baseImage: z.string().nullable().default(null), + registryAddress: z.string().nullable().default(null), }), node: z.object({ /** This node's managed-swap target, GiB (updateNodeSettings). */ diff --git a/packages/shared/src/settings.ts b/packages/shared/src/settings.ts index 17494725..eec58356 100644 --- a/packages/shared/src/settings.ts +++ b/packages/shared/src/settings.ts @@ -142,6 +142,29 @@ export const runtimeSettingsSchema = z.object({ * unlimited: the cap is what keeps a fork bomb inside its own sandbox. */ pidsLimit: z.number().int().min(PIDS_LIMIT_MIN), + /** + * The image a sandbox without a template boots from — the fleet's base + * image (images/Dockerfile), a bare reference like + * `dormice-base:20260831`. A fleet setting since the fourth cut + * (2026-09-15): one base for every node, pulled from the fleet's + * registry by a node that lacks it (gateway.ts nodeConfigBundleSchema + * carries it in the bundle). Changing it is the base's re-point, the + * template's `registerTemplate` for template-less sandboxes: each one + * converges onto it at its next cold wake. Null = none set — a node then + * falls back to its own DORMICE_BASE_IMAGE, the knob's old home, and + * says so, or refuses to build a template-less sandbox. + */ + baseImage: z.string().nullable(), + /** + * The fleet's image registry, host and port (`10.0.0.5:5000`), where a + * node that lacks an image pulls it from — `/`, + * tagged back under the bare name so nothing else changes. Null = no + * registry (a laptop, the exam): a missing image is then an honest error + * naming the host. Seeded from the gateway's DORMICE_REGISTRY_ADDRESS + * and read-only over the wire in this cut — moving a fleet to another + * registry is an operator's action, not a console knob yet. + */ + registryAddress: z.string().nullable(), /** ISO 8601 of the last updateSettings; null = still exactly the first-boot seed. */ updatedAt: z.string().nullable(), }); @@ -186,6 +209,14 @@ export const updateSettingsRequestSchema = z error: `pidsLimit must be at least ${PIDS_LIMIT_MIN} — below that a sandbox cannot boot its own runtime`, }) .optional(), + /** The fleet's base image, a bare image reference; never null — a fleet cannot un-know its base, only re-point it. */ + baseImage: z + .string() + .regex(/^\S+$/, { + error: + 'baseImage must be an image reference like dormice-base:20260831 — no spaces', + }) + .optional(), }) .refine( (patch) => @@ -194,10 +225,11 @@ export const updateSettingsRequestSchema = z patch.defaultPolicy !== undefined || patch.s3 !== undefined || patch.sandboxDomain !== undefined || - patch.sandboxDomainAliases !== undefined, + patch.sandboxDomainAliases !== undefined || + patch.baseImage !== undefined, { message: - 'updateSettings needs at least one of sandboxDefaults, defaultPolicy, s3, sandboxDomain, sandboxDomainAliases, pidsLimit', + 'updateSettings needs at least one of sandboxDefaults, defaultPolicy, s3, sandboxDomain, sandboxDomainAliases, pidsLimit, baseImage', }, ); From ff2ff51aa7bbb9ae1ad4d09c9fd9e5dee0220502 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 16:58:47 +0800 Subject: [PATCH 60/89] The upgrade is the fleet's: the gateway's machine first, then every node told at its check-in, one at a time, once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three upgrade verbs answer at the gateway (admin gate; the 501s and UNNAMED_VERBS are gone). checkUpgrade compares the gateway's own build against origin/main. applyUpgrade with no node runs install.sh on the gateway's machine through the daemon's Updater, reached through a new @dormice/server/updater subpath — its constructor takes the caller's own reason one-click is off instead of an executor name, and its availability() is public. getUpgradeStatus is that run plus every node's standing: current, behind, upgrading, stuck, unavailable, unreachable or unknown, each with the reason in the gateway's words. The roll rides the check-in (rolling.ts). A node reports whether it can upgrade itself; one that runs another build than the gateway and can is told — the answer carries upgrade: true — when no other node is upgrading, and the node runs its own updater. Told once: the tell is on the node's row (migration 0005, with the node's self-upgrade word), so a gateway restart neither forgets it nor repeats it; a node still on the old build twenty minutes later is stuck, never re-told on its own — a build that fails every time must not rebuild every twenty minutes on the sandboxes' CPU — and the pointer moves past it. A told node's missed check-ins around its restart read as upgrading, not unreachable, or the one-at-a-time rule would see nobody upgrading. applyUpgrade with a nodeId is the operator's hand that tells a node again, whatever the order says. The console's version page speaks for the gateway, its upgrade dialog says the nodes follow one at a time, and a table under it shows every node's standing with a Try again button on a stuck one. The SDK's applyUpgrade takes the optional nodeId. --- e2e/src/gateway.test.ts | 25 +- packages/console/messages/de/settings.json | 30 +- packages/console/messages/en/settings.json | 30 +- packages/console/messages/es/settings.json | 30 +- packages/console/messages/fr/settings.json | 30 +- packages/console/messages/ja/settings.json | 30 +- packages/console/messages/ko/settings.json | 30 +- packages/console/messages/pt-BR/settings.json | 30 +- packages/console/messages/ru/settings.json | 30 +- packages/console/messages/zh-CN/settings.json | 30 +- packages/console/messages/zh-TW/settings.json | 30 +- .../settings/components/UpgradeDialog.tsx | 11 +- .../settings/components/VersionCard.tsx | 144 ++++- .../src/features/settings/hooks/useUpgrade.ts | 21 +- packages/console/src/lib/api.ts | 10 +- .../gateway/drizzle/0005_fleet-upgrade.sql | 2 + .../gateway/drizzle/meta/0005_snapshot.json | 502 ++++++++++++++++++ packages/gateway/drizzle/meta/_journal.json | 7 + packages/gateway/src/app.test.ts | 21 +- packages/gateway/src/app.ts | 24 +- packages/gateway/src/db/schema.ts | 9 + packages/gateway/src/fleet.ts | 53 +- packages/gateway/src/main.ts | 27 + packages/gateway/src/rolling.test.ts | 293 ++++++++++ packages/gateway/src/rolling.ts | 217 ++++++++ packages/gateway/src/routes/native.ts | 22 - packages/gateway/src/routes/nodes.ts | 31 +- packages/gateway/src/routes/upgrade.test.ts | 232 ++++++++ packages/gateway/src/routes/upgrade.ts | 96 ++++ packages/gateway/src/testing.ts | 26 +- packages/sdk/src/client.ts | 9 +- packages/server/package.json | 4 + packages/server/src/app.ts | 16 +- packages/server/src/check-in.test.ts | 56 ++ packages/server/src/check-in.ts | 38 ++ packages/server/src/main.ts | 35 +- packages/server/src/updater.test.ts | 11 +- packages/server/src/updater.ts | 46 +- packages/server/tsup.config.ts | 9 +- packages/shared/src/gateway.ts | 18 + packages/shared/src/upgrade.ts | 81 ++- website/content/docs/console.mdx | 15 +- website/content/docs/http-api.mdx | 6 +- website/content/docs/upgrading.mdx | 24 + 44 files changed, 2246 insertions(+), 195 deletions(-) create mode 100644 packages/gateway/drizzle/0005_fleet-upgrade.sql create mode 100644 packages/gateway/drizzle/meta/0005_snapshot.json create mode 100644 packages/gateway/src/rolling.test.ts create mode 100644 packages/gateway/src/rolling.ts create mode 100644 packages/gateway/src/routes/upgrade.test.ts create mode 100644 packages/gateway/src/routes/upgrade.ts diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts index e22e4d04..7b6b5b6b 100644 --- a/e2e/src/gateway.test.ts +++ b/e2e/src/gateway.test.ts @@ -493,11 +493,26 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { expect(history.points.length).toBeGreaterThanOrEqual(1); }); - it('the upgrade verbs are an honest 501 until their cut; a misspelled verb is a 404', async () => { - await expect(viaGateway().checkUpgrade()).rejects.toMatchObject({ - status: 501, - message: expect.stringMatching(/until the upgrade cut/), - }); + it("the upgrade verbs answer at the door: the gateway's own build and standing, every node's standing beside it; a misspelled verb is a 404", async () => { + // Deliberately not applyUpgrade: it would re-run install.sh on the + // machine running the exam. checkUpgrade reaches for origin/main — + // its outcome is data either way (a check, or a checkError). + const check = await viaGateway().checkUpgrade(); + expect(check.check !== null || check.checkError !== null).toBe(true); + const s = await viaGateway().getUpgradeStatus(); + expect(typeof s.available).toBe('boolean'); + if (!s.available) expect(s.unavailableReason).not.toBeNull(); + // The exam's gateway and nodes are one build (or, built outside a + // checkout, none): every node is current, or unknown with the reason. + const standing = new Map(s.nodes?.map((n) => [n.id, n])); + for (const id of ['node-b', 'node-c']) { + const node = standing.get(id); + expect(node).toBeDefined(); + expect(['current', 'unknown']).toContain(node?.state); + if (node?.state === 'unknown') expect(node.reason).not.toBeNull(); + } + // A node's own answer has no nodes to speak of. + expect((await direct('node-b').getUpgradeStatus()).nodes).toBeUndefined(); expect((await rpc('/acquireSandbx', { name: 'x' })).status).toBe(404); }); diff --git a/packages/console/messages/de/settings.json b/packages/console/messages/de/settings.json index ce2fa363..debd6ddc 100644 --- a/packages/console/messages/de/settings.json +++ b/packages/console/messages/de/settings.json @@ -84,22 +84,23 @@ "settings_policy_archive_label": "Archivieren nach Leerlauf (Sekunden)", "settings_policy_archive_desc": "Gestoppt → Archiviert: Der Datenträger wird komprimiert nach S3 hochgeladen, null lokaler Platz.", "settings_upgrade_dialog_title": "Dormice aktualisieren", - "settings_upgrade_dialog_desc": "Führt install.sh auf diesem Server erneut aus (erneut ausführen = aktualisieren); das Ganze dauert wenige Minuten.", + "settings_upgrade_dialog_desc": "Aktualisiert die ganze Flotte: install.sh läuft zuerst auf der Gateway-Maschine erneut (Erneut-Ausführen ist das Upgrade); sobald das Gateway mit dem neuen Build zurück ist, folgen die übrigen Knoten nacheinander, jeder bei seinem nächsten Check-in. Insgesamt wenige Minuten, mit mehr Knoten länger.", "settings_upgrade_step1": "Neuesten Code holen und neu bauen; die Dauer hängt vom Netz des Servers ab.", - "settings_upgrade_step2_pre": "Nach erfolgreichem Build startet der Daemon neu: ", + "settings_upgrade_step2_pre": "Nach erfolgreichem Build starten das Gateway und der Knoten auf dieser Maschine neu: ", "settings_upgrade_step2_em": "laufende Terminals, Befehlsausführungen und Datei-Watches werden getrennt", "settings_upgrade_step2_post": "; Sandboxes und Datenträger bleiben unberührt, der Abgleich übernimmt.", "settings_upgrade_step3": "Schlägt der Build fehl, rollt der Code automatisch auf die aktuelle Version zurück und wird neu gebaut; der Daemon startet nicht neu und bedient weiter.", - "settings_upgrade_step4": "Während des Neustarts ist die Konsole kurz nicht erreichbar; dieser Dialog wartet, bis sie mit der neuen Version zurück ist.", + "settings_upgrade_step4": "Während des Neustarts verliert die Konsole kurz die Verbindung; dieser Dialog wartet, bis das Gateway mit der neuen Version zurück ist.", + "settings_upgrade_step5": "Die übrigen Knoten aktualisieren einer nach dem anderen: Jeder wird bei seinem nächsten Check-in benachrichtigt, führt install.sh selbst aus, startet einige Dutzend Sekunden neu und meldet sich mit dem neuen Build, bevor der nächste benachrichtigt wird. Ein Knoten, der nach 20 Minuten nicht zurück ist, wird als „festgefahren“ markiert und Ihnen überlassen; er wird nie von selbst erneut versucht.", "settings_upgrade_launch_failed": "Upgrade konnte nicht gestartet werden: {error}", "settings_upgrade_start": "Upgrade starten", "settings_upgrade_watch_running": "Upgrade läuft", "settings_upgrade_watch_succeeded": "Upgrade abgeschlossen", "settings_upgrade_watch_rolled_back": "Upgrade fehlgeschlagen, zurückgerollt", "settings_upgrade_watch_failed": "Upgrade fehlgeschlagen", - "settings_upgrade_desc_unreachable": "Der Daemon startet gerade neu; ein kurzer Verbindungsverlust gehört zum Upgrade, wir warten auf seine Rückkehr. Bleibt es lange hierbei, melden Sie sich per ssh an und prüfen Sie journalctl -u dormice-upgrade.", + "settings_upgrade_desc_unreachable": "Das Gateway startet neu; ein kurzer Verbindungsverlust gehört zum Upgrade, und wir warten auf seine Rückkehr. Bleibt es lange hier stehen, per ssh einloggen und journalctl -u dormice-upgrade prüfen.", "settings_upgrade_desc_running": "Das Upgrade läuft in einer systemd-Unit auf dem Server; das Schließen dieses Dialogs unterbricht es nicht.", - "settings_upgrade_success_1": "Der Daemon ist mit der neuen Version zurück und hat die doctor-Prüfung bestanden", + "settings_upgrade_success_1": "Das Gateway ist mit der neuen Version zurück und hat die doctor-Prüfung bestanden", "settings_upgrade_success_running": ", läuft jetzt", "settings_upgrade_success_2": ". Diese Seite ist noch die alte Konsole; klicken Sie auf „Konsole neu laden“, um die neue zu laden.", "settings_upgrade_desc_rolled_back": "Der Build schlug fehl; der Code wurde auf die Version vor dem Upgrade zurückgerollt und neu gebaut. Der Daemon startete nicht neu und bedient weiter. Ursache beheben und erneut versuchen.", @@ -115,7 +116,7 @@ "settings_upgrade_more_commits": "{n} ältere Commits nicht angezeigt", "settings_upgrade_ignore": "Diese Version ignorieren", "settings_upgrade_go": "Aktualisieren", - "settings_version_card_desc": "Der Build, den der Daemon gerade ausführt, und ob es auf dem entfernten main einen neueren gibt", + "settings_version_card_desc": "Der Build, den das Gateway ausführt, ob remote main einen neueren hat, und ob jeder Knoten aufgeholt hat", "settings_check_updates": "Nach Updates suchen", "settings_upgrade_running_banner": "Upgrade läuft gerade", "settings_view_progress": "Fortschritt anzeigen", @@ -133,10 +134,25 @@ "settings_up_to_date_cached": " (Ergebnis aus dem Cache; „Nach Updates suchen“ prüft neu)", "settings_upgradable_badge": "Aktualisierbar", "settings_upgrade_to_latest": "Auf neueste Version aktualisieren", - "settings_oneclick_unavailable": "Ein-Klick-Upgrade ist auf diesem Daemon nicht verfügbar", + "settings_oneclick_unavailable": "Das Ein-Klick-Upgrade ist auf dieser Gateway-Maschine nicht verfügbar", "settings_paren_open": " (", "settings_paren_close": ")", "settings_oneclick_manual": "; install.sh auf dem Server erneut ausführen genügt (erneut ausführen = aktualisieren, der API-Token rotiert dabei nicht).", + "settings_nodes_title": "Knoten", + "settings_nodes_desc": "Jeder Knoten am Build des Gateways gemessen: gleich = aufgeholt; ein anderer Build und selbst aktualisierbar = wird bei einem Check-in benachrichtigt, wenn er an der Reihe ist.", + "settings_nodes_col_node": "Knoten", + "settings_nodes_col_build": "Build", + "settings_nodes_col_state": "Zustand", + "settings_nodes_state_current": "Aktuell", + "settings_nodes_state_behind": "Wartet", + "settings_nodes_state_upgrading": "Aktualisiert", + "settings_nodes_state_stuck": "Festgefahren", + "settings_nodes_state_unavailable": "Kein Selbst-Upgrade", + "settings_nodes_state_unreachable": "Nicht erreichbar", + "settings_nodes_state_unknown": "Unbekannt", + "settings_nodes_retry": "Erneut versuchen", + "settings_nodes_retry_done": "Knoten {id} benachrichtigt; er aktualisiert beim nächsten Check-in erneut", + "settings_nodes_retry_failed": "Knoten {id} konnte nicht benachrichtigt werden: {error}", "settings_archive_card_title": "Archivspeicher (S3)", "settings_archive_card_desc": "Datenträger untätiger Sandboxes wandern komprimiert in einen S3-kompatiblen Speicher, null lokaler Platz, und werden beim nächsten acquire automatisch wiederhergestellt. Wohnt im Ledger; Änderungen wirken sofort, ohne Neustart.", "settings_archive_not_configured": "Nicht konfiguriert — Sandboxes bleiben höchstens „Gestoppt“ und ihre Datenträger belegen weiter lokalen Platz.", diff --git a/packages/console/messages/en/settings.json b/packages/console/messages/en/settings.json index d3eae1a7..2a44ba56 100644 --- a/packages/console/messages/en/settings.json +++ b/packages/console/messages/en/settings.json @@ -84,22 +84,23 @@ "settings_policy_archive_label": "Archive after idle (seconds)", "settings_policy_archive_desc": "Stopped to archived: the disk is compressed and uploaded to S3, with zero local footprint.", "settings_upgrade_dialog_title": "Upgrade Dormice", - "settings_upgrade_dialog_desc": "Re-runs install.sh on this server (re-running is upgrading); the whole process takes a few minutes.", + "settings_upgrade_dialog_desc": "Upgrades the whole fleet: install.sh re-runs on the gateway's machine first (re-running is upgrading), and once the gateway is back on the new build the other nodes follow, one at a time, each at its next check-in. A few minutes in all, longer with more nodes.", "settings_upgrade_step1": "Pull the latest code and rebuild; duration depends on the server's network.", - "settings_upgrade_step2_pre": "After a successful build the daemon restarts: ", + "settings_upgrade_step2_pre": "After a successful build the gateway and the node on this machine restart: ", "settings_upgrade_step2_em": "in-progress terminals, command executions, and file watches will disconnect", "settings_upgrade_step2_post": "; sandboxes and disks are unaffected, and the reconciler takes over.", "settings_upgrade_step3": "If the build fails, the code automatically rolls back to the current version and rebuilds; the daemon does not restart and keeps serving.", - "settings_upgrade_step4": "The console briefly loses connection during the restart; this dialog waits for it to come back with the new version.", + "settings_upgrade_step4": "The console briefly loses connection during the restart; this dialog waits for the gateway to come back with the new version.", + "settings_upgrade_step5": "The other nodes upgrade one at a time: each is told at its next check-in, runs install.sh itself, restarts for a few tens of seconds and checks in on the new build before the next one is told. A node not back after 20 minutes is marked stuck and left to you; it is never retried on its own.", "settings_upgrade_launch_failed": "Failed to start upgrade: {error}", "settings_upgrade_start": "Start upgrade", "settings_upgrade_watch_running": "Upgrade in progress", "settings_upgrade_watch_succeeded": "Upgrade complete", "settings_upgrade_watch_rolled_back": "Upgrade failed, rolled back", "settings_upgrade_watch_failed": "Upgrade failed", - "settings_upgrade_desc_unreachable": "The daemon is restarting; a brief loss of connection is an expected part of the upgrade, and we are waiting for it to return. If it stays here for a long time, ssh in and check journalctl -u dormice-upgrade.", + "settings_upgrade_desc_unreachable": "The gateway is restarting; a brief loss of connection is an expected part of the upgrade, and we are waiting for it to return. If it stays here for a long time, ssh in and check journalctl -u dormice-upgrade.", "settings_upgrade_desc_running": "The upgrade runs in a systemd unit on the server; closing this dialog will not interrupt it.", - "settings_upgrade_success_1": "The daemon is back with the new version and passed the doctor check", + "settings_upgrade_success_1": "The gateway is back with the new version and passed the doctor check", "settings_upgrade_success_running": ", now running", "settings_upgrade_success_2": ". This page is still the old console; click Reload console to load the new build.", "settings_upgrade_desc_rolled_back": "The build failed; the code was rolled back to the pre-upgrade version and rebuilt. The daemon did not restart and keeps serving. Fix the cause and try again.", @@ -115,7 +116,7 @@ "settings_upgrade_more_commits": "{n} earlier commits not shown", "settings_upgrade_ignore": "Ignore this version", "settings_upgrade_go": "Upgrade", - "settings_version_card_desc": "The build the daemon is currently running, and whether remote main has a newer one", + "settings_version_card_desc": "The build the gateway is running, whether remote main has a newer one, and whether every node has caught up", "settings_check_updates": "Check updates", "settings_upgrade_running_banner": "Upgrade in progress", "settings_view_progress": "View progress", @@ -133,10 +134,25 @@ "settings_up_to_date_cached": " (cached result; click Check updates to re-test)", "settings_upgradable_badge": "Upgradable", "settings_upgrade_to_latest": "Upgrade to latest", - "settings_oneclick_unavailable": "One-click upgrade is not available on this daemon", + "settings_oneclick_unavailable": "One-click upgrade is not available on this gateway's machine", "settings_paren_open": " (", "settings_paren_close": ")", "settings_oneclick_manual": "; re-run install.sh on the server to upgrade (re-running is upgrading and does not rotate the API token).", + "settings_nodes_title": "Nodes", + "settings_nodes_desc": "Each node judged against the gateway's build: the same = caught up; another build and able to upgrade itself = told at a check-in when its turn comes.", + "settings_nodes_col_node": "Node", + "settings_nodes_col_build": "Build", + "settings_nodes_col_state": "State", + "settings_nodes_state_current": "Current", + "settings_nodes_state_behind": "Behind", + "settings_nodes_state_upgrading": "Upgrading", + "settings_nodes_state_stuck": "Stuck", + "settings_nodes_state_unavailable": "Cannot self-upgrade", + "settings_nodes_state_unreachable": "Unreachable", + "settings_nodes_state_unknown": "Unknown", + "settings_nodes_retry": "Try again", + "settings_nodes_retry_done": "Node {id} told; it upgrades again at its next check-in", + "settings_nodes_retry_failed": "Could not tell node {id}: {error}", "settings_archive_card_title": "Archive store (S3)", "settings_archive_card_desc": "Idle sandboxes' disks compress into any S3-compatible store at zero local cost, and restore on the next acquire. Lives in the ledger; changes apply immediately, no restart.", "settings_archive_not_configured": "Not configured — sandboxes park at stopped and their disks keep occupying local space.", diff --git a/packages/console/messages/es/settings.json b/packages/console/messages/es/settings.json index 7ed5f16d..f4f33aec 100644 --- a/packages/console/messages/es/settings.json +++ b/packages/console/messages/es/settings.json @@ -84,22 +84,23 @@ "settings_policy_archive_label": "Archivar tras inactividad (segundos)", "settings_policy_archive_desc": "Detenido → Archivado: el disco se comprime y se sube a S3, sin ocupar nada en local.", "settings_upgrade_dialog_title": "Actualizar Dormice", - "settings_upgrade_dialog_desc": "Vuelve a ejecutar install.sh en este servidor (repetirlo es actualizar); el proceso completo tarda unos minutos.", + "settings_upgrade_dialog_desc": "Actualiza toda la flota: install.sh se vuelve a ejecutar primero en la máquina del gateway (volver a ejecutarlo es actualizar) y, cuando el gateway regresa con la nueva compilación, los demás nodos lo siguen de uno en uno, cada uno en su siguiente registro. Unos minutos en total; más con más nodos.", "settings_upgrade_step1": "Se descarga el código más reciente y se recompila; la duración depende de la red del servidor.", - "settings_upgrade_step2_pre": "Tras una compilación correcta el daemon se reinicia: ", + "settings_upgrade_step2_pre": "Tras una compilación correcta, el gateway y el nodo de esta máquina se reinician: ", "settings_upgrade_step2_em": "las terminales, las ejecuciones de comandos y las vigilancias de archivos en curso se desconectarán", "settings_upgrade_step2_post": "; los sandboxes y los discos no se ven afectados, y el reconciliador toma el relevo.", "settings_upgrade_step3": "Si la compilación falla, el código vuelve automáticamente a la versión actual y se recompila; el daemon no se reinicia y sigue atendiendo.", - "settings_upgrade_step4": "La consola pierde la conexión un momento durante el reinicio; este diálogo espera a que vuelva con la nueva versión.", + "settings_upgrade_step4": "La consola pierde la conexión brevemente durante el reinicio; este diálogo espera a que el gateway vuelva con la nueva versión.", + "settings_upgrade_step5": "Los demás nodos se actualizan de uno en uno: a cada uno se le avisa en su siguiente registro, ejecuta install.sh por sí mismo, se reinicia durante unas decenas de segundos y se registra con la nueva compilación antes de avisar al siguiente. Un nodo que no ha vuelto tras 20 minutos se marca como atascado y se deja en tus manos; nunca se reintenta solo.", "settings_upgrade_launch_failed": "No se pudo iniciar la actualización: {error}", "settings_upgrade_start": "Iniciar la actualización", "settings_upgrade_watch_running": "Actualización en curso", "settings_upgrade_watch_succeeded": "Actualización completada", "settings_upgrade_watch_rolled_back": "La actualización falló y se revirtió", "settings_upgrade_watch_failed": "La actualización falló", - "settings_upgrade_desc_unreachable": "El daemon se está reiniciando; perder la conexión un momento es parte esperada de la actualización y estamos esperando a que vuelva. Si se queda aquí mucho tiempo, conéctate por ssh y revisa journalctl -u dormice-upgrade.", + "settings_upgrade_desc_unreachable": "El gateway se está reiniciando; una breve pérdida de conexión es parte esperada de la actualización y estamos esperando a que vuelva. Si se queda aquí mucho tiempo, entra por ssh y revisa journalctl -u dormice-upgrade.", "settings_upgrade_desc_running": "La actualización corre en una unidad de systemd del servidor; cerrar este diálogo no la interrumpe.", - "settings_upgrade_success_1": "El daemon volvió con la nueva versión y pasó la revisión de doctor", + "settings_upgrade_success_1": "El gateway ha vuelto con la nueva versión y ha pasado la comprobación de doctor", "settings_upgrade_success_running": ", ahora en ejecución", "settings_upgrade_success_2": ". Esta página sigue siendo la consola antigua; usa Recargar consola para cargar la nueva compilación.", "settings_upgrade_desc_rolled_back": "La compilación falló; el código volvió a la versión previa a la actualización y se recompiló. El daemon no se reinició y sigue atendiendo. Corrige la causa y vuelve a intentarlo.", @@ -115,7 +116,7 @@ "settings_upgrade_more_commits": "{n} commits anteriores sin mostrar", "settings_upgrade_ignore": "Ignorar esta versión", "settings_upgrade_go": "Actualizar", - "settings_version_card_desc": "La compilación que el daemon está ejecutando y si en el main remoto hay una más nueva", + "settings_version_card_desc": "La compilación que ejecuta el gateway, si el main remoto tiene una más nueva y si todos los nodos se han puesto al día", "settings_check_updates": "Buscar actualizaciones", "settings_upgrade_running_banner": "Actualización en curso", "settings_view_progress": "Ver el progreso", @@ -133,10 +134,25 @@ "settings_up_to_date_cached": " (resultado en caché; usa Buscar actualizaciones para repetir la comprobación)", "settings_upgradable_badge": "Actualizable", "settings_upgrade_to_latest": "Actualizar a la última", - "settings_oneclick_unavailable": "La actualización con un clic no está disponible en este daemon", + "settings_oneclick_unavailable": "La actualización con un clic no está disponible en la máquina de este gateway", "settings_paren_open": " (", "settings_paren_close": ")", "settings_oneclick_manual": "; vuelve a ejecutar install.sh en el servidor para actualizar (repetirlo es actualizar y no rota el token de API).", + "settings_nodes_title": "Nodos", + "settings_nodes_desc": "Cada nodo juzgado contra la compilación del gateway: igual = al día; otra compilación y capaz de actualizarse solo = se le avisa en un registro cuando le toca.", + "settings_nodes_col_node": "Nodo", + "settings_nodes_col_build": "Compilación", + "settings_nodes_col_state": "Estado", + "settings_nodes_state_current": "Al día", + "settings_nodes_state_behind": "Pendiente", + "settings_nodes_state_upgrading": "Actualizando", + "settings_nodes_state_stuck": "Atascado", + "settings_nodes_state_unavailable": "No se autoactualiza", + "settings_nodes_state_unreachable": "Inalcanzable", + "settings_nodes_state_unknown": "Desconocido", + "settings_nodes_retry": "Reintentar", + "settings_nodes_retry_done": "Nodo {id} avisado; se actualiza de nuevo en su siguiente registro", + "settings_nodes_retry_failed": "No se pudo avisar al nodo {id}: {error}", "settings_archive_card_title": "Almacenamiento de archivado (S3)", "settings_archive_card_desc": "El disco de los sandboxes inactivos se comprime y se sube a un almacenamiento compatible con S3, sin ocupar nada en local, y se restaura solo en el siguiente acquire. Vive en el libro de registro; los cambios se aplican de inmediato, sin reiniciar.", "settings_archive_not_configured": "Sin configurar — los sandboxes se quedan como muy frío en «Detenido» y su disco sigue ocupando espacio local.", diff --git a/packages/console/messages/fr/settings.json b/packages/console/messages/fr/settings.json index 67fc5f6a..20b85f01 100644 --- a/packages/console/messages/fr/settings.json +++ b/packages/console/messages/fr/settings.json @@ -84,22 +84,23 @@ "settings_policy_archive_label": "Archiver après inactivité (secondes)", "settings_policy_archive_desc": "Arrêtée → Archivée : le disque est compressé et envoyé sur S3, zéro empreinte locale.", "settings_upgrade_dialog_title": "Mettre à niveau Dormice", - "settings_upgrade_dialog_desc": "Relance install.sh sur ce serveur (relancer, c'est mettre à niveau) ; l'ensemble prend quelques minutes.", + "settings_upgrade_dialog_desc": "Met à niveau toute la flotte : install.sh est d'abord relancé sur la machine de la gateway (relancer, c'est mettre à niveau), puis, une fois la gateway revenue avec le nouveau build, les autres nœuds suivent un par un, chacun à son prochain check-in. Quelques minutes en tout, davantage avec plus de nœuds.", "settings_upgrade_step1": "Récupère le code le plus récent et reconstruit ; la durée dépend du réseau du serveur.", - "settings_upgrade_step2_pre": "Après une construction réussie, le daemon redémarre : ", + "settings_upgrade_step2_pre": "Après un build réussi, la gateway et le nœud de cette machine redémarrent : ", "settings_upgrade_step2_em": "les terminaux, exécutions de commandes et surveillances de fichiers en cours seront déconnectés", "settings_upgrade_step2_post": " ; sandbox et disques ne sont pas affectés, le réconciliateur reprend la main.", "settings_upgrade_step3": "Si la construction échoue, le code revient automatiquement à la version actuelle et se reconstruit ; le daemon ne redémarre pas et continue de servir.", - "settings_upgrade_step4": "La console perd brièvement la connexion pendant le redémarrage ; cette fenêtre attend son retour avec la nouvelle version.", + "settings_upgrade_step4": "La console perd brièvement la connexion pendant le redémarrage ; cette boîte de dialogue attend que la gateway revienne avec la nouvelle version.", + "settings_upgrade_step5": "Les autres nœuds se mettent à niveau un par un : chacun est prévenu à son prochain check-in, lance install.sh lui-même, redémarre quelques dizaines de secondes et se présente avec le nouveau build avant que le suivant soit prévenu. Un nœud qui n'est pas revenu après 20 minutes est marqué « bloqué » et vous est laissé ; il n'est jamais retenté de lui-même.", "settings_upgrade_launch_failed": "Impossible de lancer la mise à niveau : {error}", "settings_upgrade_start": "Démarrer la mise à niveau", "settings_upgrade_watch_running": "Mise à niveau en cours", "settings_upgrade_watch_succeeded": "Mise à niveau terminée", "settings_upgrade_watch_rolled_back": "Mise à niveau échouée, retour arrière effectué", "settings_upgrade_watch_failed": "Mise à niveau échouée", - "settings_upgrade_desc_unreachable": "Le daemon redémarre ; une brève perte de connexion fait partie du déroulement normal de la mise à niveau, nous attendons son retour. Si cela dure trop longtemps, connectez-vous en ssh et consultez journalctl -u dormice-upgrade.", + "settings_upgrade_desc_unreachable": "La gateway redémarre ; une brève perte de connexion fait partie de la mise à niveau, et nous attendons son retour. Si cela reste ainsi longtemps, connectez-vous en ssh et consultez journalctl -u dormice-upgrade.", "settings_upgrade_desc_running": "La mise à niveau s'exécute dans une unité systemd sur le serveur ; fermer cette fenêtre ne l'interrompt pas.", - "settings_upgrade_success_1": "Le daemon est revenu avec la nouvelle version et a passé la vérification doctor", + "settings_upgrade_success_1": "La gateway est revenue avec la nouvelle version et a passé la vérification doctor", "settings_upgrade_success_running": ", exécute maintenant", "settings_upgrade_success_2": ". Cette page est encore l'ancienne console ; cliquez sur « Recharger la console » pour charger la nouvelle version.", "settings_upgrade_desc_rolled_back": "La construction a échoué ; le code est revenu à la version d'avant la mise à niveau et a été reconstruit. Le daemon n'a pas redémarré et continue de servir. Corrigez la cause, puis réessayez.", @@ -115,7 +116,7 @@ "settings_upgrade_more_commits": "{n} commits plus anciens non affichés", "settings_upgrade_ignore": "Ignorer cette version", "settings_upgrade_go": "Mettre à niveau", - "settings_version_card_desc": "La version que le daemon exécute actuellement, et si la branche main distante en a une plus récente", + "settings_version_card_desc": "Le build que la gateway exécute, si le main distant en a un plus récent, et si chaque nœud a rattrapé", "settings_check_updates": "Vérifier les mises à jour", "settings_upgrade_running_banner": "Mise à niveau en cours", "settings_view_progress": "Voir la progression", @@ -133,10 +134,25 @@ "settings_up_to_date_cached": " (résultat en cache ; cliquez sur « Vérifier les mises à jour » pour retester)", "settings_upgradable_badge": "À mettre à niveau", "settings_upgrade_to_latest": "Mettre à niveau vers la dernière version", - "settings_oneclick_unavailable": "La mise à niveau en un clic n'est pas disponible sur ce daemon", + "settings_oneclick_unavailable": "La mise à niveau en un clic n'est pas disponible sur la machine de cette gateway", "settings_paren_open": " (", "settings_paren_close": ")", "settings_oneclick_manual": " ; relancez install.sh sur le serveur pour mettre à niveau (relancer, c'est mettre à niveau ; le token API n'est pas modifié).", + "settings_nodes_title": "Nœuds", + "settings_nodes_desc": "Chaque nœud jugé sur le build de la gateway : identique = à jour ; un autre build et capable de se mettre à niveau lui-même = prévenu à un check-in quand vient son tour.", + "settings_nodes_col_node": "Nœud", + "settings_nodes_col_build": "Build", + "settings_nodes_col_state": "État", + "settings_nodes_state_current": "À jour", + "settings_nodes_state_behind": "En attente", + "settings_nodes_state_upgrading": "Mise à niveau", + "settings_nodes_state_stuck": "Bloqué", + "settings_nodes_state_unavailable": "Pas d’auto-mise à niveau", + "settings_nodes_state_unreachable": "Injoignable", + "settings_nodes_state_unknown": "Inconnu", + "settings_nodes_retry": "Réessayer", + "settings_nodes_retry_done": "Nœud {id} prévenu ; il se met à niveau à nouveau à son prochain check-in", + "settings_nodes_retry_failed": "Impossible de prévenir le nœud {id} : {error}", "settings_archive_card_title": "Stockage d'archivage (S3)", "settings_archive_card_desc": "Le disque des sandbox inactives est compressé vers un stockage compatible S3, sans empreinte locale, et restauré automatiquement au prochain acquire. Stocké dans le registre ; les changements prennent effet immédiatement, sans redémarrage.", "settings_archive_not_configured": "Non configuré — les sandbox s'arrêtent au plus froid à « Arrêtée » et leurs disques continuent d'occuper l'espace local.", diff --git a/packages/console/messages/ja/settings.json b/packages/console/messages/ja/settings.json index 89331da0..a68f78d7 100644 --- a/packages/console/messages/ja/settings.json +++ b/packages/console/messages/ja/settings.json @@ -84,22 +84,23 @@ "settings_policy_archive_label": "アイドル後にアーカイブするまで(秒)", "settings_policy_archive_desc": "停止中 → アーカイブ済み:ディスクを圧縮して S3 にアップロードし、ローカル使用量はゼロになります。", "settings_upgrade_dialog_title": "Dormice のアップグレード", - "settings_upgrade_dialog_desc": "このサーバーで install.sh を再実行します(再実行 = アップグレード)。全体で数分程度かかります。", + "settings_upgrade_dialog_desc": "フリート全体をアップグレードします:まずゲートウェイ機で install.sh を再実行し(再実行がアップグレードです)、ゲートウェイが新ビルドで戻った後、残りのノードがそれぞれ次のチェックイン時に 1 台ずつ続きます。全体で数分、ノードが多いほど長くなります。", "settings_upgrade_step1": "最新のコードを取得して再ビルドします。所要時間はサーバーのネットワーク次第です。", - "settings_upgrade_step2_pre": "ビルド成功後に daemon が再起動します:", + "settings_upgrade_step2_pre": "ビルド成功後、ゲートウェイとこのマシン上のノードが再起動します:", "settings_upgrade_step2_em": "進行中のターミナル、コマンド実行、ファイル監視は切断されます", "settings_upgrade_step2_post": "。サンドボックスとディスクは影響を受けず、リコンサイラーが現場を引き継ぎます。", "settings_upgrade_step3": "ビルドに失敗した場合は現在のバージョンへ自動でロールバックして再ビルドします。daemon は再起動せず、通常どおり稼働し続けます。", - "settings_upgrade_step4": "再起動中はコンソールが一時的に切断されます。このダイアログは、新しいバージョンで戻ってくるまで待機します。", + "settings_upgrade_step4": "再起動中はコンソールが一時的に切断されます。このダイアログはゲートウェイが新バージョンで戻るのを待ちます。", + "settings_upgrade_step5": "残りのノードは 1 台ずつアップグレードします:各ノードは次のチェックイン時に通知を受け、自ら install.sh を実行し、数十秒の再起動後に新ビルドでチェックインしてから次のノードが通知されます。20 分経っても戻らないノードは「スタック」と表示され人手に委ねられ、自動では再試行しません。", "settings_upgrade_launch_failed": "アップグレードを開始できません:{error}", "settings_upgrade_start": "アップグレード開始", "settings_upgrade_watch_running": "アップグレード進行中", "settings_upgrade_watch_succeeded": "アップグレード完了", "settings_upgrade_watch_rolled_back": "アップグレードに失敗し、自動でロールバックしました", "settings_upgrade_watch_failed": "アップグレードに失敗しました", - "settings_upgrade_desc_unreachable": "daemon は再起動中です。短時間の切断はアップグレードの想定内の段階で、復帰を待っています。長時間ここで止まる場合は、ssh で接続して journalctl -u dormice-upgrade を確認してください。", + "settings_upgrade_desc_unreachable": "ゲートウェイが再起動中です。一時的な切断はアップグレードの想定内の段階で、戻るのを待っています。長く止まったままなら、ssh で入って journalctl -u dormice-upgrade を確認してください。", "settings_upgrade_desc_running": "アップグレードはサーバーの systemd unit 内で実行されるため、このダイアログを閉じても中断されません。", - "settings_upgrade_success_1": "daemon は新しいバージョンで復帰し、doctor チェックに合格しました", + "settings_upgrade_success_1": "ゲートウェイが新バージョンで戻り、doctor チェックに合格しました", "settings_upgrade_success_running": "。現在実行中:", "settings_upgrade_success_2": "。このページはまだ旧バージョンのコンソールです。「コンソールを再読み込み」をクリックして新しいビルドを読み込んでください。", "settings_upgrade_desc_rolled_back": "ビルドに失敗したため、コードはアップグレード前のバージョンへロールバックして再ビルドされました。daemon は再起動しておらず、通常どおり稼働しています。原因を修正してから再試行してください。", @@ -115,7 +116,7 @@ "settings_upgrade_more_commits": "ほかに {n} 件の古いコミットは未表示です", "settings_upgrade_ignore": "このバージョンを無視", "settings_upgrade_go": "アップグレードへ", - "settings_version_card_desc": "daemon が現在実行しているビルドと、リモートの main により新しいバージョンがあるかどうか", + "settings_version_card_desc": "ゲートウェイが実行中のビルド、リモート main に新しいものがあるか、そして各ノードが追いついたか", "settings_check_updates": "更新を確認", "settings_upgrade_running_banner": "アップグレード進行中", "settings_view_progress": "進捗を表示", @@ -133,10 +134,25 @@ "settings_up_to_date_cached": "(キャッシュ済みの結果です。「更新を確認」をクリックすると再判定します)", "settings_upgradable_badge": "アップグレード可能", "settings_upgrade_to_latest": "最新へアップグレード", - "settings_oneclick_unavailable": "この daemon ではワンクリックアップグレードを利用できません", + "settings_oneclick_unavailable": "このゲートウェイ機ではワンクリックアップグレードは使えません", "settings_paren_open": "(", "settings_paren_close": ")", "settings_oneclick_manual": "。サーバーで install.sh を再実行すればアップグレードできます(再実行 = アップグレードで、API トークンはローテーションされません)。", + "settings_nodes_title": "ノード", + "settings_nodes_desc": "各ノードをゲートウェイのビルドと比べます:同じ=追いついている。異なり、かつ自己アップグレード可能=順番が来たときチェックインで通知されます。", + "settings_nodes_col_node": "ノード", + "settings_nodes_col_build": "ビルド", + "settings_nodes_col_state": "状態", + "settings_nodes_state_current": "最新", + "settings_nodes_state_behind": "待機中", + "settings_nodes_state_upgrading": "アップグレード中", + "settings_nodes_state_stuck": "スタック", + "settings_nodes_state_unavailable": "自己アップグレード不可", + "settings_nodes_state_unreachable": "到達不能", + "settings_nodes_state_unknown": "不明", + "settings_nodes_retry": "再試行", + "settings_nodes_retry_done": "ノード {id} に通知しました。次のチェックイン時に再びアップグレードします", + "settings_nodes_retry_failed": "ノード {id} に通知できません:{error}", "settings_archive_card_title": "アーカイブストレージ(S3)", "settings_archive_card_desc": "アイドルなサンドボックスのディスクを圧縮して S3 互換ストレージにアップロードし、ローカル使用量はゼロになります。次回の acquire で自動的に復元されます。台帳に保存され、変更は再起動なしで即時に反映されます。", "settings_archive_not_configured": "未設定 — サンドボックスは「停止中」までしか冷えず、ディスクはローカル領域を占め続けます。", diff --git a/packages/console/messages/ko/settings.json b/packages/console/messages/ko/settings.json index 8af88dcd..4cbf6147 100644 --- a/packages/console/messages/ko/settings.json +++ b/packages/console/messages/ko/settings.json @@ -84,22 +84,23 @@ "settings_policy_archive_label": "유휴 후 아카이브까지(초)", "settings_policy_archive_desc": "중지됨 → 아카이브됨: 디스크를 압축해 S3에 업로드하며 로컬 점유는 0입니다.", "settings_upgrade_dialog_title": "Dormice 업그레이드", - "settings_upgrade_dialog_desc": "이 서버에서 install.sh를 다시 실행합니다(다시 실행이 곧 업그레이드). 전체 과정은 몇 분 정도 걸립니다.", + "settings_upgrade_dialog_desc": "플릿 전체를 업그레이드합니다: 먼저 게이트웨이 머신에서 install.sh를 다시 실행하고(다시 실행이 곧 업그레이드), 게이트웨이가 새 빌드로 돌아오면 나머지 노드가 각자의 다음 체크인 때 한 대씩 이어서 올라갑니다. 전체 몇 분, 노드가 많으면 더 걸립니다.", "settings_upgrade_step1": "최신 코드를 가져와 다시 빌드합니다. 소요 시간은 서버 네트워크에 따라 다릅니다.", - "settings_upgrade_step2_pre": "빌드가 성공하면 daemon이 재시작됩니다: ", + "settings_upgrade_step2_pre": "빌드 성공 후 게이트웨이와 이 머신의 노드가 재시작됩니다: ", "settings_upgrade_step2_em": "진행 중인 터미널, 명령 실행, 파일 감시가 끊어집니다", "settings_upgrade_step2_post": ". 샌드박스와 디스크는 영향받지 않으며, 정합기가 현장을 이어받습니다.", "settings_upgrade_step3": "빌드가 실패하면 현재 버전으로 자동 롤백해 다시 빌드하며, daemon은 재시작하지 않고 평소처럼 서비스합니다.", - "settings_upgrade_step4": "재시작 동안 콘솔이 잠시 끊기며, 이 대화 상자는 daemon이 새 버전과 함께 돌아올 때까지 기다립니다.", + "settings_upgrade_step4": "재시작 중 콘솔이 잠시 끊깁니다. 이 대화상자는 게이트웨이가 새 버전으로 돌아올 때까지 기다립니다.", + "settings_upgrade_step5": "나머지 노드는 한 번에 한 대씩 업그레이드합니다: 각 노드는 다음 체크인 때 통지를 받고 스스로 install.sh를 실행해 수십 초 재시작한 뒤 새 빌드로 체크인하고, 그 다음에 다음 노드가 통지됩니다. 20분이 지나도 돌아오지 않는 노드는 「막힘」으로 표시되어 사람에게 맡겨지며, 자동으로 재시도하지 않습니다.", "settings_upgrade_launch_failed": "업그레이드를 시작할 수 없습니다: {error}", "settings_upgrade_start": "업그레이드 시작", "settings_upgrade_watch_running": "업그레이드 진행 중", "settings_upgrade_watch_succeeded": "업그레이드 완료", "settings_upgrade_watch_rolled_back": "업그레이드 실패, 자동 롤백됨", "settings_upgrade_watch_failed": "업그레이드 실패", - "settings_upgrade_desc_unreachable": "daemon이 재시작 중입니다. 잠깐의 연결 끊김은 업그레이드의 예상된 단계이며, 돌아오기를 기다리고 있습니다. 오랫동안 이 상태에 머문다면 ssh로 접속해 journalctl -u dormice-upgrade를 확인하세요.", + "settings_upgrade_desc_unreachable": "게이트웨이가 재시작 중입니다. 잠시 끊기는 것은 업그레이드의 예상된 단계이며 돌아오길 기다리고 있습니다. 오래 멈춰 있으면 ssh로 들어가 journalctl -u dormice-upgrade를 확인하세요.", "settings_upgrade_desc_running": "업그레이드는 서버의 systemd unit에서 실행되므로 이 대화 상자를 닫아도 중단되지 않습니다.", - "settings_upgrade_success_1": "daemon이 새 버전으로 돌아와 doctor 검사를 통과했습니다", + "settings_upgrade_success_1": "게이트웨이가 새 버전으로 돌아와 doctor 검사를 통과했습니다", "settings_upgrade_success_running": " — 현재 실행 중: ", "settings_upgrade_success_2": ". 이 페이지는 아직 이전 콘솔입니다. '콘솔 새로 고침'을 눌러 새 버전을 불러오세요.", "settings_upgrade_desc_rolled_back": "빌드가 실패해 코드가 업그레이드 이전 버전으로 롤백되어 다시 빌드되었습니다. daemon은 재시작하지 않았고 평소처럼 서비스 중입니다. 원인을 고친 뒤 다시 시도하세요.", @@ -115,7 +116,7 @@ "settings_upgrade_more_commits": "이전 커밋 {n}개는 표시하지 않음", "settings_upgrade_ignore": "이 버전 무시", "settings_upgrade_go": "업그레이드", - "settings_version_card_desc": "daemon이 현재 실행 중인 빌드와, 원격 main에 더 새로운 버전이 있는지 여부", + "settings_version_card_desc": "게이트웨이가 실행 중인 빌드, 원격 main에 더 새로운 것이 있는지, 그리고 각 노드가 따라왔는지", "settings_check_updates": "업데이트 확인", "settings_upgrade_running_banner": "업그레이드 진행 중", "settings_view_progress": "진행 상황 보기", @@ -133,10 +134,25 @@ "settings_up_to_date_cached": " (캐시된 결과입니다. '업데이트 확인'을 눌러 다시 확인하세요)", "settings_upgradable_badge": "업그레이드 가능", "settings_upgrade_to_latest": "최신으로 업그레이드", - "settings_oneclick_unavailable": "이 daemon에서는 원클릭 업그레이드를 사용할 수 없습니다", + "settings_oneclick_unavailable": "이 게이트웨이 머신에서는 원클릭 업그레이드를 사용할 수 없습니다", "settings_paren_open": " (", "settings_paren_close": ")", "settings_oneclick_manual": ". 서버에서 install.sh를 다시 실행하면 업그레이드됩니다(다시 실행이 곧 업그레이드이며, API token은 교체되지 않습니다).", + "settings_nodes_title": "노드", + "settings_nodes_desc": "각 노드를 게이트웨이 빌드와 비교합니다: 같음=따라옴; 다르고 스스로 업그레이드 가능=차례가 오면 체크인에서 통지됩니다.", + "settings_nodes_col_node": "노드", + "settings_nodes_col_build": "빌드", + "settings_nodes_col_state": "상태", + "settings_nodes_state_current": "최신", + "settings_nodes_state_behind": "대기 중", + "settings_nodes_state_upgrading": "업그레이드 중", + "settings_nodes_state_stuck": "막힘", + "settings_nodes_state_unavailable": "자체 업그레이드 불가", + "settings_nodes_state_unreachable": "접근 불가", + "settings_nodes_state_unknown": "알 수 없음", + "settings_nodes_retry": "다시 시도", + "settings_nodes_retry_done": "노드 {id}에 통지했습니다. 다음 체크인 때 다시 업그레이드합니다", + "settings_nodes_retry_failed": "노드 {id}에 통지할 수 없습니다: {error}", "settings_archive_card_title": "아카이브 스토리지(S3)", "settings_archive_card_desc": "유휴 샌드박스의 디스크를 압축해 S3 호환 스토리지로 업로드하며 로컬 점유는 0이고, 다음 acquire 때 자동으로 복원됩니다. 장부에 저장되어 변경 즉시 적용되며 재시작이 필요 없습니다.", "settings_archive_not_configured": "미설정 — 샌드박스는 '중지됨'까지만 식으며, 디스크가 로컬 공간을 계속 차지합니다.", diff --git a/packages/console/messages/pt-BR/settings.json b/packages/console/messages/pt-BR/settings.json index a7101ab3..a38e26f1 100644 --- a/packages/console/messages/pt-BR/settings.json +++ b/packages/console/messages/pt-BR/settings.json @@ -84,22 +84,23 @@ "settings_policy_archive_label": "Arquivar após ociosidade (segundos)", "settings_policy_archive_desc": "Parado → Arquivado: o disco é comprimido e enviado ao S3, zero espaço local.", "settings_upgrade_dialog_title": "Atualizar o Dormice", - "settings_upgrade_dialog_desc": "Reexecuta o install.sh neste servidor (reexecutar é atualizar); o processo todo leva alguns minutos.", + "settings_upgrade_dialog_desc": "Atualiza toda a frota: o install.sh roda de novo primeiro na máquina do gateway (rodar de novo é atualizar) e, quando o gateway volta com o novo build, os demais nós seguem um por vez, cada um no seu próximo check-in. Alguns minutos no total; mais com mais nós.", "settings_upgrade_step1": "Baixa o código mais recente e reconstrói; a duração depende da rede do servidor.", - "settings_upgrade_step2_pre": "Após um build bem-sucedido, o daemon reinicia: ", + "settings_upgrade_step2_pre": "Após um build bem-sucedido, o gateway e o nó desta máquina reiniciam: ", "settings_upgrade_step2_em": "terminais, execuções de comando e observadores de arquivo em andamento serão desconectados", "settings_upgrade_step2_post": "; sandboxes e discos não são afetados, e o reconciliador assume a cena.", "settings_upgrade_step3": "Se o build falhar, o código volta automaticamente para a versão atual e é reconstruído; o daemon não reinicia e continua servindo.", - "settings_upgrade_step4": "O console perde a conexão por um instante durante o reinício; este diálogo espera ele voltar com a nova versão.", + "settings_upgrade_step4": "O console perde a conexão brevemente durante o reinício; este diálogo espera o gateway voltar com a nova versão.", + "settings_upgrade_step5": "Os demais nós atualizam um por vez: cada um é avisado no seu próximo check-in, roda o install.sh por conta própria, reinicia por algumas dezenas de segundos e faz check-in com o novo build antes que o próximo seja avisado. Um nó que não voltou após 20 minutos é marcado como travado e deixado para você; nunca é tentado de novo sozinho.", "settings_upgrade_launch_failed": "Falha ao iniciar a atualização: {error}", "settings_upgrade_start": "Iniciar atualização", "settings_upgrade_watch_running": "Atualização em andamento", "settings_upgrade_watch_succeeded": "Atualização concluída", "settings_upgrade_watch_rolled_back": "Atualização falhou, revertida", "settings_upgrade_watch_failed": "Atualização falhou", - "settings_upgrade_desc_unreachable": "O daemon está reiniciando; perder a conexão por um momento é parte esperada da atualização, e estamos aguardando ele voltar. Se ficar muito tempo aqui, entre via ssh e veja journalctl -u dormice-upgrade.", + "settings_upgrade_desc_unreachable": "O gateway está reiniciando; uma breve perda de conexão é parte esperada da atualização e estamos esperando ele voltar. Se ficar aqui por muito tempo, entre por ssh e veja journalctl -u dormice-upgrade.", "settings_upgrade_desc_running": "A atualização roda em uma unit do systemd no servidor; fechar este diálogo não a interrompe.", - "settings_upgrade_success_1": "O daemon voltou com a nova versão e passou na verificação do doctor", + "settings_upgrade_success_1": "O gateway voltou com a nova versão e passou na verificação do doctor", "settings_upgrade_success_running": ", rodando agora", "settings_upgrade_success_2": ". Esta página ainda é o console antigo; clique em Recarregar console para carregar a nova versão.", "settings_upgrade_desc_rolled_back": "O build falhou; o código foi revertido para a versão anterior à atualização e reconstruído. O daemon não reiniciou e continua servindo. Corrija a causa e tente de novo.", @@ -115,7 +116,7 @@ "settings_upgrade_more_commits": "{n} commits anteriores não exibidos", "settings_upgrade_ignore": "Ignorar esta versão", "settings_upgrade_go": "Atualizar", - "settings_version_card_desc": "O build que o daemon está executando agora e se o main remoto tem um mais novo", + "settings_version_card_desc": "O build que o gateway está rodando, se o main remoto tem um mais novo e se todos os nós acompanharam", "settings_check_updates": "Verificar atualizações", "settings_upgrade_running_banner": "Atualização em andamento", "settings_view_progress": "Ver progresso", @@ -133,10 +134,25 @@ "settings_up_to_date_cached": " (resultado em cache; clique em Verificar atualizações para testar de novo)", "settings_upgradable_badge": "Atualizável", "settings_upgrade_to_latest": "Atualizar para a mais recente", - "settings_oneclick_unavailable": "A atualização em um clique não está disponível neste daemon", + "settings_oneclick_unavailable": "A atualização com um clique não está disponível na máquina deste gateway", "settings_paren_open": " (", "settings_paren_close": ")", "settings_oneclick_manual": "; reexecute o install.sh no servidor para atualizar (reexecutar é atualizar e não rotaciona o token de API).", + "settings_nodes_title": "Nós", + "settings_nodes_desc": "Cada nó julgado pelo build do gateway: igual = em dia; outro build e capaz de se atualizar sozinho = avisado em um check-in quando chegar sua vez.", + "settings_nodes_col_node": "Nó", + "settings_nodes_col_build": "Build", + "settings_nodes_col_state": "Estado", + "settings_nodes_state_current": "Em dia", + "settings_nodes_state_behind": "Aguardando", + "settings_nodes_state_upgrading": "Atualizando", + "settings_nodes_state_stuck": "Travado", + "settings_nodes_state_unavailable": "Não se autoatualiza", + "settings_nodes_state_unreachable": "Inacessível", + "settings_nodes_state_unknown": "Desconhecido", + "settings_nodes_retry": "Tentar de novo", + "settings_nodes_retry_done": "Nó {id} avisado; ele atualiza de novo no próximo check-in", + "settings_nodes_retry_failed": "Não foi possível avisar o nó {id}: {error}", "settings_archive_card_title": "Armazenamento de arquivamento (S3)", "settings_archive_card_desc": "O disco dos sandboxes ociosos é comprimido e enviado para qualquer armazenamento compatível com S3, sem ocupar nada local, e volta sozinho no próximo acquire. Mora no ledger; mudanças entram em vigor imediatamente, sem reiniciar.", "settings_archive_not_configured": "Não configurado — os sandboxes esfriam no máximo até Parado e o disco deles continua ocupando espaço local.", diff --git a/packages/console/messages/ru/settings.json b/packages/console/messages/ru/settings.json index 9c796535..3d57c3d5 100644 --- a/packages/console/messages/ru/settings.json +++ b/packages/console/messages/ru/settings.json @@ -84,22 +84,23 @@ "settings_policy_archive_label": "Архивация после простоя (сек)", "settings_policy_archive_desc": "«Остановлена → В архиве»: диск сжимается и выгружается в S3, локально ничего не занимает.", "settings_upgrade_dialog_title": "Обновление Dormice", - "settings_upgrade_dialog_desc": "Повторно запускает install.sh на этом сервере (повторный запуск и есть обновление); весь процесс занимает несколько минут.", + "settings_upgrade_dialog_desc": "Обновляет весь флот: сначала install.sh перезапускается на машине шлюза (перезапуск и есть обновление), а когда шлюз возвращается с новой сборкой, остальные узлы следуют за ним по одному, каждый при своём следующем чек-ине. Несколько минут в сумме, больше при большем числе узлов.", "settings_upgrade_step1": "Забрать свежий код и пересобрать; длительность зависит от сети сервера.", - "settings_upgrade_step2_pre": "После успешной сборки daemon перезапускается: ", + "settings_upgrade_step2_pre": "После успешной сборки шлюз и узел на этой машине перезапускаются: ", "settings_upgrade_step2_em": "открытые терминалы, выполняющиеся команды и наблюдатели файлов отключатся", "settings_upgrade_step2_post": "; песочницы и диски не пострадают, дальше всё подхватит сверка.", "settings_upgrade_step3": "Если сборка не удалась, код автоматически откатывается на текущую версию и пересобирается; daemon не перезапускается и продолжает работать.", - "settings_upgrade_step4": "Во время перезапуска консоль ненадолго теряет связь; это окно дождётся её возвращения с новой версией.", + "settings_upgrade_step4": "Во время перезапуска консоль ненадолго теряет соединение; этот диалог ждёт, пока шлюз вернётся с новой версией.", + "settings_upgrade_step5": "Остальные узлы обновляются по одному: каждый получает указание при следующем чек-ине, сам запускает install.sh, перезапускается на несколько десятков секунд и отмечается с новой сборкой, прежде чем указание получит следующий. Узел, не вернувшийся через 20 минут, помечается как «застрявший» и остаётся за вами; сам он никогда не повторяет попытку.", "settings_upgrade_launch_failed": "Не удалось запустить обновление: {error}", "settings_upgrade_start": "Начать обновление", "settings_upgrade_watch_running": "Обновление выполняется", "settings_upgrade_watch_succeeded": "Обновление завершено", "settings_upgrade_watch_rolled_back": "Обновление не удалось, выполнен откат", "settings_upgrade_watch_failed": "Обновление не удалось", - "settings_upgrade_desc_unreachable": "Daemon перезапускается; кратковременная потеря связи — ожидаемая часть обновления, ждём его возвращения. Если ожидание затянулось, зайдите по ssh и посмотрите journalctl -u dormice-upgrade.", + "settings_upgrade_desc_unreachable": "Шлюз перезапускается; кратковременная потеря соединения — ожидаемая часть обновления, и мы ждём его возвращения. Если это длится долго, зайдите по ssh и посмотрите journalctl -u dormice-upgrade.", "settings_upgrade_desc_running": "Обновление выполняется в systemd-юните на сервере; закрытие этого окна его не прервёт.", - "settings_upgrade_success_1": "Daemon вернулся с новой версией и прошёл проверку doctor", + "settings_upgrade_success_1": "Шлюз вернулся с новой версией и прошёл проверку doctor", "settings_upgrade_success_running": ", сейчас работает", "settings_upgrade_success_2": ". Эта страница всё ещё старая консоль; нажмите «Перезагрузить консоль», чтобы загрузить новую.", "settings_upgrade_desc_rolled_back": "Сборка не удалась; код откатился на предыдущую версию и был пересобран. Daemon не перезапускался и работает как прежде. Устраните причину и попробуйте снова.", @@ -115,7 +116,7 @@ "settings_upgrade_more_commits": "не показаны более ранние коммиты: {n}", "settings_upgrade_ignore": "Пропустить эту версию", "settings_upgrade_go": "Обновить", - "settings_version_card_desc": "Сборка, на которой сейчас работает daemon, и наличие более новой в удалённом main", + "settings_version_card_desc": "Сборка, которую выполняет шлюз, есть ли более новая в удалённом main, и догнали ли её все узлы", "settings_check_updates": "Проверить обновления", "settings_upgrade_running_banner": "Идёт обновление", "settings_view_progress": "Показать прогресс", @@ -133,10 +134,25 @@ "settings_up_to_date_cached": " (кэшированный результат; нажмите «Проверить обновления» для повторной проверки)", "settings_upgradable_badge": "Есть обновление", "settings_upgrade_to_latest": "Обновить до последней", - "settings_oneclick_unavailable": "Обновление в один клик недоступно на этом daemon", + "settings_oneclick_unavailable": "Обновление в один клик недоступно на машине этого шлюза", "settings_paren_open": " (", "settings_paren_close": ")", "settings_oneclick_manual": "; для обновления повторно запустите install.sh на сервере (повторный запуск и есть обновление, API-токен не меняется).", + "settings_nodes_title": "Узлы", + "settings_nodes_desc": "Каждый узел сравнивается со сборкой шлюза: та же = догнал; другая и способен обновиться сам = получит указание при чек-ине, когда придёт его очередь.", + "settings_nodes_col_node": "Узел", + "settings_nodes_col_build": "Сборка", + "settings_nodes_col_state": "Состояние", + "settings_nodes_state_current": "Актуален", + "settings_nodes_state_behind": "Ожидает", + "settings_nodes_state_upgrading": "Обновляется", + "settings_nodes_state_stuck": "Застрял", + "settings_nodes_state_unavailable": "Не обновляется сам", + "settings_nodes_state_unreachable": "Недоступен", + "settings_nodes_state_unknown": "Неизвестно", + "settings_nodes_retry": "Повторить", + "settings_nodes_retry_done": "Узлу {id} дано указание; он обновится снова при следующем чек-ине", + "settings_nodes_retry_failed": "Не удалось дать указание узлу {id}: {error}", "settings_archive_card_title": "Хранилище архивов (S3)", "settings_archive_card_desc": "Диски простаивающих песочниц сжимаются и уходят в любое S3-совместимое хранилище, локально не занимая ничего, а при следующем acquire восстанавливаются. Хранится в реестре; изменения действуют сразу, без перезапуска.", "settings_archive_not_configured": "Не настроено — песочницы остывают только до «Остановлена», и их диски продолжают занимать место локально.", diff --git a/packages/console/messages/zh-CN/settings.json b/packages/console/messages/zh-CN/settings.json index 389f3961..7be8005d 100644 --- a/packages/console/messages/zh-CN/settings.json +++ b/packages/console/messages/zh-CN/settings.json @@ -84,22 +84,23 @@ "settings_policy_archive_label": "空闲多久后归档(秒)", "settings_policy_archive_desc": "已停止 → 已归档:磁盘压缩上传 S3,本地零占用。", "settings_upgrade_dialog_title": "升级 Dormice", - "settings_upgrade_dialog_desc": "在这台服务器上重跑 install.sh(重跑即升级),全程约几分钟。", + "settings_upgrade_dialog_desc": "升级整个舰队:先在网关机上重跑 install.sh(重跑即升级),网关带着新版本回来后,其余节点在各自的下一次报到时被逐台带起。全程约几分钟,节点多则更久。", "settings_upgrade_step1": "拉取最新代码并重新构建,耗时取决于服务器网络。", - "settings_upgrade_step2_pre": "构建成功后 daemon 重启:", + "settings_upgrade_step2_pre": "构建成功后网关与这台机器上的节点重启:", "settings_upgrade_step2_em": "进行中的终端、命令执行、文件监听会断开", "settings_upgrade_step2_post": ";沙箱与磁盘不受影响,对账器会接管现场。", "settings_upgrade_step3": "构建失败会自动回退到当前版本重新构建,daemon 不重启, 照常服务。", - "settings_upgrade_step4": "重启期间控制台会短暂失联,这个弹窗会等它带着新版本回来。", + "settings_upgrade_step4": "重启期间控制台会短暂失联,这个弹窗会等网关带着新版本回来。", + "settings_upgrade_step5": "其余节点一次只升一台:每台在下一次报到时被告知、自己跑 install.sh、重启几十秒后带着新版本报到,再轮到下一台。20 分钟还没回来的节点标为「卡住」交给人处理,不会自动重试。", "settings_upgrade_launch_failed": "无法发起升级:{error}", "settings_upgrade_start": "开始升级", "settings_upgrade_watch_running": "升级进行中", "settings_upgrade_watch_succeeded": "升级完成", "settings_upgrade_watch_rolled_back": "升级失败,已自动回退", "settings_upgrade_watch_failed": "升级失败", - "settings_upgrade_desc_unreachable": "daemon 正在重启,短暂失联是升级的预期环节;正在等它回来。如果长时间停在这里,ssh 上去看 journalctl -u dormice-upgrade。", + "settings_upgrade_desc_unreachable": "网关正在重启,短暂失联是升级的预期环节;正在等它回来。如果长时间停在这里,ssh 上去看 journalctl -u dormice-upgrade。", "settings_upgrade_desc_running": "升级在服务器的 systemd unit 里执行,关掉弹窗也不会中断。", - "settings_upgrade_success_1": "daemon 已带着新版本回来并通过 doctor 验收", + "settings_upgrade_success_1": "网关已带着新版本回来并通过 doctor 验收", "settings_upgrade_success_running": ",当前运行", "settings_upgrade_success_2": "。当前页面还是旧版控制台,点「刷新控制台」加载新版。", "settings_upgrade_desc_rolled_back": "构建失败,代码已回退到升级前的版本并重新构建,daemon 未重启、照常服务。修复原因后可再试。", @@ -115,7 +116,7 @@ "settings_upgrade_more_commits": "还有 {n} 个更早的提交未列出", "settings_upgrade_ignore": "忽略此版本", "settings_upgrade_go": "去升级", - "settings_version_card_desc": "daemon 当前运行的构建,以及远端 main 上有没有更新的版本", + "settings_version_card_desc": "网关当前运行的构建、远端 main 上有没有更新的版本,以及各节点跟上了没有", "settings_check_updates": "检查更新", "settings_upgrade_running_banner": "升级正在进行", "settings_view_progress": "查看进度", @@ -133,10 +134,25 @@ "settings_up_to_date_cached": "(缓存结果,点「检查更新」重测)", "settings_upgradable_badge": "可升级", "settings_upgrade_to_latest": "升级到最新", - "settings_oneclick_unavailable": "一键升级在这台 daemon 上不可用", + "settings_oneclick_unavailable": "一键升级在这台网关机上不可用", "settings_paren_open": "(", "settings_paren_close": ")", "settings_oneclick_manual": ";在服务器上重跑 install.sh 即可升级(重跑即升级,不轮换 API token)。", + "settings_nodes_title": "各节点", + "settings_nodes_desc": "按网关的构建判每台节点:一样=已跟上;不一样且能自升=轮到它时在报到里被带起。", + "settings_nodes_col_node": "节点", + "settings_nodes_col_build": "构建", + "settings_nodes_col_state": "状态", + "settings_nodes_state_current": "已跟上", + "settings_nodes_state_behind": "待升级", + "settings_nodes_state_upgrading": "升级中", + "settings_nodes_state_stuck": "卡住", + "settings_nodes_state_unavailable": "不能自升", + "settings_nodes_state_unreachable": "不可达", + "settings_nodes_state_unknown": "未知", + "settings_nodes_retry": "再试一次", + "settings_nodes_retry_done": "已通知节点 {id},它在下一次报到时重新升级", + "settings_nodes_retry_failed": "无法通知节点 {id}:{error}", "settings_archive_card_title": "归档存储(S3)", "settings_archive_card_desc": "闲置沙箱的磁盘压缩上传到 S3 兼容存储、本地零占用,下次 acquire 自动恢复。住在账本里,改了立即生效,不用重启。", "settings_archive_not_configured": "未配置 — 沙箱最冷停在「已停止」,磁盘一直占本地空间。", diff --git a/packages/console/messages/zh-TW/settings.json b/packages/console/messages/zh-TW/settings.json index 7b05f074..643d7a53 100644 --- a/packages/console/messages/zh-TW/settings.json +++ b/packages/console/messages/zh-TW/settings.json @@ -84,22 +84,23 @@ "settings_policy_archive_label": "閒置多久後封存(秒)", "settings_policy_archive_desc": "已停止 → 已封存:磁碟壓縮上傳 S3,本機零佔用。", "settings_upgrade_dialog_title": "升級 Dormice", - "settings_upgrade_dialog_desc": "在這台伺服器上重跑 install.sh(重跑即升級),全程約幾分鐘。", + "settings_upgrade_dialog_desc": "升級整個艦隊:先在閘道機上重跑 install.sh(重跑即升級),閘道帶著新版本回來後,其餘節點在各自的下一次報到時被逐台帶起。全程約幾分鐘,節點多則更久。", "settings_upgrade_step1": "拉取最新程式碼並重新建置,耗時取決於伺服器網路。", - "settings_upgrade_step2_pre": "建置成功後 daemon 重啟:", + "settings_upgrade_step2_pre": "建置成功後閘道與這台機器上的節點重啟:", "settings_upgrade_step2_em": "進行中的終端機、指令執行、檔案監聽會中斷", "settings_upgrade_step2_post": ";沙箱與磁碟不受影響,對帳器會接管現場。", "settings_upgrade_step3": "建置失敗會自動回退到目前版本重新建置,daemon 不重啟,照常服務。", - "settings_upgrade_step4": "重啟期間主控台會短暫失聯,這個對話框會等它帶著新版本回來。", + "settings_upgrade_step4": "重啟期間控制台會短暫失聯,這個對話框會等閘道帶著新版本回來。", + "settings_upgrade_step5": "其餘節點一次只升一台:每台在下一次報到時被告知、自己跑 install.sh、重啟幾十秒後帶著新版本報到,再輪到下一台。20 分鐘還沒回來的節點標為「卡住」交給人處理,不會自動重試。", "settings_upgrade_launch_failed": "無法發起升級:{error}", "settings_upgrade_start": "開始升級", "settings_upgrade_watch_running": "升級進行中", "settings_upgrade_watch_succeeded": "升級完成", "settings_upgrade_watch_rolled_back": "升級失敗,已自動回退", "settings_upgrade_watch_failed": "升級失敗", - "settings_upgrade_desc_unreachable": "daemon 正在重啟,短暫失聯是升級的預期環節;正在等它回來。如果長時間停在這裡,ssh 上去看 journalctl -u dormice-upgrade。", + "settings_upgrade_desc_unreachable": "閘道正在重啟,短暫失聯是升級的預期環節;正在等它回來。如果長時間停在這裡,ssh 上去看 journalctl -u dormice-upgrade。", "settings_upgrade_desc_running": "升級在伺服器的 systemd unit 裡執行,關掉對話框也不會中斷。", - "settings_upgrade_success_1": "daemon 已帶著新版本回來並通過 doctor 驗收", + "settings_upgrade_success_1": "閘道已帶著新版本回來並通過 doctor 驗收", "settings_upgrade_success_running": ",目前執行", "settings_upgrade_success_2": "。目前頁面還是舊版主控台,點「重新載入主控台」載入新版。", "settings_upgrade_desc_rolled_back": "建置失敗,程式碼已回退到升級前的版本並重新建置,daemon 未重啟、照常服務。修復原因後可再試。", @@ -115,7 +116,7 @@ "settings_upgrade_more_commits": "還有 {n} 個更早的提交未列出", "settings_upgrade_ignore": "忽略此版本", "settings_upgrade_go": "去升級", - "settings_version_card_desc": "daemon 目前執行的建置,以及遠端 main 上有沒有更新的版本", + "settings_version_card_desc": "閘道目前執行的建置、遠端 main 上有沒有更新的版本,以及各節點跟上了沒有", "settings_check_updates": "檢查更新", "settings_upgrade_running_banner": "升級正在進行", "settings_view_progress": "查看進度", @@ -133,10 +134,25 @@ "settings_up_to_date_cached": "(快取結果,點「檢查更新」重測)", "settings_upgradable_badge": "可升級", "settings_upgrade_to_latest": "升級到最新", - "settings_oneclick_unavailable": "一鍵升級在這台 daemon 上不可用", + "settings_oneclick_unavailable": "一鍵升級在這台閘道機上不可用", "settings_paren_open": "(", "settings_paren_close": ")", "settings_oneclick_manual": ";在伺服器上重跑 install.sh 即可升級(重跑即升級,不輪替 API token)。", + "settings_nodes_title": "各節點", + "settings_nodes_desc": "按閘道的建置判每台節點:一樣=已跟上;不一樣且能自升=輪到它時在報到裡被帶起。", + "settings_nodes_col_node": "節點", + "settings_nodes_col_build": "建置", + "settings_nodes_col_state": "狀態", + "settings_nodes_state_current": "已跟上", + "settings_nodes_state_behind": "待升級", + "settings_nodes_state_upgrading": "升級中", + "settings_nodes_state_stuck": "卡住", + "settings_nodes_state_unavailable": "不能自升", + "settings_nodes_state_unreachable": "不可達", + "settings_nodes_state_unknown": "未知", + "settings_nodes_retry": "再試一次", + "settings_nodes_retry_done": "已通知節點 {id},它在下一次報到時重新升級", + "settings_nodes_retry_failed": "無法通知節點 {id}:{error}", "settings_archive_card_title": "封存儲存(S3)", "settings_archive_card_desc": "閒置沙箱的磁碟壓縮上傳到 S3 相容儲存、本機零佔用,下次 acquire 自動還原。住在帳本裡,改了立即生效,不用重新啟動。", "settings_archive_not_configured": "未設定 — 沙箱最冷停在「已停止」,磁碟一直佔本機空間。", diff --git a/packages/console/src/features/settings/components/UpgradeDialog.tsx b/packages/console/src/features/settings/components/UpgradeDialog.tsx index 0c29b9af..af694c35 100644 --- a/packages/console/src/features/settings/components/UpgradeDialog.tsx +++ b/packages/console/src/features/settings/components/UpgradeDialog.tsx @@ -15,12 +15,14 @@ import { ApiError, applyUpgrade, getUpgradeStatus } from '@/lib/api'; import { m } from '@/paraglide/messages'; /** - * 一键升级的两幕:确认(把会发生什么说全 — 包括 daemon 重启会打断 - * 进行中的终端/exec/watch,以及构建失败自动回退)→ 观察(2 秒轮询 + * 一键升级的两幕:确认(把会发生什么说全 — 包括网关机重启会打断 + * 进行中的终端/exec/watch,构建失败自动回退,以及 2026-09-15 刀 4 起 + * 其余节点在报到里被逐台带起、20 分钟不回来标卡住)→ 观察(2 秒轮询 * getUpgradeStatus,实时滚日志)。轮询不用查询库而是自己的 setTimeout - * 链:daemon 重启造成的失联是升级的预期环节,要显示「重启中」继续等, + * 链:网关重启造成的失联是升级的预期环节,要显示「重启中」继续等, * 而不是当错误处理。终局以 install.sh 写下的报告为准 — succeeded 意味 - * 着 daemon 已带着新版本回来并通过 doctor,不是"脚本跑完了"。 + * 着网关已带着新版本回来并通过 doctor,不是"脚本跑完了";节点的滚动 + * 在版本卡的节点表里继续画,不归这个弹窗。 * * 防「上一次的旧报告」误判靠报告身份:发起前记下现存报告的 startedAt * 作基线,只接受非基线的终局 — applyUpgrade 刚返回时读到的还是上一轮 @@ -159,6 +161,7 @@ export function UpgradeDialog({
  • {m.settings_upgrade_step3()}
  • {m.settings_upgrade_step4()}
    • +
  • {m.settings_upgrade_step5()}
    • {launchError !== null && (

    diff --git a/packages/console/src/features/settings/components/VersionCard.tsx b/packages/console/src/features/settings/components/VersionCard.tsx index 7a03c073..fa63687e 100644 --- a/packages/console/src/features/settings/components/VersionCard.tsx +++ b/packages/console/src/features/settings/components/VersionCard.tsx @@ -1,8 +1,9 @@ -import type { CheckUpgradeResponse } from '@dormice/shared'; +import type { CheckUpgradeResponse, NodeUpgradeView } from '@dormice/shared'; import { RefreshIcon } from '@hugeicons/core-free-icons'; import { HugeiconsIcon } from '@hugeicons/react'; -import { useQueryClient } from '@tanstack/react-query'; +import { useMutation, useQueryClient } from '@tanstack/react-query'; import { useEffect, useRef, useState } from 'react'; +import { toast } from 'sonner'; import { Badge } from '@/components/ui/badge'; import { Button } from '@/components/ui/button'; import { @@ -14,6 +15,15 @@ import { CardTitle, } from '@/components/ui/card'; import { Spinner } from '@/components/ui/spinner'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { applyUpgrade } from '@/lib/api'; import { formatDateTime } from '@/lib/datetime'; import { m } from '@/paraglide/messages'; import { @@ -24,11 +34,16 @@ import { import { UpgradeDialog } from './UpgradeDialog'; /** - * daemon 自己的版本与升级窗口。「版本」= 构建进 dist 的 git commit - * (还没有发版 tag,main 上每个提交都过验收链);比较由 daemon 服务端 - * 裁决(upgradable 字段),这里只负责显示 — 与沙箱「可升级」徽章同一 - * 纪律。检查失败如实显示 checkError,绝不把失败装成「已是最新」。 - * 「升级」按钮只在 daemon 自报一键可用时出现,否则给手动路径与原因。 + * 网关的版本与舰队的升级窗口(2026-09-15 刀 4 起主语是网关:三动词在 + * 门口答,当前=网关构建,「升级」=先升网关机、再在报到里逐台带起节点)。 + * 「版本」= 构建进 dist 的 git commit(还没有发版 tag,main 上每个提交 + * 都过验收链);比较由服务端裁决(upgradable 字段),这里只负责显示 — + * 与沙箱「可升级」徽章同一纪律。检查失败如实显示 checkError,绝不把 + * 失败装成「已是最新」。「升级」按钮只在网关自报一键可用时出现,否则 + * 给手动路径与原因。卡片下方一张节点表:每台对着网关构建的站位,由 + * 网关裁决(state 字段);「卡住」的行给「再试一次」= applyUpgrade + * {nodeId},人工重告 — 网关自己绝不重告(构建总失败的节点不能每 20 + * 分钟白烤一遍)。 */ export function VersionCard() { const { data, isPending, isError, error } = useCheckUpgrade(); @@ -128,6 +143,9 @@ export function VersionCard() { } /> )} + {status.data?.nodes !== undefined && status.data.nodes.length > 0 && ( + + )} ); } + +const STATE_LABEL: Record string> = { + current: m.settings_nodes_state_current, + behind: m.settings_nodes_state_behind, + upgrading: m.settings_nodes_state_upgrading, + stuck: m.settings_nodes_state_stuck, + unavailable: m.settings_nodes_state_unavailable, + unreachable: m.settings_nodes_state_unreachable, + unknown: m.settings_nodes_state_unknown, +}; + +/** 徽章色阶:跟上=静;待升/升级中=琥珀(在动);卡住=红(要人);其余=灰(说明在 title)。 */ +function stateClass(state: NodeUpgradeView['state']): string { + switch (state) { + case 'behind': + case 'upgrading': + return 'border-amber-500/40 bg-amber-500/10 text-amber-600 dark:text-amber-400'; + case 'stuck': + return 'border-destructive/40 bg-destructive/10 text-destructive'; + default: + return ''; + } +} + +/** + * 节点表:网关裁决的站位,原话进 title。只有「卡住」给动作 — 待升的 + * 会自己轮到,升级中的在跑,不能自升/不可达/未知的原因都在 reason 里, + * 按钮改不了物理事实。 + */ +function NodesTable({ nodes }: { nodes: NodeUpgradeView[] }) { + const queryClient = useQueryClient(); + const retell = useMutation({ + mutationFn: (id: string) => applyUpgrade(id), + onSuccess: (_data, id) => { + toast.success(m.settings_nodes_retry_done({ id })); + void queryClient.invalidateQueries({ queryKey: ['upgradeStatus'] }); + }, + onError: (error, id) => { + toast.error( + m.settings_nodes_retry_failed({ + id, + error: error instanceof Error ? error.message : String(error), + }), + ); + }, + }); + return ( +

    +
    +
    {m.settings_nodes_title()}
    +

    + {m.settings_nodes_desc()} +

    +
    +
    + + + + {m.settings_nodes_col_node()} + {m.settings_nodes_col_build()} + {m.settings_nodes_col_state()} + + + + + {nodes.map((node) => ( + + {node.id} + + {node.build === null ? ( + + {m.common_unknown()} + + ) : ( + + {node.build.commit} + + )} + + + + {node.state === 'upgrading' && } + {STATE_LABEL[node.state]()} + + + + {node.state === 'stuck' && ( + + )} + + + ))} + +
    +
    +
    + ); +} diff --git a/packages/console/src/features/settings/hooks/useUpgrade.ts b/packages/console/src/features/settings/hooks/useUpgrade.ts index 6c2636f1..904f33d1 100644 --- a/packages/console/src/features/settings/hooks/useUpgrade.ts +++ b/packages/console/src/features/settings/hooks/useUpgrade.ts @@ -28,11 +28,13 @@ export function useForceCheckUpgrade() { /** * 升级执行窗:一键升级可不可用、systemd unit 是否活着、上一次运行的 - * 报告。全是本机读数(systemctl + 状态文件),不打网络 — 版本卡拿它 - * 决定「升级」按钮还是手动指引。升级弹窗里的 2 秒轮询是弹窗自己的 - * setTimeout 链(daemon 重启的失联是预期环节,查询库的重试语义不合身); - * 这里只在有升级在跑时自轮询 — 横幅与「上次运行」的结局要能自愈, - * 不能指望弹窗一直开着。 + * 报告,以及(2026-09-15 刀 4 起)各节点对着网关构建的站位。网关机 + * 本机读数(systemctl + 状态文件)加报到里的内存,不打网络 — 版本卡 + * 拿它决定「升级」按钮还是手动指引,并画节点表。升级弹窗里的 2 秒 + * 轮询是弹窗自己的 setTimeout 链(网关重启的失联是预期环节,查询库 + * 的重试语义不合身);这里在有升级在跑、或有节点待升/升级中时自轮询 + * — 横幅、「上次运行」的结局与节点表的滚动都要能自愈,不能指望弹窗 + * 一直开着。 */ export function useUpgradeStatus() { return useQuery({ @@ -40,7 +42,14 @@ export function useUpgradeStatus() { queryFn: getUpgradeStatus, staleTime: 15_000, retry: false, - refetchInterval: (query) => (query.state.data?.running ? 5000 : false), + refetchInterval: (query) => { + const data = query.state.data; + if (data === undefined) return false; + const rolling = data.nodes?.some( + (n) => n.state === 'behind' || n.state === 'upgrading', + ); + return data.running || rolling ? 5000 : false; + }, }); } diff --git a/packages/console/src/lib/api.ts b/packages/console/src/lib/api.ts index 9200945e..43515395 100644 --- a/packages/console/src/lib/api.ts +++ b/packages/console/src/lib/api.ts @@ -261,8 +261,14 @@ export const checkUpgrade = (force = false) => // The one-click upgrade: the daemon hands install.sh to a systemd unit // that outlives its own restart, then answers { started: true }. Progress // lives in getUpgradeStatus; expect the daemon to restart near the end. -export const applyUpgrade = () => - rpc('/applyUpgrade', {}); +// Without a node: the gateway's machine upgrades and the fleet rolls +// behind it. With one: the operator tells that node again (a node the +// status lists as stuck). +export const applyUpgrade = (nodeId?: string) => + rpc( + '/applyUpgrade', + nodeId === undefined ? {} : { nodeId }, + ); // The upgrade execution window: availability, unit liveness (from systemd, // not the status file's claim), the last run's report and the log tail. diff --git a/packages/gateway/drizzle/0005_fleet-upgrade.sql b/packages/gateway/drizzle/0005_fleet-upgrade.sql new file mode 100644 index 00000000..834ef887 --- /dev/null +++ b/packages/gateway/drizzle/0005_fleet-upgrade.sql @@ -0,0 +1,2 @@ +ALTER TABLE `nodes` ADD `self_upgrade` text;--> statement-breakpoint +ALTER TABLE `nodes` ADD `upgrade_told_at` text; \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/0005_snapshot.json b/packages/gateway/drizzle/meta/0005_snapshot.json new file mode 100644 index 00000000..cbf9e1ac --- /dev/null +++ b/packages/gateway/drizzle/meta/0005_snapshot.json @@ -0,0 +1,502 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "310aabe3-0483-455e-ab72-060b6292d2a6", + "prevId": "54d3f360-1488-4db9-99a7-2d393ba2dd1f", + "tables": { + "api_keys": { + "name": "api_keys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + "key_hash" + ], + "isUnique": true + }, + "api_keys_active_name_idx": { + "name": "api_keys_active_name_idx", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"api_keys\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "console_account": { + "name": "console_account", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_secret": { + "name": "session_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "fleet_state_samples": { + "name": "fleet_state_samples", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active": { + "name": "active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "frozen": { + "name": "frozen", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stopped": { + "name": "stopped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "restoring": { + "name": "restoring", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "fleet_state_samples_at_idx": { + "name": "fleet_state_samples_at_idx", + "columns": [ + "at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "nodes": { + "name": "nodes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "added_at": { + "name": "added_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_gb": { + "name": "swap_gb", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_check_in_at": { + "name": "last_check_in_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "interval_seconds": { + "name": "interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config_version": { + "name": "config_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "build": { + "name": "build", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reading": { + "name": "reading", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "self_upgrade": { + "name": "self_upgrade", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "upgrade_told_at": { + "name": "upgrade_told_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_cpus": { + "name": "sandbox_cpus", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_memory_gb": { + "name": "sandbox_memory_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_disk_gb": { + "name": "sandbox_disk_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_freeze_after_seconds": { + "name": "default_freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_stop_after_seconds": { + "name": "default_stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_archive_after_seconds": { + "name": "default_archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_endpoint": { + "name": "s3_endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_access_key_id": { + "name": "s3_access_key_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_secret_access_key": { + "name": "s3_secret_access_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_region": { + "name": "s3_region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_force_path_style": { + "name": "s3_force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain": { + "name": "sandbox_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain_aliases": { + "name": "sandbox_domain_aliases", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pids_limit": { + "name": "pids_limit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_image": { + "name": "base_image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registry_address": { + "name": "registry_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/_journal.json b/packages/gateway/drizzle/meta/_journal.json index 7b1be0a2..b616232d 100644 --- a/packages/gateway/drizzle/meta/_journal.json +++ b/packages/gateway/drizzle/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1789459862989, "tag": "0004_fleet-base-image", "breakpoints": true + }, + { + "idx": 5, + "version": "6", + "when": 1789461434618, + "tag": "0005_fleet-upgrade", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/gateway/src/app.test.ts b/packages/gateway/src/app.test.ts index 2e71c244..9fe0ff10 100644 --- a/packages/gateway/src/app.test.ts +++ b/packages/gateway/src/app.test.ts @@ -1203,11 +1203,11 @@ describe('using, destroying, and the cache', () => { expect(a.lookups()).toBe(before + 1); }); - it('the upgrade verbs are an honest 501, a misspelled verb a 404, a body without a name a 400', async () => { + it("the upgrade verbs answer at the door (the gateway's own, routes/upgrade.test.ts), a misspelled verb is a 404, a body without a name a 400", async () => { const h = await gateway(['a']); const upgrade = await rpc(h, '/checkUpgrade'); - expect(upgrade.status).toBe(501); - expect(message(upgrade)).toContain('call the node directly'); + expect(upgrade.status).toBe(200); + expect(upgrade.body).toMatchObject({ current: null, check: null }); expect((await rpc(h, '/acquireSandbx', { name: 'x' })).status).toBe(404); expect((await rpc(h, '/execCommand', { command: 'x' })).status).toBe(400); expect(h.nodes[0]?.hits).toEqual([]); @@ -1880,13 +1880,16 @@ describe('the fleet-wide lists and the by-node readings', () => { expect(message(refused)).toContain('no node has checked in yet'); }); - it('the upgrade verbs alone are still an honest 501', async () => { + it("the upgrade verbs are the gateway's own now, not a node's: no node is asked", async () => { const h = await gateway(['b']); - for (const verb of ['checkUpgrade', 'applyUpgrade', 'getUpgradeStatus']) { - const r = await rpc(h, `/${verb}`, {}); - expect(r.status).toBe(501); - expect(message(r)).toContain('until the upgrade cut'); - } + const before = h.nodes[0]?.hits.length ?? 0; + const s = await rpc(h, '/getUpgradeStatus', {}); + expect(s.status).toBe(200); + expect(s.body).toMatchObject({ + available: false, + nodes: [{ id: 'b', state: 'unknown' }], + }); + expect(h.nodes[0]?.hits.length ?? 0).toBe(before); }); }); diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts index d6bed52d..93e5a0f5 100644 --- a/packages/gateway/src/app.ts +++ b/packages/gateway/src/app.ts @@ -1,5 +1,8 @@ import http from 'node:http'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; import type { KeyedQueue } from '@dormice/server/keyed-queue'; +import { Updater } from '@dormice/server/updater'; import { sandboxDomainsInForce } from '@dormice/shared'; import fastifyCookie from '@fastify/cookie'; import fastify, { @@ -28,6 +31,7 @@ import type { Fleet } from './fleet'; import type { Ingress } from './ingress'; import type { PlacementKnobs } from './placement'; import { createRawFaces } from './raw'; +import { Rolling } from './rolling'; import { apiKeyRoutes } from './routes/api-keys'; import { consoleRoutes } from './routes/console'; import { e2bControlRoutes } from './routes/e2b'; @@ -39,6 +43,7 @@ import { checkInRoutes, nodeRoutes } from './routes/nodes'; import { observeRoutes } from './routes/observe'; import { settingsRoutes } from './routes/settings'; import { templateRoutes } from './routes/templates'; +import { upgradeRoutes } from './routes/upgrade'; import { type BuildInfo, readBuildInfo } from './version'; export interface GatewayAppDeps { @@ -79,6 +84,14 @@ export interface GatewayAppDeps { * Defaults to reading process.env; tests inject a fixed map. */ sources?: ConfigSources; + /** + * The gateway machine's upgrade window (the daemon's Updater over this + * checkout; routes/upgrade.ts). main.ts injects one that knows the + * checkout; the default knows none, so checkUpgrade answers an honest + * checkError and applyUpgrade refuses — tests never reach the network + * or systemd by accident. + */ + updater?: Updater; } type SettingsProbe = NonNullable< @@ -124,10 +137,18 @@ export function buildGatewayApp({ probeS3, ask, sources = configSources(), + updater = new Updater({ + repoDir: null, + build, + statusDir: path.join(tmpdir(), 'dormice-gateway-upgrade'), + }), }: GatewayAppDeps) { const loggerInstance = typeof logger === 'boolean' ? pino({ enabled: logger }) : logger; const token = config.DORMICE_API_TOKEN; + // The fleet upgrade's live half, judged against this gateway's build: + // the check-ins ask it, the upgrade routes read and steer it. + const rolling = new Rolling(fleet, build); // The faces keyed on a header sit in front of Fastify, exactly as the // daemon's port proxy does (server/app.ts): refuse what is not an @@ -271,7 +292,7 @@ export function buildGatewayApp({ await reply.code(401).send({ message: 'missing or invalid API token' }); } }); - await nodesFace.register(checkInRoutes, { fleet, db }); + await nodesFace.register(checkInRoutes, { fleet, db, rolling }); }); // The sandbox gate: everything that addresses a sandbox — and the @@ -301,6 +322,7 @@ export function buildGatewayApp({ }); await admin.register(templateRoutes, { db, fleet, ask: askVerb }); await admin.register(ingressRoutes, { ingress }); + await admin.register(upgradeRoutes, { updater, fleet, rolling }); }); // The web console: account + session endpoints (open — setup and login diff --git a/packages/gateway/src/db/schema.ts b/packages/gateway/src/db/schema.ts index c978352d..20aed4ea 100644 --- a/packages/gateway/src/db/schema.ts +++ b/packages/gateway/src/db/schema.ts @@ -64,6 +64,15 @@ export const nodes = sqliteTable('nodes', { build: text('build'), /** JSON, shared nodeReadingSchema; null until the first check-in. */ reading: text('reading'), + /** JSON `{available, reason}` — whether the node can upgrade itself, its own word at its last check-in; null = it did not say (a build before the fourth cut) or never checked in. */ + selfUpgrade: text('self_upgrade'), + /** + * ISO 8601 UTC — when the fleet upgrade last told this node to upgrade + * itself (rolling.ts); null = never, or the tell was fulfilled (the node + * came back on the gateway's build). On the row so a gateway restart + * mid-roll neither forgets a node it told nor tells it twice. + */ + upgradeToldAt: text('upgrade_told_at'), }); export type NodeRow = typeof nodes.$inferSelect; diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 12c93324..6c924273 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -8,7 +8,7 @@ import { type SandboxStateCounts, } from '@dormice/shared'; import { eq } from 'drizzle-orm'; -import type { z } from 'zod'; +import { z } from 'zod'; import type { Db } from './db/db'; import { type NodeRow, nodes } from './db/schema'; import { bumpConfigVersion } from './db/settings'; @@ -42,10 +42,21 @@ export interface NodeState { intervalSeconds: number | null; build: BuildInfo | null; reading: NodeReading | null; + /** Whether the node can upgrade itself, its own word (shared checkInRequestSchema.selfUpgrade); null = it did not say. */ + selfUpgrade: SelfUpgrade | null; + /** When the fleet upgrade last told this node to upgrade (rolling.ts); null = never, or fulfilled. */ + upgradeToldAt: Date | null; placedSinceCheckIn: number; placedIds: Set; } +export type SelfUpgrade = NonNullable; + +const selfUpgradeSchema = z.object({ + available: z.boolean(), + reason: z.string().nullable(), +}); + /** * What the fleet says for itself — a row it could not read back, a row it * could not write. Pino's shape (main.ts passes the gateway's logger); @@ -179,6 +190,7 @@ type RowShare = Pick< | 'configVersion' | 'build' | 'reading' + | 'selfUpgrade' >; /** @@ -232,11 +244,24 @@ export class Fleet { row.reading, nodeReadingSchema, ), + selfUpgrade: this.parseJson( + row.id, + 'selfUpgrade', + row.selfUpgrade, + selfUpgradeSchema, + ), + upgradeToldAt: this.parseDate(row.upgradeToldAt), placedSinceCheckIn: 0, placedIds: new Set(), }; } + private parseDate(iso: string | null): Date | null { + if (iso === null) return null; + const date = new Date(iso); + return Number.isNaN(date.getTime()) ? null : date; + } + private parseJson( nodeId: string, column: string, @@ -301,6 +326,10 @@ export class Fleet { configVersion: report.configVersion, build: report.build === null ? null : JSON.stringify(report.build), reading: JSON.stringify(report.reading), + selfUpgrade: + report.selfUpgrade === undefined + ? null + : JSON.stringify(report.selfUpgrade), }; if (node === undefined) { const addedAt = now.toISOString(); @@ -318,6 +347,8 @@ export class Fleet { intervalSeconds: null, build: null, reading: null, + selfUpgrade: null, + upgradeToldAt: null, placedSinceCheckIn: 0, placedIds: new Set(), }; @@ -348,11 +379,31 @@ export class Fleet { node.build = report.build; node.reading = report.reading; node.configVersion = report.configVersion; + node.selfUpgrade = report.selfUpgrade ?? null; node.placedSinceCheckIn = 0; node.placedIds.clear(); return { node, joined, movedFrom }; } + /** + * The fleet upgrade's one mark on a node: when it was told to upgrade, + * or null once the tell is fulfilled (rolling.ts). Written through, + * not best-effort — the tell rides on the check-in's answer, and a + * gateway that forgot it told a node would tell it again after a + * restart, the one thing the rolling upgrade promises not to do; a + * write that fails fails the check-in, and the node is told at the next. + */ + setUpgradeToldAt(id: string, at: Date | null): void { + const node = this.members.get(id); + if (node === undefined) return; + this.db + .update(nodes) + .set({ upgradeToldAt: at === null ? null : at.toISOString() }) + .where(eq(nodes.id, id)) + .run(); + node.upgradeToldAt = at; + } + /** * The row's share of a check-in, best-effort: a write that fails — a * full disk, a file gone read-only — is said once, and the check-in is diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts index 92a1de99..d354366e 100644 --- a/packages/gateway/src/main.ts +++ b/packages/gateway/src/main.ts @@ -1,8 +1,11 @@ import { existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { KeyedQueue } from '@dormice/server/keyed-queue'; import { acquireSingleWriterLock } from '@dormice/server/lock'; import { closeWithGrace, trackConnections } from '@dormice/server/shutdown'; +import { Updater } from '@dormice/server/updater'; import { pino } from 'pino'; import { z } from 'zod'; import { buildGatewayApp } from './app'; @@ -84,6 +87,29 @@ log.info( : 'dormice-gateway build: no version identity (built outside a git checkout)', ); +// The gateway machine's upgrade window — the daemon's Updater over the +// same checkout (dist/main.js sits at packages/gateway/dist, three hops +// under the repo root, as the daemon's does). applyUpgrade at the door +// runs install.sh here, which restarts this gateway and the node beside +// it; the other nodes follow at their check-ins (rolling.ts). The run +// reports beside the gateway's database; an in-memory database is not an +// install, and one-click is honestly off there. +const repoRoot = fileURLToPath(new URL('../../..', import.meta.url)); +const inMemory = config.DORMICE_GATEWAY_DB_PATH === ':memory:'; +const updater = new Updater({ + repoDir: existsSync(path.join(repoRoot, '.git')) ? repoRoot : null, + build, + statusDir: inMemory + ? path.join(tmpdir(), 'dormice-gateway-upgrade') + : path.join(path.dirname(config.DORMICE_GATEWAY_DB_PATH), 'upgrade'), + ...(inMemory + ? { + unavailable: + 'the gateway runs on an in-memory database — not an install that install.sh can upgrade', + } + : {}), +}); + // The managed front door, present exactly when the knob names a file // (the archiver precedent). The upstream is this gateway: the fleet's one // public Caddy sits in front of the one door. @@ -123,6 +149,7 @@ const app = buildGatewayApp({ build, consoleDistDir: existsSync(consoleDistDir) ? consoleDistDir : undefined, ingress, + updater, }); // Same red line as the daemon: loopback only, host not configurable — the diff --git a/packages/gateway/src/rolling.test.ts b/packages/gateway/src/rolling.test.ts new file mode 100644 index 00000000..9e451267 --- /dev/null +++ b/packages/gateway/src/rolling.test.ts @@ -0,0 +1,293 @@ +import { fileURLToPath } from 'node:url'; +import type { BuildInfo } from '@dormice/shared'; +import { describe, expect, it } from 'vitest'; +import { migrateDb, openDb } from './db/db'; +import { nodes } from './db/schema'; +import { Fleet, type NodeState } from './fleet'; +import { + Rolling, + rollingDecision, + UPGRADE_TOLD_TIMEOUT_MS, + upgradeStateOf, +} from './rolling'; +import { checkInOf } from './testing'; + +// The fleet upgrade's rules, pure: every state a node can stand in +// against the gateway's build, and who is told when. The wire (the +// check-in's answer, the routes) is routes/upgrade.test.ts's. + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const NOW = new Date('2026-09-15T12:00:00.000Z'); +const GATEWAY: BuildInfo = { + commit: 'new0001', + title: 'the new build', + committedAt: '2026-09-15T00:00:00.000Z', +}; +const OLD: BuildInfo = { + commit: 'old0001', + title: 'the old build', + committedAt: '2026-09-14T00:00:00.000Z', +}; +const CAN = { available: true, reason: null }; +const CANNOT = { + available: false, + reason: + 'systemd-run is not available — one-click upgrade needs a systemd host', +}; + +function fleetOver() { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + return { db, fleet: new Fleet(db) }; +} + +function reporting( + fleet: Fleet, + id: string, + over: Parameters[2] = {}, + at = NOW, +): NodeState { + const outcome = fleet.checkIn(checkInOf(id, `http://${id}:80`, over), at); + if ('refused' in outcome) throw new Error(outcome.refused); + return outcome.node; +} + +describe('upgradeStateOf', () => { + it('current on the same commit; behind on another when the node can upgrade itself and was not told', () => { + const { fleet } = fleetOver(); + const same = reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: CAN }); + expect(upgradeStateOf(same, GATEWAY, NOW)).toEqual({ + state: 'current', + reason: null, + }); + const behind = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }); + expect(upgradeStateOf(behind, GATEWAY, NOW)).toEqual({ + state: 'behind', + reason: null, + }); + }); + + it('unknown when either side has no build identity; a row that never checked in says so', () => { + const { db, fleet } = fleetOver(); + const node = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); + expect(upgradeStateOf(node, null, NOW)).toMatchObject({ + state: 'unknown', + reason: expect.stringMatching(/gateway carries no build identity/), + }); + const bare = reporting(fleet, 'b', { build: null, selfUpgrade: CAN }); + expect(upgradeStateOf(bare, GATEWAY, NOW)).toMatchObject({ + state: 'unknown', + reason: expect.stringMatching(/node reports no build identity/), + }); + db.insert(nodes) + .values({ id: 'n', endpoint: 'http://n:80', addedAt: NOW.toISOString() }) + .run(); + const never = new Fleet(db).get('n'); + if (!never) throw new Error('row lost'); + expect(upgradeStateOf(never, GATEWAY, NOW)).toEqual({ + state: 'unknown', + reason: 'the node has never checked in', + }); + }); + + it('unreachable outranks behind and current: a node two of its intervals silent is not judged on its build — unless it was told, whose silence is the restart', () => { + const { fleet } = fleetOver(); + const later = new Date(NOW.getTime() + 31_000); + const node = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); + expect(upgradeStateOf(node, GATEWAY, later)).toEqual({ + state: 'unreachable', + reason: 'has not checked in for 31s', + }); + const current = reporting(fleet, 'b', { build: GATEWAY, selfUpgrade: CAN }); + expect(upgradeStateOf(current, GATEWAY, later).state).toBe('unreachable'); + fleet.setUpgradeToldAt('a', NOW); + expect(upgradeStateOf(node, GATEWAY, later)).toEqual({ + state: 'upgrading', + reason: 'told 31s ago, still on old0001 (has not checked in for 31s)', + }); + }); + + it("unavailable, with the node's own reason, when it cannot upgrade itself — or did not say", () => { + const { fleet } = fleetOver(); + const cannot = reporting(fleet, 'a', { build: OLD, selfUpgrade: CANNOT }); + expect(upgradeStateOf(cannot, GATEWAY, NOW)).toEqual({ + state: 'unavailable', + reason: `${CANNOT.reason} — run install.sh on it`, + }); + const silent = reporting(fleet, 'b', { build: OLD }); + expect(upgradeStateOf(silent, GATEWAY, NOW)).toMatchObject({ + state: 'unavailable', + reason: expect.stringMatching( + /runs old0001 and does not say whether it can upgrade itself/, + ), + }); + }); + + it('told: upgrading within the timeout, stuck past it — with when it was told, what it still runs, and where to look', () => { + const { fleet } = fleetOver(); + const node = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); + fleet.setUpgradeToldAt('a', NOW); + const soon = new Date(NOW.getTime() + 90_000); + // Still checking in during its build: upgrading, plainly. + reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }, soon); + expect(upgradeStateOf(node, GATEWAY, soon)).toEqual({ + state: 'upgrading', + reason: 'told 90s ago, still on old0001', + }); + const late = new Date(NOW.getTime() + UPGRADE_TOLD_TIMEOUT_MS); + reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }, late); + expect(upgradeStateOf(node, GATEWAY, late)).toMatchObject({ + state: 'stuck', + reason: expect.stringMatching( + /^told to upgrade at 2026-09-15T12:00:00\.000Z and still on old0001 20 minutes later — read journalctl -u dormice-upgrade/, + ), + }); + // Back on the gateway's build: current, whatever the tell says. + reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: CAN }, late); + expect(upgradeStateOf(node, GATEWAY, late).state).toBe('current'); + }); +}); + +describe('rollingDecision', () => { + it('tells a node behind when no other is upgrading; not while one is; a stuck one does not hold the pointer', () => { + const { fleet } = fleetOver(); + const a = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); + const b = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }); + const all = fleet.all(); + expect(rollingDecision(all, GATEWAY, a, NOW)).toBe(true); + fleet.setUpgradeToldAt('a', NOW); + expect(rollingDecision(all, GATEWAY, b, NOW)).toBe(false); + // a restarts near the end of its upgrade and misses check-ins: still + // upgrading, and b still waits. + const restarting = new Date(NOW.getTime() + 60_000); + reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }, restarting); + expect(rollingDecision(all, GATEWAY, b, restarting)).toBe(false); + // Twenty minutes on, a is stuck, not upgrading: b's turn comes. + const late = new Date(NOW.getTime() + UPGRADE_TOLD_TIMEOUT_MS); + reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }, late); + expect(rollingDecision(all, GATEWAY, b, late)).toBe(true); + // And a stuck node is never told again on its own. + reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }, late); + expect(rollingDecision(all, GATEWAY, a, late)).toBe(false); + }); + + it('never tells a node that is current, unavailable, unreachable or unknown', () => { + const { fleet } = fleetOver(); + const current = reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: CAN }); + const cannot = reporting(fleet, 'b', { build: OLD, selfUpgrade: CANNOT }); + const bare = reporting(fleet, 'c', { build: null, selfUpgrade: CAN }); + const gone = reporting(fleet, 'd', { build: OLD, selfUpgrade: CAN }); + gone.lastCheckInAt = new Date(NOW.getTime() - 40_000); + for (const node of [current, cannot, bare, gone]) { + expect(rollingDecision(fleet.all(), GATEWAY, node, NOW)).toBe(false); + } + expect(rollingDecision(fleet.all(), null, cannot, NOW)).toBe(false); + }); +}); + +describe('Rolling', () => { + it('onCheckIn tells once and writes the tell to the row; a fulfilled tell is cleared; a restart remembers who was told', () => { + const { db, fleet } = fleetOver(); + const rolling = new Rolling(fleet, GATEWAY); + const a = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); + const b = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }); + expect(rolling.onCheckIn(a, NOW)).toBe(true); + expect(a.upgradeToldAt).toEqual(NOW); + expect(rolling.onCheckIn(b, NOW)).toBe(false); + // a's next check-in, still old: not told again. + const later = new Date(NOW.getTime() + 15_000); + reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }, later); + expect(rolling.onCheckIn(a, later)).toBe(false); + // The row remembers across a restart. + const restarted = new Fleet(db); + expect(restarted.get('a')?.upgradeToldAt).toEqual(NOW); + expect(new Rolling(restarted, GATEWAY).states(later)).toMatchObject([ + { id: 'a', state: 'upgrading', toldAt: NOW.toISOString() }, + { id: 'b', state: 'behind', toldAt: null }, + ]); + // a comes back on the new build: its tell is fulfilled and cleared, + // and b's turn comes at its next check-in. + reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: CAN }, later); + expect(rolling.onCheckIn(a, later)).toBe(false); + expect(a.upgradeToldAt).toBeNull(); + expect(new Fleet(db).get('a')?.upgradeToldAt).toBeNull(); + expect(rolling.onCheckIn(b, later)).toBe(true); + }); + + it("requestRetell honors the operator's hand at the next check-in even while another node upgrades, and refuses in words where it would do nothing", () => { + const { fleet } = fleetOver(); + const rolling = new Rolling(fleet, GATEWAY); + const a = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); + const b = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }); + expect(rolling.onCheckIn(a, NOW)).toBe(true); + // b is behind and a is upgrading: b would wait — unless the operator says so. + expect(rolling.onCheckIn(b, NOW)).toBe(false); + expect(rolling.requestRetell(b, NOW)).toBeNull(); + expect(rolling.onCheckIn(b, NOW)).toBe(true); + // A stuck node is the case the hand exists for. + const late = new Date(NOW.getTime() + UPGRADE_TOLD_TIMEOUT_MS); + reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }, late); + expect(upgradeStateOf(a, GATEWAY, late).state).toBe('stuck'); + expect(rolling.onCheckIn(a, late)).toBe(false); + expect(rolling.requestRetell(a, late)).toBeNull(); + expect(rolling.onCheckIn(a, late)).toBe(true); + expect(a.upgradeToldAt).toEqual(late); + + const current = reporting( + fleet, + 'c', + { build: GATEWAY, selfUpgrade: CAN }, + late, + ); + expect(rolling.requestRetell(current, late)).toMatchObject({ + status: 400, + message: expect.stringMatching(/already runs the gateway's build/), + }); + const cannot = reporting( + fleet, + 'd', + { build: OLD, selfUpgrade: CANNOT }, + late, + ); + expect(rolling.requestRetell(cannot, late)).toMatchObject({ + status: 400, + message: expect.stringMatching(/cannot be told to upgrade: systemd-run/), + }); + const gone = reporting(fleet, 'e', { build: OLD, selfUpgrade: CAN }, late); + gone.lastCheckInAt = new Date(late.getTime() - 40_000); + expect(rolling.requestRetell(gone, late)).toMatchObject({ + status: 409, + message: expect.stringMatching(/not checking in/), + }); + expect(new Rolling(fleet, null).requestRetell(a, late)).toMatchObject({ + status: 400, + message: expect.stringMatching(/gateway carries no build identity/), + }); + }); + + it('states lists every node in id order with its standing, build and tell', () => { + const { fleet } = fleetOver(); + const rolling = new Rolling(fleet, GATEWAY); + reporting(fleet, 'c', { build: GATEWAY, selfUpgrade: CAN }); + const a = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); + reporting(fleet, 'b', { build: OLD }); + expect(rolling.onCheckIn(a, NOW)).toBe(true); + expect(rolling.states(new Date(NOW.getTime() + 5_000))).toEqual([ + { + id: 'a', + build: OLD, + state: 'upgrading', + toldAt: NOW.toISOString(), + reason: 'told 5s ago, still on old0001', + }, + { + id: 'b', + build: OLD, + state: 'unavailable', + toldAt: null, + reason: expect.stringMatching(/does not say whether it can upgrade/), + }, + { id: 'c', build: GATEWAY, state: 'current', toldAt: null, reason: null }, + ]); + }); +}); diff --git a/packages/gateway/src/rolling.ts b/packages/gateway/src/rolling.ts new file mode 100644 index 00000000..3750bc66 --- /dev/null +++ b/packages/gateway/src/rolling.ts @@ -0,0 +1,217 @@ +import type { + BuildInfo, + NodeUpgradeState, + NodeUpgradeView, +} from '@dormice/shared'; +import { downReason, type Fleet, type NodeState } from './fleet'; + +/** + * The fleet upgrade, rolled over the nodes by their check-ins (design + * record #32: the gateway changes a node's target, the node upgrades + * itself when it sees it, and the next node's turn comes when it is back + * on the new build). The gateway's machine upgrades first — install.sh + * there restarts its gateway and its node together — and from then on + * every node that reports another build than the gateway's is behind. + * + * Told at a check-in, one node at a time: the answer carries `upgrade: + * true` (shared checkInResponseSchema), the node runs its own updater + * (install.sh in a systemd unit, back on the new build within minutes), + * and no other node is told while one is upgrading — the fleet loses one + * node's sandboxes for a few tens of seconds, never two nodes' at once. + * + * Told once. A node still on the old build UPGRADE_TOLD_TIMEOUT_MS after + * its tell is stuck: named as such with where to look, and never re-told + * on its own — a node whose build fails every time would otherwise + * rebuild every twenty minutes, on the CPU its sandboxes run on. The + * pointer moves past it (a stuck node is not "upgrading"), and the + * operator's applyUpgrade {nodeId} is the hand that tells it again, + * whatever the rolling order says at that moment. The tell is on the + * node's row (nodes.upgrade_told_at), so a gateway restart mid-roll + * neither forgets a node it told nor tells it twice; the operator's + * re-tell is memory — a gateway restarted before the node's next check-in + * forgets it, and the operator clicks again. + */ + +/** How long a told node has to come back on the new build before it is stuck: a pull, a build and a restart take a few minutes; twenty is a build that failed. */ +export const UPGRADE_TOLD_TIMEOUT_MS = 20 * 60_000; + +/** + * One node's standing against the gateway's build, from its last + * check-in (shared upgrade.ts nodeUpgradeViewSchema has each state's + * meaning). Pure: the same inputs give the same answer, and the suite + * walks every branch. + */ +export function upgradeStateOf( + node: NodeState, + gatewayBuild: BuildInfo | null, + now: Date, +): { state: NodeUpgradeState; reason: string | null } { + if (gatewayBuild === null) { + return { + state: 'unknown', + reason: + 'the gateway carries no build identity (built outside a git checkout) — nothing to compare the node against', + }; + } + if (node.build === null) { + return { + state: 'unknown', + reason: + node.lastCheckInAt === null + ? 'the node has never checked in' + : 'the node reports no build identity (built outside a git checkout)', + }; + } + const down = downReason(node, now); + if (node.build.commit === gatewayBuild.commit) { + return down === null + ? { state: 'current', reason: null } + : { state: 'unreachable', reason: down }; + } + // A told node is upgrading or stuck whether or not it is checking in: + // its daemon restarts near the end of install.sh and misses a check-in + // or two by design, and were that silence read as "unreachable" the + // one-at-a-time rule would see nobody upgrading and tell the next node + // into the same minute. The silence is said in the reason instead. + if (node.upgradeToldAt !== null) { + const sinceMs = now.getTime() - node.upgradeToldAt.getTime(); + const silence = down === null ? '' : ` (${down})`; + if (sinceMs < UPGRADE_TOLD_TIMEOUT_MS) { + return { + state: 'upgrading', + reason: `told ${Math.round(sinceMs / 1000)}s ago, still on ${node.build.commit}${silence}`, + }; + } + return { + state: 'stuck', + reason: `told to upgrade at ${node.upgradeToldAt.toISOString()} and still on ${node.build.commit} ${Math.round(sinceMs / 60_000)} minutes later${silence} — read journalctl -u dormice-upgrade and the upgrade log on the node, then tell it again (applyUpgrade with its nodeId)`, + }; + } + if (down !== null) { + return { state: 'unreachable', reason: down }; + } + if (node.selfUpgrade === null) { + return { + state: 'unavailable', + reason: `the node runs ${node.build.commit} and does not say whether it can upgrade itself (a build from before the fleet upgrade) — run install.sh on it`, + }; + } + if (!node.selfUpgrade.available) { + return { + state: 'unavailable', + reason: `${node.selfUpgrade.reason ?? 'the node cannot upgrade itself'} — run install.sh on it`, + }; + } + return { state: 'behind', reason: null }; +} + +/** + * Whether this node, checking in now, is told to upgrade: it is behind + * (upgradeStateOf), and no other node is upgrading right now — the one at + * a time rule. Pure, over the fleet's current standings. + */ +export function rollingDecision( + nodes: readonly NodeState[], + gatewayBuild: BuildInfo | null, + node: NodeState, + now: Date, +): boolean { + if (upgradeStateOf(node, gatewayBuild, now).state !== 'behind') return false; + return !nodes.some( + (other) => + other.id !== node.id && + upgradeStateOf(other, gatewayBuild, now).state === 'upgrading', + ); +} + +/** + * The fleet upgrade's live half: the gateway's build to judge against, + * the operator's pending re-tells, and the check-in's verdicts — one + * object the check-in route and the upgrade routes share. + */ +export class Rolling { + /** Nodes the operator told to upgrade again (applyUpgrade {nodeId}), told at their next check-in whatever the order says. Memory: see the module comment. */ + private readonly retell = new Set(); + + constructor( + private readonly fleet: Fleet, + private readonly gatewayBuild: BuildInfo | null, + ) {} + + /** + * The check-in's verdict for a node that just reported, in order: a + * fulfilled tell is cleared (the node is back on the gateway's build); + * a pending re-tell is honored; otherwise the rolling rule decides. Any + * tell is written to the row before the answer carries it. Answers + * whether the node is told now. + */ + onCheckIn(node: NodeState, now: Date): boolean { + const { state } = upgradeStateOf(node, this.gatewayBuild, now); + if (state === 'current' && node.upgradeToldAt !== null) { + this.fleet.setUpgradeToldAt(node.id, null); + this.retell.delete(node.id); + return false; + } + const tell = + (this.retell.has(node.id) && + (state === 'behind' || state === 'stuck' || state === 'upgrading')) || + rollingDecision(this.fleet.all(), this.gatewayBuild, node, now); + if (!tell) return false; + this.fleet.setUpgradeToldAt(node.id, now); + this.retell.delete(node.id); + return true; + } + + /** + * The operator's re-tell (applyUpgrade {nodeId}): honored at the node's + * next check-in. Refused in words when it would do nothing — a node on + * the gateway's build, one that cannot upgrade itself, one whose build + * is unknown — and told to wait for an unreachable one; a node merely + * behind or upgrading is taken too (the operator's hand outranks the + * order). Answers the refusal, or null when the re-tell is pending. + */ + requestRetell( + node: NodeState, + now: Date, + ): { status: 400 | 409; message: string } | null { + const { state, reason } = upgradeStateOf(node, this.gatewayBuild, now); + switch (state) { + case 'current': + return { + status: 400, + message: `node ${node.id} already runs the gateway's build (${node.build?.commit ?? 'unknown'}) — nothing to upgrade`, + }; + case 'unavailable': + case 'unknown': + return { + status: 400, + message: `node ${node.id} cannot be told to upgrade: ${reason ?? state}`, + }; + case 'unreachable': + return { + status: 409, + message: `node ${node.id} is not checking in (${reason}) — it is told at its next check-in; retry once it is back, or remove it if it is gone for good`, + }; + default: + this.retell.add(node.id); + return null; + } + } + + /** Every node's standing right now (getUpgradeStatus.nodes), in node-id order. */ + states(now: Date): NodeUpgradeView[] { + return this.fleet + .all() + .sort((a, b) => a.id.localeCompare(b.id)) + .map((node) => { + const { state, reason } = upgradeStateOf(node, this.gatewayBuild, now); + return { + id: node.id, + build: node.build, + state, + toldAt: node.upgradeToldAt?.toISOString() ?? null, + reason, + }; + }); + } +} diff --git a/packages/gateway/src/routes/native.ts b/packages/gateway/src/routes/native.ts index 56947e4f..7e598aac 100644 --- a/packages/gateway/src/routes/native.ts +++ b/packages/gateway/src/routes/native.ts @@ -58,20 +58,6 @@ export const BY_NODE_VERBS = [ 'getHostMetricsHistory', ] as const; -/** - * The verbs that address the daemon and that the gateway does not route - * yet: the upgrade, whose fleet-wide form (the gateway upgrades itself, - * then rolls the nodes one at a time) is the fourth cut's. Until then - * each answers an honest 501 naming the alternative. (The lists merged - * and the host readings went by node in the third cut; keys, settings, - * templates and ingress left with the configuration authority.) - */ -export const UNNAMED_VERBS = [ - 'checkUpgrade', - 'applyUpgrade', - 'getUpgradeStatus', -] as const; - export interface NativeRoutesOptions { fleet: Fleet; finder: Finder; @@ -93,14 +79,6 @@ export const nativeRoutes: FastifyPluginAsyncZod = async ( (_request, body, done) => done(null, body), ); - for (const verb of UNNAMED_VERBS) { - app.post(`/${verb}`, async (_request, reply) => - reply.code(501).send({ - message: `${verb} is not routed by the gateway until the upgrade cut — call the node directly`, - }), - ); - } - for (const verb of BY_NODE_VERBS) { app.post(`/${verb}`, async (request, reply) => { const body = request.body as Buffer | undefined; diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index dd44092f..cb1f659c 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -16,10 +16,13 @@ import { readNodeConfig } from '../db/node-config'; import { readConfigVersion } from '../db/settings'; import { downReason, type Fleet, type NodeState } from '../fleet'; import { RETRY_AFTER_SECONDS } from '../raw'; +import type { Rolling } from '../rolling'; export interface CheckInRoutesOptions { fleet: Fleet; db: Db; + /** The fleet upgrade's verdict for each check-in (rolling.ts). */ + rolling: Rolling; } export interface NodeRoutesOptions { @@ -37,12 +40,15 @@ function refusal(statusCode: number, message: string): Error { * own gate in app.ts: the fleet token and nothing else. The answer is the * configuration version, and the whole bundle when the node's differs * (design record #22, shared nodeConfigBundleSchema): the check-in is the - * pull. No record is kept of who was told what — the node states what it - * runs at every check-in, and the comparison is the whole protocol. + * pull. Of the configuration no record is kept of who was told what — + * the node states what it runs at every check-in, and the comparison is + * the whole protocol. The fleet upgrade rides the same answer (`upgrade: + * true`, rolling.ts) and is the one thing remembered: a node is told + * once, on its row. */ export const checkInRoutes: FastifyPluginAsyncZod< CheckInRoutesOptions -> = async (app, { fleet, db }) => { +> = async (app, { fleet, db, rolling }) => { /** * Per node, the ids it was last reported to share an endpoint with * (sorted, joined) — so the warning below is said when the situation @@ -124,6 +130,17 @@ export const checkInRoutes: FastifyPluginAsyncZod< 'the node no longer shares its endpoint with another', ); } + // The fleet upgrade's turn for this node, if it is its turn: said + // here, once — the tell is the event, and the node's own log has the + // run. + const told = rolling.onCheckIn(node, new Date()); + if (told) { + request.log.info( + { nodeId: node.id, build: node.build?.commit ?? null }, + 'the node is told to upgrade itself: it runs another build than the gateway and no other node is upgrading', + ); + } + const upgrade = told ? { upgrade: true as const } : {}; const version = readConfigVersion(db); const runs = request.body.configVersion; if (runs === version) { @@ -133,7 +150,7 @@ export const checkInRoutes: FastifyPluginAsyncZod< 'the node now runs the current configuration version', ); } - return { configVersion: version }; + return { configVersion: version, ...upgrade }; } const gap = `${String(runs)}→${version}`; if (bundleSaid.get(node.id) !== gap) { @@ -145,7 +162,11 @@ export const checkInRoutes: FastifyPluginAsyncZod< : 'a node runs another configuration version; the bundle rides on this answer', ); } - return { configVersion: version, config: readNodeConfig(db, node) }; + return { + configVersion: version, + config: readNodeConfig(db, node), + ...upgrade, + }; }, ); }; diff --git a/packages/gateway/src/routes/upgrade.test.ts b/packages/gateway/src/routes/upgrade.test.ts new file mode 100644 index 00000000..ee07a301 --- /dev/null +++ b/packages/gateway/src/routes/upgrade.test.ts @@ -0,0 +1,232 @@ +import type { BuildInfo } from '@dormice/shared'; +import { + checkInResponseSchema, + checkUpgradeResponseSchema, + getUpgradeStatusResponseSchema, +} from '@dormice/shared'; +import { describe, expect, it } from 'vitest'; +import { UPGRADE_TOLD_TIMEOUT_MS } from '../rolling'; +import { checkInOf, TEST_TOKEN, testGateway } from '../testing'; + +// The fleet upgrade over the wire: the check-in that carries the tell, +// the three verbs at the door. The rules themselves are rolling.test.ts's; +// the launch of install.sh is the daemon's updater suite's — here the +// gateway runs from no checkout, so applyUpgrade without a node is the +// honest 400 and nothing reaches systemd. + +const authed = { authorization: `Bearer ${TEST_TOKEN}` }; +type App = ReturnType['app']; + +const GATEWAY: BuildInfo = { + commit: 'new0001', + title: 'the new build', + committedAt: '2026-09-15T00:00:00.000Z', +}; +const OLD: BuildInfo = { + commit: 'old0001', + title: 'the old build', + committedAt: '2026-09-14T00:00:00.000Z', +}; +const CAN = { available: true, reason: null }; + +function rpc( + app: App, + url: string, + payload: object = {}, + headers: Record = authed, +) { + return app.inject({ method: 'POST', url, headers, payload }); +} + +async function checkIn( + app: App, + id: string, + over: Parameters[2] = {}, +) { + const res = await rpc( + app, + '/checkIn', + checkInOf(id, `http://${id}:80`, over), + ); + expect(res.statusCode).toBe(200); + return checkInResponseSchema.parse(res.json()); +} + +async function status(app: App) { + const res = await rpc(app, '/getUpgradeStatus'); + expect(res.statusCode).toBe(200); + return getUpgradeStatusResponseSchema.parse(res.json()); +} + +describe('the fleet upgrade over the check-in', () => { + it("a node on another build is told once, the next one waits, and the answer stops telling once the node is back on the gateway's build", async () => { + const { app, fleet } = testGateway({}, { build: GATEWAY }); + // Both behind. a checks in first: told. b: not while a upgrades. + expect( + (await checkIn(app, 'a', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBe(true); + expect( + (await checkIn(app, 'b', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBeUndefined(); + // a again, still old: told once, not twice. + expect( + (await checkIn(app, 'a', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBeUndefined(); + expect((await status(app)).nodes).toMatchObject([ + { id: 'a', state: 'upgrading' }, + { id: 'b', state: 'behind' }, + ]); + // a is back on the new build: current, and b's turn. + expect( + (await checkIn(app, 'a', { build: GATEWAY, selfUpgrade: CAN })).upgrade, + ).toBeUndefined(); + expect( + (await checkIn(app, 'b', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBe(true); + expect((await status(app)).nodes).toMatchObject([ + { id: 'a', state: 'current', toldAt: null }, + { id: 'b', state: 'upgrading' }, + ]); + // The tell rides beside a bundle when one is due, on the same answer. + const { app: app2 } = testGateway({}, { build: GATEWAY }); + const answer = await checkIn(app2, 'c', { + build: OLD, + selfUpgrade: CAN, + configVersion: null, + }); + expect(answer.config).toBeDefined(); + expect(answer.upgrade).toBe(true); + void fleet; + }); + + it('a node still old twenty minutes after its tell is stuck and is not re-told; the operator re-tells it, and it hears at its next check-in', async () => { + const { app, fleet } = testGateway({}, { build: GATEWAY }); + expect( + (await checkIn(app, 'a', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBe(true); + const a = fleet.get('a'); + if (!a) throw new Error('node lost'); + a.upgradeToldAt = new Date(Date.now() - UPGRADE_TOLD_TIMEOUT_MS - 1000); + expect( + (await checkIn(app, 'a', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBeUndefined(); + const stuck = (await status(app)).nodes?.find((n) => n.id === 'a'); + expect(stuck).toMatchObject({ + state: 'stuck', + reason: expect.stringMatching(/still on old0001 2\d minutes later/), + }); + // Meanwhile the pointer moved on: b is told. + expect( + (await checkIn(app, 'b', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBe(true); + // The operator's hand: a is told again at its next check-in, even + // while b upgrades. + const retold = await rpc(app, '/applyUpgrade', { nodeId: 'a' }); + expect(retold.statusCode).toBe(200); + expect(retold.json()).toEqual({ started: true }); + expect( + (await checkIn(app, 'a', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBe(true); + expect((await status(app)).nodes?.find((n) => n.id === 'a')?.state).toBe( + 'upgrading', + ); + }); + + it('nodes that cannot upgrade themselves, did not say, or carry no build are listed with the reason and never told', async () => { + const { app } = testGateway({}, { build: GATEWAY }); + expect( + ( + await checkIn(app, 'a', { + build: OLD, + selfUpgrade: { + available: false, + reason: 'the process does not run from a git checkout', + }, + }) + ).upgrade, + ).toBeUndefined(); + expect((await checkIn(app, 'b', { build: OLD })).upgrade).toBeUndefined(); + expect( + (await checkIn(app, 'c', { build: null, selfUpgrade: CAN })).upgrade, + ).toBeUndefined(); + expect((await status(app)).nodes).toEqual([ + expect.objectContaining({ + id: 'a', + state: 'unavailable', + reason: + 'the process does not run from a git checkout — run install.sh on it', + }), + expect.objectContaining({ + id: 'b', + state: 'unavailable', + reason: expect.stringMatching(/does not say whether it can upgrade/), + }), + expect.objectContaining({ id: 'c', state: 'unknown' }), + ]); + // A gateway without a build identity judges nobody. + const { app: bare } = testGateway(); + expect( + (await checkIn(bare, 'a', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBeUndefined(); + expect((await status(bare)).nodes).toEqual([ + expect.objectContaining({ + id: 'a', + state: 'unknown', + reason: expect.stringMatching(/gateway carries no build identity/), + }), + ]); + }); +}); + +describe('the upgrade verbs at the door', () => { + it("checkUpgrade answers the gateway's build and, running from no checkout, an honest checkError", async () => { + const { app } = testGateway({}, { build: GATEWAY }); + const res = await rpc(app, '/checkUpgrade', {}); + expect(res.statusCode).toBe(200); + expect(checkUpgradeResponseSchema.parse(res.json())).toEqual({ + current: GATEWAY, + check: null, + checkError: expect.stringMatching(/does not run from a git checkout/), + }); + }); + + it("getUpgradeStatus is the gateway machine's run plus the nodes; applyUpgrade without a node is the honest 400 here, with one names its refusals", async () => { + const { app } = testGateway({}, { build: GATEWAY }); + await checkIn(app, 'a', { build: GATEWAY, selfUpgrade: CAN }); + const s = await status(app); + expect(s).toMatchObject({ + available: false, + unavailableReason: expect.stringMatching(/git checkout/), + running: false, + last: null, + nodes: [{ id: 'a', state: 'current' }], + }); + const fleetWide = await rpc(app, '/applyUpgrade', {}); + expect(fleetWide.statusCode).toBe(400); + expect(fleetWide.json().message).toMatch(/one-click upgrade unavailable/); + expect( + (await rpc(app, '/applyUpgrade', { nodeId: 'ghost' })).statusCode, + ).toBe(404); + const current = await rpc(app, '/applyUpgrade', { nodeId: 'a' }); + expect(current.statusCode).toBe(400); + expect(current.json().message).toMatch(/already runs the gateway's build/); + expect((await rpc(app, '/applyUpgrade', { nodeId: '' })).statusCode).toBe( + 400, + ); + }); + + it('is admin-only: a minted key gets 403 on all three', async () => { + const { app } = testGateway({}, { build: GATEWAY }); + const minted = (await rpc(app, '/createApiKey', { name: 'ci' })).json(); + for (const verb of ['checkUpgrade', 'applyUpgrade', 'getUpgradeStatus']) { + const res = await rpc( + app, + `/${verb}`, + {}, + { authorization: `Bearer ${minted.token}` }, + ); + expect(res.statusCode).toBe(403); + } + expect((await rpc(app, '/getUpgradeStatus', {}, {})).statusCode).toBe(401); + }); +}); diff --git a/packages/gateway/src/routes/upgrade.ts b/packages/gateway/src/routes/upgrade.ts new file mode 100644 index 00000000..8a54f5a6 --- /dev/null +++ b/packages/gateway/src/routes/upgrade.ts @@ -0,0 +1,96 @@ +import type { Updater } from '@dormice/server/updater'; +import { + applyUpgradeRequestSchema, + applyUpgradeResponseSchema, + checkUpgradeRequestSchema, + checkUpgradeResponseSchema, + getUpgradeStatusRequestSchema, + getUpgradeStatusResponseSchema, +} from '@dormice/shared'; +import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import type { Fleet } from '../fleet'; +import { httpError } from '../http-error'; +import type { Rolling } from '../rolling'; + +export interface UpgradeRoutesOptions { + /** The gateway's own upgrade window over its machine's checkout (the daemon's Updater, through @dormice/server/updater). */ + updater: Updater; + fleet: Fleet; + rolling: Rolling; +} + +/** + * The fleet's upgrade surface, at the door (RULES/协议.md「舰队升级」): + * checkUpgrade compares the gateway's build against origin's main; + * applyUpgrade without a node upgrades the gateway's machine — install.sh + * in a systemd unit, the daemon's own mechanism, which restarts the + * gateway and its node together, and from then on the check-ins roll the + * upgrade over the other nodes (rolling.ts); applyUpgrade with a node is + * the operator's re-tell of one node; getUpgradeStatus is the gateway + * machine's run plus every node's standing. Behind the admin gate: an + * upgrade is the fleet's configuration in the largest sense, and a leaked + * automation key must not be able to restart every machine. + */ +export const upgradeRoutes: FastifyPluginAsyncZod< + UpgradeRoutesOptions +> = async (app, { updater, fleet, rolling }) => { + app.post( + '/checkUpgrade', + { + schema: { + body: checkUpgradeRequestSchema, + response: { 200: checkUpgradeResponseSchema }, + }, + }, + async (request) => updater.check(request.body.force), + ); + + app.post( + '/applyUpgrade', + { + schema: { + body: applyUpgradeRequestSchema, + response: { 200: applyUpgradeResponseSchema }, + }, + }, + async (request) => { + const { nodeId } = request.body; + if (nodeId === undefined) { + await updater.apply(); + request.log.info( + { from: updater.current?.commit ?? null }, + "fleet upgrade launched: install.sh runs on the gateway's machine (systemd unit dormice-upgrade); the other nodes are told at their check-ins once the gateway is back on the new build", + ); + return { started: true as const }; + } + const node = fleet.get(nodeId); + if (node === undefined) { + throw httpError( + 404, + `no node with id '${nodeId}' — listNodes shows which exist`, + ); + } + const refused = rolling.requestRetell(node, new Date()); + if (refused !== null) throw httpError(refused.status, refused.message); + request.log.info( + { nodeId, build: node.build?.commit ?? null }, + 'node told to upgrade again by the operator; it hears at its next check-in', + ); + return { started: true as const }; + }, + ); + + app.post( + '/getUpgradeStatus', + { + schema: { + body: getUpgradeStatusRequestSchema, + response: { 200: getUpgradeStatusResponseSchema }, + }, + }, + async () => ({ + ...(await updater.status()), + nodes: rolling.states(new Date()), + }), + ); +}; diff --git a/packages/gateway/src/testing.ts b/packages/gateway/src/testing.ts index 82ec8847..e7cef914 100644 --- a/packages/gateway/src/testing.ts +++ b/packages/gateway/src/testing.ts @@ -1,6 +1,6 @@ import { fileURLToPath } from 'node:url'; import { KeyedQueue } from '@dormice/server/keyed-queue'; -import type { CheckInRequest, NodeReading } from '@dormice/shared'; +import type { BuildInfo, CheckInRequest, NodeReading } from '@dormice/shared'; import { buildGatewayApp } from './app'; import { type AskNode, type AskVerb, httpAskNode } from './ask'; import { NameCache } from './cache'; @@ -73,6 +73,10 @@ export function checkInOf( over: Parameters[0] & { intervalSeconds?: number; configVersion?: number | null; + /** The build the node reports; the scaffolding's default, or null for a node built outside a checkout. */ + build?: BuildInfo | null; + /** Whether the node can upgrade itself; absent = the node does not say (a build before the fourth cut). */ + selfUpgrade?: CheckInRequest['selfUpgrade']; } = {}, ): CheckInRequest { return { @@ -82,12 +86,18 @@ export function checkInOf( // A configured node by default: placement refuses one without a copy, // and most suites are about nodes that run one. configVersion: over.configVersion === undefined ? 1 : over.configVersion, - build: { - commit: 'abc1234', - title: 'a commit', - committedAt: '2026-09-14T00:00:00.000Z', - }, + build: + over.build === undefined + ? { + commit: 'abc1234', + title: 'a commit', + committedAt: '2026-09-14T00:00:00.000Z', + } + : over.build, reading: reading(over), + ...(over.selfUpgrade === undefined + ? {} + : { selfUpgrade: over.selfUpgrade }), }; } @@ -107,6 +117,8 @@ export function testGateway( ingress?: Ingress; /** Forged by default: the suites here are about the settings machinery, not S3's availability. */ probeS3?: NonNullable[0]['probeS3']>; + /** The gateway's build identity (null by default, as a source run has none) — the fleet upgrade judges the nodes against it. */ + build?: BuildInfo | null; } = {}, ) { const db = openDb(':memory:'); @@ -135,7 +147,7 @@ export function testGateway( finder, locks: new KeyedQueue(), logger: false, - build: null, + build: opts.build ?? null, consoleDistDir: opts.consoleDistDir, ingress: opts.ingress, ask: opts.askVerb, diff --git a/packages/sdk/src/client.ts b/packages/sdk/src/client.ts index 0f0eeed2..0fcdf145 100644 --- a/packages/sdk/src/client.ts +++ b/packages/sdk/src/client.ts @@ -395,8 +395,13 @@ export class Dormice { * Expect the daemon to restart near the end: in-flight execs, terminals * and watchers break, sandboxes and their disks are untouched. */ - async applyUpgrade(): Promise { - const data = await this.rpc('applyUpgrade', {}); + async applyUpgrade(options?: { + /** At the gateway: tell this one node to upgrade again at its next check-in (a node the fleet upgrade lists as stuck). Absent: upgrade the gateway's machine and roll the fleet. */ + nodeId?: string; + }): Promise { + const data = await this.rpc('applyUpgrade', { + ...(options?.nodeId === undefined ? {} : { nodeId: options.nodeId }), + }); return applyUpgradeResponseSchema.parse(data); } diff --git a/packages/server/package.json b/packages/server/package.json index aafba06e..c69cc686 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -37,6 +37,10 @@ "./history": { "types": "./dist/history.d.ts", "default": "./dist/history.js" + }, + "./updater": { + "types": "./dist/updater.d.ts", + "default": "./dist/updater.js" } }, "files": [ diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index a626704a..4150c166 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -88,6 +88,20 @@ export interface AppDeps { * Building the app is separate from listening so tests can inject requests * without opening a port. */ +/** + * The daemon's own reason one-click is off: an upgrade is a real + * install's move (install.sh, systemd, a Docker host), never a test + * double's — an e2e daemon on the fake executor must not be able to + * re-run install.sh on a developer's machine. + */ +export function fakeExecutorUnavailable( + executor: 'fake' | 'docker', +): string | undefined { + return executor === 'docker' + ? undefined + : 'one-click upgrade is for a real install (docker executor) — this daemon runs the fake executor'; +} + export function buildApp({ config, db, @@ -100,7 +114,7 @@ export function buildApp({ repoDir: null, build: readBuildInfo(), statusDir: nodePath.join(config.DORMICE_DATA_DIR, 'upgrade'), - executor: config.DORMICE_EXECUTOR, + unavailable: fakeExecutorUnavailable(config.DORMICE_EXECUTOR), }), }: AppDeps) { // Always a pino instance (booleans are normalized into one): two fastify() diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index fe1baa4f..66a70865 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -384,6 +384,62 @@ describe('CheckIn', () => { ]); }); + it("reports whether this node can upgrade itself, and runs its upgrade when the answer says so — a launch that fails is a warning, not the tick's failure", async () => { + let sent = 0; + const gw = await gateway(() => { + sent += 1; + return { + status: 200, + body: JSON.stringify({ + configVersion: 1, + ...(sent === 2 || sent === 3 ? { upgrade: true } : {}), + }), + }; + }); + const { log, warns, infos, details } = logSpy(); + let launches = 0; + const opts = options(gw.endpoint, log, { + selfUpgrade: async () => ({ + available: false, + reason: 'systemd-run is not available', + }), + applyUpgrade: async () => { + launches += 1; + if (launches === 2) throw new Error('an upgrade is already running'); + }, + }); + applyNodeConfig(opts.db, testBundle({}, 1)); + const checkIn = new CheckIn(opts); + await checkIn.once(); + expect(checkInRequestSchema.parse(gw.seen[0]?.body).selfUpgrade).toEqual({ + available: false, + reason: 'systemd-run is not available', + }); + expect(launches).toBe(0); + // Told: the upgrade is launched, said as its own line. + await checkIn.once(); + expect(launches).toBe(1); + expect(infos).toEqual([ + expect.stringMatching(/this node's turn to upgrade has come/), + ]); + expect(warns).toEqual([]); + // Told again while one runs: the launch's 409 is a warning; the + // check-in itself succeeded. + await checkIn.once(); + expect(launches).toBe(2); + expect(warns).toEqual([expect.stringMatching(/could not be launched/)]); + expect((details[0] as { error: string }).error).toMatch(/already running/); + // Not told: nothing launched, and a check-in without an updater + // wired reports no selfUpgrade at all. + await checkIn.once(); + expect(launches).toBe(2); + const bare = new CheckIn(options(gw.endpoint, logSpy().log)); + await bare.once(); + expect( + checkInRequestSchema.parse(gw.seen.at(-1)?.body).selfUpgrade, + ).toBeUndefined(); + }); + it('ticks on its interval from start() and stops on stop()', async () => { const gw = await gateway(() => answering(1)); const { log } = logSpy(); diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index d91a8ce1..3f3b0008 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -57,6 +57,16 @@ export interface CheckInOptions { configVersion: () => number | null; /** Makes a bundle the gateway answered with real on this node (node-config.ts applyConfig). */ applyConfig: (bundle: NodeConfigBundle) => Promise; + /** + * Whether this node can upgrade itself when told, and why not (the + * updater's availability, updater.ts) — reported at every check-in so + * the gateway rolls the fleet upgrade over the nodes that can and names + * the rest. Optional for the suites that embed a check-in without an + * updater; the daemon always wires it. + */ + selfUpgrade?: () => Promise<{ available: boolean; reason: string | null }>; + /** Runs this node's own upgrade (updater.apply) when the gateway's answer says `upgrade: true`. */ + applyUpgrade?: () => Promise; log: CheckInLog; /** Test seam; production uses the platform's fetch. */ fetchImpl?: typeof fetch; @@ -101,6 +111,11 @@ function describe(error: unknown): string { return why === undefined ? message : `${message} (${why})`; } +/** The daemon always wires applyUpgrade; a suite that does not, told to upgrade, hears why nothing happened. */ +async function unavailableUpgrade(): Promise { + throw new Error('this check-in has no updater to run an upgrade with'); +} + export class CheckIn { private timer: NodeJS.Timeout | undefined; private closing = false; @@ -129,6 +144,9 @@ export class CheckIn { build: opts.build, reading: await opts.readReading(), configVersion: opts.configVersion(), + ...(opts.selfUpgrade === undefined + ? {} + : { selfUpgrade: await opts.selfUpgrade() }), }; const res = await (opts.fetchImpl ?? fetch)(`${opts.gateway}/checkIn`, { method: 'POST', @@ -178,6 +196,26 @@ export class CheckIn { ); } } + if (answer.upgrade === true) { + // The gateway's turn for this node in the fleet upgrade: run the + // same one-click upgrade an operator would (install.sh in a + // systemd unit, updater.ts). Said as its own line, not this tick's + // failure: the check-in itself succeeded, and the gateway tells a + // node once — a launch that fails here is the operator's to read + // (the gateway shows the node as stuck twenty minutes on, and + // applyUpgrade {nodeId} at the gateway tells it again). + opts.log.info( + `the gateway says this node's turn to upgrade has come — launching install.sh (systemd unit dormice-upgrade)`, + ); + try { + await (opts.applyUpgrade ?? unavailableUpgrade)(); + } catch (error) { + opts.log.warn( + { error: describe(error) }, + 'the upgrade the gateway asked for could not be launched; the gateway lists this node as stuck once twenty minutes have passed, and applyUpgrade {nodeId} there tells it again', + ); + } + } } catch (error) { const message = describe(error); const failure = message.replace(/\d+/g, '#'); diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index cf041245..17ff3407 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -2,7 +2,7 @@ import { existsSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { pino } from 'pino'; -import { buildApp } from './app'; +import { buildApp, fakeExecutorUnavailable } from './app'; import { Archiver } from './archive/archiver'; import { LedgerArchiveStore } from './archive/ledger-store'; import { CheckIn, readNodeReading } from './check-in'; @@ -206,6 +206,21 @@ const build = readBuildInfo(); // unknown; a sample a few milliseconds before it would make that first // reading a percentage over the sliver in between, near 0 or near 100 by // luck (found by review, 2026-09-14). +// The daemon's own upgrade window compares the commit baked into this +// build against the checkout it runs from — main.js sits at +// packages/server/dist (src/main.ts at packages/server/src: same depth), +// so three hops up is the repo root either way. No checkout (a dist +// copied elsewhere) means checking is honestly unavailable, not guessed. +// Built before the check-in: the check-in reports whether this node can +// upgrade itself, and runs the upgrade when the gateway says so. +const repoRoot = fileURLToPath(new URL('../../..', import.meta.url)); +const updater = new Updater({ + repoDir: existsSync(path.join(repoRoot, '.git')) ? repoRoot : null, + build, + statusDir: path.join(config.DORMICE_DATA_DIR, 'upgrade'), + unavailable: fakeExecutorUnavailable(config.DORMICE_EXECUTOR), +}); + const nodeEndpoint = config.DORMICE_NODE_ENDPOINT ?? `http://127.0.0.1:${config.DORMICE_PORT}`; const checkInCpu = new CpuSampler(); @@ -229,21 +244,13 @@ const checkIn = new CheckIn({ beat, baseImageFallback: config.DORMICE_BASE_IMAGE, }), + selfUpgrade: async () => { + const reason = await updater.availability(); + return { available: reason === null, reason }; + }, + applyUpgrade: () => updater.apply(), log, }); - -// The daemon's own upgrade window compares the commit baked into this -// build against the checkout it runs from — main.js sits at -// packages/server/dist (src/main.ts at packages/server/src: same depth), -// so three hops up is the repo root either way. No checkout (a dist -// copied elsewhere) means checking is honestly unavailable, not guessed. -const repoRoot = fileURLToPath(new URL('../../..', import.meta.url)); -const updater = new Updater({ - repoDir: existsSync(path.join(repoRoot, '.git')) ? repoRoot : null, - build, - statusDir: path.join(config.DORMICE_DATA_DIR, 'upgrade'), - executor: config.DORMICE_EXECUTOR, -}); log.info( build ? `dormice build ${build.commit} (${build.title})` diff --git a/packages/server/src/updater.test.ts b/packages/server/src/updater.test.ts index fa7f74d1..4a5763fb 100644 --- a/packages/server/src/updater.test.ts +++ b/packages/server/src/updater.test.ts @@ -7,6 +7,7 @@ import { } from '@dormice/shared'; import { execaSync } from 'execa'; import { beforeAll, describe, expect, it } from 'vitest'; +import { fakeExecutorUnavailable } from './app'; import { type RunCommand, Updater, type UpdaterOptions } from './updater'; import type { BuildInfo } from './version'; @@ -62,7 +63,6 @@ function updaterFor(overrides: Partial = {}): Updater { repoDir: clone, build: installedBuild, statusDir: mkdtempSync(path.join(tmpdir(), 'dormice-status-')), - executor: 'docker', run: okRun, ...overrides, }); @@ -173,15 +173,20 @@ describe('Updater.check', () => { }); describe('Updater.apply and status', () => { - it('refuses one-click on the fake executor and without a checkout', async () => { - const fake = updaterFor({ executor: 'fake' }); + it("refuses one-click for the caller's own reason (the daemon's fake executor) and without a checkout; availability() is the same word the check-in reports", async () => { + const fake = updaterFor({ + unavailable: fakeExecutorUnavailable('fake'), + }); await expect(fake.apply()).rejects.toMatchObject({ statusCode: 400 }); const status = await fake.status(); expect(status.available).toBe(false); expect(status.unavailableReason).toMatch(/fake executor/); + expect(await fake.availability()).toBe(status.unavailableReason); + expect(fakeExecutorUnavailable('docker')).toBeUndefined(); const noRepo = updaterFor({ repoDir: null }); expect((await noRepo.status()).unavailableReason).toMatch(/git checkout/); + expect(await updaterFor().availability()).toBeNull(); }); it('launches install.sh in a transient unit built from daemon-side paths only', async () => { diff --git a/packages/server/src/updater.ts b/packages/server/src/updater.ts index 71d37043..2faa9db8 100644 --- a/packages/server/src/updater.ts +++ b/packages/server/src/updater.ts @@ -12,12 +12,15 @@ import { httpError } from './http-error'; import type { BuildInfo } from './version'; /** - * The daemon's own upgrade window. Versions are git commits (trunk-based, - * no release tags yet), and the question "is a newer Dormice available?" - * is answered by comparing the commit baked into this build against the - * origin's main — fetched through the checkout's own `origin` remote, so - * an install done with `--mirror cn` (whose clone URL carries the mirror - * prefix) checks through the same mirror for free. + * A process's own upgrade window — the daemon's, and since the fourth cut + * the gateway's (imported through the `@dormice/server/updater` subpath; + * the gateway's machine is upgraded by the same install.sh, and its + * process launches it the same way). Versions are git commits + * (trunk-based, no release tags yet), and the question "is a newer + * Dormice available?" is answered by comparing the commit baked into this + * build against the origin's main — fetched through the checkout's own + * `origin` remote, so an install done with `--mirror cn` (whose clone URL + * carries the mirror prefix) checks through the same mirror for free. * * `git fetch` updates .git only and never touches the working tree or the * running process — checking is always safe. The result is cached so a @@ -72,8 +75,13 @@ export interface UpdaterOptions { build: BuildInfo | null; /** Where the upgrade unit writes status.json and its log: /upgrade. */ statusDir: string; - /** One-click upgrade is a real install's move; the fake executor refuses. */ - executor: 'fake' | 'docker'; + /** + * The caller's own reason one-click is off, when it has one: the daemon + * says so on the fake executor (a real install's move, not a test + * double's), the gateway on an in-memory database. Checked first; the + * checkout, install.sh and systemd-run probes follow. + */ + unavailable?: string; run?: RunCommand; } @@ -83,7 +91,7 @@ export class Updater { private readonly repoDir: string | null; private readonly build: BuildInfo | null; private readonly statusDir: string; - private readonly executor: 'fake' | 'docker'; + private readonly unavailable: string | undefined; private readonly run: RunCommand; private cache: { at: number; check: Check } | null = null; /** Probed once — every input (executor, checkout, systemd) is boot-stable. */ @@ -93,7 +101,7 @@ export class Updater { this.repoDir = options.repoDir; this.build = options.build; this.statusDir = options.statusDir; - this.executor = options.executor; + this.unavailable = options.unavailable; this.run = options.run ?? defaultRun; } @@ -108,7 +116,7 @@ export class Updater { current: this.build, check: null, checkError: - 'the daemon does not run from a git checkout — nothing to compare against', + 'the process does not run from a git checkout — nothing to compare against', }; } if (this.build === null) { @@ -202,7 +210,7 @@ export class Updater { // name must be reusable for the next upgrade. '--collect', '--description', - 'Dormice self-upgrade (install.sh)', + 'Dormice upgrade (install.sh)', '/bin/bash', '-c', command, @@ -254,7 +262,13 @@ export class Updater { }; } - private async availability(): Promise { + /** + * Why one-click is off, or null when it is on — probed once (every + * input is boot-stable). Public for the node's check-in, which reports + * it to the gateway: the fleet upgrade rolls over the nodes that can + * upgrade themselves and names the rest with this reason. + */ + async availability(): Promise { if (this.availabilityReason === undefined) { this.availabilityReason = await this.probeAvailability(); } @@ -262,11 +276,11 @@ export class Updater { } private async probeAvailability(): Promise { - if (this.executor !== 'docker') { - return 'one-click upgrade is for a real install (docker executor) — this daemon runs the fake executor'; + if (this.unavailable !== undefined) { + return this.unavailable; } if (this.repoDir === null) { - return 'the daemon does not run from a git checkout'; + return 'the process does not run from a git checkout'; } if (!existsSync(path.join(this.repoDir, 'deploy', 'install.sh'))) { return 'deploy/install.sh is missing from the checkout'; diff --git a/packages/server/tsup.config.ts b/packages/server/tsup.config.ts index 9451ed75..8360bdbb 100644 --- a/packages/server/tsup.config.ts +++ b/packages/server/tsup.config.ts @@ -25,10 +25,10 @@ const commitTime = git('log -1 --format=%cI'); export default defineConfig({ // Subpath entries beyond the root: mini-s3 for the e2e harness; auth, - // keyed-queue, lock, shutdown, s3-store and history for the gateway, - // which reuses the daemon's small self-contained pieces without loading - // its executor (the root's import graph drags dockerode, execa and the - // AWS SDK in). + // keyed-queue, lock, shutdown, s3-store, history and updater for the + // gateway, which reuses the daemon's small self-contained pieces + // without loading its executor (the root's import graph drags + // dockerode, execa and the AWS SDK in). entry: [ 'src/index.ts', 'src/main.ts', @@ -39,6 +39,7 @@ export default defineConfig({ 'src/shutdown.ts', 'src/archive/s3-store.ts', 'src/history.ts', + 'src/updater.ts', ], format: ['esm'], dts: true, diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts index f9f75f05..dfbdad53 100644 --- a/packages/shared/src/gateway.ts +++ b/packages/shared/src/gateway.ts @@ -135,6 +135,17 @@ export const checkInRequestSchema = z.object({ * the gateway has to remember about who was told what. */ configVersion: z.number().int().nullable(), + /** + * Whether this node can upgrade itself when told (its updater's + * availability: a git checkout, install.sh, systemd-run; upgrade.ts), + * and why not when it cannot. The gateway rolls an upgrade only over + * nodes that can; the rest it lists as `unavailable` with the reason. + * Optional on the wire: a node on a build before the fourth cut does + * not say, and its check-in is taken. + */ + selfUpgrade: z + .object({ available: z.boolean(), reason: z.string().nullable() }) + .optional(), }); export type CheckInRequest = z.infer; @@ -188,6 +199,13 @@ export const checkInResponseSchema = z.object({ configVersion: z.number().int().positive(), /** Present exactly when the node's reported version differs from the gateway's: the whole bundle to apply. */ config: nodeConfigBundleSchema.optional(), + /** + * Present exactly when the gateway tells this node to upgrade itself + * now (upgrade.ts: the fleet upgrade rolls over the nodes one at a + * time, each told once at a check-in) — the node runs its own + * applyUpgrade and comes back on the gateway's build. + */ + upgrade: z.literal(true).optional(), }); export type CheckInResponse = z.infer; diff --git a/packages/shared/src/upgrade.ts b/packages/shared/src/upgrade.ts index 6ac4f25c..fbd66148 100644 --- a/packages/shared/src/upgrade.ts +++ b/packages/shared/src/upgrade.ts @@ -1,4 +1,16 @@ import { z } from 'zod'; +import { buildInfoSchema } from './gateway'; + +/** + * The upgrade verbs, on a node and at the gateway. On a node they are + * the node's own (the emergency path: ssh in, curl its loopback). At the + * gateway they are the fleet's (the fourth cut, 2026-09-15): checkUpgrade + * compares the gateway's build, applyUpgrade upgrades the gateway's + * machine — and once the gateway runs the new build, every node behind it + * is told at its check-in to upgrade itself, one node at a time + * (gateway.ts checkInResponseSchema.upgrade), and getUpgradeStatus lists + * each node's standing. + */ /** * checkUpgrade() — is a newer Dormice available for this daemon? @@ -79,19 +91,31 @@ export const checkUpgradeResponseSchema = z.object({ export type CheckUpgradeResponse = z.infer; /** - * applyUpgrade() — the one-click upgrade. The daemon launches install.sh + * applyUpgrade() — the one-click upgrade. The process launches install.sh * (re-running it IS the upgrade — one script for manual and one-click) in * a systemd transient unit, detached from its own lifetime: the upgrade's - * last step restarts the daemon, and a child process would die with its - * parent mid-build. The unit name doubles as the mutex — a second apply - * while one runs is refused with 409, the Coolify double-click corruption - * made structural. The verb takes no parameters on purpose: nothing from - * the request ever reaches a root command line. + * last step restarts the process, and a child would die with its parent + * mid-build. The unit name doubles as the mutex — a second apply while + * one runs is refused with 409, the Coolify double-click corruption made + * structural. Nothing from the request ever reaches a root command line. + * + * At the gateway, without `nodeId`, this is the fleet upgrade: the + * gateway's machine upgrades (its gateway and its node together), and the + * nodes behind follow — each is told at its check-in, one at a time, once. + * With `nodeId` it is the operator telling one node again: a node told + * once that is still on the old build twenty minutes later is `stuck` + * (getUpgradeStatus), never re-told on its own — a node whose build keeps + * failing must not rebuild every twenty minutes on the sandboxes' CPU — + * and this is the hand that re-tells it. On a node `nodeId` is meaningless + * and refused. * * Refused (400) when one-click is unavailable — fake executor, no git * checkout, no systemd. Watch progress with getUpgradeStatus. */ -export const applyUpgradeRequestSchema = z.object({}); +export const applyUpgradeRequestSchema = z.object({ + /** At the gateway: tell this one node to upgrade at its next check-in, whatever the rolling order says. Absent: upgrade the gateway's machine, then roll the fleet. */ + nodeId: z.string().min(1).optional(), +}); export type ApplyUpgradeRequest = z.infer; @@ -133,8 +157,47 @@ export type GetUpgradeStatusRequest = z.infer< typeof getUpgradeStatusRequestSchema >; +/** + * Where one node stands against the gateway's build, as the gateway + * judges it from the node's last check-in (gateway rolling.ts): + * current the node runs the gateway's build + * behind another build, able to upgrade itself, not told yet — its + * turn comes when no other node is upgrading + * upgrading told within the last twenty minutes, not back yet + * stuck told, still on the old build twenty minutes on — never + * re-told on its own; applyUpgrade {nodeId} is the hand + * unavailable another build, but the node cannot upgrade itself (its + * own reason: no checkout, no systemd, an older build that + * does not say) — run install.sh on it by hand + * unreachable not checking in (two of its intervals silent) + * unknown no build identity to compare, the node's or the gateway's + */ +export const NODE_UPGRADE_STATES = [ + 'current', + 'behind', + 'upgrading', + 'stuck', + 'unavailable', + 'unreachable', + 'unknown', +] as const; + +export type NodeUpgradeState = (typeof NODE_UPGRADE_STATES)[number]; + +export const nodeUpgradeViewSchema = z.object({ + id: z.string(), + build: buildInfoSchema.nullable(), + state: z.enum(NODE_UPGRADE_STATES), + /** ISO 8601 UTC — when the node was last told to upgrade; null = never, or its last tell was fulfilled. */ + toldAt: z.iso.datetime().nullable(), + /** In the gateway's words, for every state but current and behind: why it is stuck, unavailable, unreachable or unknown; how long it has been upgrading. */ + reason: z.string().nullable(), +}); + +export type NodeUpgradeView = z.infer; + export const getUpgradeStatusResponseSchema = z.object({ - /** Can this daemon one-click upgrade itself at all? */ + /** Can this process one-click upgrade its machine at all? */ available: z.boolean(), /** Why not, when available is false — the console shows the manual path instead. */ unavailableReason: z.string().nullable(), @@ -144,6 +207,8 @@ export const getUpgradeStatusResponseSchema = z.object({ last: upgradeRunSchema.nullable(), /** The tail of the run's output — real progress, straight from the script. */ log: z.string().nullable(), + /** The gateway's answer carries every node's standing; a node's own answer has no nodes to speak of. */ + nodes: z.array(nodeUpgradeViewSchema).optional(), }); export type GetUpgradeStatusResponse = z.infer< diff --git a/website/content/docs/console.mdx b/website/content/docs/console.mdx index 59e76514..40489b9f 100644 --- a/website/content/docs/console.mdx +++ b/website/content/docs/console.mdx @@ -149,13 +149,16 @@ Three more pages round out the operator view: else is the effective environment configuration, read-only, with secrets shown as present-or-absent only; changing those happens in `/etc/dormice/env` plus a restart, never from the browser. -- **Version** — the build the daemon is running (a git commit — trunk +- **Version** — the build the gateway is running (a git commit — trunk commit titles are the changelog) compared against the latest, with a - one-click [upgrade](/docs/upgrading) and live progress where the host - supports it. The console also checks once when you open it: if an - upgrade is available, a dialog shows what it would bring — dismiss it - for this visit, or ignore that version until a newer one appears. The - sidebar badge stays lit either way, until you actually upgrade. + one-click [fleet upgrade](/docs/upgrading#upgrade-a-fleet) and live + progress where the host supports it, and a table of every node's + standing against the gateway's build: current, behind, upgrading, + stuck (with a **Try again** button), or why it cannot upgrade itself. + The console also checks once when you open it: if an upgrade is + available, a dialog shows what it would bring — dismiss it for this + visit, or ignore that version until a newer one appears. The sidebar + badge stays lit either way, until you actually upgrade. A command palette (⌘K) jumps to any page or sandbox. diff --git a/website/content/docs/http-api.mdx b/website/content/docs/http-api.mdx index 3a62ceeb..6dcd4378 100644 --- a/website/content/docs/http-api.mdx +++ b/website/content/docs/http-api.mdx @@ -25,8 +25,10 @@ to the node the request names. The **daemon** (`127.0.0.1:3676`) is a node: it answers the per-sandbox, list and host verbs for itself; asked for a verb that lives at the gateway, it answers `404` naming the gateway. Point clients at the gateway; the rows below say which door -answers what. The three upgrade verbs are the one thing the gateway does -not route yet (an honest `501`). +answers what. The three upgrade verbs answer at both doors: at the +gateway they are the [fleet upgrade](/docs/upgrading#upgrade-a-fleet) +(the gateway's machine first, then every node behind it, one at a +time); on a node they are that node's own, the emergency path. Two rules cover the whole surface: diff --git a/website/content/docs/upgrading.mdx b/website/content/docs/upgrading.mdx index a7992c27..524cc459 100644 --- a/website/content/docs/upgrading.mdx +++ b/website/content/docs/upgrading.mdx @@ -27,6 +27,30 @@ script file mid-run while bash keeps executing the *old* bytes — measured on a real upgrade. Run a local copy twice, or just pipe from the repo. +## Upgrade a fleet + +A fleet upgrades from its gateway, and one action upgrades every +machine. On the gateway's machine, re-running the installer — or the +console's **Upgrade** button, or `POST /applyUpgrade` at the gateway — +pulls, rebuilds and restarts the gateway and the node on that machine. +Once the gateway is back on the new build, every other node hears at +its next check-in (within fifteen seconds) that its turn has come, and +runs the same installer itself: one node at a time, so the fleet is +never missing more than one node's sandboxes for the few tens of +seconds a restart takes. The version page shows each node's standing: +current, behind, upgrading, or stuck. + +A node is told exactly once. If it has not come back on the new build +twenty minutes later, the gateway marks it **stuck** and leaves it +alone — a node whose build keeps failing must not rebuild every twenty +minutes on the CPU its sandboxes run on. Read `journalctl -u +dormice-upgrade` and `/var/lib/dormice/upgrade/upgrade.log` on that +node, fix the cause, and press **Try again** on the version page (or +`POST /applyUpgrade {"nodeId": "..."}`), which tells that node again at +its next check-in. A node that cannot upgrade itself — no git checkout, +no systemd — is listed with its reason; run the installer on it by +hand. + ## What survives an upgrade - **Your configuration.** `/etc/dormice/env` is never touched once it From 1cb64d13148fa181bac4bc70784166445ecac045 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 17:11:58 +0800 Subject: [PATCH 61/89] A single machine's ledger is imported into the gateway once, before the gateway's first start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit packages/gateway/dist/import.js (import-ledger.ts behind it) carries what the daemon held as the fleet's configuration before the authority moved to the gateway — its settings row, templates, API keys, console account — and its last thirty days of fleet history into the gateway's tables, in one transaction, and pre-creates the node's row so its swap target survives the cut. Refused with exit 2 when the gateway's settings row already exists: this is for a first start, where the gateway would otherwise seed from the env and the node pull that, and months of operator settings would quietly go. The node's ledger is read through a read-only handle with plain SQL naming each column, asking only for the columns the table has — the ledger on a machine has the shape of whatever build its daemon last ran, not this one's. The pre-move spellings of "off" ('' and NULL) become the gateway's NULL, an archive default with no store is dropped, the base image comes from the node's copy, then its env, then the gateway's seed. The report is counts; no value is ever printed. install.sh runs it in the next step. --- packages/gateway/src/import-ledger.test.ts | 320 ++++++++++++++++ packages/gateway/src/import-ledger.ts | 412 +++++++++++++++++++++ packages/gateway/src/import.ts | 87 +++++ packages/gateway/tsup.config.ts | 5 +- 4 files changed, 822 insertions(+), 2 deletions(-) create mode 100644 packages/gateway/src/import-ledger.test.ts create mode 100644 packages/gateway/src/import-ledger.ts create mode 100644 packages/gateway/src/import.ts diff --git a/packages/gateway/src/import-ledger.test.ts b/packages/gateway/src/import-ledger.test.ts new file mode 100644 index 00000000..a876c6ed --- /dev/null +++ b/packages/gateway/src/import-ledger.test.ts @@ -0,0 +1,320 @@ +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + migrateDb as migrateNodeDb, + openDb as openNodeDb, +} from '@dormice/server'; +import type Database from 'better-sqlite3'; +import { describe, expect, it } from 'vitest'; +import { loadConfig } from './config'; +import { getConsoleAccount } from './db/account'; +import { listApiKeys } from './db/api-keys'; +import { migrateDb, openDb } from './db/db'; +import { fleetStateSamples, nodes } from './db/schema'; +import { readConfigVersion, readS3Settings, readSettings } from './db/settings'; +import { listTemplates } from './db/templates'; +import { Fleet } from './fleet'; +import { importNodeLedger, parseEnvFile } from './import-ledger'; + +// The one-time import against a real node ledger: the daemon's own +// migrations build it (through @dormice/server), plain SQL fills it the +// way months of a single machine did — a pre-move settings row with the +// old spellings of "off", templates, keys with a revoked one, a console +// account, a fleet history straddling the thirty-day cut. + +const MIGRATIONS = fileURLToPath(new URL('../drizzle', import.meta.url)); +const NODE_MIGRATIONS = fileURLToPath( + new URL('../../server/drizzle', import.meta.url), +); +const TOKEN = 'fleet-token-fleet-token-fleet-token-fleet'; +const NOW = new Date('2026-09-15T12:00:00.000Z'); + +function gatewayDb(env: Record = {}) { + const db = openDb(':memory:'); + migrateDb(db, MIGRATIONS); + const config = loadConfig({ + DORMICE_API_TOKEN: TOKEN, + DORMICE_GATEWAY_DB_PATH: '/var/lib/dormice-gateway/gateway.db', + ...env, + }); + return { db, config }; +} + +/** A node's ledger on disk, at this build's schema, with the daemon's tables filled by SQL. */ +function nodeLedger(fill: (raw: Database.Database) => void): string { + const dir = mkdtempSync(path.join(tmpdir(), 'dormice-import-')); + const file = path.join(dir, 'dormice.db'); + const db = openNodeDb(file); + migrateNodeDb(db, NODE_MIGRATIONS); + fill(db.$client); + db.$client.close(); + return file; +} + +const iso = (offsetDays: number) => + new Date(NOW.getTime() - offsetDays * 86_400_000).toISOString(); + +/** The machine's settings as the daemon left them before the move: two-state columns spelled the old way. */ +function fillProduction(raw: Database.Database) { + raw + .prepare( + `INSERT INTO runtime_settings (id, sandbox_cpus, sandbox_memory_gb, sandbox_disk_gb, + default_freeze_after_seconds, default_stop_after_seconds, default_archive_after_seconds, + swap_gb, s3_endpoint, s3_bucket, s3_access_key_id, s3_secret_access_key, s3_region, + s3_force_path_style, sandbox_domain, sandbox_domain_aliases, pids_limit, updated_at) + VALUES (1, 2, 4, 20, 300, NULL, 604800, 8, + 'https://oss.example.com', 'prod-bucket', 'AKIA-not-real', 'secret-not-real', 'cn-beijing', 0, + '', NULL, 4096, '2026-08-19T10:00:00.000Z')`, + ) + .run(); + raw + .prepare( + `INSERT INTO templates (name, image, created_at, updated_at) VALUES + ('clawsgo_20260808_base', 'clawsgo_20260808_base:20260907', '2026-08-08T00:00:00.000Z', '2026-09-07T00:00:00.000Z'), + ('rarvis', 'rarvis_20260808_base:20260901', '2026-08-08T00:00:00.000Z', '2026-09-01T00:00:00.000Z')`, + ) + .run(); + raw + .prepare( + `INSERT INTO api_keys (id, name, key_hash, prefix, created_at, last_used_at, expires_at, disabled_at, revoked_at) VALUES + ('k1', 'clawsgo', 'h1', 'aaaaaaaa', '2026-07-19T00:00:00.000Z', '2026-09-15T00:00:00.000Z', NULL, NULL, NULL), + ('k2', 'ci', 'h2', 'bbbbbbbb', '2026-07-20T00:00:00.000Z', NULL, '2026-12-31T23:59:59.999Z', NULL, NULL), + ('k3', 'old', 'h3', 'cccccccc', '2026-07-01T00:00:00.000Z', NULL, NULL, NULL, '2026-07-19T00:00:00.000Z')`, + ) + .run(); + raw + .prepare( + `INSERT INTO console_account (id, username, password_hash, session_secret, created_at, updated_at) + VALUES (1, 'admin', 'scrypt$16384$8$1$salt$hash', 'session-secret-not-real', '2026-07-19T00:00:00.000Z', '2026-07-19T00:00:00.000Z')`, + ) + .run(); + const sample = raw.prepare( + `INSERT INTO fleet_snapshots (at, active, frozen, stopped, archived, restoring, total) + VALUES (?, ?, 0, 0, 0, 0, ?)`, + ); + // 35 rows inside the thirty days, 5 older. + for (let i = 0; i < 40; i += 1) { + const daysAgo = i < 35 ? i * 0.5 : 31 + (i - 35); + sample.run(iso(daysAgo), 10 + i, 10 + i); + } +} + +const NODE_ENV = parseEnvFile(` +# written by install.sh +DORMICE_EXECUTOR=docker +DORMICE_API_TOKEN="${TOKEN}" +DORMICE_BASE_IMAGE=dormice-base:20260718 +# moved to /etc/dormice/gateway.env (2026-09-14): DORMICE_SANDBOX_DOMAIN=sandbox.example.com +DORMICE_DATA_DIR=/var/lib/dormice +`); + +describe('parseEnvFile', () => { + it('reads KEY=VALUE, skips blanks and comments, strips one pair of quotes', () => { + expect(NODE_ENV).toEqual({ + DORMICE_EXECUTOR: 'docker', + DORMICE_API_TOKEN: TOKEN, + DORMICE_BASE_IMAGE: 'dormice-base:20260718', + DORMICE_DATA_DIR: '/var/lib/dormice', + }); + expect(parseEnvFile("A='x=y'\nB=\nC")).toEqual({ A: 'x=y', B: '' }); + }); +}); + +describe('importNodeLedger', () => { + it("carries a single machine's settings into the gateway's row, translating the old spellings; pre-creates the node row with its swap target; copies templates, keys, the account and thirty days of history — and counts, never values", () => { + const ledger = nodeLedger(fillProduction); + const { db, config } = gatewayDb({ + DORMICE_REGISTRY_ADDRESS: '10.0.0.5:5000', + }); + const outcome = importNodeLedger(db, config, { + nodeDbPath: ledger, + nodeEnv: NODE_ENV, + now: NOW, + }); + expect(outcome).toEqual({ + code: 0, + counts: { + settings: 1, + nodes: 1, + templates: 2, + apiKeys: 3, + consoleAccount: 1, + fleetStateSamples: 35, + }, + }); + expect(JSON.stringify(outcome)).not.toContain('secret-not-real'); + + expect(readSettings(db)).toEqual({ + sandboxDefaults: { cpus: 2, memoryGb: 4, diskGb: 20 }, + defaultPolicy: { + freezeAfterSeconds: 300, + stopAfterSeconds: null, + archiveAfterSeconds: 604800, + }, + s3: { + endpoint: 'https://oss.example.com', + bucket: 'prod-bucket', + region: 'cn-beijing', + forcePathStyle: false, + }, + // '' was the pre-move "off"; NULL aliases were "none". + sandboxDomain: null, + sandboxDomainAliases: [], + pidsLimit: 4096, + // The node's env, the base image's old home; the registry is the seed's. + baseImage: 'dormice-base:20260718', + registryAddress: '10.0.0.5:5000', + // The operator's edits carried: not the seed. + updatedAt: '2026-08-19T10:00:00.000Z', + }); + expect(readS3Settings(db)).toMatchObject({ + accessKeyId: 'AKIA-not-real', + secretAccessKey: 'secret-not-real', + }); + expect(readConfigVersion(db)).toBe(1); + + // The node's row, never checked in, with the target its daemon will + // reconcile its swap to at the first check-in; a fleet built on it + // loads it as a node that has never checked in. + expect(db.select().from(nodes).all()).toMatchObject([ + { + id: 'node-1', + endpoint: 'http://127.0.0.1:3676', + swapGb: 8, + lastCheckInAt: null, + reading: null, + }, + ]); + const node = new Fleet(db).get('node-1'); + expect(node?.swapGb).toBe(8); + expect(node?.lastCheckInAt).toBeNull(); + + expect(listTemplates(db).map((t) => [t.name, t.image])).toEqual([ + ['clawsgo_20260808_base', 'clawsgo_20260808_base:20260907'], + ['rarvis', 'rarvis_20260808_base:20260901'], + ]); + const keys = listApiKeys(db); + expect(keys.map((k) => [k.name, k.revokedAt !== null]).sort()).toEqual([ + ['ci', false], + ['clawsgo', false], + ['old', true], + ]); + expect(keys.find((k) => k.name === 'ci')?.expiresAt).toBe( + '2026-12-31T23:59:59.999Z', + ); + expect(getConsoleAccount(db)).toMatchObject({ + username: 'admin', + sessionSecret: 'session-secret-not-real', + }); + const samples = db.select().from(fleetStateSamples).all(); + expect(samples).toHaveLength(35); + expect(Math.min(...samples.map((s) => s.active))).toBe(10); + expect(Math.max(...samples.map((s) => s.active))).toBe(44); + }); + + it('a second run is refused with 2 and writes nothing; the node id, endpoint and port come from the env when set', () => { + const ledger = nodeLedger(fillProduction); + const { db, config } = gatewayDb(); + const env = { + ...NODE_ENV, + DORMICE_NODE_ID: 'iZ2ze0uezt9j0ca8bgrhqsZ', + DORMICE_NODE_ENDPOINT: 'http://10.0.0.7:80/', + }; + expect( + importNodeLedger(db, config, { + nodeDbPath: ledger, + nodeEnv: env, + now: NOW, + }).code, + ).toBe(0); + expect(db.select().from(nodes).all()).toMatchObject([ + { id: 'iZ2ze0uezt9j0ca8bgrhqsZ', endpoint: 'http://10.0.0.7:80' }, + ]); + const again = importNodeLedger(db, config, { + nodeDbPath: ledger, + nodeEnv: env, + now: NOW, + }); + expect(again).toMatchObject({ + code: 2, + message: expect.stringMatching(/already holds a settings row/), + }); + expect(listTemplates(db)).toHaveLength(2); + expect(db.select().from(nodes).all()).toHaveLength(1); + // A different port in the env shapes the loopback default. + const { db: db2, config: config2 } = gatewayDb(); + importNodeLedger(db2, config2, { + nodeDbPath: ledger, + nodeEnv: { ...NODE_ENV, DORMICE_PORT: '3700' }, + now: NOW, + }); + expect(db2.select().from(nodes).all()[0]?.endpoint).toBe( + 'http://127.0.0.1:3700', + ); + }); + + it("reads a ledger of another build's shape: an extra column is never asked for, a missing one reads as NULL, an archive default with the store off is dropped; a node already on this cut keeps its copy's base image", () => { + const ledger = nodeLedger((raw) => { + fillProduction(raw); + // The test machine's leftover from an abandoned branch. + raw.exec('ALTER TABLE api_keys ADD COLUMN delegate text'); + // A ledger from before the fourth cut's columns and the config copy. + raw.exec('ALTER TABLE runtime_settings DROP COLUMN base_image'); + raw.exec('ALTER TABLE runtime_settings DROP COLUMN registry_address'); + raw.exec('ALTER TABLE runtime_settings DROP COLUMN config_version'); + raw.exec('ALTER TABLE runtime_settings DROP COLUMN updated_at'); + // The store off the old way, a default that still says archive. + raw.exec( + "UPDATE runtime_settings SET s3_endpoint = '', default_archive_after_seconds = 604800, pids_limit = NULL WHERE id = 1", + ); + }); + const { db, config } = gatewayDb({ DORMICE_SANDBOX_PIDS_LIMIT: '2048' }); + const outcome = importNodeLedger(db, config, { + nodeDbPath: ledger, + nodeEnv: {}, + now: NOW, + }); + expect(outcome.code).toBe(0); + expect(readSettings(db)).toMatchObject({ + s3: null, + defaultPolicy: expect.objectContaining({ archiveAfterSeconds: null }), + // NULL in the ledger: the seed's. + pidsLimit: 2048, + // No copy column, no env: none — the gateway's late seed or the + // console fills it. + baseImage: null, + registryAddress: null, + updatedAt: null, + }); + expect(listApiKeys(db)).toHaveLength(3); + + const onThisCut = nodeLedger((raw) => { + fillProduction(raw); + raw.exec( + "UPDATE runtime_settings SET base_image = 'dormice-base:20260901' WHERE id = 1", + ); + }); + const fresh = gatewayDb(); + importNodeLedger(fresh.db, fresh.config, { + nodeDbPath: onThisCut, + nodeEnv: NODE_ENV, + now: NOW, + }); + expect(readSettings(fresh.db).baseImage).toBe('dormice-base:20260901'); + }); + + it("a ledger without a settings row is not a daemon's: refused in words, nothing written", () => { + const ledger = nodeLedger(() => {}); + const { db, config } = gatewayDb(); + expect(() => + importNodeLedger(db, config, { + nodeDbPath: ledger, + nodeEnv: {}, + now: NOW, + }), + ).toThrow(/has no runtime_settings row/); + expect(db.select().from(nodes).all()).toEqual([]); + }); +}); diff --git a/packages/gateway/src/import-ledger.ts b/packages/gateway/src/import-ledger.ts new file mode 100644 index 00000000..cf4dccd7 --- /dev/null +++ b/packages/gateway/src/import-ledger.ts @@ -0,0 +1,412 @@ +import Database from 'better-sqlite3'; +import type { Config } from './config'; +import type { Db } from './db/db'; +import { FLEET_SAMPLE_KEEP_DAYS } from './db/fleet-samples'; +import { + apiKeys, + consoleAccount, + fleetStateSamples, + nodes, + settings, + templates, +} from './db/schema'; +import { seedRow } from './db/settings'; + +/** + * The one-time import of a single machine's ledger into the gateway's + * tables (the fourth cut, 2026-09-15): what the daemon held as the + * fleet's configuration before the authority moved to the gateway + * (design record #22) — its settings row, its templates, its API keys, + * its console account — and its fleet history, so the dashboard's + * thirty-day curve does not break at the cut. install.sh runs it once, + * before the gateway's first start: the gateway seeds its settings row + * at that start and the node pulls the bundle at its first check-in, so + * an import that came later would find the row taken and the node + * already running on the env seeds — with the S3 store, the default + * policy, the domain aliases and the templates the operator set over + * months quietly gone (a restore from archive would fail, a template + * sandbox would wake to "template not registered"). Refused, with exit + * 2, when the gateway's settings row exists: this is for a first start, + * and a backup file can be imported into a fresh gateway database the + * same way. + * + * The node's ledger is read through a read-only handle with plain SQL + * naming each column — not the daemon's drizzle schema, whose shape is + * this build's, where the ledger on the machine is whatever build the + * daemon last ran: a column the ledger lacks reads as NULL (the Beijing + * node at the cut sits before the config-copy migration), an extra + * column it carries is never asked for (the test machine's api_keys has + * a `delegate` left by an abandoned branch). Values are never printed: + * the settings row holds the S3 secret and the keys table the hashes; + * the report is counts. + */ + +export interface ImportCounts { + settings: number; + nodes: number; + templates: number; + apiKeys: number; + consoleAccount: number; + fleetStateSamples: number; +} + +export type ImportOutcome = + | { code: 0; counts: ImportCounts } + /** The gateway's settings row exists: nothing written (exit 2 at the CLI). */ + | { code: 2; message: string }; + +export interface ImportInput { + /** The node's ledger (DORMICE_DB_PATH), read-only. */ + nodeDbPath: string; + /** The node's env file, parsed (parseEnvFile): its identity — node id, endpoint, port, base image. */ + nodeEnv: Record; + now?: Date; +} + +/** + * `KEY=VALUE` lines, the systemd EnvironmentFile dialect install.sh + * writes: blank lines and `#` comments skipped, one pair of surrounding + * quotes stripped. Nothing more — the file is ours. + */ +export function parseEnvFile(text: string): Record { + const env: Record = {}; + for (const raw of text.split('\n')) { + const line = raw.trim(); + if (line === '' || line.startsWith('#')) continue; + const eq = line.indexOf('='); + if (eq === -1) continue; + const key = line.slice(0, eq).trim(); + let value = line.slice(eq + 1).trim(); + if ( + value.length >= 2 && + ((value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'"))) + ) { + value = value.slice(1, -1); + } + env[key] = value; + } + return env; +} + +/** One row of the node's `runtime_settings`, the columns the import reads — NULL where the ledger lacks the column. */ +interface NodeSettingsRow { + sandbox_cpus: number; + sandbox_memory_gb: number; + sandbox_disk_gb: number; + default_freeze_after_seconds: number; + default_stop_after_seconds: number | null; + default_archive_after_seconds: number | null; + swap_gb: number | null; + s3_endpoint: string | null; + s3_bucket: string | null; + s3_access_key_id: string | null; + s3_secret_access_key: string | null; + s3_region: string | null; + s3_force_path_style: number | null; + sandbox_domain: string | null; + sandbox_domain_aliases: string | null; + pids_limit: number | null; + base_image: string | null; + registry_address: string | null; + updated_at: string | null; +} + +const NODE_SETTINGS_COLUMNS: Array = [ + 'sandbox_cpus', + 'sandbox_memory_gb', + 'sandbox_disk_gb', + 'default_freeze_after_seconds', + 'default_stop_after_seconds', + 'default_archive_after_seconds', + 'swap_gb', + 's3_endpoint', + 's3_bucket', + 's3_access_key_id', + 's3_secret_access_key', + 's3_region', + 's3_force_path_style', + 'sandbox_domain', + 'sandbox_domain_aliases', + 'pids_limit', + 'base_image', + 'registry_address', + 'updated_at', +]; + +const API_KEY_COLUMNS = [ + 'id', + 'name', + 'key_hash', + 'prefix', + 'created_at', + 'last_used_at', + 'expires_at', + 'disabled_at', + 'revoked_at', +] as const; + +const CONSOLE_ACCOUNT_COLUMNS = [ + 'id', + 'username', + 'password_hash', + 'session_secret', + 'created_at', + 'updated_at', +] as const; + +const TEMPLATE_COLUMNS = ['name', 'image', 'created_at', 'updated_at'] as const; + +const SAMPLE_COLUMNS = [ + 'at', + 'active', + 'frozen', + 'stopped', + 'archived', + 'restoring', + 'total', +] as const; + +/** SQLite's bound-parameter ceiling is generous (32766), but a 79,603-row history is inserted in slices all the same. */ +const INSERT_CHUNK = 500; + +export function importNodeLedger( + db: Db, + config: Config, + input: ImportInput, +): ImportOutcome { + const now = input.now ?? new Date(); + const existing = db.select({ id: settings.id }).from(settings).get(); + if (existing !== undefined) { + return { + code: 2, + message: `the gateway database at ${config.DORMICE_GATEWAY_DB_PATH} already holds a settings row — the import is for a gateway's first start; nothing was written`, + }; + } + const source = new Database(input.nodeDbPath, { + readonly: true, + fileMustExist: true, + }); + try { + const read = new LedgerReader(source); + const nodeSettings = read.settingsRow(); + if (nodeSettings === undefined) { + throw new Error( + `${input.nodeDbPath} has no runtime_settings row — not a daemon's ledger, or one that never started`, + ); + } + const seed = seedRow(config); + const s3On = nonEmpty(nodeSettings.s3_endpoint) !== null; + const row: typeof settings.$inferInsert = { + ...seed, + sandboxCpus: nodeSettings.sandbox_cpus, + sandboxMemoryGb: nodeSettings.sandbox_memory_gb, + sandboxDiskGb: nodeSettings.sandbox_disk_gb, + defaultFreezeAfterSeconds: nodeSettings.default_freeze_after_seconds, + defaultStopAfterSeconds: nodeSettings.default_stop_after_seconds, + // A default that archives with no store would be the standing lie + // the gateway's updateSettings refuses; the node's own seeding + // forced null here too. + defaultArchiveAfterSeconds: s3On + ? nodeSettings.default_archive_after_seconds + : null, + // The six as one unit: on, all six from the node; off ('' or NULL — + // the pre-move row's two spellings of off), all six NULL. + s3Endpoint: s3On ? nodeSettings.s3_endpoint : null, + s3Bucket: s3On ? nodeSettings.s3_bucket : null, + s3AccessKeyId: s3On ? nodeSettings.s3_access_key_id : null, + s3SecretAccessKey: s3On ? nodeSettings.s3_secret_access_key : null, + s3Region: s3On ? nodeSettings.s3_region : null, + s3ForcePathStyle: s3On + ? nodeSettings.s3_force_path_style === null + ? null + : nodeSettings.s3_force_path_style !== 0 + : null, + sandboxDomain: nonEmpty(nodeSettings.sandbox_domain), + sandboxDomainAliases: nodeSettings.sandbox_domain_aliases ?? '[]', + pidsLimit: nodeSettings.pids_limit ?? seed.pidsLimit, + // The base image's homes, newest first: the node's copy (a node + // already on the fourth cut), the node's env (its old home), the + // gateway's seed. + baseImage: + nodeSettings.base_image ?? + nonEmpty(input.nodeEnv.DORMICE_BASE_IMAGE ?? null) ?? + seed.baseImage, + registryAddress: seed.registryAddress ?? nodeSettings.registry_address, + // The operator's last edit, carried: this row is not the seed. + updatedAt: nodeSettings.updated_at, + }; + const nodeId = nonEmpty(input.nodeEnv.DORMICE_NODE_ID ?? null) ?? 'node-1'; + const endpoint = + nonEmpty(input.nodeEnv.DORMICE_NODE_ENDPOINT ?? null) ?? + `http://127.0.0.1:${nonEmpty(input.nodeEnv.DORMICE_PORT ?? null) ?? '3676'}`; + const cutoff = new Date( + now.getTime() - FLEET_SAMPLE_KEEP_DAYS * 86_400_000, + ).toISOString(); + const templateRows = read.rows('templates', TEMPLATE_COLUMNS); + const keyRows = read.rows('api_keys', API_KEY_COLUMNS); + const accountRows = read.rows('console_account', CONSOLE_ACCOUNT_COLUMNS); + const sampleRows = read.rows( + 'fleet_snapshots', + SAMPLE_COLUMNS, + 'WHERE at >= ?', + [cutoff], + ); + const counts: ImportCounts = { + settings: 1, + nodes: 1, + templates: templateRows.length, + apiKeys: keyRows.length, + consoleAccount: accountRows.length, + fleetStateSamples: sampleRows.length, + }; + db.transaction((tx) => { + tx.insert(settings).values(row).run(); + // The node's row, so its swap target survives the cut (the daemon + // reconciles its managed swap from the bundle's node row; a fleet + // that forgot the target would stop growing swap after the next + // reboot). Never checked in yet: fleet.ts loads it as such, and + // the node's first check-in fills the rest. + tx.insert(nodes) + .values({ + id: nodeId, + endpoint: endpoint.replace(/\/+$/, ''), + addedAt: now.toISOString(), + swapGb: nodeSettings.swap_gb ?? 0, + }) + .run(); + for (const chunk of chunks(templateRows)) { + tx.insert(templates) + .values( + chunk.map((r) => ({ + name: r.name as string, + image: r.image as string, + createdAt: r.created_at as string, + updatedAt: r.updated_at as string, + })), + ) + .run(); + } + for (const chunk of chunks(keyRows)) { + tx.insert(apiKeys) + .values( + chunk.map((r) => ({ + id: r.id as string, + name: r.name as string, + keyHash: r.key_hash as string, + prefix: r.prefix as string, + createdAt: r.created_at as string, + lastUsedAt: r.last_used_at as string | null, + expiresAt: r.expires_at as string | null, + disabledAt: r.disabled_at as string | null, + revokedAt: r.revoked_at as string | null, + })), + ) + .run(); + } + for (const chunk of chunks(accountRows)) { + tx.insert(consoleAccount) + .values( + chunk.map((r) => ({ + id: r.id as number, + username: r.username as string, + passwordHash: r.password_hash as string, + sessionSecret: r.session_secret as string, + createdAt: r.created_at as string, + updatedAt: r.updated_at as string, + })), + ) + .run(); + } + for (const chunk of chunks(sampleRows)) { + tx.insert(fleetStateSamples) + .values( + chunk.map((r) => ({ + at: r.at as string, + active: r.active as number, + frozen: r.frozen as number, + stopped: r.stopped as number, + archived: r.archived as number, + restoring: r.restoring as number, + total: r.total as number, + })), + ) + .run(); + } + }); + return { code: 0, counts }; + } finally { + source.close(); + } +} + +/** '' and NULL are both "unset" in a pre-move row (schema.ts on the node has the story); the gateway knows NULL alone. */ +function nonEmpty(value: string | null): string | null { + return value === null || value === '' ? null : value; +} + +function* chunks(rows: T[]): Generator { + for (let i = 0; i < rows.length; i += INSERT_CHUNK) { + yield rows.slice(i, i + INSERT_CHUNK); + } +} + +/** + * Plain SQL over the node's ledger, asking only for columns the table + * has: `PRAGMA table_info` first, then a SELECT that names the present + * ones and reads the absent as NULL — the ledger's shape is the build + * the daemon last ran, not this one's. + */ +class LedgerReader { + constructor(private readonly db: Database.Database) {} + + private columnsOf(table: string): Set { + const info = this.db + .prepare(`PRAGMA table_info(${quoteIdent(table)})`) + .all() as Array<{ name: string }>; + return new Set(info.map((c) => c.name)); + } + + private hasTable(table: string): boolean { + return ( + this.db + .prepare( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", + ) + .get(table) !== undefined + ); + } + + settingsRow(): NodeSettingsRow | undefined { + if (!this.hasTable('runtime_settings')) return undefined; + const present = this.columnsOf('runtime_settings'); + const select = NODE_SETTINGS_COLUMNS.map((c) => + present.has(c) ? quoteIdent(c) : `NULL AS ${quoteIdent(c)}`, + ).join(', '); + return this.db + .prepare(`SELECT ${select} FROM runtime_settings WHERE id = 1`) + .get() as NodeSettingsRow | undefined; + } + + /** Every row of a table, the named columns only (absent columns read NULL; a table the ledger lacks reads empty). */ + rows( + table: string, + columns: readonly string[], + where = '', + params: unknown[] = [], + ): Array> { + if (!this.hasTable(table)) return []; + const present = this.columnsOf(table); + const select = columns + .map((c) => (present.has(c) ? quoteIdent(c) : `NULL AS ${quoteIdent(c)}`)) + .join(', '); + return this.db + .prepare(`SELECT ${select} FROM ${quoteIdent(table)} ${where}`) + .all(...params) as Array>; + } +} + +/** Identifiers here are this module's own constants, never input; quoted all the same. */ +function quoteIdent(name: string): string { + return `"${name.replaceAll('"', '""')}"`; +} diff --git a/packages/gateway/src/import.ts b/packages/gateway/src/import.ts new file mode 100644 index 00000000..a872d58f --- /dev/null +++ b/packages/gateway/src/import.ts @@ -0,0 +1,87 @@ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { acquireSingleWriterLock } from '@dormice/server/lock'; +import { z } from 'zod'; +import { loadConfig } from './config'; +import { migrateDb, openDb } from './db/db'; +import { importNodeLedger, parseEnvFile } from './import-ledger'; + +/** + * `node dist/import.js --node-db --node-env ` — the + * one-time import of a single machine's ledger into the gateway's + * database (import-ledger.ts has what and why), run by install.sh on the + * gateway's machine before the gateway's first start. The gateway's own + * configuration comes from the environment, as for the gateway itself + * (install.sh sources gateway.env): DORMICE_GATEWAY_DB_PATH says where + * to write, the fleet seeds fill what the node's row does not say. + * + * Exit 0 with the counts as JSON on stdout; 2 when the gateway's + * settings row exists (nothing written); 1 for everything else, one + * line on stderr. No value of any row is ever printed. + */ +const USAGE = + 'usage: import.js --node-db --node-env '; + +function parseArgs(argv: string[]): { nodeDb: string; nodeEnv: string } { + let nodeDb: string | undefined; + let nodeEnv: string | undefined; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--node-db') nodeDb = argv[++i]; + else if (arg?.startsWith('--node-db=')) + nodeDb = arg.slice('--node-db='.length); + else if (arg === '--node-env') nodeEnv = argv[++i]; + else if (arg?.startsWith('--node-env=')) + nodeEnv = arg.slice('--node-env='.length); + else throw new Error(`unknown argument ${arg}\n${USAGE}`); + } + if (!nodeDb || !nodeEnv) throw new Error(USAGE); + return { nodeDb, nodeEnv }; +} + +function main(): number { + const { nodeDb, nodeEnv } = parseArgs(process.argv.slice(2)); + const config = loadConfig(); + if (config.DORMICE_GATEWAY_DB_PATH === ':memory:') { + throw new Error( + 'DORMICE_GATEWAY_DB_PATH is :memory: — an import needs the gateway database file the gateway will start on', + ); + } + // The gateway's own lock over its file: an import under a running + // gateway's feet would hand it a settings row it never saw. + const lock = acquireSingleWriterLock( + config.DORMICE_GATEWAY_DB_PATH, + `a gateway is running against ${config.DORMICE_GATEWAY_DB_PATH} — stop it (systemctl stop dormice-gateway) before importing`, + ); + try { + const db = openDb(config.DORMICE_GATEWAY_DB_PATH); + migrateDb(db, fileURLToPath(new URL('../drizzle', import.meta.url))); + const outcome = importNodeLedger(db, config, { + nodeDbPath: nodeDb, + nodeEnv: parseEnvFile(readFileSync(nodeEnv, 'utf8')), + }); + if (outcome.code === 2) { + process.stderr.write(`${outcome.message}\n`); + return 2; + } + process.stdout.write(`${JSON.stringify(outcome.counts)}\n`); + return 0; + } finally { + lock.close(); + } +} + +try { + process.exitCode = main(); +} catch (error) { + process.stderr.write( + `import.js: ${ + error instanceof z.ZodError + ? z.prettifyError(error) + : error instanceof Error + ? error.message + : String(error) + }\n`, + ); + process.exitCode = 1; +} diff --git a/packages/gateway/tsup.config.ts b/packages/gateway/tsup.config.ts index 1ab151a5..17130127 100644 --- a/packages/gateway/tsup.config.ts +++ b/packages/gateway/tsup.config.ts @@ -26,8 +26,9 @@ export default defineConfig({ // index is the library surface the SDK's and the CLI's suites embed a // gateway through (the verbs they test for keys, settings and templates // answer at the gateway), so it ships with declarations like the - // daemon's. - entry: ['src/main.ts', 'src/index.ts'], + // daemon's. import.ts is the one-time ledger import install.sh runs + // before the gateway's first start (import-ledger.ts). + entry: ['src/main.ts', 'src/index.ts', 'src/import.ts'], format: ['esm'], dts: true, clean: true, From 6591b03e26e4242a88e2b7acd416fa3fe814f2c6 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 17:16:05 +0800 Subject: [PATCH 62/89] The node's ledger drops the three tables that moved to the gateway, now that the import can carry them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit api_keys, console_account and fleet_snapshots were kept on the node, unread and unwritten, until the one-time import into the gateway existed; it does now (migration 0027 drops them, and runtime_settings' updated_at with them). install.sh orders the two: the import before the gateway's first start, the node's restart — and with it this migration — after. The import's test builds its node ledger from the daemon's migrations up to 0026, the shape the previous build leaves behind. --- packages/gateway/src/import-ledger.test.ts | 43 +- .../server/drizzle/0027_drop-moved-tables.sql | 4 + .../server/drizzle/meta/0027_snapshot.json | 587 ++++++++++++++++++ packages/server/drizzle/meta/_journal.json | 7 + packages/server/src/db/schema.ts | 109 +--- 5 files changed, 646 insertions(+), 104 deletions(-) create mode 100644 packages/server/drizzle/0027_drop-moved-tables.sql create mode 100644 packages/server/drizzle/meta/0027_snapshot.json diff --git a/packages/gateway/src/import-ledger.test.ts b/packages/gateway/src/import-ledger.test.ts index a876c6ed..5571e427 100644 --- a/packages/gateway/src/import-ledger.test.ts +++ b/packages/gateway/src/import-ledger.test.ts @@ -1,4 +1,10 @@ -import { mkdtempSync } from 'node:fs'; +import { + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -42,12 +48,43 @@ function gatewayDb(env: Record = {}) { return { db, config }; } -/** A node's ledger on disk, at this build's schema, with the daemon's tables filled by SQL. */ +/** + * The daemon's migrations up to and including `lastTag`, as a migrations + * folder of their own: the import reads the ledger the PREVIOUS build + * left, which still holds the tables this build's migration 0027 drops — + * a ledger built with every migration would have nothing to import. + */ +function nodeMigrationsUpTo(lastTag: string): string { + const dir = mkdtempSync(path.join(tmpdir(), 'dormice-node-migrations-')); + mkdirSync(path.join(dir, 'meta')); + const journal = JSON.parse( + readFileSync(path.join(NODE_MIGRATIONS, 'meta', '_journal.json'), 'utf8'), + ) as { entries: Array<{ tag: string }> }; + const cut = journal.entries.findIndex((e) => e.tag === lastTag); + if (cut === -1) throw new Error(`no node migration tagged ${lastTag}`); + const entries = journal.entries.slice(0, cut + 1); + writeFileSync( + path.join(dir, 'meta', '_journal.json'), + JSON.stringify({ ...journal, entries }), + ); + for (const entry of entries) { + copyFileSync( + path.join(NODE_MIGRATIONS, `${entry.tag}.sql`), + path.join(dir, `${entry.tag}.sql`), + ); + } + return dir; +} + +/** The last daemon migration before the fourth cut dropped the moved tables. */ +const PRE_DROP_MIGRATIONS = nodeMigrationsUpTo('0026_fleet-base-image'); + +/** A node's ledger on disk, at the previous build's schema, with the daemon's tables filled by SQL. */ function nodeLedger(fill: (raw: Database.Database) => void): string { const dir = mkdtempSync(path.join(tmpdir(), 'dormice-import-')); const file = path.join(dir, 'dormice.db'); const db = openNodeDb(file); - migrateNodeDb(db, NODE_MIGRATIONS); + migrateNodeDb(db, PRE_DROP_MIGRATIONS); fill(db.$client); db.$client.close(); return file; diff --git a/packages/server/drizzle/0027_drop-moved-tables.sql b/packages/server/drizzle/0027_drop-moved-tables.sql new file mode 100644 index 00000000..dc3c60ae --- /dev/null +++ b/packages/server/drizzle/0027_drop-moved-tables.sql @@ -0,0 +1,4 @@ +DROP TABLE `api_keys`;--> statement-breakpoint +DROP TABLE `console_account`;--> statement-breakpoint +DROP TABLE `fleet_snapshots`;--> statement-breakpoint +ALTER TABLE `runtime_settings` DROP COLUMN `updated_at`; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0027_snapshot.json b/packages/server/drizzle/meta/0027_snapshot.json new file mode 100644 index 00000000..93174673 --- /dev/null +++ b/packages/server/drizzle/meta/0027_snapshot.json @@ -0,0 +1,587 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "e6364cb1-554b-407d-903a-37f1214c5b22", + "prevId": "8cdf1b9f-73e5-46d1-a0cc-bf448d1887d9", + "tables": { + "daemon_secrets": { + "name": "daemon_secrets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "envd_signing_secret": { + "name": "envd_signing_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "host_metrics_samples": { + "name": "host_metrics_samples", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "cpu_used_pct": { + "name": "cpu_used_pct", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_available_bytes": { + "name": "mem_available_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_total_bytes": { + "name": "swap_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_used_bytes": { + "name": "swap_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_total_bytes": { + "name": "disk_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_used_bytes": { + "name": "disk_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_available_bytes": { + "name": "disk_available_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "runtime_settings": { + "name": "runtime_settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_version": { + "name": "config_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config_applied_at": { + "name": "config_applied_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_cpus": { + "name": "sandbox_cpus", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_memory_gb": { + "name": "sandbox_memory_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_disk_gb": { + "name": "sandbox_disk_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_freeze_after_seconds": { + "name": "default_freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_stop_after_seconds": { + "name": "default_stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_archive_after_seconds": { + "name": "default_archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_gb": { + "name": "swap_gb", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "s3_endpoint": { + "name": "s3_endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_access_key_id": { + "name": "s3_access_key_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_secret_access_key": { + "name": "s3_secret_access_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_region": { + "name": "s3_region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_force_path_style": { + "name": "s3_force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain": { + "name": "sandbox_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain_aliases": { + "name": "sandbox_domain_aliases", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pids_limit": { + "name": "pids_limit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_image": { + "name": "base_image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registry_address": { + "name": "registry_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sandbox_metrics_samples": { + "name": "sandbox_metrics_samples", + "columns": { + "sandbox_id": { + "name": "sandbox_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "at": { + "name": "at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cpu_count": { + "name": "cpu_count", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cpu_used_pct": { + "name": "cpu_used_pct", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_used_bytes": { + "name": "mem_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_total_bytes": { + "name": "mem_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mem_cache_bytes": { + "name": "mem_cache_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_used_bytes": { + "name": "swap_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "swap_total_bytes": { + "name": "swap_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_used_bytes": { + "name": "disk_used_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "disk_total_bytes": { + "name": "disk_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "sandbox_metrics_samples_sandbox_at_idx": { + "name": "sandbox_metrics_samples_sandbox_at_idx", + "columns": [ + "sandbox_id", + "at" + ], + "isUnique": false + }, + "sandbox_metrics_samples_at_idx": { + "name": "sandbox_metrics_samples_at_idx", + "columns": [ + "at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sandboxes": { + "name": "sandboxes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "freeze_after_seconds": { + "name": "freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stop_after_seconds": { + "name": "stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "archive_after_seconds": { + "name": "archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpus": { + "name": "cpus", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "memory_gb": { + "name": "memory_gb", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_gb": { + "name": "disk_gb", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_active_at": { + "name": "last_active_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_exit_at": { + "name": "last_exit_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_exit_code": { + "name": "last_exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_exit_cause": { + "name": "last_exit_cause", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "envs": { + "name": "envs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "on_deadline": { + "name": "on_deadline", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "paused_by_user": { + "name": "paused_by_user", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "sandboxes_name_unique": { + "name": "sandboxes_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index e62752d5..fa261a73 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -190,6 +190,13 @@ "when": 1789460306678, "tag": "0026_fleet-base-image", "breakpoints": true + }, + { + "idx": 27, + "version": "6", + "when": 1789463530946, + "tag": "0027_drop-moved-tables", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index ce24734d..502b27bb 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -1,12 +1,10 @@ import { SANDBOX_STATES, SHELL_EXIT_CAUSES } from '@dormice/shared'; -import { sql } from 'drizzle-orm'; import { index, integer, real, sqliteTable, text, - uniqueIndex, } from 'drizzle-orm/sqlite-core'; /** @@ -159,32 +157,6 @@ export const sandboxMetricsSamples = sqliteTable( export type SandboxMetricsSampleRow = typeof sandboxMetricsSamples.$inferSelect; -/** - * LEGACY, read by nothing on the node since the third cut (2026-09-15): - * the fleet's state counts per sampler tick, as this node sampled them - * while it was a product of its own. The fleet's history is the gateway's - * table now (gateway db/schema.ts fleet_state_samples, summed over every - * node at each check-in), and the sampler writes here no more. The table - * and its rows stay until the fourth cut's import tool has carried a - * production node's last 30 days into the gateway — a single-node fleet's - * history is the same figure — so the console's 30-day curve does not - * break at the cut-over; the DROP ships with that import, beside - * api_keys and console_account. Not a bug to delete early: the import - * reads it. - */ -export const fleetSnapshots = sqliteTable('fleet_snapshots', { - /** ISO 8601 UTC; one row per tick, so time itself is the key. */ - at: text('at').primaryKey(), - active: integer('active').notNull(), - frozen: integer('frozen').notNull(), - stopped: integer('stopped').notNull(), - archived: integer('archived').notNull(), - restoring: integer('restoring').notNull(), - total: integer('total').notNull(), -}); - -export type FleetSnapshotRow = typeof fleetSnapshots.$inferSelect; - /** * The host machine's own resource history, one row per sampler tick — the * historical sibling of getHostMetrics' snapshot. The original ruling @@ -217,27 +189,6 @@ export const hostMetricsSamples = sqliteTable('host_metrics_samples', { export type HostMetricsSampleRow = typeof hostMetricsSamples.$inferSelect; -/** - * The console's one human account — the gateway's table since 2026-09-14 - * (the console is served there; packages/gateway/src/db/schema.ts has the - * living definition). Kept in the node's ledger, unread and unwritten, - * until the one-time import of a single machine's old tables into the - * gateway (cut 4) has run; it is dropped then, not before — a migration - * that dropped it now would delete the operator's account ahead of the - * step that moves it. - */ -export const consoleAccount = sqliteTable('console_account', { - id: integer('id').primaryKey(), - username: text('username').notNull(), - /** Self-describing scrypt string: scrypt$N$r$p$$. */ - passwordHash: text('password_hash').notNull(), - sessionSecret: text('session_secret').notNull(), - createdAt: text('created_at').notNull(), - updatedAt: text('updated_at').notNull(), -}); - -export type ConsoleAccountRow = typeof consoleAccount.$inferSelect; - /** * The daemon's own secrets — one row, fixed id, same singleton pattern as * console_account. envdSigningSecret is the HMAC key behind every envd @@ -269,10 +220,14 @@ export type DaemonSecretsRow = typeof daemonSecrets.$inferSelect; * * config_version says which bundle the row is. NULL means the row is not * a copy at all: the daemon's own settings from before the move (seeded - * from the env, edited from the console), kept for the one-time import - * into the gateway (cut 4) and never read by the node again — the node - * reports "no copy" and takes the gateway's bundle at its first check-in, - * which overwrites every column. + * from the env, edited from the console), never read by the node again — + * the node reports "no copy" and takes the gateway's bundle at its first + * check-in, which overwrites every column. What such a row held for the + * fleet was carried into the gateway by the fourth cut's import + * (gateway import-ledger.ts), which install.sh runs before the gateway's + * first start; the same cut dropped the three tables that moved with the + * authority (api_keys, console_account, fleet_snapshots) and the row's + * updated_at, all read by nothing on a node (migration 0027). * * Typed columns, not a JSON blob: the schema IS the vocabulary, and a knob * that exists but is invisible to migrations would drift silently. @@ -332,54 +287,6 @@ export const runtimeSettings = sqliteTable('runtime_settings', { baseImage: text('base_image'), /** The fleet's image registry, host:port (shared settings.ts registryAddress); NULL = no registry, a missing image is a plain error. */ registryAddress: text('registry_address'), - /** The old single-machine row's last edit — the gateway's timestamp now; kept for the cut-4 import, never written by the node. */ - updatedAt: text('updated_at'), }); export type RuntimeSettingsRow = typeof runtimeSettings.$inferSelect; - -/** - * API keys — the gateway's table since 2026-09-14: keys are minted and - * judged at the fleet's one door (packages/gateway/src/db/schema.ts has - * the living definition), and toward a node the gateway speaks the fleet - * token alone. Kept in the node's ledger, unread and unwritten, until the - * cut-4 import into the gateway has run — the same reasoning as - * console_account above: a key an operator's automation still holds must - * move, not vanish. - */ -export const apiKeys = sqliteTable( - 'api_keys', - { - /** UUID, never an autoincrement — ids must stay unique across machines. */ - id: text('id').primaryKey(), - name: text('name').notNull(), - /** sha256 hex of the bare 64-hex key material. The key itself is never stored. */ - keyHash: text('key_hash').notNull().unique(), - /** First 8 hex chars of the key, for display — 32 bits, no meaningful entropy. */ - prefix: text('prefix').notNull(), - createdAt: text('created_at').notNull(), - /** Null = never used. Written with 60s granularity, not per request. */ - lastUsedAt: text('last_used_at'), - /** - * Null = never expires. Always written through normalizeIso (exact - * toISOString shape) so the liveness filter's string comparison against - * "now" is chronologically sound — wire input has variable precision. - */ - expiresAt: text('expires_at'), - /** - * Null = enabled. The reversible half of revocation: set/cleared by - * updateApiKey, and the name stays held while disabled — only revoke - * frees a name. - */ - disabledAt: text('disabled_at'), - /** Null = active. Set once by revokeApiKey; never cleared. */ - revokedAt: text('revoked_at'), - }, - (table) => [ - uniqueIndex('api_keys_active_name_idx') - .on(table.name) - .where(sql`${table.revokedAt} IS NULL`), - ], -); - -export type ApiKeyRow = typeof apiKeys.$inferSelect; From 3be1ab0f3ce784058805dbf5784ec0dc3d773a77 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 17:31:23 +0800 Subject: [PATCH 63/89] install.sh installs two roles, runs the fleet registry, backs up before it restarts, imports the old ledger once, and re-points the right Caddy file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The role is not a file: /etc/dormice/env names the gateway, and a remote one makes a node machine. The first install of a node takes --role node --gateway (--node-id, --node-endpoint optional; the token from the environment, never a flag); re-runs on either role take no flags. A node machine gets the daemon alone, a Caddy on :80 to it with no marker and no source-IP gate (the security group is the fence), learns the fleet's registry and base image from the gateway's getConfig, pins the registry's certificate on first sight with its fingerprint printed, and pulls the base image for doctor's probes. The gateway's machine gets the fleet registry: distribution's static binary, pinned and checksummed, as a systemd unit over TLS (a self-signed ten-year certificate, SAN the listening address) with an htpasswd whose one user takes the fleet token — a registry over plain HTTP cannot take basic auth, so the choice was a lock with TLS or no lock. Docker trusts the certificate through certs.d, no restart. The base image is pushed once; its tag and the registry's address become the gateway's late seeds (appended to an existing gateway.env), and a new env file no longer carries DORMICE_BASE_IMAGE. Before any unit restarts, both databases are snapshotted with SQLite's online backup API (three kept). When the gateway's database does not exist and the daemon's ledger holds a settings row, import.js carries the old configuration over before the gateway's first start; a failure ends the run with nothing restarted. The Caddy re-point targets the file the gateway owns (DORMICE_INGRESS_FILE), only under its marker, and any other file under /etc/caddy still aimed at the daemon is named with its line, not touched. Every curl that needs the token reads it from stdin. dor doctor says the base image is the fleet's setting when the env lacks it, names the registry pull as the fix when there is one, and gains a check that the fleet registry answers over the pinned certificate. --- deploy/dormice-gateway.service | 4 +- deploy/dormice-registry.service | 27 ++ deploy/dormice.service | 5 +- deploy/install.sh | 741 +++++++++++++++++++++++++++----- packages/cli/src/doctor.test.ts | 79 +++- packages/cli/src/doctor.ts | 63 ++- 6 files changed, 808 insertions(+), 111 deletions(-) create mode 100644 deploy/dormice-registry.service diff --git a/deploy/dormice-gateway.service b/deploy/dormice-gateway.service index 5ecd549a..a1dccdd8 100644 --- a/deploy/dormice-gateway.service +++ b/deploy/dormice-gateway.service @@ -6,8 +6,8 @@ # comments only there — systemd's EnvironmentFile treats an inline comment # as part of the value). install.sh restarts this unit before the daemon's, # so both run one commit and the daemon's first check-in lands on the new -# gateway. The `--role node` install for a machine without a gateway is a -# later step. +# gateway. A machine of its own joins as a node with `install.sh --role +# node --gateway :3677` and runs no gateway unit. [Unit] Description=Dormice gateway (fleet front door) Wants=network-online.target diff --git a/deploy/dormice-registry.service b/deploy/dormice-registry.service new file mode 100644 index 00000000..4a9abd06 --- /dev/null +++ b/deploy/dormice-registry.service @@ -0,0 +1,27 @@ +# Dormice image registry: the fleet's one image store, beside the gateway +# (design record #33). The base image install.sh builds is pushed here +# once, template images are pushed by the operator, and every node pulls +# an image it lacks from here under the fleet token (the registry's +# htpasswd holds the same credential as user `dormice`). Standard CNCF +# distribution, the static binary pinned and checksummed by install.sh, +# serving TLS with a self-signed certificate whose SAN is this machine's +# address — the same file is trusted by Docker on the gateway machine +# (/etc/docker/certs.d) and copied to each node when it joins. Config in +# /etc/dormice/registry/registry.yml, images under +# /var/lib/dormice-gateway/registry. OTEL_TRACES_EXPORTER=none: the +# binary otherwise ships traces to localhost:4318 at every request and +# logs the connection refusals. +[Unit] +Description=Dormice image registry (fleet image store) +Wants=network-online.target +After=network-online.target +Before=dormice-gateway.service + +[Service] +ExecStart=/usr/local/bin/registry serve /etc/dormice/registry/registry.yml +Environment=OTEL_TRACES_EXPORTER=none +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/dormice.service b/deploy/dormice.service index 2a8d1e05..995a80ce 100644 --- a/deploy/dormice.service +++ b/deploy/dormice.service @@ -4,8 +4,9 @@ # as part of the value). It takes the fleet's configuration from its # gateway at check-in and, holding no copy yet, waits for the gateway # before it listens — hence After= the gateway unit; not Requires=, since -# a node on a machine of its own has no local gateway and keeps retrying -# the remote one on its own. +# a node on a machine of its own (`install.sh --role node`) has no local +# gateway unit — After= a unit that does not exist is nothing — and keeps +# retrying the remote one on its own. [Unit] Description=Dormice daemon (agent sandbox node) Wants=network-online.target diff --git a/deploy/install.sh b/deploy/install.sh index bd11989c..48f417a3 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -1,18 +1,45 @@ #!/usr/bin/env bash # Dormice installer: turns a bare Ubuntu/Debian x86_64 host into a running -# Dormice — a gateway (the fleet's one door: configuration, keys, the web -# console) and a daemon (the node that runs the sandboxes) on one machine, -# a fleet of one — then proves it by running `dor doctor`. +# Dormice machine and proves it by running `dor doctor`. Two roles, one +# script: # -# curl -fsSL https://raw.githubusercontent.com/BitMiracle-AI/Dormice/main/deploy/install.sh | bash +# The gateway's machine (the default): the gateway (the fleet's one door: +# configuration, keys, the web console), the fleet's image registry, and +# a daemon (a node that runs sandboxes) — on one machine, a fleet of one. +# +# curl -fsSL https://raw.githubusercontent.com/BitMiracle-AI/Dormice/main/deploy/install.sh | bash +# +# A node machine (--role node): a daemon alone, joined to a gateway on +# another machine. Told the gateway's address once, at its first install, +# and nothing else — the fleet's settings, the base image and the +# templates come from the gateway at its first check-in, images it lacks +# from the fleet's registry. The fleet token comes from the environment +# (never a flag: flags show in `ps`; exported, not prefixed — a prefix +# would bind to curl, not to bash): +# +# export DORMICE_API_TOKEN= +# curl -fsSL .../install.sh | bash -s -- --role node --gateway http://10.0.0.5:3677 +# +# The role is not a file: /etc/dormice/env names the gateway +# (DORMICE_GATEWAY_ENDPOINT), and a remote one is what makes a machine a +# node machine. Re-runs — upgrades — need no flags on either role. # # Flags (pass after `bash -s --` when piping): -# --mirror cn use mainland-China mirrors for every download -# --swap-gb N size of the swapfile to create when the host has no swap -# (default 16 — the configuration freezing was measured on) -# --status-dir D write status.json into D as the run progresses — how the -# daemon's one-click upgrade reads the outcome back; a -# manual run doesn't need it +# --mirror cn use mainland-China mirrors for every download +# --swap-gb N size of the swapfile to create when the host has no +# swap (default 16 — the configuration freezing was +# measured on) +# --status-dir D write status.json into D as the run progresses — how +# the one-click upgrade reads the outcome back; a manual +# run doesn't need it +# --role node first install of a node machine (see above) +# --gateway URL the gateway this node joins (--role node, first install) +# --node-id ID this node's name in the fleet (default: the hostname) +# --node-endpoint URL where the gateway reaches this node (default: +# http://:80) +# --registry-addr H:P the address the fleet registry listens on and the +# nodes pull from (gateway's machine; default: this +# machine's private address, port 5000) # # Four promises, mirroring `dor doctor`: # - Idempotent. Every step checks before it acts; a step whose outcome is @@ -45,12 +72,23 @@ GVISOR_DATE=20260622 RUNSC_SHA512=6df95d09363dbd9ee5d5c889c1549b457e1783b039ff60a8f9f16f8c94c774a2ca2eef5b1c370e36b863f6b0407b53ba3c69051c6ef051253843dabf89a6de4e SHIM_SHA512=87c63197836574b7a2c057d2c0647d2badb679187f0b9175ecf78ac52207cdaa3f101629d3e5d165c95930ca35fe81bc26bb90fcf08e09b99c2ee047b6235ce2 +# The fleet's image registry: CNCF distribution's static binary, pinned by +# the sha256 GitHub publishes beside the release tarball (checked +# 2026-09-15). A binary and a systemd unit, like caddy and runsc — not a +# container: no Docker Hub pull to mirror first, and a docker restart does +# not take the fleet's image store down with it. +REGISTRY_VERSION=3.1.1 +REGISTRY_SHA256=6f330a3ba9ea1d23a6ee189f449d792595240585bb2f159123d76ac594f70dd8 +REGISTRY_PORT=5000 + REPO_URL=https://github.com/BitMiracle-AI/Dormice.git INSTALL_DIR=/opt/dormice ENV_FILE=/etc/dormice/env GATEWAY_ENV_FILE=/etc/dormice/gateway.env DATA_DIR=/var/lib/dormice GATEWAY_DATA_DIR=/var/lib/dormice-gateway +REGISTRY_CONF_DIR=/etc/dormice/registry +REGISTRY_DIR=$GATEWAY_DATA_DIR/registry DAEMON_JSON=/etc/docker/daemon.json PORT=3676 GATEWAY_PORT=3677 @@ -59,6 +97,11 @@ GATEWAY_PORT=3677 MIRROR='' SWAP_GB=16 STATUS_DIR='' +ROLE_FLAG='' +GATEWAY_FLAG='' +NODE_ID_FLAG='' +NODE_ENDPOINT_FLAG='' +REGISTRY_ADDR_FLAG='' while [ $# -gt 0 ]; do case "$1" in --mirror) MIRROR="${2:?--mirror needs a value}"; shift 2 ;; @@ -67,13 +110,27 @@ while [ $# -gt 0 ]; do --swap-gb=*) SWAP_GB="${1#*=}"; shift ;; --status-dir) STATUS_DIR="${2:?--status-dir needs a value}"; shift 2 ;; --status-dir=*) STATUS_DIR="${1#*=}"; shift ;; - *) echo "install.sh: unknown flag $1 (known: --mirror cn, --swap-gb N, --status-dir D)" >&2; exit 1 ;; + --role) ROLE_FLAG="${2:?--role needs a value}"; shift 2 ;; + --role=*) ROLE_FLAG="${1#*=}"; shift ;; + --gateway) GATEWAY_FLAG="${2:?--gateway needs a value}"; shift 2 ;; + --gateway=*) GATEWAY_FLAG="${1#*=}"; shift ;; + --node-id) NODE_ID_FLAG="${2:?--node-id needs a value}"; shift 2 ;; + --node-id=*) NODE_ID_FLAG="${1#*=}"; shift ;; + --node-endpoint) NODE_ENDPOINT_FLAG="${2:?--node-endpoint needs a value}"; shift 2 ;; + --node-endpoint=*) NODE_ENDPOINT_FLAG="${1#*=}"; shift ;; + --registry-addr) REGISTRY_ADDR_FLAG="${2:?--registry-addr needs a value}"; shift 2 ;; + --registry-addr=*) REGISTRY_ADDR_FLAG="${1#*=}"; shift ;; + *) echo "install.sh: unknown flag $1 (known: --mirror cn, --swap-gb N, --status-dir D, --role node, --gateway URL, --node-id ID, --node-endpoint URL, --registry-addr HOST:PORT)" >&2; exit 1 ;; esac done if [ -n "$MIRROR" ] && [ "$MIRROR" != cn ]; then echo "install.sh: --mirror only knows \"cn\", got \"$MIRROR\"" >&2 exit 1 fi +if [ -n "$ROLE_FLAG" ] && [ "$ROLE_FLAG" != node ]; then + echo "install.sh: --role only knows \"node\" (the default install is the gateway's machine), got \"$ROLE_FLAG\"" >&2 + exit 1 +fi # log() doubles as the phase tracker: when something fails under `set -e` # there is no message to report, but "which step" is always known. @@ -83,6 +140,47 @@ note() { printf ' %s\n' "$*"; } DIE_MSG='' die() { DIE_MSG="$*"; printf '\ninstall.sh: %s\n' "$*" >&2; exit 1; } +# ---- role ----------------------------------------------------------------- +# Not a file of its own: the daemon's env names its gateway, and a remote +# one is what makes this a node machine — the daemon's config reads it the +# same way (a remote gateway refuses the default node id). Read before +# anything is installed, so a mistaken flag dies before it changes the +# machine. A first install with no env is the gateway's machine unless +# --role node says otherwise; a re-run needs no flag on either role. +url_host() { printf '%s' "$1" | sed -E 's#^[A-Za-z][A-Za-z0-9+.-]*://##; s#/.*$##; s#^\[([^]]*)\].*$#\1#; s#:[0-9]+$##'; } +is_loopback_url() { + case "$(url_host "$1")" in + 127.*|localhost|::1|'') return 0 ;; + *) return 1 ;; + esac +} +ENV_GATEWAY='' +[ -f "$ENV_FILE" ] && ENV_GATEWAY=$(sed -n 's/^DORMICE_GATEWAY_ENDPOINT=//p' "$ENV_FILE" | head -1) +if [ -n "$ENV_GATEWAY" ] && ! is_loopback_url "$ENV_GATEWAY"; then + ROLE=node + GATEWAY_URL=${ENV_GATEWAY%/} + if [ -n "$GATEWAY_FLAG" ] && [ "${GATEWAY_FLAG%/}" != "$GATEWAY_URL" ]; then + die "$ENV_FILE says this node's gateway is $GATEWAY_URL, --gateway says ${GATEWAY_FLAG%/} — edit DORMICE_GATEWAY_ENDPOINT in the env file if the gateway really moved, then re-run without the flag" + fi +elif [ "$ROLE_FLAG" = node ]; then + if [ -f "$ENV_FILE" ]; then + die "$ENV_FILE exists and names a gateway on this machine (or none) — this is the gateway's machine; --role node is for a machine that has never been installed. To turn it into a node, stop and disable dormice-gateway, move the env file aside, and re-run" + fi + [ -n "$GATEWAY_FLAG" ] || die "--role node needs --gateway http://:$GATEWAY_PORT — the gateway this node joins" + case "$GATEWAY_FLAG" in + http://*|https://*) ;; + *) die "--gateway must be a full URL like http://10.0.0.5:$GATEWAY_PORT, got \"$GATEWAY_FLAG\"" ;; + esac + is_loopback_url "$GATEWAY_FLAG" && die "--gateway names this machine ($GATEWAY_FLAG) — a node machine joins a gateway on another machine; the default install (no --role) is the gateway's machine" + [ -n "${DORMICE_API_TOKEN:-}" ] || die "--role node needs the fleet token in the environment: DORMICE_API_TOKEN= bash -s -- --role node --gateway $GATEWAY_FLAG (a flag would show in ps)" + [ "${#DORMICE_API_TOKEN}" -ge 32 ] || die 'DORMICE_API_TOKEN must be at least 32 characters — copy it from the gateway machine: grep ^DORMICE_API_TOKEN /etc/dormice/env' + ROLE=node + GATEWAY_URL=${GATEWAY_FLAG%/} +else + ROLE=gateway + GATEWAY_URL="http://127.0.0.1:$GATEWAY_PORT" +fi + # ---- outcome reporting and the build rollback -------------------------------- # status.json is the one file the daemon's one-click upgrade reads back; # without --status-dir every write is a no-op and a manual run behaves as @@ -580,47 +678,15 @@ ln -sf "$INSTALL_DIR/packages/cli/dist/main.js" /usr/local/bin/dormice ln -sf "$INSTALL_DIR/packages/cli/dist/main.js" /usr/local/bin/dor note "built; \`dormice\` and \`dor\` linked into /usr/local/bin" -# ---- sandbox base image ------------------------------------------------------ -log 'sandbox base image' -existing_image='' -[ -f "$ENV_FILE" ] && existing_image=$(sed -n 's/^DORMICE_BASE_IMAGE=//p' "$ENV_FILE") -if [ -n "$existing_image" ] && docker image inspect "$existing_image" >/dev/null 2>&1; then - base_image=$existing_image - note "[skip] $base_image (from $ENV_FILE) is present" -else - base_image="dormice-base:$(date +%Y%m%d)" - if [ "$MIRROR" = cn ] && ! docker image inspect ubuntu:24.04 >/dev/null 2>&1; then - # Personal registry mirrors in mainland China often proxy only an image - # whitelist; daocloud + retag is the measured workaround. - docker pull -q docker.m.daocloud.io/library/ubuntu:24.04 - docker tag docker.m.daocloud.io/library/ubuntu:24.04 ubuntu:24.04 - docker rmi -f docker.m.daocloud.io/library/ubuntu:24.04 >/dev/null - fi - if [ "$MIRROR" = cn ]; then - # http on purpose: the base image has no CA certificates until this very - # layer installs them, so an https mirror cannot even handshake. apt's - # integrity comes from GPG signatures, not TLS (the default - # archive.ubuntu.com is http too). - docker build -t "$base_image" \ - --build-arg UBUNTU_MIRROR=http://mirrors.aliyun.com/ubuntu/ \ - --build-arg NODE_DIST=https://npmmirror.com/mirrors/node \ - --build-arg PIP_INDEX=https://mirrors.aliyun.com/pypi/simple/ \ - --build-arg NPM_REGISTRY=https://registry.npmmirror.com \ - "$INSTALL_DIR/images" - else - docker build -t "$base_image" "$INSTALL_DIR/images" - fi - note "built $base_image from images/Dockerfile" -fi - # ---- ingress (Caddy reverse proxy) ------------------------------------------- # Gateway and daemon bind 127.0.0.1 by design; Caddy on :80 is what makes -# http:///console reachable from a browser. It proxies to the +# them reachable from outside. On the gateway's machine it proxies to the # GATEWAY — the fleet's one door, which serves the console and forwards -# the sandbox verbs to the daemon. The Caddyfile below is also what the -# gateway rewrites when the operator binds a domain in the console -# (setIngress) — Caddy then obtains and renews the TLS certificate on its -# own. Pinned binary with checksum, same posture as gVisor. +# the sandbox verbs — and its file is also what the gateway rewrites when +# the operator binds a domain in the console (setIngress); Caddy then +# obtains and renews the TLS certificate on its own. On a node machine it +# proxies to the daemon: the gateway reaches the node through it. Pinned +# binary with checksum, same posture as gVisor. log 'ingress (Caddy reverse proxy)' CADDY_VERSION=2.10.0 CADDY_SHA512=626682d623ca04356ab3c9a93a82386cfde6d8243b11f2d0eea9e97ba630c7ada62373401e96b72c6690c98ae8dd004d61fafe477f5249690d5cb251ebbfd2d9 @@ -629,10 +695,14 @@ if command -v caddy >/dev/null; then note "[skip] caddy is installed ($(caddy version | cut -d' ' -f1))" elif ss -ltnH 'sport = :80' 2>/dev/null | grep -q .; then # Another server owns port 80: never fight it. The operator keeps their - # proxy (point it at the gateway, 127.0.0.1:$GATEWAY_PORT); web domain - # binding stays off. + # proxy (point it at the gateway, or at the daemon on a node machine); + # web domain binding stays off. note "port 80 is already in use and caddy is not installed — skipping the ingress layer" - note "point your own reverse proxy at 127.0.0.1:$GATEWAY_PORT (the gateway); the console's domain binding stays disabled" + if [ "$ROLE" = node ]; then + note "point your own reverse proxy at 127.0.0.1:$PORT (the daemon) — the gateway reaches this node through it" + else + note "point your own reverse proxy at 127.0.0.1:$GATEWAY_PORT (the gateway); the console's domain binding stays disabled" + fi else caddy_url="https://github.com/caddyserver/caddy/releases/download/v$CADDY_VERSION/caddy_${CADDY_VERSION}_linux_amd64.tar.gz" [ "$MIRROR" = cn ] && caddy_url="https://ghfast.top/$caddy_url" @@ -647,11 +717,56 @@ INGRESS_FILE_READY='' CADDY_REPOINTED='' if command -v caddy >/dev/null; then mkdir -p /etc/caddy - if [ ! -f "$CADDYFILE" ]; then - # The marker below is the ownership contract: the gateway refuses to - # rewrite a Caddyfile that lacks it. Kept in sync by hand with - # packages/gateway/src/ingress.ts. - cat >"$CADDYFILE" <"$CADDYFILE" <"$CADDYFILE" </dev/null; then } } EOF - note "wrote $CADDYFILE (plain HTTP on :80 to the gateway — bind domains in the console's domains page for HTTPS)" - INGRESS_FILE_READY=1 - elif grep -q 'Managed by Dormice' "$CADDYFILE"; then - if grep -q "reverse_proxy 127.0.0.1:$PORT\b" "$CADDYFILE"; then - # A file from before the gateway became the door (2026-09-14): the - # catch-all still points at the daemon, where the console no longer - # lives. Re-pointed in place; the bound domains, if any, are rewritten - # the same way by the gateway at the next setIngress. - sed -i "s|reverse_proxy 127.0.0.1:$PORT\b|reverse_proxy 127.0.0.1:$GATEWAY_PORT|g" "$CADDYFILE" - note "re-pointed $CADDYFILE from the daemon ($PORT) to the gateway ($GATEWAY_PORT) — the console lives there now" - CADDY_REPOINTED=1 + note "wrote $CADDYFILE (plain HTTP on :80 to the gateway — bind domains in the console's domains page for HTTPS)" + else + note "$ingress_target (DORMICE_INGRESS_FILE) does not exist yet — the gateway writes it at the first domain bind" + fi + INGRESS_FILE_READY=1 + elif grep -q 'Managed by Dormice' "$ingress_target"; then + if grep -q "reverse_proxy 127.0.0.1:$PORT\b" "$ingress_target"; then + # A file from before the gateway became the door (2026-09-14): the + # catch-all still points at the daemon, where the console no longer + # lives. Re-pointed in place; the bound domains, if any, are + # rewritten the same way by the gateway at the next setIngress. + sed -i "s|reverse_proxy 127.0.0.1:$PORT\b|reverse_proxy 127.0.0.1:$GATEWAY_PORT|g" "$ingress_target" + note "re-pointed $ingress_target from the daemon ($PORT) to the gateway ($GATEWAY_PORT) — the console lives there now" + CADDY_REPOINTED=1 + else + note "[skip] $ingress_target is managed by Dormice — left to the gateway" + fi + INGRESS_FILE_READY=1 else - note "[skip] $CADDYFILE is managed by Dormice — left to the gateway" + note "$ingress_target exists but was not written by Dormice — left untouched; domain binding will refuse to overwrite it" + INGRESS_FILE_READY=1 + fi + # The hand-written files (an outer Caddyfile importing the fragment, + # with the wildcard sandbox domain block): a proxy line there still + # aimed at the daemon sends the sandbox domain past the gateway's port + # proxy face. Named with file and line; the operator edits their own + # file. + leftovers=$(grep -rn "reverse_proxy 127.0.0.1:$PORT\b" /etc/caddy 2>/dev/null | grep -v "^$ingress_target:" || true) + if [ -n "$leftovers" ]; then + note "WARNING: these lines under /etc/caddy still proxy to the daemon ($PORT) — change them to 127.0.0.1:$GATEWAY_PORT by hand (the gateway is the door for every face, the sandbox domain included) and reload caddy:" + printf '%s\n' "$leftovers" | sed 's/^/ /' fi - INGRESS_FILE_READY=1 - else - note "$CADDYFILE exists but was not written by Dormice — left untouched; domain binding will refuse to overwrite it" - INGRESS_FILE_READY=1 fi if [ ! -f /etc/systemd/system/caddy.service ]; then cat >/etc/systemd/system/caddy.service </dev/null 2>&1 || systemctl restart caddy + # shellcheck disable=SC2086 # the reload command is the operator's own words, split on purpose + (cd / && $ingress_reload >/dev/null 2>&1) || systemctl restart caddy note 'reloaded caddy with the re-pointed config' else note '[skip] caddy is running' @@ -709,14 +839,48 @@ fi # ---- daemon configuration ---------------------------------------------------- # The daemon's env is the node's identity and its machine: token, executor, -# image, ledger, data dir. The fleet's operator knobs (sandbox defaults, -# the archive store, the sandbox domain, the managed front door) are the -# gateway's since 2026-09-14 — its env seeds them once, the console edits -# them, and the daemon takes them from its check-in. +# ledger, data dir — and, on a node machine, which gateway it belongs to +# and where that gateway reaches it. The fleet's operator knobs (sandbox +# defaults, the base image, the archive store, the sandbox domain, the +# managed front door) are the gateway's since 2026-09-14: its env seeds +# them once, the console edits them, and the daemon takes them from its +# check-in. A DORMICE_BASE_IMAGE line in an older env file stays as the +# daemon's fallback while the fleet names none. log "daemon configuration ($ENV_FILE)" install -d -m 700 "$DATA_DIR" if [ -f "$ENV_FILE" ]; then note "[skip] exists — kept as is (your API token is never rotated); delete it to regenerate" +elif [ "$ROLE" = node ]; then + install -d -m 755 /etc/dormice + NODE_ID=${NODE_ID_FLAG:-$(hostname)} + if [ -n "$NODE_ENDPOINT_FLAG" ]; then + NODE_ENDPOINT=${NODE_ENDPOINT_FLAG%/} + else + # The address this machine speaks to the gateway from — the one the + # gateway can speak back to — on :80, the door Caddy opens above. + gateway_host=$(url_host "$GATEWAY_URL") + gateway_ip=$(getent ahostsv4 "$gateway_host" 2>/dev/null | awk 'NR==1{print $1}') + [ -n "$gateway_ip" ] || die "cannot resolve the gateway's host $gateway_host — check --gateway, or pass --node-endpoint http://:80 as well" + node_ip=$(ip -4 route get "$gateway_ip" 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}') + [ -n "$node_ip" ] || die "cannot tell which address this machine reaches $gateway_ip from — pass --node-endpoint http://:80" + NODE_ENDPOINT="http://$node_ip:80" + fi + cat >"$ENV_FILE" </dev/null 2>&1; then + base_image=$existing_image + note "[skip] $base_image is present" +else + base_image="dormice-base:$(date +%Y%m%d)" + if [ "$MIRROR" = cn ] && ! docker image inspect ubuntu:24.04 >/dev/null 2>&1; then + # Personal registry mirrors in mainland China often proxy only an image + # whitelist; daocloud + retag is the measured workaround. + docker pull -q docker.m.daocloud.io/library/ubuntu:24.04 + docker tag docker.m.daocloud.io/library/ubuntu:24.04 ubuntu:24.04 + docker rmi -f docker.m.daocloud.io/library/ubuntu:24.04 >/dev/null + fi + if [ "$MIRROR" = cn ]; then + # http on purpose: the base image has no CA certificates until this very + # layer installs them, so an https mirror cannot even handshake. apt's + # integrity comes from GPG signatures, not TLS (the default + # archive.ubuntu.com is http too). + docker build -t "$base_image" \ + --build-arg UBUNTU_MIRROR=http://mirrors.aliyun.com/ubuntu/ \ + --build-arg NODE_DIST=https://npmmirror.com/mirrors/node \ + --build-arg PIP_INDEX=https://mirrors.aliyun.com/pypi/simple/ \ + --build-arg NPM_REGISTRY=https://registry.npmmirror.com \ + "$INSTALL_DIR/images" + else + docker build -t "$base_image" "$INSTALL_DIR/images" + fi + note "built $base_image from images/Dockerfile" +fi +# The seed, appended once to an existing gateway.env too (a column born +# after the row: the gateway fills its settings from this while empty, +# and the console edits it from then on). +if ! grep -q '^DORMICE_BASE_IMAGE=' "$GATEWAY_ENV_FILE"; then + { + echo "# The fleet's base image (the image template-less sandboxes boot from);" + echo '# a first-boot seed — the console edits the value in force.' + echo "DORMICE_BASE_IMAGE=$base_image" + } >>"$GATEWAY_ENV_FILE" + note "added DORMICE_BASE_IMAGE=$base_image to $GATEWAY_ENV_FILE (the fleet's setting from here on)" +fi +fi + +# ---- image registry (gateway's machine) -------------------------------------- +# The fleet's one image store (design record #33): a node that lacks an +# image pulls it from here — the base image install.sh pushes below, the +# template images the operator pushes. TLS with a self-signed certificate +# and the fleet token as the password (user `dormice`, bcrypt htpasswd): +# not a preference — a registry over plain HTTP cannot take basic auth +# at all (the distribution documentation says so), so the choice is TLS +# with a lock or no lock, and a store that decides what code every node +# runs gets the lock (design record #34 draws the fleet's one credential; +# this is the same one, no second secret). Docker on this machine trusts +# the certificate through /etc/docker/certs.d — picked up per pull, no +# dockerd restart — and a node copies the same file when it joins. +if [ "$ROLE" = gateway ]; then +log "image registry (distribution v$REGISTRY_VERSION)" +if [ -x /usr/local/bin/registry ] && /usr/local/bin/registry --version 2>/dev/null | grep -q "v$REGISTRY_VERSION\b"; then + note "[skip] registry v$REGISTRY_VERSION is installed" +else + registry_url="https://github.com/distribution/distribution/releases/download/v$REGISTRY_VERSION/registry_${REGISTRY_VERSION}_linux_amd64.tar.gz" + [ "$MIRROR" = cn ] && registry_url="https://ghfast.top/$registry_url" + curl -fsSL -o /tmp/registry.tar.gz "$registry_url" + echo "$REGISTRY_SHA256 /tmp/registry.tar.gz" | sha256sum -c - >/dev/null + tar -C /tmp -xzf /tmp/registry.tar.gz registry + install -m 755 /tmp/registry /usr/local/bin/registry + rm -f /tmp/registry.tar.gz /tmp/registry + note "installed registry v$REGISTRY_VERSION to /usr/local/bin" +fi +# Where it listens and where the nodes pull from: the flag, else what the +# gateway's env already says (re-runs), else the address this machine +# speaks to the world from (the default route's source — on a cloud VPC +# the private address the other machines reach it by; docker0's 172.17.0.1 +# never is). A machine whose main address is public listens on it; the +# lock is what makes that acceptable. +REGISTRY_ADDR=$REGISTRY_ADDR_FLAG +[ -n "$REGISTRY_ADDR" ] || REGISTRY_ADDR=$(sed -n 's/^DORMICE_REGISTRY_ADDRESS=//p' "$GATEWAY_ENV_FILE" | head -1) +if [ -z "$REGISTRY_ADDR" ]; then + registry_host=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}') + [ -n "$registry_host" ] || registry_host=$(hostname -I 2>/dev/null | awk '{print $1}') + [ -n "$registry_host" ] || die "cannot tell this machine's address for the registry to listen on — pass --registry-addr
    :$REGISTRY_PORT" + REGISTRY_ADDR="$registry_host:$REGISTRY_PORT" +fi +registry_host=${REGISTRY_ADDR%:*} +# The SAN as openssl req takes it, and as openssl x509 prints it back. +case "$registry_host" in + *[!0-9.]*) registry_san="DNS:$registry_host"; registry_san_printed="DNS:$registry_host" ;; + *) registry_san="IP:$registry_host"; registry_san_printed="IP Address:$registry_host" ;; +esac +install -d -m 700 "$REGISTRY_CONF_DIR" "$REGISTRY_DIR" +# The certificate: ten years, SAN = the registry's address, its own CA +# (self-signed). Regenerated when the address moved out of its SAN. +if [ -f "$REGISTRY_CONF_DIR/tls.crt" ] && openssl x509 -in "$REGISTRY_CONF_DIR/tls.crt" -noout -ext subjectAltName 2>/dev/null | grep -q "$registry_san_printed"; then + note "[skip] certificate for $registry_san is in place" +else + openssl req -x509 -newkey rsa:2048 -nodes -days 3650 \ + -keyout "$REGISTRY_CONF_DIR/tls.key" -out "$REGISTRY_CONF_DIR/tls.crt" \ + -subj '/CN=dormice-registry' -addext "subjectAltName=$registry_san" >/dev/null 2>&1 + chmod 600 "$REGISTRY_CONF_DIR/tls.key" + note "issued a self-signed certificate for $registry_san (10 years) — sha256 $(openssl x509 -in "$REGISTRY_CONF_DIR/tls.crt" -noout -fingerprint -sha256 | cut -d= -f2)" +fi +# The lock: user dormice, password the fleet token, bcrypt (the only hash +# the registry's htpasswd takes). caddy hashes it — reading the plaintext +# from stdin, never a command line. Written once: the token never rotates. +if [ -f "$REGISTRY_CONF_DIR/htpasswd" ]; then + note '[skip] htpasswd is in place' +else + command -v caddy >/dev/null || die 'caddy is needed to hash the registry password (caddy hash-password) — the ingress step above did not install it because port 80 is taken; install caddy by hand or free port 80, then re-run' + hashed=$(printf '%s\n' "$API_TOKEN" | caddy hash-password 2>/dev/null) + case "$hashed" in + \$2*) ;; + *) die 'caddy hash-password did not produce a bcrypt hash — is caddy >= 2.4?' ;; + esac + printf 'dormice:%s\n' "$hashed" >"$REGISTRY_CONF_DIR/htpasswd" + chmod 600 "$REGISTRY_CONF_DIR/htpasswd" + note "wrote $REGISTRY_CONF_DIR/htpasswd (user dormice, password = the fleet token)" +fi +registry_conf=$(cat </dev/null)" = "$registry_conf" ]; then + note "[skip] registry.yml is in place (listening on $REGISTRY_ADDR)" +else + printf '%s\n' "$registry_conf" >"$REGISTRY_CONF_DIR/registry.yml" + REGISTRY_CHANGED=1 + note "wrote $REGISTRY_CONF_DIR/registry.yml (listening on $REGISTRY_ADDR, TLS, htpasswd)" +fi +if ! cmp -s "$INSTALL_DIR/deploy/dormice-registry.service" /etc/systemd/system/dormice-registry.service; then + cp "$INSTALL_DIR/deploy/dormice-registry.service" /etc/systemd/system/dormice-registry.service + systemctl daemon-reload + REGISTRY_CHANGED=1 +fi +systemctl enable dormice-registry >/dev/null 2>&1 +if [ -n "$REGISTRY_CHANGED" ] || [ "$(systemctl is-active dormice-registry)" != active ]; then + systemctl restart dormice-registry +fi +# Docker's trust in the certificate: the file's presence is the whole +# mechanism — no daemon.json edit, no restart. +install -d "/etc/docker/certs.d/$REGISTRY_ADDR" +if ! cmp -s "$REGISTRY_CONF_DIR/tls.crt" "/etc/docker/certs.d/$REGISTRY_ADDR/ca.crt"; then + install -m 644 "$REGISTRY_CONF_DIR/tls.crt" "/etc/docker/certs.d/$REGISTRY_ADDR/ca.crt" + note "trusted the certificate for docker: /etc/docker/certs.d/$REGISTRY_ADDR/ca.crt" +fi +for _ in $(seq 1 40); do + registry_code=$(curl -s -o /dev/null -w '%{http_code}' --cacert "$REGISTRY_CONF_DIR/tls.crt" "https://$REGISTRY_ADDR/v2/" 2>/dev/null || true) + [ "$registry_code" = 401 ] && break + sleep 0.5 +done +[ "$registry_code" = 401 ] \ + || die "the registry did not answer on https://$REGISTRY_ADDR/v2/ (got '${registry_code:-nothing}', expected 401 asking for the credential) — check: journalctl -u dormice-registry -n 50" +note "registry is answering on https://$REGISTRY_ADDR (TLS, asks for the fleet credential)" +if ! grep -q '^DORMICE_REGISTRY_ADDRESS=' "$GATEWAY_ENV_FILE"; then + { + echo "# The fleet's image registry (host:port), run by this machine's" + echo '# dormice-registry unit: nodes pull the images they lack from here.' + echo "DORMICE_REGISTRY_ADDRESS=$REGISTRY_ADDR" + } >>"$GATEWAY_ENV_FILE" + note "added DORMICE_REGISTRY_ADDRESS=$REGISTRY_ADDR to $GATEWAY_ENV_FILE" +fi +# The base image into the store, once per tag (a manifest already there +# is skipped). Template images are the operator's to push — templates.mdx +# has the three lines; the registry credential is the fleet token. +manifest_code=$(curl_basic_config | curl -s -K - -o /dev/null -w '%{http_code}' --cacert "$REGISTRY_CONF_DIR/tls.crt" \ + -H 'Accept: application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json' \ + "https://$REGISTRY_ADDR/v2/${base_image%:*}/manifests/${base_image##*:}" 2>/dev/null || true) +if [ "$manifest_code" = 200 ]; then + note "[skip] $base_image is in the registry" +else + printf '%s' "$API_TOKEN" | docker login "$REGISTRY_ADDR" -u dormice --password-stdin >/dev/null 2>&1 \ + || die "docker login to https://$REGISTRY_ADDR refused the fleet credential — if the token changed after $REGISTRY_CONF_DIR/htpasswd was written, delete that file and re-run" + docker tag "$base_image" "$REGISTRY_ADDR/$base_image" + docker push -q "$REGISTRY_ADDR/$base_image" >/dev/null + # The credential does not stay in /root/.docker/config.json: the daemon + # presents it per pull from memory. + docker logout "$REGISTRY_ADDR" >/dev/null 2>&1 || true + note "pushed $base_image to the registry as $REGISTRY_ADDR/$base_image" +fi +fi + +# ---- joining the fleet (node machine) ---------------------------------------- +# What a node needs from its gateway before its daemon starts: the fleet +# registry's certificate, pinned into Docker's trust store on first sight +# (the ssh posture: the fingerprint is printed, and an intranet +# man-in-the-middle at install time is outside the threat model of design +# record #34), and the base image pulled ahead so `dor doctor`'s container +# probes have it — the daemon would pull it on its own at its first +# bundle, but a minute later, in the background. +if [ "$ROLE" = node ]; then +log "joining the fleet at $GATEWAY_URL" +fleet_config=$(curl_auth_config | curl -fsS -K - -X POST -H 'content-type: application/json' -d '{}' "$GATEWAY_URL/getConfig" 2>/dev/null) \ + || die "the gateway at $GATEWAY_URL did not answer getConfig — is it running, is :$GATEWAY_PORT open to this machine, is DORMICE_API_TOKEN the gateway machine's token? (curl -fsS $GATEWAY_URL/healthz answers without a token)" +read -r FLEET_REGISTRY FLEET_BASE_IMAGE </dev/null \ + | openssl x509 -outform PEM >"$cert_dir/ca.crt" 2>/dev/null \ + || die "could not fetch the registry's certificate from https://$FLEET_REGISTRY — is :${FLEET_REGISTRY##*:} on the gateway machine open to this machine?" + [ -s "$cert_dir/ca.crt" ] || die "https://$FLEET_REGISTRY presented no certificate" + note "pinned the registry's certificate on first sight: $cert_dir/ca.crt — sha256 $(openssl x509 -in "$cert_dir/ca.crt" -noout -fingerprint -sha256 | cut -d= -f2)" + note "compare it with the gateway machine's: openssl x509 -in $REGISTRY_CONF_DIR/tls.crt -noout -fingerprint -sha256" + fi + registry_code=$(curl -s -o /dev/null -w '%{http_code}' --cacert "$cert_dir/ca.crt" "https://$FLEET_REGISTRY/v2/" 2>/dev/null || true) + [ "$registry_code" = 401 ] || die "the registry at https://$FLEET_REGISTRY did not answer as expected (got '${registry_code:-nothing}', expected 401 asking for the credential)" + if [ "$FLEET_BASE_IMAGE" != - ]; then + if docker image inspect "$FLEET_BASE_IMAGE" >/dev/null 2>&1; then + note "[skip] base image $FLEET_BASE_IMAGE is present" + else + printf '%s' "$API_TOKEN" | docker login "$FLEET_REGISTRY" -u dormice --password-stdin >/dev/null 2>&1 \ + || die "docker login to https://$FLEET_REGISTRY refused the fleet credential — DORMICE_API_TOKEN here must be the gateway machine's token" + docker pull -q "$FLEET_REGISTRY/$FLEET_BASE_IMAGE" >/dev/null \ + || die "could not pull $FLEET_REGISTRY/$FLEET_BASE_IMAGE — on the gateway machine, re-run install.sh (it pushes the base image), then re-run here" + docker tag "$FLEET_REGISTRY/$FLEET_BASE_IMAGE" "$FLEET_BASE_IMAGE" + docker logout "$FLEET_REGISTRY" >/dev/null 2>&1 || true + note "pulled the fleet's base image $FLEET_BASE_IMAGE from the registry" + fi + fi +elif [ "$FLEET_BASE_IMAGE" != - ] && ! docker image inspect "$FLEET_BASE_IMAGE" >/dev/null 2>&1; then + note "WARNING: the fleet has no registry and this machine lacks the base image $FLEET_BASE_IMAGE — build or load it here under that name before creating sandboxes" +fi +fi + +# ---- backups (before anything restarts) -------------------------------------- +# A copy of every database this run may migrate, taken through SQLite's +# online backup API (a consistent snapshot even while the daemon writes; +# the hosts have no sqlite3 CLI), before any unit is restarted: the new +# daemon migrates its ledger forward at boot, and a downgrade past a +# migration is not a one-click affair. Three kept, the oldest dropped. +log 'backups' +backup_db() { # + [ -f "$1" ] || return 0 + mkdir -p "$2" + (cd "$INSTALL_DIR/packages/server" && node -e ' +const Database = require("better-sqlite3"); +const [src, dst] = process.argv.slice(1); +const db = new Database(src, { readonly: true }); +db.backup(dst).then(() => { db.close(); }).catch((err) => { console.error(err.message); process.exit(1); }); +' "$1" "$2/$(basename "$1")") || die "backup of $1 failed" + chmod 600 "$2/$(basename "$1")" +} +BACKUP_DIR="$DATA_DIR/backups/$(date -u +%Y%m%dT%H%M%SZ)-${OLD_SHA:-fresh}" +backed='' +if [ -f "$DATA_DIR/dormice.db" ]; then + backup_db "$DATA_DIR/dormice.db" "$BACKUP_DIR" + backed="$backed dormice.db" +fi +if [ "$ROLE" = gateway ] && [ -f "$GATEWAY_DATA_DIR/gateway.db" ]; then + backup_db "$GATEWAY_DATA_DIR/gateway.db" "$BACKUP_DIR" + backed="$backed gateway.db" +fi +if [ -n "$backed" ]; then + chmod 700 "$BACKUP_DIR" + note "backed up$backed to $BACKUP_DIR" + # shellcheck disable=SC2012 # ls sorts the timestamped names; the directory is ours + ls -1d "$DATA_DIR"/backups/*/ 2>/dev/null | sort | head -n -3 | while read -r old; do + rm -rf "$old" + note "dropped old backup $old" + done +else + note '[skip] no database yet — a first install has nothing to back up' +fi + +# ---- the old ledger's configuration into the gateway (once) ------------------ +# A machine that ran as a single daemon before the gateway existed holds +# the fleet's configuration in its ledger — the S3 store, the default +# policy, the domain aliases, the templates, the API keys, the console +# account — and it must be in the gateway's database BEFORE the gateway's +# first start: at that start the gateway seeds its settings from the env +# and the daemon takes that at its first check-in, and months of operator +# settings would be quietly gone (a restore from archive would fail, a +# template sandbox would wake to "not registered"). The import is the +# gateway package's own tool (import-ledger.ts has the translation); a +# failure here ends the run before any unit restarts, with the ledger +# untouched. +if [ "$ROLE" = gateway ]; then +log 'importing the single-machine ledger into the gateway' +if [ -f "$GATEWAY_DATA_DIR/gateway.db" ]; then + note '[skip] the gateway database exists — the import is for its first start' +elif [ ! -f "$DATA_DIR/dormice.db" ]; then + note '[skip] no daemon ledger — a fresh install has nothing to import' +else + has_settings=$(cd "$INSTALL_DIR/packages/server" && node -e ' +const Database = require("better-sqlite3"); +const db = new Database(process.argv[1], { readonly: true }); +try { console.log(db.prepare("SELECT count(*) AS n FROM runtime_settings").get().n); } catch { console.log(0); } +db.close(); +' "$DATA_DIR/dormice.db") + if [ "$has_settings" = 0 ]; then + note '[skip] the daemon ledger holds no settings row — nothing to import' + else + imported=$( + set -a + # shellcheck source=/dev/null + . "$GATEWAY_ENV_FILE" + set +a + node "$INSTALL_DIR/packages/gateway/dist/import.js" --node-db "$DATA_DIR/dormice.db" --node-env "$ENV_FILE" + ) || die "the import of $DATA_DIR/dormice.db into the gateway failed — nothing was restarted; fix the cause and re-run (the gateway database, if half-written, is at $GATEWAY_DATA_DIR/gateway.db: delete it before the re-run)" + note "imported into the gateway: $imported" + fi +fi +fi # ---- systemd services -------------------------------------------------------- -# Two units, the gateway first: a daemon without a configuration copy takes -# its first bundle from its gateway before it listens, and a re-run just -# built both dists — the two processes of a fleet of one run one commit, -# never two. Restart, not start: both are crash-only by design, so -# restarting them is always safe. +# The gateway's machine: two units, the gateway first — a daemon without a +# configuration copy takes its first bundle from its gateway before it +# listens, and a re-run just built both dists: the two processes of a +# fleet of one run one commit, never two. A node machine: the daemon +# alone, joined to its remote gateway. Restart, not start: all crash-only +# by design, so restarting them is always safe. log 'systemd services' -cp "$INSTALL_DIR/deploy/dormice-gateway.service" /etc/systemd/system/dormice-gateway.service cp "$INSTALL_DIR/deploy/dormice.service" /etc/systemd/system/dormice.service -systemctl daemon-reload -systemctl enable dormice-gateway dormice >/dev/null 2>&1 -systemctl restart dormice-gateway -for _ in $(seq 1 60); do - curl -fsS "http://127.0.0.1:$GATEWAY_PORT/healthz" >/dev/null 2>&1 && break - sleep 0.5 -done -curl -fsS "http://127.0.0.1:$GATEWAY_PORT/healthz" >/dev/null 2>&1 \ - || die "the gateway did not answer /healthz on 127.0.0.1:$GATEWAY_PORT — check: journalctl -u dormice-gateway -n 50" -note "gateway is answering on 127.0.0.1:$GATEWAY_PORT" -systemctl restart dormice -note 'enabled and (re)started both' +if [ "$ROLE" = gateway ]; then + cp "$INSTALL_DIR/deploy/dormice-gateway.service" /etc/systemd/system/dormice-gateway.service + systemctl daemon-reload + systemctl enable dormice-gateway dormice >/dev/null 2>&1 + systemctl restart dormice-gateway + for _ in $(seq 1 60); do + curl -fsS "http://127.0.0.1:$GATEWAY_PORT/healthz" >/dev/null 2>&1 && break + sleep 0.5 + done + curl -fsS "http://127.0.0.1:$GATEWAY_PORT/healthz" >/dev/null 2>&1 \ + || die "the gateway did not answer /healthz on 127.0.0.1:$GATEWAY_PORT — check: journalctl -u dormice-gateway -n 50" + note "gateway is answering on 127.0.0.1:$GATEWAY_PORT" + systemctl restart dormice + note 'enabled and (re)started both' +else + systemctl daemon-reload + systemctl enable dormice >/dev/null 2>&1 + systemctl restart dormice + note 'enabled and (re)started the daemon' +fi # ---- verification: the install has not succeeded until doctor says so -------- log 'verification' @@ -838,13 +1355,21 @@ done curl -fsS "http://127.0.0.1:$PORT/healthz" >/dev/null 2>&1 \ || die "the daemon did not answer /healthz on 127.0.0.1:$PORT — check: journalctl -u dormice -n 50 (a daemon with no configuration copy waits for its gateway before it listens)" note "daemon is answering on 127.0.0.1:$PORT" -# Both env files: doctor reads the node's knobs from the daemon's and the -# fleet's seeds (the S3 set, the managed front door) from the gateway's. set -a # shellcheck source=/dev/null . "$ENV_FILE" -# shellcheck source=/dev/null -. "$GATEWAY_ENV_FILE" +if [ "$ROLE" = gateway ]; then + # Both env files: doctor reads the node's knobs from the daemon's and the + # fleet's seeds (the base image, the S3 set, the registry, the managed + # front door) from the gateway's. + # shellcheck source=/dev/null + . "$GATEWAY_ENV_FILE" +else + # A node's env names no image and no registry — the fleet's, learned + # from the gateway above — so doctor is told them for this run alone. + [ "$FLEET_BASE_IMAGE" != - ] && export DORMICE_BASE_IMAGE=$FLEET_BASE_IMAGE + [ "$FLEET_REGISTRY" != - ] && export DORMICE_REGISTRY_ADDRESS=$FLEET_REGISTRY +fi set +a dor doctor @@ -854,13 +1379,23 @@ dor doctor status_write succeeded printf '\nDormice is installed.\n' +if [ "$ROLE" = node ]; then + printf ' role: node %s of gateway %s, reached at %s\n' "$(sed -n 's/^DORMICE_NODE_ID=//p' "$ENV_FILE")" "$GATEWAY_URL" "$(sed -n 's/^DORMICE_NODE_ENDPOINT=//p' "$ENV_FILE")" + printf ' daemon logs: journalctl -u dormice -f\n' + printf ' the fleet is driven from its gateway: console, keys, settings, templates, upgrades all answer there.\n' + printf ' cloud firewall: allow :80 on this machine from the gateway machine only.\n' + exit 0 +fi printf ' API token: grep ^DORMICE_API_TOKEN %s\n' "$ENV_FILE" printf ' gateway logs: journalctl -u dormice-gateway -f (the door: console, keys, settings, templates)\n' printf ' daemon logs: journalctl -u dormice -f (the node: sandboxes)\n' +printf ' registry: https://%s (TLS, user dormice, password = the API token; journalctl -u dormice-registry -f)\n' "$REGISTRY_ADDR" printf ' CLI: export DORMICE_ENDPOINT=http://127.0.0.1:%s DORMICE_API_TOKEN=; dor sandbox ls\n' "$GATEWAY_PORT" printf ' (the gateway is the door for every verb; a node answers only the sandbox and host verbs\n' printf ' for itself on 127.0.0.1:%s)\n' "$PORT" printf ' Both processes listen on 127.0.0.1 only, by design — exposing them is a reverse proxy'"'"'s job.\n' +printf ' add a node: on another machine of the same network, with :%s and :%s here open to it:\n' "$GATEWAY_PORT" "$REGISTRY_PORT" +printf ' DORMICE_API_TOKEN= bash install.sh --role node --gateway http://%s:%s\n' "${REGISTRY_ADDR%:*}" "$GATEWAY_PORT" if [ "$(systemctl is-active caddy 2>/dev/null)" = active ]; then printf ' console: http:///console (Caddy on :80 -> the gateway; open your cloud firewall for\n' printf ' 80/443, then bind domains in the domains page for automatic HTTPS)\n' diff --git a/packages/cli/src/doctor.test.ts b/packages/cli/src/doctor.test.ts index 15dc2ab2..42cfbd3e 100644 --- a/packages/cli/src/doctor.test.ts +++ b/packages/cli/src/doctor.test.ts @@ -85,6 +85,8 @@ function fakeHost( [FIREWALL_UNIT_PATH]: FIREWALL_UNIT_GOOD, '/etc/caddy/Caddyfile': '# Managed by Dormice — setIngress rewrites this file.\n\n:80 {\n\treverse_proxy 127.0.0.1:3676\n}\n', + '/etc/docker/certs.d/10.0.0.5:5000/ca.crt': + '-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n', ...overrides.files, }; const commands: Record = { @@ -109,6 +111,8 @@ function fakeHost( 'zstd --version': ok('zstd command line interface v1.5.5\n'), 'caddy version': ok('v2.10.0 h1:fakehash\n'), 'systemctl is-active caddy': ok('active\n'), + 'curl -sS -o /dev/null -w %{http_code} --cacert /etc/docker/certs.d/10.0.0.5:5000/ca.crt https://10.0.0.5:5000/v2/': + ok('401'), ...overrides.commands, }; const calls: string[] = []; @@ -127,6 +131,7 @@ function fakeHost( DORMICE_S3_ACCESS_KEY_ID: 'minio-user', DORMICE_S3_SECRET_ACCESS_KEY: 'minio-secret', DORMICE_INGRESS_FILE: '/etc/caddy/Caddyfile', + DORMICE_REGISTRY_ADDRESS: '10.0.0.5:5000', }, calls, run: async (cmd, args) => { @@ -636,19 +641,46 @@ describe('daemon configuration', () => { }); }); - it('without a base image, image check and probes skip with directions', async () => { + it("without a base image, image check and probes skip with directions — the base image is the fleet's setting", async () => { const { results, failed } = await runDoctor( fakeHost({ env: { DORMICE_API_TOKEN: TOKEN } }), ); expect(results['base-image']).toMatchObject({ status: 'skip', - detail: expect.stringContaining('DORMICE_BASE_IMAGE'), + detail: expect.stringMatching(/DORMICE_BASE_IMAGE.*fleet's setting/), }); expect(statusOf(results, 'probe-gvisor')).toBe('skip'); expect(statusOf(results, 'absolute-paths')).toBe('skip'); + expect(statusOf(results, 'registry')).toBe('skip'); expect(failed).toBe(false); }); + it('a missing base image names the pull from the fleet registry when there is one, the build otherwise', async () => { + const missing = { + commands: { [`docker image inspect ${IMAGE}`]: boom('No such image') }, + }; + const withRegistry = await runDoctor(fakeHost(missing)); + expect(withRegistry.results['base-image']).toMatchObject({ + status: 'fail', + fix: expect.stringContaining(`docker pull 10.0.0.5:5000/${IMAGE}`), + }); + const alone = await runDoctor( + fakeHost({ + ...missing, + env: { + DORMICE_API_TOKEN: TOKEN, + DORMICE_EXECUTOR: 'docker', + DORMICE_BASE_IMAGE: IMAGE, + DORMICE_DB_PATH: '/var/lib/dormice/dormice.db', + }, + }), + ); + expect(alone.results['base-image']).toMatchObject({ + status: 'fail', + fix: expect.stringContaining('images/Dockerfile'), + }); + }); + it('scarce disk space warns with the measured number', async () => { const { results } = await runDoctor( fakeHost({ @@ -667,6 +699,49 @@ describe('daemon configuration', () => { }); }); +describe('fleet image registry', () => { + it('reachable over TLS and asking for the credential passes; a missing pinned certificate, a dead address and an unexpected status each fail with their own fix', async () => { + const healthy = await runDoctor(fakeHost()); + expect(healthy.results.registry).toMatchObject({ + status: 'pass', + detail: expect.stringContaining('asks for the fleet credential'), + }); + const unpinned = await runDoctor( + fakeHost({ + files: { '/etc/docker/certs.d/10.0.0.5:5000/ca.crt': undefined }, + }), + ); + expect(unpinned.results.registry).toMatchObject({ + status: 'fail', + detail: expect.stringContaining('ca.crt is missing'), + }); + const dead = await runDoctor( + fakeHost({ + commands: { + 'curl -sS -o /dev/null -w %{http_code} --cacert /etc/docker/certs.d/10.0.0.5:5000/ca.crt https://10.0.0.5:5000/v2/': + boom('curl: (7) Failed to connect to 10.0.0.5 port 5000'), + }, + }), + ); + expect(dead.results.registry).toMatchObject({ + status: 'fail', + detail: expect.stringContaining('Failed to connect'), + }); + const odd = await runDoctor( + fakeHost({ + commands: { + 'curl -sS -o /dev/null -w %{http_code} --cacert /etc/docker/certs.d/10.0.0.5:5000/ca.crt https://10.0.0.5:5000/v2/': + ok('502'), + }, + }), + ); + expect(odd.results.registry).toMatchObject({ + status: 'fail', + detail: expect.stringContaining('answered 502'), + }); + }); +}); + describe('container probes', () => { it('a real host kernel in the probe means gVisor is not isolating', async () => { const { results, failed } = await runDoctor( diff --git a/packages/cli/src/doctor.ts b/packages/cli/src/doctor.ts index 1633ec1a..0d1d69bc 100644 --- a/packages/cli/src/doctor.ts +++ b/packages/cli/src/doctor.ts @@ -719,10 +719,17 @@ const CHECKS: DoctorCheck[] = [ title: 'base image available', needs: ['docker-daemon'], run: async (ctx) => { + // The base image is the fleet's setting since the fourth cut + // (2026-09-15): the gateway's env seeds it, the console edits it, + // a node takes it at check-in and pulls it from the fleet registry + // when the host lacks it. Doctor stays an offline preflight and + // reads the environment it was given: install.sh loads the + // gateway's env on the gateway's machine, and on a node machine + // tells doctor the value it learned from the gateway for this run. const image = baseImage(ctx); if (!image) { return skip( - 'DORMICE_BASE_IMAGE is not set — set it to check the image and enable the container probes', + "DORMICE_BASE_IMAGE is not set in this environment — the base image is the fleet's setting (console › settings; the gateway's env seeds it), and a node pulls it from the fleet registry when it lacks it; export DORMICE_BASE_IMAGE= to check the image here and enable the container probes", ); } const res = await ctx.run('docker', ['image', 'inspect', image]); @@ -730,7 +737,59 @@ const CHECKS: DoctorCheck[] = [ ? pass(`${image} is present locally`) : fail( `${image} is not present locally`, - 'build it from images/Dockerfile — doctor never pulls images itself', + ctx.env.DORMICE_REGISTRY_ADDRESS + ? `pull it from the fleet registry: docker pull ${ctx.env.DORMICE_REGISTRY_ADDRESS}/${image} && docker tag ${ctx.env.DORMICE_REGISTRY_ADDRESS}/${image} ${image} (the daemon does this on its own at its next configuration bundle) — doctor never pulls images itself` + : 'build it from images/Dockerfile — doctor never pulls images itself', + ); + }, + }, + { + id: 'registry', + title: 'fleet image registry reachable', + run: async (ctx) => { + // The fleet's image store (install.sh runs one beside the gateway, + // TLS with a self-signed certificate Docker trusts through + // /etc/docker/certs.d). Reachable and asking for the credential is + // the whole check: a 401 from /v2/ proves the address, the TLS trust + // and that a registry answers — without a credential on any command + // line. The pull itself is the daemon's, per image. + const address = ctx.env.DORMICE_REGISTRY_ADDRESS; + if (!address) { + return skip( + "DORMICE_REGISTRY_ADDRESS not set in this environment — no fleet registry (the gateway's variable; without one every node must have its images staged by hand)", + ); + } + const ca = `/etc/docker/certs.d/${address}/ca.crt`; + if ((await ctx.readTextFile(ca)) === undefined) { + return fail( + `${ca} is missing — docker cannot trust the registry's certificate, so every pull from ${address} fails`, + "re-run install.sh: on the gateway's machine it writes the certificate there; on a node it pins the gateway's on first sight", + ); + } + const res = await ctx.run('curl', [ + '-sS', + '-o', + '/dev/null', + '-w', + '%{http_code}', + '--cacert', + ca, + `https://${address}/v2/`, + ]); + const code = res.stdout.trim(); + if (!res.ok) { + return fail( + `https://${address}/v2/ did not answer: ${res.stderr.trim() || 'no reason given'}`, + "on the gateway's machine: systemctl status dormice-registry; on a node: is :5000 on the gateway machine open to this one?", + ); + } + return code === '401' || code === '200' + ? pass( + `https://${address}/v2/ answers ${code} — reachable over TLS${code === '401' ? ', asks for the fleet credential' : ''}`, + ) + : fail( + `https://${address}/v2/ answered ${code}, not the 401 a registry gives without a credential`, + 'journalctl -u dormice-registry on the gateway machine', ); }, }, From 03003d9e6f67396e6351a0b6fffbedeba55c8b84 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 17:31:23 +0800 Subject: [PATCH 64/89] e2e: the base image re-pointed at the gateway reaches the nodes' copies and their image verdicts; getConfig carries the two new settings --- e2e/src/gateway.test.ts | 43 +++++++++++++++++++++++++++++++++++++++++ e2e/src/native.test.ts | 6 ++++++ 2 files changed, 49 insertions(+) diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts index 7b6b5b6b..87d4b920 100644 --- a/e2e/src/gateway.test.ts +++ b/e2e/src/gateway.test.ts @@ -124,6 +124,49 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { } }); + it("the fleet's base image is a setting: re-pointed at the gateway, it reaches the nodes' copies within a check-in and every template-less sandbox reads as upgradable", async () => { + const before = (await rpc('/getConfig')).body as { + configVersion: number; + settings: { baseImage: string | null; registryAddress: string | null }; + }; + // The exam's gateway seeds the fake's own base and no registry. + expect(before.settings.baseImage).toBe('fake-base'); + expect(before.settings.registryAddress).toBeNull(); + const staged = await direct('node-b').acquireSandbox('gw-base'); + try { + const { status } = await rpc('/updateSettings', { + baseImage: 'fake-base-2', + }); + expect(status).toBe(200); + await configSettled(gateway(), token()); + // The node's own copy resolves the next image; the merged list at + // the door reads the same verdict. + const onNode = await direct('node-b').listSandboxImages(); + const row = onNode.images.find((i) => i.sandboxId === staged.sandbox.id); + expect(row).toMatchObject({ + image: 'fake-base', + nextImage: 'fake-base-2', + upgradable: true, + }); + const atDoor = await viaGateway().listSandboxImages(); + expect( + atDoor.images.find((i) => i.sandboxId === staged.sandbox.id)?.nextImage, + ).toBe('fake-base-2'); + // Back, and the verdict follows. + await rpc('/updateSettings', { baseImage: 'fake-base' }); + await configSettled(gateway(), token()); + expect( + (await direct('node-b').listSandboxImages()).images.find( + (i) => i.sandboxId === staged.sandbox.id, + )?.upgradable, + ).toBe(false); + } finally { + await direct('node-b').destroySandbox('gw-base'); + await rpc('/updateSettings', { baseImage: 'fake-base' }); + await configSettled(gateway(), token()); + } + }); + it('a template registered at the gateway is usable on every node; removal asks the nodes and is refused while one holds a sandbox on it', async () => { await viaGateway().registerTemplate('gw-tpl', 'img:gw-tpl'); await configSettled(gateway(), token()); diff --git a/e2e/src/native.test.ts b/e2e/src/native.test.ts index 4ebabf2b..f8c5b2e0 100644 --- a/e2e/src/native.test.ts +++ b/e2e/src/native.test.ts @@ -808,6 +808,12 @@ describe('the observability verbs over a real daemon', () => { defaultSeconds: 7 * 24 * 60 * 60, }); expect(config.configVersion).toBeGreaterThanOrEqual(1); + // The fleet's base image and registry are settings since the fourth + // cut: seeded by the exam's gateway env (an image name, no registry). + expect(config.settings.baseImage).toBe( + process.env.DORMICE_BASE_IMAGE ?? 'fake-base', + ); + expect(config.settings.registryAddress).toBeNull(); // The node answers no configuration verb of its own anymore. await expect(client().getConfig()).rejects.toMatchObject({ status: 404 }); }); From 2566db4f9a923f413d2519611b338868de3afb91 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 17:31:23 +0800 Subject: [PATCH 65/89] Docs: adding a node, the fleet upgrade, the base image as a setting, images through the fleet registry, the two new doctor checks --- .changeset/fleet-install-and-upgrade.md | 7 ++++ README.md | 7 ++++ website/content/docs/architecture.mdx | 11 ++++-- website/content/docs/configuration.mdx | 26 +++++++----- website/content/docs/console.mdx | 4 ++ website/content/docs/doctor.mdx | 17 +++++++- website/content/docs/installation.mdx | 50 ++++++++++++++++++++---- website/content/docs/templates.mdx | 28 ++++++++++--- website/content/docs/troubleshooting.mdx | 21 +++++++++- website/content/docs/upgrading.mdx | 39 ++++++++++++------ 10 files changed, 167 insertions(+), 43 deletions(-) create mode 100644 .changeset/fleet-install-and-upgrade.md diff --git a/.changeset/fleet-install-and-upgrade.md b/.changeset/fleet-install-and-upgrade.md new file mode 100644 index 00000000..1d97e268 --- /dev/null +++ b/.changeset/fleet-install-and-upgrade.md @@ -0,0 +1,7 @@ +--- +"@dormice/shared": minor +"@dormice/sdk": minor +"@dormice/cli": minor +--- + +The fleet installs and upgrades as one. The base image is a fleet setting (`settings.baseImage`, seeded from the gateway's `DORMICE_BASE_IMAGE`, edited with `updateSettings { baseImage }`) beside a read-only `registryAddress`; both ride the configuration bundle to every node, and a node pulls an image it lacks from the fleet registry. The upgrade verbs answer at the gateway: `applyUpgrade` upgrades the gateway's machine and then every node behind it, told one at a time at its check-in; `applyUpgrade { nodeId }` tells a stuck node again; `getUpgradeStatus.nodes[]` lists each node's standing. The check-in carries `selfUpgrade` and can answer `upgrade: true`. `dor doctor` reads the fleet's base image from the environment the installer gives it and checks the fleet registry is reachable over TLS. diff --git a/README.md b/README.md index 607b8dbc..f0179c49 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,13 @@ a battery of read-only checks — three of them boot a real gVisor container — that decides whether the install actually succeeded; `dor doctor` can be re-run on its own at any time. +A second machine joins the same fleet with one more command +(`--role node --gateway http://:3677`, the token in the +environment) and needs no settings of its own; upgrades then run from +the gateway, one node at a time. See the +[installation](website/content/docs/installation.mdx) and +[upgrading](website/content/docs/upgrading.mdx) docs. + ## Quick start `@dormice/sdk` is the native TypeScript client. (Not on npm yet — the diff --git a/website/content/docs/architecture.mdx b/website/content/docs/architecture.mdx index 5faa1101..4592aee6 100644 --- a/website/content/docs/architecture.mdx +++ b/website/content/docs/architecture.mdx @@ -126,10 +126,13 @@ older than their template's current one; the console shows it as an Pick something else if: -- **You need a fleet.** One machine, one daemon, by design — that is - where the simplicity comes from. Multi-machine sharding is a future - direction (the schema already carries the fields), not a current - feature. +- **You need one giant sandbox pool with cross-machine scheduling.** A + Dormice fleet is one gateway in front of independent nodes, each a + complete single-machine daemon: the gateway places a new sandbox on a + node and keeps finding it there for the rest of its life. Nothing + moves a live sandbox between machines, and nothing shares state + between nodes but the gateway's configuration and the image registry. + That is where the simplicity comes from. - **You want a managed service.** No hosted anything, no SLA. That is E2B's product, and it is good at it. - **Your threat model demands hardware virtualization.** Sandboxes are diff --git a/website/content/docs/configuration.mdx b/website/content/docs/configuration.mdx index bb5374e5..4cd4f689 100644 --- a/website/content/docs/configuration.mdx +++ b/website/content/docs/configuration.mdx @@ -13,10 +13,11 @@ A Dormice host runs two processes, each with its own environment file: and keeps a copy in its own database, so it keeps serving while the gateway is away. `/etc/dormice/env`. -A single machine runs both — a fleet of one. All variables are -`DORMICE_*`, validated once at startup: a bad value refuses to boot with -a named error instead of surfacing later as confusing runtime behavior. -Edit a file and restart its process: +A single machine runs both — a fleet of one; a node machine +(`install.sh --role node`) runs the daemon alone, its env naming the +gateway. All variables are `DORMICE_*`, validated once at startup: a +bad value refuses to boot with a named error instead of surfacing later +as confusing runtime behavior. Edit a file and restart its process: ```sh systemctl restart dormice-gateway # after editing gateway.env @@ -103,6 +104,13 @@ sandboxes [archive](/docs/archiving) after 7 more idle days by default. | `DORMICE_S3_REGION` | `us-east-1` | Region, if your store cares. | | `DORMICE_S3_FORCE_PATH_STYLE` | `false` | Path-style addressing: MinIO needs `true`; the clouds route by subdomain. | +#### Images + +| Variable | Default | What it does | +| --- | --- | --- | +| `DORMICE_BASE_IMAGE` | — | Seed for the fleet's **base image**, the image a sandbox without a template boots from — a bare reference like `dormice-base:20260831`. `install.sh` builds one on the gateway's machine, pushes it to the fleet registry and records it here; the console's settings page re-points it (existing template-less sandboxes swap onto the new image at their next cold wake). A node that lacks the image pulls it from the registry. | +| `DORMICE_REGISTRY_ADDRESS` | — | Seed for the fleet's image registry, `host:port` (no scheme) — where a node pulls an image it lacks, prefixing this to the bare name and tagging it back. `install.sh` runs the registry beside the gateway on the machine's private address, port 5000, over TLS with a self-signed certificate (trusted through `/etc/docker/certs.d`, pinned on each node when it joins), user `dormice` and the fleet token as the password. Read-only over the wire. | + ## Daemon (node) `/etc/dormice/env` — the node's identity and its machine. Nothing here @@ -112,9 +120,9 @@ is a fleet setting. | --- | --- | --- | | `DORMICE_PORT` | `3676` | The daemon's port: the sandbox verbs, the E2B surface, the sandbox port proxy. Always bound to 127.0.0.1. | | `DORMICE_DB_PATH` | `data/dormice.db` | Where the node's SQLite database lives (sandboxes, metrics history, its copy of the fleet configuration). Must be an **absolute path** in docker mode — a relative path depends on the start directory, and a wrong start directory means an empty database facing real sandboxes. | -| `DORMICE_NODE_ID` | `node-1` | This node's name — in its sandboxes' rows and to its gateway, which tells nodes apart by it. The default serves a node beside its gateway; a node whose gateway is on another machine must state its own. | -| `DORMICE_GATEWAY_ENDPOINT` | `http://127.0.0.1:3677` | The gateway this daemon is a node of. It checks in there every `DORMICE_CHECK_IN_INTERVAL_SECONDS` — readings, build, where it can be reached, which configuration version it runs — and takes the fleet's configuration from the answer. A daemon with no copy yet waits for its gateway before it listens. | -| `DORMICE_NODE_ENDPOINT` | `http://127.0.0.1:` | Where the gateway reaches this node. Required (an origin: scheme, host, port — no path) when the gateway is on another machine. | +| `DORMICE_NODE_ID` | `node-1` | This node's name — in its sandboxes' rows and to its gateway, which tells nodes apart by it. The default serves a node beside its gateway; a node whose gateway is on another machine must state its own (`install.sh --role node` writes the hostname, or `--node-id`). | +| `DORMICE_GATEWAY_ENDPOINT` | `http://127.0.0.1:3677` | The gateway this daemon is a node of. It checks in there every `DORMICE_CHECK_IN_INTERVAL_SECONDS` — readings, build, where it can be reached, which configuration version it runs, whether it can upgrade itself — and takes the fleet's configuration (and, its turn come, the order to upgrade) from the answer. A daemon with no copy yet waits for its gateway before it listens. A remote address here is what makes the machine a node machine to `install.sh`. | +| `DORMICE_NODE_ENDPOINT` | `http://127.0.0.1:` | Where the gateway reaches this node. Required (an origin: scheme, host, port — no path) when the gateway is on another machine; `install.sh --role node` writes this machine's address toward the gateway on port 80, where its Caddy forwards to the daemon. | | `DORMICE_CHECK_IN_INTERVAL_SECONDS` | `15` | How often the node checks in; the gateway reads two missed check-ins as down. At most `86400` (a day). | ### Executor @@ -122,7 +130,7 @@ is a fleet setting. | Variable | Default | What it does | | --- | --- | --- | | `DORMICE_EXECUTOR` | `fake` | What runs the sandboxes: `fake` is an in-memory executor so a bare checkout works on any machine (development, tests); `docker` runs real gVisor sandboxes and needs a prepared Linux host — see [Installation](/docs/installation). | -| `DORMICE_BASE_IMAGE` | — | The image sandboxes boot from. **Required in docker mode**; `install.sh` builds one and records it here. | +| `DORMICE_BASE_IMAGE` | — | The node's **fallback** for the fleet's base image while the fleet's settings name none (the knob's home until 2026-09-15; see the gateway's `DORMICE_BASE_IMAGE` above). A node whose fleet names one logs at boot that this line can go; a node with neither refuses to build a template-less sandbox, naming where to set it. | | `DORMICE_DATA_DIR` | `/var/lib/dormice` | Sandbox disk images and their mount points (docker mode). Must be an absolute path in docker mode. | ### Lifecycle engine @@ -138,5 +146,5 @@ The `dor` CLI (and nothing else) reads two variables to find Dormice: | Variable | What it does | | --- | --- | -| `DORMICE_ENDPOINT` | A base URL: the gateway (`http://127.0.0.1:3677`) for the template and API-key commands and everything the gateway forwards; the daemon (`http://127.0.0.1:3676`) for `sandbox ls` and the other list/observation commands the gateway does not answer yet. | +| `DORMICE_ENDPOINT` | A base URL: the gateway (`http://127.0.0.1:3677`) — the door for every verb, `sandbox ls` and the upgrade included; a node's own address (`http://127.0.0.1:3676`) only to ask that one node about itself. | | `DORMICE_API_TOKEN` | The fleet's token, or a key minted at the gateway (keys open the gateway, not the daemon). | diff --git a/website/content/docs/console.mdx b/website/content/docs/console.mdx index 40489b9f..131c4722 100644 --- a/website/content/docs/console.mdx +++ b/website/content/docs/console.mdx @@ -149,6 +149,10 @@ Three more pages round out the operator view: else is the effective environment configuration, read-only, with secrets shown as present-or-absent only; changing those happens in `/etc/dormice/env` plus a restart, never from the browser. +- **Settings** also carries the fleet's **base image** row: the image a + template-less sandbox boots from, re-pointed here (push the new image + to the fleet registry first; every node pulls it ahead and swaps each + template-less sandbox at its next cold wake). - **Version** — the build the gateway is running (a git commit — trunk commit titles are the changelog) compared against the latest, with a one-click [fleet upgrade](/docs/upgrading#upgrade-a-fleet) and live diff --git a/website/content/docs/doctor.mdx b/website/content/docs/doctor.mdx index 4c95d68b..d15446a5 100644 --- a/website/content/docs/doctor.mdx +++ b/website/content/docs/doctor.mdx @@ -114,8 +114,21 @@ know why; the Linux terms in it are covered in config file that doesn't exist yet is a *warn* (the first bind creates it), and so is one the gateway did not write: `setIngress` refuses to overwrite it, so web domain binding is effectively off. -- **base image available** — present locally; the fix names - `images/Dockerfile`, and doctor never pulls. +- **base image available** — present locally. The base image is the + fleet's setting (`DORMICE_BASE_IMAGE` in the gateway's env seeds it, + the console edits it); `install.sh` runs doctor with that value on + both roles — on a node, the value it learned from the gateway for the + run. Skipped with directions when the environment has none. The fix + names the pull from the fleet registry when there is one (the daemon + does that pull itself at its next configuration bundle), else + `images/Dockerfile`; doctor never pulls. +- **fleet image registry reachable** — skipped when + `DORMICE_REGISTRY_ADDRESS` is not set (no fleet registry; every node + then needs its images staged by hand). Set, the registry's certificate + must be pinned at `/etc/docker/certs.d/
    /ca.crt` (or docker + cannot pull from it at all — re-run `install.sh`) and + `https://
    /v2/` must answer over TLS: a `401` asking for the + credential is the pass, without a credential on any command line. - **docker-mode paths absolute** — a relative `DORMICE_DB_PATH` depends on the start directory, and a wrong start directory opens an *empty database next to real sandboxes* — the exact mismatch the daemon's diff --git a/website/content/docs/installation.mdx b/website/content/docs/installation.mdx index 6a10daed..52ce35f4 100644 --- a/website/content/docs/installation.mdx +++ b/website/content/docs/installation.mdx @@ -28,12 +28,14 @@ The installer sets up everything — runtime), [gVisor](/docs/core-concepts#gvisor) (the extra isolation layer around each sandbox), [swap](/docs/core-concepts#swap-and-freezing) (the disk-as-overflow-memory -that makes freezing work), the network hardening, and Dormice's two +that makes freezing work), the network hardening, and Dormice's processes as systemd services (so they start on boot and restart after a crash): the **gateway** — the front door that serves the web console -and holds the fleet's settings and API keys — and the **daemon** — the -node that runs the sandboxes. On one machine they are a fleet of one; -more machines can join the same gateway later. +and holds the fleet's settings and API keys — the **image registry** — +the fleet's store for the sandbox base image and your template images, +which every node pulls from — and the **daemon** — the node that runs +the sandboxes. On one machine they are a fleet of one; more machines +join the same gateway with one more command, below. It finishes by running `dor doctor`, a battery of read-only checks that verifies the install actually works. You can re-run `dor doctor` at any time. @@ -41,6 +43,40 @@ time. **The installer is safe to re-run.** Running it again upgrades the code and repairs anything that drifted, and never rotates your API token. +## Add a second machine + +A fleet grows one machine at a time, and every machine after the first +is a **node**: a daemon that runs sandboxes and belongs to the gateway +on the first machine. It needs no settings of its own — the fleet's +settings, the base image and the templates come from the gateway at its +first check-in, and any image it lacks comes from the fleet's image +registry, which the first machine runs beside the gateway. + +On the new machine (same private network as the gateway; its cloud +firewall must let it reach the gateway machine's ports `3677` and +`5000`, and let the gateway machine reach its port `80`): + +```sh +export DORMICE_API_TOKEN= +curl -fsSL https://raw.githubusercontent.com/BitMiracle-AI/Dormice/main/deploy/install.sh \ + | bash -s -- --role node --gateway http://:3677 +``` + +The token travels in the environment, never as a flag (flags show in +`ps`). The installer prepares the machine the same way, writes an env +file naming the gateway, pins the registry's certificate on first sight +(printing its fingerprint — compare it with the gateway machine's), pulls +the base image, and starts the daemon, which checks in and appears on +the console's nodes page within fifteen seconds. Two optional flags: +`--node-id` (the node's name in the fleet; default: the hostname) and +`--node-endpoint` (where the gateway reaches it; default: this machine's +address toward the gateway, port 80 — the installer's Caddy forwards +that port to the daemon). + +Re-running the installer on a node needs no flags: the env file says +which gateway it belongs to. Upgrades come from the gateway — see +[Upgrading](/docs/upgrading#upgrade-a-fleet). + ## Get your API token The installer generates an [API @@ -70,9 +106,9 @@ ssh -L 3677:127.0.0.1:3677 -L 3676:127.0.0.1:3676 root@your-host ``` Keep that command running, and use `http://127.0.0.1:3677` (the -gateway: console, keys, settings, and every sandbox verb) and -`http://127.0.0.1:3676` (the daemon, for the list and observation verbs -the gateway does not answer yet) as if they were local. +gateway — the door for every verb: console, keys, settings, sandboxes, +the upgrade) as if it were local; `3676` is the daemon's own address, +for asking that one node about itself. **Option 2: Bind a domain** — serves the API, the E2B surface, and the web console at once, over HTTPS. The installer already placed a reverse diff --git a/website/content/docs/templates.mdx b/website/content/docs/templates.mdx index d245d2c8..5f47b555 100644 --- a/website/content/docs/templates.mdx +++ b/website/content/docs/templates.mdx @@ -21,14 +21,28 @@ the rest. ## Create a template -Build an image (any way you like) and register its name: +Build an image (any way you like), push it to the fleet's image +registry, and register its name: ```sh docker build -t my-agent-env:v1 . +docker tag my-agent-env:v1 /my-agent-env:v1 +docker push /my-agent-env:v1 # user dormice, password = the API token dor template add my-agent-env my-agent-env:v1 dor template ls ``` +The template names the **bare** image (`my-agent-env:v1`), never the +registry-prefixed one: a node that lacks the image pulls it from the +fleet's registry — prefixing the address itself, and tagging it back +under the bare name — the moment a configuration bundle names it (ahead +of any sandbox needing it) and again at creation if it is still missing. +The registry address is on the console's settings page +(`DORMICE_REGISTRY_ADDRESS` in the gateway's env). A fleet of one machine +can skip the push: the image is already on the only node. An image whose +name carries its own registry (`ghcr.io/you/tool:v1`) is pulled from +there as written. + ## Create sandboxes from a template Both API surfaces accept a template name at creation: @@ -101,12 +115,14 @@ home directory, non-root user at uid 1000. - **Registration is configuration.** The image is not checked for existence when you register — register first, build later, if you - like. If it is still missing when a sandbox needs it, creation fails - with a named error - (`image X is not on this host — docker pull or build it, then retry`). + like. A node that lacks the image pulls it from the fleet's registry; + if the registry lacks it too, creation fails with a named error + (`image X is not on this host, and pulling /X from the fleet + registry failed … — push it from a machine that has it: docker tag X + /X && docker push /X`). - **`base` is reserved.** Through the E2B surface, `Sandbox.create()` - with no template argument means the daemon's base image; the name - `base` cannot be registered. + with no template argument means the fleet's base image (a setting on + the console's settings page); the name `base` cannot be registered. - **Templates apply at creation only.** An acquire with a `template` on a name that already has a sandbox keeps the sandbox it finds, template unchanged — but an unknown name is still refused, never silently diff --git a/website/content/docs/troubleshooting.mdx b/website/content/docs/troubleshooting.mdx index 9c0151c8..042de9f2 100644 --- a/website/content/docs/troubleshooting.mdx +++ b/website/content/docs/troubleshooting.mdx @@ -138,8 +138,25 @@ image probe warns about exactly this. By design: [registration is configuration](/docs/templates) — the image is not checked when a name is registered, so the error surfaces at -create time, named. `docker pull` or build the image on the daemon host, -then retry. +create time, named. A node first tries the fleet's image registry; the +message says which failed. No registry in the fleet: build or +`docker pull` the image on that node under the bare name. A registry +that lacks it: run the push command the message spells out from a +machine that has the image. A pull that fails otherwise: the node's +`/etc/docker/certs.d//ca.crt` must be the registry's +certificate (re-run the installer to pin it) and the fleet token its +credential. + +## A node shows "stuck" on the version page + +The gateway told the node to upgrade itself, and twenty minutes later +it still reports the old build. The gateway never tells it again on its +own (a build that fails every time must not rebuild every twenty +minutes on the sandboxes' CPU). On that node read `journalctl -u +dormice-upgrade` and `/var/lib/dormice/upgrade/upgrade.log`, fix the +cause — a mirror that hung, a full disk — and press **Try again** on the +version page, which tells the node once more at its next check-in. +See [Upgrade a fleet](/docs/upgrading#upgrade-a-fleet). ## A paused container won't `docker rm` by hand diff --git a/website/content/docs/upgrading.mdx b/website/content/docs/upgrading.mdx index 524cc459..27a4caf1 100644 --- a/website/content/docs/upgrading.mdx +++ b/website/content/docs/upgrading.mdx @@ -53,12 +53,17 @@ hand. ## What survives an upgrade -- **Your configuration.** `/etc/dormice/env` is never touched once it - exists — the API token is never rotated, and the recorded - `DORMICE_BASE_IMAGE` is kept (delete the file to regenerate). -- **The database and every disk.** Everything under `/var/lib/dormice` - stays; schema migrations run automatically when the new daemon boots — - there is no manual migration step. +- **Your configuration.** `/etc/dormice/env` and + `/etc/dormice/gateway.env` are never rewritten once they exist — the + API token is never rotated (a knob born after your install is + appended with its default, never overwritten). +- **The databases and every disk.** Everything under `/var/lib/dormice` + and `/var/lib/dormice-gateway` stays; schema migrations run + automatically when the new processes boot — there is no manual + migration step. Before it restarts anything, the installer takes an + online snapshot of both databases into `/var/lib/dormice/backups/` + (the three most recent are kept): a migration is not reversible by + re-running an older installer, and the snapshot is the way back. - **The sandboxes.** A daemon restart does not touch containers: active ones keep running, frozen ones stay frozen. Before the daemon starts listening it reconciles its records against what Docker actually @@ -79,20 +84,28 @@ already-present gVisor is kept as-is, not upgraded. ## Upgrade sandbox images -The daemon upgrade above changes no sandbox's image — images move on a +The Dormice upgrade above changes no sandbox's image — images move on a separate, per-sandbox path, because a container swap is visible to -whoever is using the sandbox and should happen on your schedule: +whoever is using the sandbox and should happen on your schedule. The +base image is a fleet setting: build the new one on the gateway +machine, push it to the fleet's registry, and re-point the setting on +the console's settings page (or with `updateSettings`): ```sh docker build -t dormice-base:NEW -f images/Dockerfile images/ -# point DORMICE_BASE_IMAGE at the new tag in /etc/dormice/env, then: -systemctl restart dormice +docker tag dormice-base:NEW /dormice-base:NEW +docker push /dormice-base:NEW # user dormice, password = the API token +# then set "Base image" to dormice-base:NEW in console › settings dor sandbox rebuild # per sandbox, when it suits you ``` -[Rebuild](/docs/persistence#rebuild-a-sandbox) swaps the container and -keeps `/home/user` — nothing under it is lost. Sandboxes created from a -[template](/docs/templates) follow the same rhythm, with +The registry address is on the settings page (`DORMICE_REGISTRY_ADDRESS` +in `/etc/dormice/gateway.env`). Every node hears the new setting at its +next check-in, pulls the image ahead of time, and swaps each +template-less sandbox onto it at its next cold wake — no restart +anywhere. [Rebuild](/docs/persistence#rebuild-a-sandbox) swaps the +container and keeps `/home/user` — nothing under it is lost. Sandboxes +created from a [template](/docs/templates) follow the same rhythm, with `dor template add` re-pointing the name first. Sandboxes you never rebuild simply keep running on the image they were born with. From 4b451e40c1e77a6269e1311fac4170b1ae710b60 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 19:20:15 +0800 Subject: [PATCH 66/89] A node newer than the gateway reads ahead, never behind: not told, its tell fulfilled, the gateway's own upgrade the remedy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rolling.ts judged any build other than the gateway's as behind, so a node whose build was newer — a commit that landed on main after the gateway's machine upgraded and before that node's turn came, or install.sh run on the node by hand — was told to upgrade, pulled the main head it already ran, and twenty minutes on read stuck with a remedy (tell it again) that repeated the mistake. Found by review, reproduced with the pure functions. Behind now means older: the commit's time orders the two (main is trunk-based and linear). A newer build is `ahead`: never told, unreachable when silent like a current one, refused by applyUpgrade {nodeId} with the reason, and its check-in fulfils any standing tell. Two commits in one second tie and read behind — the one misjudgement left, costing a rebuild. The eighth state reaches the wire enum, the version card (amber, no button), its ten locales and the upgrading doc. --- packages/console/messages/de/settings.json | 1 + packages/console/messages/en/settings.json | 1 + packages/console/messages/es/settings.json | 1 + packages/console/messages/fr/settings.json | 1 + packages/console/messages/ja/settings.json | 1 + packages/console/messages/ko/settings.json | 1 + packages/console/messages/pt-BR/settings.json | 1 + packages/console/messages/ru/settings.json | 1 + packages/console/messages/zh-CN/settings.json | 1 + packages/console/messages/zh-TW/settings.json | 1 + .../settings/components/VersionCard.tsx | 7 ++- packages/gateway/src/rolling.test.ts | 53 +++++++++++++++++++ packages/gateway/src/rolling.ts | 42 +++++++++++++-- packages/shared/src/upgrade.ts | 9 +++- website/content/docs/upgrading.mdx | 5 +- 15 files changed, 116 insertions(+), 10 deletions(-) diff --git a/packages/console/messages/de/settings.json b/packages/console/messages/de/settings.json index debd6ddc..8d3c2131 100644 --- a/packages/console/messages/de/settings.json +++ b/packages/console/messages/de/settings.json @@ -144,6 +144,7 @@ "settings_nodes_col_build": "Build", "settings_nodes_col_state": "Zustand", "settings_nodes_state_current": "Aktuell", + "settings_nodes_state_ahead": "Neuer als das Gateway", "settings_nodes_state_behind": "Wartet", "settings_nodes_state_upgrading": "Aktualisiert", "settings_nodes_state_stuck": "Festgefahren", diff --git a/packages/console/messages/en/settings.json b/packages/console/messages/en/settings.json index 2a44ba56..ed0293a8 100644 --- a/packages/console/messages/en/settings.json +++ b/packages/console/messages/en/settings.json @@ -144,6 +144,7 @@ "settings_nodes_col_build": "Build", "settings_nodes_col_state": "State", "settings_nodes_state_current": "Current", + "settings_nodes_state_ahead": "Ahead of gateway", "settings_nodes_state_behind": "Behind", "settings_nodes_state_upgrading": "Upgrading", "settings_nodes_state_stuck": "Stuck", diff --git a/packages/console/messages/es/settings.json b/packages/console/messages/es/settings.json index f4f33aec..560e14ab 100644 --- a/packages/console/messages/es/settings.json +++ b/packages/console/messages/es/settings.json @@ -144,6 +144,7 @@ "settings_nodes_col_build": "Compilación", "settings_nodes_col_state": "Estado", "settings_nodes_state_current": "Al día", + "settings_nodes_state_ahead": "Más nuevo que el gateway", "settings_nodes_state_behind": "Pendiente", "settings_nodes_state_upgrading": "Actualizando", "settings_nodes_state_stuck": "Atascado", diff --git a/packages/console/messages/fr/settings.json b/packages/console/messages/fr/settings.json index 20b85f01..0bccbb36 100644 --- a/packages/console/messages/fr/settings.json +++ b/packages/console/messages/fr/settings.json @@ -144,6 +144,7 @@ "settings_nodes_col_build": "Build", "settings_nodes_col_state": "État", "settings_nodes_state_current": "À jour", + "settings_nodes_state_ahead": "Plus récent que la passerelle", "settings_nodes_state_behind": "En attente", "settings_nodes_state_upgrading": "Mise à niveau", "settings_nodes_state_stuck": "Bloqué", diff --git a/packages/console/messages/ja/settings.json b/packages/console/messages/ja/settings.json index a68f78d7..ce94ecfe 100644 --- a/packages/console/messages/ja/settings.json +++ b/packages/console/messages/ja/settings.json @@ -144,6 +144,7 @@ "settings_nodes_col_build": "ビルド", "settings_nodes_col_state": "状態", "settings_nodes_state_current": "最新", + "settings_nodes_state_ahead": "ゲートウェイより新しい", "settings_nodes_state_behind": "待機中", "settings_nodes_state_upgrading": "アップグレード中", "settings_nodes_state_stuck": "スタック", diff --git a/packages/console/messages/ko/settings.json b/packages/console/messages/ko/settings.json index 4cbf6147..6c976429 100644 --- a/packages/console/messages/ko/settings.json +++ b/packages/console/messages/ko/settings.json @@ -144,6 +144,7 @@ "settings_nodes_col_build": "빌드", "settings_nodes_col_state": "상태", "settings_nodes_state_current": "최신", + "settings_nodes_state_ahead": "게이트웨이보다 최신", "settings_nodes_state_behind": "대기 중", "settings_nodes_state_upgrading": "업그레이드 중", "settings_nodes_state_stuck": "막힘", diff --git a/packages/console/messages/pt-BR/settings.json b/packages/console/messages/pt-BR/settings.json index a38e26f1..1d6ee50b 100644 --- a/packages/console/messages/pt-BR/settings.json +++ b/packages/console/messages/pt-BR/settings.json @@ -144,6 +144,7 @@ "settings_nodes_col_build": "Build", "settings_nodes_col_state": "Estado", "settings_nodes_state_current": "Em dia", + "settings_nodes_state_ahead": "Mais novo que o gateway", "settings_nodes_state_behind": "Aguardando", "settings_nodes_state_upgrading": "Atualizando", "settings_nodes_state_stuck": "Travado", diff --git a/packages/console/messages/ru/settings.json b/packages/console/messages/ru/settings.json index 3d57c3d5..d3700c41 100644 --- a/packages/console/messages/ru/settings.json +++ b/packages/console/messages/ru/settings.json @@ -144,6 +144,7 @@ "settings_nodes_col_build": "Сборка", "settings_nodes_col_state": "Состояние", "settings_nodes_state_current": "Актуален", + "settings_nodes_state_ahead": "Новее шлюза", "settings_nodes_state_behind": "Ожидает", "settings_nodes_state_upgrading": "Обновляется", "settings_nodes_state_stuck": "Застрял", diff --git a/packages/console/messages/zh-CN/settings.json b/packages/console/messages/zh-CN/settings.json index 7be8005d..a325443e 100644 --- a/packages/console/messages/zh-CN/settings.json +++ b/packages/console/messages/zh-CN/settings.json @@ -144,6 +144,7 @@ "settings_nodes_col_build": "构建", "settings_nodes_col_state": "状态", "settings_nodes_state_current": "已跟上", + "settings_nodes_state_ahead": "比网关新", "settings_nodes_state_behind": "待升级", "settings_nodes_state_upgrading": "升级中", "settings_nodes_state_stuck": "卡住", diff --git a/packages/console/messages/zh-TW/settings.json b/packages/console/messages/zh-TW/settings.json index 643d7a53..7f8d42a4 100644 --- a/packages/console/messages/zh-TW/settings.json +++ b/packages/console/messages/zh-TW/settings.json @@ -144,6 +144,7 @@ "settings_nodes_col_build": "建置", "settings_nodes_col_state": "狀態", "settings_nodes_state_current": "已跟上", + "settings_nodes_state_ahead": "比網關新", "settings_nodes_state_behind": "待升級", "settings_nodes_state_upgrading": "升級中", "settings_nodes_state_stuck": "卡住", diff --git a/packages/console/src/features/settings/components/VersionCard.tsx b/packages/console/src/features/settings/components/VersionCard.tsx index fa63687e..2bd67977 100644 --- a/packages/console/src/features/settings/components/VersionCard.tsx +++ b/packages/console/src/features/settings/components/VersionCard.tsx @@ -43,7 +43,8 @@ import { UpgradeDialog } from './UpgradeDialog'; * 给手动路径与原因。卡片下方一张节点表:每台对着网关构建的站位,由 * 网关裁决(state 字段);「卡住」的行给「再试一次」= applyUpgrade * {nodeId},人工重告 — 网关自己绝不重告(构建总失败的节点不能每 20 - * 分钟白烤一遍)。 + * 分钟白烤一遍)。「比网关新」的节点不给按钮:该升的是网关,理由在 + * title 里说。 */ export function VersionCard() { const { data, isPending, isError, error } = useCheckUpgrade(); @@ -290,6 +291,7 @@ function UpgradePreview({ const STATE_LABEL: Record string> = { current: m.settings_nodes_state_current, + ahead: m.settings_nodes_state_ahead, behind: m.settings_nodes_state_behind, upgrading: m.settings_nodes_state_upgrading, stuck: m.settings_nodes_state_stuck, @@ -298,9 +300,10 @@ const STATE_LABEL: Record string> = { unknown: m.settings_nodes_state_unknown, }; -/** 徽章色阶:跟上=静;待升/升级中=琥珀(在动);卡住=红(要人);其余=灰(说明在 title)。 */ +/** 徽章色阶:跟上=静;待升/升级中/比网关新=琥珀(在动,或要人先升网关);卡住=红(要人);其余=灰(说明在 title)。 */ function stateClass(state: NodeUpgradeView['state']): string { switch (state) { + case 'ahead': case 'behind': case 'upgrading': return 'border-amber-500/40 bg-amber-500/10 text-amber-600 dark:text-amber-400'; diff --git a/packages/gateway/src/rolling.test.ts b/packages/gateway/src/rolling.test.ts index 9e451267..ce13a311 100644 --- a/packages/gateway/src/rolling.test.ts +++ b/packages/gateway/src/rolling.test.ts @@ -28,6 +28,12 @@ const OLD: BuildInfo = { title: 'the old build', committedAt: '2026-09-14T00:00:00.000Z', }; +/** A commit that landed on main after the gateway upgraded — where a node told mid-roll ends up. */ +const NEWER: BuildInfo = { + commit: 'new0002', + title: 'landed on main mid-roll', + committedAt: '2026-09-15T00:05:00.000Z', +}; const CAN = { available: true, reason: null }; const CANNOT = { available: false, @@ -123,6 +129,27 @@ describe('upgradeStateOf', () => { }); }); + it("ahead, never told, when the node's build is newer than the gateway's; silent, it is unreachable like any other; a same-second tie reads behind", () => { + const { fleet } = fleetOver(); + const node = reporting(fleet, 'a', { build: NEWER, selfUpgrade: CAN }); + expect(upgradeStateOf(node, GATEWAY, NOW)).toEqual({ + state: 'ahead', + reason: + "runs new0002 (committed 2026-09-15T00:05:00.000Z), newer than the gateway's new0001 — a fleet upgrades from its gateway: upgrade the gateway (applyUpgrade there), and this node reads current", + }); + expect(rollingDecision(fleet.all(), GATEWAY, node, NOW)).toBe(false); + expect( + upgradeStateOf(node, GATEWAY, new Date(NOW.getTime() + 31_000)), + ).toEqual({ state: 'unreachable', reason: 'has not checked in for 31s' }); + // The commit's time is the order; two commits in one second cannot be + // told apart, and the tie reads behind. + const tied = reporting(fleet, 'b', { + build: { ...GATEWAY, commit: 'tie0001' }, + selfUpgrade: CAN, + }); + expect(upgradeStateOf(tied, GATEWAY, NOW).state).toBe('behind'); + }); + it('told: upgrading within the timeout, stuck past it — with when it was told, what it still runs, and where to look', () => { const { fleet } = fleetOver(); const node = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); @@ -265,6 +292,32 @@ describe('Rolling', () => { }); }); + it('a told node that comes back newer than the gateway is ahead: its tell is fulfilled and cleared, it is not told again, the operator cannot re-tell it, and it holds nobody', () => { + const { db, fleet } = fleetOver(); + const rolling = new Rolling(fleet, GATEWAY); + const a = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); + expect(rolling.onCheckIn(a, NOW)).toBe(true); + // Its install.sh pulled main's head, which moved on since the gateway + // upgraded: back on a newer build than the gateway's. + const later = new Date(NOW.getTime() + 120_000); + reporting(fleet, 'a', { build: NEWER, selfUpgrade: CAN }, later); + expect(rolling.onCheckIn(a, later)).toBe(false); + expect(a.upgradeToldAt).toBeNull(); + expect(new Fleet(db).get('a')?.upgradeToldAt).toBeNull(); + expect(rolling.states(later)).toMatchObject([ + { id: 'a', state: 'ahead', toldAt: null }, + ]); + expect(rolling.requestRetell(a, later)).toMatchObject({ + status: 400, + message: expect.stringMatching( + /cannot be told to upgrade: runs new0002 .* newer than the gateway's new0001/, + ), + }); + // Not upgrading, so it holds no pointer: b's turn comes. + const b = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }, later); + expect(rolling.onCheckIn(b, later)).toBe(true); + }); + it('states lists every node in id order with its standing, build and tell', () => { const { fleet } = fleetOver(); const rolling = new Rolling(fleet, GATEWAY); diff --git a/packages/gateway/src/rolling.ts b/packages/gateway/src/rolling.ts index 3750bc66..2dd3cc57 100644 --- a/packages/gateway/src/rolling.ts +++ b/packages/gateway/src/rolling.ts @@ -11,7 +11,7 @@ import { downReason, type Fleet, type NodeState } from './fleet'; * itself when it sees it, and the next node's turn comes when it is back * on the new build). The gateway's machine upgrades first — install.sh * there restarts its gateway and its node together — and from then on - * every node that reports another build than the gateway's is behind. + * every node that reports an older build than the gateway's is behind. * * Told at a check-in, one node at a time: the answer carries `upgrade: * true` (shared checkInResponseSchema), the node runs its own updater @@ -30,6 +30,15 @@ import { downReason, type Fleet, type NodeState } from './fleet'; * neither forgets a node it told nor tells it twice; the operator's * re-tell is memory — a gateway restarted before the node's next check-in * forgets it, and the operator clicks again. + * + * Behind means older. A node whose build is newer than the gateway's — a + * commit that landed on main after the gateway's machine upgraded and + * before this node's turn came, or install.sh run on the node by hand — + * is `ahead`, never told: install.sh pulls main's head, which is where + * the node already is, so a tell would rebuild it for nothing and, twenty + * minutes on, read it stuck with a remedy that repeats the mistake (found + * by review, 2026-09-15). The gateway's own upgrade is what brings an + * ahead node to current, and its reason says so. */ /** How long a told node has to come back on the new build before it is stuck: a pull, a build and a restart take a few minutes; twenty is a build that failed. */ @@ -68,6 +77,24 @@ export function upgradeStateOf( ? { state: 'current', reason: null } : { state: 'unreachable', reason: down }; } + // Newer than the gateway's build is ahead, not behind (the module + // comment has why), judged before the tell: a told node that comes back + // on a newer build has upgraded, and reads ahead — not upgrading. The + // commit's time is the order: main is trunk-based and linear, so a later + // committer time is a later commit. Two commits in one second (a rebase + // re-commits several in a burst) tie, and a tie reads behind — a node + // ahead by a same-second commit is the one case misjudged, and a tell + // it survives as before. + if ( + Date.parse(node.build.committedAt) > Date.parse(gatewayBuild.committedAt) + ) { + return down === null + ? { + state: 'ahead', + reason: `runs ${node.build.commit} (committed ${node.build.committedAt}), newer than the gateway's ${gatewayBuild.commit} — a fleet upgrades from its gateway: upgrade the gateway (applyUpgrade there), and this node reads current`, + } + : { state: 'unreachable', reason: down }; + } // A told node is upgrading or stuck whether or not it is checking in: // its daemon restarts near the end of install.sh and misses a check-in // or two by design, and were that silence read as "unreachable" the @@ -140,14 +167,18 @@ export class Rolling { /** * The check-in's verdict for a node that just reported, in order: a - * fulfilled tell is cleared (the node is back on the gateway's build); + * fulfilled tell is cleared (the node is off the old build — on the + * gateway's, or ahead of it); * a pending re-tell is honored; otherwise the rolling rule decides. Any * tell is written to the row before the answer carries it. Answers * whether the node is told now. */ onCheckIn(node: NodeState, now: Date): boolean { const { state } = upgradeStateOf(node, this.gatewayBuild, now); - if (state === 'current' && node.upgradeToldAt !== null) { + if ( + (state === 'current' || state === 'ahead') && + node.upgradeToldAt !== null + ) { this.fleet.setUpgradeToldAt(node.id, null); this.retell.delete(node.id); return false; @@ -165,8 +196,8 @@ export class Rolling { /** * The operator's re-tell (applyUpgrade {nodeId}): honored at the node's * next check-in. Refused in words when it would do nothing — a node on - * the gateway's build, one that cannot upgrade itself, one whose build - * is unknown — and told to wait for an unreachable one; a node merely + * the gateway's build or ahead of it, one that cannot upgrade itself, + * one whose build is unknown — and told to wait for an unreachable one; a node merely * behind or upgrading is taken too (the operator's hand outranks the * order). Answers the refusal, or null when the re-tell is pending. */ @@ -181,6 +212,7 @@ export class Rolling { status: 400, message: `node ${node.id} already runs the gateway's build (${node.build?.commit ?? 'unknown'}) — nothing to upgrade`, }; + case 'ahead': case 'unavailable': case 'unknown': return { diff --git a/packages/shared/src/upgrade.ts b/packages/shared/src/upgrade.ts index fbd66148..8486ce6f 100644 --- a/packages/shared/src/upgrade.ts +++ b/packages/shared/src/upgrade.ts @@ -161,7 +161,11 @@ export type GetUpgradeStatusRequest = z.infer< * Where one node stands against the gateway's build, as the gateway * judges it from the node's last check-in (gateway rolling.ts): * current the node runs the gateway's build - * behind another build, able to upgrade itself, not told yet — its + * ahead a build newer than the gateway's — a commit that landed on + * main while the fleet was rolling, or install.sh run on the + * node by hand — never told; the gateway's own upgrade + * brings it to current + * behind an older build, able to upgrade itself, not told yet — its * turn comes when no other node is upgrading * upgrading told within the last twenty minutes, not back yet * stuck told, still on the old build twenty minutes on — never @@ -174,6 +178,7 @@ export type GetUpgradeStatusRequest = z.infer< */ export const NODE_UPGRADE_STATES = [ 'current', + 'ahead', 'behind', 'upgrading', 'stuck', @@ -190,7 +195,7 @@ export const nodeUpgradeViewSchema = z.object({ state: z.enum(NODE_UPGRADE_STATES), /** ISO 8601 UTC — when the node was last told to upgrade; null = never, or its last tell was fulfilled. */ toldAt: z.iso.datetime().nullable(), - /** In the gateway's words, for every state but current and behind: why it is stuck, unavailable, unreachable or unknown; how long it has been upgrading. */ + /** In the gateway's words, for every state but current and behind: why it is ahead, stuck, unavailable, unreachable or unknown; how long it has been upgrading. */ reason: z.string().nullable(), }); diff --git a/website/content/docs/upgrading.mdx b/website/content/docs/upgrading.mdx index 27a4caf1..fb78ec6b 100644 --- a/website/content/docs/upgrading.mdx +++ b/website/content/docs/upgrading.mdx @@ -38,7 +38,10 @@ its next check-in (within fifteen seconds) that its turn has come, and runs the same installer itself: one node at a time, so the fleet is never missing more than one node's sandboxes for the few tens of seconds a restart takes. The version page shows each node's standing: -current, behind, upgrading, or stuck. +current, behind, upgrading, stuck — or ahead: a node on a build newer +than the gateway's (a commit landed on `main` while the fleet was +rolling, or the installer was run on that node by hand) is left alone, +and reads current once the gateway itself is upgraded. A node is told exactly once. If it has not come back on the new build twenty minutes later, the gateway marks it **stuck** and leaves it From 79c3a726777fd01a39712b01477b4256db27b4fa Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 19:20:15 +0800 Subject: [PATCH 67/89] install.sh: a failed import removes the half-made gateway database, the gateway machine's node stops before the gateway restarts, a failed push or pull logs out, and a moved registry address is refused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit import.js creates the gateway database (its migrations) before it reads the ledger, and install.sh guards the import on that file's existence: a failure past the migrations left a file with tables and no settings row, a re-run skipped the import, the gateway seeded from the env, and the daemon's next boot dropped the tables the import carries — keys and console account gone, the backup directory the only copy. Found by review, reproduced with the built tool. The file did not exist before the step: on failure it is removed, and the re-run imports again. The gateway restarted first with the old daemon still running. In the second or two before its own restart, the daemon's check-in could land on the new gateway, read as behind and be told to upgrade — into the very unit that was running (one chance in eight per upgrade at a fifteen-second interval). The daemon now stops first and starts last; a gateway that does not answer /healthz has the daemon started again before the run dies. docker login was followed by a push (gateway) or a pull (node) whose failure exited under set -e before the logout, leaving the fleet token in /root/.docker/config.json: both log out on failure too. --registry-addr on a re-run moved the listener while the env line and the settings row kept the old address, every node pulling from where the registry no longer was: a differing flag is refused, as --gateway is against a node's env. The pull hint (executor, templates doc) names docker login before tag and push, and the upgrading doc says what install.sh does edit in the env files. --- deploy/install.sh | 77 +++++++++++++++---- .../server/src/executor/docker-images.test.ts | 2 +- packages/server/src/executor/docker.ts | 2 +- website/content/docs/templates.mdx | 5 +- website/content/docs/upgrading.mdx | 10 ++- 5 files changed, 74 insertions(+), 22 deletions(-) diff --git a/deploy/install.sh b/deploy/install.sh index 48f417a3..c1f9c8c4 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -38,8 +38,9 @@ # --node-endpoint URL where the gateway reaches this node (default: # http://:80) # --registry-addr H:P the address the fleet registry listens on and the -# nodes pull from (gateway's machine; default: this -# machine's private address, port 5000) +# nodes pull from (gateway's machine, first install; +# default: this machine's private address, port 5000 +# — a re-run keeps the address gateway.env holds) # # Four promises, mirroring `dor doctor`: # - Idempotent. Every step checks before it acts; a step whose outcome is @@ -1055,8 +1056,19 @@ fi # the private address the other machines reach it by; docker0's 172.17.0.1 # never is). A machine whose main address is public listens on it; the # lock is what makes that acceptable. +env_registry_addr=$(sed -n 's/^DORMICE_REGISTRY_ADDRESS=//p' "$GATEWAY_ENV_FILE" | head -1) REGISTRY_ADDR=$REGISTRY_ADDR_FLAG -[ -n "$REGISTRY_ADDR" ] || REGISTRY_ADDR=$(sed -n 's/^DORMICE_REGISTRY_ADDRESS=//p' "$GATEWAY_ENV_FILE" | head -1) +if [ -n "$REGISTRY_ADDR" ] && [ -n "$env_registry_addr" ] && [ "$REGISTRY_ADDR" != "$env_registry_addr" ]; then + # Once seeded, the address is the fleet's setting: the gateway's row + # carries it to every node, and each node pins the certificate under it + # and pulls from it. A flag on a re-run would move the listener and + # nothing else — the env line and the row stand — and every node would + # pull from where the registry no longer is. Refused, as --gateway is + # against a node's env (found by review, 2026-09-15); moving the + # registry is not a re-run's job in this version. + die "$GATEWAY_ENV_FILE says the fleet registry is at $env_registry_addr, --registry-addr says $REGISTRY_ADDR — the address is the fleet's setting and does not move with a flag; re-run without it" +fi +[ -n "$REGISTRY_ADDR" ] || REGISTRY_ADDR=$env_registry_addr if [ -z "$REGISTRY_ADDR" ]; then registry_host=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}') [ -n "$registry_host" ] || registry_host=$(hostname -I 2>/dev/null | awk '{print $1}') @@ -1170,9 +1182,15 @@ else printf '%s' "$API_TOKEN" | docker login "$REGISTRY_ADDR" -u dormice --password-stdin >/dev/null 2>&1 \ || die "docker login to https://$REGISTRY_ADDR refused the fleet credential — if the token changed after $REGISTRY_CONF_DIR/htpasswd was written, delete that file and re-run" docker tag "$base_image" "$REGISTRY_ADDR/$base_image" - docker push -q "$REGISTRY_ADDR/$base_image" >/dev/null - # The credential does not stay in /root/.docker/config.json: the daemon - # presents it per pull from memory. + # The credential does not stay in /root/.docker/config.json, whatever + # the push comes to: the daemon presents it per pull from memory, and a + # push that fails must not leave the fleet token on disk (`set -e` would + # exit before a logout that only followed success; found by review, + # 2026-09-15). + if ! docker push -q "$REGISTRY_ADDR/$base_image" >/dev/null; then + docker logout "$REGISTRY_ADDR" >/dev/null 2>&1 || true + die "docker push of $REGISTRY_ADDR/$base_image failed — the registry's side of it: journalctl -u dormice-registry -n 50; fix the cause and re-run" + fi docker logout "$REGISTRY_ADDR" >/dev/null 2>&1 || true note "pushed $base_image to the registry as $REGISTRY_ADDR/$base_image" fi @@ -1218,8 +1236,11 @@ if [ "$FLEET_REGISTRY" != - ]; then else printf '%s' "$API_TOKEN" | docker login "$FLEET_REGISTRY" -u dormice --password-stdin >/dev/null 2>&1 \ || die "docker login to https://$FLEET_REGISTRY refused the fleet credential — DORMICE_API_TOKEN here must be the gateway machine's token" - docker pull -q "$FLEET_REGISTRY/$FLEET_BASE_IMAGE" >/dev/null \ - || die "could not pull $FLEET_REGISTRY/$FLEET_BASE_IMAGE — on the gateway machine, re-run install.sh (it pushes the base image), then re-run here" + if ! docker pull -q "$FLEET_REGISTRY/$FLEET_BASE_IMAGE" >/dev/null; then + # Logged out on failure too: the fleet token must not stay on disk. + docker logout "$FLEET_REGISTRY" >/dev/null 2>&1 || true + die "could not pull $FLEET_REGISTRY/$FLEET_BASE_IMAGE — on the gateway machine, re-run install.sh (it pushes the base image), then re-run here" + fi docker tag "$FLEET_REGISTRY/$FLEET_BASE_IMAGE" "$FLEET_BASE_IMAGE" docker logout "$FLEET_REGISTRY" >/dev/null 2>&1 || true note "pulled the fleet's base image $FLEET_BASE_IMAGE from the registry" @@ -1284,6 +1305,9 @@ fi # untouched. if [ "$ROLE" = gateway ]; then log 'importing the single-machine ledger into the gateway' +# The file is the pre-check only — a running gateway holds the lock the +# tool would need, and a gateway that has started has its row; the tool's +# own refusal, on the settings row, is the arbiter (import-ledger.ts). if [ -f "$GATEWAY_DATA_DIR/gateway.db" ]; then note '[skip] the gateway database exists — the import is for its first start' elif [ ! -f "$DATA_DIR/dormice.db" ]; then @@ -1304,7 +1328,19 @@ db.close(); . "$GATEWAY_ENV_FILE" set +a node "$INSTALL_DIR/packages/gateway/dist/import.js" --node-db "$DATA_DIR/dormice.db" --node-env "$ENV_FILE" - ) || die "the import of $DATA_DIR/dormice.db into the gateway failed — nothing was restarted; fix the cause and re-run (the gateway database, if half-written, is at $GATEWAY_DATA_DIR/gateway.db: delete it before the re-run)" + ) || { + # The tool creates the gateway database (its migrations) before it + # reads the ledger, so a failure past that point leaves a file with + # tables and no settings row. Left there, a re-run would see the + # file above and skip the import, the gateway would seed its + # settings from the env, and the daemon's next boot would drop the + # tables the import carries — the operator's keys and console + # account gone with no word said (found by review, 2026-09-15). The + # file did not exist before this step: removing it puts the machine + # back exactly where it was, and the re-run imports again. + rm -f "$GATEWAY_DATA_DIR/gateway.db" "$GATEWAY_DATA_DIR/gateway.db-wal" "$GATEWAY_DATA_DIR/gateway.db-shm" "$GATEWAY_DATA_DIR/gateway.db.lock" + die "the import of $DATA_DIR/dormice.db into the gateway failed — nothing was restarted, and the half-made gateway database was removed so that the re-run imports again; fix the cause and re-run" + } note "imported into the gateway: $imported" fi fi @@ -1315,23 +1351,36 @@ fi # configuration copy takes its first bundle from its gateway before it # listens, and a re-run just built both dists: the two processes of a # fleet of one run one commit, never two. A node machine: the daemon -# alone, joined to its remote gateway. Restart, not start: all crash-only -# by design, so restarting them is always safe. +# alone, joined to its remote gateway. Restarted, not merely started: all +# crash-only by design, so restarting them is always safe. log 'systemd services' cp "$INSTALL_DIR/deploy/dormice.service" /etc/systemd/system/dormice.service if [ "$ROLE" = gateway ]; then cp "$INSTALL_DIR/deploy/dormice-gateway.service" /etc/systemd/system/dormice-gateway.service systemctl daemon-reload systemctl enable dormice-gateway dormice >/dev/null 2>&1 + # The daemon goes down first and comes up last: this machine's two + # processes upgrade as one. Gateway first with the old daemon still + # running, the old daemon's check-in could land on the new gateway in + # the second or two before its own restart — read as a node behind, told + # to upgrade, it would try to start the very install.sh unit that is + # running, log the refusal, and hold the fleet's one-at-a-time slot for + # an interval (found by review, 2026-09-15). Stopped, it says nothing + # until it is the new build. + systemctl stop dormice systemctl restart dormice-gateway for _ in $(seq 1 60); do curl -fsS "http://127.0.0.1:$GATEWAY_PORT/healthz" >/dev/null 2>&1 && break sleep 0.5 done - curl -fsS "http://127.0.0.1:$GATEWAY_PORT/healthz" >/dev/null 2>&1 \ - || die "the gateway did not answer /healthz on 127.0.0.1:$GATEWAY_PORT — check: journalctl -u dormice-gateway -n 50" + if ! curl -fsS "http://127.0.0.1:$GATEWAY_PORT/healthz" >/dev/null 2>&1; then + # The daemon must not stay down for the gateway's failure: it serves + # its sandboxes without one, and its check-in keeps trying. + systemctl start dormice + die "the gateway did not answer /healthz on 127.0.0.1:$GATEWAY_PORT — check: journalctl -u dormice-gateway -n 50 (the daemon was started again)" + fi note "gateway is answering on 127.0.0.1:$GATEWAY_PORT" - systemctl restart dormice + systemctl start dormice note 'enabled and (re)started both' else systemctl daemon-reload diff --git a/packages/server/src/executor/docker-images.test.ts b/packages/server/src/executor/docker-images.test.ts index 473b8a5d..aaec3204 100644 --- a/packages/server/src/executor/docker-images.test.ts +++ b/packages/server/src/executor/docker-images.test.ts @@ -189,7 +189,7 @@ describe('DockerExecutor.ensureImage', () => { await expect( executor(stub, '10.0.0.5:5000').ensureImage('tpl:1'), ).rejects.toThrow( - /image tpl:1 is not on this host, and pulling 10\.0\.0\.5:5000\/tpl:1 from the fleet registry failed: manifest unknown.*docker tag tpl:1 10\.0\.0\.5:5000\/tpl:1 && docker push 10\.0\.0\.5:5000\/tpl:1/, + /image tpl:1 is not on this host, and pulling 10\.0\.0\.5:5000\/tpl:1 from the fleet registry failed: manifest unknown.*docker login 10\.0\.0\.5:5000 -u dormice .*docker tag tpl:1 10\.0\.0\.5:5000\/tpl:1 && docker push 10\.0\.0\.5:5000\/tpl:1/, ); expect(stub.calls.tagged).toEqual([]); }); diff --git a/packages/server/src/executor/docker.ts b/packages/server/src/executor/docker.ts index b4c380b5..de19e5bb 100644 --- a/packages/server/src/executor/docker.ts +++ b/packages/server/src/executor/docker.ts @@ -405,7 +405,7 @@ export class DockerExecutor implements Executor { const why = err instanceof Error ? err.message : String(err); throw new Error( fromFleet - ? `image ${image} is not on this host, and pulling ${source} from the fleet registry failed: ${why} — push it from a machine that has it: docker tag ${image} ${source} && docker push ${source}` + ? `image ${image} is not on this host, and pulling ${source} from the fleet registry failed: ${why} — push it from a machine that has it: docker login ${registry} -u dormice (the password is the fleet token), then docker tag ${image} ${source} && docker push ${source}` : `image ${image} is not on this host, and pulling it failed: ${why}`, ); } diff --git a/website/content/docs/templates.mdx b/website/content/docs/templates.mdx index 5f47b555..b6e7cf23 100644 --- a/website/content/docs/templates.mdx +++ b/website/content/docs/templates.mdx @@ -118,8 +118,9 @@ home directory, non-root user at uid 1000. like. A node that lacks the image pulls it from the fleet's registry; if the registry lacks it too, creation fails with a named error (`image X is not on this host, and pulling /X from the fleet - registry failed … — push it from a machine that has it: docker tag X - /X && docker push /X`). + registry failed … — push it from a machine that has it: docker login + -u dormice (the password is the fleet token), then docker + tag X /X && docker push /X`). - **`base` is reserved.** Through the E2B surface, `Sandbox.create()` with no template argument means the fleet's base image (a setting on the console's settings page); the name `base` cannot be registered. diff --git a/website/content/docs/upgrading.mdx b/website/content/docs/upgrading.mdx index fb78ec6b..b867c2cb 100644 --- a/website/content/docs/upgrading.mdx +++ b/website/content/docs/upgrading.mdx @@ -56,10 +56,12 @@ hand. ## What survives an upgrade -- **Your configuration.** `/etc/dormice/env` and - `/etc/dormice/gateway.env` are never rewritten once they exist — the - API token is never rotated (a knob born after your install is - appended with its default, never overwritten). +- **Your configuration.** The API token in `/etc/dormice/env` is never + rotated, and neither env file is regenerated once it exists. Two + edits are made in place and nothing else: a knob born after your + install is appended with its default, and on the move to a gateway the + fleet knobs the daemon's env used to hold are copied into + `/etc/dormice/gateway.env` and commented out where they were. - **The databases and every disk.** Everything under `/var/lib/dormice` and `/var/lib/dormice-gateway` stays; schema migrations run automatically when the new processes boot — there is no manual From 24d583b64814b534eb40f51b92c62a5012329df6 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 22:26:11 +0800 Subject: [PATCH 68/89] A check-in that reads current or ahead spends the operator's pending re-tell too; --registry-addr is refused on a node; the tie comment says where it can and cannot bite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rolling.onCheckIn cleared the operator's re-tell only together with a tell on the row. A node re-told while behind, then upgraded by hand to a build ahead of the gateway's, kept its re-tell in memory: the moment it read behind again — the gateway upgraded past it — it was told at once, whatever the one-at-a-time rule said, and the fleet lost two nodes' sandboxes together. The re-tell was for the node that was; a check-in off the old build spends it. --registry-addr on a node machine was read by nobody: the registry is the gateway machine's, and a node pulls from the address its gateway names. Refused before anything is installed, as the other mistaken flags are. The import's cleanup removes the lock's journal file too. The ahead/behind tie: the comment claimed same-second commits are a rebase's rarity. They are not rare in this history (24 of main's last 300 commits share a second with a neighbour), but the two builds compared were each built at a branch head, and two heads a second apart would be two pushes a second apart — a hand-built mid-series checkout is the one way to the misjudgement, and it costs that node one rebuild. --- deploy/install.sh | 6 +++++- packages/gateway/src/rolling.test.ts | 11 +++++++++++ packages/gateway/src/rolling.ts | 26 +++++++++++++++++--------- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/deploy/install.sh b/deploy/install.sh index c1f9c8c4..bf40a7b4 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -181,6 +181,10 @@ else ROLE=gateway GATEWAY_URL="http://127.0.0.1:$GATEWAY_PORT" fi +# The registry is the gateway machine's; a node pulls from the address the +# fleet's settings name, and a flag here would be taken for a setting and +# silently do nothing (found by review, 2026-09-15). +[ "$ROLE" = node ] && [ -n "$REGISTRY_ADDR_FLAG" ] && die "--registry-addr is the gateway machine's flag — a node pulls from the registry its gateway names; re-run without it" # ---- outcome reporting and the build rollback -------------------------------- # status.json is the one file the daemon's one-click upgrade reads back; @@ -1338,7 +1342,7 @@ db.close(); # account gone with no word said (found by review, 2026-09-15). The # file did not exist before this step: removing it puts the machine # back exactly where it was, and the re-run imports again. - rm -f "$GATEWAY_DATA_DIR/gateway.db" "$GATEWAY_DATA_DIR/gateway.db-wal" "$GATEWAY_DATA_DIR/gateway.db-shm" "$GATEWAY_DATA_DIR/gateway.db.lock" + rm -f "$GATEWAY_DATA_DIR/gateway.db" "$GATEWAY_DATA_DIR/gateway.db-wal" "$GATEWAY_DATA_DIR/gateway.db-shm" "$GATEWAY_DATA_DIR/gateway.db.lock" "$GATEWAY_DATA_DIR/gateway.db.lock-journal" die "the import of $DATA_DIR/dormice.db into the gateway failed — nothing was restarted, and the half-made gateway database was removed so that the re-run imports again; fix the cause and re-run" } note "imported into the gateway: $imported" diff --git a/packages/gateway/src/rolling.test.ts b/packages/gateway/src/rolling.test.ts index ce13a311..1faf95bb 100644 --- a/packages/gateway/src/rolling.test.ts +++ b/packages/gateway/src/rolling.test.ts @@ -316,6 +316,17 @@ describe('Rolling', () => { // Not upgrading, so it holds no pointer: b's turn comes. const b = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }, later); expect(rolling.onCheckIn(b, later)).toBe(true); + // A pending re-tell is spent by an ahead check-in too: c, behind and + // re-told by the operator while b upgrades, is upgraded by hand to a + // newer build, then put back on the old one — the hand was for the + // node that was, and c waits its turn like any other. + const c = reporting(fleet, 'c', { build: OLD, selfUpgrade: CAN }, later); + expect(rolling.onCheckIn(c, later)).toBe(false); + expect(rolling.requestRetell(c, later)).toBeNull(); + reporting(fleet, 'c', { build: NEWER, selfUpgrade: CAN }, later); + expect(rolling.onCheckIn(c, later)).toBe(false); + reporting(fleet, 'c', { build: OLD, selfUpgrade: CAN }, later); + expect(rolling.onCheckIn(c, later)).toBe(false); }); it('states lists every node in id order with its standing, build and tell', () => { diff --git a/packages/gateway/src/rolling.ts b/packages/gateway/src/rolling.ts index 2dd3cc57..7da48f1e 100644 --- a/packages/gateway/src/rolling.ts +++ b/packages/gateway/src/rolling.ts @@ -81,10 +81,14 @@ export function upgradeStateOf( // comment has why), judged before the tell: a told node that comes back // on a newer build has upgraded, and reads ahead — not upgrading. The // commit's time is the order: main is trunk-based and linear, so a later - // committer time is a later commit. Two commits in one second (a rebase - // re-commits several in a burst) tie, and a tie reads behind — a node - // ahead by a same-second commit is the one case misjudged, and a tell - // it survives as before. + // committer time is a later commit. Commits in one second tie, and a tie + // reads behind. Ties are common in the history (a rebased series is + // re-committed in a burst: 24 of main's last 300 commits share a second + // with their neighbour, measured 2026-09-15) but not between the two + // builds compared here: each was built by install.sh at its branch's + // head at the time, and two heads a second apart would be two pushes a + // second apart. A hand-built checkout of a mid-series commit is the one + // way to reach the misjudgement, and it costs that node one rebuild. if ( Date.parse(node.build.committedAt) > Date.parse(gatewayBuild.committedAt) ) { @@ -175,11 +179,15 @@ export class Rolling { */ onCheckIn(node: NodeState, now: Date): boolean { const { state } = upgradeStateOf(node, this.gatewayBuild, now); - if ( - (state === 'current' || state === 'ahead') && - node.upgradeToldAt !== null - ) { - this.fleet.setUpgradeToldAt(node.id, null); + if (state === 'current' || state === 'ahead') { + // Off the old build: the tell, if any, is fulfilled, and so is the + // operator's pending re-tell — that hand was for the node that was + // behind, and a re-tell left standing would fire the moment this + // node read behind again, whatever the order said (found by + // review, 2026-09-15). + if (node.upgradeToldAt !== null) { + this.fleet.setUpgradeToldAt(node.id, null); + } this.retell.delete(node.id); return false; } From 58a9cfcaf6042549a561834b1359500b5aec8096 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 22:59:20 +0800 Subject: [PATCH 69/89] removeNode forgets the node's pending re-tell; --node-id and --node-endpoint are refused on the gateway's machine and when they contradict a node's env file The operator's re-tell (applyUpgrade {nodeId}) lived in Rolling's memory until the node was told or checked in off the old build. A node re-told, then removed, left it standing: a machine re-imaged under the same id joined as a new node and was told at its first check-in, past the one-at-a-time order. Removal forgets the hand along with the row. --node-id and --node-endpoint were read only at a node machine's first install. On the gateway's machine, or on a node's re-run with a value that contradicts /etc/dormice/env, they were taken for a change and did nothing. Refused before anything is installed, with the edit that does change it, as --registry-addr and --gateway already are. --- deploy/install.sh | 26 ++++++++++++++++++--- packages/gateway/src/app.ts | 2 +- packages/gateway/src/rolling.test.ts | 13 +++++++++++ packages/gateway/src/rolling.ts | 12 +++++++++- packages/gateway/src/routes/nodes.ts | 5 +++- packages/gateway/src/routes/upgrade.test.ts | 16 +++++++++++++ 6 files changed, 68 insertions(+), 6 deletions(-) diff --git a/deploy/install.sh b/deploy/install.sh index bf40a7b4..b36c3d7c 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -34,9 +34,11 @@ # run doesn't need it # --role node first install of a node machine (see above) # --gateway URL the gateway this node joins (--role node, first install) -# --node-id ID this node's name in the fleet (default: the hostname) -# --node-endpoint URL where the gateway reaches this node (default: -# http://:80) +# --node-id ID this node's name in the fleet (--role node, first +# install; default: the hostname) +# --node-endpoint URL where the gateway reaches this node (--role node, +# first install; default: http://:80) # --registry-addr H:P the address the fleet registry listens on and the # nodes pull from (gateway's machine, first install; # default: this machine's private address, port 5000 @@ -163,6 +165,18 @@ if [ -n "$ENV_GATEWAY" ] && ! is_loopback_url "$ENV_GATEWAY"; then if [ -n "$GATEWAY_FLAG" ] && [ "${GATEWAY_FLAG%/}" != "$GATEWAY_URL" ]; then die "$ENV_FILE says this node's gateway is $GATEWAY_URL, --gateway says ${GATEWAY_FLAG%/} — edit DORMICE_GATEWAY_ENDPOINT in the env file if the gateway really moved, then re-run without the flag" fi + # The env file is the node's identity on a re-run, as it is the gateway's + # address: a flag that repeats it is harmless, one that contradicts it + # would be taken for a change and silently do nothing (found by review, + # 2026-09-15). + env_node_id=$(sed -n 's/^DORMICE_NODE_ID=//p' "$ENV_FILE" | head -1) + if [ -n "$NODE_ID_FLAG" ] && [ "$NODE_ID_FLAG" != "$env_node_id" ]; then + die "$ENV_FILE says this node is $env_node_id, --node-id says $NODE_ID_FLAG — the id is the node's name in the gateway's rows and its sandboxes'; to re-join under another, edit DORMICE_NODE_ID in the env file and removeNode the old id at the gateway, then re-run without the flag" + fi + env_node_endpoint=$(sed -n 's/^DORMICE_NODE_ENDPOINT=//p' "$ENV_FILE" | head -1) + if [ -n "$NODE_ENDPOINT_FLAG" ] && [ "${NODE_ENDPOINT_FLAG%/}" != "${env_node_endpoint%/}" ]; then + die "$ENV_FILE says the gateway reaches this node at $env_node_endpoint, --node-endpoint says ${NODE_ENDPOINT_FLAG%/} — edit DORMICE_NODE_ENDPOINT in the env file if the address really changed (the daemon reports it at its next check-in), then re-run without the flag" + fi elif [ "$ROLE_FLAG" = node ]; then if [ -f "$ENV_FILE" ]; then die "$ENV_FILE exists and names a gateway on this machine (or none) — this is the gateway's machine; --role node is for a machine that has never been installed. To turn it into a node, stop and disable dormice-gateway, move the env file aside, and re-run" @@ -185,6 +199,12 @@ fi # fleet's settings name, and a flag here would be taken for a setting and # silently do nothing (found by review, 2026-09-15). [ "$ROLE" = node ] && [ -n "$REGISTRY_ADDR_FLAG" ] && die "--registry-addr is the gateway machine's flag — a node pulls from the registry its gateway names; re-run without it" +# And the node's identity flags are a node machine's: the gateway machine's +# node is the daemon beside the gateway, named by its own env (config.ts +# DORMICE_NODE_ID), and a flag here would be read by nobody (same review). +if [ "$ROLE" = gateway ] && { [ -n "$NODE_ID_FLAG" ] || [ -n "$NODE_ENDPOINT_FLAG" ]; }; then + die "--node-id and --node-endpoint are a node machine's flags (--role node, first install) — this is the gateway's machine, whose node is the daemon beside the gateway (DORMICE_NODE_ID in $ENV_FILE names it); re-run without them" +fi # ---- outcome reporting and the build rollback -------------------------------- # status.json is the one file the daemon's one-click upgrade reads back; diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts index 93e5a0f5..34241e42 100644 --- a/packages/gateway/src/app.ts +++ b/packages/gateway/src/app.ts @@ -312,7 +312,7 @@ export function buildGatewayApp({ app.register(async (admin) => { admin.addHook('onRequest', adminAuth); await admin.register(apiKeyRoutes, { db }); - await admin.register(nodeRoutes, { fleet, cache: finder.cache }); + await admin.register(nodeRoutes, { fleet, cache: finder.cache, rolling }); await admin.register(settingsRoutes, { config, db, diff --git a/packages/gateway/src/rolling.test.ts b/packages/gateway/src/rolling.test.ts index 1faf95bb..0278a25e 100644 --- a/packages/gateway/src/rolling.test.ts +++ b/packages/gateway/src/rolling.test.ts @@ -327,6 +327,19 @@ describe('Rolling', () => { expect(rolling.onCheckIn(c, later)).toBe(false); reporting(fleet, 'c', { build: OLD, selfUpgrade: CAN }, later); expect(rolling.onCheckIn(c, later)).toBe(false); + // And by the node's removal: d, re-told, is removed and a machine + // under the same id joins behind — a new node, waiting its turn. + const d = reporting(fleet, 'd', { build: OLD, selfUpgrade: CAN }, later); + expect(rolling.requestRetell(d, later)).toBeNull(); + fleet.remove('d'); + rolling.forget('d'); + const again = reporting( + fleet, + 'd', + { build: OLD, selfUpgrade: CAN }, + later, + ); + expect(rolling.onCheckIn(again, later)).toBe(false); }); it('states lists every node in id order with its standing, build and tell', () => { diff --git a/packages/gateway/src/rolling.ts b/packages/gateway/src/rolling.ts index 7da48f1e..6d911117 100644 --- a/packages/gateway/src/rolling.ts +++ b/packages/gateway/src/rolling.ts @@ -161,7 +161,7 @@ export function rollingDecision( * object the check-in route and the upgrade routes share. */ export class Rolling { - /** Nodes the operator told to upgrade again (applyUpgrade {nodeId}), told at their next check-in whatever the order says. Memory: see the module comment. */ + /** Nodes the operator told to upgrade again (applyUpgrade {nodeId}), told at their next check-in whatever the order says. Memory: see the module comment. Spent by the tell, by a check-in off the old build (onCheckIn), or by the node's removal (forget). */ private readonly retell = new Set(); constructor( @@ -238,6 +238,16 @@ export class Rolling { } } + /** + * The node is gone (removeNode): its pending re-tell goes with it. The + * hand was for that node; a machine re-imaged under the same id joins + * as a new node, and must wait its turn like one — not be told at its + * first check-in past the order (found by review, 2026-09-15). + */ + forget(id: string): void { + this.retell.delete(id); + } + /** Every node's standing right now (getUpgradeStatus.nodes), in node-id order. */ states(now: Date): NodeUpgradeView[] { return this.fleet diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index cb1f659c..00663a91 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -28,6 +28,8 @@ export interface CheckInRoutesOptions { export interface NodeRoutesOptions { fleet: Fleet; cache: NameCache; + /** Forgets a removed node's pending re-tell (rolling.ts forget). */ + rolling: Rolling; } /** A refusal in the native dialect, rendered by the app's error handler as `{ message }` under its status. */ @@ -178,7 +180,7 @@ export const checkInRoutes: FastifyPluginAsyncZod< */ export const nodeRoutes: FastifyPluginAsyncZod = async ( app, - { fleet, cache }, + { fleet, cache, rolling }, ) => { app.post( '/listNodes', @@ -278,6 +280,7 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( } const removed = fleet.remove(request.body.id); const evicted = cache.evictNode(request.body.id); + rolling.forget(request.body.id); if (removed) { request.log.warn( { nodeId: request.body.id, evicted }, diff --git a/packages/gateway/src/routes/upgrade.test.ts b/packages/gateway/src/routes/upgrade.test.ts index ee07a301..a5fa8539 100644 --- a/packages/gateway/src/routes/upgrade.test.ts +++ b/packages/gateway/src/routes/upgrade.test.ts @@ -130,6 +130,22 @@ describe('the fleet upgrade over the check-in', () => { expect((await status(app)).nodes?.find((n) => n.id === 'a')?.state).toBe( 'upgrading', ); + // The hand does not outlive the node: c, behind and re-told, goes + // silent and is removed; a machine under the same id joins behind + // while a upgrades, and waits its turn. + expect( + (await checkIn(app, 'c', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBeUndefined(); + expect((await rpc(app, '/applyUpgrade', { nodeId: 'c' })).statusCode).toBe( + 200, + ); + const c = fleet.get('c'); + if (!c) throw new Error('node lost'); + c.lastCheckInAt = new Date(Date.now() - 40_000); + expect((await rpc(app, '/removeNode', { id: 'c' })).statusCode).toBe(200); + expect( + (await checkIn(app, 'c', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBeUndefined(); }); it('nodes that cannot upgrade themselves, did not say, or carry no build are listed with the reason and never told', async () => { From 95375570e814abf1ff6dd0d49f3dd010c26eee2f Mon Sep 17 00:00:00 2001 From: Annactswell Date: Tue, 15 Sep 2026 23:25:48 +0800 Subject: [PATCH 70/89] The operator's hand on a stuck node puts it back in line instead of telling it past the order; install.sh judges the four fleet flags against the role in one place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit applyUpgrade {nodeId} used to mean "tell this node at its next check-in, whatever the one-at-a-time order says", kept as a set of node ids in the gateway's memory until the tell. Three reviews in one day each found a way for that memory to outlive the node it was for — a check-in that read current or ahead, the node's removal — and tell two nodes into one minute. The roll promises one node down at a time; a hand that could break the promise was the wrong hand. Now the hand forgets a stuck node's tell on its row: the node reads behind and is told at its turn, after the node upgrading now, never beside it. Refused everywhere else in words — 409 on a node upgrading, whose tell is what the rule counts; 400 on one behind, in line already. Rolling keeps nothing of its own: every verdict is a function of the rows, the gateway's build and the clock. The console's toast, the SDK's and the docs' wording follow. install.sh judged each fleet flag where it was consumed, hundreds of lines and a role branch apart — and each review found one flag the other role's machine, or a re-run with a contradicting value, silently ignored. The four (--gateway, --node-id, --node-endpoint, --registry-addr) are now judged once, at the top, with the role known and nothing installed: the other role's flag is refused; a flag that contradicts the env file's line is refused with the edit that does change the value; one that repeats it is harmless. --- deploy/install.sh | 78 ++++++------ packages/console/messages/de/settings.json | 2 +- packages/console/messages/en/settings.json | 2 +- packages/console/messages/es/settings.json | 2 +- packages/console/messages/fr/settings.json | 2 +- packages/console/messages/ja/settings.json | 2 +- packages/console/messages/ko/settings.json | 2 +- packages/console/messages/pt-BR/settings.json | 2 +- packages/console/messages/ru/settings.json | 2 +- packages/console/messages/zh-CN/settings.json | 2 +- packages/console/messages/zh-TW/settings.json | 2 +- .../settings/components/VersionCard.tsx | 4 +- packages/gateway/src/app.ts | 2 +- packages/gateway/src/rolling.test.ts | 73 ++++++------ packages/gateway/src/rolling.ts | 112 +++++++++--------- packages/gateway/src/routes/nodes.ts | 5 +- packages/gateway/src/routes/upgrade.test.ts | 40 ++++--- packages/gateway/src/routes/upgrade.ts | 8 +- packages/sdk/src/client.ts | 2 +- packages/server/src/check-in.ts | 4 +- packages/shared/src/upgrade.ts | 13 +- website/content/docs/troubleshooting.mdx | 3 +- website/content/docs/upgrading.mdx | 9 +- 23 files changed, 188 insertions(+), 185 deletions(-) diff --git a/deploy/install.sh b/deploy/install.sh index b36c3d7c..2e8caa02 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -43,6 +43,10 @@ # nodes pull from (gateway's machine, first install; # default: this machine's private address, port 5000 # — a re-run keeps the address gateway.env holds) +# The four fleet flags (--gateway, --node-id, --node-endpoint, +# --registry-addr) are judged against the role before anything is +# installed: one on the other role's machine, or one contradicting the +# env file on a re-run, is refused — never silently ignored. # # Four promises, mirroring `dor doctor`: # - Idempotent. Every step checks before it acts; a step whose outcome is @@ -162,21 +166,6 @@ ENV_GATEWAY='' if [ -n "$ENV_GATEWAY" ] && ! is_loopback_url "$ENV_GATEWAY"; then ROLE=node GATEWAY_URL=${ENV_GATEWAY%/} - if [ -n "$GATEWAY_FLAG" ] && [ "${GATEWAY_FLAG%/}" != "$GATEWAY_URL" ]; then - die "$ENV_FILE says this node's gateway is $GATEWAY_URL, --gateway says ${GATEWAY_FLAG%/} — edit DORMICE_GATEWAY_ENDPOINT in the env file if the gateway really moved, then re-run without the flag" - fi - # The env file is the node's identity on a re-run, as it is the gateway's - # address: a flag that repeats it is harmless, one that contradicts it - # would be taken for a change and silently do nothing (found by review, - # 2026-09-15). - env_node_id=$(sed -n 's/^DORMICE_NODE_ID=//p' "$ENV_FILE" | head -1) - if [ -n "$NODE_ID_FLAG" ] && [ "$NODE_ID_FLAG" != "$env_node_id" ]; then - die "$ENV_FILE says this node is $env_node_id, --node-id says $NODE_ID_FLAG — the id is the node's name in the gateway's rows and its sandboxes'; to re-join under another, edit DORMICE_NODE_ID in the env file and removeNode the old id at the gateway, then re-run without the flag" - fi - env_node_endpoint=$(sed -n 's/^DORMICE_NODE_ENDPOINT=//p' "$ENV_FILE" | head -1) - if [ -n "$NODE_ENDPOINT_FLAG" ] && [ "${NODE_ENDPOINT_FLAG%/}" != "${env_node_endpoint%/}" ]; then - die "$ENV_FILE says the gateway reaches this node at $env_node_endpoint, --node-endpoint says ${NODE_ENDPOINT_FLAG%/} — edit DORMICE_NODE_ENDPOINT in the env file if the address really changed (the daemon reports it at its next check-in), then re-run without the flag" - fi elif [ "$ROLE_FLAG" = node ]; then if [ -f "$ENV_FILE" ]; then die "$ENV_FILE exists and names a gateway on this machine (or none) — this is the gateway's machine; --role node is for a machine that has never been installed. To turn it into a node, stop and disable dormice-gateway, move the env file aside, and re-run" @@ -195,15 +184,41 @@ else ROLE=gateway GATEWAY_URL="http://127.0.0.1:$GATEWAY_PORT" fi -# The registry is the gateway machine's; a node pulls from the address the -# fleet's settings name, and a flag here would be taken for a setting and -# silently do nothing (found by review, 2026-09-15). -[ "$ROLE" = node ] && [ -n "$REGISTRY_ADDR_FLAG" ] && die "--registry-addr is the gateway machine's flag — a node pulls from the registry its gateway names; re-run without it" -# And the node's identity flags are a node machine's: the gateway machine's -# node is the daemon beside the gateway, named by its own env (config.ts -# DORMICE_NODE_ID), and a flag here would be read by nobody (same review). -if [ "$ROLE" = gateway ] && { [ -n "$NODE_ID_FLAG" ] || [ -n "$NODE_ENDPOINT_FLAG" ]; }; then - die "--node-id and --node-endpoint are a node machine's flags (--role node, first install) — this is the gateway's machine, whose node is the daemon beside the gateway (DORMICE_NODE_ID in $ENV_FILE names it); re-run without them" + +# ---- the fleet flags against the role ----------------------------------------- +# Every flag but --mirror, --swap-gb and --status-dir names this machine's +# place in the fleet, and each belongs to one role: a node machine's say +# which gateway it joins and how it is known there, the gateway machine's +# where its registry listens. Judged here, once, with the role known and +# nothing installed yet. On the other role's machine a flag is read by +# nobody; on a re-run the env files hold the value, and a flag would be +# taken for a change and silently do nothing — three reviews in one day +# found one such flag each (2026-09-15), because each flag was judged +# where it was consumed, hundreds of lines and a role branch away from the +# others. A flag that repeats the env file's line is harmless; one that +# contradicts it is refused, with the edit that does change the value. +flag_against_env() { # + [ -n "$2" ] && [ -f "$3" ] || return 0 + have=$(sed -n "s/^$4=//p" "$3" | head -1) + [ -n "$have" ] || return 0 + [ "${2%/}" = "${have%/}" ] && return 0 + die "$3 says $4=$have, $1 says $2 — $5" +} +if [ "$ROLE" = gateway ]; then + for f in "--gateway=$GATEWAY_FLAG" "--node-id=$NODE_ID_FLAG" "--node-endpoint=$NODE_ENDPOINT_FLAG"; do + [ -n "${f#*=}" ] || continue + die "${f%%=*} is a node machine's flag (--role node, first install) — this is the gateway's machine, whose node is the daemon beside the gateway (DORMICE_NODE_ID in $ENV_FILE names it); re-run without it" + done + flag_against_env --registry-addr "$REGISTRY_ADDR_FLAG" "$GATEWAY_ENV_FILE" DORMICE_REGISTRY_ADDRESS \ + "the address is the fleet's setting: the gateway's row carries it to every node, and each pins the registry's certificate under it; it does not move with a flag — re-run without it" +else + [ -z "$REGISTRY_ADDR_FLAG" ] || die "--registry-addr is the gateway machine's flag — a node pulls from the registry its gateway names; re-run without it" + flag_against_env --gateway "$GATEWAY_FLAG" "$ENV_FILE" DORMICE_GATEWAY_ENDPOINT \ + "edit DORMICE_GATEWAY_ENDPOINT in the env file if the gateway really moved, then re-run without the flag" + flag_against_env --node-id "$NODE_ID_FLAG" "$ENV_FILE" DORMICE_NODE_ID \ + "the id is the node's name in the gateway's rows and its sandboxes'; to re-join under another, edit DORMICE_NODE_ID in the env file and removeNode the old id at the gateway, then re-run without the flag" + flag_against_env --node-endpoint "$NODE_ENDPOINT_FLAG" "$ENV_FILE" DORMICE_NODE_ENDPOINT \ + "edit DORMICE_NODE_ENDPOINT in the env file if the address really changed (the daemon reports it at its next check-in), then re-run without the flag" fi # ---- outcome reporting and the build rollback -------------------------------- @@ -1080,19 +1095,10 @@ fi # the private address the other machines reach it by; docker0's 172.17.0.1 # never is). A machine whose main address is public listens on it; the # lock is what makes that acceptable. +# (A flag contradicting the env line died at the top, with the other fleet +# flags.) env_registry_addr=$(sed -n 's/^DORMICE_REGISTRY_ADDRESS=//p' "$GATEWAY_ENV_FILE" | head -1) -REGISTRY_ADDR=$REGISTRY_ADDR_FLAG -if [ -n "$REGISTRY_ADDR" ] && [ -n "$env_registry_addr" ] && [ "$REGISTRY_ADDR" != "$env_registry_addr" ]; then - # Once seeded, the address is the fleet's setting: the gateway's row - # carries it to every node, and each node pins the certificate under it - # and pulls from it. A flag on a re-run would move the listener and - # nothing else — the env line and the row stand — and every node would - # pull from where the registry no longer is. Refused, as --gateway is - # against a node's env (found by review, 2026-09-15); moving the - # registry is not a re-run's job in this version. - die "$GATEWAY_ENV_FILE says the fleet registry is at $env_registry_addr, --registry-addr says $REGISTRY_ADDR — the address is the fleet's setting and does not move with a flag; re-run without it" -fi -[ -n "$REGISTRY_ADDR" ] || REGISTRY_ADDR=$env_registry_addr +REGISTRY_ADDR=${REGISTRY_ADDR_FLAG:-$env_registry_addr} if [ -z "$REGISTRY_ADDR" ]; then registry_host=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}') [ -n "$registry_host" ] || registry_host=$(hostname -I 2>/dev/null | awk '{print $1}') diff --git a/packages/console/messages/de/settings.json b/packages/console/messages/de/settings.json index 8d3c2131..725b9645 100644 --- a/packages/console/messages/de/settings.json +++ b/packages/console/messages/de/settings.json @@ -152,7 +152,7 @@ "settings_nodes_state_unreachable": "Nicht erreichbar", "settings_nodes_state_unknown": "Unbekannt", "settings_nodes_retry": "Erneut versuchen", - "settings_nodes_retry_done": "Knoten {id} benachrichtigt; er aktualisiert beim nächsten Check-in erneut", + "settings_nodes_retry_done": "Knoten {id} ist wieder in der Reihe; er wird beim nächsten Check-in benachrichtigt, sobald kein anderer Knoten aktualisiert", "settings_nodes_retry_failed": "Knoten {id} konnte nicht benachrichtigt werden: {error}", "settings_archive_card_title": "Archivspeicher (S3)", "settings_archive_card_desc": "Datenträger untätiger Sandboxes wandern komprimiert in einen S3-kompatiblen Speicher, null lokaler Platz, und werden beim nächsten acquire automatisch wiederhergestellt. Wohnt im Ledger; Änderungen wirken sofort, ohne Neustart.", diff --git a/packages/console/messages/en/settings.json b/packages/console/messages/en/settings.json index ed0293a8..99fd0cad 100644 --- a/packages/console/messages/en/settings.json +++ b/packages/console/messages/en/settings.json @@ -152,7 +152,7 @@ "settings_nodes_state_unreachable": "Unreachable", "settings_nodes_state_unknown": "Unknown", "settings_nodes_retry": "Try again", - "settings_nodes_retry_done": "Node {id} told; it upgrades again at its next check-in", + "settings_nodes_retry_done": "Node {id} is back in line; it is told at its next check-in once no other node is upgrading", "settings_nodes_retry_failed": "Could not tell node {id}: {error}", "settings_archive_card_title": "Archive store (S3)", "settings_archive_card_desc": "Idle sandboxes' disks compress into any S3-compatible store at zero local cost, and restore on the next acquire. Lives in the ledger; changes apply immediately, no restart.", diff --git a/packages/console/messages/es/settings.json b/packages/console/messages/es/settings.json index 560e14ab..307f81c2 100644 --- a/packages/console/messages/es/settings.json +++ b/packages/console/messages/es/settings.json @@ -152,7 +152,7 @@ "settings_nodes_state_unreachable": "Inalcanzable", "settings_nodes_state_unknown": "Desconocido", "settings_nodes_retry": "Reintentar", - "settings_nodes_retry_done": "Nodo {id} avisado; se actualiza de nuevo en su siguiente registro", + "settings_nodes_retry_done": "El nodo {id} vuelve a la cola; se le avisa en su siguiente registro cuando ningún otro nodo esté actualizándose", "settings_nodes_retry_failed": "No se pudo avisar al nodo {id}: {error}", "settings_archive_card_title": "Almacenamiento de archivado (S3)", "settings_archive_card_desc": "El disco de los sandboxes inactivos se comprime y se sube a un almacenamiento compatible con S3, sin ocupar nada en local, y se restaura solo en el siguiente acquire. Vive en el libro de registro; los cambios se aplican de inmediato, sin reiniciar.", diff --git a/packages/console/messages/fr/settings.json b/packages/console/messages/fr/settings.json index 0bccbb36..0d850eab 100644 --- a/packages/console/messages/fr/settings.json +++ b/packages/console/messages/fr/settings.json @@ -152,7 +152,7 @@ "settings_nodes_state_unreachable": "Injoignable", "settings_nodes_state_unknown": "Inconnu", "settings_nodes_retry": "Réessayer", - "settings_nodes_retry_done": "Nœud {id} prévenu ; il se met à niveau à nouveau à son prochain check-in", + "settings_nodes_retry_done": "Le nœud {id} est de retour dans la file ; il est prévenu à son prochain check-in dès qu'aucun autre nœud ne se met à niveau", "settings_nodes_retry_failed": "Impossible de prévenir le nœud {id} : {error}", "settings_archive_card_title": "Stockage d'archivage (S3)", "settings_archive_card_desc": "Le disque des sandbox inactives est compressé vers un stockage compatible S3, sans empreinte locale, et restauré automatiquement au prochain acquire. Stocké dans le registre ; les changements prennent effet immédiatement, sans redémarrage.", diff --git a/packages/console/messages/ja/settings.json b/packages/console/messages/ja/settings.json index ce94ecfe..3a029f63 100644 --- a/packages/console/messages/ja/settings.json +++ b/packages/console/messages/ja/settings.json @@ -152,7 +152,7 @@ "settings_nodes_state_unreachable": "到達不能", "settings_nodes_state_unknown": "不明", "settings_nodes_retry": "再試行", - "settings_nodes_retry_done": "ノード {id} に通知しました。次のチェックイン時に再びアップグレードします", + "settings_nodes_retry_done": "ノード {id} は列に戻りました。他のノードがアップグレード中でなければ、次のチェックイン時に通知されます", "settings_nodes_retry_failed": "ノード {id} に通知できません:{error}", "settings_archive_card_title": "アーカイブストレージ(S3)", "settings_archive_card_desc": "アイドルなサンドボックスのディスクを圧縮して S3 互換ストレージにアップロードし、ローカル使用量はゼロになります。次回の acquire で自動的に復元されます。台帳に保存され、変更は再起動なしで即時に反映されます。", diff --git a/packages/console/messages/ko/settings.json b/packages/console/messages/ko/settings.json index 6c976429..80e20d9a 100644 --- a/packages/console/messages/ko/settings.json +++ b/packages/console/messages/ko/settings.json @@ -152,7 +152,7 @@ "settings_nodes_state_unreachable": "접근 불가", "settings_nodes_state_unknown": "알 수 없음", "settings_nodes_retry": "다시 시도", - "settings_nodes_retry_done": "노드 {id}에 통지했습니다. 다음 체크인 때 다시 업그레이드합니다", + "settings_nodes_retry_done": "노드 {id}가 대기열로 돌아왔습니다. 다른 노드가 업그레이드 중이 아니면 다음 체크인 때 통지됩니다", "settings_nodes_retry_failed": "노드 {id}에 통지할 수 없습니다: {error}", "settings_archive_card_title": "아카이브 스토리지(S3)", "settings_archive_card_desc": "유휴 샌드박스의 디스크를 압축해 S3 호환 스토리지로 업로드하며 로컬 점유는 0이고, 다음 acquire 때 자동으로 복원됩니다. 장부에 저장되어 변경 즉시 적용되며 재시작이 필요 없습니다.", diff --git a/packages/console/messages/pt-BR/settings.json b/packages/console/messages/pt-BR/settings.json index 1d6ee50b..fe903a8e 100644 --- a/packages/console/messages/pt-BR/settings.json +++ b/packages/console/messages/pt-BR/settings.json @@ -152,7 +152,7 @@ "settings_nodes_state_unreachable": "Inacessível", "settings_nodes_state_unknown": "Desconhecido", "settings_nodes_retry": "Tentar de novo", - "settings_nodes_retry_done": "Nó {id} avisado; ele atualiza de novo no próximo check-in", + "settings_nodes_retry_done": "O nó {id} voltou para a fila; ele é avisado no próximo check-in quando nenhum outro nó estiver atualizando", "settings_nodes_retry_failed": "Não foi possível avisar o nó {id}: {error}", "settings_archive_card_title": "Armazenamento de arquivamento (S3)", "settings_archive_card_desc": "O disco dos sandboxes ociosos é comprimido e enviado para qualquer armazenamento compatível com S3, sem ocupar nada local, e volta sozinho no próximo acquire. Mora no ledger; mudanças entram em vigor imediatamente, sem reiniciar.", diff --git a/packages/console/messages/ru/settings.json b/packages/console/messages/ru/settings.json index d3700c41..9938a2de 100644 --- a/packages/console/messages/ru/settings.json +++ b/packages/console/messages/ru/settings.json @@ -152,7 +152,7 @@ "settings_nodes_state_unreachable": "Недоступен", "settings_nodes_state_unknown": "Неизвестно", "settings_nodes_retry": "Повторить", - "settings_nodes_retry_done": "Узлу {id} дано указание; он обновится снова при следующем чек-ине", + "settings_nodes_retry_done": "Узел {id} снова в очереди; он получит указание при следующем чек-ине, когда ни один другой узел не обновляется", "settings_nodes_retry_failed": "Не удалось дать указание узлу {id}: {error}", "settings_archive_card_title": "Хранилище архивов (S3)", "settings_archive_card_desc": "Диски простаивающих песочниц сжимаются и уходят в любое S3-совместимое хранилище, локально не занимая ничего, а при следующем acquire восстанавливаются. Хранится в реестре; изменения действуют сразу, без перезапуска.", diff --git a/packages/console/messages/zh-CN/settings.json b/packages/console/messages/zh-CN/settings.json index a325443e..539c0561 100644 --- a/packages/console/messages/zh-CN/settings.json +++ b/packages/console/messages/zh-CN/settings.json @@ -152,7 +152,7 @@ "settings_nodes_state_unreachable": "不可达", "settings_nodes_state_unknown": "未知", "settings_nodes_retry": "再试一次", - "settings_nodes_retry_done": "已通知节点 {id},它在下一次报到时重新升级", + "settings_nodes_retry_done": "节点 {id} 已回到队列;没有别的节点在升级时,它在下一次报到被带起", "settings_nodes_retry_failed": "无法通知节点 {id}:{error}", "settings_archive_card_title": "归档存储(S3)", "settings_archive_card_desc": "闲置沙箱的磁盘压缩上传到 S3 兼容存储、本地零占用,下次 acquire 自动恢复。住在账本里,改了立即生效,不用重启。", diff --git a/packages/console/messages/zh-TW/settings.json b/packages/console/messages/zh-TW/settings.json index 7f8d42a4..8a94d3d6 100644 --- a/packages/console/messages/zh-TW/settings.json +++ b/packages/console/messages/zh-TW/settings.json @@ -152,7 +152,7 @@ "settings_nodes_state_unreachable": "不可達", "settings_nodes_state_unknown": "未知", "settings_nodes_retry": "再試一次", - "settings_nodes_retry_done": "已通知節點 {id},它在下一次報到時重新升級", + "settings_nodes_retry_done": "節點 {id} 已回到隊列;沒有別的節點在升級時,它在下一次報到被帶起", "settings_nodes_retry_failed": "無法通知節點 {id}:{error}", "settings_archive_card_title": "封存儲存(S3)", "settings_archive_card_desc": "閒置沙箱的磁碟壓縮上傳到 S3 相容儲存、本機零佔用,下次 acquire 自動還原。住在帳本裡,改了立即生效,不用重新啟動。", diff --git a/packages/console/src/features/settings/components/VersionCard.tsx b/packages/console/src/features/settings/components/VersionCard.tsx index 2bd67977..57765502 100644 --- a/packages/console/src/features/settings/components/VersionCard.tsx +++ b/packages/console/src/features/settings/components/VersionCard.tsx @@ -42,8 +42,8 @@ import { UpgradeDialog } from './UpgradeDialog'; * 失败装成「已是最新」。「升级」按钮只在网关自报一键可用时出现,否则 * 给手动路径与原因。卡片下方一张节点表:每台对着网关构建的站位,由 * 网关裁决(state 字段);「卡住」的行给「再试一次」= applyUpgrade - * {nodeId},人工重告 — 网关自己绝不重告(构建总失败的节点不能每 20 - * 分钟白烤一遍)。「比网关新」的节点不给按钮:该升的是网关,理由在 + * {nodeId}=忘掉那次告知、回到队列按序再被带起,不越过正在升级的那台 + * — 网关自己绝不重告(构建总失败的节点不能每 20 分钟白烤一遍)。「比网关新」的节点不给按钮:该升的是网关,理由在 * title 里说。 */ export function VersionCard() { diff --git a/packages/gateway/src/app.ts b/packages/gateway/src/app.ts index 34241e42..93e5a0f5 100644 --- a/packages/gateway/src/app.ts +++ b/packages/gateway/src/app.ts @@ -312,7 +312,7 @@ export function buildGatewayApp({ app.register(async (admin) => { admin.addHook('onRequest', adminAuth); await admin.register(apiKeyRoutes, { db }); - await admin.register(nodeRoutes, { fleet, cache: finder.cache, rolling }); + await admin.register(nodeRoutes, { fleet, cache: finder.cache }); await admin.register(settingsRoutes, { config, db, diff --git a/packages/gateway/src/rolling.test.ts b/packages/gateway/src/rolling.test.ts index 0278a25e..b6df028e 100644 --- a/packages/gateway/src/rolling.test.ts +++ b/packages/gateway/src/rolling.test.ts @@ -241,22 +241,41 @@ describe('Rolling', () => { expect(rolling.onCheckIn(b, later)).toBe(true); }); - it("requestRetell honors the operator's hand at the next check-in even while another node upgrades, and refuses in words where it would do nothing", () => { - const { fleet } = fleetOver(); + it("retell forgets a stuck node's tell: it reads behind, waits for the node upgrading and is told at its turn; every other state is refused in words", () => { + const { db, fleet } = fleetOver(); const rolling = new Rolling(fleet, GATEWAY); const a = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); const b = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }); expect(rolling.onCheckIn(a, NOW)).toBe(true); - // b is behind and a is upgrading: b would wait — unless the operator says so. expect(rolling.onCheckIn(b, NOW)).toBe(false); - expect(rolling.requestRetell(b, NOW)).toBeNull(); - expect(rolling.onCheckIn(b, NOW)).toBe(true); - // A stuck node is the case the hand exists for. + // b is behind and in line already; a is upgrading and its tell is the + // one-at-a-time rule's count — neither is the hand's to touch. + expect(rolling.retell(b, NOW)).toMatchObject({ + status: 400, + message: expect.stringMatching(/behind and in line/), + }); + expect(rolling.retell(a, NOW)).toMatchObject({ + status: 409, + message: expect.stringMatching( + /is upgrading \(told 0s ago, still on old0001\)/, + ), + }); + expect(a.upgradeToldAt).toEqual(NOW); + // Twenty minutes on, a is stuck and b's turn came. const late = new Date(NOW.getTime() + UPGRADE_TOLD_TIMEOUT_MS); reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }, late); - expect(upgradeStateOf(a, GATEWAY, late).state).toBe('stuck'); + reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }, late); expect(rolling.onCheckIn(a, late)).toBe(false); - expect(rolling.requestRetell(a, late)).toBeNull(); + expect(rolling.onCheckIn(b, late)).toBe(true); + // The hand: a's tell is forgotten on the row, and a reads behind. + expect(rolling.retell(a, late)).toBeNull(); + expect(a.upgradeToldAt).toBeNull(); + expect(new Fleet(db).get('a')?.upgradeToldAt).toBeNull(); + expect(upgradeStateOf(a, GATEWAY, late).state).toBe('behind'); + // Not past b: a waits while b upgrades, and is told once b is back. + expect(rolling.onCheckIn(a, late)).toBe(false); + reporting(fleet, 'b', { build: GATEWAY, selfUpgrade: CAN }, late); + expect(rolling.onCheckIn(b, late)).toBe(false); expect(rolling.onCheckIn(a, late)).toBe(true); expect(a.upgradeToldAt).toEqual(late); @@ -266,7 +285,7 @@ describe('Rolling', () => { { build: GATEWAY, selfUpgrade: CAN }, late, ); - expect(rolling.requestRetell(current, late)).toMatchObject({ + expect(rolling.retell(current, late)).toMatchObject({ status: 400, message: expect.stringMatching(/already runs the gateway's build/), }); @@ -276,23 +295,23 @@ describe('Rolling', () => { { build: OLD, selfUpgrade: CANNOT }, late, ); - expect(rolling.requestRetell(cannot, late)).toMatchObject({ + expect(rolling.retell(cannot, late)).toMatchObject({ status: 400, message: expect.stringMatching(/cannot be told to upgrade: systemd-run/), }); const gone = reporting(fleet, 'e', { build: OLD, selfUpgrade: CAN }, late); gone.lastCheckInAt = new Date(late.getTime() - 40_000); - expect(rolling.requestRetell(gone, late)).toMatchObject({ - status: 409, + expect(rolling.retell(gone, late)).toMatchObject({ + status: 400, message: expect.stringMatching(/not checking in/), }); - expect(new Rolling(fleet, null).requestRetell(a, late)).toMatchObject({ + expect(new Rolling(fleet, null).retell(a, late)).toMatchObject({ status: 400, message: expect.stringMatching(/gateway carries no build identity/), }); }); - it('a told node that comes back newer than the gateway is ahead: its tell is fulfilled and cleared, it is not told again, the operator cannot re-tell it, and it holds nobody', () => { + it('a told node that comes back newer than the gateway is ahead: its tell is fulfilled and cleared, it is not told again, the hand refuses it, and it holds nobody', () => { const { db, fleet } = fleetOver(); const rolling = new Rolling(fleet, GATEWAY); const a = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); @@ -307,7 +326,7 @@ describe('Rolling', () => { expect(rolling.states(later)).toMatchObject([ { id: 'a', state: 'ahead', toldAt: null }, ]); - expect(rolling.requestRetell(a, later)).toMatchObject({ + expect(rolling.retell(a, later)).toMatchObject({ status: 400, message: expect.stringMatching( /cannot be told to upgrade: runs new0002 .* newer than the gateway's new0001/, @@ -316,30 +335,6 @@ describe('Rolling', () => { // Not upgrading, so it holds no pointer: b's turn comes. const b = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }, later); expect(rolling.onCheckIn(b, later)).toBe(true); - // A pending re-tell is spent by an ahead check-in too: c, behind and - // re-told by the operator while b upgrades, is upgraded by hand to a - // newer build, then put back on the old one — the hand was for the - // node that was, and c waits its turn like any other. - const c = reporting(fleet, 'c', { build: OLD, selfUpgrade: CAN }, later); - expect(rolling.onCheckIn(c, later)).toBe(false); - expect(rolling.requestRetell(c, later)).toBeNull(); - reporting(fleet, 'c', { build: NEWER, selfUpgrade: CAN }, later); - expect(rolling.onCheckIn(c, later)).toBe(false); - reporting(fleet, 'c', { build: OLD, selfUpgrade: CAN }, later); - expect(rolling.onCheckIn(c, later)).toBe(false); - // And by the node's removal: d, re-told, is removed and a machine - // under the same id joins behind — a new node, waiting its turn. - const d = reporting(fleet, 'd', { build: OLD, selfUpgrade: CAN }, later); - expect(rolling.requestRetell(d, later)).toBeNull(); - fleet.remove('d'); - rolling.forget('d'); - const again = reporting( - fleet, - 'd', - { build: OLD, selfUpgrade: CAN }, - later, - ); - expect(rolling.onCheckIn(again, later)).toBe(false); }); it('states lists every node in id order with its standing, build and tell', () => { diff --git a/packages/gateway/src/rolling.ts b/packages/gateway/src/rolling.ts index 6d911117..2fb8b484 100644 --- a/packages/gateway/src/rolling.ts +++ b/packages/gateway/src/rolling.ts @@ -23,13 +23,22 @@ import { downReason, type Fleet, type NodeState } from './fleet'; * its tell is stuck: named as such with where to look, and never re-told * on its own — a node whose build fails every time would otherwise * rebuild every twenty minutes, on the CPU its sandboxes run on. The - * pointer moves past it (a stuck node is not "upgrading"), and the - * operator's applyUpgrade {nodeId} is the hand that tells it again, - * whatever the rolling order says at that moment. The tell is on the - * node's row (nodes.upgrade_told_at), so a gateway restart mid-roll - * neither forgets a node it told nor tells it twice; the operator's - * re-tell is memory — a gateway restarted before the node's next check-in - * forgets it, and the operator clicks again. + * pointer moves past it (a stuck node is not "upgrading"). The operator's + * applyUpgrade {nodeId} puts it back in line: its tell is forgotten, it + * reads behind again, and the roll tells it at its turn — after the node + * upgrading now, if there is one, never beside it. Not a tell past the + * order: the first version's hand was one ("told at its next check-in, + * whatever the order says"), kept in the gateway's memory, and three + * reviews in one day found three ways for that memory to outlive the + * node it was for and tell two nodes into one minute (2026-09-15). One + * node down at a time is the one thing the roll promises; a hand that + * could break it was the wrong hand, and with it gone nothing is + * remembered but the row. + * + * The row: the tell is nodes.upgrade_told_at, so a gateway restart + * mid-roll neither forgets a node it told nor tells it twice, and every + * verdict here is a function of the rows, the gateway's build and the + * clock. * * Behind means older. A node whose build is newer than the gateway's — a * commit that landed on main after the gateway's machine upgraded and @@ -115,7 +124,7 @@ export function upgradeStateOf( } return { state: 'stuck', - reason: `told to upgrade at ${node.upgradeToldAt.toISOString()} and still on ${node.build.commit} ${Math.round(sinceMs / 60_000)} minutes later${silence} — read journalctl -u dormice-upgrade and the upgrade log on the node, then tell it again (applyUpgrade with its nodeId)`, + reason: `told to upgrade at ${node.upgradeToldAt.toISOString()} and still on ${node.build.commit} ${Math.round(sinceMs / 60_000)} minutes later${silence} — read journalctl -u dormice-upgrade and the upgrade log on the node, then put it back in line (applyUpgrade with its nodeId): it is told again at its turn`, }; } if (down !== null) { @@ -156,98 +165,87 @@ export function rollingDecision( } /** - * The fleet upgrade's live half: the gateway's build to judge against, - * the operator's pending re-tells, and the check-in's verdicts — one - * object the check-in route and the upgrade routes share. + * The fleet upgrade's live half: the gateway's build to judge against and + * the fleet's rows — one object the check-in route and the upgrade routes + * share. Nothing of its own: every verdict is a function of the rows, the + * gateway's build and the clock (the module comment has why). */ export class Rolling { - /** Nodes the operator told to upgrade again (applyUpgrade {nodeId}), told at their next check-in whatever the order says. Memory: see the module comment. Spent by the tell, by a check-in off the old build (onCheckIn), or by the node's removal (forget). */ - private readonly retell = new Set(); - constructor( private readonly fleet: Fleet, private readonly gatewayBuild: BuildInfo | null, ) {} /** - * The check-in's verdict for a node that just reported, in order: a - * fulfilled tell is cleared (the node is off the old build — on the - * gateway's, or ahead of it); - * a pending re-tell is honored; otherwise the rolling rule decides. Any - * tell is written to the row before the answer carries it. Answers - * whether the node is told now. + * The check-in's verdict for a node that just reported: a fulfilled tell + * is cleared (the node is off the old build — on the gateway's, or ahead + * of it); otherwise the rolling rule decides, and a tell is written to + * the row before the answer carries it. Answers whether the node is told + * now. */ onCheckIn(node: NodeState, now: Date): boolean { const { state } = upgradeStateOf(node, this.gatewayBuild, now); if (state === 'current' || state === 'ahead') { - // Off the old build: the tell, if any, is fulfilled, and so is the - // operator's pending re-tell — that hand was for the node that was - // behind, and a re-tell left standing would fire the moment this - // node read behind again, whatever the order said (found by - // review, 2026-09-15). if (node.upgradeToldAt !== null) { this.fleet.setUpgradeToldAt(node.id, null); } - this.retell.delete(node.id); return false; } - const tell = - (this.retell.has(node.id) && - (state === 'behind' || state === 'stuck' || state === 'upgrading')) || - rollingDecision(this.fleet.all(), this.gatewayBuild, node, now); - if (!tell) return false; + if (!rollingDecision(this.fleet.all(), this.gatewayBuild, node, now)) { + return false; + } this.fleet.setUpgradeToldAt(node.id, now); - this.retell.delete(node.id); return true; } /** - * The operator's re-tell (applyUpgrade {nodeId}): honored at the node's - * next check-in. Refused in words when it would do nothing — a node on - * the gateway's build or ahead of it, one that cannot upgrade itself, - * one whose build is unknown — and told to wait for an unreachable one; a node merely - * behind or upgrading is taken too (the operator's hand outranks the - * order). Answers the refusal, or null when the re-tell is pending. + * The operator's hand on a stuck node (applyUpgrade {nodeId}): its tell + * is forgotten, it reads behind, and the roll tells it at its turn. + * Refused in words everywhere else. Nothing to forget on a node that is + * current, ahead, behind (in line already), unavailable, unreachable or + * unknown. And not on one upgrading: its tell is what the one-at-a-time + * rule counts, and forgotten, the rule would see nobody upgrading and + * tell the next node into the same minute — the twenty minutes to stuck + * are the roll's promise, not a delay to be skipped. Answers the + * refusal, or null when the tell was forgotten. */ - requestRetell( + retell( node: NodeState, now: Date, ): { status: 400 | 409; message: string } | null { const { state, reason } = upgradeStateOf(node, this.gatewayBuild, now); switch (state) { + case 'stuck': + this.fleet.setUpgradeToldAt(node.id, null); + return null; + case 'upgrading': + return { + status: 409, + message: `node ${node.id} is upgrading (${reason}) — it reads stuck twenty minutes after its tell if it is still on the old build, and can be put back in line from there; wait`, + }; case 'current': return { status: 400, message: `node ${node.id} already runs the gateway's build (${node.build?.commit ?? 'unknown'}) — nothing to upgrade`, }; - case 'ahead': - case 'unavailable': - case 'unknown': + case 'behind': return { status: 400, - message: `node ${node.id} cannot be told to upgrade: ${reason ?? state}`, + message: `node ${node.id} is behind and in line — it is told at its next check-in once no other node is upgrading; nothing to do`, }; case 'unreachable': return { - status: 409, - message: `node ${node.id} is not checking in (${reason}) — it is told at its next check-in; retry once it is back, or remove it if it is gone for good`, + status: 400, + message: `node ${node.id} is not checking in (${reason}) — back and behind, it is told at its turn; gone for good, remove it`, }; default: - this.retell.add(node.id); - return null; + return { + status: 400, + message: `node ${node.id} cannot be told to upgrade: ${reason ?? state}`, + }; } } - /** - * The node is gone (removeNode): its pending re-tell goes with it. The - * hand was for that node; a machine re-imaged under the same id joins - * as a new node, and must wait its turn like one — not be told at its - * first check-in past the order (found by review, 2026-09-15). - */ - forget(id: string): void { - this.retell.delete(id); - } - /** Every node's standing right now (getUpgradeStatus.nodes), in node-id order. */ states(now: Date): NodeUpgradeView[] { return this.fleet diff --git a/packages/gateway/src/routes/nodes.ts b/packages/gateway/src/routes/nodes.ts index 00663a91..cb1f659c 100644 --- a/packages/gateway/src/routes/nodes.ts +++ b/packages/gateway/src/routes/nodes.ts @@ -28,8 +28,6 @@ export interface CheckInRoutesOptions { export interface NodeRoutesOptions { fleet: Fleet; cache: NameCache; - /** Forgets a removed node's pending re-tell (rolling.ts forget). */ - rolling: Rolling; } /** A refusal in the native dialect, rendered by the app's error handler as `{ message }` under its status. */ @@ -180,7 +178,7 @@ export const checkInRoutes: FastifyPluginAsyncZod< */ export const nodeRoutes: FastifyPluginAsyncZod = async ( app, - { fleet, cache, rolling }, + { fleet, cache }, ) => { app.post( '/listNodes', @@ -280,7 +278,6 @@ export const nodeRoutes: FastifyPluginAsyncZod = async ( } const removed = fleet.remove(request.body.id); const evicted = cache.evictNode(request.body.id); - rolling.forget(request.body.id); if (removed) { request.log.warn( { nodeId: request.body.id, evicted }, diff --git a/packages/gateway/src/routes/upgrade.test.ts b/packages/gateway/src/routes/upgrade.test.ts index a5fa8539..0f9a321f 100644 --- a/packages/gateway/src/routes/upgrade.test.ts +++ b/packages/gateway/src/routes/upgrade.test.ts @@ -99,7 +99,7 @@ describe('the fleet upgrade over the check-in', () => { void fleet; }); - it('a node still old twenty minutes after its tell is stuck and is not re-told; the operator re-tells it, and it hears at its next check-in', async () => { + it('a node still old twenty minutes after its tell is stuck and is not re-told; the operator puts it back in line, and it is told at its turn — not beside the node upgrading', async () => { const { app, fleet } = testGateway({}, { build: GATEWAY }); expect( (await checkIn(app, 'a', { build: OLD, selfUpgrade: CAN })).upgrade, @@ -119,33 +119,35 @@ describe('the fleet upgrade over the check-in', () => { expect( (await checkIn(app, 'b', { build: OLD, selfUpgrade: CAN })).upgrade, ).toBe(true); - // The operator's hand: a is told again at its next check-in, even - // while b upgrades. + // The operator's hand: a is back in line — behind, not told while b + // upgrades, told once b is back. const retold = await rpc(app, '/applyUpgrade', { nodeId: 'a' }); expect(retold.statusCode).toBe(200); expect(retold.json()).toEqual({ started: true }); + expect((await status(app)).nodes).toMatchObject([ + { id: 'a', state: 'behind', toldAt: null }, + { id: 'b', state: 'upgrading' }, + ]); expect( (await checkIn(app, 'a', { build: OLD, selfUpgrade: CAN })).upgrade, - ).toBe(true); - expect((await status(app)).nodes?.find((n) => n.id === 'a')?.state).toBe( - 'upgrading', - ); - // The hand does not outlive the node: c, behind and re-told, goes - // silent and is removed; a machine under the same id joins behind - // while a upgrades, and waits its turn. - expect( - (await checkIn(app, 'c', { build: OLD, selfUpgrade: CAN })).upgrade, ).toBeUndefined(); - expect((await rpc(app, '/applyUpgrade', { nodeId: 'c' })).statusCode).toBe( - 200, + // The hand on b, upgrading, is refused — its tell is the rule's count; + // on a, behind and in line, there is nothing to do. + const onUpgrading = await rpc(app, '/applyUpgrade', { nodeId: 'b' }); + expect(onUpgrading.statusCode).toBe(409); + expect(onUpgrading.json().message).toMatch(/is upgrading/); + expect((await rpc(app, '/applyUpgrade', { nodeId: 'a' })).statusCode).toBe( + 400, ); - const c = fleet.get('c'); - if (!c) throw new Error('node lost'); - c.lastCheckInAt = new Date(Date.now() - 40_000); - expect((await rpc(app, '/removeNode', { id: 'c' })).statusCode).toBe(200); expect( - (await checkIn(app, 'c', { build: OLD, selfUpgrade: CAN })).upgrade, + (await checkIn(app, 'b', { build: GATEWAY, selfUpgrade: CAN })).upgrade, ).toBeUndefined(); + expect( + (await checkIn(app, 'a', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBe(true); + expect((await status(app)).nodes?.find((n) => n.id === 'a')?.state).toBe( + 'upgrading', + ); }); it('nodes that cannot upgrade themselves, did not say, or carry no build are listed with the reason and never told', async () => { diff --git a/packages/gateway/src/routes/upgrade.ts b/packages/gateway/src/routes/upgrade.ts index 8a54f5a6..baf6d62c 100644 --- a/packages/gateway/src/routes/upgrade.ts +++ b/packages/gateway/src/routes/upgrade.ts @@ -25,8 +25,8 @@ export interface UpgradeRoutesOptions { * applyUpgrade without a node upgrades the gateway's machine — install.sh * in a systemd unit, the daemon's own mechanism, which restarts the * gateway and its node together, and from then on the check-ins roll the - * upgrade over the other nodes (rolling.ts); applyUpgrade with a node is - * the operator's re-tell of one node; getUpgradeStatus is the gateway + * upgrade over the other nodes (rolling.ts); applyUpgrade with a node puts + * one stuck node back in line; getUpgradeStatus is the gateway * machine's run plus every node's standing. Behind the admin gate: an * upgrade is the fleet's configuration in the largest sense, and a leaked * automation key must not be able to restart every machine. @@ -70,11 +70,11 @@ export const upgradeRoutes: FastifyPluginAsyncZod< `no node with id '${nodeId}' — listNodes shows which exist`, ); } - const refused = rolling.requestRetell(node, new Date()); + const refused = rolling.retell(node, new Date()); if (refused !== null) throw httpError(refused.status, refused.message); request.log.info( { nodeId, build: node.build?.commit ?? null }, - 'node told to upgrade again by the operator; it hears at its next check-in', + 'a stuck node was put back in line by the operator: its tell is forgotten, and it is told again at its turn', ); return { started: true as const }; }, diff --git a/packages/sdk/src/client.ts b/packages/sdk/src/client.ts index 0fcdf145..86a5b440 100644 --- a/packages/sdk/src/client.ts +++ b/packages/sdk/src/client.ts @@ -396,7 +396,7 @@ export class Dormice { * and watchers break, sandboxes and their disks are untouched. */ async applyUpgrade(options?: { - /** At the gateway: tell this one node to upgrade again at its next check-in (a node the fleet upgrade lists as stuck). Absent: upgrade the gateway's machine and roll the fleet. */ + /** At the gateway: put a stuck node back in line — its tell is forgotten and the roll tells it again at its turn (400 on any other state, 409 while it is upgrading). Absent: upgrade the gateway's machine and roll the fleet. */ nodeId?: string; }): Promise { const data = await this.rpc('applyUpgrade', { diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index 3f3b0008..2dd7096e 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -203,7 +203,7 @@ export class CheckIn { // failure: the check-in itself succeeded, and the gateway tells a // node once — a launch that fails here is the operator's to read // (the gateway shows the node as stuck twenty minutes on, and - // applyUpgrade {nodeId} at the gateway tells it again). + // applyUpgrade {nodeId} at the gateway puts it back in line). opts.log.info( `the gateway says this node's turn to upgrade has come — launching install.sh (systemd unit dormice-upgrade)`, ); @@ -212,7 +212,7 @@ export class CheckIn { } catch (error) { opts.log.warn( { error: describe(error) }, - 'the upgrade the gateway asked for could not be launched; the gateway lists this node as stuck once twenty minutes have passed, and applyUpgrade {nodeId} there tells it again', + 'the upgrade the gateway asked for could not be launched; the gateway lists this node as stuck once twenty minutes have passed, and applyUpgrade {nodeId} there puts it back in line', ); } } diff --git a/packages/shared/src/upgrade.ts b/packages/shared/src/upgrade.ts index 8486ce6f..07bd0070 100644 --- a/packages/shared/src/upgrade.ts +++ b/packages/shared/src/upgrade.ts @@ -102,18 +102,20 @@ export type CheckUpgradeResponse = z.infer; * At the gateway, without `nodeId`, this is the fleet upgrade: the * gateway's machine upgrades (its gateway and its node together), and the * nodes behind follow — each is told at its check-in, one at a time, once. - * With `nodeId` it is the operator telling one node again: a node told + * With `nodeId` it is the operator's hand on one stuck node: a node told * once that is still on the old build twenty minutes later is `stuck` * (getUpgradeStatus), never re-told on its own — a node whose build keeps * failing must not rebuild every twenty minutes on the sandboxes' CPU — - * and this is the hand that re-tells it. On a node `nodeId` is meaningless - * and refused. + * and this puts it back in line: its tell is forgotten, it reads behind, + * and the roll tells it at its turn, after the node upgrading now if + * there is one, never beside it. On a node `nodeId` is meaningless and + * refused. * * Refused (400) when one-click is unavailable — fake executor, no git * checkout, no systemd. Watch progress with getUpgradeStatus. */ export const applyUpgradeRequestSchema = z.object({ - /** At the gateway: tell this one node to upgrade at its next check-in, whatever the rolling order says. Absent: upgrade the gateway's machine, then roll the fleet. */ + /** At the gateway: put this stuck node back in line — its tell is forgotten, and the roll tells it again at its turn (400 on any other state, 409 while it is upgrading). Absent: upgrade the gateway's machine, then roll the fleet. */ nodeId: z.string().min(1).optional(), }); @@ -169,7 +171,8 @@ export type GetUpgradeStatusRequest = z.infer< * turn comes when no other node is upgrading * upgrading told within the last twenty minutes, not back yet * stuck told, still on the old build twenty minutes on — never - * re-told on its own; applyUpgrade {nodeId} is the hand + * re-told on its own; applyUpgrade {nodeId} puts it back + * in line * unavailable another build, but the node cannot upgrade itself (its * own reason: no checkout, no systemd, an older build that * does not say) — run install.sh on it by hand diff --git a/website/content/docs/troubleshooting.mdx b/website/content/docs/troubleshooting.mdx index 042de9f2..300b4bbc 100644 --- a/website/content/docs/troubleshooting.mdx +++ b/website/content/docs/troubleshooting.mdx @@ -155,7 +155,8 @@ own (a build that fails every time must not rebuild every twenty minutes on the sandboxes' CPU). On that node read `journalctl -u dormice-upgrade` and `/var/lib/dormice/upgrade/upgrade.log`, fix the cause — a mirror that hung, a full disk — and press **Try again** on the -version page, which tells the node once more at its next check-in. +version page, which puts the node back in line: it is told once more at +its next check-in, once no other node is upgrading. See [Upgrade a fleet](/docs/upgrading#upgrade-a-fleet). ## A paused container won't `docker rm` by hand diff --git a/website/content/docs/upgrading.mdx b/website/content/docs/upgrading.mdx index b867c2cb..3ce8bad2 100644 --- a/website/content/docs/upgrading.mdx +++ b/website/content/docs/upgrading.mdx @@ -49,10 +49,11 @@ alone — a node whose build keeps failing must not rebuild every twenty minutes on the CPU its sandboxes run on. Read `journalctl -u dormice-upgrade` and `/var/lib/dormice/upgrade/upgrade.log` on that node, fix the cause, and press **Try again** on the version page (or -`POST /applyUpgrade {"nodeId": "..."}`), which tells that node again at -its next check-in. A node that cannot upgrade itself — no git checkout, -no systemd — is listed with its reason; run the installer on it by -hand. +`POST /applyUpgrade {"nodeId": "..."}`), which puts that node back in +line: the gateway forgets it was told and tells it again at its next +check-in once no other node is upgrading — one node at a time holds for +the retry too. A node that cannot upgrade itself — no git checkout, no +systemd — is listed with its reason; run the installer on it by hand. ## What survives an upgrade From 69d96cbc1f3a62aadb9923d76ffe58be6381ba0d Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 00:44:25 +0800 Subject: [PATCH 71/89] applyUpgrade on a node refuses nodeId instead of upgrading whichever node it was sent to; Rolling.retell is unstick; the docs stop calling the console's overview dark and the list verbs unrouted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The node's applyUpgrade took the gateway's nodeId field and dropped it, then launched its own install.sh — shared upgrade.ts had said the field is refused on a node since the fourth cut, and nothing refused it. Now a 400 that names the gateway's verb; the fake executor's 400 still answers the bare call (app.test pins the order). The hand on a stuck node forgets its tell and never tells again since 9537557; the method was still called retell after the semantics it replaced. unstick, in the gateway and the console. console.mdx said the overview and the sandbox list stay dark until the gateway answers those verbs (it has since the third cut), that a node "has not reported since the gateway started" (the grace went with the fourth cut's rows), and that the workbench shows lifecycle events (the activity table went with the second); quickstart.mdx sent the list verbs to the daemon's port. The SDK's nodeId doc gains the node's answer. --- .../settings/components/VersionCard.tsx | 6 ++-- packages/gateway/src/rolling.test.ts | 18 +++++------ packages/gateway/src/rolling.ts | 2 +- packages/gateway/src/routes/upgrade.ts | 2 +- packages/sdk/src/client.ts | 2 +- packages/server/src/app.test.ts | 14 +++++++++ packages/server/src/routes/upgrade.ts | 12 +++++++ website/content/docs/console.mdx | 31 +++++++------------ website/content/docs/quickstart.mdx | 7 ++--- 9 files changed, 55 insertions(+), 39 deletions(-) diff --git a/packages/console/src/features/settings/components/VersionCard.tsx b/packages/console/src/features/settings/components/VersionCard.tsx index 57765502..2f6119aa 100644 --- a/packages/console/src/features/settings/components/VersionCard.tsx +++ b/packages/console/src/features/settings/components/VersionCard.tsx @@ -321,7 +321,7 @@ function stateClass(state: NodeUpgradeView['state']): string { */ function NodesTable({ nodes }: { nodes: NodeUpgradeView[] }) { const queryClient = useQueryClient(); - const retell = useMutation({ + const unstick = useMutation({ mutationFn: (id: string) => applyUpgrade(id), onSuccess: (_data, id) => { toast.success(m.settings_nodes_retry_done({ id })); @@ -387,8 +387,8 @@ function NodesTable({ nodes }: { nodes: NodeUpgradeView[] }) { diff --git a/packages/gateway/src/rolling.test.ts b/packages/gateway/src/rolling.test.ts index b6df028e..b14bb7d6 100644 --- a/packages/gateway/src/rolling.test.ts +++ b/packages/gateway/src/rolling.test.ts @@ -241,7 +241,7 @@ describe('Rolling', () => { expect(rolling.onCheckIn(b, later)).toBe(true); }); - it("retell forgets a stuck node's tell: it reads behind, waits for the node upgrading and is told at its turn; every other state is refused in words", () => { + it("unstick forgets a stuck node's tell: it reads behind, waits for the node upgrading and is told at its turn; every other state is refused in words", () => { const { db, fleet } = fleetOver(); const rolling = new Rolling(fleet, GATEWAY); const a = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); @@ -250,11 +250,11 @@ describe('Rolling', () => { expect(rolling.onCheckIn(b, NOW)).toBe(false); // b is behind and in line already; a is upgrading and its tell is the // one-at-a-time rule's count — neither is the hand's to touch. - expect(rolling.retell(b, NOW)).toMatchObject({ + expect(rolling.unstick(b, NOW)).toMatchObject({ status: 400, message: expect.stringMatching(/behind and in line/), }); - expect(rolling.retell(a, NOW)).toMatchObject({ + expect(rolling.unstick(a, NOW)).toMatchObject({ status: 409, message: expect.stringMatching( /is upgrading \(told 0s ago, still on old0001\)/, @@ -268,7 +268,7 @@ describe('Rolling', () => { expect(rolling.onCheckIn(a, late)).toBe(false); expect(rolling.onCheckIn(b, late)).toBe(true); // The hand: a's tell is forgotten on the row, and a reads behind. - expect(rolling.retell(a, late)).toBeNull(); + expect(rolling.unstick(a, late)).toBeNull(); expect(a.upgradeToldAt).toBeNull(); expect(new Fleet(db).get('a')?.upgradeToldAt).toBeNull(); expect(upgradeStateOf(a, GATEWAY, late).state).toBe('behind'); @@ -285,7 +285,7 @@ describe('Rolling', () => { { build: GATEWAY, selfUpgrade: CAN }, late, ); - expect(rolling.retell(current, late)).toMatchObject({ + expect(rolling.unstick(current, late)).toMatchObject({ status: 400, message: expect.stringMatching(/already runs the gateway's build/), }); @@ -295,17 +295,17 @@ describe('Rolling', () => { { build: OLD, selfUpgrade: CANNOT }, late, ); - expect(rolling.retell(cannot, late)).toMatchObject({ + expect(rolling.unstick(cannot, late)).toMatchObject({ status: 400, message: expect.stringMatching(/cannot be told to upgrade: systemd-run/), }); const gone = reporting(fleet, 'e', { build: OLD, selfUpgrade: CAN }, late); gone.lastCheckInAt = new Date(late.getTime() - 40_000); - expect(rolling.retell(gone, late)).toMatchObject({ + expect(rolling.unstick(gone, late)).toMatchObject({ status: 400, message: expect.stringMatching(/not checking in/), }); - expect(new Rolling(fleet, null).retell(a, late)).toMatchObject({ + expect(new Rolling(fleet, null).unstick(a, late)).toMatchObject({ status: 400, message: expect.stringMatching(/gateway carries no build identity/), }); @@ -326,7 +326,7 @@ describe('Rolling', () => { expect(rolling.states(later)).toMatchObject([ { id: 'a', state: 'ahead', toldAt: null }, ]); - expect(rolling.retell(a, later)).toMatchObject({ + expect(rolling.unstick(a, later)).toMatchObject({ status: 400, message: expect.stringMatching( /cannot be told to upgrade: runs new0002 .* newer than the gateway's new0001/, diff --git a/packages/gateway/src/rolling.ts b/packages/gateway/src/rolling.ts index 2fb8b484..da10846f 100644 --- a/packages/gateway/src/rolling.ts +++ b/packages/gateway/src/rolling.ts @@ -209,7 +209,7 @@ export class Rolling { * are the roll's promise, not a delay to be skipped. Answers the * refusal, or null when the tell was forgotten. */ - retell( + unstick( node: NodeState, now: Date, ): { status: 400 | 409; message: string } | null { diff --git a/packages/gateway/src/routes/upgrade.ts b/packages/gateway/src/routes/upgrade.ts index baf6d62c..ced11857 100644 --- a/packages/gateway/src/routes/upgrade.ts +++ b/packages/gateway/src/routes/upgrade.ts @@ -70,7 +70,7 @@ export const upgradeRoutes: FastifyPluginAsyncZod< `no node with id '${nodeId}' — listNodes shows which exist`, ); } - const refused = rolling.retell(node, new Date()); + const refused = rolling.unstick(node, new Date()); if (refused !== null) throw httpError(refused.status, refused.message); request.log.info( { nodeId, build: node.build?.commit ?? null }, diff --git a/packages/sdk/src/client.ts b/packages/sdk/src/client.ts index 86a5b440..32bc4ab6 100644 --- a/packages/sdk/src/client.ts +++ b/packages/sdk/src/client.ts @@ -396,7 +396,7 @@ export class Dormice { * and watchers break, sandboxes and their disks are untouched. */ async applyUpgrade(options?: { - /** At the gateway: put a stuck node back in line — its tell is forgotten and the roll tells it again at its turn (400 on any other state, 409 while it is upgrading). Absent: upgrade the gateway's machine and roll the fleet. */ + /** At the gateway: put a stuck node back in line — its tell is forgotten and the roll tells it again at its turn (400 on any other state, 409 while it is upgrading). Absent: upgrade the gateway's machine and roll the fleet. On a node the field is refused (400): a node upgrades only itself. */ nodeId?: string; }): Promise { const data = await this.rpc('applyUpgrade', { diff --git a/packages/server/src/app.test.ts b/packages/server/src/app.test.ts index 990bf3ea..1d9d207a 100644 --- a/packages/server/src/app.test.ts +++ b/packages/server/src/app.test.ts @@ -2097,3 +2097,17 @@ describe('the request log', () => { expect(lines.join('\n')).not.toContain('secret-sig'); }); }); + +describe('the upgrade verbs on a node', () => { + it("applyUpgrade refuses nodeId: the hand on a stuck node is the gateway's verb, and a node upgrades only itself", async () => { + const { app } = testApp(); + const hand = await rpc(app, '/applyUpgrade', { nodeId: 'node-b' }); + expect(hand.statusCode).toBe(400); + expect(hand.json().message).toMatch(/nodeId is the gateway's/); + // Without one, the refusal is the updater's own — the fake executor + // cannot one-click — so the nodeId verdict comes first, not instead. + const own = await rpc(app, '/applyUpgrade', {}); + expect(own.statusCode).toBe(400); + expect(own.json().message).toMatch(/one-click upgrade unavailable/); + }); +}); diff --git a/packages/server/src/routes/upgrade.ts b/packages/server/src/routes/upgrade.ts index 07bbd45b..fe0b5c67 100644 --- a/packages/server/src/routes/upgrade.ts +++ b/packages/server/src/routes/upgrade.ts @@ -7,6 +7,7 @@ import { getUpgradeStatusResponseSchema, } from '@dormice/shared'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; +import { httpError } from '../http-error'; import type { Updater } from '../updater'; export interface UpgradeRoutesOptions { @@ -45,6 +46,17 @@ export const upgradeRoutes: FastifyPluginAsyncZod< }, }, async (request) => { + // The verb's `nodeId` half is the gateway's (the hand that puts a + // stuck node back in line, gateway routes/upgrade.ts); on a node it + // names nothing. Refused, not dropped: taken silently, a hand meant + // for one node would upgrade whichever node it was sent to (found + // by review, 2026-09-16). + if (request.body.nodeId !== undefined) { + throw httpError( + 400, + "nodeId is the gateway's: applyUpgrade {nodeId} at the gateway puts a stuck node back in line; here, applyUpgrade {} upgrades this node itself", + ); + } await updater.apply(); request.log.info( { from: updater.current?.commit ?? null }, diff --git a/website/content/docs/console.mdx b/website/content/docs/console.mdx index 131c4722..a6ed572c 100644 --- a/website/content/docs/console.mdx +++ b/website/content/docs/console.mdx @@ -11,13 +11,6 @@ an explicit click.** (The console UI is currently Chinese-language.) -**Work in progress:** the console moved to the gateway with the fleet's -configuration. The pages that read the fleet-wide list and observation -verbs — the overview and the sandbox list — stay dark until the gateway -answers those verbs itself, the next step of the move; the settings, -domains, templates, API keys and version pages, and the per-sandbox -workbench, work. - ## Signing in The console has one account with a username and password. On first @@ -41,8 +34,8 @@ concurrency peak, the total by lifecycle state and the disk-overcommit figure (promised vs actually occupied — the same numbers as [`getFleetMetrics`](/docs/metrics)), the stacked concurrency curve over the chosen window, and a nodes card with each machine's CPU, memory and -data-disk levels. When a node has not reported since the gateway -started, the card says so — the figures above are then a lower bound. +data-disk levels. A node that is not checking in is marked as such — +the figures above are then a lower bound. ## Nodes @@ -94,8 +87,8 @@ middle, and a monitor rail on the right, with draggable dividers: without waking: the lifecycle card, CPU/memory/disk vitals with one-hour sparklines (full 1h/24h/7d trend charts from the daemon's [metrics history](/docs/metrics) open in a dialog), the E2B process - table (a stopped sandbox honestly shows none), this sandbox's recent - lifecycle events, and its identity details. + table (a stopped sandbox honestly shows none), and its identity + details. ## Templates @@ -109,7 +102,7 @@ with the daemon's own message naming the sandboxes. Mint revocable credentials without leaving the browser ([what they are](/docs/http-api#api-keys)). The create dialog takes a name and an optional expiry date, then shows the key **exactly once** — -the daemon stores only a hash — with a copy button, a "copy connection +the gateway stores only a hash — with a copy button, a "copy connection config" button (endpoint + credential, ready for CI secrets), and no second chance. The table keeps every key ever minted, revoked ones included, as your rotation history: name, masked prefix, creation time, @@ -118,16 +111,16 @@ revoking), and status. Rows can be edited in place (rename, change or clear the expiry), selected in bulk for revocation, and revoking takes effect on the key's next request. `DORMICE_API_TOKEN` itself appears as a pinned, read-only "default" row so all credentials are visible in one -table — but it lives in the server's environment, so retiring it means -editing `/etc/dormice/env`, and it (or a console session) is also the -only credential the key-management actions themselves accept: keys -cannot manage keys. +table — but it lives in the env files (`/etc/dormice/gateway.env` and +every node's `/etc/dormice/env`), so retiring it means editing those, +and it (or a console session) is also the only credential the +key-management actions themselves accept: keys cannot manage keys. ## Domains, settings, and version Three more pages round out the operator view: -- **Domains** — two sections. Console domains bind to the daemon's +- **Domains** — two sections. Console domains bind to the gateway's front door (the managed reverse proxy), with a copyable DNS record guide and a live per-domain probe that shows DNS and certificate progress until each one turns green — see @@ -171,8 +164,8 @@ A command palette (⌘K) jumps to any page or sandbox. Copy-paste snippets for the SDK, the official E2B packages, and curl, pre-filled with the endpoint the browser itself is using. The API token is never displayed — the console does not hold it and cannot read it -back; fetch it from the daemon host -(`grep ^DORMICE_API_TOKEN /etc/dormice/env`). +back; fetch it from the gateway machine +(`grep ^DORMICE_API_TOKEN /etc/dormice/gateway.env`). ## What you will not find diff --git a/website/content/docs/quickstart.mdx b/website/content/docs/quickstart.mdx index 7c77cff7..a9ec5e63 100644 --- a/website/content/docs/quickstart.mdx +++ b/website/content/docs/quickstart.mdx @@ -20,11 +20,8 @@ export DORMICE_ENDPOINT=http://127.0.0.1:3677 export DORMICE_API_TOKEN= # on the host: grep ^DORMICE_API_TOKEN /etc/dormice/env ``` -The gateway answers everything this page does. Two CLI forms read the -sandbox list — `dor sandbox ls`, and `dor sandbox meta ` without -labels — and the gateway does not route the list verbs yet: for those, -point `DORMICE_ENDPOINT` at the daemon (`http://127.0.0.1:3676`) until -it does. +The gateway answers everything this page does, the sandbox list +included — it merges the lists of every node. ## 2. Get the SDK From adda883497d3f31430e3965850625b8e0d829f13 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 01:56:30 +0800 Subject: [PATCH 72/89] pulseFileGrowth delivers nothing after stop(): a stat already in flight could pulse once more when the file had grown meanwhile --- packages/server/src/archive/archiver.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/server/src/archive/archiver.ts b/packages/server/src/archive/archiver.ts index 94f2909e..2b1ddaf7 100644 --- a/packages/server/src/archive/archiver.ts +++ b/packages/server/src/archive/archiver.ts @@ -59,7 +59,11 @@ function clampPercent(fraction: number): number { * evented, because the writer (tar) offers no progress hooks and a growing * output file IS its progress. Only actual growth pulses: a wedged writer * goes silent, and the heartbeat watchdog hears exactly that. A file not - * born yet is no growth either. + * born yet is no growth either. stop() ends the sampling for good: a stat + * already in flight when it is called delivers nothing — clearing the + * interval alone let that last sample pulse after stop() whenever the + * file had grown meanwhile (seen as a flaky test under a full parallel + * run, 2026-09-16). */ export function pulseFileGrowth( filePath: string, @@ -67,10 +71,11 @@ export function pulseFileGrowth( everyMs = 15_000, ): { stop(): void } { let lastSize = -1; + let stopped = false; const timer = setInterval(() => { void stat(filePath).then( ({ size }) => { - if (size > lastSize) { + if (!stopped && size > lastSize) { lastSize = size; onPulse(); } @@ -80,7 +85,12 @@ export function pulseFileGrowth( }, everyMs); // Sampling must never be what keeps the process alive. timer.unref(); - return { stop: () => clearInterval(timer) }; + return { + stop: () => { + stopped = true; + clearInterval(timer); + }, + }; } /** From ff0f372ed00715c47bcfaed689c3641dbadd95d8 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 01:56:30 +0800 Subject: [PATCH 73/89] A node joins through the gateway machine's Caddy on :80, not the gateway's own loopback port; install.sh re-points Caddy to the gateway only once the gateway answers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway listens on 127.0.0.1 only, like the daemon, so the `--gateway http://:3677` the docs gave a node's first install would have died at the fleet-join step. The docs, the installer's own usage text and its closing hint now name :80 — the gateway machine's Caddy — and the join failure says why. The installer used to re-point the gateway machine's Caddy from the daemon (3676) to the gateway (3677) in the ingress step, before the gateway was ever started: on a machine moving to the gateway, the public API face would proxy to a dark port from that reload until the gateway's first start — past the registry install, the base image push, the backups and the import, minutes on a production ledger. The re-point now happens in the services step, after the gateway answers /healthz and while the daemon is stopped anyway; a gateway that does not come up leaves the door on the daemon, which is started again. README: a fleet is a current feature, sharded rather than distributed; "one daemon per machine" is the invariant, not "one machine". --- README.md | 14 +++++---- deploy/install.sh | 44 ++++++++++++++++++--------- website/content/docs/installation.mdx | 15 ++++++--- 3 files changed, 49 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index f0179c49..90affc1a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![CI](https://github.com/BitMiracle-AI/Dormice/actions/workflows/ci.yml/badge.svg)](https://github.com/BitMiracle-AI/Dormice/actions/workflows/ci.yml) [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) -**The SQLite of agent sandboxes** — a self-hosted sandbox platform for AI agents. One machine, sandboxes that live forever, idle costs nothing. +**The SQLite of agent sandboxes** — a self-hosted sandbox platform for AI agents. One machine or a fleet of them, sandboxes that live forever, idle costs nothing. > **Status: early development.** The daemon, its lifecycle engine, the SDK, the CLI, the web console, the real Docker + gVisor executor, the S3 archiver, and the E2B-compatible API work end to end — the full create → freeze → stop → archive → restore cycle, command execution, file I/O, and the official `e2b` SDK against real infrastructure. Nothing here is ready for production yet. @@ -32,7 +32,7 @@ container — that decides whether the install actually succeeded; `dor doctor` can be re-run on its own at any time. A second machine joins the same fleet with one more command -(`--role node --gateway http://:3677`, the token in the +(`--role node --gateway http://:80`, the token in the environment) and needs no settings of its own; upgrades then run from the gateway, one node at a time. See the [installation](website/content/docs/installation.mdx) and @@ -222,7 +222,7 @@ verifies it, but these are the facts underneath: disable inter-container traffic (`"icc": false` in `daemon.json`). The daemon binds to 127.0.0.1 only, by design without a knob; exposing it is a reverse proxy's job. -- **One machine, one daemon.** The daemon enforces this with a lock next to +- **One daemon per machine.** The daemon enforces this with a lock next to its ledger and refuses to start when its ledger and the machine's reality cannot belong together. @@ -230,9 +230,11 @@ verifies it, but these are the facts underneath: Pick something else if: -- **You need a fleet.** One machine, one daemon, by design — that is where - the simplicity comes from. Multi-machine sharding is a future direction - (the schema already carries the fields), not a current feature. +- **You need sandboxes that move between machines, or a fleet across + regions.** A fleet is one gateway and the machines on its private + network — sharded, not distributed, by design: a sandbox lives on the + machine it was created on, and a node that is down takes its sandboxes + with it until it is back. - **You want a managed service.** No hosted anything, no SLA. That is E2B's product, and it is good at it. - **Your threat model demands hardware virtualization.** Sandboxes are diff --git a/deploy/install.sh b/deploy/install.sh index 2e8caa02..6d33def0 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -18,7 +18,11 @@ # would bind to curl, not to bash): # # export DORMICE_API_TOKEN= -# curl -fsSL .../install.sh | bash -s -- --role node --gateway http://10.0.0.5:3677 +# curl -fsSL .../install.sh | bash -s -- --role node --gateway http://10.0.0.5:80 +# +# :80 there is the gateway machine's Caddy (or the operator's own reverse +# proxy in front of the gateway): the gateway itself listens on loopback +# only, like the daemon, so its own port answers no other machine. # # The role is not a file: /etc/dormice/env names the gateway # (DORMICE_GATEWAY_ENDPOINT), and a remote one is what makes a machine a @@ -754,7 +758,7 @@ else note "installed caddy v$CADDY_VERSION to /usr/local/bin" fi INGRESS_FILE_READY='' -CADDY_REPOINTED='' +CADDY_REPOINT_PENDING='' if command -v caddy >/dev/null; then mkdir -p /etc/caddy if [ "$ROLE" = node ]; then @@ -824,11 +828,15 @@ EOF if grep -q "reverse_proxy 127.0.0.1:$PORT\b" "$ingress_target"; then # A file from before the gateway became the door (2026-09-14): the # catch-all still points at the daemon, where the console no longer - # lives. Re-pointed in place; the bound domains, if any, are - # rewritten the same way by the gateway at the next setIngress. - sed -i "s|reverse_proxy 127.0.0.1:$PORT\b|reverse_proxy 127.0.0.1:$GATEWAY_PORT|g" "$ingress_target" - note "re-pointed $ingress_target from the daemon ($PORT) to the gateway ($GATEWAY_PORT) — the console lives there now" - CADDY_REPOINTED=1 + # lives. Re-pointed in place — but in the services step below, once + # the gateway answers: done here, the machine's public API face + # would proxy to a port nobody listens on from this line until the + # gateway's first start, past the registry install, the base image + # push, the backups and the import — minutes on a production + # ledger (found by review, 2026-09-16). The bound domains, if any, + # are rewritten the same way by the gateway at the next setIngress. + note "$ingress_target still proxies to the daemon ($PORT) — re-pointed to the gateway ($GATEWAY_PORT) once it answers, below" + CADDY_REPOINT_PENDING=1 else note "[skip] $ingress_target is managed by Dormice — left to the gateway" fi @@ -868,10 +876,6 @@ EOF if [ "$(systemctl is-active caddy)" != active ]; then systemctl start caddy note 'started caddy' - elif [ -n "$CADDY_REPOINTED" ]; then - # shellcheck disable=SC2086 # the reload command is the operator's own words, split on purpose - (cd / && $ingress_reload >/dev/null 2>&1) || systemctl restart caddy - note 'reloaded caddy with the re-pointed config' else note '[skip] caddy is running' fi @@ -1237,7 +1241,7 @@ fi if [ "$ROLE" = node ]; then log "joining the fleet at $GATEWAY_URL" fleet_config=$(curl_auth_config | curl -fsS -K - -X POST -H 'content-type: application/json' -d '{}' "$GATEWAY_URL/getConfig" 2>/dev/null) \ - || die "the gateway at $GATEWAY_URL did not answer getConfig — is it running, is :$GATEWAY_PORT open to this machine, is DORMICE_API_TOKEN the gateway machine's token? (curl -fsS $GATEWAY_URL/healthz answers without a token)" + || die "the gateway at $GATEWAY_URL did not answer getConfig — is it running; is that address its machine's Caddy on :80 (the gateway itself listens on loopback only, so http://:$GATEWAY_PORT answers no other machine); is :80 there open to this machine; is DORMICE_API_TOKEN the gateway machine's token? (curl -fsS $GATEWAY_URL/healthz answers without a token)" read -r FLEET_REGISTRY FLEET_BASE_IMAGE </dev/null 2>&1) || systemctl restart caddy + note "re-pointed $ingress_target from the daemon ($PORT) to the gateway ($GATEWAY_PORT) and reloaded caddy — the console lives there now" + fi systemctl start dormice note 'enabled and (re)started both' else @@ -1473,8 +1488,9 @@ printf ' CLI: export DORMICE_ENDPOINT=http://127.0.0.1:%s DORMICE_API_ printf ' (the gateway is the door for every verb; a node answers only the sandbox and host verbs\n' printf ' for itself on 127.0.0.1:%s)\n' "$PORT" printf ' Both processes listen on 127.0.0.1 only, by design — exposing them is a reverse proxy'"'"'s job.\n' -printf ' add a node: on another machine of the same network, with :%s and :%s here open to it:\n' "$GATEWAY_PORT" "$REGISTRY_PORT" -printf ' DORMICE_API_TOKEN= bash install.sh --role node --gateway http://%s:%s\n' "${REGISTRY_ADDR%:*}" "$GATEWAY_PORT" +printf ' add a node: on another machine of the same network, with :80 (Caddy, the gateway'"'"'s door) and :%s here\n' "$REGISTRY_PORT" +printf ' open to it and its own :80 open to this machine:\n' +printf ' export DORMICE_API_TOKEN=; bash install.sh --role node --gateway http://%s:80\n' "${REGISTRY_ADDR%:*}" if [ "$(systemctl is-active caddy 2>/dev/null)" = active ]; then printf ' console: http:///console (Caddy on :80 -> the gateway; open your cloud firewall for\n' printf ' 80/443, then bind domains in the domains page for automatic HTTPS)\n' diff --git a/website/content/docs/installation.mdx b/website/content/docs/installation.mdx index 52ce35f4..8af0eb52 100644 --- a/website/content/docs/installation.mdx +++ b/website/content/docs/installation.mdx @@ -53,13 +53,14 @@ first check-in, and any image it lacks comes from the fleet's image registry, which the first machine runs beside the gateway. On the new machine (same private network as the gateway; its cloud -firewall must let it reach the gateway machine's ports `3677` and -`5000`, and let the gateway machine reach its port `80`): +firewall must let it reach the gateway machine's ports `80` — Caddy, +the gateway's door; the gateway itself listens on loopback only — and +`5000`, the registry, and let the gateway machine reach its port `80`): ```sh export DORMICE_API_TOKEN= curl -fsSL https://raw.githubusercontent.com/BitMiracle-AI/Dormice/main/deploy/install.sh \ - | bash -s -- --role node --gateway http://:3677 + | bash -s -- --role node --gateway http://:80 ``` The token travels in the environment, never as a flag (flags show in @@ -77,6 +78,12 @@ Re-running the installer on a node needs no flags: the env file says which gateway it belongs to. Upgrades come from the gateway — see [Upgrading](/docs/upgrading#upgrade-a-fleet). +One symptom worth knowing in advance: a node that appears on the nodes +page as reachable, yet whose sandboxes fail to create with a `502`, has +a firewall open one way only. Check-ins run from the node to the +gateway; everything else runs from the gateway to the node's port `80`, +and that is the rule the `502` is naming. + ## Get your API token The installer generates an [API @@ -175,7 +182,7 @@ affect using Dormice. reach it (`iptables -I DOCKER-USER -d 169.254.0.0/16 -j DROP`, persisted), and stops sandboxes from talking to each other (`"icc": false` in Docker's `daemon.json`). -- **One machine, one daemon.** The daemon enforces this with a lock and +- **One daemon per machine.** The daemon enforces this with a lock and refuses to start when its database and the machine's reality don't match. From 1b7d073a49216ca8040f60e46002682a36e881fd Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 02:56:09 +0800 Subject: [PATCH 74/89] install.sh builds native modules from the Node headers it shipped, and under --mirror cn takes better-sqlite3's prebuilt binary from npmmirror MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit better-sqlite3's install fetches a prebuilt binary from GitHub and, failing that, compiles — for which node-gyp fetches the running Node's headers from nodejs.org. On a fresh cn-beijing VM both timed out and the fresh install died in the build (2026-09-16). The headers are inside the Node tarball the script unpacks into /opt, so node-gyp is pointed there whenever that Node is the one running; with --mirror cn the prebuilt binary comes from npmmirror's copy of the GitHub releases, so the compile is not needed at all. A host whose own Node passed the version check has no headers under /opt and fetches as before. --- deploy/install.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/deploy/install.sh b/deploy/install.sh index 6d33def0..d31f87f9 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -662,7 +662,21 @@ fi # the same path for the fresh install, the upgrade, and the rollback. build_repo() { cd "$INSTALL_DIR" + # better-sqlite3 is a native module: its install fetches a prebuilt binary + # from GitHub and, failing that, compiles — for which node-gyp fetches this + # Node's headers from nodejs.org. Two hosts a mainland machine may not + # reach: both timed out on a fresh cn-beijing VM and the install died in + # the build (2026-09-16). The headers ship inside the Node tarball this + # script unpacks into /opt, so node-gyp is pointed there whenever that Node + # is the one running (a host whose own Node passed the version check has + # no headers here and fetches as before); under --mirror cn the prebuilt + # binary comes from npmmirror's copy of the GitHub releases. + local node_home="/opt/node-$NODE_VERSION-linux-x64" + if [ -f "$node_home/include/node/node.h" ] && [ "$(readlink -f "$(command -v node)")" = "$node_home/bin/node" ]; then + export npm_config_nodedir="$node_home" + fi if [ "$MIRROR" = cn ]; then + export npm_config_better_sqlite3_binary_host_mirror=https://npmmirror.com/mirrors/better-sqlite3 npm_config_registry=https://registry.npmmirror.com pnpm install --frozen-lockfile else pnpm install --frozen-lockfile From fd81d5c7db6d9202b88f3e5b2e0943ed7967f977 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 03:47:33 +0800 Subject: [PATCH 75/89] install.sh fetches Docker's install script with retries, and under --mirror cn installs the same packages from USTC's mirror when the script's mainland mirror fails Two things failed on fresh mainland VMs an hour apart (2026-09-16): get.docker.com reset the connection now and then (the third attempt got through), and Docker's own mainland mirror of its apt repository served a Packages index whose size did not match its Release file for over an hour ("Mirror sync in progress?"), so the install died in the Docker step both times; the script's other mainland mirror lagged the package list the script installs (no docker-model-plugin) and is no use as a second try. So the fetch retries, and under --mirror cn the script is a first attempt: when it fails, or cannot be fetched at all, the same five packages are installed from USTC's mirror of the repository by hand, with its signing key and an apt source written the way the script writes them. Off the mainland an unreachable get.docker.com stays a plain refusal that says what to do. --- deploy/install.sh | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/deploy/install.sh b/deploy/install.sh index d31f87f9..e84d0bc9 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -333,17 +333,44 @@ if [ ! -x /usr/local/bin/node ]; then fi # ---- Docker ---------------------------------------------------------------- +# The packages Docker's install script installs, from a mirror of its apt +# repository — the fallback for the script's own mainland mirrors (below). +install_docker_from_mirror() { + local base="$1" id codename + # shellcheck disable=SC1091 # the host's own os-release, not a script of ours + read -r id codename < <(. /etc/os-release && echo "$ID $VERSION_CODENAME") + note "Docker's install script could not install from its mainland mirror — installing the same packages from $base" + install -m 0755 -d /etc/apt/keyrings + curl -fsSL --retry 3 --retry-all-errors -o /etc/apt/keyrings/docker.asc "$base/linux/$id/gpg" + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] $base/linux/$id $codename stable" >/etc/apt/sources.list.d/docker.list + apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +} + log 'Docker' if docker version --format '{{.Server.Version}}' >/dev/null 2>&1; then note "[skip] dockerd $(docker version --format '{{.Server.Version}}') is running" else - curl -fsSL -o /tmp/get-docker.sh https://get.docker.com - if [ "$MIRROR" = cn ]; then - sh /tmp/get-docker.sh --mirror Aliyun + # Two things fail on a mainland VM, found on two fresh ones an hour apart + # (2026-09-16): get.docker.com resets the connection now and then (the third + # attempt got through), and Docker's own mainland mirror of its apt + # repository is not always in sync ("File has unexpected size … Mirror sync + # in progress?" for over an hour; the script's other mainland mirror lagged + # the package list the script installs). So under --mirror cn the script is + # a first attempt, and the same packages come from USTC's mirror of the + # repository when it fails — or when the script cannot be fetched at all. + if curl -fsSL --retry 3 --retry-all-errors -o /tmp/get-docker.sh https://get.docker.com; then + if [ "$MIRROR" = cn ]; then + sh /tmp/get-docker.sh --mirror Aliyun || install_docker_from_mirror https://mirrors.ustc.edu.cn/docker-ce + else + sh /tmp/get-docker.sh + fi + rm -f /tmp/get-docker.sh + elif [ "$MIRROR" = cn ]; then + install_docker_from_mirror https://mirrors.ustc.edu.cn/docker-ce else - sh /tmp/get-docker.sh + die 'could not fetch https://get.docker.com — install Docker Engine by hand (https://docs.docker.com/engine/install/), then re-run' fi - rm /tmp/get-docker.sh systemctl enable --now docker note "installed dockerd $(docker version --format '{{.Server.Version}}')" fi From 3256445ed5d63e4b2546691aab99484edbeb86ac Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 12:58:51 +0800 Subject: [PATCH 76/89] install.sh installs Docker Engine from Docker's apt repository with the signing key's fingerprint pinned, not through the convenience script and its mirror fallbacks; node-gyp is pointed at the running Node's own headers; the node-role messages name the gateway machine's :80 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Docker step had grown three paths in a day (fetch get.docker.com with retries, run it against Docker's mainland mirror, fall back to USTC's mirror of the repository when that failed) around one unpinned remote script run as root — the one download in the installer without a checksum. Docker's own documentation gives production hosts the apt repository recipe; that is the one path now, with the repository's base URL the only thing --mirror cn changes, and the signing key's fingerprint compared with the one Docker publishes before apt is told to trust it (a mirror serves the key and the packages alike). Proven in an ubuntu:24.04 container against download.docker.com and against USTC's mirror. npm_config_nodedir was tied to the path this script unpacks its pinned Node into; the rule is simpler — a running Node whose prefix carries include/node has its own headers, and node-gyp is pointed there. Two --role node refusals still gave http://…:3677 as the example, the port ff0f372 established answers no other machine; the gateway unit's header comment said the same. --- deploy/dormice-gateway.service | 4 +- deploy/install.sh | 104 ++++++++++++++++----------------- 2 files changed, 55 insertions(+), 53 deletions(-) diff --git a/deploy/dormice-gateway.service b/deploy/dormice-gateway.service index a1dccdd8..8f7af02c 100644 --- a/deploy/dormice-gateway.service +++ b/deploy/dormice-gateway.service @@ -7,7 +7,9 @@ # as part of the value). install.sh restarts this unit before the daemon's, # so both run one commit and the daemon's first check-in lands on the new # gateway. A machine of its own joins as a node with `install.sh --role -# node --gateway :3677` and runs no gateway unit. +# node --gateway http://:80` — this machine's Caddy, the +# gateway's door; the gateway itself listens on loopback only — and runs +# no gateway unit. [Unit] Description=Dormice gateway (fleet front door) Wants=network-online.target diff --git a/deploy/install.sh b/deploy/install.sh index e84d0bc9..708ff89c 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -92,6 +92,13 @@ REGISTRY_VERSION=3.1.1 REGISTRY_SHA256=6f330a3ba9ea1d23a6ee189f449d792595240585bb2f159123d76ac594f70dd8 REGISTRY_PORT=5000 +# Docker Engine comes from Docker's apt repository (or a mirror of it), and +# apt trusts what that repository's signing key signed — a key fetched from +# the same place as the packages, so it is the key that must be pinned: +# the fingerprint Docker publishes in its install documentation (checked +# 2026-09-16 against download.docker.com and USTC's mirror). +DOCKER_GPG_FINGERPRINT=9DC858229FC7DD38854AE2D88D81803C0EBFCD88 + REPO_URL=https://github.com/BitMiracle-AI/Dormice.git INSTALL_DIR=/opt/dormice ENV_FILE=/etc/dormice/env @@ -174,10 +181,10 @@ elif [ "$ROLE_FLAG" = node ]; then if [ -f "$ENV_FILE" ]; then die "$ENV_FILE exists and names a gateway on this machine (or none) — this is the gateway's machine; --role node is for a machine that has never been installed. To turn it into a node, stop and disable dormice-gateway, move the env file aside, and re-run" fi - [ -n "$GATEWAY_FLAG" ] || die "--role node needs --gateway http://:$GATEWAY_PORT — the gateway this node joins" + [ -n "$GATEWAY_FLAG" ] || die "--role node needs --gateway http://:80 — the gateway this node joins, through its machine's Caddy on :80 (the gateway itself listens on loopback only)" case "$GATEWAY_FLAG" in http://*|https://*) ;; - *) die "--gateway must be a full URL like http://10.0.0.5:$GATEWAY_PORT, got \"$GATEWAY_FLAG\"" ;; + *) die "--gateway must be a full URL like http://10.0.0.5:80, got \"$GATEWAY_FLAG\"" ;; esac is_loopback_url "$GATEWAY_FLAG" && die "--gateway names this machine ($GATEWAY_FLAG) — a node machine joins a gateway on another machine; the default install (no --role) is the gateway's machine" [ -n "${DORMICE_API_TOKEN:-}" ] || die "--role node needs the fleet token in the environment: DORMICE_API_TOKEN= bash -s -- --role node --gateway $GATEWAY_FLAG (a flag would show in ps)" @@ -292,16 +299,17 @@ grep -qw memory /sys/fs/cgroup/cgroup.controllers 2>/dev/null \ note "Linux x86_64, root, cgroup v2 — ok" # ---- base packages --------------------------------------------------------- -log 'base packages (git, curl, openssl, zstd)' +log 'base packages (git, curl, openssl, zstd, gpg)' missing='' # zstd: the archiver's tar -I zstd runs on the host at every archive/restore. -for tool in git curl openssl zstd; do +# gpg: checks the fingerprint of Docker's signing key below (package gnupg). +for tool in git curl openssl zstd gpg; do command -v "$tool" >/dev/null || missing="$missing $tool" done if [ -n "$missing" ]; then apt-get update -q # shellcheck disable=SC2086 # word splitting is the point - apt-get install -qy ca-certificates $missing + apt-get install -qy ca-certificates ${missing/ gpg/ gnupg} note "installed:$missing" else note '[skip] all present' @@ -333,46 +341,37 @@ if [ ! -x /usr/local/bin/node ]; then fi # ---- Docker ---------------------------------------------------------------- -# The packages Docker's install script installs, from a mirror of its apt -# repository — the fallback for the script's own mainland mirrors (below). -install_docker_from_mirror() { - local base="$1" id codename - # shellcheck disable=SC1091 # the host's own os-release, not a script of ours - read -r id codename < <(. /etc/os-release && echo "$ID $VERSION_CODENAME") - note "Docker's install script could not install from its mainland mirror — installing the same packages from $base" - install -m 0755 -d /etc/apt/keyrings - curl -fsSL --retry 3 --retry-all-errors -o /etc/apt/keyrings/docker.asc "$base/linux/$id/gpg" - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] $base/linux/$id $codename stable" >/etc/apt/sources.list.d/docker.list - apt-get update -qq - DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -} - +# Docker Engine from Docker's apt repository — the recipe its documentation +# gives for production hosts (a keyring, a source line, the packages), not +# its convenience script: that is unpinned code from the network run as +# root, where every other binary this installer fetches is checksummed, and +# its mainland fallbacks failed in turn on two fresh VMs an hour apart +# (2026-09-16: get.docker.com resetting the connection, the aliyun mirror +# of the repository out of sync for over an hour, the Azure mirror lagging +# the package list). One path: the repository from Docker, under --mirror +# cn from USTC's mirror of it, and the signing key's fingerprint checked +# against the one Docker publishes before apt is told to trust it. log 'Docker' if docker version --format '{{.Server.Version}}' >/dev/null 2>&1; then note "[skip] dockerd $(docker version --format '{{.Server.Version}}') is running" else - # Two things fail on a mainland VM, found on two fresh ones an hour apart - # (2026-09-16): get.docker.com resets the connection now and then (the third - # attempt got through), and Docker's own mainland mirror of its apt - # repository is not always in sync ("File has unexpected size … Mirror sync - # in progress?" for over an hour; the script's other mainland mirror lagged - # the package list the script installs). So under --mirror cn the script is - # a first attempt, and the same packages come from USTC's mirror of the - # repository when it fails — or when the script cannot be fetched at all. - if curl -fsSL --retry 3 --retry-all-errors -o /tmp/get-docker.sh https://get.docker.com; then - if [ "$MIRROR" = cn ]; then - sh /tmp/get-docker.sh --mirror Aliyun || install_docker_from_mirror https://mirrors.ustc.edu.cn/docker-ce - else - sh /tmp/get-docker.sh - fi - rm -f /tmp/get-docker.sh - elif [ "$MIRROR" = cn ]; then - install_docker_from_mirror https://mirrors.ustc.edu.cn/docker-ce - else - die 'could not fetch https://get.docker.com — install Docker Engine by hand (https://docs.docker.com/engine/install/), then re-run' - fi + docker_repo=https://download.docker.com + [ "$MIRROR" = cn ] && docker_repo=https://mirrors.ustc.edu.cn/docker-ce + # shellcheck disable=SC1091 # the host's own os-release, not a script of ours + read -r os_id os_codename < <(. /etc/os-release && echo "$ID $VERSION_CODENAME") + [ -n "${os_codename:-}" ] || die "/etc/os-release names no VERSION_CODENAME — Docker's repository is laid out by release codename; install Docker Engine by hand (https://docs.docker.com/engine/install/), then re-run" + install -m 0755 -d /etc/apt/keyrings + curl -fsSL --retry 3 --retry-all-errors -o /etc/apt/keyrings/docker.asc "$docker_repo/linux/$os_id/gpg" + docker_fpr=$(gpg --show-keys --with-fingerprint --with-colons /etc/apt/keyrings/docker.asc 2>/dev/null | awk -F: '/^fpr/ { print $10; exit }') + [ "$docker_fpr" = "$DOCKER_GPG_FINGERPRINT" ] \ + || die "the signing key at $docker_repo/linux/$os_id/gpg has fingerprint ${docker_fpr:-none}, not Docker's $DOCKER_GPG_FINGERPRINT — that is not Docker's repository; try without --mirror, or install Docker Engine by hand (https://docs.docker.com/engine/install/) and re-run" + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] $docker_repo/linux/$os_id $os_codename stable" >/etc/apt/sources.list.d/docker.list + apt-get update -qq + # The engine, its CLI, containerd, and buildx — the builder `docker build` + # runs the base image through. + apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin systemctl enable --now docker - note "installed dockerd $(docker version --format '{{.Server.Version}}')" + note "installed dockerd $(docker version --format '{{.Server.Version}}') from $docker_repo" fi # ---- daemon.json: icc off + log rotation ----------------------------------- @@ -689,17 +688,18 @@ fi # the same path for the fresh install, the upgrade, and the rollback. build_repo() { cd "$INSTALL_DIR" - # better-sqlite3 is a native module: its install fetches a prebuilt binary - # from GitHub and, failing that, compiles — for which node-gyp fetches this - # Node's headers from nodejs.org. Two hosts a mainland machine may not - # reach: both timed out on a fresh cn-beijing VM and the install died in - # the build (2026-09-16). The headers ship inside the Node tarball this - # script unpacks into /opt, so node-gyp is pointed there whenever that Node - # is the one running (a host whose own Node passed the version check has - # no headers here and fetches as before); under --mirror cn the prebuilt - # binary comes from npmmirror's copy of the GitHub releases. - local node_home="/opt/node-$NODE_VERSION-linux-x64" - if [ -f "$node_home/include/node/node.h" ] && [ "$(readlink -f "$(command -v node)")" = "$node_home/bin/node" ]; then + # better-sqlite3 is a native module: its install takes a prebuilt binary + # from GitHub — under --mirror cn from npmmirror's copy of those releases, + # GitHub being the first host a mainland machine cannot reach (a fresh VM + # timed out on it and then on nodejs.org, 2026-09-16) — and compiles + # otherwise, for which node-gyp downloads this Node's headers from + # nodejs.org: needlessly when the interpreter came with them. A Node + # unpacked from its tarball (the one this script puts in /opt) carries + # them under include/node, and node-gyp is pointed at the running Node's + # own copy whenever there is one. + local node_home + node_home=$(dirname "$(dirname "$(readlink -f "$(command -v node)")")") + if [ -f "$node_home/include/node/node.h" ]; then export npm_config_nodedir="$node_home" fi if [ "$MIRROR" = cn ]; then @@ -1452,7 +1452,7 @@ if [ "$ROLE" = gateway ]; then # The daemon must not stay down for the gateway's failure: it serves # its sandboxes without one, and its check-in keeps trying. systemctl start dormice - die "the gateway did not answer /healthz on 127.0.0.1:$GATEWAY_PORT — check: journalctl -u dormice-gateway -n 50 (the daemon was started again)" + die "the gateway did not answer /healthz on 127.0.0.1:$GATEWAY_PORT — check: journalctl -u dormice-gateway -n 50 (the daemon was started again; on this machine's first run with a gateway it holds no configuration copy yet and waits for the gateway before it listens)" fi note "gateway is answering on 127.0.0.1:$GATEWAY_PORT" if [ -n "$CADDY_REPOINT_PENDING" ]; then From 6cbe18dc47a56c58271f4ae30bcb2d311f7d266d Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 13:21:11 +0800 Subject: [PATCH 77/89] The one-click upgrade runs the installer of the build it installs, fetched from the branch the checkout tracks, not the tree's copy that belongs to the build being replaced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An upgrade is "bring this machine to that commit", and only that commit's install.sh knows the host-side steps its build needs — a sysctl floor, a runtime flag, a new unit. apply() copied the tree's script, the one of the build being replaced, so twice in production the code arrived and the host-side step did not, until the new installer was re-run by hand (2026-09-01 --allow-suid, 2026-09-09 the inotify floor); the cut-over rules grew a standing "run install.sh by hand for a version that changed it". Now apply() fetches the tracked branch's head and writes its deploy/install.sh into the status directory before launching it — still outside the tree, whose file the script's own git pull replaces mid-run. The ref is the branch the checkout tracks (branch..remote and .merge, the way git pull reads it), for check() as well: it compared against origin main while install.sh pulled the checkout's upstream, and on a series-branch checkout the two disagreed — the version page read the build as ahead of main while the pull brought the branch. A detached HEAD or an untracked branch is an honest checkError, and one-click reports itself unavailable for that reason before a node could be told and fail. The mirror is judged from the remote's configured URL, before git's url.insteadOf rewrite, which lets the suite point the fixture's mirror URL at a local path and keep the fetch off the network. --- packages/server/src/updater.test.ts | 121 ++++++++++++++++++++++++---- packages/server/src/updater.ts | 108 ++++++++++++++++++++----- 2 files changed, 192 insertions(+), 37 deletions(-) diff --git a/packages/server/src/updater.test.ts b/packages/server/src/updater.test.ts index 4a5763fb..b747dad0 100644 --- a/packages/server/src/updater.test.ts +++ b/packages/server/src/updater.test.ts @@ -1,4 +1,10 @@ -import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { @@ -73,8 +79,8 @@ beforeAll(() => { origin = path.join(root, 'origin'); clone = path.join(root, 'clone'); execaSync('git', ['init', '-q', '-b', 'main', origin]); - // The tree carries a stand-in installer: apply() copies and launches - // deploy/install.sh, and availability checks it exists. + // The tree carries a stand-in installer: apply() launches the + // deploy/install.sh of the build it installs, fetched from the origin. mkdirSync(path.join(origin, 'deploy')); writeFileSync(path.join(origin, 'deploy', 'install.sh'), '#!/bin/bash\n'); execaSync('git', ['add', '-A'], { cwd: origin }); @@ -85,7 +91,7 @@ beforeAll(() => { }); describe('Updater.check', () => { - it('reports up to date when the build matches origin main', async () => { + it("reports up to date when the build matches the tracked branch's head", async () => { const updater = updaterFor(); const answer = await updater.check(); const parsed = checkUpgradeResponseSchema.parse(answer); @@ -148,7 +154,38 @@ describe('Updater.check', () => { } }); - it('is honest without a checkout, without a build identity, and on a dead remote', async () => { + it('follows the branch the checkout tracks, not main', async () => { + // A checkout on a series branch (the test machine's shape between + // cuts): what check() compares against, what install.sh pulls and + // whose install.sh apply() runs must be the same ref, and that ref is + // the branch's upstream — main would read this build as ahead. + execaSync('git', ['checkout', '-q', '-b', 'series'], { cwd: origin }); + const onSeries = commit(origin, 'on the series branch'); + execaSync('git', ['checkout', '-q', 'main'], { cwd: origin }); + const series = mkdtempSync(path.join(tmpdir(), 'dormice-series-')); + execaSync('git', ['clone', '-q', '-b', 'series', origin, series]); + const updater = updaterFor({ repoDir: series, build: buildAt(series) }); + const upToDate = await updater.check(); + expect(upToDate.checkError).toBeNull(); + expect(upToDate.check).toMatchObject({ + behindBy: 0, + aheadBy: 0, + latest: onSeries, + }); + + execaSync('git', ['checkout', '-q', 'series'], { cwd: origin }); + const later = commit(origin, 'later on the series branch'); + execaSync('git', ['checkout', '-q', 'main'], { cwd: origin }); + const behind = await updater.check(true); + expect(behind.check).toMatchObject({ + behindBy: 1, + aheadBy: 0, + upgradable: true, + latest: later, + }); + }); + + it('is honest without a checkout, without a build identity, on an untracked branch and on a dead remote', async () => { const noRepo = updaterFor({ repoDir: null }); expect((await noRepo.check()).checkError).toMatch(/git checkout/); @@ -165,10 +202,39 @@ describe('Updater.check', () => { ['remote', 'add', 'origin', path.join(broken, 'does-not-exist')], { cwd: broken }, ); + // A remote, but a branch that tracks nothing: there is no ref to + // compare against or to pull along, and one-click says so too. + const untracked = updaterFor({ repoDir: broken, build: buildAt(broken) }); + expect((await untracked.check()).checkError).toMatch(/tracks no upstream/); + expect(await untracked.availability()).toMatch(/tracks no upstream/); + + execaSync('git', ['config', 'branch.main.remote', 'origin'], { + cwd: broken, + }); + execaSync('git', ['config', 'branch.main.merge', 'refs/heads/main'], { + cwd: broken, + }); const deadRemote = updaterFor({ repoDir: broken, build: buildAt(broken) }); const dead = await deadRemote.check(); expect(dead.check).toBeNull(); expect(dead.checkError).toMatch(/fetch failed/); + // One-click is available on paper (the branch tracks a remote), and + // the launch fails honestly when the installer cannot be fetched. + expect(await deadRemote.availability()).toBeNull(); + await expect(deadRemote.apply()).rejects.toMatchObject({ + statusCode: 500, + message: expect.stringContaining('could not fetch the installer'), + }); + + const detached = mkdtempSync(path.join(tmpdir(), 'dormice-detached-')); + execaSync('git', ['clone', '-q', origin, detached]); + execaSync('git', ['checkout', '-q', '--detach'], { cwd: detached }); + const offBranch = updaterFor({ + repoDir: detached, + build: buildAt(detached), + }); + expect(await offBranch.availability()).toMatch(/detached HEAD/); + await expect(offBranch.apply()).rejects.toMatchObject({ statusCode: 400 }); }); }); @@ -207,8 +273,8 @@ describe('Updater.apply and status', () => { expect(launch?.args).toContain('dormice-upgrade'); expect(launch?.args).toContain('--collect'); const command = launch?.args.at(-1) ?? ''; - // The copy, not the tree's file (git pull would replace it mid-read), - // reporting into the status dir, output tee'd next to it. + // A file in the status dir, not the tree's (git pull would replace it + // mid-read), reporting into the status dir, output tee'd next to it. expect(command).toContain(`${statusDir}/install.sh`); expect(command).toContain('--status-dir'); expect(command).toContain('upgrade.log'); @@ -217,19 +283,40 @@ describe('Updater.apply and status', () => { expect(existsSync(path.join(statusDir, 'install.sh'))).toBe(true); }); + it("runs the installer of the build being installed, not the tree's copy", async () => { + // The origin moves on with a changed installer: the upgrade must run + // that one — it alone knows the host-side steps its build needs — and + // the tree, still at the old build, is not where it comes from. + const newInstaller = '#!/bin/bash\necho the new installer\n'; + writeFileSync(path.join(origin, 'deploy', 'install.sh'), newInstaller); + execaSync('git', ['add', '-A'], { cwd: origin }); + commit(origin, 'a changed installer'); + const statusDir = mkdtempSync(path.join(tmpdir(), 'dormice-status-')); + const updater = updaterFor({ statusDir }); + await updater.apply(); + expect(readFileSync(path.join(statusDir, 'install.sh'), 'utf8')).toBe( + newInstaller, + ); + expect(readFileSync(path.join(clone, 'deploy', 'install.sh'), 'utf8')).toBe( + '#!/bin/bash\n', + ); + }); + it('passes --mirror cn when the origin was cloned through the mirror', async () => { const mirrored = mkdtempSync(path.join(tmpdir(), 'dormice-mirrored-')); execaSync('git', ['clone', '-q', origin, mirrored]); - execaSync( - 'git', - [ - 'remote', - 'set-url', - 'origin', - 'https://ghfast.top/https://github.com/BitMiracle-AI/Dormice.git', - ], - { cwd: mirrored }, - ); + // The remote's URL as an install with --mirror cn writes it; git is + // told to reach the fixture origin in its place (url.insteadOf), so + // the fetch apply() does stays off the network. The mirror is judged + // from the URL as configured, not as rewritten. + const mirrorUrl = + 'https://ghfast.top/https://github.com/BitMiracle-AI/Dormice.git'; + execaSync('git', ['remote', 'set-url', 'origin', mirrorUrl], { + cwd: mirrored, + }); + execaSync('git', ['config', `url.${origin}.insteadOf`, mirrorUrl], { + cwd: mirrored, + }); const calls: string[] = []; const updater = updaterFor({ repoDir: mirrored, diff --git a/packages/server/src/updater.ts b/packages/server/src/updater.ts index 2faa9db8..f669fb68 100644 --- a/packages/server/src/updater.ts +++ b/packages/server/src/updater.ts @@ -1,5 +1,4 @@ -import { existsSync } from 'node:fs'; -import { copyFile, mkdir, open, readFile, stat } from 'node:fs/promises'; +import { mkdir, open, readFile, stat, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { type CheckUpgradeResponse, @@ -18,8 +17,9 @@ import type { BuildInfo } from './version'; * process launches it the same way). Versions are git commits * (trunk-based, no release tags yet), and the question "is a newer * Dormice available?" is answered by comparing the commit baked into this - * build against the origin's main — fetched through the checkout's own - * `origin` remote, so an install done with `--mirror cn` (whose clone URL + * build against the head of the branch the checkout tracks — origin/main + * for every install the installer made — fetched through the checkout's + * own remote, so an install done with `--mirror cn` (whose clone URL * carries the mirror prefix) checks through the same mirror for free. * * `git fetch` updates .git only and never touches the working tree or the @@ -31,9 +31,17 @@ import type { BuildInfo } from './version'; * upgrade's last step restarts it, killing its own children), so apply() * hands install.sh to a systemd transient unit and steps aside — the unit * outlives the restart, tees its output where status() can read it, and - * its name is the mutex against a double-click. The upgrade command line - * is composed entirely from daemon-side paths: nothing from any request - * ever reaches it. + * its name is the mutex against a double-click. The install.sh it hands + * over is the one of the build being installed, read from the fetched + * head (`git show FETCH_HEAD:deploy/install.sh`) into the status + * directory — not the tree's copy, which is the build being replaced: an + * upgrade is "bring this machine to that commit", and only that commit's + * installer knows the host-side steps it needs (a sysctl floor, a runtime + * flag, a new unit). Run with the old script, twice in production the code + * arrived and the host-side step did not, until someone re-ran the new + * installer by hand (2026-09-01 --allow-suid, 2026-09-09 the inotify + * floor). The upgrade command line is composed entirely from daemon-side + * paths: nothing from any request ever reaches it. */ const CHECK_CACHE_MS = 3600_000; @@ -147,10 +155,45 @@ export class Updater { } } + /** + * The line this checkout follows: the remote and branch its HEAD tracks + * — the one ref for the three things that must agree: what check() + * compares against, what install.sh's `git pull --ff-only` brings, and + * whose install.sh apply() runs. Read the way git reads it for pull + * (branch..remote and .merge), so a checkout on a series branch + * follows that branch and a clone of main follows main. Throws, in + * words, for a detached HEAD or a branch that tracks nothing. + */ + private async upstream(): Promise<{ remote: string; branch: string }> { + let head: string; + try { + head = await this.git(['symbolic-ref', '--short', '--quiet', 'HEAD']); + } catch { + throw new Error( + 'the checkout is on a detached HEAD, not a branch — install.sh pulls --ff-only along a branch that tracks its remote', + ); + } + let remote: string; + let merge: string; + try { + remote = await this.git(['config', '--get', `branch.${head}.remote`]); + merge = await this.git(['config', '--get', `branch.${head}.merge`]); + } catch { + throw new Error( + `branch ${head} tracks no upstream — install.sh pulls --ff-only from the branch it tracks (git branch --set-upstream-to=origin/main, on an install of main)`, + ); + } + return { remote, branch: merge.replace(/^refs\/heads\//, '') }; + } + + /** The head of the tracked branch, into FETCH_HEAD — written by every fetch regardless of the clone's refspec configuration. */ + private async fetchUpstream(): Promise { + const { remote, branch } = await this.upstream(); + await this.git(['fetch', '--quiet', remote, branch], FETCH_TIMEOUT_MS); + } + private async compare(currentCommit: string): Promise { - // FETCH_HEAD instead of origin/main: it is written by every fetch - // regardless of the clone's refspec configuration. - await this.git(['fetch', '--quiet', 'origin', 'main'], FETCH_TIMEOUT_MS); + await this.fetchUpstream(); const behindBy = Number( await this.git(['rev-list', '--count', `${currentCommit}..FETCH_HEAD`]), ); @@ -184,9 +227,14 @@ export class Updater { /** * Launch the one-click upgrade: install.sh in a systemd transient unit. - * The script is copied out of the tree first — its own first step is - * `git pull`, which must not replace the file bash is reading. The - * mirror choice is derived from the origin URL (an install done with + * The script is the one of the build being installed — the tracked + * branch's head, fetched now, `git show`n into the status directory + * (the module comment has why); outside the tree, because the script's + * own first step is `git pull`, which must not replace the file bash is + * reading. A commit that lands between this fetch and that pull would + * put the tree one commit past the script — seconds of drift at most, + * and the node's next check-in reports the tree's build. The mirror + * choice is derived from the remote's URL (an install done with * --mirror cn cloned through the mirror prefix), so no separate knob. */ async apply(): Promise { @@ -198,9 +246,19 @@ export class Updater { if (this.repoDir === null) throw new Error('unreachable'); await mkdir(this.statusDir, { recursive: true }); const script = path.join(this.statusDir, 'install.sh'); - await copyFile(path.join(this.repoDir, 'deploy', 'install.sh'), script); + try { + await this.fetchUpstream(); + // execa strips the blob's final newline; put it back. + const content = await this.git(['show', 'FETCH_HEAD:deploy/install.sh']); + await writeFile(script, `${content}\n`); + } catch (error) { + throw httpError( + 500, + `could not fetch the installer of the build to install: ${error instanceof Error ? error.message : String(error)}`, + ); + } const args = ['--status-dir', this.statusDir]; - if (await this.originUsesMirror()) args.push('--mirror', 'cn'); + if (await this.remoteUsesMirror()) args.push('--mirror', 'cn'); const logFile = path.join(this.statusDir, 'upgrade.log'); const command = `exec bash ${quote(script)} ${args.map(quote).join(' ')} >${quote(logFile)} 2>&1`; const launch = await this.run('systemd-run', [ @@ -282,8 +340,13 @@ export class Updater { if (this.repoDir === null) { return 'the process does not run from a git checkout'; } - if (!existsSync(path.join(this.repoDir, 'deploy', 'install.sh'))) { - return 'deploy/install.sh is missing from the checkout'; + // The branch to pull along and to take the installer from must be + // known before a node reports it can upgrade itself: told without one, + // it would fail the launch and read stuck twenty minutes on. + try { + await this.upstream(); + } catch (error) { + return error instanceof Error ? error.message : String(error); } // Presence of systemd-run covers the platform question too — a // non-systemd host simply does not have it. @@ -294,9 +357,11 @@ export class Updater { return null; } - private async originUsesMirror(): Promise { + /** Whether the tracked remote was cloned through the mainland mirror prefix — its configured URL as written, before any url.insteadOf rewrite git applies when fetching. */ + private async remoteUsesMirror(): Promise { try { - const url = await this.git(['remote', 'get-url', 'origin']); + const { remote } = await this.upstream(); + const url = await this.git(['config', '--get', `remote.${remote}.url`]); return url.includes('ghfast.top'); } catch { return false; @@ -363,7 +428,10 @@ export class Updater { `git ${args[0]} failed: ${stderr.slice(0, 300) || `exit ${result.exitCode ?? 'unknown'}`}`, ); } - return result.stdout.trim(); + // As git printed it, less the final newline execa strips: every caller + // reads whole lines, and the one that reads a file (`git show` in + // apply) puts that newline back. + return result.stdout; } } From 9e980dbda88204a094c1b776b0dc5433af837d2a Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 13:21:11 +0800 Subject: [PATCH 78/89] upgrading.mdx says the one-click upgrade runs the installer of the version it installs, so a host-side step the new version adds lands with the code --- website/content/docs/upgrading.mdx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/website/content/docs/upgrading.mdx b/website/content/docs/upgrading.mdx index 3ce8bad2..9e9c5b94 100644 --- a/website/content/docs/upgrading.mdx +++ b/website/content/docs/upgrading.mdx @@ -21,7 +21,14 @@ curl -fsSL https://raw.githubusercontent.com/BitMiracle-AI/Dormice/main/deploy/i Every step checks before it acts — a step whose outcome is already in place says `[skip]` and touches nothing. -**Use the `curl | bash` form for upgrades.** If you instead run a local +The console's **Upgrade** button and `POST /applyUpgrade` run the same +installer — the installer *of the version being installed*, fetched from +the repository before it starts, not the copy on disk that belongs to the +version being replaced. A step the new version adds to the host (a +kernel parameter, a runtime flag, a new unit) therefore lands with the +code, and there is nothing to re-run by hand afterwards. + +**Use the `curl | bash` form for upgrades by hand.** If you instead run a local copy (`bash /opt/dormice/deploy/install.sh`), its `git pull` updates the script file mid-run while bash keeps executing the *old* bytes — measured on a real upgrade. Run a local copy twice, or just pipe from @@ -32,7 +39,8 @@ the repo. A fleet upgrades from its gateway, and one action upgrades every machine. On the gateway's machine, re-running the installer — or the console's **Upgrade** button, or `POST /applyUpgrade` at the gateway — -pulls, rebuilds and restarts the gateway and the node on that machine. +pulls, rebuilds and restarts the gateway and the node on that machine, +running the new version's installer. Once the gateway is back on the new build, every other node hears at its next check-in (within fifteen seconds) that its turn has come, and runs the same installer itself: one node at a time, so the fleet is From b0c59b4b9130414f3998598df99e06213747b5ec Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 13:24:43 +0800 Subject: [PATCH 79/89] =?UTF-8?q?The=20node's=20check-in=20says=20it=20lau?= =?UTF-8?q?nches=20the=20new=20build's=20install.sh=20when=20its=20turn=20?= =?UTF-8?q?to=20upgrade=20comes=20=E2=80=94=20the=20script=20the=20updater?= =?UTF-8?q?=20now=20fetches,=20not=20the=20tree's?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/server/src/check-in.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index 2dd7096e..4869977d 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -198,14 +198,14 @@ export class CheckIn { } if (answer.upgrade === true) { // The gateway's turn for this node in the fleet upgrade: run the - // same one-click upgrade an operator would (install.sh in a - // systemd unit, updater.ts). Said as its own line, not this tick's + // same one-click upgrade an operator would (the new build's + // install.sh in a systemd unit, updater.ts). Said as its own line, not this tick's // failure: the check-in itself succeeded, and the gateway tells a // node once — a launch that fails here is the operator's to read // (the gateway shows the node as stuck twenty minutes on, and // applyUpgrade {nodeId} at the gateway puts it back in line). opts.log.info( - `the gateway says this node's turn to upgrade has come — launching install.sh (systemd unit dormice-upgrade)`, + `the gateway says this node's turn to upgrade has come — launching the new build's install.sh (systemd unit dormice-upgrade)`, ); try { await (opts.applyUpgrade ?? unavailableUpgrade)(); From 7eb7b9cb4038d39885693ad86ffba42fd5c43efd Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 13:43:45 +0800 Subject: [PATCH 80/89] The gateway exam's note on checkUpgrade names what it fetches now: the head of the branch the checkout tracks, which a detached HEAD on CI has none of --- e2e/src/gateway.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/src/gateway.test.ts b/e2e/src/gateway.test.ts index 87d4b920..93093edd 100644 --- a/e2e/src/gateway.test.ts +++ b/e2e/src/gateway.test.ts @@ -538,8 +538,9 @@ describe.skipIf(skip)('the gateway in front of two daemons', () => { it("the upgrade verbs answer at the door: the gateway's own build and standing, every node's standing beside it; a misspelled verb is a 404", async () => { // Deliberately not applyUpgrade: it would re-run install.sh on the - // machine running the exam. checkUpgrade reaches for origin/main — - // its outcome is data either way (a check, or a checkError). + // machine running the exam. checkUpgrade fetches the head of the + // branch the checkout tracks (none on CI's detached HEAD) — its + // outcome is data either way (a check, or a checkError). const check = await viaGateway().checkUpgrade(); expect(check.check !== null || check.checkError !== null).toBe(true); const s = await viaGateway().getUpgradeStatus(); From 104478f84a7de6a2b81ee207dc6b28893627b8aa Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 18:23:14 +0800 Subject: [PATCH 81/89] =?UTF-8?q?A=20told=20node=20has=20fulfilled=20its?= =?UTF-8?q?=20tell=20when=20it=20reports=20another=20commit=20than=20the?= =?UTF-8?q?=20one=20it=20was=20told=20on,=20not=20only=20when=20it=20repor?= =?UTF-8?q?ts=20the=20gateway's=20=E2=80=94=20a=20gateway=20upgraded=20aga?= =?UTF-8?q?in=20while=20the=20node=20built=20no=20longer=20reads=20that=20?= =?UTF-8?q?node=20as=20upgrading=20for=20twenty=20minutes=20and=20then=20s?= =?UTF-8?q?tuck?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The row now records the commit the node ran when told (nodes.upgrade_told_build, migration 0006) beside the moment. A node back on any other commit is judged afresh: current, ahead, or behind and told again at its turn. Before, "still on the commit I was told on" was measured against the gateway's current build, so a fix pushed on the heels of the first push held the whole roll for twenty minutes with a reason that said the node had not moved when it had. A tell recorded before the column existed stands until the node reads current or ahead, as every tell once did. --- .../drizzle/0006_fleet-upgrade-told-build.sql | 1 + .../gateway/drizzle/meta/0006_snapshot.json | 509 ++++++++++++++++++ packages/gateway/drizzle/meta/_journal.json | 7 + packages/gateway/src/db/schema.ts | 11 +- packages/gateway/src/fleet.ts | 27 +- packages/gateway/src/rolling.test.ts | 67 ++- packages/gateway/src/rolling.ts | 59 +- website/content/docs/upgrading.mdx | 9 +- 8 files changed, 658 insertions(+), 32 deletions(-) create mode 100644 packages/gateway/drizzle/0006_fleet-upgrade-told-build.sql create mode 100644 packages/gateway/drizzle/meta/0006_snapshot.json diff --git a/packages/gateway/drizzle/0006_fleet-upgrade-told-build.sql b/packages/gateway/drizzle/0006_fleet-upgrade-told-build.sql new file mode 100644 index 00000000..4ad1f03b --- /dev/null +++ b/packages/gateway/drizzle/0006_fleet-upgrade-told-build.sql @@ -0,0 +1 @@ +ALTER TABLE `nodes` ADD `upgrade_told_build` text; \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/0006_snapshot.json b/packages/gateway/drizzle/meta/0006_snapshot.json new file mode 100644 index 00000000..931485c6 --- /dev/null +++ b/packages/gateway/drizzle/meta/0006_snapshot.json @@ -0,0 +1,509 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "5e0e0a1d-f29b-4024-aec2-6cabf0d092c4", + "prevId": "310aabe3-0483-455e-ab72-060b6292d2a6", + "tables": { + "api_keys": { + "name": "api_keys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + "key_hash" + ], + "isUnique": true + }, + "api_keys_active_name_idx": { + "name": "api_keys_active_name_idx", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"api_keys\".\"revoked_at\" IS NULL" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "console_account": { + "name": "console_account", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_secret": { + "name": "session_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "fleet_state_samples": { + "name": "fleet_state_samples", + "columns": { + "at": { + "name": "at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active": { + "name": "active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "frozen": { + "name": "frozen", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stopped": { + "name": "stopped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "restoring": { + "name": "restoring", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "fleet_state_samples_at_idx": { + "name": "fleet_state_samples_at_idx", + "columns": [ + "at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "nodes": { + "name": "nodes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "added_at": { + "name": "added_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "swap_gb": { + "name": "swap_gb", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_check_in_at": { + "name": "last_check_in_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "interval_seconds": { + "name": "interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config_version": { + "name": "config_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "build": { + "name": "build", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reading": { + "name": "reading", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "self_upgrade": { + "name": "self_upgrade", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "upgrade_told_at": { + "name": "upgrade_told_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "upgrade_told_build": { + "name": "upgrade_told_build", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_cpus": { + "name": "sandbox_cpus", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_memory_gb": { + "name": "sandbox_memory_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sandbox_disk_gb": { + "name": "sandbox_disk_gb", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_freeze_after_seconds": { + "name": "default_freeze_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_stop_after_seconds": { + "name": "default_stop_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_archive_after_seconds": { + "name": "default_archive_after_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_endpoint": { + "name": "s3_endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_access_key_id": { + "name": "s3_access_key_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_secret_access_key": { + "name": "s3_secret_access_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_region": { + "name": "s3_region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3_force_path_style": { + "name": "s3_force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain": { + "name": "sandbox_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandbox_domain_aliases": { + "name": "sandbox_domain_aliases", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pids_limit": { + "name": "pids_limit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_image": { + "name": "base_image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registry_address": { + "name": "registry_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/gateway/drizzle/meta/_journal.json b/packages/gateway/drizzle/meta/_journal.json index b616232d..0fbe6e70 100644 --- a/packages/gateway/drizzle/meta/_journal.json +++ b/packages/gateway/drizzle/meta/_journal.json @@ -43,6 +43,13 @@ "when": 1789461434618, "tag": "0005_fleet-upgrade", "breakpoints": true + }, + { + "idx": 6, + "version": "6", + "when": 1789553929327, + "tag": "0006_fleet-upgrade-told-build", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/gateway/src/db/schema.ts b/packages/gateway/src/db/schema.ts index 20aed4ea..b2d39f60 100644 --- a/packages/gateway/src/db/schema.ts +++ b/packages/gateway/src/db/schema.ts @@ -69,10 +69,19 @@ export const nodes = sqliteTable('nodes', { /** * ISO 8601 UTC — when the fleet upgrade last told this node to upgrade * itself (rolling.ts); null = never, or the tell was fulfilled (the node - * came back on the gateway's build). On the row so a gateway restart + * came back on another build). On the row so a gateway restart * mid-roll neither forgets a node it told nor tells it twice. */ upgradeToldAt: text('upgrade_told_at'), + /** + * The commit the node ran when it was told — what "fulfilled" is judged + * against: a node reporting any other commit did what it was told, + * whether or not that commit is the gateway's by now (the gateway may + * have upgraded again meanwhile). Null beside a tell only on a row + * written before this column existed; such a tell stands until the + * node reads current or ahead. + */ + upgradeToldBuild: text('upgrade_told_build'), }); export type NodeRow = typeof nodes.$inferSelect; diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 6c924273..75e49a32 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -46,6 +46,8 @@ export interface NodeState { selfUpgrade: SelfUpgrade | null; /** When the fleet upgrade last told this node to upgrade (rolling.ts); null = never, or fulfilled. */ upgradeToldAt: Date | null; + /** The commit the node ran when it was told — the tell is fulfilled the moment it reports another (rolling.ts); null with a tell = a row written before this was recorded. */ + upgradeToldBuild: string | null; placedSinceCheckIn: number; placedIds: Set; } @@ -251,6 +253,7 @@ export class Fleet { selfUpgradeSchema, ), upgradeToldAt: this.parseDate(row.upgradeToldAt), + upgradeToldBuild: row.upgradeToldBuild, placedSinceCheckIn: 0, placedIds: new Set(), }; @@ -349,6 +352,7 @@ export class Fleet { reading: null, selfUpgrade: null, upgradeToldAt: null, + upgradeToldBuild: null, placedSinceCheckIn: 0, placedIds: new Set(), }; @@ -386,22 +390,27 @@ export class Fleet { } /** - * The fleet upgrade's one mark on a node: when it was told to upgrade, - * or null once the tell is fulfilled (rolling.ts). Written through, - * not best-effort — the tell rides on the check-in's answer, and a - * gateway that forgot it told a node would tell it again after a - * restart, the one thing the rolling upgrade promises not to do; a - * write that fails fails the check-in, and the node is told at the next. + * The fleet upgrade's one mark on a node: when it was told to upgrade + * and what it ran at that moment, or null once the tell is fulfilled + * (rolling.ts). Written through, not best-effort — the tell rides on + * the check-in's answer, and a gateway that forgot it told a node would + * tell it again after a restart, the one thing the rolling upgrade + * promises not to do; a write that fails fails the check-in, and the + * node is told at the next. */ - setUpgradeToldAt(id: string, at: Date | null): void { + setUpgradeTold(id: string, told: { at: Date; build: string } | null): void { const node = this.members.get(id); if (node === undefined) return; this.db .update(nodes) - .set({ upgradeToldAt: at === null ? null : at.toISOString() }) + .set({ + upgradeToldAt: told === null ? null : told.at.toISOString(), + upgradeToldBuild: told === null ? null : told.build, + }) .where(eq(nodes.id, id)) .run(); - node.upgradeToldAt = at; + node.upgradeToldAt = told === null ? null : told.at; + node.upgradeToldBuild = told === null ? null : told.build; } /** diff --git a/packages/gateway/src/rolling.test.ts b/packages/gateway/src/rolling.test.ts index b14bb7d6..e87a1ba8 100644 --- a/packages/gateway/src/rolling.test.ts +++ b/packages/gateway/src/rolling.test.ts @@ -1,5 +1,6 @@ import { fileURLToPath } from 'node:url'; import type { BuildInfo } from '@dormice/shared'; +import { eq } from 'drizzle-orm'; import { describe, expect, it } from 'vitest'; import { migrateDb, openDb } from './db/db'; import { nodes } from './db/schema'; @@ -106,7 +107,7 @@ describe('upgradeStateOf', () => { }); const current = reporting(fleet, 'b', { build: GATEWAY, selfUpgrade: CAN }); expect(upgradeStateOf(current, GATEWAY, later).state).toBe('unreachable'); - fleet.setUpgradeToldAt('a', NOW); + fleet.setUpgradeTold('a', { at: NOW, build: OLD.commit }); expect(upgradeStateOf(node, GATEWAY, later)).toEqual({ state: 'upgrading', reason: 'told 31s ago, still on old0001 (has not checked in for 31s)', @@ -153,7 +154,7 @@ describe('upgradeStateOf', () => { it('told: upgrading within the timeout, stuck past it — with when it was told, what it still runs, and where to look', () => { const { fleet } = fleetOver(); const node = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); - fleet.setUpgradeToldAt('a', NOW); + fleet.setUpgradeTold('a', { at: NOW, build: OLD.commit }); const soon = new Date(NOW.getTime() + 90_000); // Still checking in during its build: upgrading, plainly. reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }, soon); @@ -182,7 +183,7 @@ describe('rollingDecision', () => { const b = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }); const all = fleet.all(); expect(rollingDecision(all, GATEWAY, a, NOW)).toBe(true); - fleet.setUpgradeToldAt('a', NOW); + fleet.setUpgradeTold('a', { at: NOW, build: OLD.commit }); expect(rollingDecision(all, GATEWAY, b, NOW)).toBe(false); // a restarts near the end of its upgrade and misses check-ins: still // upgrading, and b still waits. @@ -311,6 +312,66 @@ describe('Rolling', () => { }); }); + it('a told node that comes back on another commit than it was told on has fulfilled its tell, even when the gateway moved on meanwhile: cleared, behind again, and told again at its turn — not upgrading, not stuck', () => { + const { db, fleet } = fleetOver(); + // The gateway was on GATEWAY when a was told; a pulled that head and + // built. Before it came back, the gateway's machine was upgraded + // again (a fix pushed minutes after the first), so the gateway now + // runs NEWER and a comes back on GATEWAY — older than the gateway's. + const first = new Rolling(fleet, GATEWAY); + const a = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); + const b = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }); + expect(first.onCheckIn(a, NOW)).toBe(true); + expect(a.upgradeToldBuild).toBe(OLD.commit); + expect(new Fleet(db).get('a')?.upgradeToldBuild).toBe(OLD.commit); + const rolling = new Rolling(fleet, NEWER); + const back = new Date(NOW.getTime() + 70_000); + // Its tell stands while it still reports the commit it was told on. + expect(upgradeStateOf(a, NEWER, back)).toMatchObject({ + state: 'upgrading', + }); + reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }, back); + expect(rolling.onCheckIn(b, back)).toBe(false); + // Back on GATEWAY: another commit than told on — fulfilled. Not + // upgrading, not stuck; behind the gateway's NEWER, and told again at + // once, nobody else being mid-upgrade. + reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: CAN }, back); + expect(upgradeStateOf(a, NEWER, back)).toEqual({ + state: 'behind', + reason: null, + }); + expect(rolling.onCheckIn(a, back)).toBe(true); + expect(a.upgradeToldAt).toEqual(back); + expect(a.upgradeToldBuild).toBe(GATEWAY.commit); + expect(rolling.states(back)).toMatchObject([ + { id: 'a', state: 'upgrading', toldAt: back.toISOString() }, + { id: 'b', state: 'behind', toldAt: null }, + ]); + // Twenty minutes on the same commit is stuck, as before — and now the + // reason's "still on" is true by construction. + const late = new Date(back.getTime() + UPGRADE_TOLD_TIMEOUT_MS); + reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: CAN }, late); + expect(upgradeStateOf(a, NEWER, late)).toMatchObject({ + state: 'stuck', + reason: expect.stringMatching(/still on new0001 20 minutes later/), + }); + // A tell from a row written before the commit was recorded stands + // until the node reads current or ahead, as every tell once did. + reporting(fleet, 'c', { build: OLD, selfUpgrade: CAN }, late); + fleet.setUpgradeTold('c', { at: late, build: OLD.commit }); + db.update(nodes) + .set({ upgradeToldBuild: null }) + .where(eq(nodes.id, 'c')) + .run(); + const cOld = new Fleet(db).get('c'); + if (!cOld) throw new Error('row lost'); + expect(cOld.upgradeToldBuild).toBeNull(); + cOld.build = GATEWAY; + expect(upgradeStateOf(cOld, NEWER, late).state).toBe('upgrading'); + cOld.build = NEWER; + expect(upgradeStateOf(cOld, NEWER, late).state).toBe('current'); + }); + it('a told node that comes back newer than the gateway is ahead: its tell is fulfilled and cleared, it is not told again, the hand refuses it, and it holds nobody', () => { const { db, fleet } = fleetOver(); const rolling = new Rolling(fleet, GATEWAY); diff --git a/packages/gateway/src/rolling.ts b/packages/gateway/src/rolling.ts index da10846f..b3123324 100644 --- a/packages/gateway/src/rolling.ts +++ b/packages/gateway/src/rolling.ts @@ -35,10 +35,20 @@ import { downReason, type Fleet, type NodeState } from './fleet'; * could break it was the wrong hand, and with it gone nothing is * remembered but the row. * - * The row: the tell is nodes.upgrade_told_at, so a gateway restart + * The row: the tell is nodes.upgrade_told_at and, beside it, the commit + * the node ran when told (upgrade_told_build) — so a gateway restart * mid-roll neither forgets a node it told nor tells it twice, and every * verdict here is a function of the rows, the gateway's build and the - * clock. + * clock. Fulfilled means the node reports another commit than the one + * it was told on, not the gateway's commit: the gateway may have been + * upgraded again while the node built (a fix pushed on the heels of the + * first push — the shape of every review day), and a node coming back on + * the first push's commit has done exactly what it was told. Judged as + * "still not on my build" it read upgrading for twenty minutes, then + * stuck with a reason that said it had not moved when it had — and the + * one-at-a-time rule held the whole roll for those twenty minutes (found + * by review, 2026-09-16). Back on another commit it is judged afresh: + * current, ahead, or behind and in line for the next tell. * * Behind means older. A node whose build is newer than the gateway's — a * commit that landed on main after the gateway's machine upgraded and @@ -112,8 +122,16 @@ export function upgradeStateOf( // its daemon restarts near the end of install.sh and misses a check-in // or two by design, and were that silence read as "unreachable" the // one-at-a-time rule would see nobody upgrading and tell the next node - // into the same minute. The silence is said in the reason instead. - if (node.upgradeToldAt !== null) { + // into the same minute. The silence is said in the reason instead. The + // tell stands while the node still reports the commit it was told on + // (the module comment has why that, and not the gateway's commit, is + // the measure); a tell recorded without one — a row from before the + // column — stands until the node reads current or ahead, above. + if ( + node.upgradeToldAt !== null && + (node.upgradeToldBuild === null || + node.upgradeToldBuild === node.build.commit) + ) { const sinceMs = now.getTime() - node.upgradeToldAt.getTime(); const silence = down === null ? '' : ` (${down})`; if (sinceMs < UPGRADE_TOLD_TIMEOUT_MS) { @@ -178,23 +196,32 @@ export class Rolling { /** * The check-in's verdict for a node that just reported: a fulfilled tell - * is cleared (the node is off the old build — on the gateway's, or ahead - * of it); otherwise the rolling rule decides, and a tell is written to - * the row before the answer carries it. Answers whether the node is told - * now. + * is cleared from the row (the node is off the commit it was told on — + * on the gateway's, ahead of it, or behind it once more because the + * gateway moved on meanwhile: upgradeStateOf no longer reads it as + * upgrading or stuck); then the rolling rule decides, and a tell is + * written to the row before the answer carries it. Answers whether the + * node is told now — which a node just back from one upgrade may be, + * when the gateway is already past the build it came back on. */ onCheckIn(node: NodeState, now: Date): boolean { const { state } = upgradeStateOf(node, this.gatewayBuild, now); - if (state === 'current' || state === 'ahead') { - if (node.upgradeToldAt !== null) { - this.fleet.setUpgradeToldAt(node.id, null); - } - return false; + if ( + node.upgradeToldAt !== null && + state !== 'upgrading' && + state !== 'stuck' + ) { + this.fleet.setUpgradeTold(node.id, null); } - if (!rollingDecision(this.fleet.all(), this.gatewayBuild, node, now)) { + if ( + state !== 'behind' || + !rollingDecision(this.fleet.all(), this.gatewayBuild, node, now) + ) { return false; } - this.fleet.setUpgradeToldAt(node.id, now); + // Behind is judged only of a node with a build (upgradeStateOf). + if (node.build === null) return false; + this.fleet.setUpgradeTold(node.id, { at: now, build: node.build.commit }); return true; } @@ -216,7 +243,7 @@ export class Rolling { const { state, reason } = upgradeStateOf(node, this.gatewayBuild, now); switch (state) { case 'stuck': - this.fleet.setUpgradeToldAt(node.id, null); + this.fleet.setUpgradeTold(node.id, null); return null; case 'upgrading': return { diff --git a/website/content/docs/upgrading.mdx b/website/content/docs/upgrading.mdx index 9e9c5b94..6ad0eb0e 100644 --- a/website/content/docs/upgrading.mdx +++ b/website/content/docs/upgrading.mdx @@ -51,10 +51,13 @@ than the gateway's (a commit landed on `main` while the fleet was rolling, or the installer was run on that node by hand) is left alone, and reads current once the gateway itself is upgraded. -A node is told exactly once. If it has not come back on the new build -twenty minutes later, the gateway marks it **stuck** and leaves it +A node is told exactly once. If twenty minutes later it still reports +the build it was told on, the gateway marks it **stuck** and leaves it alone — a node whose build keeps failing must not rebuild every twenty -minutes on the CPU its sandboxes run on. Read `journalctl -u +minutes on the CPU its sandboxes run on. (A node that comes back on +another build has done what it was told, even if the gateway was +upgraded again meanwhile: it is judged afresh, and told again at its +turn if it is behind once more.) Read `journalctl -u dormice-upgrade` and `/var/lib/dormice/upgrade/upgrade.log` on that node, fix the cause, and press **Try again** on the version page (or `POST /applyUpgrade {"nodeId": "..."}`), which puts that node back in From 1310400f82517c0784d3a1671af162cb2828463c Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 18:23:18 +0800 Subject: [PATCH 82/89] A node's auth.ts is the fleet token and nothing else: the scrypt, session-cookie and admin-gate halves went to the gateway with the console in the second cut and were dead here, as was the @dormice/server/auth subpath nobody imported MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit What remains is tokensEqual and requireApiAuth(isCredential) — the one hook the node's two faces share. The gateway keeps its own auth.ts, the only copy of the console's credential code now; the stale comments in the daemon's copy ("the gateway never mounts the console") went with it. --- packages/server/package.json | 4 - packages/server/src/app.ts | 11 +- packages/server/src/auth.ts | 234 +++------------------------------ packages/server/tsup.config.ts | 3 +- 4 files changed, 24 insertions(+), 228 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index c69cc686..ea5b0015 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -14,10 +14,6 @@ "types": "./dist/archive/mini-s3.d.ts", "default": "./dist/archive/mini-s3.js" }, - "./auth": { - "types": "./dist/auth.d.ts", - "default": "./dist/auth.js" - }, "./keyed-queue": { "types": "./dist/keyed-queue.d.ts", "default": "./dist/keyed-queue.js" diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 4150c166..0d7d7c27 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -247,14 +247,13 @@ export function buildApp({ // The one adjudication of "does this bare credential open the door": // the fleet token, constant-time compared — the only credential a node - // knows. Minted API keys are the gateway's to judge; it forwards under - // this token. Both faces — the native Bearer header and the E2B - // X-API-KEY hook — feed this same closure: one truth, two dialects. No - // session leg: the console lives at the gateway, so no cookie is ever - // valid here. + // knows (auth.ts). Minted API keys are the gateway's to judge; it + // forwards under this token. Both faces — the native Bearer header and + // the E2B X-API-KEY hook — feed this same closure: one truth, two + // dialects. const isCredential = (bare: string): boolean => tokensEqual(bare, config.DORMICE_API_TOKEN); - const apiAuth = requireApiAuth(isCredential, () => null); + const apiAuth = requireApiAuth(isCredential); // The envd/signed-URL derivation base. Captured once — unlike a session // secret there is no verb that rotates it (see db/secrets.ts) — and NOT diff --git a/packages/server/src/auth.ts b/packages/server/src/auth.ts index 5e6c4abd..a882ce86 100644 --- a/packages/server/src/auth.ts +++ b/packages/server/src/auth.ts @@ -1,27 +1,17 @@ -import { - createHash, - createHmac, - randomBytes, - type ScryptOptions, - scrypt, - timingSafeEqual, -} from 'node:crypto'; -import type { FastifyRequest, onRequestAsyncHookHandler } from 'fastify'; +import { createHash, timingSafeEqual } from 'node:crypto'; +import type { onRequestAsyncHookHandler } from 'fastify'; -// Hand-rolled instead of util.promisify: promisify picks the overload -// without the options argument, and the cost parameters live there. -function scryptAsync( - password: string, - salt: Buffer, - keylen: number, - options: ScryptOptions, -): Promise { - return new Promise((resolve, reject) => { - scrypt(password, salt, keylen, options, (err, key) => - err ? reject(err) : resolve(key), - ); - }); -} +/** + * A node's whole authentication: the fleet token, and nothing else. Since + * the configuration authority moved to the gateway (design record #22, + * 2026-09-14) a node knows one credential — the token every node and the + * gateway share (#34) — and judges nothing about who is behind it: minted + * API keys, the console's session cookie and the admin gate are the + * gateway's (packages/gateway/src/auth.ts), which speaks this token toward + * the node whoever called it. The two faces of a node — the native Bearer + * header and the E2B X-API-KEY hook (e2b/control.ts) — feed the one + * closure app.ts builds over tokensEqual: one truth, two dialects. + */ const sha256 = (value: string) => createHash('sha256').update(value).digest(); @@ -31,155 +21,14 @@ export function tokensEqual(presented: string, expected: string): boolean { } /** - * Password hashing for the console account: scrypt (in node:crypto, zero - * dependencies — the whole reason it wins over bcrypt/argon2 here). The - * parameters ride inside the stored string, so they can change later - * without invalidating old hashes. - * - * N=2^14, r=8, p=1 (~16 MiB, tens of ms): the standard interactive-login - * cost. The online-guessing defense is the login throttle; this cost is - * for the offline case, a stolen ledger file. - */ -const SCRYPT_N = 16384; -const SCRYPT_R = 8; -const SCRYPT_P = 1; -const SCRYPT_KEYLEN = 32; - -export async function hashPassword(password: string): Promise { - const salt = randomBytes(16); - const hash = await scryptAsync(password, salt, SCRYPT_KEYLEN, { - N: SCRYPT_N, - r: SCRYPT_R, - p: SCRYPT_P, - }); - return [ - 'scrypt', - SCRYPT_N, - SCRYPT_R, - SCRYPT_P, - salt.toString('base64'), - hash.toString('base64'), - ].join('$'); -} - -export async function verifyPassword( - password: string, - stored: string, -): Promise { - const [scheme, n, r, p, saltB64, hashB64] = stored.split('$'); - if (scheme !== 'scrypt' || !n || !r || !p || !saltB64 || !hashB64) { - return false; - } - const expected = Buffer.from(hashB64, 'base64'); - const actual = await scryptAsync( - password, - Buffer.from(saltB64, 'base64'), - expected.length, - { N: Number(n), r: Number(r), p: Number(p) }, - ); - return actual.length === expected.length && timingSafeEqual(actual, expected); -} - -/** The session-cookie HMAC key: random, stored on the account row. */ -export function mintSessionSecret(): string { - return randomBytes(32).toString('hex'); -} - -/** - * The web console's session cookie. Stateless on purpose: the daemon is - * crash-only, and an in-memory session table would log every operator out - * on each restart. The value carries its own expiry and an HMAC over it — - * the same pattern as the envd access token — so a restart changes nothing. - * - * The HMAC key is the account's sessionSecret, not the API token: the two - * credentials rotate independently. Re-running setup (password change or - * reset) regenerates the secret and voids every session — the semantics a - * password change should have — while rotating the API token leaves the - * console signed in. - */ -export const SESSION_COOKIE = 'dormice_session'; -export const SESSION_TTL_SECONDS = 7 * 24 * 60 * 60; - -/** - * Cookie-authenticated requests must also carry this header. A cross-origin - * page cannot send a custom header without a CORS preflight, and the daemon - * answers no preflights — this closes the hole SameSite leaves open - * (SameSite ignores the port, so another local web app counts as same-site). - */ -export const CONSOLE_HEADER = 'x-dormice-console'; - -function sessionHmac(secret: string, expiresAtSeconds: number): string { - return createHmac('sha256', secret) - .update(`console-session:${expiresAtSeconds}`) - .digest('hex'); -} - -export function mintSession(secret: string, nowMs = Date.now()): string { - const expiresAt = Math.floor(nowMs / 1000) + SESSION_TTL_SECONDS; - return `${expiresAt}.${sessionHmac(secret, expiresAt)}`; -} - -export function verifySession( - secret: string, - value: string, - nowMs = Date.now(), -): boolean { - const dot = value.indexOf('.'); - if (dot < 0) return false; - const expiresAt = Number(value.slice(0, dot)); - // The expiry is plaintext in the cookie — nothing secret to compare in - // constant time. The HMAC comparison below is the constant-time one. - if (!Number.isInteger(expiresAt) || expiresAt * 1000 <= nowMs) return false; - return tokensEqual(value.slice(dot + 1), sessionHmac(secret, expiresAt)); -} - -/** - * The console-session leg shared verbatim by both auth hooks: cookie - * present, an account exists (secret non-null), the CSRF header rode along - * (see CONSOLE_HEADER), and the HMAC verifies. The session secret is - * fetched per request, not captured at startup: setup can replace the - * account (and its secret) while the daemon runs, and the arbiter must - * judge against the current one. Null means no account exists yet — no - * cookie can be valid. - */ -function sessionCookieValid( - request: FastifyRequest, - getSessionSecret: () => string | null, -): boolean { - // The jar exists only where the app registered @fastify/cookie: the - // daemon does; the gateway never mounts the console and passes a getter - // that answers null, so no cookie can pass there. Typed structurally so - // this module stands alone as the `@dormice/server/auth` subpath entry - // without that plugin's request augmentation in its build graph. - const jar = (request as { cookies?: Record }) - .cookies; - const cookie = jar?.[SESSION_COOKIE]; - const secret = getSessionSecret(); - return Boolean( - cookie && - secret !== null && - request.headers[CONSOLE_HEADER] !== undefined && - verifySession(secret, cookie), - ); -} - -/** - * The single arbiter of who may call the API (/healthz stays open — - * liveness probes have no secrets). Two credentials open the same door: - * a Bearer credential (SDK, CLI, curl — the env token or any live API - * key, adjudicated by isCredential) and the web console's session - * cookie (which additionally requires the console header, see above). A - * second route surface with its own auth would be a second truth. - * - * isCredential judges bare tokens, so both faces (this Bearer header and - * the E2B X-API-KEY hook) feed it the same canonical form — one closure, - * one truth, two dialects. The 'Bearer ' prefix is public framing, not a - * secret, so stripping it needs no constant time; the secret comparisons - * live inside isCredential. + * The single arbiter of who may call a node's API (/healthz stays open — + * liveness probes have no secrets): a Bearer credential that isCredential + * accepts. The 'Bearer ' prefix is public framing, not a secret, so + * stripping it needs no constant time; the secret comparison lives inside + * isCredential. */ export function requireApiAuth( isCredential: (bareToken: string) => boolean, - getSessionSecret: () => string | null, ): onRequestAsyncHookHandler { return async (request, reply) => { const header = request.headers.authorization; @@ -187,53 +36,6 @@ export function requireApiAuth( if (bare !== null && isCredential(bare)) { return; } - if (sessionCookieValid(request, getSessionSecret)) { - return; - } - await reply.code(401).send({ message: 'missing or invalid API token' }); - }; -} - -/** - * The admin gate for the apiKey management verbs and updateSettings: only - * the env token (Bearer) or a console session may pass. A key that is - * otherwise valid gets an honest 403 instead of a silent 401 — - * key-manages-key would let one leaked credential mint itself an unrevoked - * successor and revoke every legitimate peer, and a leaked automation key - * must not be able to raise the very limits that contain it; the refusal - * names the rule. The console-setup door (routes/console.ts) rests on the - * same doctrine: a machine credential must not escalate into managing the - * daemon. - * - * Leg order matters twice. The isLiveApiKey lookup runs only after both - * accepting legs failed, so a console session with a stray key header - * still passes, and the ledger is consulted only for requests already - * being refused. And isLiveApiKey must be a pure read that never stamps - * lastUsedAt — the request is being refused, not honored. A disabled or - * expired key is no longer a valid credential and falls through to the - * same 401 as garbage: a 403 for it would leak that the row exists. - */ -export function requireAdminAuth( - isEnvToken: (bareToken: string) => boolean, - isLiveApiKey: (bareToken: string) => boolean, - getSessionSecret: () => string | null, -): onRequestAsyncHookHandler { - return async (request, reply) => { - const header = request.headers.authorization; - const bare = header?.startsWith('Bearer ') ? header.slice(7) : null; - if (bare !== null && isEnvToken(bare)) { - return; - } - if (sessionCookieValid(request, getSessionSecret)) { - return; - } - if (bare !== null && isLiveApiKey(bare)) { - await reply.code(403).send({ - message: - 'API keys cannot manage API keys or settings — use DORMICE_API_TOKEN or the console', - }); - return; - } await reply.code(401).send({ message: 'missing or invalid API token' }); }; } diff --git a/packages/server/tsup.config.ts b/packages/server/tsup.config.ts index 8360bdbb..75aabf83 100644 --- a/packages/server/tsup.config.ts +++ b/packages/server/tsup.config.ts @@ -24,7 +24,7 @@ function git(args: string): string { const commitTime = git('log -1 --format=%cI'); export default defineConfig({ - // Subpath entries beyond the root: mini-s3 for the e2e harness; auth, + // Subpath entries beyond the root: mini-s3 for the e2e harness; // keyed-queue, lock, shutdown, s3-store, history and updater for the // gateway, which reuses the daemon's small self-contained pieces // without loading its executor (the root's import graph drags @@ -33,7 +33,6 @@ export default defineConfig({ 'src/index.ts', 'src/main.ts', 'src/archive/mini-s3.ts', - 'src/auth.ts', 'src/keyed-queue.ts', 'src/db/lock.ts', 'src/shutdown.ts', From ec5eae398ed5dd72b7ba4054833c20c71d2d8b67 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 18:39:46 +0800 Subject: [PATCH 83/89] =?UTF-8?q?The=20node's=20sandbox=20census=20is=20on?= =?UTF-8?q?e=20GROUP=20BY,=20not=20every=20row=20loaded=20and=20counted=20?= =?UTF-8?q?in=20JavaScript=20=E2=80=94=20the=20check-in=20asks=20it=20ever?= =?UTF-8?q?y=20fifteen=20seconds,=20getHostMetrics=20on=20every=20poll,=20?= =?UTF-8?q?and=20a=20production=20ledger=20holds=20tens=20of=20thousands?= =?UTF-8?q?=20of=20rows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit countSandboxesByState(db) replaces countByState(listSandboxes(db)) at both call sites; the metrics sampler keeps loading the rows it samples one by one. --- packages/server/src/check-in.ts | 4 ++-- packages/server/src/db/ledger.test.ts | 24 +++++++++++++++++++++++- packages/server/src/db/ledger.ts | 24 +++++++++++++++++++----- packages/server/src/routes/host.ts | 5 ++--- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index 4869977d..301c304a 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -6,7 +6,7 @@ import { type NodeReading, } from '@dormice/shared'; import type { Db } from './db/db'; -import { countByState, listSandboxes } from './db/ledger'; +import { countSandboxesByState } from './db/ledger'; import type { Executor } from './executor/executor'; import { type CpuSampler, readHostReading } from './host-metrics'; import type { SwapControl } from './swap'; @@ -27,7 +27,7 @@ export async function readNodeReading( executor: Executor, swap?: SwapControl, ): Promise { - const { byState, total } = countByState(listSandboxes(db)); + const { byState, total } = countSandboxesByState(db); return { ...(await readHostReading(cpu, dataDir)), sandboxes: { total, byState }, diff --git a/packages/server/src/db/ledger.test.ts b/packages/server/src/db/ledger.test.ts index ac0bf5ab..1007724a 100644 --- a/packages/server/src/db/ledger.test.ts +++ b/packages/server/src/db/ledger.test.ts @@ -3,7 +3,13 @@ import { fileURLToPath } from 'node:url'; import { DEFAULT_LIFECYCLE_POLICY } from '@dormice/shared'; import { describe, expect, it } from 'vitest'; import { type Db, migrateDb, openDb } from './db'; -import { createSandbox, findByName, touch, transition } from './ledger'; +import { + countSandboxesByState, + createSandbox, + findByName, + touch, + transition, +} from './ledger'; const MIGRATIONS = fileURLToPath(new URL('../../drizzle', import.meta.url)); @@ -31,6 +37,22 @@ describe('ledger', () => { expect(findByName(db, 'someone-else')).toBeUndefined(); }); + it('counts the census by state in SQL, every state present, zero where empty', () => { + const db = testDb(); + expect(countSandboxesByState(db)).toEqual({ + byState: { active: 0, frozen: 0, stopped: 0, archived: 0, restoring: 0 }, + total: 0, + }); + create(db, 'a'); + create(db, 'b'); + const { id } = create(db, 'c'); + transition(db, id, 'frozen'); + expect(countSandboxesByState(db)).toEqual({ + byState: { active: 2, frozen: 1, stopped: 0, archived: 0, restoring: 0 }, + total: 3, + }); + }); + it('enforces one sandbox per name at the database level', () => { const db = testDb(); create(db); diff --git a/packages/server/src/db/ledger.ts b/packages/server/src/db/ledger.ts index 4a043261..0331aa5c 100644 --- a/packages/server/src/db/ledger.ts +++ b/packages/server/src/db/ledger.ts @@ -4,7 +4,7 @@ import { SANDBOX_STATES, type SandboxState, } from '@dormice/shared'; -import { eq } from 'drizzle-orm'; +import { count, eq } from 'drizzle-orm'; import type { Db } from './db'; import { type SandboxRow, sandboxes } from './schema'; @@ -117,16 +117,30 @@ export function listSandboxes(db: Db): SandboxRow[] { return db.select().from(sandboxes).all(); } -/** State census over a listing — getHostMetrics and the metrics sampler share it. */ -export function countByState(rows: SandboxRow[]): { +/** + * The state census, straight from SQL — one GROUP BY, never every row + * loaded and counted in JavaScript: the check-in asks it every fifteen + * seconds and getHostMetrics on every poll, and a production ledger holds + * tens of thousands of rows (Beijing, 2026-09). Every state is present, + * zero where the table has none. + */ +export function countSandboxesByState(db: Db): { byState: Record; total: number; } { const byState = Object.fromEntries( SANDBOX_STATES.map((state) => [state, 0]), ) as Record; - for (const row of rows) byState[row.state] += 1; - return { byState, total: rows.length }; + let total = 0; + for (const row of db + .select({ state: sandboxes.state, n: count() }) + .from(sandboxes) + .groupBy(sandboxes.state) + .all()) { + byState[row.state] = row.n; + total += row.n; + } + return { byState, total }; } /** diff --git a/packages/server/src/routes/host.ts b/packages/server/src/routes/host.ts index d6b100d4..3adab5b1 100644 --- a/packages/server/src/routes/host.ts +++ b/packages/server/src/routes/host.ts @@ -6,7 +6,7 @@ import { import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import type { Config } from '../config'; import type { Db } from '../db/db'; -import { countByState, listSandboxes } from '../db/ledger'; +import { countSandboxesByState } from '../db/ledger'; import { bucketHostSamples, queryHostCpuPeak, @@ -49,8 +49,7 @@ export const hostRoutes: FastifyPluginAsyncZod = async ( }, }, async () => { - const rows = listSandboxes(db); - const { byState, total } = countByState(rows); + const { byState, total } = countSandboxesByState(db); return { ...(await readHostReading(cpu, config.DORMICE_DATA_DIR)), From 6b48047d6447ab29650ca25906941663c16120f3 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 18:39:46 +0800 Subject: [PATCH 84/89] =?UTF-8?q?A=20sandbox's=20nodeId=20on=20the=20wire?= =?UTF-8?q?=20is=20the=20id=20of=20the=20node=20answering=20for=20it,=20no?= =?UTF-8?q?t=20the=20id=20the=20node=20had=20when=20the=20row=20was=20born?= =?UTF-8?q?=20=E2=80=94=20a=20node=20renamed=20since=20would=20otherwise?= =?UTF-8?q?=20list=20its=20older=20sandboxes=20under=20a=20node=20that=20n?= =?UTF-8?q?o=20longer=20exists,=20a=20ghost=20beside=20the=20real=20one=20?= =?UTF-8?q?in=20a=20fleet's=20merged=20list?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every row of a ledger is this node's; the row's node_id column stays as the birth record. The view takes the daemon's DORMICE_NODE_ID beside its endpoint (one Self for every row), and the shared schema says what the field means now. --- packages/server/src/app.test.ts | 27 ++++++++++++++++++- packages/server/src/routes/sandboxes.ts | 36 ++++++++++++++++++------- packages/shared/src/sandbox.ts | 2 +- 3 files changed, 53 insertions(+), 12 deletions(-) diff --git a/packages/server/src/app.test.ts b/packages/server/src/app.test.ts index 1d9d207a..a81a5299 100644 --- a/packages/server/src/app.test.ts +++ b/packages/server/src/app.test.ts @@ -16,7 +16,7 @@ import { MemStore } from './archive/mem-store'; import { objectKey } from './archive/store'; import { loadConfig } from './config'; import { migrateDb, openDb } from './db/db'; -import { findById, transition } from './db/ledger'; +import { createSandbox, findById, transition } from './db/ledger'; import { FakeExecutor } from './executor/fake'; import { KeyedQueue } from './keyed-queue'; import { ARCHIVE_DEFAULT_SECONDS } from './policy'; @@ -921,6 +921,31 @@ describe('POST /listSandboxes', () => { ); expect(states).toEqual({ alice: 'frozen', bob: 'active' }); }); + + it("answers this node's id for every row, including one born under the id the node had before it was renamed", async () => { + const { app, db } = testApp(); + await acquire(app, { name: 'today' }); + createSandbox(db, { + id: 'aaaaaaaa-0000-4000-8000-000000000001', + name: 'from-before', + nodeId: 'node-1', + policy: DEFAULT_LIFECYCLE_POLICY, + }); + const res = await rpc(app, '/listSandboxes'); + expect(res.statusCode).toBe(200); + const ids = res + .json() + .sandboxes.map((s: { name: string; nodeId: string }) => [ + s.name, + s.nodeId, + ]); + expect(Object.fromEntries(ids)).toEqual({ + today: 'node-test', + 'from-before': 'node-test', + }); + const one = await acquire(app, { name: 'from-before' }); + expect(one.json().sandbox.nodeId).toBe('node-test'); + }); }); describe('POST /rebuildSandbox', () => { diff --git a/packages/server/src/routes/sandboxes.ts b/packages/server/src/routes/sandboxes.ts index 1fda4e7d..caa0edc2 100644 --- a/packages/server/src/routes/sandboxes.ts +++ b/packages/server/src/routes/sandboxes.ts @@ -96,22 +96,34 @@ type AcquireOutcome = | { status: 'ready'; created: boolean; row: SandboxRow } | { status: 'restoring'; row: SandboxRow; progress: RestoreProgress }; +/** The daemon answering: its node id and where it is dialled — the same for every row of its ledger. */ +interface Self { + nodeId: string; + endpoint: string; +} + /** - * Ledger row -> wire shape: nest the flat policy columns, attach the - * endpoint, resolve the spec (NULL knobs collapse onto the global defaults + * Ledger row -> wire shape: nest the flat policy columns, attach who + * answers, resolve the spec (NULL knobs collapse onto the global defaults * — the wire reports the values in force, never a two-source riddle). + * `nodeId` is this daemon's DORMICE_NODE_ID, not the row's column: every + * row of a ledger is this node's, and the column is the id the node had + * when the row was born — a node renamed since (the test machine's, from + * `node-1` to its hostname) would otherwise list its old sandboxes under a + * node that no longer exists, and a fleet's merged list would show a + * ghost node beside the real one (found by review, 2026-09-16). */ function toSandbox( row: SandboxRow, - endpoint: string, + self: Self, defaults: SandboxResourceDefaults, ): Sandbox { return { id: row.id, name: row.name, state: row.state, - nodeId: row.nodeId, - endpoint, + nodeId: self.nodeId, + endpoint: self.endpoint, policy: { freezeAfterSeconds: row.freezeAfterSeconds, stopAfterSeconds: row.stopAfterSeconds, @@ -153,14 +165,18 @@ function serializeMetadata( export const sandboxRoutes: FastifyPluginAsyncZod< SandboxRoutesOptions > = async (app, { config, db, executor, locks, watchers, archiver }) => { - // Every sandbox lives on this daemon today, so the endpoint is our own - // address; with sharding it may point at another node. - const endpoint = `http://127.0.0.1:${config.DORMICE_PORT}`; + // Who answers for every row of this ledger: this node, at its own + // loopback address (a caller through the gateway keeps using the + // gateway's address; the field's honest limits are in shared sandbox.ts). + const self: Self = { + nodeId: config.DORMICE_NODE_ID, + endpoint: `http://127.0.0.1:${config.DORMICE_PORT}`, + }; // The single-row view: defaults read live so a console edit shows in the // very next response. List responses read the defaults once instead. const view = (row: SandboxRow) => - toSandbox(row, endpoint, readRuntimeSettings(db).sandboxDefaults); + toSandbox(row, self, readRuntimeSettings(db).sandboxDefaults); // Both verbs run inside the key's queue slot (see KeyedQueue): each is a // check followed by an act with seconds of executor work in between, and @@ -368,7 +384,7 @@ export const sandboxRoutes: FastifyPluginAsyncZod< const defaults = readRuntimeSettings(db).sandboxDefaults; return { sandboxes: listSandboxes(db).map((row) => - toSandbox(row, endpoint, defaults), + toSandbox(row, self, defaults), ), }; }, diff --git a/packages/shared/src/sandbox.ts b/packages/shared/src/sandbox.ts index b017a10b..f05de99b 100644 --- a/packages/shared/src/sandbox.ts +++ b/packages/shared/src/sandbox.ts @@ -110,7 +110,7 @@ export const sandboxSchema = z.object({ id: z.string(), name: sandboxNameSchema, state: z.enum(SANDBOX_STATES), - /** Machine that owns this sandbox. Single-machine today; the field keeps the ledger shardable. */ + /** The node that runs this sandbox — the DORMICE_NODE_ID of the daemon answering (listNodes at the gateway lists them), never a stamp from the row's birth. */ nodeId: z.string(), /** * Base URL of the daemon that owns this sandbox. Honest limits today: it From 5ab997f97c9798f28b82c86f7ff98380b5a8d066 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 18:53:45 +0800 Subject: [PATCH 85/89] =?UTF-8?q?A=20node=20told=20to=20upgrade=20owes=20t?= =?UTF-8?q?he=20launch=20until=20it=20has=20made=20one:=20refused=20becaus?= =?UTF-8?q?e=20an=20upgrade=20unit=20is=20already=20running=20on=20the=20m?= =?UTF-8?q?achine=20=E2=80=94=20its=20previous=20upgrade=20still=20running?= =?UTF-8?q?=20doctor=20when=20the=20restarted=20daemon's=20first=20check-i?= =?UTF-8?q?n=20is=20answered=20with=20the=20next=20tell=20=E2=80=94=20it?= =?UTF-8?q?=20tries=20again=20at=20the=20next=20check-in=20instead=20of=20?= =?UTF-8?q?dropping=20the=20tell=20with=20a=20warning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on the test machine 2026-09-16: an eight-second window between the daemon's first check-in on the new build and the end of the installer that brought it, and the gateway tells once — so a fix pushed on the heels of a fleet upgrade left the node reading upgrading for twenty minutes and then stuck, with the whole roll waiting behind it. Any other refusal (one-click unavailable, systemd-run failing) still drops the debt with the warning: trying again would change nothing there. --- packages/server/src/check-in.test.ts | 71 ++++++++++++++++++++++++--- packages/server/src/check-in.ts | 73 ++++++++++++++++++++++------ 2 files changed, 123 insertions(+), 21 deletions(-) diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index 66a70865..0171a638 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -405,7 +405,12 @@ describe('CheckIn', () => { }), applyUpgrade: async () => { launches += 1; - if (launches === 2) throw new Error('an upgrade is already running'); + if (launches === 2) { + throw Object.assign( + new Error('failed to launch the upgrade: launch failed'), + { statusCode: 500 }, + ); + } }, }); applyNodeConfig(opts.db, testBundle({}, 1)); @@ -423,16 +428,17 @@ describe('CheckIn', () => { expect.stringMatching(/this node's turn to upgrade has come/), ]); expect(warns).toEqual([]); - // Told again while one runs: the launch's 409 is a warning; the - // check-in itself succeeded. + // Told again while the launch fails for a reason that will not pass + // (systemd-run itself failing, a 500): a warning; the check-in itself + // succeeded, and the debt is dropped — the next tick, not told, + // launches nothing. await checkIn.once(); expect(launches).toBe(2); expect(warns).toEqual([expect.stringMatching(/could not be launched/)]); - expect((details[0] as { error: string }).error).toMatch(/already running/); - // Not told: nothing launched, and a check-in without an updater - // wired reports no selfUpgrade at all. + expect((details[0] as { error: string }).error).toMatch(/launch failed/); await checkIn.once(); expect(launches).toBe(2); + // A check-in without an updater wired reports no selfUpgrade at all. const bare = new CheckIn(options(gw.endpoint, logSpy().log)); await bare.once(); expect( @@ -440,6 +446,59 @@ describe('CheckIn', () => { ).toBeUndefined(); }); + it("a tell whose launch is refused because an upgrade unit is still running is owed: said once, tried again at every check-in, launched once the unit has ended — the restarted daemon's first check-in is answered with the next tell while its previous install.sh still runs doctor", async () => { + let sent = 0; + const gw = await gateway(() => { + sent += 1; + return { + status: 200, + body: JSON.stringify({ + configVersion: 1, + ...(sent === 1 ? { upgrade: true } : {}), + }), + }; + }); + const { log, warns, infos } = logSpy(); + let unitRunning = true; + let launches = 0; + const opts = options(gw.endpoint, log, { + selfUpgrade: async () => ({ available: true, reason: null }), + applyUpgrade: async () => { + if (unitRunning) { + throw Object.assign( + new Error( + 'an upgrade is already running — wait for it to finish (systemd unit dormice-upgrade)', + ), + { statusCode: 409 }, + ); + } + launches += 1; + }, + }); + applyNodeConfig(opts.db, testBundle({}, 1)); + const checkIn = new CheckIn(opts); + // Told at the first check-in; the unit of the previous upgrade is + // still alive: refused, owed, said once. + await checkIn.once(); + await checkIn.once(); + expect(launches).toBe(0); + expect(warns).toEqual([]); + expect(infos).toEqual([ + expect.stringMatching(/turn to upgrade has come/), + expect.stringMatching(/still running on this node/), + ]); + // The unit ends; the next check-in — not told again, the gateway + // tells once — launches the owed upgrade and says so. + unitRunning = false; + await checkIn.once(); + expect(launches).toBe(1); + expect(infos.at(-1)).toMatch(/owed is launched now/); + // Nothing owed any more: later ticks launch nothing. + await checkIn.once(); + expect(launches).toBe(1); + expect(warns).toEqual([]); + }); + it('ticks on its interval from start() and stops on stop()', async () => { const gw = await gateway(() => answering(1)); const { log } = logSpy(); diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index 301c304a..2ea1d892 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -121,9 +121,56 @@ export class CheckIn { private closing = false; /** The failure the gateway is currently in (its sentence with the numbers blanked, so a 409 that says "3s ago" and then "4s ago" is one failure), or null while it answers. */ private failing: string | null = null; + /** A tell taken and not yet launched (launchOwedUpgrade). Memory of this process: a daemon restarted before it launched forgets the debt, and the gateway reads it stuck twenty minutes after the tell — the honest outcome, put back in line by the operator. */ + private owedUpgrade = false; + /** Whether the "still finishing" line was said for the debt now owed. */ + private waitingSaid = false; constructor(private readonly opts: CheckInOptions) {} + /** + * Launches the upgrade this node owes: the same one-click an operator + * would (the new build's install.sh in a systemd unit, updater.ts). + * Refused because an upgrade unit is already running on this machine + * (the updater's 409 — its previous upgrade finishing, or one an + * operator started by hand), the debt stands and the launch is tried + * again at the next check-in, said once; refused for any other reason + * (one-click unavailable, systemd-run failing), the debt is dropped + * with a warning — nothing here would change by trying again — and the + * gateway lists this node as stuck twenty minutes after its tell, where + * applyUpgrade {nodeId} puts it back in line. Never this tick's + * failure: the check-in itself succeeded. + */ + private async launchOwedUpgrade(): Promise { + const { opts } = this; + try { + await (opts.applyUpgrade ?? unavailableUpgrade)(); + if (this.waitingSaid) { + opts.log.info( + 'the upgrade this node owed is launched now that the previous upgrade unit has ended', + ); + } + this.owedUpgrade = false; + this.waitingSaid = false; + } catch (error) { + if ((error as { statusCode?: unknown }).statusCode === 409) { + if (!this.waitingSaid) { + this.waitingSaid = true; + opts.log.info( + 'an upgrade unit is still running on this node (its previous upgrade finishing); the upgrade the gateway asked for is launched at a later check-in, once it has ended', + ); + } + return; + } + this.owedUpgrade = false; + this.waitingSaid = false; + opts.log.warn( + { error: describe(error) }, + 'the upgrade the gateway asked for could not be launched; the gateway lists this node as stuck once twenty minutes have passed, and applyUpgrade {nodeId} there puts it back in line', + ); + } + } + start(): void { this.schedule(0); } @@ -197,25 +244,21 @@ export class CheckIn { } } if (answer.upgrade === true) { - // The gateway's turn for this node in the fleet upgrade: run the - // same one-click upgrade an operator would (the new build's - // install.sh in a systemd unit, updater.ts). Said as its own line, not this tick's - // failure: the check-in itself succeeded, and the gateway tells a - // node once — a launch that fails here is the operator's to read - // (the gateway shows the node as stuck twenty minutes on, and - // applyUpgrade {nodeId} at the gateway puts it back in line). + // The gateway's turn for this node in the fleet upgrade: the node + // owes an upgrade from here until it has launched one (below). + // Owed, not launched on the spot: the gateway tells a node once, + // and the launch may be refused right now for a reason that is + // this node's own and passes by itself — its previous upgrade's + // unit still running the installer's last step (doctor) when the + // restarted daemon's first check-in is already answered with the + // next tell (measured 2026-09-16: an eight-second window, the + // whole roll stuck on it). opts.log.info( `the gateway says this node's turn to upgrade has come — launching the new build's install.sh (systemd unit dormice-upgrade)`, ); - try { - await (opts.applyUpgrade ?? unavailableUpgrade)(); - } catch (error) { - opts.log.warn( - { error: describe(error) }, - 'the upgrade the gateway asked for could not be launched; the gateway lists this node as stuck once twenty minutes have passed, and applyUpgrade {nodeId} there puts it back in line', - ); - } + this.owedUpgrade = true; } + if (this.owedUpgrade) await this.launchOwedUpgrade(); } catch (error) { const message = describe(error); const failure = message.replace(/\d+/g, '#'); From 8d309fd2a8677313f7f0e61fe750601c26f902fe Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 18:53:45 +0800 Subject: [PATCH 86/89] upgrading.mdx says a node told while its previous upgrade is still finishing launches the next one at a later check-in --- website/content/docs/upgrading.mdx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/website/content/docs/upgrading.mdx b/website/content/docs/upgrading.mdx index 6ad0eb0e..972e21ec 100644 --- a/website/content/docs/upgrading.mdx +++ b/website/content/docs/upgrading.mdx @@ -57,7 +57,9 @@ alone — a node whose build keeps failing must not rebuild every twenty minutes on the CPU its sandboxes run on. (A node that comes back on another build has done what it was told, even if the gateway was upgraded again meanwhile: it is judged afresh, and told again at its -turn if it is behind once more.) Read `journalctl -u +turn if it is behind once more. Told while its previous upgrade is +still finishing — the installer's last step is `dor doctor` — it +launches the next one at a later check-in, once that has ended.) Read `journalctl -u dormice-upgrade` and `/var/lib/dormice/upgrade/upgrade.log` on that node, fix the cause, and press **Try again** on the version page (or `POST /applyUpgrade {"nodeId": "..."}`), which puts that node back in From c78f059f4ea5f6215183d7bebda32700d9d6e8fd Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 20:14:51 +0800 Subject: [PATCH 87/89] =?UTF-8?q?A=20node's=20tell=20is=20one=20value=20on?= =?UTF-8?q?=20the=20gateway=20=E2=80=94=20when,=20and=20on=20what=20commit?= =?UTF-8?q?=20=E2=80=94=20not=20two=20nullable=20fields=20read=20together:?= =?UTF-8?q?=20fleet.ts=20reads=20the=20row's=20two=20columns=20into=20`upg?= =?UTF-8?q?radeTold`=20or=20null,=20and=20rolling.ts's=20verdicts=20have?= =?UTF-8?q?=20no=20branch=20for=20a=20tell=20recorded=20without=20its=20co?= =?UTF-8?q?mmit=20(a=20row=20from=20before=20migration=200006=20holding=20?= =?UTF-8?q?one=20reads=20as=20no=20tell);=20a=20tell=20that=20reads=20curr?= =?UTF-8?q?ent=20or=20ahead=20while=20the=20node=20is=20still=20on=20the?= =?UTF-8?q?=20commit=20it=20was=20told=20on=20=E2=80=94=20the=20gateway=20?= =?UTF-8?q?went=20back=20to=20an=20older=20build=20=E2=80=94=20is=20forgot?= =?UTF-8?q?ten=20too,=20so=20the=20node=20is=20not=20read=20stuck=20the=20?= =?UTF-8?q?day=20the=20gateway=20passes=20it=20again?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/gateway/src/db/schema.ts | 4 +- packages/gateway/src/fleet.ts | 45 ++++++++++---- packages/gateway/src/rolling.test.ts | 69 ++++++++++++++------- packages/gateway/src/rolling.ts | 50 +++++++++------ packages/gateway/src/routes/upgrade.test.ts | 5 +- 5 files changed, 116 insertions(+), 57 deletions(-) diff --git a/packages/gateway/src/db/schema.ts b/packages/gateway/src/db/schema.ts index b2d39f60..dbbd9dca 100644 --- a/packages/gateway/src/db/schema.ts +++ b/packages/gateway/src/db/schema.ts @@ -78,8 +78,8 @@ export const nodes = sqliteTable('nodes', { * against: a node reporting any other commit did what it was told, * whether or not that commit is the gateway's by now (the gateway may * have upgraded again meanwhile). Null beside a tell only on a row - * written before this column existed; such a tell stands until the - * node reads current or ahead. + * written before this column existed; fleet.ts reads such a tell as + * none. */ upgradeToldBuild: text('upgrade_told_build'), }); diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index 75e49a32..e9f49fc2 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -44,16 +44,25 @@ export interface NodeState { reading: NodeReading | null; /** Whether the node can upgrade itself, its own word (shared checkInRequestSchema.selfUpgrade); null = it did not say. */ selfUpgrade: SelfUpgrade | null; - /** When the fleet upgrade last told this node to upgrade (rolling.ts); null = never, or fulfilled. */ - upgradeToldAt: Date | null; - /** The commit the node ran when it was told — the tell is fulfilled the moment it reports another (rolling.ts); null with a tell = a row written before this was recorded. */ - upgradeToldBuild: string | null; + /** + * The fleet upgrade's tell on this node (rolling.ts): when it was told, + * and the commit it ran at that moment — one value, since a tell is + * judged by the two together (fulfilled the moment the node reports + * another commit). Null = never told, or the last tell was fulfilled. + */ + upgradeTold: Tell | null; placedSinceCheckIn: number; placedIds: Set; } export type SelfUpgrade = NonNullable; +/** A tell as the row holds it: nodes.upgrade_told_at and nodes.upgrade_told_build. */ +export interface Tell { + at: Date; + build: string; +} + const selfUpgradeSchema = z.object({ available: z.boolean(), reason: z.string().nullable(), @@ -252,13 +261,25 @@ export class Fleet { row.selfUpgrade, selfUpgradeSchema, ), - upgradeToldAt: this.parseDate(row.upgradeToldAt), - upgradeToldBuild: row.upgradeToldBuild, + upgradeTold: this.parseTell(row), placedSinceCheckIn: 0, placedIds: new Set(), }; } + /** + * The tell is its two columns together; a row holding one without the + * other is read as no tell. That is a row written before + * upgrade_told_build existed (migration 0006, 2026-09-16) whose tell + * was in flight at the upgrade — none exists outside the test + * machine's history, and the cost would be one node told again. + */ + private parseTell(row: NodeRow): Tell | null { + const at = this.parseDate(row.upgradeToldAt); + if (at === null || row.upgradeToldBuild === null) return null; + return { at, build: row.upgradeToldBuild }; + } + private parseDate(iso: string | null): Date | null { if (iso === null) return null; const date = new Date(iso); @@ -351,8 +372,7 @@ export class Fleet { build: null, reading: null, selfUpgrade: null, - upgradeToldAt: null, - upgradeToldBuild: null, + upgradeTold: null, placedSinceCheckIn: 0, placedIds: new Set(), }; @@ -398,19 +418,18 @@ export class Fleet { * promises not to do; a write that fails fails the check-in, and the * node is told at the next. */ - setUpgradeTold(id: string, told: { at: Date; build: string } | null): void { + setUpgradeTold(id: string, tell: Tell | null): void { const node = this.members.get(id); if (node === undefined) return; this.db .update(nodes) .set({ - upgradeToldAt: told === null ? null : told.at.toISOString(), - upgradeToldBuild: told === null ? null : told.build, + upgradeToldAt: tell === null ? null : tell.at.toISOString(), + upgradeToldBuild: tell === null ? null : tell.build, }) .where(eq(nodes.id, id)) .run(); - node.upgradeToldAt = told === null ? null : told.at; - node.upgradeToldBuild = told === null ? null : told.build; + node.upgradeTold = tell; } /** diff --git a/packages/gateway/src/rolling.test.ts b/packages/gateway/src/rolling.test.ts index e87a1ba8..2087e6c0 100644 --- a/packages/gateway/src/rolling.test.ts +++ b/packages/gateway/src/rolling.test.ts @@ -220,7 +220,7 @@ describe('Rolling', () => { const a = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); const b = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }); expect(rolling.onCheckIn(a, NOW)).toBe(true); - expect(a.upgradeToldAt).toEqual(NOW); + expect(a.upgradeTold).toEqual({ at: NOW, build: OLD.commit }); expect(rolling.onCheckIn(b, NOW)).toBe(false); // a's next check-in, still old: not told again. const later = new Date(NOW.getTime() + 15_000); @@ -228,7 +228,10 @@ describe('Rolling', () => { expect(rolling.onCheckIn(a, later)).toBe(false); // The row remembers across a restart. const restarted = new Fleet(db); - expect(restarted.get('a')?.upgradeToldAt).toEqual(NOW); + expect(restarted.get('a')?.upgradeTold).toEqual({ + at: NOW, + build: OLD.commit, + }); expect(new Rolling(restarted, GATEWAY).states(later)).toMatchObject([ { id: 'a', state: 'upgrading', toldAt: NOW.toISOString() }, { id: 'b', state: 'behind', toldAt: null }, @@ -237,8 +240,8 @@ describe('Rolling', () => { // and b's turn comes at its next check-in. reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: CAN }, later); expect(rolling.onCheckIn(a, later)).toBe(false); - expect(a.upgradeToldAt).toBeNull(); - expect(new Fleet(db).get('a')?.upgradeToldAt).toBeNull(); + expect(a.upgradeTold).toBeNull(); + expect(new Fleet(db).get('a')?.upgradeTold).toBeNull(); expect(rolling.onCheckIn(b, later)).toBe(true); }); @@ -261,7 +264,7 @@ describe('Rolling', () => { /is upgrading \(told 0s ago, still on old0001\)/, ), }); - expect(a.upgradeToldAt).toEqual(NOW); + expect(a.upgradeTold).toEqual({ at: NOW, build: OLD.commit }); // Twenty minutes on, a is stuck and b's turn came. const late = new Date(NOW.getTime() + UPGRADE_TOLD_TIMEOUT_MS); reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }, late); @@ -270,15 +273,15 @@ describe('Rolling', () => { expect(rolling.onCheckIn(b, late)).toBe(true); // The hand: a's tell is forgotten on the row, and a reads behind. expect(rolling.unstick(a, late)).toBeNull(); - expect(a.upgradeToldAt).toBeNull(); - expect(new Fleet(db).get('a')?.upgradeToldAt).toBeNull(); + expect(a.upgradeTold).toBeNull(); + expect(new Fleet(db).get('a')?.upgradeTold).toBeNull(); expect(upgradeStateOf(a, GATEWAY, late).state).toBe('behind'); // Not past b: a waits while b upgrades, and is told once b is back. expect(rolling.onCheckIn(a, late)).toBe(false); reporting(fleet, 'b', { build: GATEWAY, selfUpgrade: CAN }, late); expect(rolling.onCheckIn(b, late)).toBe(false); expect(rolling.onCheckIn(a, late)).toBe(true); - expect(a.upgradeToldAt).toEqual(late); + expect(a.upgradeTold).toEqual({ at: late, build: OLD.commit }); const current = reporting( fleet, @@ -322,8 +325,11 @@ describe('Rolling', () => { const a = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); const b = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }); expect(first.onCheckIn(a, NOW)).toBe(true); - expect(a.upgradeToldBuild).toBe(OLD.commit); - expect(new Fleet(db).get('a')?.upgradeToldBuild).toBe(OLD.commit); + expect(a.upgradeTold).toEqual({ at: NOW, build: OLD.commit }); + expect(new Fleet(db).get('a')?.upgradeTold).toEqual({ + at: NOW, + build: OLD.commit, + }); const rolling = new Rolling(fleet, NEWER); const back = new Date(NOW.getTime() + 70_000); // Its tell stands while it still reports the commit it was told on. @@ -341,8 +347,7 @@ describe('Rolling', () => { reason: null, }); expect(rolling.onCheckIn(a, back)).toBe(true); - expect(a.upgradeToldAt).toEqual(back); - expect(a.upgradeToldBuild).toBe(GATEWAY.commit); + expect(a.upgradeTold).toEqual({ at: back, build: GATEWAY.commit }); expect(rolling.states(back)).toMatchObject([ { id: 'a', state: 'upgrading', toldAt: back.toISOString() }, { id: 'b', state: 'behind', toldAt: null }, @@ -355,8 +360,9 @@ describe('Rolling', () => { state: 'stuck', reason: expect.stringMatching(/still on new0001 20 minutes later/), }); - // A tell from a row written before the commit was recorded stands - // until the node reads current or ahead, as every tell once did. + // A row from before the commit was recorded (migration 0006) holding + // a tell without one is read as no tell: the node stands as its + // build says. reporting(fleet, 'c', { build: OLD, selfUpgrade: CAN }, late); fleet.setUpgradeTold('c', { at: late, build: OLD.commit }); db.update(nodes) @@ -365,11 +371,32 @@ describe('Rolling', () => { .run(); const cOld = new Fleet(db).get('c'); if (!cOld) throw new Error('row lost'); - expect(cOld.upgradeToldBuild).toBeNull(); - cOld.build = GATEWAY; - expect(upgradeStateOf(cOld, NEWER, late).state).toBe('upgrading'); - cOld.build = NEWER; - expect(upgradeStateOf(cOld, NEWER, late).state).toBe('current'); + expect(cOld.upgradeTold).toBeNull(); + expect(upgradeStateOf(cOld, NEWER, late).state).toBe('behind'); + }); + + it('a told node that reads current or ahead while still on the commit it was told on — the gateway went back to an older build — has its tell forgotten, so it is not read stuck the day the gateway passes it again', () => { + const { fleet } = fleetOver(); + const a = reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: CAN }); + const first = new Rolling(fleet, NEWER); + expect(first.onCheckIn(a, NOW)).toBe(true); + // The gateway's machine is put back on GATEWAY by hand: a reads + // current, still on the commit it was told on, and the tell goes. + const back = new Rolling(fleet, GATEWAY); + const later = new Date(NOW.getTime() + 60_000); + reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: CAN }, later); + expect(back.onCheckIn(a, later)).toBe(false); + expect(a.upgradeTold).toBeNull(); + // Further back, to OLD: a reads ahead, and a tell would go the same way. + fleet.setUpgradeTold('a', { at: NOW, build: GATEWAY.commit }); + expect(new Rolling(fleet, OLD).onCheckIn(a, later)).toBe(false); + expect(a.upgradeTold).toBeNull(); + // Twenty minutes on, the gateway is on NEWER again: a is behind and + // told afresh — not stuck on a tell from before the detour. + const late = new Date(NOW.getTime() + UPGRADE_TOLD_TIMEOUT_MS + 60_000); + reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: CAN }, late); + expect(upgradeStateOf(a, NEWER, late).state).toBe('behind'); + expect(first.onCheckIn(a, late)).toBe(true); }); it('a told node that comes back newer than the gateway is ahead: its tell is fulfilled and cleared, it is not told again, the hand refuses it, and it holds nobody', () => { @@ -382,8 +409,8 @@ describe('Rolling', () => { const later = new Date(NOW.getTime() + 120_000); reporting(fleet, 'a', { build: NEWER, selfUpgrade: CAN }, later); expect(rolling.onCheckIn(a, later)).toBe(false); - expect(a.upgradeToldAt).toBeNull(); - expect(new Fleet(db).get('a')?.upgradeToldAt).toBeNull(); + expect(a.upgradeTold).toBeNull(); + expect(new Fleet(db).get('a')?.upgradeTold).toBeNull(); expect(rolling.states(later)).toMatchObject([ { id: 'a', state: 'ahead', toldAt: null }, ]); diff --git a/packages/gateway/src/rolling.ts b/packages/gateway/src/rolling.ts index b3123324..6c0d1d38 100644 --- a/packages/gateway/src/rolling.ts +++ b/packages/gateway/src/rolling.ts @@ -3,7 +3,7 @@ import type { NodeUpgradeState, NodeUpgradeView, } from '@dormice/shared'; -import { downReason, type Fleet, type NodeState } from './fleet'; +import { downReason, type Fleet, type NodeState, type Tell } from './fleet'; /** * The fleet upgrade, rolled over the nodes by their check-ins (design @@ -63,6 +63,22 @@ import { downReason, type Fleet, type NodeState } from './fleet'; /** How long a told node has to come back on the new build before it is stuck: a pull, a build and a restart take a few minutes; twenty is a build that failed. */ export const UPGRADE_TOLD_TIMEOUT_MS = 20 * 60_000; +/** + * Whether the node's tell stands: it was told, and still reports the + * commit it was told on. Reporting another, it has done what it was told + * (the module comment has why that, and not the gateway's commit, is the + * measure) and the tell is fulfilled. + */ +function tellStands( + node: NodeState, +): node is NodeState & { upgradeTold: Tell; build: BuildInfo } { + return ( + node.upgradeTold !== null && + node.build !== null && + node.build.commit === node.upgradeTold.build + ); +} + /** * One node's standing against the gateway's build, from its last * check-in (shared upgrade.ts nodeUpgradeViewSchema has each state's @@ -122,17 +138,9 @@ export function upgradeStateOf( // its daemon restarts near the end of install.sh and misses a check-in // or two by design, and were that silence read as "unreachable" the // one-at-a-time rule would see nobody upgrading and tell the next node - // into the same minute. The silence is said in the reason instead. The - // tell stands while the node still reports the commit it was told on - // (the module comment has why that, and not the gateway's commit, is - // the measure); a tell recorded without one — a row from before the - // column — stands until the node reads current or ahead, above. - if ( - node.upgradeToldAt !== null && - (node.upgradeToldBuild === null || - node.upgradeToldBuild === node.build.commit) - ) { - const sinceMs = now.getTime() - node.upgradeToldAt.getTime(); + // into the same minute. The silence is said in the reason instead. + if (tellStands(node)) { + const sinceMs = now.getTime() - node.upgradeTold.at.getTime(); const silence = down === null ? '' : ` (${down})`; if (sinceMs < UPGRADE_TOLD_TIMEOUT_MS) { return { @@ -142,7 +150,7 @@ export function upgradeStateOf( } return { state: 'stuck', - reason: `told to upgrade at ${node.upgradeToldAt.toISOString()} and still on ${node.build.commit} ${Math.round(sinceMs / 60_000)} minutes later${silence} — read journalctl -u dormice-upgrade and the upgrade log on the node, then put it back in line (applyUpgrade with its nodeId): it is told again at its turn`, + reason: `told to upgrade at ${node.upgradeTold.at.toISOString()} and still on ${node.build.commit} ${Math.round(sinceMs / 60_000)} minutes later${silence} — read journalctl -u dormice-upgrade and the upgrade log on the node, then put it back in line (applyUpgrade with its nodeId): it is told again at its turn`, }; } if (down !== null) { @@ -206,21 +214,23 @@ export class Rolling { */ onCheckIn(node: NodeState, now: Date): boolean { const { state } = upgradeStateOf(node, this.gatewayBuild, now); + // Fulfilled: the node is off the commit it was told on. Or moot: it + // reads current or ahead while still on it — the gateway went back + // to an older build (install.sh by hand on its machine), and a tell + // kept through that would read the node stuck the day the gateway + // passed it again. if ( - node.upgradeToldAt !== null && - state !== 'upgrading' && - state !== 'stuck' + node.upgradeTold !== null && + (!tellStands(node) || state === 'current' || state === 'ahead') ) { this.fleet.setUpgradeTold(node.id, null); } if ( - state !== 'behind' || + node.build === null || !rollingDecision(this.fleet.all(), this.gatewayBuild, node, now) ) { return false; } - // Behind is judged only of a node with a build (upgradeStateOf). - if (node.build === null) return false; this.fleet.setUpgradeTold(node.id, { at: now, build: node.build.commit }); return true; } @@ -284,7 +294,7 @@ export class Rolling { id: node.id, build: node.build, state, - toldAt: node.upgradeToldAt?.toISOString() ?? null, + toldAt: node.upgradeTold?.at.toISOString() ?? null, reason, }; }); diff --git a/packages/gateway/src/routes/upgrade.test.ts b/packages/gateway/src/routes/upgrade.test.ts index 0f9a321f..a6744ad7 100644 --- a/packages/gateway/src/routes/upgrade.test.ts +++ b/packages/gateway/src/routes/upgrade.test.ts @@ -106,7 +106,10 @@ describe('the fleet upgrade over the check-in', () => { ).toBe(true); const a = fleet.get('a'); if (!a) throw new Error('node lost'); - a.upgradeToldAt = new Date(Date.now() - UPGRADE_TOLD_TIMEOUT_MS - 1000); + a.upgradeTold = { + at: new Date(Date.now() - UPGRADE_TOLD_TIMEOUT_MS - 1000), + build: OLD.commit, + }; expect( (await checkIn(app, 'a', { build: OLD, selfUpgrade: CAN })).upgrade, ).toBeUndefined(); From 04e1f4d62bf62a73cf920ceed929defead0d9fac Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 20:16:58 +0800 Subject: [PATCH 88/89] =?UTF-8?q?A=20node=20says=20at=20every=20check-in?= =?UTF-8?q?=20whether=20an=20upgrade=20unit=20is=20running=20on=20its=20ma?= =?UTF-8?q?chine,=20and=20the=20gateway=20reads=20such=20a=20node=20as=20u?= =?UTF-8?q?pgrading=20and=20does=20not=20tell=20it=20=E2=80=94=20instead?= =?UTF-8?q?=20of=20the=20node=20taking=20a=20tell=20it=20cannot=20launch?= =?UTF-8?q?=20as=20a=20debt=20and=20retrying:=20the=20eight=20seconds=20of?= =?UTF-8?q?=20doctor=20after=20the=20restarted=20daemon's=20first=20check-?= =?UTF-8?q?in=20are=20a=20fact=20the=20node=20knows=20(systemctl=20is-acti?= =?UTF-8?q?ve=20dormice-upgrade,=20what=20its=20own=20409=20already=20aske?= =?UTF-8?q?d),=20so=20the=20node=20states=20it=20and=20the=20gateway=20wai?= =?UTF-8?q?ts=20it=20out,=20telling=20at=20the=20first=20check-in=20that?= =?UTF-8?q?=20says=20the=20unit=20has=20ended;=20an=20installer=20run=20on?= =?UTF-8?q?=20a=20node=20by=20hand=20shows=20on=20the=20version=20page=20a?= =?UTF-8?q?s=20upgrading=20for=20the=20same=20reason,=20and=20the=20check-?= =?UTF-8?q?in's=20owedUpgrade/waitingSaid=20memory=20goes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/gateway/src/fleet.ts | 13 +-- packages/gateway/src/rolling.test.ts | 63 +++++++++++++- packages/gateway/src/rolling.ts | 42 +++++++++- packages/gateway/src/routes/upgrade.test.ts | 33 +++++++- packages/server/src/check-in.test.ts | 73 +++++----------- packages/server/src/check-in.ts | 93 +++++++-------------- packages/server/src/main.ts | 8 +- packages/server/src/updater.ts | 15 +++- packages/shared/src/gateway.ts | 37 +++++--- packages/shared/src/upgrade.ts | 5 +- website/content/docs/upgrading.mdx | 9 +- 11 files changed, 240 insertions(+), 151 deletions(-) diff --git a/packages/gateway/src/fleet.ts b/packages/gateway/src/fleet.ts index e9f49fc2..52d6cc15 100644 --- a/packages/gateway/src/fleet.ts +++ b/packages/gateway/src/fleet.ts @@ -6,9 +6,11 @@ import { nodeReadingSchema, type SandboxDisks, type SandboxStateCounts, + type SelfUpgrade, + selfUpgradeSchema, } from '@dormice/shared'; import { eq } from 'drizzle-orm'; -import { z } from 'zod'; +import type { z } from 'zod'; import type { Db } from './db/db'; import { type NodeRow, nodes } from './db/schema'; import { bumpConfigVersion } from './db/settings'; @@ -42,7 +44,7 @@ export interface NodeState { intervalSeconds: number | null; build: BuildInfo | null; reading: NodeReading | null; - /** Whether the node can upgrade itself, its own word (shared checkInRequestSchema.selfUpgrade); null = it did not say. */ + /** The node's own word on upgrading itself — can it, and is an upgrade unit running on it now (shared selfUpgradeSchema); null = it did not say. */ selfUpgrade: SelfUpgrade | null; /** * The fleet upgrade's tell on this node (rolling.ts): when it was told, @@ -55,19 +57,12 @@ export interface NodeState { placedIds: Set; } -export type SelfUpgrade = NonNullable; - /** A tell as the row holds it: nodes.upgrade_told_at and nodes.upgrade_told_build. */ export interface Tell { at: Date; build: string; } -const selfUpgradeSchema = z.object({ - available: z.boolean(), - reason: z.string().nullable(), -}); - /** * What the fleet says for itself — a row it could not read back, a row it * could not write. Pino's shape (main.ts passes the gateway's logger); diff --git a/packages/gateway/src/rolling.test.ts b/packages/gateway/src/rolling.test.ts index 2087e6c0..4a6d04ea 100644 --- a/packages/gateway/src/rolling.test.ts +++ b/packages/gateway/src/rolling.test.ts @@ -35,12 +35,15 @@ const NEWER: BuildInfo = { title: 'landed on main mid-roll', committedAt: '2026-09-15T00:05:00.000Z', }; -const CAN = { available: true, reason: null }; +const CAN = { available: true, reason: null, running: false }; const CANNOT = { available: false, reason: 'systemd-run is not available — one-click upgrade needs a systemd host', + running: false, }; +/** A node whose machine has an upgrade unit alive: its previous upgrade finishing, or install.sh by hand. */ +const BUSY = { ...CAN, running: true }; function fleetOver() { const db = openDb(':memory:'); @@ -425,6 +428,64 @@ describe('Rolling', () => { expect(rolling.onCheckIn(b, later)).toBe(true); }); + it('a node that says an upgrade unit is running on it is upgrading, told or not: not told, counted by the one-at-a-time rule, the hand refused, a fulfilled tell on it still forgotten — and told at the first check-in that says the unit has ended', () => { + const { fleet } = fleetOver(); + const rolling = new Rolling(fleet, NEWER); + // a was told on OLD by a gateway then on GATEWAY, pulled and built + // GATEWAY, restarted — and its first check-in back finds the gateway + // on NEWER already, while its own installer still runs doctor. + const a = reporting(fleet, 'a', { build: OLD, selfUpgrade: CAN }); + fleet.setUpgradeTold('a', { at: NOW, build: OLD.commit }); + const back = new Date(NOW.getTime() + 70_000); + reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: BUSY }, back); + expect(upgradeStateOf(a, NEWER, back)).toEqual({ + state: 'upgrading', + reason: expect.stringMatching( + /^an upgrade unit is running on the node \(dormice-upgrade\)/, + ), + }); + expect(rolling.onCheckIn(a, back)).toBe(false); + // The tell it fulfilled is gone from the row, upgrading or not. + expect(a.upgradeTold).toBeNull(); + // b, behind, waits: a's unit is the count. + const b = reporting(fleet, 'b', { build: OLD, selfUpgrade: CAN }, back); + expect(rolling.onCheckIn(b, back)).toBe(false); + expect(rolling.unstick(a, back)).toMatchObject({ + status: 409, + message: expect.stringMatching( + /not on a tell, so nothing to put back in line/, + ), + }); + // The unit ends: a says so at its next check-in, reads behind, and is + // told — on the commit it now runs. + const ended = new Date(back.getTime() + 15_000); + reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: CAN }, ended); + expect(rolling.onCheckIn(a, ended)).toBe(true); + expect(a.upgradeTold).toEqual({ at: ended, build: GATEWAY.commit }); + // Told and its unit alive: upgrading on its tell, the unit said + // beside it; twenty minutes on, stuck — a build that hangs. + const hung = new Date(ended.getTime() + 30_000); + reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: BUSY }, hung); + expect(upgradeStateOf(a, NEWER, hung)).toEqual({ + state: 'upgrading', + reason: + 'told 30s ago, still on new0001 (an upgrade unit is running on it)', + }); + const late = new Date(ended.getTime() + UPGRADE_TOLD_TIMEOUT_MS); + reporting(fleet, 'a', { build: GATEWAY, selfUpgrade: BUSY }, late); + expect(upgradeStateOf(a, NEWER, late)).toMatchObject({ + state: 'stuck', + reason: expect.stringMatching( + /20 minutes later \(an upgrade unit is running on it\)/, + ), + }); + // Quiet for good after saying "running", and no tell to clock it: + // unreachable, not upgrading forever. + fleet.setUpgradeTold('a', null); + a.lastCheckInAt = new Date(late.getTime() - 40_000); + expect(upgradeStateOf(a, NEWER, late).state).toBe('unreachable'); + }); + it('states lists every node in id order with its standing, build and tell', () => { const { fleet } = fleetOver(); const rolling = new Rolling(fleet, GATEWAY); diff --git a/packages/gateway/src/rolling.ts b/packages/gateway/src/rolling.ts index 6c0d1d38..d3475ea3 100644 --- a/packages/gateway/src/rolling.ts +++ b/packages/gateway/src/rolling.ts @@ -50,6 +50,19 @@ import { downReason, type Fleet, type NodeState, type Tell } from './fleet'; * by review, 2026-09-16). Back on another commit it is judged afresh: * current, ahead, or behind and in line for the next tell. * + * A node's own upgrade unit counts as upgrading too, told or not. The + * node reports whether dormice-upgrade is alive on its machine + * (selfUpgrade.running), and one that says so is not told: its previous + * upgrade's installer is finishing — doctor runs on for seconds after + * the daemon it restarted is back — or an operator ran install.sh on it + * by hand. A tell landing in those seconds was refused by the unit's + * mutex on the node and, said once, held the roll until the node read + * stuck (measured 2026-09-16: an eight-second window at the end of every + * upgrade, hit by every fix pushed on the heels of a push). The node + * states the fact, the gateway waits it out and tells at the first + * check-in that says the unit has ended — no memory of a tell on the + * node, and a hand-run upgrade shows on the version page as what it is. + * * Behind means older. A node whose build is newer than the gateway's — a * commit that landed on main after the gateway's machine upgraded and * before this node's turn came, or install.sh run on the node by hand — @@ -141,21 +154,39 @@ export function upgradeStateOf( // into the same minute. The silence is said in the reason instead. if (tellStands(node)) { const sinceMs = now.getTime() - node.upgradeTold.at.getTime(); - const silence = down === null ? '' : ` (${down})`; + const aside = + down !== null + ? ` (${down})` + : node.selfUpgrade?.running === true + ? ' (an upgrade unit is running on it)' + : ''; if (sinceMs < UPGRADE_TOLD_TIMEOUT_MS) { return { state: 'upgrading', - reason: `told ${Math.round(sinceMs / 1000)}s ago, still on ${node.build.commit}${silence}`, + reason: `told ${Math.round(sinceMs / 1000)}s ago, still on ${node.build.commit}${aside}`, }; } return { state: 'stuck', - reason: `told to upgrade at ${node.upgradeTold.at.toISOString()} and still on ${node.build.commit} ${Math.round(sinceMs / 60_000)} minutes later${silence} — read journalctl -u dormice-upgrade and the upgrade log on the node, then put it back in line (applyUpgrade with its nodeId): it is told again at its turn`, + reason: `told to upgrade at ${node.upgradeTold.at.toISOString()} and still on ${node.build.commit} ${Math.round(sinceMs / 60_000)} minutes later${aside} — read journalctl -u dormice-upgrade and the upgrade log on the node, then put it back in line (applyUpgrade with its nodeId): it is told again at its turn`, }; } if (down !== null) { return { state: 'unreachable', reason: down }; } + // An upgrade unit alive on the node's machine is an upgrade in + // progress whoever started it (the module comment has the two ways): + // waited out — not told, and counted by the one-at-a-time rule. Judged + // after the silence, unlike a tell: a tell has the twenty-minute clock + // to bound it and this has none, so a node that said "running" and + // went quiet for good reads unreachable, not upgrading forever. + if (node.selfUpgrade?.running === true) { + return { + state: 'upgrading', + reason: + 'an upgrade unit is running on the node (dormice-upgrade) — its previous upgrade finishing, or install.sh run there by hand; it is told at its turn once that has ended', + }; + } if (node.selfUpgrade === null) { return { state: 'unavailable', @@ -258,7 +289,10 @@ export class Rolling { case 'upgrading': return { status: 409, - message: `node ${node.id} is upgrading (${reason}) — it reads stuck twenty minutes after its tell if it is still on the old build, and can be put back in line from there; wait`, + message: + node.upgradeTold === null + ? `node ${node.id} is upgrading (${reason}) — not on a tell, so nothing to put back in line; it reads behind once the unit has ended and is told at its turn` + : `node ${node.id} is upgrading (${reason}) — it reads stuck twenty minutes after its tell if it is still on the old build, and can be put back in line from there; wait`, }; case 'current': return { diff --git a/packages/gateway/src/routes/upgrade.test.ts b/packages/gateway/src/routes/upgrade.test.ts index a6744ad7..ebaa427c 100644 --- a/packages/gateway/src/routes/upgrade.test.ts +++ b/packages/gateway/src/routes/upgrade.test.ts @@ -27,7 +27,7 @@ const OLD: BuildInfo = { title: 'the old build', committedAt: '2026-09-14T00:00:00.000Z', }; -const CAN = { available: true, reason: null }; +const CAN = { available: true, reason: null, running: false }; function rpc( app: App, @@ -153,6 +153,36 @@ describe('the fleet upgrade over the check-in', () => { ); }); + it('a node that says an upgrade unit is running on it is not told and reads upgrading with the unit as the reason; it is told once it says the unit has ended; a node on a build from before `running` was said is read as not running', async () => { + const { app } = testGateway({}, { build: GATEWAY }); + expect( + ( + await checkIn(app, 'a', { + build: OLD, + selfUpgrade: { ...CAN, running: true }, + }) + ).upgrade, + ).toBeUndefined(); + expect((await status(app)).nodes?.find((n) => n.id === 'a')).toMatchObject({ + state: 'upgrading', + toldAt: null, + reason: expect.stringMatching(/an upgrade unit is running on the node/), + }); + expect( + (await checkIn(app, 'a', { build: OLD, selfUpgrade: CAN })).upgrade, + ).toBe(true); + // Before 2026-09-16 a node's selfUpgrade had no `running`: taken, and + // read as not running. + const old = await rpc(app, '/checkIn', { + ...checkInOf('b', 'http://b:80', { build: OLD }), + selfUpgrade: { available: true, reason: null }, + }); + expect(old.statusCode).toBe(200); + expect((await status(app)).nodes?.find((n) => n.id === 'b')).toMatchObject({ + state: 'behind', + }); + }); + it('nodes that cannot upgrade themselves, did not say, or carry no build are listed with the reason and never told', async () => { const { app } = testGateway({}, { build: GATEWAY }); expect( @@ -162,6 +192,7 @@ describe('the fleet upgrade over the check-in', () => { selfUpgrade: { available: false, reason: 'the process does not run from a git checkout', + running: false, }, }) ).upgrade, diff --git a/packages/server/src/check-in.test.ts b/packages/server/src/check-in.test.ts index 0171a638..a3426714 100644 --- a/packages/server/src/check-in.test.ts +++ b/packages/server/src/check-in.test.ts @@ -384,7 +384,7 @@ describe('CheckIn', () => { ]); }); - it("reports whether this node can upgrade itself, and runs its upgrade when the answer says so — a launch that fails is a warning, not the tick's failure", async () => { + it("reports whether this node can upgrade itself and whether an upgrade unit runs on it, and runs its upgrade when the answer says so — a launch that fails is a warning, not the tick's failure", async () => { let sent = 0; const gw = await gateway(() => { sent += 1; @@ -402,6 +402,7 @@ describe('CheckIn', () => { selfUpgrade: async () => ({ available: false, reason: 'systemd-run is not available', + running: false, }), applyUpgrade: async () => { launches += 1; @@ -419,6 +420,7 @@ describe('CheckIn', () => { expect(checkInRequestSchema.parse(gw.seen[0]?.body).selfUpgrade).toEqual({ available: false, reason: 'systemd-run is not available', + running: false, }); expect(launches).toBe(0); // Told: the upgrade is launched, said as its own line. @@ -428,10 +430,9 @@ describe('CheckIn', () => { expect.stringMatching(/this node's turn to upgrade has come/), ]); expect(warns).toEqual([]); - // Told again while the launch fails for a reason that will not pass - // (systemd-run itself failing, a 500): a warning; the check-in itself - // succeeded, and the debt is dropped — the next tick, not told, - // launches nothing. + // Told again while the launch fails (systemd-run itself failing, a + // 500): a warning; the check-in itself succeeded, and the next tick, + // not told, launches nothing — the gateway tells once. await checkIn.once(); expect(launches).toBe(2); expect(warns).toEqual([expect.stringMatching(/could not be launched/)]); @@ -446,57 +447,23 @@ describe('CheckIn', () => { ).toBeUndefined(); }); - it("a tell whose launch is refused because an upgrade unit is still running is owed: said once, tried again at every check-in, launched once the unit has ended — the restarted daemon's first check-in is answered with the next tell while its previous install.sh still runs doctor", async () => { - let sent = 0; - const gw = await gateway(() => { - sent += 1; - return { - status: 200, - body: JSON.stringify({ - configVersion: 1, - ...(sent === 1 ? { upgrade: true } : {}), - }), - }; - }); - const { log, warns, infos } = logSpy(); - let unitRunning = true; - let launches = 0; + it('says on the wire when an upgrade unit is running on this machine — what keeps the gateway from telling it while its previous upgrade finishes', async () => { + const gw = await gateway(() => answering(1)); + const { log } = logSpy(); const opts = options(gw.endpoint, log, { - selfUpgrade: async () => ({ available: true, reason: null }), - applyUpgrade: async () => { - if (unitRunning) { - throw Object.assign( - new Error( - 'an upgrade is already running — wait for it to finish (systemd unit dormice-upgrade)', - ), - { statusCode: 409 }, - ); - } - launches += 1; - }, + selfUpgrade: async () => ({ + available: true, + reason: null, + running: true, + }), }); applyNodeConfig(opts.db, testBundle({}, 1)); - const checkIn = new CheckIn(opts); - // Told at the first check-in; the unit of the previous upgrade is - // still alive: refused, owed, said once. - await checkIn.once(); - await checkIn.once(); - expect(launches).toBe(0); - expect(warns).toEqual([]); - expect(infos).toEqual([ - expect.stringMatching(/turn to upgrade has come/), - expect.stringMatching(/still running on this node/), - ]); - // The unit ends; the next check-in — not told again, the gateway - // tells once — launches the owed upgrade and says so. - unitRunning = false; - await checkIn.once(); - expect(launches).toBe(1); - expect(infos.at(-1)).toMatch(/owed is launched now/); - // Nothing owed any more: later ticks launch nothing. - await checkIn.once(); - expect(launches).toBe(1); - expect(warns).toEqual([]); + await new CheckIn(opts).once(); + expect(checkInRequestSchema.parse(gw.seen[0]?.body).selfUpgrade).toEqual({ + available: true, + reason: null, + running: true, + }); }); it('ticks on its interval from start() and stops on stop()', async () => { diff --git a/packages/server/src/check-in.ts b/packages/server/src/check-in.ts index 2ea1d892..97e2c531 100644 --- a/packages/server/src/check-in.ts +++ b/packages/server/src/check-in.ts @@ -4,6 +4,7 @@ import { checkInResponseSchema, type NodeConfigBundle, type NodeReading, + type SelfUpgrade, } from '@dormice/shared'; import type { Db } from './db/db'; import { countSandboxesByState } from './db/ledger'; @@ -58,13 +59,16 @@ export interface CheckInOptions { /** Makes a bundle the gateway answered with real on this node (node-config.ts applyConfig). */ applyConfig: (bundle: NodeConfigBundle) => Promise; /** - * Whether this node can upgrade itself when told, and why not (the - * updater's availability, updater.ts) — reported at every check-in so - * the gateway rolls the fleet upgrade over the nodes that can and names - * the rest. Optional for the suites that embed a check-in without an - * updater; the daemon always wires it. + * This node's word on upgrading itself (shared selfUpgradeSchema): + * whether it can when told and why not (the updater's availability), + * and whether an upgrade unit is running on this machine now (the + * updater's running()) — reported at every check-in so the gateway + * rolls the fleet upgrade over the nodes that can, names the rest, and + * does not tell a node whose previous upgrade is still finishing. + * Optional for the suites that embed a check-in without an updater; the + * daemon always wires it. */ - selfUpgrade?: () => Promise<{ available: boolean; reason: string | null }>; + selfUpgrade?: () => Promise; /** Runs this node's own upgrade (updater.apply) when the gateway's answer says `upgrade: true`. */ applyUpgrade?: () => Promise; log: CheckInLog; @@ -121,56 +125,9 @@ export class CheckIn { private closing = false; /** The failure the gateway is currently in (its sentence with the numbers blanked, so a 409 that says "3s ago" and then "4s ago" is one failure), or null while it answers. */ private failing: string | null = null; - /** A tell taken and not yet launched (launchOwedUpgrade). Memory of this process: a daemon restarted before it launched forgets the debt, and the gateway reads it stuck twenty minutes after the tell — the honest outcome, put back in line by the operator. */ - private owedUpgrade = false; - /** Whether the "still finishing" line was said for the debt now owed. */ - private waitingSaid = false; constructor(private readonly opts: CheckInOptions) {} - /** - * Launches the upgrade this node owes: the same one-click an operator - * would (the new build's install.sh in a systemd unit, updater.ts). - * Refused because an upgrade unit is already running on this machine - * (the updater's 409 — its previous upgrade finishing, or one an - * operator started by hand), the debt stands and the launch is tried - * again at the next check-in, said once; refused for any other reason - * (one-click unavailable, systemd-run failing), the debt is dropped - * with a warning — nothing here would change by trying again — and the - * gateway lists this node as stuck twenty minutes after its tell, where - * applyUpgrade {nodeId} puts it back in line. Never this tick's - * failure: the check-in itself succeeded. - */ - private async launchOwedUpgrade(): Promise { - const { opts } = this; - try { - await (opts.applyUpgrade ?? unavailableUpgrade)(); - if (this.waitingSaid) { - opts.log.info( - 'the upgrade this node owed is launched now that the previous upgrade unit has ended', - ); - } - this.owedUpgrade = false; - this.waitingSaid = false; - } catch (error) { - if ((error as { statusCode?: unknown }).statusCode === 409) { - if (!this.waitingSaid) { - this.waitingSaid = true; - opts.log.info( - 'an upgrade unit is still running on this node (its previous upgrade finishing); the upgrade the gateway asked for is launched at a later check-in, once it has ended', - ); - } - return; - } - this.owedUpgrade = false; - this.waitingSaid = false; - opts.log.warn( - { error: describe(error) }, - 'the upgrade the gateway asked for could not be launched; the gateway lists this node as stuck once twenty minutes have passed, and applyUpgrade {nodeId} there puts it back in line', - ); - } - } - start(): void { this.schedule(0); } @@ -244,21 +201,29 @@ export class CheckIn { } } if (answer.upgrade === true) { - // The gateway's turn for this node in the fleet upgrade: the node - // owes an upgrade from here until it has launched one (below). - // Owed, not launched on the spot: the gateway tells a node once, - // and the launch may be refused right now for a reason that is - // this node's own and passes by itself — its previous upgrade's - // unit still running the installer's last step (doctor) when the - // restarted daemon's first check-in is already answered with the - // next tell (measured 2026-09-16: an eight-second window, the - // whole roll stuck on it). + // The gateway's turn for this node in the fleet upgrade: the same + // one-click an operator would (the new build's install.sh in a + // systemd unit, updater.ts). A launch that fails is a warning, + // never this tick's failure — the check-in itself succeeded — and + // the gateway, which tells a node once, lists it as stuck twenty + // minutes on, where applyUpgrade {nodeId} puts it back in line. + // Refused for an upgrade unit already running here is not among + // the reasons by construction: this very check-in reported that + // unit (selfUpgrade.running), and the gateway tells no node that + // is upgrading; only a unit started between the reading and the + // answer gets here, and earns the warning. opts.log.info( `the gateway says this node's turn to upgrade has come — launching the new build's install.sh (systemd unit dormice-upgrade)`, ); - this.owedUpgrade = true; + try { + await (opts.applyUpgrade ?? unavailableUpgrade)(); + } catch (error) { + opts.log.warn( + { error: describe(error) }, + 'the upgrade the gateway asked for could not be launched; the gateway lists this node as stuck once twenty minutes have passed, and applyUpgrade {nodeId} there puts it back in line', + ); + } } - if (this.owedUpgrade) await this.launchOwedUpgrade(); } catch (error) { const message = describe(error); const failure = message.replace(/\d+/g, '#'); diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index 17ff3407..d96a2363 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -246,7 +246,13 @@ const checkIn = new CheckIn({ }), selfUpgrade: async () => { const reason = await updater.availability(); - return { available: reason === null, reason }; + return { + available: reason === null, + reason, + // Asked of systemd at every check-in; spared where nothing could be + // running (no systemd, no checkout, the fake executor). + running: reason === null && (await updater.running()), + }; }, applyUpgrade: () => updater.apply(), log, diff --git a/packages/server/src/updater.ts b/packages/server/src/updater.ts index f669fb68..fe8e64ce 100644 --- a/packages/server/src/updater.ts +++ b/packages/server/src/updater.ts @@ -278,7 +278,7 @@ export class Updater { // than one way ("already exists"; "was already loaded or has a // fragment file" on systemd 255, caught on real hardware) — so ask // systemd whether the unit is alive instead of parsing prose. - if (await this.unitActive()) { + if (await this.running()) { throw httpError( 409, 'an upgrade is already running — wait for it to finish (systemd unit dormice-upgrade)', @@ -301,7 +301,7 @@ export class Updater { */ async status(): Promise { const reason = await this.availability(); - const running = await this.unitActive(); + const running = await this.running(); let last = await this.readRun(); if (last !== null && last.state === 'running' && !running) { last = { @@ -368,7 +368,16 @@ export class Updater { } } - private async unitActive(): Promise { + /** + * Whether an upgrade unit is alive on this machine right now — systemd's + * word, never the status file's. For status(), for the 409 apply() + * answers a double-click, and for the node's check-in, which reports it + * so the gateway does not tell a node whose previous upgrade is still + * finishing: install.sh's last step, doctor, runs on for seconds after + * the daemon it restarted is back (eight, measured 2026-09-16), and a + * tell landing in them met this unit's mutex. + */ + async running(): Promise { const result = await this.run('systemctl', [ 'is-active', '--quiet', diff --git a/packages/shared/src/gateway.ts b/packages/shared/src/gateway.ts index dfbdad53..549ca791 100644 --- a/packages/shared/src/gateway.ts +++ b/packages/shared/src/gateway.ts @@ -118,6 +118,30 @@ export type NodeReading = z.infer; * knows what it runs and where it can be reached; the gateway only listens * and compares, and never keeps a record of what it told whom. */ +/** + * A node's own word on upgrading itself, carried in every check-in. + * `available`: whether it can upgrade itself when told (its updater's + * availability: a git checkout, install.sh, systemd-run; upgrade.ts), and + * `reason` why not when it cannot — the gateway rolls an upgrade only + * over nodes that can, and lists the rest as `unavailable` with the + * reason. `running`: whether an upgrade unit (dormice-upgrade) is alive + * on its machine right now — its previous upgrade's installer finishing + * (doctor runs on for seconds after the daemon it restarted is back), or + * install.sh started there by hand. A node that says so is upgrading, + * whoever started it, and the gateway does not tell it: a tell landing in + * those seconds was refused by the unit's mutex on the node and, said + * once, held the whole roll until the node read stuck (measured + * 2026-09-16). Who knows the truth speaks; the gateway compares. Default + * false: a node on a build before 2026-09-16 does not say. + */ +export const selfUpgradeSchema = z.object({ + available: z.boolean(), + reason: z.string().nullable(), + running: z.boolean().default(false), +}); + +export type SelfUpgrade = z.infer; + export const checkInRequestSchema = z.object({ /** DORMICE_NODE_ID — the node's name in every sandbox's `nodeId`. */ nodeId: z.string().min(1), @@ -135,17 +159,8 @@ export const checkInRequestSchema = z.object({ * the gateway has to remember about who was told what. */ configVersion: z.number().int().nullable(), - /** - * Whether this node can upgrade itself when told (its updater's - * availability: a git checkout, install.sh, systemd-run; upgrade.ts), - * and why not when it cannot. The gateway rolls an upgrade only over - * nodes that can; the rest it lists as `unavailable` with the reason. - * Optional on the wire: a node on a build before the fourth cut does - * not say, and its check-in is taken. - */ - selfUpgrade: z - .object({ available: z.boolean(), reason: z.string().nullable() }) - .optional(), + /** This node's own word on upgrading itself (selfUpgradeSchema). Optional on the wire: a node on a build before the fourth cut does not say, and its check-in is taken. */ + selfUpgrade: selfUpgradeSchema.optional(), }); export type CheckInRequest = z.infer; diff --git a/packages/shared/src/upgrade.ts b/packages/shared/src/upgrade.ts index 07bd0070..05bb4f2f 100644 --- a/packages/shared/src/upgrade.ts +++ b/packages/shared/src/upgrade.ts @@ -169,7 +169,10 @@ export type GetUpgradeStatusRequest = z.infer< * brings it to current * behind an older build, able to upgrade itself, not told yet — its * turn comes when no other node is upgrading - * upgrading told within the last twenty minutes, not back yet + * upgrading told within the last twenty minutes, not back yet — or, + * by its own word, an upgrade unit is running on it (its + * previous upgrade's installer finishing, or install.sh + * run there by hand); not told until that has ended * stuck told, still on the old build twenty minutes on — never * re-told on its own; applyUpgrade {nodeId} puts it back * in line diff --git a/website/content/docs/upgrading.mdx b/website/content/docs/upgrading.mdx index 972e21ec..16ab81b7 100644 --- a/website/content/docs/upgrading.mdx +++ b/website/content/docs/upgrading.mdx @@ -57,9 +57,12 @@ alone — a node whose build keeps failing must not rebuild every twenty minutes on the CPU its sandboxes run on. (A node that comes back on another build has done what it was told, even if the gateway was upgraded again meanwhile: it is judged afresh, and told again at its -turn if it is behind once more. Told while its previous upgrade is -still finishing — the installer's last step is `dor doctor` — it -launches the next one at a later check-in, once that has ended.) Read `journalctl -u +turn if it is behind once more. While its previous upgrade is still +finishing — the installer's last step is `dor doctor`, which runs on for +a few seconds after the daemon is back — the node reports the running +unit and is shown as upgrading, not told; the tell comes at its first +check-in after the unit has ended. An installer you run on a node by +hand shows the same way.) Read `journalctl -u dormice-upgrade` and `/var/lib/dormice/upgrade/upgrade.log` on that node, fix the cause, and press **Try again** on the version page (or `POST /applyUpgrade {"nodeId": "..."}`), which puts that node back in From afcbda4e693eaed451f8f501e1528f951fa85536 Mon Sep 17 00:00:00 2001 From: Annactswell Date: Wed, 16 Sep 2026 20:29:03 +0800 Subject: [PATCH 89/89] =?UTF-8?q?The=20updater=20asks=20systemd=20whether?= =?UTF-8?q?=20the=20upgrade=20unit=20is=20alive=20with=20list-units,=20not?= =?UTF-8?q?=20is-active:=20is-active=20makes=20systemd=20load=20the=20unit?= =?UTF-8?q?=20by=20name,=20and=20for=20a=20transient=20unit=20that=20has?= =?UTF-8?q?=20ended=20and=20been=20collected=20that=20reopens=20its=20frag?= =?UTF-8?q?ment=20under=20/run/systemd/transient=20=E2=80=94=20gone=20?= =?UTF-8?q?=E2=80=94=20and=20logs=20a=20notice,=20two=20lines=20per=20ask?= =?UTF-8?q?=20(systemd=20255,=20caught=20on=20the=20test=20machine=20the?= =?UTF-8?q?=20first=20quarter-hour=20the=20node=20reported=20`running`=20a?= =?UTF-8?q?t=20every=20check-in:=20eleven=20thousand=20journal=20lines=20a?= =?UTF-8?q?=20day=20per=20node);=20list-units=20only=20lists=20what=20syst?= =?UTF-8?q?emd=20holds=20in=20memory=20and=20loads=20nothing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/server/src/updater.test.ts | 41 ++++++++++++++++------------- packages/server/src/updater.ts | 17 +++++++++--- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/packages/server/src/updater.test.ts b/packages/server/src/updater.test.ts index b747dad0..0361a3ef 100644 --- a/packages/server/src/updater.test.ts +++ b/packages/server/src/updater.test.ts @@ -333,16 +333,24 @@ describe('Updater.apply and status', () => { // The refusal wording is systemd 255's, verbatim from the real machine // — the adjudication must not depend on it: the unit's liveness is // what makes this a "someone is already upgrading", not the prose. + // Liveness is systemctl list-units listing the unit (its real line). const updater = updaterFor({ run: async (file, args) => - file === 'systemctl' || args[0] === '--version' - ? { exitCode: 0, stdout: '', stderr: '' } - : { - exitCode: 1, - stdout: '', - stderr: - 'Failed to start transient service unit: Unit dormice-upgrade.service was already loaded or has a fragment file.', - }, + file === 'systemctl' + ? { + exitCode: 0, + stdout: + 'dormice-upgrade.service loaded active running Dormice upgrade (install.sh)\n', + stderr: '', + } + : args[0] === '--version' + ? { exitCode: 0, stdout: '', stderr: '' } + : { + exitCode: 1, + stdout: '', + stderr: + 'Failed to start transient service unit: Unit dormice-upgrade.service was already loaded or has a fragment file.', + }, }); await expect(updater.apply()).rejects.toMatchObject({ statusCode: 409 }); }); @@ -353,7 +361,8 @@ describe('Updater.apply and status', () => { args[0] === '--version' ? { exitCode: 0, stdout: '', stderr: '' } : file === 'systemctl' - ? { exitCode: 3, stdout: '', stderr: '' } + ? // list-units lists nothing: the unit is not in memory. + { exitCode: 0, stdout: '', stderr: '' } : { exitCode: 1, stdout: '', @@ -380,14 +389,11 @@ describe('Updater.apply and status', () => { }), ); writeFileSync(path.join(statusDir, 'upgrade.log'), '==> build\nboom\n'); - // systemd-run answers the availability probe; systemctl says the unit - // is not active — the "running" claim in the file is a dead process. + // systemd-run answers the availability probe; systemctl lists no such + // unit — the "running" claim in the file is a dead process. const updater = updaterFor({ statusDir, - run: async (file) => - file === 'systemctl' - ? { exitCode: 3, stdout: '', stderr: '' } - : { exitCode: 0, stdout: '', stderr: '' }, + run: async () => ({ exitCode: 0, stdout: '', stderr: '' }), }); const status = getUpgradeStatusResponseSchema.parse(await updater.status()); expect(status.available).toBe(true); @@ -410,10 +416,7 @@ describe('Updater.apply and status', () => { writeFileSync(path.join(statusDir, 'status.json'), JSON.stringify(run)); const updater = updaterFor({ statusDir, - run: async (file) => - file === 'systemctl' - ? { exitCode: 3, stdout: '', stderr: '' } - : { exitCode: 0, stdout: '', stderr: '' }, + run: async () => ({ exitCode: 0, stdout: '', stderr: '' }), }); const status = await updater.status(); expect(status.last).toEqual(run); diff --git a/packages/server/src/updater.ts b/packages/server/src/updater.ts index fe8e64ce..3389128d 100644 --- a/packages/server/src/updater.ts +++ b/packages/server/src/updater.ts @@ -376,14 +376,25 @@ export class Updater { * finishing: install.sh's last step, doctor, runs on for seconds after * the daemon it restarted is back (eight, measured 2026-09-16), and a * tell landing in them met this unit's mutex. + * + * Asked with list-units, not is-active: is-active makes systemd try to + * load the unit by name, and for a transient unit that has ended and + * been collected that means reopening its fragment under + * /run/systemd/transient — gone — and logging a notice about it, two + * lines per ask (systemd 255, measured 2026-09-16); asked every check-in, + * eleven thousand journal lines a day on every node. list-units only + * lists what systemd has in memory: a line when the unit is alive, nothing + * when it is not, and no load attempted either way. */ async running(): Promise { const result = await this.run('systemctl', [ - 'is-active', - '--quiet', + 'list-units', + '--plain', + '--no-legend', + '--state=active', `${UNIT}.service`, ]); - return result.exitCode === 0; + return result.exitCode === 0 && result.stdout.trim() !== ''; } private async readRun(): Promise {